Merge pull request #1897 from theopolis/remove_rdb

Refactor backing storage

CMakeLists.txt

@@ -139,12 +139,22 @@ set(OSQUERY_REQUIRE_RUNTIMES
 if(DEFINED ENV{DEBUG})
   set(DEBUG TRUE)
   set(CMAKE_BUILD_TYPE "Debug")
-  add_compile_options(-g -O0)
+  add_compile_options(
+    -g
+    -O0
+    -fstandalone-debug
+  )
   add_definitions(-DDEBUG)
   WARNING_LOG("Setting DEBUG build")
 elseif(DEFINED ENV{SANITIZE})
   # make sanitize (cannot make debug sanitize)
-  add_compile_options(-g -O1 -fno-omit-frame-pointer -fno-optimize-sibling-calls)
+  add_compile_options(
+    -g
+    -O1
+    -fstandalone-debug
+    -fno-omit-frame-pointer
+    -fno-optimize-sibling-calls
+  )
   add_definitions(-DNDEBUG)
   if(DEFINED ENV{SANITIZE_THREAD})
     add_compile_options(-fsanitize=thread)
@@ -347,12 +357,12 @@ endif()
 find_package(Glog REQUIRED)
 find_package(Gflags REQUIRED)
 find_package(Gtest REQUIRED)
-find_package(RocksDB REQUIRED)
 find_package(Thrift 0.9.3 REQUIRED)
 
 # If using the RocksDB LITE version our code must also define ROCKSDB_LITE=1
-if(ROCKSDB_LITE_FOUND)
+if(NOT DEFINED ENV{SKIP_ROCKSDB})
-  add_definitions(-DROCKSDB_LITE=1)
+  set(ROCKSDB TRUE)
+  find_package(RocksDB REQUIRED)
 endif()
 
 # Python is used for table spec generation and formating.

include/osquery/database.h

@@ -10,6 +10,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <map>
 #include <string>
 #include <vector>
@@ -21,6 +22,18 @@
 
 namespace osquery {
 
+/**
+ * @brief A list of supported backing storage categories: called domains.
+ *
+ * RocksDB has a concept of "column families" which are kind of like tables
+ * in other databases. kDomainds is populated with a list of all column
+ * families. If a string exists in kDomains, it's a column family in the
+ * database.
+ *
+ * For SQLite-backed storage these are tables using a keyed index.
+ */
+extern const std::vector<std::string> kDomains;
+
 /**
  * @brief A backing storage domain name, used for key/value based storage.
  *
@@ -46,9 +59,6 @@ extern const std::string kEvents;
  */
 extern const std::string kLogs;
 
-/// An ordered list of column type names.
-extern const std::vector<std::string> kDomains;
-
 /////////////////////////////////////////////////////////////////////////////
 // Row
 /////////////////////////////////////////////////////////////////////////////
@@ -394,142 +404,6 @@ Status serializeQueryLogItemAsEvents(const QueryLogItem& item,
 Status serializeQueryLogItemAsEventsJSON(const QueryLogItem& i,
                                          std::vector<std::string>& items);
 
-/////////////////////////////////////////////////////////////////////////////
-// DistributedQueryRequest
-/////////////////////////////////////////////////////////////////////////////
-
-/**
- * @brief Small struct containing the query and ID information for a
- * distributed query
- */
-struct DistributedQueryRequest {
- public:
-  explicit DistributedQueryRequest() {}
-  explicit DistributedQueryRequest(const std::string& q, const std::string& i)
-      : query(q), id(i) {}
-
-  /// equals operator
-  bool operator==(const DistributedQueryRequest& comp) const {
-    return (comp.query == query) && (comp.id == id);
-  }
-
-  std::string query;
-  std::string id;
-};
-
-/**
- * @brief Serialize a DistributedQueryRequest into a property tree
- *
- * @param r the DistributedQueryRequest to serialize
- * @param tree the output property tree
- *
- * @return Status indicating the success or failure of the operation
- */
-Status serializeDistributedQueryRequest(const DistributedQueryRequest& r,
-                                        boost::property_tree::ptree& tree);
-
-/**
- * @brief Serialize a DistributedQueryRequest object into a JSON string
- *
- * @param r the DistributedQueryRequest to serialize
- * @param json the output JSON string
- *
- * @return Status indicating the success or failure of the operation
- */
-Status serializeDistributedQueryRequestJSON(const DistributedQueryRequest& r,
-                                            std::string& json);
-
-/**
- * @brief Deserialize a DistributedQueryRequest object from a property tree
- *
- * @param tree the input property tree
- * @param r the output DistributedQueryRequest structure
- *
- * @return Status indicating the success or failure of the operation
- */
-Status deserializeDistributedQueryRequest(
-    const boost::property_tree::ptree& tree, DistributedQueryRequest& r);
-
-/**
- * @brief Deserialize a DistributedQueryRequest object from a JSON string
- *
- * @param json the input JSON string
- * @param r the output DistributedQueryRequest structure
- *
- * @return Status indicating the success or failure of the operation
- */
-Status deserializeDistributedQueryRequestJSON(const std::string& json,
-                                              DistributedQueryRequest& r);
-
-/////////////////////////////////////////////////////////////////////////////
-// DistributedQueryResult
-/////////////////////////////////////////////////////////////////////////////
-
-/**
- * @brief Small struct containing the results of a distributed query
- */
-struct DistributedQueryResult {
- public:
-  explicit DistributedQueryResult() {}
-  explicit DistributedQueryResult(const DistributedQueryRequest& req,
-                                  const QueryData& res)
-      : request(req), results(res) {}
-
-  /// equals operator
-  bool operator==(const DistributedQueryResult& comp) const {
-    return (comp.request == request) && (comp.results == results);
-  }
-
-  DistributedQueryRequest request;
-  QueryData results;
-};
-
-/**
- * @brief Serialize a DistributedQueryResult into a property tree
- *
- * @param r the DistributedQueryResult to serialize
- * @param tree the output property tree
- *
- * @return Status indicating the success or failure of the operation
- */
-Status serializeDistributedQueryResult(const DistributedQueryResult& r,
-                                       boost::property_tree::ptree& tree);
-
-/**
- * @brief Serialize a DistributedQueryResult object into a JSON string
- *
- * @param r the DistributedQueryResult to serialize
- * @param json the output JSON string
- *
- * @return Status indicating the success or failure of the operation
- */
-Status serializeDistributedQueryResultJSON(const DistributedQueryResult& r,
-                                           std::string& json);
-
-/**
- * @brief Deserialize a DistributedQueryResult object from a property tree
- *
- * @param tree the input property tree
- * @param r the output DistributedQueryResult structure
- *
- * @return Status indicating the success or failure of the operation
- */
-Status deserializeDistributedQueryResult(
-    const boost::property_tree::ptree& tree, DistributedQueryResult& r);
-
-/**
- * @brief Deserialize a DistributedQueryResult object from a JSON string
- *
- * @param json the input JSON string
- * @param r the output DistributedQueryResult structure
- *
- * @return Status indicating the success or failure of the operation
- */
-Status deserializeDistributedQueryResultJSON(const std::string& json,
-                                             DistributedQueryResult& r);
-
-/////////////////////////////////////////////////////////////////////////////
-
 /**
  * @brief An osquery backing storage (database) type that persists executions.
  *
@@ -543,7 +417,7 @@ Status deserializeDistributedQueryResultJSON(const std::string& json,
  * to removing RocksDB as a dependency for the osquery SDK.
  */
 class DatabasePlugin : public Plugin {
- protected:
+ public:
   /**
    * @brief Perform a domain and key lookup from the backing store.
    *
@@ -580,15 +454,79 @@ class DatabasePlugin : public Plugin {
   /// Data removal method.
   virtual Status remove(const std::string& domain, const std::string& k) = 0;
 
-  /// Key/index lookup method.
   virtual Status scan(const std::string& domain,
                       std::vector<std::string>& results,
+                      const std::string& prefix,
                       size_t max = 0) const {
     return Status(0, "Not used");
   }
 
+  /**
+   * @brief Shutdown the database and release initialization resources.
+   *
+   * Assume that a plugin may override ::tearDown and choose to close resources
+   * when the registry is stopping. Most plugins will implement a mutex around
+   * initialization and destruction and assume ::setUp and ::tearDown will
+   * dictate the flow in most situations.
+   */
+  virtual ~DatabasePlugin() {}
+
+  /**
+   * @brief Support the registry calling API for extensions.
+   *
+   * The database plugin "fast-calls" directly to local plugins.
+   * Extensions cannot use an extension-local backing store so their requests
+   * are routed like all other plugins.
+   */
+  Status call(const PluginRequest& request, PluginResponse& response) override;
+
  public:
-  Status call(const PluginRequest& request, PluginResponse& response);
+  /// Database-specific workflow: reset the originally request instance.
+  virtual Status reset() final;
+
+  /// Database-specific workflow: perform an initialize, then reset.
+  bool checkDB();
+
+  /// Require all DBHandle accesses to open a read and write handle.
+  static void setRequireWrite(bool rw) { kDBHandleOptionRequireWrite = rw; }
+
+  /// Allow DBHandle creations.
+  static void setAllowOpen(bool ao) { kDBHandleOptionAllowOpen = ao; }
+
+ public:
+  /// Control availability of the RocksDB handle (default false).
+  static bool kDBHandleOptionAllowOpen;
+
+  /// The database must be opened in a R/W mode (default false).
+  static bool kDBHandleOptionRequireWrite;
+
+  /// A queryable mutex around database sanity checking.
+  static std::atomic<bool> kCheckingDB;
+
+ public:
+  /**
+   * @brief Allow the initializer to check the active database plugin.
+   *
+   * Unlink the initializer's ::initActivePlugin helper method, the database
+   * plugin should always be within the core. There is no need to discover
+   * the active plugin via the registry or extensions API.
+   *
+   * The database should setUp in preparation for accesses.
+   */
+  static bool initPlugin();
+
+  /// Allow shutdown before exit.
+  static void shutdown();
+
+ protected:
+  /// The database was opened in a ReadOnly mode.
+  bool read_only_{false};
+
+  /// True if the database was started in an in-memory mode.
+  bool in_memory_{false};
+
+  /// Original requested path on disk.
+  std::string path_;
 };
 
 /**
@@ -631,6 +569,12 @@ Status scanDatabaseKeys(const std::string& domain,
                         std::vector<std::string>& keys,
                         size_t max = 0);
 
+/// Get a list of keys for a given domain.
+Status scanDatabaseKeys(const std::string& domain,
+                        std::vector<std::string>& keys,
+                        const std::string& prefix,
+                        size_t max = 0);
+
 /// Allow callers to scan each column family and print each value.
 void dumpDatabase();
 

include/osquery/distributed.h

@@ -19,6 +19,136 @@
 
 namespace osquery {
 
+/**
+ * @brief Small struct containing the query and ID information for a
+ * distributed query
+ */
+struct DistributedQueryRequest {
+ public:
+  explicit DistributedQueryRequest() {}
+  explicit DistributedQueryRequest(const std::string& q, const std::string& i)
+      : query(q), id(i) {}
+
+  /// equals operator
+  bool operator==(const DistributedQueryRequest& comp) const {
+    return (comp.query == query) && (comp.id == id);
+  }
+
+  std::string query;
+  std::string id;
+};
+
+/**
+ * @brief Serialize a DistributedQueryRequest into a property tree
+ *
+ * @param r the DistributedQueryRequest to serialize
+ * @param tree the output property tree
+ *
+ * @return Status indicating the success or failure of the operation
+ */
+Status serializeDistributedQueryRequest(const DistributedQueryRequest& r,
+                                        boost::property_tree::ptree& tree);
+
+/**
+ * @brief Serialize a DistributedQueryRequest object into a JSON string
+ *
+ * @param r the DistributedQueryRequest to serialize
+ * @param json the output JSON string
+ *
+ * @return Status indicating the success or failure of the operation
+ */
+Status serializeDistributedQueryRequestJSON(const DistributedQueryRequest& r,
+                                            std::string& json);
+
+/**
+ * @brief Deserialize a DistributedQueryRequest object from a property tree
+ *
+ * @param tree the input property tree
+ * @param r the output DistributedQueryRequest structure
+ *
+ * @return Status indicating the success or failure of the operation
+ */
+Status deserializeDistributedQueryRequest(
+    const boost::property_tree::ptree& tree, DistributedQueryRequest& r);
+
+/**
+ * @brief Deserialize a DistributedQueryRequest object from a JSON string
+ *
+ * @param json the input JSON string
+ * @param r the output DistributedQueryRequest structure
+ *
+ * @return Status indicating the success or failure of the operation
+ */
+Status deserializeDistributedQueryRequestJSON(const std::string& json,
+                                              DistributedQueryRequest& r);
+
+/////////////////////////////////////////////////////////////////////////////
+// DistributedQueryResult
+/////////////////////////////////////////////////////////////////////////////
+
+/**
+ * @brief Small struct containing the results of a distributed query
+ */
+struct DistributedQueryResult {
+ public:
+  explicit DistributedQueryResult() {}
+  explicit DistributedQueryResult(const DistributedQueryRequest& req,
+                                  const QueryData& res)
+      : request(req), results(res) {}
+
+  /// equals operator
+  bool operator==(const DistributedQueryResult& comp) const {
+    return (comp.request == request) && (comp.results == results);
+  }
+
+  DistributedQueryRequest request;
+  QueryData results;
+};
+
+/**
+ * @brief Serialize a DistributedQueryResult into a property tree
+ *
+ * @param r the DistributedQueryResult to serialize
+ * @param tree the output property tree
+ *
+ * @return Status indicating the success or failure of the operation
+ */
+Status serializeDistributedQueryResult(const DistributedQueryResult& r,
+                                       boost::property_tree::ptree& tree);
+
+/**
+ * @brief Serialize a DistributedQueryResult object into a JSON string
+ *
+ * @param r the DistributedQueryResult to serialize
+ * @param json the output JSON string
+ *
+ * @return Status indicating the success or failure of the operation
+ */
+Status serializeDistributedQueryResultJSON(const DistributedQueryResult& r,
+                                           std::string& json);
+
+/**
+ * @brief Deserialize a DistributedQueryResult object from a property tree
+ *
+ * @param tree the input property tree
+ * @param r the output DistributedQueryResult structure
+ *
+ * @return Status indicating the success or failure of the operation
+ */
+Status deserializeDistributedQueryResult(
+    const boost::property_tree::ptree& tree, DistributedQueryResult& r);
+
+/**
+ * @brief Deserialize a DistributedQueryResult object from a JSON string
+ *
+ * @param json the input JSON string
+ * @param r the output DistributedQueryResult structure
+ *
+ * @return Status indicating the success or failure of the operation
+ */
+Status deserializeDistributedQueryResultJSON(const std::string& json,
+                                             DistributedQueryResult& r);
+
 class DistributedPlugin : public Plugin {
  public:
   /**

include/osquery/extensions.h

@@ -16,7 +16,6 @@
 
 namespace osquery {
 
-DECLARE_int32(worker_threads);
 DECLARE_string(extensions_socket);
 DECLARE_string(extensions_autoload);
 DECLARE_string(extensions_timeout);
@@ -41,11 +40,7 @@ typedef std::map<RouteUUID, ExtensionInfo> ExtensionList;
 
 inline std::string getExtensionSocket(
     RouteUUID uuid, const std::string& path = FLAGS_extensions_socket) {
-  if (uuid == 0) {
+  return (uuid == 0) ? path : path + "." + std::to_string(uuid);
-    return path;
-  } else {
-    return path + "." + std::to_string(uuid);
-  }
 }
 
 /// External (extensions) SQL implementation of the osquery query API.

include/osquery/registry.h

@@ -12,8 +12,8 @@
 
 #include <map>
 #include <mutex>
-#include <vector>
 #include <set>
+#include <vector>
 
 #include <boost/noncopyable.hpp>
 #include <boost/property_tree/ptree.hpp>

osquery/CMakeLists.txt

@@ -14,10 +14,16 @@ set(OSQUERY_ADDITIONAL_LINKS "")
 set(OSQUERY_ADDITIONAL_TESTS "")
 set(OSQUERY_TABLES_TESTS "")
 
+# Add all and extra for osquery code.
+add_compile_options(
+  -Wall
+  -Wextra
+  -Wno-unused-parameter
+)
+
 # The core set of osquery libraries most discovered with find_package.
 set(OSQUERY_LIBS
   # This includes librocksdb[_lite] and libsnappy.
-  ${ROCKSDB_LIBRARIES}
   ${THRIFT_LIBRARY}
   ${GLOG_LIBRARY}
   ${GFLAGS_LIBRARY}
@@ -30,12 +36,10 @@ set(OSQUERY_LIBS
   z
 )
 
-# Add all and extra for osquery code.
+# If building with RocksDB (default) append associated libraries.
-add_compile_options(
+if(ROCKSDB)
-  -Wall
+  set(OSQUERY_LIBS ${OSQUERY_LIBS} ${ROCKSDB_LIBRARIES})
-  -Wextra
+endif()
-  -Wno-unused-parameter
-)
 
 if(NOT FREEBSD)
   set(OSQUERY_LIBS ${OSQUERY_LIBS} dl)

osquery/config/config.cpp

@@ -18,9 +18,9 @@
 
 #include <osquery/config.h>
 #include <osquery/database.h>
+#include <osquery/filesystem.h>
 #include <osquery/flags.h>
 #include <osquery/hash.h>
-#include <osquery/filesystem.h>
 #include <osquery/logger.h>
 #include <osquery/packs.h>
 #include <osquery/registry.h>
@@ -249,8 +249,9 @@ void Config::removeFiles(const std::string& source) {
   }
 }
 
-void Config::scheduledQueries(std::function<
+void Config::scheduledQueries(
-    void(const std::string& name, const ScheduledQuery& query)> predicate) {
+    std::function<void(const std::string& name, const ScheduledQuery& query)>
+        predicate) {
   ReadLock rlock(config_schedule_mutex_);
   for (const PackRef& pack : *schedule_) {
     for (const auto& it : pack->getSchedule()) {
@@ -675,18 +676,12 @@ Status Config::getMD5(std::string& hash) {
 
 const std::shared_ptr<ConfigParserPlugin> Config::getParser(
     const std::string& parser) {
-  std::shared_ptr<ConfigParserPlugin> config_parser = nullptr;
+  if (!Registry::exists("config_parser", parser, true)) {
-  try {
+    return nullptr;
-    auto plugin = Registry::get("config_parser", parser);
-    config_parser = std::dynamic_pointer_cast<ConfigParserPlugin>(plugin);
-  } catch (const std::out_of_range& e) {
-    LOG(ERROR) << "Error getting config parser plugin " << parser << ": "
-               << e.what();
-  } catch (const std::bad_cast& e) {
-    LOG(ERROR) << "Error casting " << parser
-               << " as a ConfigParserPlugin: " << e.what();
   }
-  return config_parser;
+
+  auto plugin = Registry::get("config_parser", parser);
+  return std::dynamic_pointer_cast<ConfigParserPlugin>(plugin);
 }
 
 void Config::files(

osquery/core/init.cpp

@@ -30,7 +30,6 @@
 #include <osquery/registry.h>
 
 #include "osquery/core/watcher.h"
-#include "osquery/database/db_handle.h"
 #include "osquery/dispatcher/dispatcher.h"
 
 #if defined(__linux__) || defined(__FreeBSD__)
@@ -105,6 +104,8 @@ static inline bool hasWorkerVariable() {
 
 volatile std::sig_atomic_t kHandledSignal{0};
 
+static inline bool isWatcher() { return (osquery::Watcher::getWorker() > 0); }
+
 void signalHandler(int signal) {
   // Inform exit status of main threads blocked by service joins.
   if (kHandledSignal == 0) {
@@ -112,10 +113,8 @@ void signalHandler(int signal) {
   }
 
   // Handle signals based on a tri-state (worker, watcher, neither).
-  pid_t worker_pid = osquery::Watcher::getWorker();
-  bool is_watcher = worker_pid > 0;
   if (signal == SIGHUP) {
-    if (!is_watcher || hasWorkerVariable()) {
+    if (!isWatcher() || hasWorkerVariable()) {
       // Reload configuration.
     }
   } else if (signal == SIGTERM || signal == SIGINT || signal == SIGABRT) {
@@ -127,7 +126,7 @@ void signalHandler(int signal) {
     std::signal(signal, SIG_DFL);
 
     // The watcher waits for the worker to die.
-    if (is_watcher) {
+    if (isWatcher()) {
       // Bind the fate of the worker to this watcher.
       osquery::Watcher::bindFates();
     } else {
@@ -145,7 +144,7 @@ void signalHandler(int signal) {
     raise((kHandledSignal != 0) ? kHandledSignal : SIGALRM);
   }
 
-  if (is_watcher) {
+  if (isWatcher()) {
     // The signal should be proliferated through the process group.
     // Otherwise the watcher could 'forward' the signal to workers and
     // managed extension processes.
@@ -166,7 +165,9 @@ DECLARE_string(distributed_plugin);
 DECLARE_bool(disable_distributed);
 DECLARE_string(config_plugin);
 DECLARE_bool(config_check);
+DECLARE_bool(config_dump);
 DECLARE_bool(database_dump);
+DECLARE_string(database_path);
 
 ToolType kToolType = OSQUERY_TOOL_UNKNOWN;
 
@@ -421,14 +422,27 @@ void Initializer::initActivePlugin(const std::string& type,
     timeout = kExtensionInitializeLatencyUS * 10;
   }
 
-  while (!Registry::setActive(type, name)) {
+  // Attempt to set the request plugin as active.
-    if (!Watcher::hasManagedExtensions() || delay > timeout) {
+  Status status;
-      LOG(ERROR) << "Active " << type << " plugin not found: " << name;
+  do {
-      osquery::shutdown(EXIT_CATASTROPHIC);
+    status = Registry::setActive(type, name);
+    if (status.ok()) {
+      // The plugin was found, and is not active.
+      return;
     }
+
+    if (!Watcher::hasManagedExtensions()) {
+      // The plugin was found locally, and is not active, problem.
+      break;
+    }
+    // The plugin is not local and is not active, wait and retry.
     delay += kExtensionInitializeLatencyUS;
     ::usleep(kExtensionInitializeLatencyUS);
-  }
+  } while (delay < timeout);
+
+  LOG(ERROR) << "Cannot activate " << name << " " << type
+             << " plugin: " << status.getMessage();
+  osquery::shutdown(EXIT_CATASTROPHIC);
 }
 
 void Initializer::start() const {
@@ -436,18 +450,26 @@ void Initializer::start() const {
   osquery::loadModules();
 
   // Pre-extension manager initialization options checking.
-  if (FLAGS_config_check && !Watcher::hasManagedExtensions()) {
+  // If the shell or daemon does not need extensions and it will exit quickly,
+  // prefer to disable the extension manager.
+  if ((FLAGS_config_check || FLAGS_config_dump) &&
+      !Watcher::hasManagedExtensions()) {
     FLAGS_disable_extensions = true;
   }
 
-  // A daemon must always have R/W access to the database.
+  // A watcher should not need access to the backing store.
-  DBHandle::setAllowOpen(true);
+  // If there are spurious access then warning logs will be emitted since the
-  DBHandle::setRequireWrite(tool_ == OSQUERY_TOOL_DAEMON);
+  // set-allow-open will never be called.
-  if (!DBHandle::checkDB()) {
+  if (!isWatcher()) {
-    LOG(ERROR) << RLOG(1629) << binary_
+    DatabasePlugin::setAllowOpen(true);
-               << " initialize failed: Could not open RocksDB";
+    // A daemon must always have R/W access to the database.
-    auto retcode = (isWorker()) ? EXIT_CATASTROPHIC : EXIT_FAILURE;
+    DatabasePlugin::setRequireWrite(tool_ == OSQUERY_TOOL_DAEMON);
-    ::exit(retcode);
+    if (!DatabasePlugin::initPlugin()) {
+      LOG(ERROR) << RLOG(1629) << binary_
+                 << " initialize failed: Could not initialize database";
+      auto retcode = (isWorker()) ? EXIT_CATASTROPHIC : EXIT_FAILURE;
+      ::exit(retcode);
+    }
   }
 
   // Bind to an extensions socket and wait for registry additions.
@@ -516,6 +538,7 @@ void shutdown(int retcode, bool wait) {
 
   // Hopefully release memory used by global string constructors in gflags.
   GFLAGS_NAMESPACE::ShutDownCommandLineFlags();
+  DatabasePlugin::shutdown();
   ::exit(retcode);
 }
 }

osquery/core/test_util.cpp

@@ -16,18 +16,20 @@
 #include <signal.h>
 #include <time.h>
 
-#include <boost/property_tree/json_parser.hpp>
 #include <boost/filesystem/operations.hpp>
+#include <boost/property_tree/json_parser.hpp>
 
+#include <osquery/core.h>
+#include <osquery/database.h>
 #include <osquery/filesystem.h>
 #include <osquery/logger.h>
 
 #include "osquery/core/test_util.h"
-#include "osquery/database/db_handle.h"
 
 namespace fs = boost::filesystem;
 
 namespace osquery {
+
 std::string kFakeDirectory = "";
 
 #ifdef DARWIN
@@ -80,10 +82,14 @@ void initTesting() {
   FLAGS_modules_autoload = kTestWorkingDirectory + "unittests-mod.load";
   FLAGS_disable_logging = true;
 
-  // Create a default DBHandle instance before unittests.
+  // Tests need a database plugin.
-  (void)DBHandle::getInstance();
+  // Set up the database instance for the unittests.
+  DatabasePlugin::setAllowOpen(true);
+  Registry::setActive("database", "ephemeral");
 }
 
+void shutdownTesting() { DatabasePlugin::shutdown(); }
+
 std::map<std::string, std::string> getTestConfigMap() {
   std::string content;
   readFile(kTestDataPath + "test_parse_items.conf", content);

osquery/core/test_util.h

@@ -29,7 +29,7 @@ namespace osquery {
 void initTesting();
 
 /// Cleanup/stop function for tests and benchmarks.
-void cleanupTesting();
+void shutdownTesting();
 
 /// Any SQL-dependent tests should use kTestQuery for a pre-populated example.
 const std::string kTestQuery = "SELECT * FROM test_table";

osquery/core/watcher.cpp

@@ -11,8 +11,8 @@
 #include <cstring>
 
 #include <math.h>
-#include <sys/wait.h>
 #include <signal.h>
+#include <sys/wait.h>
 
 #include <boost/filesystem.hpp>
 
@@ -431,7 +431,7 @@ void WatcherWatcherRunner::start() {
       VLOG(1) << "osqueryd worker (" << getpid()
               << ") detected killed watcher (" << watcher_ << ")";
       // The watcher watcher is a thread. Do not join services after removing.
-      ::exit(EXIT_FAILURE);
+      raise(SIGKILL);
     }
     interruptableSleep(getWorkerLimit(INTERVAL) * 1000);
   }

osquery/database/CMakeLists.txt

@@ -1,15 +1,41 @@
 ADD_OSQUERY_LIBRARY(TRUE osquery_database
   database.cpp
-)
-
-# osquery_database_internal should be an 'additional' CORE=False lib
-ADD_OSQUERY_LIBRARY(TRUE osquery_database_internal
-  db_handle.cpp
   query.cpp
+
+  # Add 'core' plugins that do not required additional libraries.
+  plugins/ephemeral.cpp
 )
 
+# Begin with just the SQLite database plugin.
+set(OSQUERY_DATABASE_PLUGINS
+  plugins/sqlite.cpp
+)
+
+set(OSQUERY_DATABASE_PLUGIN_TESTS
+  database/tests/plugin_tests.cpp
+  database/plugins/tests/sqlite_tests.cpp
+)
+
+# Optionally (but by default), add the RocksDB database plugin.
+if(ROCKSDB)
+  if(ROCKSDB_LITE_FOUND)
+    add_definitions(-DROCKSDB_LITE=1)
+  endif()
+  set(OSQUERY_DATABASE_PLUGINS ${OSQUERY_DATABASE_PLUGINS} plugins/rocksdb.cpp)
+  set(OSQUERY_DATABASE_PLUGIN_TESTS ${OSQUERY_DATABASE_PLUGIN_TESTS} database/plugins/tests/rocksdb_tests.cpp)
+else()
+  add_definitions(-DSKIP_ROCKSDB)
+endif()
+
+# Plugins (which do not include the shim/ephemeral) are 'additional'.
+ADD_OSQUERY_LIBRARY(FALSE osquery_database_plugins ${OSQUERY_DATABASE_PLUGINS})
+
+# Non-plugin tests are core.
 file(GLOB OSQUERY_DATABASE_TESTS "tests/*.cpp")
 ADD_OSQUERY_TEST(TRUE ${OSQUERY_DATABASE_TESTS})
 
+# Plugin tests are additional.
+ADD_OSQUERY_TEST(FALSE ${OSQUERY_DATABASE_PLUGIN_TESTS})
+
 file(GLOB OSQUERY_DATABASE_BENCHMARKS "benchmarks/*.cpp")
 ADD_OSQUERY_BENCHMARK(${OSQUERY_DATABASE_BENCHMARKS})

osquery/database/benchmarks/database_benchmarks.cpp

@@ -10,11 +10,10 @@
 
 #include <benchmark/benchmark.h>
 
-#include <osquery/filesystem.h>
 #include <osquery/database.h>
+#include <osquery/filesystem.h>
 
 #include "osquery/core/test_util.h"
-#include "osquery/database/db_handle.h"
 #include "osquery/database/query.h"
 
 namespace osquery {
@@ -76,8 +75,22 @@ static void DATABASE_query_results(benchmark::State& state) {
   }
 }
 
-BENCHMARK(DATABASE_query_results)->ArgPair(1, 1)->ArgPair(10, 10)->ArgPair(10,
+BENCHMARK(DATABASE_query_results)
-                                                                           100);
+    ->ArgPair(1, 1)
+    ->ArgPair(10, 10)
+    ->ArgPair(10, 100);
+
+static void DATABASE_get(benchmark::State& state) {
+  setDatabaseValue(kPersistentSettings, "benchmark", "1");
+  while (state.KeepRunning()) {
+    std::string value;
+    getDatabaseValue(kPersistentSettings, "benchmark", value);
+  }
+  // All benchmarks will share a single database handle.
+  deleteDatabaseValue(kPersistentSettings, "benchmark");
+}
+
+BENCHMARK(DATABASE_get);
 
 static void DATABASE_store(benchmark::State& state) {
   while (state.KeepRunning()) {
@@ -112,12 +125,14 @@ static void DATABASE_store_append(benchmark::State& state) {
 
   size_t k = 0;
   while (state.KeepRunning()) {
-    setDatabaseValue(kPersistentSettings, "key" + std::to_string(k++), content);
+    setDatabaseValue(kPersistentSettings, "key" + std::to_string(k), content);
+    deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(k));
+    k++;
   }
 
   // All benchmarks will share a single database handle.
   for (size_t i = 0; i < k; ++i) {
-    deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(i));
+    // deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(i));
   }
 }
 

osquery/database/database.cpp

@@ -8,12 +8,7 @@
  *
  */
 
-#include <algorithm>
-#include <iostream>
-#include <sstream>
 #include <set>
-#include <string>
-#include <vector>
 
 #include <boost/lexical_cast.hpp>
 #include <boost/property_tree/json_parser.hpp>
@@ -27,6 +22,37 @@ namespace osquery {
 
 CLI_FLAG(bool, database_dump, false, "Dump the contents of the backing store");
 
+CLI_FLAG(string,
+         database_path,
+         "/var/osquery/osquery.db",
+         "If using a disk-based backing store, specify a path");
+FLAG_ALIAS(std::string, db_path, database_path);
+
+CLI_FLAG(bool,
+         database_in_memory,
+         false,
+         "Keep osquery backing-store in memory");
+FLAG_ALIAS(bool, use_in_memory_database, database_in_memory);
+
+#if defined(SKIP_ROCKSDB)
+#define DATABASE_PLUGIN "sqlite"
+#else
+#define DATABASE_PLUGIN "rocksdb"
+#endif
+const std::string kInternalDatabase = DATABASE_PLUGIN;
+
+const std::string kPersistentSettings = "configurations";
+const std::string kQueries = "queries";
+const std::string kEvents = "events";
+const std::string kLogs = "logs";
+
+const std::vector<std::string> kDomains = {kPersistentSettings, kQueries,
+                                           kEvents, kLogs};
+
+bool DatabasePlugin::kDBHandleOptionAllowOpen(false);
+bool DatabasePlugin::kDBHandleOptionRequireWrite(false);
+std::atomic<bool> DatabasePlugin::kCheckingDB(false);
+
 /////////////////////////////////////////////////////////////////////////////
 // Row - the representation of a row in a set of database results. Row is a
 // simple map where individual column names are keys, which map to the Row's
@@ -360,133 +386,6 @@ Status serializeQueryLogItemAsEventsJSON(const QueryLogItem& i,
   return Status(0, "OK");
 }
 
-/////////////////////////////////////////////////////////////////////////////
-// DistributedQueryRequest - small struct containing the query and ID
-// information for a distributed query
-/////////////////////////////////////////////////////////////////////////////
-
-Status serializeDistributedQueryRequest(const DistributedQueryRequest& r,
-                                        pt::ptree& tree) {
-  tree.put("query", r.query);
-  tree.put("id", r.id);
-  return Status(0, "OK");
-}
-
-Status serializeDistributedQueryRequestJSON(const DistributedQueryRequest& r,
-                                            std::string& json) {
-  pt::ptree tree;
-  auto s = serializeDistributedQueryRequest(r, tree);
-  if (!s.ok()) {
-    return s;
-  }
-  std::stringstream ss;
-  try {
-    pt::write_json(ss, tree, false);
-  } catch (const pt::ptree_error& e) {
-    return Status(1, "Error serializing JSON: " + std::string(e.what()));
-  }
-  json = ss.str();
-
-  return Status(0, "OK");
-}
-
-Status deserializeDistributedQueryRequest(const pt::ptree& tree,
-                                          DistributedQueryRequest& r) {
-  r.query = tree.get<std::string>("query", "");
-  r.id = tree.get<std::string>("id", "");
-  return Status(0, "OK");
-}
-
-Status deserializeDistributedQueryRequestJSON(const std::string& json,
-                                              DistributedQueryRequest& r) {
-  std::stringstream ss(json);
-  pt::ptree tree;
-  try {
-    pt::read_json(ss, tree);
-  } catch (const pt::ptree_error& e) {
-    return Status(1, "Error serializing JSON: " + std::string(e.what()));
-  }
-  return deserializeDistributedQueryRequest(tree, r);
-}
-
-/////////////////////////////////////////////////////////////////////////////
-// DistributedQueryResult - small struct containing the results of a
-// distributed query
-/////////////////////////////////////////////////////////////////////////////
-
-Status serializeDistributedQueryResult(const DistributedQueryResult& r,
-                                       pt::ptree& tree) {
-  pt::ptree request;
-  auto s = serializeDistributedQueryRequest(r.request, request);
-  if (!s.ok()) {
-    return s;
-  }
-
-  pt::ptree results;
-  s = serializeQueryData(r.results, results);
-  if (!s.ok()) {
-    return s;
-  }
-
-  tree.add_child("request", request);
-  tree.add_child("results", results);
-
-  return Status(0, "OK");
-}
-
-Status serializeDistributedQueryResultJSON(const DistributedQueryResult& r,
-                                           std::string& json) {
-  pt::ptree tree;
-  auto s = serializeDistributedQueryResult(r, tree);
-  if (!s.ok()) {
-    return s;
-  }
-  std::stringstream ss;
-  try {
-    pt::write_json(ss, tree, false);
-  } catch (const pt::ptree_error& e) {
-    return Status(1, "Error serializing JSON: " + std::string(e.what()));
-  }
-  json = ss.str();
-
-  return Status(0, "OK");
-}
-
-Status deserializeDistributedQueryResult(const pt::ptree& tree,
-                                         DistributedQueryResult& r) {
-  DistributedQueryRequest request;
-  auto s =
-      deserializeDistributedQueryRequest(tree.get_child("request"), request);
-  if (!s.ok()) {
-    return s;
-  }
-
-  QueryData results;
-  s = deserializeQueryData(tree.get_child("results"), results);
-  if (!s.ok()) {
-    return s;
-  }
-
-  r.request = request;
-  r.results = results;
-
-  return Status(0, "OK");
-}
-
-Status deserializeDistributedQueryResultJSON(const std::string& json,
-                                             DistributedQueryResult& r) {
-  std::stringstream ss(json);
-  pt::ptree tree;
-  try {
-    pt::read_json(ss, tree);
-  } catch (const pt::ptree_error& e) {
-    return Status(1, "Error serializing JSON: " + std::string(e.what()));
-  }
-  return deserializeDistributedQueryResult(tree, r);
-}
-
-/////////////////////////////////////////////////////////////////////////////
-
 bool addUniqueRowToQueryData(QueryData& q, const Row& r) {
   if (std::find(q.begin(), q.end(), r) != q.end()) {
     return false;
@@ -495,6 +394,41 @@ bool addUniqueRowToQueryData(QueryData& q, const Row& r) {
   return true;
 }
 
+bool DatabasePlugin::initPlugin() {
+  // Initialize the database plugin using the flag.
+  return Registry::setActive("database", kInternalDatabase).ok();
+}
+
+void DatabasePlugin::shutdown() {
+  auto datbase_registry = Registry::registry("database");
+  for (auto& plugin : datbase_registry->names()) {
+    datbase_registry->remove(plugin);
+  }
+}
+
+Status DatabasePlugin::reset() {
+  tearDown();
+  return setUp();
+}
+
+bool DatabasePlugin::checkDB() {
+  kCheckingDB = true;
+  bool result = true;
+  try {
+    auto status = setUp();
+    if (kDBHandleOptionRequireWrite && read_only_) {
+      result = false;
+    }
+    tearDown();
+    result = status.ok();
+  } catch (const std::exception& e) {
+    VLOG(1) << "Database plugin check failed: " << e.what();
+    result = false;
+  }
+  kCheckingDB = false;
+  return result;
+}
+
 Status DatabasePlugin::call(const PluginRequest& request,
                             PluginResponse& response) {
   if (request.count("action") == 0) {
@@ -526,7 +460,7 @@ Status DatabasePlugin::call(const PluginRequest& request,
     if (request.count("max") > 0) {
       max = std::stoul(request.at("max"));
     }
-    auto status = this->scan(domain, keys, max);
+    auto status = this->scan(domain, keys, request.at("prefix"), max);
     for (const auto& key : keys) {
       response.push_back({{"k", key}});
     }
@@ -536,51 +470,97 @@ Status DatabasePlugin::call(const PluginRequest& request,
   return Status(1, "Unknown database plugin action");
 }
 
+static inline std::shared_ptr<DatabasePlugin> getDatabasePlugin() {
+  if (!Registry::exists("database", Registry::getActive("database"), true)) {
+    return nullptr;
+  }
+
+  auto plugin = Registry::get("database", Registry::getActive("database"));
+  return std::dynamic_pointer_cast<DatabasePlugin>(plugin);
+}
+
 Status getDatabaseValue(const std::string& domain,
                         const std::string& key,
                         std::string& value) {
-  PluginRequest request = {{"action", "get"}, {"domain", domain}, {"key", key}};
+  if (Registry::external()) {
-  PluginResponse response;
+    // External registries (extensions) do not have databases active.
-  auto status = Registry::call("database", "rocks", request, response);
+    // It is not possible to use an extension-based database.
-  if (!status.ok()) {
+    PluginRequest request = {
+        {"action", "get"}, {"domain", domain}, {"key", key}};
+    PluginResponse response;
+    auto status = Registry::call("database", request, response);
+    if (status.ok()) {
+      // Set value from the internally-known "v" key.
+      if (response.size() > 0 && response[0].count("v") > 0) {
+        value = response[0].at("v");
+      }
+    }
     return status;
+  } else {
+    auto plugin = getDatabasePlugin();
+    return plugin->get(domain, key, value);
   }
-
-  // Set value from the internally-known "v" key.
-  if (response.size() > 0 && response[0].count("v") > 0) {
-    value = response[0].at("v");
-  }
-  return status;
 }
 
 Status setDatabaseValue(const std::string& domain,
                         const std::string& key,
                         const std::string& value) {
-  PluginRequest request = {
+  if (Registry::external()) {
-      {"action", "put"}, {"domain", domain}, {"key", key}, {"value", value}};
+    // External registries (extensions) do not have databases active.
-  return Registry::call("database", "rocks", request);
+    // It is not possible to use an extension-based database.
+    PluginRequest request = {
+        {"action", "put"}, {"domain", domain}, {"key", key}, {"value", value}};
+    return Registry::call("database", request);
+  } else {
+    auto plugin = getDatabasePlugin();
+    return plugin->put(domain, key, value);
+  }
 }
 
 Status deleteDatabaseValue(const std::string& domain, const std::string& key) {
-  PluginRequest request = {
+  if (Registry::external()) {
-      {"action", "remove"}, {"domain", domain}, {"key", key}};
+    // External registries (extensions) do not have databases active.
-  return Registry::call("database", "rocks", request);
+    // It is not possible to use an extension-based database.
+    PluginRequest request = {
+        {"action", "remove"}, {"domain", domain}, {"key", key}};
+    return Registry::call("database", request);
+  } else {
+    auto plugin = getDatabasePlugin();
+    return plugin->remove(domain, key);
+  }
 }
 
 Status scanDatabaseKeys(const std::string& domain,
                         std::vector<std::string>& keys,
                         size_t max) {
-  PluginRequest request = {
+  return scanDatabaseKeys(domain, keys, "", max);
-      {"action", "scan"}, {"domain", domain}, {"max", std::to_string(max)}};
+}
-  PluginResponse response;
-  auto status = Registry::call("database", "rocks", request, response);
 
-  for (const auto& item : response) {
+/// Get a list of keys for a given domain.
-    if (item.count("k") > 0) {
+Status scanDatabaseKeys(const std::string& domain,
-      keys.push_back(item.at("k"));
+                        std::vector<std::string>& keys,
+                        const std::string& prefix,
+                        size_t max) {
+  if (Registry::external()) {
+    // External registries (extensions) do not have databases active.
+    // It is not possible to use an extension-based database.
+    PluginRequest request = {{"action", "scan"},
+                             {"domain", domain},
+                             {"prefix", prefix},
+                             {"max", std::to_string(max)}};
+    PluginResponse response;
+    auto status = Registry::call("database", request, response);
+
+    for (const auto& item : response) {
+      if (item.count("k") > 0) {
+        keys.push_back(item.at("k"));
+      }
     }
+    return status;
+  } else {
+    auto plugin = getDatabasePlugin();
+    return plugin->scan(domain, keys, prefix, max);
   }
-  return status;
 }
 
 void dumpDatabase() {

osquery/database/db_handle.cpp

@@ -1,400 +0,0 @@
-/*
- *  Copyright (c) 2014-present, Facebook, Inc.
- *  All rights reserved.
- *
- *  This source code is licensed under the BSD-style license found in the
- *  LICENSE file in the root directory of this source tree. An additional grant
- *  of patent rights can be found in the PATENTS file in the same directory.
- *
- */
-
-#include <algorithm>
-#include <mutex>
-#include <stdexcept>
-
-#include <sys/stat.h>
-
-#include <snappy.h>
-
-#include <rocksdb/env.h>
-#include <rocksdb/options.h>
-
-#include <osquery/database.h>
-#include <osquery/filesystem.h>
-#include <osquery/logger.h>
-#include <osquery/status.h>
-
-#include "osquery/database/db_handle.h"
-
-namespace osquery {
-
-class RocksDatabasePlugin : public DatabasePlugin {
- public:
-  /// Data retrieval method.
-  Status get(const std::string& domain,
-             const std::string& key,
-             std::string& value) const override;
-
-  /// Data storage method.
-  Status put(const std::string& domain,
-             const std::string& key,
-             const std::string& value) override;
-
-  /// Data removal method.
-  Status remove(const std::string& domain, const std::string& k) override;
-
-  /// Key/index lookup method.
-  Status scan(const std::string& domain,
-              std::vector<std::string>& results,
-              size_t max = 0) const override;
-};
-
-/// Backing-storage provider for osquery internal/core.
-REGISTER_INTERNAL(RocksDatabasePlugin, "database", "rocks");
-
-/////////////////////////////////////////////////////////////////////////////
-// Constants
-/////////////////////////////////////////////////////////////////////////////
-
-bool DBHandle::kDBHandleOptionAllowOpen = false;
-bool DBHandle::kDBHandleOptionRequireWrite = false;
-
-const std::string kPersistentSettings = "configurations";
-const std::string kQueries = "queries";
-const std::string kEvents = "events";
-const std::string kLogs = "logs";
-
-/**
- * @brief A const vector of column families in RocksDB
- *
- * RocksDB has a concept of "column families" which are kind of like tables
- * in other databases. kDomainds is populated with a list of all column
- * families. If a string exists in kDomains, it's a column family in the
- * database.
- */
-const std::vector<std::string> kDomains = {
-    kPersistentSettings, kQueries, kEvents, kLogs};
-
-CLI_FLAG(string,
-         database_path,
-         "/var/osquery/osquery.db",
-         "If using a disk-based backing store, specify a path");
-FLAG_ALIAS(std::string, db_path, database_path);
-
-CLI_FLAG(bool,
-         database_in_memory,
-         false,
-         "Keep osquery backing-store in memory");
-FLAG_ALIAS(bool, use_in_memory_database, database_in_memory);
-
-/////////////////////////////////////////////////////////////////////////////
-// constructors and destructors
-/////////////////////////////////////////////////////////////////////////////
-
-/// A queryable mutex around database sanity checking.
-static bool kCheckingDB = false;
-
-void GlogRocksDBLogger::Logv(const char* format, va_list ap) {
-  // Convert RocksDB log to string and check if header or level-ed log.
-  char buffer[501] = {0};
-  vsnprintf(buffer, 500, format, ap);
-  va_end(ap);
-  if (buffer[0] != '[') {
-    return;
-  }
-
-  if (buffer[1] == 'E' || buffer[1] == 'W') {
-    LOG(INFO) << "RocksDB: " << buffer;
-  }
-}
-
-DBHandle::DBHandle(const std::string& path, bool in_memory)
-    : path_(path), in_memory_(in_memory) {
-  if (!kDBHandleOptionAllowOpen) {
-    LOG(WARNING) << RLOG(1629) << "Not allowed to create DBHandle instance";
-  }
-
-  // Set meta-data (mostly) handling options.
-  options_.create_if_missing = true;
-  options_.create_missing_column_families = true;
-  options_.info_log_level = rocksdb::ERROR_LEVEL;
-  options_.log_file_time_to_roll = 0;
-  options_.keep_log_file_num = 10;
-  options_.max_log_file_size = 1024 * 1024 * 1;
-  options_.stats_dump_period_sec = 0;
-
-  // Performance and optimization settings.
-  options_.compression = rocksdb::kNoCompression;
-  options_.compaction_style = rocksdb::kCompactionStyleLevel;
-  options_.arena_block_size = (4 * 1024);
-  options_.write_buffer_size = (4 * 1024) * 100; // 100 blocks.
-  options_.max_write_buffer_number = 2;
-  options_.min_write_buffer_number_to_merge = 1;
-  options_.max_background_compactions = 2;
-  options_.max_background_flushes = 2;
-
-  // Create an environment to replace the default logger.
-  if (logger_ == nullptr) {
-    logger_ = std::make_shared<GlogRocksDBLogger>();
-  }
-  options_.info_log = logger_;
-
-  column_families_.push_back(rocksdb::ColumnFamilyDescriptor(
-      rocksdb::kDefaultColumnFamilyName, options_));
-
-  for (const auto& cf_name : kDomains) {
-    column_families_.push_back(
-        rocksdb::ColumnFamilyDescriptor(cf_name, options_));
-  }
-
-  // Make the magic happen.
-  open();
-}
-
-void DBHandle::open() {
-  if (in_memory_) {
-    // Remove when MemEnv is included in librocksdb
-    // options_.env = rocksdb::NewMemEnv(rocksdb::Env::Default());
-    throw std::runtime_error("Cannot start in-memory RocksDB: Requires MemEnv");
-  }
-
-  if (pathExists(path_).ok() && !isReadable(path_).ok()) {
-    throw std::runtime_error("Cannot read RocksDB path: " + path_);
-  }
-
-  if (!kCheckingDB) {
-    VLOG(1) << "Opening RocksDB handle: " << path_;
-  }
-
-  auto s =
-      rocksdb::DB::Open(options_, path_, column_families_, &handles_, &db_);
-  if (!s.ok() || db_ == nullptr) {
-    if (kDBHandleOptionRequireWrite) {
-      // A failed open in R/W mode is a runtime error.
-      throw std::runtime_error(s.ToString());
-    }
-
-    if (!kCheckingDB) {
-      VLOG(1) << "Opening RocksDB failed: Continuing with read-only support";
-    }
-#if !defined(ROCKSDB_LITE)
-    // RocksDB LITE does not support readonly mode.
-    // The database was readable but could not be opened, either (1) it is not
-    // writable or (2) it is already opened by another process.
-    // Try to open the database in a ReadOnly mode.
-    rocksdb::DB::OpenForReadOnly(
-        options_, path_, column_families_, &handles_, &db_);
-#endif
-    // Also disable event publishers.
-    Flag::updateValue("disable_events", "true");
-    read_only_ = true;
-  }
-
-  // RocksDB may not create/append a directory with acceptable permissions.
-  if (!read_only_ && chmod(path_.c_str(), S_IRWXU) != 0) {
-    throw std::runtime_error("Cannot set permissions on RocksDB path: " +
-                             path_);
-  }
-}
-
-DBHandle::~DBHandle() { close(); }
-
-void DBHandle::close() {
-  for (auto handle : handles_) {
-    delete handle;
-  }
-
-  if (db_ != nullptr) {
-    delete db_;
-  }
-}
-
-/////////////////////////////////////////////////////////////////////////////
-// getInstance methods
-/////////////////////////////////////////////////////////////////////////////
-
-DBHandleRef DBHandle::getInstance() {
-  return getInstance(FLAGS_database_path, FLAGS_database_in_memory);
-}
-
-bool DBHandle::checkDB() {
-  // Allow database instances to check if a status/sanity check was requested.
-  kCheckingDB = true;
-  try {
-    auto handle = DBHandle(FLAGS_database_path, FLAGS_database_in_memory);
-    kCheckingDB = false;
-    if (kDBHandleOptionRequireWrite && handle.read_only_) {
-      return false;
-    }
-  } catch (const std::exception& e) {
-    kCheckingDB = false;
-    VLOG(1) << e.what();
-    return false;
-  }
-  return true;
-}
-
-DBHandleRef DBHandle::getInstanceInMemory() { return getInstance("", true); }
-
-DBHandleRef DBHandle::getInstanceAtPath(const std::string& path) {
-  return getInstance(path, false);
-}
-
-DBHandleRef DBHandle::getInstance(const std::string& path, bool in_memory) {
-  static DBHandleRef db_handle = DBHandleRef(new DBHandle(path, in_memory));
-  return db_handle;
-}
-
-void DBHandle::resetInstance(const std::string& path, bool in_memory) {
-  close();
-
-  path_ = path;
-  in_memory_ = in_memory;
-  open();
-}
-
-/////////////////////////////////////////////////////////////////////////////
-// getters and setters
-/////////////////////////////////////////////////////////////////////////////
-
-rocksdb::DB* DBHandle::getDB() const { return db_; }
-
-rocksdb::ColumnFamilyHandle* DBHandle::getHandleForColumnFamily(
-    const std::string& cf) const {
-  try {
-    for (size_t i = 0; i < kDomains.size(); i++) {
-      if (kDomains[i] == cf) {
-        return handles_[i];
-      }
-    }
-  } catch (const std::exception& e) {
-    // pass through and return nullptr
-  }
-  return nullptr;
-}
-
-/////////////////////////////////////////////////////////////////////////////
-// Data manipulation methods
-/////////////////////////////////////////////////////////////////////////////
-
-Status DBHandle::Get(const std::string& domain,
-                     const std::string& key,
-                     std::string& value) const {
-  if (getDB() == nullptr) {
-    return Status(1, "Database not opened");
-  }
-  auto cfh = getHandleForColumnFamily(domain);
-  if (cfh == nullptr) {
-    return Status(1, "Could not get column family for " + domain);
-  }
-  auto s = getDB()->Get(rocksdb::ReadOptions(), cfh, key, &value);
-  return Status(s.code(), s.ToString());
-}
-
-Status DBHandle::Put(const std::string& domain,
-                     const std::string& key,
-                     const std::string& value) const {
-  if (read_only_) {
-    return Status(0, "Database in readonly mode");
-  }
-
-  auto cfh = getHandleForColumnFamily(domain);
-  if (cfh == nullptr) {
-    return Status(1, "Could not get column family for " + domain);
-  }
-  auto s = getDB()->Put(rocksdb::WriteOptions(), cfh, key, value);
-  if (s.code() != 0 && s.IsIOError()) {
-    // An error occurred, check if it is an IO error and remove the offending
-    // specific filename or log name.
-    std::string error_string = s.ToString();
-    size_t error_pos = error_string.find_last_of(":");
-    if (error_pos != std::string::npos) {
-      return Status(s.code(), "IOError: " + error_string.substr(error_pos + 2));
-    }
-  }
-  return Status(s.code(), s.ToString());
-}
-
-Status DBHandle::Delete(const std::string& domain,
-                        const std::string& key) const {
-  if (read_only_) {
-    return Status(0, "Database in readonly mode");
-  }
-
-  auto cfh = getHandleForColumnFamily(domain);
-  if (cfh == nullptr) {
-    return Status(1, "Could not get column family for " + domain);
-  }
-  auto options = rocksdb::WriteOptions();
-
-  // We could sync here, but large deletes will cause multi-syncs.
-  // For example: event record expirations found in an expired index.
-  // options.sync = true;
-  auto s = getDB()->Delete(options, cfh, key);
-  return Status(s.code(), s.ToString());
-}
-
-Status DBHandle::Scan(const std::string& domain,
-                      std::vector<std::string>& results,
-                      size_t max) const {
-  // Trampoline into scan prefix with an empty prefix requirement.
-  return ScanPrefix(domain, results, "", max);
-}
-
-Status DBHandle::ScanPrefix(const std::string& domain,
-                            std::vector<std::string>& results,
-                            const std::string& prefix,
-                            size_t max) const {
-  if (getDB() == nullptr) {
-    return Status(1, "Database not opened");
-  }
-
-  auto cfh = getHandleForColumnFamily(domain);
-  if (cfh == nullptr) {
-    return Status(1, "Could not get column family for " + domain);
-  }
-  auto options = rocksdb::ReadOptions();
-  options.verify_checksums = false;
-  options.fill_cache = false;
-  auto it = getDB()->NewIterator(rocksdb::ReadOptions(), cfh);
-  if (it == nullptr) {
-    return Status(1, "Could not get iterator for " + domain);
-  }
-
-  size_t count = 0;
-  for (it->SeekToFirst(); it->Valid(); it->Next()) {
-    auto key = it->key().ToString();
-    if (key.find(prefix) == 0) {
-      results.push_back(std::move(key));
-      if (max > 0 && ++count >= max) {
-        break;
-      }
-    }
-  }
-  delete it;
-  return Status(0, "OK");
-}
-
-Status RocksDatabasePlugin::get(const std::string& domain,
-                                const std::string& key,
-                                std::string& value) const {
-  return DBHandle::getInstance()->Get(domain, key, value);
-}
-
-Status RocksDatabasePlugin::put(const std::string& domain,
-                                const std::string& key,
-                                const std::string& value) {
-  return DBHandle::getInstance()->Put(domain, key, value);
-}
-
-Status RocksDatabasePlugin::remove(const std::string& domain,
-                                   const std::string& key) {
-  return DBHandle::getInstance()->Delete(domain, key);
-}
-
-Status RocksDatabasePlugin::scan(const std::string& domain,
-                                 std::vector<std::string>& results,
-                                 size_t max) const {
-  return DBHandle::getInstance()->Scan(domain, results, max);
-}
-}

osquery/database/db_handle.h

@@ -1,296 +0,0 @@
-/*
- *  Copyright (c) 2014-present, Facebook, Inc.
- *  All rights reserved.
- *
- *  This source code is licensed under the BSD-style license found in the
- *  LICENSE file in the root directory of this source tree. An additional grant
- *  of patent rights can be found in the PATENTS file in the same directory.
- *
- */
-
-#pragma once
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include <rocksdb/db.h>
-#include <rocksdb/env.h>
-#include <rocksdb/options.h>
-
-#include <boost/noncopyable.hpp>
-
-#include <osquery/core.h>
-#include <osquery/flags.h>
-
-namespace osquery {
-
-DECLARE_string(database_path);
-
-class DBHandle;
-typedef std::shared_ptr<DBHandle> DBHandleRef;
-
-class GlogRocksDBLogger : public rocksdb::Logger {
- public:
-  // We intend to override a virtual method that is overloaded.
-  using rocksdb::Logger::Logv;
-  void Logv(const char* format, va_list ap) override;
-};
-
-/**
- * @brief RAII singleton around RocksDB database access.
- *
- * Accessing RocksDB necessitates creating several pointers which must be
- * carefully memory managed. DBHandle offers you a singleton which takes
- * care of acquiring and releasing the relevant pointers and data structures
- * for you.
- */
-class DBHandle {
- public:
-  /// Removes every column family handle and single DB handle/lock.
-  ~DBHandle();
-
-  /**
-   * @brief The primary way to access the DBHandle singleton.
-   *
-   * DBHandle::getInstance() provides access to the DBHandle singleton.
-   *
-   * @code{.cpp}
-   *   auto db = DBHandle::getInstance();
-   *   std::string value;
-   *   auto status = db->Get("default", "foo", value);
-   *   if (status.ok()) {
-   *     assert(value == "bar");
-   *   }
-   * @endcode
-   *
-   * @return a shared pointer to an instance of DBHandle
-   */
-  static DBHandleRef getInstance();
-
-  /**
-   * @brief Check the sanity of the database configuration options
-   *
-   * Create a handle to the backing store using the database configuration.
-   * Catch any instance creation exceptions and release the handle immediately.
-   *
-   * @return Success if a handle was created without error.
-   */
-  static bool checkDB();
-
-  /// Require all DBHandle accesses to open a read and write handle.
-  static void setRequireWrite(bool rw) { kDBHandleOptionRequireWrite = rw; }
-
-  /// Allow DBHandle creations.
-  static void setAllowOpen(bool ao) { kDBHandleOptionAllowOpen = ao; }
-
- public:
-  /////////////////////////////////////////////////////////////////////////////
-  // Data access methods
-  /////////////////////////////////////////////////////////////////////////////
-
-  /**
-   * @brief Get data from the database for a given domain and key.
-   *
-   * @param domain the "domain" or "column family"
-   * @param key the string key
-   * @param value the output string container, will be populated with data
-   *
-   * @return operation success or failure
-   */
-  Status Get(const std::string& domain,
-             const std::string& key,
-             std::string& value) const;
-
-  /**
-   * @brief Put data into the database identified by a domain and key.
-   *
-   * @param domain the "domain" or "column family"
-   * @param key the string key
-   * @param value the data in a string container
-   *
-   * @return operation success or failure
-   */
-  Status Put(const std::string& domain,
-             const std::string& key,
-             const std::string& value) const;
-
-  /**
-   * @brief Delete data from the database given a domain and key.
-   *
-   * @param domain the "domain" or "column family"
-   * @param key the string key
-   *
-   * @return operation success or failure
-   */
-  Status Delete(const std::string& domain, const std::string& key) const;
-
-  /**
-   * @brief List the keys in a "domain"
-   *
-   * @param domain the "domain" or "column family"
-   * @param results an output list of all keys within the domain
-   * @param optional max limit the number of result keys to a max
-   *
-   * @return operation success or failure
-   */
-  Status Scan(const std::string& domain,
-              std::vector<std::string>& results,
-              size_t max = 0) const;
-
-  /**
-   * @brief List the data in a "domain"
-   *
-   * @param domain the "domain" or "column family"
-   * @param results an output list of all keys with the given prefix
-   * @param prefix require each key to contain this string prefix
-   * @param optional max limit the number of result keys to a max
-   *
-   * @return operation success or failure
-   */
-  Status ScanPrefix(const std::string& domain,
-                    std::vector<std::string>& results,
-                    const std::string& prefix,
-                    size_t max = 0) const;
-
- private:
-  /**
-   * @brief Default constructor
-   *
-   * DBHandle's constructor takes care of properly connecting to RocksDB and
-   * ensuring that all necessary column families are created. The resulting
-   * database handle can then be accessed via DBHandle::getDB() and the
-   * success of the connection can be determined by inspecting the resulting
-   * status code via DBHandle::getStatus()
-   */
-  DBHandle();
-
-  /**
-   * @brief Internal only constructor used to create instances of DBHandle.
-   *
-   * This constructor allows you to specify a few more details about how you'd
-   * like DBHandle to be used. This is only used internally, so you should
-   * never actually use it.
-   *
-   * @param path the path to create/access the database
-   * @param in_memory a boolean indicating whether or not the database should
-   * be creating in memory or not.
-   */
-  DBHandle(const std::string& path, bool in_memory);
-
-  /**
-   * @brief A method which allows you to override the database path
-   *
-   * This should only be used by unit tests. Never use it in production code.
-   *
-   * @return a shared pointer to an instance of DBHandle
-   */
-  static DBHandleRef getInstanceAtPath(const std::string& path);
-
-  /**
-   * @brief A method which gets you an in-memory RocksDB instance.
-   *
-   * This should only be used by unit tests. Never use it in production code.
-   *
-   * @return a shared pointer to an instance of DBHandle
-   */
-  static DBHandleRef getInstanceInMemory();
-
-  /**
-   * @brief A method which allows you to configure various aspects of RocksDB
-   * database options.
-   *
-   * This should only be used by unit tests. Never use it in production code.
-   *
-   * @param path the path to create/access the database
-   * @param in_memory a boolean indicating whether or not the database should
-   * be creating in memory or not.
-   *
-   * @return a shared pointer to an instance of DBHandle
-   */
-  static DBHandleRef getInstance(const std::string& path, bool in_memory);
-
-  /// Allow friend classes, such as unit tests, to reset the instance.
-  void resetInstance(const std::string& path, bool in_memory);
-
-  /// Perform the DB open work.
-  void open();
-
-  /// Perform the DB close work.
-  void close();
-
-  /**
-   * @brief Private helper around accessing the column family handle for a
-   * specific column family, based on its name
-   */
-  rocksdb::ColumnFamilyHandle* getHandleForColumnFamily(
-      const std::string& cf) const;
-
-  /**
-   * @brief Helper method which can be used to get a raw pointer to the
-   * underlying RocksDB database handle
-   *
-   * You probably shouldn't use this. DBHandle::getDB() should only be used
-   * when you're positive that it's the right thing to use.
-   *
-   * @return a pointer to the underlying RocksDB database handle
-   */
-  rocksdb::DB* getDB() const;
-
- public:
-  /// Control availability of the RocksDB handle (default false).
-  static bool kDBHandleOptionAllowOpen;
-  // The database must be opened in a R/W mode (default false).
-  static bool kDBHandleOptionRequireWrite;
-
- private:
-  /////////////////////////////////////////////////////////////////////////////
-  // Private members
-  /////////////////////////////////////////////////////////////////////////////
-
-  /// The database handle
-  rocksdb::DB* db_{nullptr};
-
-  /// RocksDB logger instance.
-  std::shared_ptr<GlogRocksDBLogger> logger_{nullptr};
-
-  /// Column family descriptors which are used to connect to RocksDB
-  std::vector<rocksdb::ColumnFamilyDescriptor> column_families_;
-
-  /// A vector of pointers to column family handles
-  std::vector<rocksdb::ColumnFamilyHandle*> handles_;
-
-  /// The RocksDB connection options that are used to connect to RocksDB
-  rocksdb::Options options_;
-
-  /// The database was opened in a ReadOnly mode.
-  bool read_only_{false};
-
-  /// Location of RocksDB on disk, blank if in-memory is true.
-  std::string path_;
-
-  /// True if the database was started in an in-memory only mode.
-  bool in_memory_{false};
-
- private:
-  friend class RocksDatabasePlugin;
-  friend class Query;
-  friend class EventSubscriberPlugin;
-
-  /////////////////////////////////////////////////////////////////////////////
-  // Unit tests which can access private members
-  /////////////////////////////////////////////////////////////////////////////
-
-  friend class DBHandleTests;
-  FRIEND_TEST(DBHandleTests, test_get);
-  FRIEND_TEST(DBHandleTests, test_put);
-  FRIEND_TEST(DBHandleTests, test_delete);
-  FRIEND_TEST(DBHandleTests, test_scan);
-  friend class QueryTests;
-  FRIEND_TEST(QueryTests, test_get_query_results);
-  FRIEND_TEST(QueryTests, test_is_query_name_in_database);
-  FRIEND_TEST(QueryTests, test_get_stored_query_names);
-  friend class EventsTests;
-  friend class EventsDatabaseTests;
-};
-}

osquery/database/plugins/ephemeral.cpp

@@ -0,0 +1,101 @@
+/*
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <osquery/database.h>
+#include <osquery/logger.h>
+
+namespace osquery {
+
+DECLARE_string(database_path);
+DECLARE_bool(database_in_memory);
+
+class EphemeralDatabasePlugin : public DatabasePlugin {
+  using DBType = std::map<std::string, std::map<std::string, std::string> >;
+
+ public:
+  /// Data retrieval method.
+  Status get(const std::string& domain,
+             const std::string& key,
+             std::string& value) const override;
+
+  /// Data storage method.
+  Status put(const std::string& domain,
+             const std::string& key,
+             const std::string& value) override;
+
+  /// Data removal method.
+  Status remove(const std::string& domain, const std::string& k) override;
+
+  /// Key/index lookup method.
+  Status scan(const std::string& domain,
+              std::vector<std::string>& results,
+              const std::string& prefix,
+              size_t max = 0) const override;
+
+ public:
+  /// Database workflow: open and setup.
+  Status setUp() override {
+    DBType().swap(db_);
+    return Status(0);
+  }
+
+ private:
+  DBType db_;
+};
+
+/// Backing-storage provider for osquery internal/core.
+REGISTER_INTERNAL(EphemeralDatabasePlugin, "database", "ephemeral");
+
+Status EphemeralDatabasePlugin::get(const std::string& domain,
+                                    const std::string& key,
+                                    std::string& value) const {
+  if (db_.count(domain) > 0 && db_.at(domain).count(key) > 0) {
+    value = db_.at(domain).at(key);
+    return Status(0);
+  } else {
+    return Status(1);
+  }
+}
+
+Status EphemeralDatabasePlugin::put(const std::string& domain,
+                                    const std::string& key,
+                                    const std::string& value) {
+  db_[domain][key] = value;
+  return Status(0);
+}
+
+Status EphemeralDatabasePlugin::remove(const std::string& domain,
+                                       const std::string& k) {
+  db_[domain].erase(k);
+  return Status(0);
+}
+
+Status EphemeralDatabasePlugin::scan(const std::string& domain,
+                                     std::vector<std::string>& results,
+                                     const std::string& prefix,
+                                     size_t max) const {
+  if (db_.count(domain) == 0) {
+    return Status(0);
+  }
+
+  for (const auto& key : db_.at(domain)) {
+    if (!prefix.empty() &&
+        !(std::mismatch(prefix.begin(), prefix.end(), key.first.begin())
+              .first == prefix.end())) {
+      continue;
+    }
+    results.push_back(key.first);
+    if (max > 0 && results.size() >= max) {
+      break;
+    }
+  }
+  return Status(0);
+}
+}

osquery/database/plugins/rocksdb.cpp

@@ -0,0 +1,350 @@
+/*
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <mutex>
+
+#include <sys/stat.h>
+
+#include <snappy.h>
+
+#include <rocksdb/db.h>
+#include <rocksdb/env.h>
+#include <rocksdb/options.h>
+
+#include <osquery/database.h>
+#include <osquery/filesystem.h>
+#include <osquery/logger.h>
+
+namespace osquery {
+
+DECLARE_string(database_path);
+DECLARE_bool(database_in_memory);
+
+class GlogRocksDBLogger : public rocksdb::Logger {
+ public:
+  // We intend to override a virtual method that is overloaded.
+  using rocksdb::Logger::Logv;
+  void Logv(const char* format, va_list ap) override;
+};
+
+class RocksDBDatabasePlugin : public DatabasePlugin {
+ public:
+  /// Data retrieval method.
+  Status get(const std::string& domain,
+             const std::string& key,
+             std::string& value) const override;
+
+  /// Data storage method.
+  Status put(const std::string& domain,
+             const std::string& key,
+             const std::string& value) override;
+
+  /// Data removal method.
+  Status remove(const std::string& domain, const std::string& k) override;
+
+  /// Key/index lookup method.
+  Status scan(const std::string& domain,
+              std::vector<std::string>& results,
+              const std::string& prefix,
+              size_t max = 0) const override;
+
+ public:
+  /// Database workflow: open and setup.
+  Status setUp() override;
+
+  /// Database workflow: close and cleanup.
+  void tearDown() override { close(); }
+
+  /// Need to tear down open resources,
+  virtual ~RocksDBDatabasePlugin() { close(); }
+
+ private:
+  /// Obtain a close lock and release resources.
+  void close();
+
+  /**
+   * @brief Private helper around accessing the column family handle for a
+   * specific column family, based on its name
+   */
+  rocksdb::ColumnFamilyHandle* getHandleForColumnFamily(
+      const std::string& cf) const;
+
+  /**
+   * @brief Helper method which can be used to get a raw pointer to the
+   * underlying RocksDB database handle
+   *
+   * @return a pointer to the underlying RocksDB database handle
+   */
+  rocksdb::DB* getDB() const;
+
+ private:
+  bool initialized_{false};
+
+  /// The database handle
+  rocksdb::DB* db_{nullptr};
+
+  /// RocksDB logger instance.
+  std::shared_ptr<GlogRocksDBLogger> logger_{nullptr};
+
+  /// Column family descriptors which are used to connect to RocksDB
+  std::vector<rocksdb::ColumnFamilyDescriptor> column_families_;
+
+  /// A vector of pointers to column family handles
+  std::vector<rocksdb::ColumnFamilyHandle*> handles_;
+
+  /// The RocksDB connection options that are used to connect to RocksDB
+  rocksdb::Options options_;
+
+  /// Deconstruction mutex.
+  std::mutex close_mutex_;
+};
+
+/// Backing-storage provider for osquery internal/core.
+REGISTER_INTERNAL(RocksDBDatabasePlugin, "database", "rocksdb");
+
+void GlogRocksDBLogger::Logv(const char* format, va_list ap) {
+  // Convert RocksDB log to string and check if header or level-ed log.
+  std::string log_line;
+  {
+    char buffer[501] = {0};
+    vsnprintf(buffer, 500, format, ap);
+    va_end(ap);
+    if (buffer[0] != '[' || (buffer[1] != 'E' && buffer[1] != 'W')) {
+      return;
+    }
+
+    log_line = buffer;
+  }
+
+  // There is a spurious warning on first open.
+  if (log_line.find("Error when reading") == std::string::npos) {
+    LOG(INFO) << "RocksDB: " << log_line;
+  }
+}
+
+Status RocksDBDatabasePlugin::setUp() {
+  if (!kDBHandleOptionAllowOpen) {
+    LOG(WARNING) << RLOG(1629) << "Not allowed to create DBHandle instance";
+  }
+
+  if (!initialized_) {
+    initialized_ = true;
+    // Set meta-data (mostly) handling options.
+    options_.create_if_missing = true;
+    options_.create_missing_column_families = true;
+    options_.info_log_level = rocksdb::ERROR_LEVEL;
+    options_.log_file_time_to_roll = 0;
+    options_.keep_log_file_num = 10;
+    options_.max_log_file_size = 1024 * 1024 * 1;
+    options_.stats_dump_period_sec = 0;
+
+    // Performance and optimization settings.
+    options_.compression = rocksdb::kNoCompression;
+    options_.compaction_style = rocksdb::kCompactionStyleLevel;
+    options_.arena_block_size = (4 * 1024);
+    options_.write_buffer_size = (4 * 1024) * 100; // 100 blocks.
+    options_.max_write_buffer_number = 2;
+    options_.min_write_buffer_number_to_merge = 1;
+    options_.max_background_compactions = 2;
+    options_.max_background_flushes = 2;
+
+    // Create an environment to replace the default logger.
+    if (logger_ == nullptr) {
+      logger_ = std::make_shared<GlogRocksDBLogger>();
+    }
+    options_.info_log = logger_;
+
+    column_families_.push_back(rocksdb::ColumnFamilyDescriptor(
+        rocksdb::kDefaultColumnFamilyName, options_));
+
+    for (const auto& cf_name : kDomains) {
+      column_families_.push_back(
+          rocksdb::ColumnFamilyDescriptor(cf_name, options_));
+    }
+  }
+
+  // Consume the current settings.
+  // A configuration update may change them, but that does not affect state.
+  path_ = FLAGS_database_path;
+  in_memory_ = FLAGS_database_in_memory;
+
+  if (in_memory_) {
+    // Remove when MemEnv is included in librocksdb
+    // options_.env = rocksdb::NewMemEnv(rocksdb::Env::Default());
+    return Status(1, "Cannot start in-memory RocksDB: Requires MemEnv");
+  }
+
+  if (pathExists(path_).ok() && !isReadable(path_).ok()) {
+    return Status(1, "Cannot read RocksDB path: " + path_);
+  }
+
+  if (!DatabasePlugin::kCheckingDB) {
+    VLOG(1) << "Opening RocksDB handle: " << path_;
+  }
+
+  // Tests may trash calls to setUp, make sure subsequent calls do not leak.
+  close();
+
+  // Attempt to create a RocksDB instance and handles.
+  auto s =
+      rocksdb::DB::Open(options_, path_, column_families_, &handles_, &db_);
+  if (!s.ok() || db_ == nullptr) {
+    if (kDBHandleOptionRequireWrite) {
+      // A failed open in R/W mode is a runtime error.
+      return Status(1, s.ToString());
+    }
+
+    if (!DatabasePlugin::kCheckingDB) {
+      VLOG(1) << "Opening RocksDB failed: Continuing with read-only support";
+    }
+#if !defined(ROCKSDB_LITE)
+    // RocksDB LITE does not support readonly mode.
+    // The database was readable but could not be opened, either (1) it is not
+    // writable or (2) it is already opened by another process.
+    // Try to open the database in a ReadOnly mode.
+    rocksdb::DB::OpenForReadOnly(options_, path_, column_families_, &handles_,
+                                 &db_);
+#endif
+    // Also disable event publishers.
+    Flag::updateValue("disable_events", "true");
+    read_only_ = true;
+  }
+
+  // RocksDB may not create/append a directory with acceptable permissions.
+  if (!read_only_ && chmod(path_.c_str(), S_IRWXU) != 0) {
+    return Status(1, "Cannot set permissions on RocksDB path: " + path_);
+  }
+  return Status(0);
+}
+
+void RocksDBDatabasePlugin::close() {
+  std::unique_lock<std::mutex> lock(close_mutex_);
+  for (auto handle : handles_) {
+    delete handle;
+  }
+  handles_.clear();
+
+  if (db_ != nullptr) {
+    delete db_;
+    db_ = nullptr;
+  }
+}
+
+rocksdb::DB* RocksDBDatabasePlugin::getDB() const { return db_; }
+
+rocksdb::ColumnFamilyHandle* RocksDBDatabasePlugin::getHandleForColumnFamily(
+    const std::string& cf) const {
+  try {
+    for (size_t i = 0; i < kDomains.size(); i++) {
+      if (kDomains[i] == cf) {
+        return handles_[i];
+      }
+    }
+  } catch (const std::exception& e) {
+    // pass through and return nullptr
+  }
+  return nullptr;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// Data manipulation methods
+/////////////////////////////////////////////////////////////////////////////
+
+Status RocksDBDatabasePlugin::get(const std::string& domain,
+                                  const std::string& key,
+                                  std::string& value) const {
+  if (getDB() == nullptr) {
+    return Status(1, "Database not opened");
+  }
+  auto cfh = getHandleForColumnFamily(domain);
+  if (cfh == nullptr) {
+    return Status(1, "Could not get column family for " + domain);
+  }
+  auto s = getDB()->Get(rocksdb::ReadOptions(), cfh, key, &value);
+  return Status(s.code(), s.ToString());
+}
+
+Status RocksDBDatabasePlugin::put(const std::string& domain,
+                                  const std::string& key,
+                                  const std::string& value) {
+  if (read_only_) {
+    return Status(0, "Database in readonly mode");
+  }
+
+  auto cfh = getHandleForColumnFamily(domain);
+  if (cfh == nullptr) {
+    return Status(1, "Could not get column family for " + domain);
+  }
+  auto s = getDB()->Put(rocksdb::WriteOptions(), cfh, key, value);
+  if (s.code() != 0 && s.IsIOError()) {
+    // An error occurred, check if it is an IO error and remove the offending
+    // specific filename or log name.
+    std::string error_string = s.ToString();
+    size_t error_pos = error_string.find_last_of(":");
+    if (error_pos != std::string::npos) {
+      return Status(s.code(), "IOError: " + error_string.substr(error_pos + 2));
+    }
+  }
+  return Status(s.code(), s.ToString());
+}
+
+Status RocksDBDatabasePlugin::remove(const std::string& domain,
+                                     const std::string& key) {
+  if (read_only_) {
+    return Status(0, "Database in readonly mode");
+  }
+
+  auto cfh = getHandleForColumnFamily(domain);
+  if (cfh == nullptr) {
+    return Status(1, "Could not get column family for " + domain);
+  }
+  auto options = rocksdb::WriteOptions();
+
+  // We could sync here, but large deletes will cause multi-syncs.
+  // For example: event record expirations found in an expired index.
+  // options.sync = true;
+  auto s = getDB()->Delete(options, cfh, key);
+  return Status(s.code(), s.ToString());
+}
+
+Status RocksDBDatabasePlugin::scan(const std::string& domain,
+                                   std::vector<std::string>& results,
+                                   const std::string& prefix,
+                                   size_t max) const {
+  if (getDB() == nullptr) {
+    return Status(1, "Database not opened");
+  }
+
+  auto cfh = getHandleForColumnFamily(domain);
+  if (cfh == nullptr) {
+    return Status(1, "Could not get column family for " + domain);
+  }
+  auto options = rocksdb::ReadOptions();
+  options.verify_checksums = false;
+  options.fill_cache = false;
+  auto it = getDB()->NewIterator(rocksdb::ReadOptions(), cfh);
+  if (it == nullptr) {
+    return Status(1, "Could not get iterator for " + domain);
+  }
+
+  size_t count = 0;
+  for (it->SeekToFirst(); it->Valid(); it->Next()) {
+    auto key = it->key().ToString();
+    if (key.find(prefix) == 0) {
+      results.push_back(std::move(key));
+      if (max > 0 && ++count >= max) {
+        break;
+      }
+    }
+  }
+  delete it;
+  return Status(0, "OK");
+}
+}

osquery/database/plugins/sqlite.cpp

@@ -0,0 +1,277 @@
+/*
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <mutex>
+
+#include <sqlite3.h>
+
+#include <sys/stat.h>
+
+#include <osquery/database.h>
+#include <osquery/filesystem.h>
+#include <osquery/logger.h>
+
+namespace osquery {
+
+DECLARE_string(database_path);
+DECLARE_bool(database_in_memory);
+
+const std::map<std::string, std::string> kDBSettings = {
+    {"synchronous", "OFF"},      {"count_changes", "OFF"},
+    {"default_temp_store", "2"}, {"auto_vacuum", "FULL"},
+    {"journal_mode", "OFF"},     {"cache_size", "1000"},
+    {"page_count", "1000"},
+};
+
+class SQLiteDatabasePlugin : public DatabasePlugin {
+ public:
+  /// Data retrieval method.
+  Status get(const std::string& domain,
+             const std::string& key,
+             std::string& value) const override;
+
+  /// Data storage method.
+  Status put(const std::string& domain,
+             const std::string& key,
+             const std::string& value) override;
+
+  /// Data removal method.
+  Status remove(const std::string& domain, const std::string& k) override;
+
+  /// Key/index lookup method.
+  Status scan(const std::string& domain,
+              std::vector<std::string>& results,
+              const std::string& prefix,
+              size_t max = 0) const override;
+
+ public:
+  /// Database workflow: open and setup.
+  Status setUp() override;
+
+  /// Database workflow: close and cleanup.
+  void tearDown() override { close(); }
+
+  /// Need to tear down open resources,
+  virtual ~SQLiteDatabasePlugin() { close(); }
+
+ private:
+  void close();
+
+ private:
+  /// The long-lived sqlite3 database.
+  sqlite3* db_{nullptr};
+
+  /// Deconstruction mutex.
+  std::mutex close_mutex_;
+};
+
+/// Backing-storage provider for osquery internal/core.
+REGISTER_INTERNAL(SQLiteDatabasePlugin, "database", "sqlite");
+
+Status SQLiteDatabasePlugin::setUp() {
+  if (!DatabasePlugin::kDBHandleOptionAllowOpen) {
+    LOG(WARNING) << RLOG(1629) << "Not allowed to create DBHandle instance";
+  }
+
+  // Consume the current settings.
+  // A configuration update may change them, but that does not affect state.
+  path_ = FLAGS_database_path;
+  in_memory_ = FLAGS_database_in_memory;
+
+  if (!in_memory_ && pathExists(path_).ok() && !isReadable(path_).ok()) {
+    return Status(1, "Cannot read database path: " + path_);
+  }
+
+  if (!DatabasePlugin::kCheckingDB) {
+    VLOG(1) << "Opening database handle: " << path_;
+  }
+
+  // Tests may trash calls to setUp, make sure subsequent calls do not leak.
+  close();
+
+  // Can actually try to create a DB in memory with: in_memory_.
+  // Open the SQLite backing storage at path_
+  auto result = sqlite3_open_v2(
+      ((in_memory_) ? ":memory:" : path_.c_str()),
+      &db_,
+      (SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE),
+      nullptr);
+
+  if (result != SQLITE_OK || db_ == nullptr) {
+    if (DatabasePlugin::kDBHandleOptionRequireWrite) {
+      close();
+      // A failed open in R/W mode is a runtime error.
+      return Status(1, "Cannot open database: " + std::to_string(result));
+    }
+
+    if (!DatabasePlugin::kCheckingDB) {
+      VLOG(1) << "Opening database failed: Continuing with read-only support";
+    }
+
+    read_only_ = true;
+  }
+
+  if (!read_only_) {
+    for (const auto& domain : kDomains) {
+      std::string q = "create table if not exists " + domain +
+                      " (key TEXT PRIMARY KEY, value TEXT);";
+      result = sqlite3_exec(db_, q.c_str(), nullptr, nullptr, nullptr);
+      if (result != SQLITE_OK) {
+        close();
+        return Status(1, "Cannot create domain: " + domain);
+      }
+    }
+
+    std::string settings;
+    for (const auto& setting : kDBSettings) {
+      settings += "PRAGMA " + setting.first + "=" + setting.second + "; ";
+    }
+    sqlite3_exec(db_, settings.c_str(), nullptr, nullptr, nullptr);
+  }
+
+  // RocksDB may not create/append a directory with acceptable permissions.
+  if (!read_only_ && chmod(path_.c_str(), S_IRWXU) != 0) {
+    close();
+    return Status(1, "Cannot set permissions on database path: " + path_);
+  }
+  return Status(0);
+}
+
+void SQLiteDatabasePlugin::close() {
+  std::unique_lock<std::mutex> lock(close_mutex_);
+  if (db_ != nullptr) {
+    sqlite3_close(db_);
+    db_ = nullptr;
+  }
+}
+
+static int getData(void* argument, int argc, char* argv[], char* column[]) {
+  if (argument == nullptr) {
+    return SQLITE_MISUSE;
+  }
+
+  QueryData* qData = (QueryData*)argument;
+  Row r;
+  for (int i = 0; i < argc; i++) {
+    if (column[i] != nullptr) {
+      r[column[i]] = (argv[i] != nullptr) ? argv[i] : "";
+    }
+  }
+  (*qData).push_back(std::move(r));
+  return 0;
+}
+
+Status SQLiteDatabasePlugin::get(const std::string& domain,
+                                 const std::string& key,
+                                 std::string& value) const {
+  QueryData results;
+  char* err = nullptr;
+  std::string q = "select value from " + domain + " where key = '" + key + "';";
+  sqlite3_exec(db_, q.c_str(), getData, &results, &err);
+  if (err != nullptr) {
+    sqlite3_free(err);
+  }
+
+  // Only assign value if the query found a result.
+  if (results.size() > 0) {
+    value = std::move(results[0]["value"]);
+    return Status(0);
+  }
+  return Status(1);
+}
+
+static void tryVacuum(sqlite3* db) {
+  std::string q =
+      "SELECT (sum(s1.pageno + 1 == s2.pageno) * 1.0 / count(*)) < 0.01 as v "
+      " FROM "
+      "(SELECT pageno FROM dbstat ORDER BY path) AS s1,"
+      "(SELECT pageno FROM dbstat ORDER BY path) AS s2 WHERE "
+      "s1.rowid + 1 = s2.rowid; ";
+
+  QueryData results;
+  sqlite3_exec(db, q.c_str(), getData, &results, nullptr);
+  if (results.size() > 0 && results[0]["v"].back() == '1') {
+    sqlite3_exec(db, "vacuum;", nullptr, nullptr, nullptr);
+  }
+}
+
+Status SQLiteDatabasePlugin::put(const std::string& domain,
+                                 const std::string& key,
+                                 const std::string& value) {
+  if (read_only_) {
+    return Status(0, "Database in readonly mode");
+  }
+
+  sqlite3_stmt* stmt = nullptr;
+  std::string q = "insert or replace into " + domain + " values (?1, ?2);";
+  sqlite3_prepare_v2(db_, q.c_str(), -1, &stmt, nullptr);
+
+  sqlite3_bind_text(stmt, 1, key.c_str(), -1, SQLITE_STATIC);
+  sqlite3_bind_text(stmt, 2, value.c_str(), -1, SQLITE_STATIC);
+  auto rc = sqlite3_step(stmt);
+  if (rc != SQLITE_DONE) {
+    return Status(1);
+  }
+
+  sqlite3_finalize(stmt);
+  if (rand() % 10 == 0) {
+    tryVacuum(db_);
+  }
+  return Status(0);
+}
+
+Status SQLiteDatabasePlugin::remove(const std::string& domain,
+                                    const std::string& key) {
+  if (read_only_) {
+    return Status(0, "Database in readonly mode");
+  }
+
+  sqlite3_stmt* stmt = nullptr;
+  std::string q = "delete from " + domain + " where key IN (?1);";
+  sqlite3_prepare_v2(db_, q.c_str(), -1, &stmt, nullptr);
+
+  sqlite3_bind_text(stmt, 1, key.c_str(), -1, SQLITE_STATIC);
+  auto rc = sqlite3_step(stmt);
+  if (rc != SQLITE_DONE) {
+    return Status(1);
+  }
+
+  sqlite3_finalize(stmt);
+  if (rand() % 10 == 0) {
+    tryVacuum(db_);
+  }
+  return Status(0);
+}
+
+Status SQLiteDatabasePlugin::scan(const std::string& domain,
+                                  std::vector<std::string>& results,
+                                  const std::string& prefix,
+                                  size_t max) const {
+  QueryData _results;
+  char* err = nullptr;
+
+  std::string q =
+      "select key from " + domain + " where key LIKE '" + prefix + "%'";
+  if (max > 0) {
+    q += " limit " + std::to_string(max);
+  }
+  sqlite3_exec(db_, q.c_str(), getData, &_results, &err);
+  if (err != nullptr) {
+    sqlite3_free(err);
+  }
+
+  // Only assign value if the query found a result.
+  for (auto& r : _results) {
+    results.push_back(std::move(r["key"]));
+  }
+
+  return Status(0, "OK");
+}
+}

osquery/database/plugins/tests/rocksdb_tests.cpp

@@ -0,0 +1,34 @@
+/*
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <osquery/sql.h>
+
+#include "osquery/database/tests/plugin_tests.h"
+
+namespace osquery {
+
+class RocksDBDatabasePluginTests : public DatabasePluginTests {
+ protected:
+  std::string name() override { return "rocksdb"; }
+};
+
+// Define the default set of database plugin operation tests.
+CREATE_DATABASE_TESTS(RocksDBDatabasePluginTests);
+
+TEST_F(RocksDBDatabasePluginTests, test_rocksdb_loglevel) {
+  // Make sure a log file was created.
+  EXPECT_FALSE(pathExists(path_ + "/LOG"));
+
+  // Make sure no log file is created.
+  // RocksDB logs are intercepted and forwarded to the GLog sink.
+  auto details = SQL::selectAllFrom("file", "path", EQUALS, path_ + "/LOG");
+  ASSERT_EQ(details.size(), 0U);
+}
+}

osquery/database/plugins/tests/sqlite_tests.cpp

@@ -0,0 +1,22 @@
+/*
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include "osquery/database/tests/plugin_tests.h"
+
+namespace osquery {
+
+class SQLiteDatabasePluginTests : public DatabasePluginTests {
+ protected:
+  std::string name() override { return "sqlite"; }
+};
+
+// Define the default set of database plugin operation tests.
+CREATE_DATABASE_TESTS(SQLiteDatabasePluginTests);
+}

osquery/database/query.cpp

@@ -14,21 +14,13 @@
 
 namespace osquery {
 
-/////////////////////////////////////////////////////////////////////////////
-// Data access methods
-/////////////////////////////////////////////////////////////////////////////
-
 Status Query::getPreviousQueryResults(QueryData& results) {
-  return getPreviousQueryResults(results, DBHandle::getInstance());
-}
-
-Status Query::getPreviousQueryResults(QueryData& results, DBHandleRef db) {
   if (!isQueryNameInDatabase()) {
     return Status(0, "Query name not found in database");
   }
 
   std::string raw;
-  auto status = db->Get(kQueries, name_, raw);
+  auto status = getDatabaseValue(kQueries, name_, raw);
   if (!status.ok()) {
     return status;
   }
@@ -41,41 +33,28 @@ Status Query::getPreviousQueryResults(QueryData& results, DBHandleRef db) {
 }
 
 std::vector<std::string> Query::getStoredQueryNames() {
-  return getStoredQueryNames(DBHandle::getInstance());
-}
-
-std::vector<std::string> Query::getStoredQueryNames(DBHandleRef db) {
   std::vector<std::string> results;
-  db->Scan(kQueries, results);
+  scanDatabaseKeys(kQueries, results);
   return results;
 }
 
 bool Query::isQueryNameInDatabase() {
-  return isQueryNameInDatabase(DBHandle::getInstance());
+  auto names = Query::getStoredQueryNames();
-}
-
-bool Query::isQueryNameInDatabase(DBHandleRef db) {
-  auto names = Query::getStoredQueryNames(db);
   return std::find(names.begin(), names.end(), name_) != names.end();
 }
 
-Status Query::addNewResults(const osquery::QueryData& qd) {
+Status Query::addNewResults(const QueryData& qd) {
-  return addNewResults(qd, DBHandle::getInstance());
-}
-
-Status Query::addNewResults(const QueryData& qd, DBHandleRef db) {
   DiffResults dr;
-  return addNewResults(qd, dr, false, db);
+  return addNewResults(qd, dr, false);
 }
 
 Status Query::addNewResults(const QueryData& qd, DiffResults& dr) {
-  return addNewResults(qd, dr, true, DBHandle::getInstance());
+  return addNewResults(qd, dr, true);
 }
 
 Status Query::addNewResults(const QueryData& current_qd,
                             DiffResults& dr,
-                            bool calculate_diff,
+                            bool calculate_diff) {
-                            DBHandleRef db) {
   // Get the rows from the last run of this query name.
   QueryData previous_qd;
   auto status = getPreviousQueryResults(previous_qd);
@@ -97,7 +76,7 @@ Status Query::addNewResults(const QueryData& current_qd,
       return status;
     }
 
-    status = db->Put(kQueries, name_, json);
+    status = setDatabaseValue(kQueries, name_, json);
     if (!status.ok()) {
       return status;
     }

osquery/database/query.h

@@ -14,10 +14,8 @@
 #include <string>
 #include <vector>
 
-#include <osquery/status.h>
 #include <osquery/database.h>
-
+#include <osquery/status.h>
-#include "osquery/database/db_handle.h"
 
 namespace osquery {
 
@@ -58,23 +56,6 @@ class Query {
    */
   Status getPreviousQueryResults(QueryData& results);
 
- private:
-  /**
-   * @brief Serialize the data in RocksDB into a useful data structure using a
-   * custom database handle
-   *
-   * This method is the same as getHistoricalQueryResults, but with the
-   * addition of a parameter which allows you to pass a custom RocksDB
-   * database handle.
-   *
-   * @param hQR the output HistoricalQueryResults struct
-   * @param db a shared pointer to a custom DBHandle
-   *
-   * @return the success or failure of the operation
-   * @see getHistoricalQueryResults
-   */
-  Status getPreviousQueryResults(QueryData& results, DBHandleRef db);
-
  public:
   /**
    * @brief Get the names of all historical queries that are stored in RocksDB
@@ -88,22 +69,6 @@ class Query {
    */
   static std::vector<std::string> getStoredQueryNames();
 
- private:
-  /**
-   * @brief Get the names of all historical queries that are stored in RocksDB
-   * using a custom database handle
-   *
-   * This method is the same as getStoredQueryNames(), but with the addition
-   * of a parameter which allows you to pass a custom RocksDB database handle.
-   *
-   * @param db a custom RocksDB database handle
-   *
-   * @return a vector containing the string names of all scheduled queries
-   *
-   * @see getStoredQueryNames()
-   */
-  static std::vector<std::string> getStoredQueryNames(DBHandleRef db);
-
  public:
   /**
    * @brief Accessor method for checking if a given scheduled query exists in
@@ -113,20 +78,6 @@ class Query {
    */
   bool isQueryNameInDatabase();
 
- private:
-  /**
-   * @brief Accessor method for checking if a given scheduled query exists in
-   * the database, using a custom database handle
-   *
-   * This method is the same as isQueryNameInDatabase(), but with the addition
-   * of a parameter which allows you to pass a custom RocksDB database handle
-   *
-   * @param db a custom RocksDB database handle
-   *
-   * @return does the scheduled query which is already exists in the database
-   */
-  bool isQueryNameInDatabase(DBHandleRef db);
-
  public:
   /**
    * @brief Add a new set of results to the persistant storage
@@ -143,24 +94,6 @@ class Query {
    */
   Status addNewResults(const QueryData& qd);
 
- private:
-  /**
-   * @brief Add a new set of results to the persistant storage using a custom
-   * database handle
-   *
-   * This method is the same as addNewResults(), but with the addition of a
-   * parameter which allows you to pass a custom RocksDB database handle
-   *
-   * @param qd the QueryData object, which has the results of the query which
-   * you would like to store
-   * @param unix_time the time that the query was executed
-   * @param db a custom RocksDB database handle
-   *
-   * @return an instance of osquery::Status indicating the success or failure
-   * of the operation
-   */
-  Status addNewResults(const QueryData& qd, DBHandleRef db);
-
  public:
   /**
    * @brief Add a new set of results to the persistent storage and get back
@@ -192,8 +125,7 @@ class Query {
    */
   Status addNewResults(const QueryData& qd,
                        DiffResults& dr,
-                       bool calculate_diff,
+                       bool calculate_diff);
-                       DBHandleRef db);
 
  public:
   /**
@@ -205,22 +137,6 @@ class Query {
    */
   Status getCurrentResults(QueryData& qd);
 
- private:
-  /**
-   * @brief A getter for the most recent result set for a scheduled query,
-   * but with the addition of a parameter which allows you to pass a custom
-   * RocksDB database handle.
-   *
-   * This method is the same as Query::getCurrentResults, but with addition of a
-   * parameter which allows you to pass a custom RocksDB database handle.
-   *
-   * @param qd the output QueryData object
-   * @param db a custom RocksDB database handle
-   *
-   * @return the success or failure of the operation
-   */
-  Status getCurrentResults(QueryData& qd, DBHandleRef db);
-
  private:
   /////////////////////////////////////////////////////////////////////////////
   // Private members
@@ -228,6 +144,7 @@ class Query {
 
   /// The scheduled query and internal
   ScheduledQuery query_;
+
   /// The scheduled query name.
   std::string name_;
 

osquery/database/tests/db_handle_tests.cpp

@@ -1,129 +0,0 @@
-/*
- *  Copyright (c) 2014-present, Facebook, Inc.
- *  All rights reserved.
- *
- *  This source code is licensed under the BSD-style license found in the
- *  LICENSE file in the root directory of this source tree. An additional grant
- *  of patent rights can be found in the PATENTS file in the same directory.
- *
- */
-
-#include <algorithm>
-
-#include <boost/filesystem/operations.hpp>
-
-#include <gtest/gtest.h>
-
-#include <osquery/filesystem.h>
-#include <osquery/flags.h>
-#include <osquery/logger.h>
-#include <osquery/sql.h>
-#include <osquery/tables.h>
-
-#include "osquery/database/db_handle.h"
-#include "osquery/core/test_util.h"
-
-namespace osquery {
-
-class DBHandleTests : public testing::Test {
- public:
-  void SetUp() {
-    // A database instance is setup during testing initialize (initTesting).
-    // We need to reset that instance to test unordered expectations.
-    path_ = kTestWorkingDirectory + std::to_string(rand() % 10000 + 20000);
-    DBHandle::getInstance()->resetInstance(path_, false);
-
-    cfh_queries_ = DBHandle::getInstance()->getHandleForColumnFamily(kQueries);
-    cfh_foobar_ =
-        DBHandle::getInstance()->getHandleForColumnFamily("foobartest");
-    db_ = DBHandle::getInstance();
-  }
-
-  void TearDown() {
-    // Clean the transient instance and reset to the testing instance.
-    boost::filesystem::remove_all(path_);
-    auto path = Flag::getValue("database_path");
-    DBHandle::getInstance()->resetInstance(path, false);
-  }
-
- public:
-  std::string path_;
-  rocksdb::ColumnFamilyHandle* cfh_queries_;
-  rocksdb::ColumnFamilyHandle* cfh_foobar_;
-  std::shared_ptr<DBHandle> db_;
-};
-
-TEST_F(DBHandleTests, test_singleton_on_disk) {
-  auto db1 = DBHandle::getInstance();
-  auto db2 = DBHandle::getInstance();
-  EXPECT_EQ(db1, db2);
-}
-
-TEST_F(DBHandleTests, test_get_handle_for_column_family) {
-  ASSERT_TRUE(cfh_queries_ != nullptr);
-  ASSERT_TRUE(cfh_foobar_ == nullptr);
-}
-
-TEST_F(DBHandleTests, test_get) {
-  db_->getDB()->Put(
-      rocksdb::WriteOptions(), cfh_queries_, "test_query_123", "{}");
-  std::string r;
-  std::string key = "test_query_123";
-  auto s = db_->Get(kQueries, key, r);
-  EXPECT_TRUE(s.ok());
-  EXPECT_EQ(s.toString(), "OK");
-  EXPECT_EQ(r, "{}");
-}
-
-TEST_F(DBHandleTests, test_put) {
-  auto s = db_->Put(kQueries, "test_put", "bar");
-  EXPECT_TRUE(s.ok());
-  EXPECT_EQ(s.toString(), "OK");
-}
-
-TEST_F(DBHandleTests, test_delete) {
-  db_->Put(kQueries, "test_delete", "baz");
-  auto s = db_->Delete(kQueries, "test_delete");
-  EXPECT_TRUE(s.ok());
-  EXPECT_EQ(s.toString(), "OK");
-}
-
-TEST_F(DBHandleTests, test_scan) {
-  db_->Put(kQueries, "test_scan_foo1", "baz");
-  db_->Put(kQueries, "test_scan_foo2", "baz");
-  db_->Put(kQueries, "test_scan_foo3", "baz");
-
-  std::vector<std::string> keys;
-  std::vector<std::string> expected = {
-      "test_scan_foo1", "test_scan_foo2", "test_scan_foo3"};
-  auto s = db_->Scan(kQueries, keys);
-  EXPECT_TRUE(s.ok());
-  EXPECT_EQ(s.toString(), "OK");
-  EXPECT_EQ(keys.size(), 3U);
-  for (const auto& i : expected) {
-    EXPECT_NE(std::find(keys.begin(), keys.end(), i), keys.end());
-  }
-}
-
-TEST_F(DBHandleTests, test_scan_limit) {
-  db_->Put(kQueries, "test_scan_foo1", "baz");
-  db_->Put(kQueries, "test_scan_foo2", "baz");
-  db_->Put(kQueries, "test_scan_foo3", "baz");
-
-  std::vector<std::string> keys;
-  auto s = db_->Scan(kQueries, keys, 2);
-  EXPECT_TRUE(s.ok());
-  EXPECT_EQ(s.toString(), "OK");
-  EXPECT_EQ(keys.size(), 2U);
-}
-
-TEST_F(DBHandleTests, test_rocksdb_loglevel) {
-  // Make sure a log file was created.
-  EXPECT_FALSE(pathExists(path_ + "/LOG"));
-
-  // Make sure no log file is created.
-  // RocksDB logs are intercepted and forwarded to the GLog sink.
-  auto details = SQL::selectAllFrom("file", "path", EQUALS, path_ + "/LOG");
-  ASSERT_EQ(details.size(), 0U);
-}
-}

osquery/database/tests/plugin_tests.cpp

@@ -0,0 +1,88 @@
+/*
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include "osquery/database/tests/plugin_tests.h"
+
+namespace osquery {
+
+class EphemeralDatabasePluginTests : public DatabasePluginTests {
+ protected:
+  std::string name() override { return "ephemeral"; }
+};
+
+// Define the default set of database plugin operation tests.
+CREATE_DATABASE_TESTS(EphemeralDatabasePluginTests);
+
+void DatabasePluginTests::testPluginCheck() {
+  // Do not worry about multiple set-active calls.
+  // For testing purposes they should be idempotent.
+  EXPECT_TRUE(Registry::setActive("database", getName()));
+
+  // Get an instance of the database plugin and call check.
+  auto plugin = Registry::get("database", getName());
+  auto db_plugin = std::dynamic_pointer_cast<DatabasePlugin>(plugin);
+  EXPECT_TRUE(db_plugin->checkDB());
+
+  // Testing relies on database resetting too.
+  EXPECT_TRUE(db_plugin->reset());
+}
+
+void DatabasePluginTests::testPut() {
+  auto s = getPlugin()->put(kQueries, "test_put", "bar");
+  EXPECT_TRUE(s.ok());
+  EXPECT_EQ(s.getMessage(), "OK");
+}
+
+void DatabasePluginTests::testGet() {
+  getPlugin()->put(kQueries, "test_get", "bar");
+
+  std::string r;
+  auto s = getPlugin()->get(kQueries, "test_get", r);
+  EXPECT_TRUE(s.ok());
+  EXPECT_EQ(s.getMessage(), "OK");
+  EXPECT_EQ(r, "bar");
+}
+
+void DatabasePluginTests::testDelete() {
+  getPlugin()->put(kQueries, "test_delete", "baz");
+  auto s = getPlugin()->remove(kQueries, "test_delete");
+  EXPECT_TRUE(s.ok());
+  EXPECT_EQ(s.getMessage(), "OK");
+}
+
+void DatabasePluginTests::testScan() {
+  getPlugin()->put(kQueries, "test_scan_foo1", "baz");
+  getPlugin()->put(kQueries, "test_scan_foo2", "baz");
+  getPlugin()->put(kQueries, "test_scan_foo3", "baz");
+
+  std::vector<std::string> keys;
+  std::vector<std::string> expected = {"test_scan_foo1", "test_scan_foo2",
+                                       "test_scan_foo3"};
+  auto s = getPlugin()->scan(kQueries, keys, "");
+  EXPECT_TRUE(s.ok());
+  EXPECT_EQ(s.getMessage(), "OK");
+  EXPECT_EQ(keys.size(), 3U);
+  for (const auto& i : expected) {
+    EXPECT_NE(std::find(keys.begin(), keys.end(), i), keys.end());
+  }
+}
+
+void DatabasePluginTests::testScanLimit() {
+  getPlugin()->put(kQueries, "test_scan_foo1", "baz");
+  getPlugin()->put(kQueries, "test_scan_foo2", "baz");
+  getPlugin()->put(kQueries, "test_scan_foo3", "baz");
+
+  std::vector<std::string> keys;
+  auto s = getPlugin()->scan(kQueries, keys, "", 2);
+  EXPECT_TRUE(s.ok());
+  EXPECT_EQ(s.getMessage(), "OK");
+  EXPECT_EQ(keys.size(), 2U);
+}
+}

osquery/database/tests/plugin_tests.h

@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#pragma once
+
+#include <gtest/gtest.h>
+
+#include <boost/filesystem/operations.hpp>
+
+#include <osquery/database.h>
+#include <osquery/flags.h>
+
+#include "osquery/core/test_util.h"
+
+/// The following test macros allow pretty test output.
+#define CREATE_DATABASE_TESTS(n)                      \
+  TEST_F(n, test_plugin_check) { testPluginCheck(); } \
+  TEST_F(n, test_put) { testPut(); }                  \
+  TEST_F(n, test_get) { testGet(); }                  \
+  TEST_F(n, test_delete) { testDelete(); }            \
+  TEST_F(n, test_scan) { testScan(); }                \
+  TEST_F(n, test_scan_limit) { testScanLimit(); }
+
+namespace osquery {
+
+DECLARE_string(database_path);
+
+class DatabasePluginTests : public testing::Test {
+ public:
+  void SetUp() override {
+    existing_plugin_ = Registry::getActive("database");
+    Registry::get("database", existing_plugin_)->tearDown();
+
+    setName(name());
+    path_ = FLAGS_database_path;
+    boost::filesystem::remove_all(path_);
+
+    auto plugin = Registry::get("database", getName());
+    plugin_ = std::dynamic_pointer_cast<DatabasePlugin>(plugin);
+    plugin_->reset();
+  }
+
+  void TearDown() override {
+    Registry::get("database", name_)->tearDown();
+    Registry::setActive("database", existing_plugin_);
+  }
+
+ protected:
+  /// Path to testing database.
+  std::string path_;
+
+ protected:
+  /// Require each plugin tester to implement a set name.
+  virtual std::string name() = 0;
+
+ private:
+  void setName(const std::string& name) { name_ = name; }
+  const std::string& getName() { return name_; }
+  std::shared_ptr<DatabasePlugin> getPlugin() { return plugin_; }
+
+ private:
+  /// Plugin name
+  std::string name_;
+
+  /// Plugin casted from setUp, ready to run tests.
+  std::shared_ptr<DatabasePlugin> plugin_{nullptr};
+
+  /// Previous active database plugin.
+  std::string existing_plugin_;
+
+ protected:
+  void testPluginCheck();
+  void testPut();
+  void testGet();
+  void testDelete();
+  void testScan();
+  void testScanLimit();
+};
+}

osquery/database/tests/query_tests.cpp

@@ -19,18 +19,9 @@
 #include "osquery/core/test_util.h"
 #include "osquery/database/query.h"
 
-const std::string kTestingQueryDBPath = "/tmp/rocksdb-osquery-querytests";
-
 namespace osquery {
 
-class QueryTests : public testing::Test {
+class QueryTests : public testing::Test {};
- public:
-  void SetUp() { db_ = DBHandle::getInstanceAtPath(kTestingQueryDBPath); }
-  void TearDown() { boost::filesystem::remove_all(kTestingQueryDBPath); }
-
- public:
-  std::shared_ptr<DBHandle> db_;
-};
 
 TEST_F(QueryTests, test_private_members) {
   auto query = getOsqueryScheduledQuery();
@@ -42,21 +33,21 @@ TEST_F(QueryTests, test_add_and_get_current_results) {
   // Test adding a "current" set of results to a scheduled query instance.
   auto query = getOsqueryScheduledQuery();
   auto cf = Query("foobar", query);
-  auto status = cf.addNewResults(getTestDBExpectedResults(), db_);
+  auto status = cf.addNewResults(getTestDBExpectedResults());
   EXPECT_TRUE(status.ok());
   EXPECT_EQ(status.toString(), "OK");
 
   // Simulate results from several schedule runs, calculate differentials.
   for (auto result : getTestDBResultStream()) {
-    // Get the results from the previous query execution (from RocksDB).
+    // Get the results from the previous query execution (from the DB).
     QueryData previous_qd;
-    auto status = cf.getPreviousQueryResults(previous_qd, db_);
+    auto status = cf.getPreviousQueryResults(previous_qd);
     EXPECT_TRUE(status.ok());
     EXPECT_EQ(status.toString(), "OK");
 
     // Add the "current" results and output the differentials.
     DiffResults dr;
-    auto s = cf.addNewResults(result.second, dr, true, db_);
+    auto s = cf.addNewResults(result.second, dr, true);
     EXPECT_TRUE(s.ok());
 
     // Call the diffing utility directly.
@@ -65,7 +56,7 @@ TEST_F(QueryTests, test_add_and_get_current_results) {
 
     // After Query::addNewResults the previous results are now current.
     QueryData qd;
-    cf.getPreviousQueryResults(qd, db_);
+    cf.getPreviousQueryResults(qd);
     EXPECT_EQ(qd, result.second);
   }
 }
@@ -74,13 +65,13 @@ TEST_F(QueryTests, test_get_query_results) {
   // Grab an expected set of query data and add it as the previous result.
   auto encoded_qd = getSerializedQueryDataJSON();
   auto query = getOsqueryScheduledQuery();
-  auto status = db_->Put(kQueries, "foobar", encoded_qd.first);
+  auto status = setDatabaseValue(kQueries, "foobar", encoded_qd.first);
   EXPECT_TRUE(status.ok());
 
   // Use the Query retrieval API to check the now "previous" result.
   QueryData previous_qd;
   auto cf = Query("foobar", query);
-  status = cf.getPreviousQueryResults(previous_qd, db_);
+  status = cf.getPreviousQueryResults(previous_qd);
   EXPECT_TRUE(status.ok());
 }
 
@@ -89,31 +80,31 @@ TEST_F(QueryTests, test_query_name_not_found_in_db) {
   QueryData previous_qd;
   auto query = getOsqueryScheduledQuery();
   auto cf = Query("not_a_real_query", query);
-  auto status = cf.getPreviousQueryResults(previous_qd, db_);
+  auto status = cf.getPreviousQueryResults(previous_qd);
-  EXPECT_TRUE(status.toString() == "Query name not found in database");
   EXPECT_TRUE(status.ok());
+  EXPECT_EQ(status.toString(), "Query name not found in database");
 }
 
 TEST_F(QueryTests, test_is_query_name_in_database) {
   auto query = getOsqueryScheduledQuery();
   auto cf = Query("foobar", query);
   auto encoded_qd = getSerializedQueryDataJSON();
-  auto status = db_->Put(kQueries, "foobar", encoded_qd.first);
+  auto status = setDatabaseValue(kQueries, "foobar", encoded_qd.first);
   EXPECT_TRUE(status.ok());
   // Now test that the query name exists.
-  EXPECT_TRUE(cf.isQueryNameInDatabase(db_));
+  EXPECT_TRUE(cf.isQueryNameInDatabase());
 }
 
 TEST_F(QueryTests, test_get_stored_query_names) {
   auto query = getOsqueryScheduledQuery();
   auto cf = Query("foobar", query);
   auto encoded_qd = getSerializedQueryDataJSON();
-  auto status = db_->Put(kQueries, "foobar", encoded_qd.first);
+  auto status = setDatabaseValue(kQueries, "foobar", encoded_qd.first);
   EXPECT_TRUE(status.ok());
 
   // Stored query names is a factory method included alongside every query.
   // It will include the set of query names with existing "previous" results.
-  auto names = cf.getStoredQueryNames(db_);
+  auto names = cf.getStoredQueryNames();
   auto in_vector = std::find(names.begin(), names.end(), "foobar");
   EXPECT_NE(in_vector, names.end());
 }

osquery/database/tests/results_tests.cpp

@@ -136,111 +136,6 @@ TEST_F(ResultsTests, test_deserialize_query_log_item_json) {
   EXPECT_EQ(output, results.second);
 }
 
-TEST_F(ResultsTests, test_serialize_distributed_query_request) {
-  DistributedQueryRequest r;
-  r.query = "foo";
-  r.id = "bar";
-
-  pt::ptree tree;
-  auto s = serializeDistributedQueryRequest(r, tree);
-  EXPECT_TRUE(s.ok());
-  EXPECT_EQ(tree.get<std::string>("query"), "foo");
-  EXPECT_EQ(tree.get<std::string>("id"), "bar");
-}
-
-TEST_F(ResultsTests, test_deserialize_distributed_query_request) {
-  pt::ptree tree;
-  tree.put<std::string>("query", "foo");
-  tree.put<std::string>("id", "bar");
-
-  DistributedQueryRequest r;
-  auto s = deserializeDistributedQueryRequest(tree, r);
-  EXPECT_TRUE(s.ok());
-  EXPECT_EQ(r.query, "foo");
-  EXPECT_EQ(r.id, "bar");
-}
-
-TEST_F(ResultsTests, test_deserialize_distributed_query_request_json) {
-  auto json =
-      "{"
-      "  \"query\": \"foo\","
-      "  \"id\": \"bar\""
-      "}";
-
-  DistributedQueryRequest r;
-  auto s = deserializeDistributedQueryRequestJSON(json, r);
-  EXPECT_TRUE(s.ok());
-  EXPECT_EQ(r.query, "foo");
-  EXPECT_EQ(r.id, "bar");
-}
-
-TEST_F(ResultsTests, test_serialize_distributed_query_result) {
-  DistributedQueryResult r;
-  r.request.query = "foo";
-  r.request.id = "bar";
-
-  Row r1;
-  r1["foo"] = "bar";
-  r.results = {r1};
-
-  pt::ptree tree;
-  auto s = serializeDistributedQueryResult(r, tree);
-  EXPECT_TRUE(s.ok());
-  EXPECT_EQ(tree.get<std::string>("request.query"), "foo");
-  EXPECT_EQ(tree.get<std::string>("request.id"), "bar");
-  auto& results = tree.get_child("results");
-  for (const auto& q : results) {
-    for (const auto& row : q.second) {
-      EXPECT_EQ(row.first, "foo");
-      EXPECT_EQ(q.second.get<std::string>(row.first), "bar");
-    }
-  }
-}
-
-TEST_F(ResultsTests, test_deserialize_distributed_query_result) {
-  pt::ptree request;
-  request.put<std::string>("id", "foo");
-  request.put<std::string>("query", "bar");
-
-  pt::ptree row;
-  row.put<std::string>("foo", "bar");
-  pt::ptree results;
-  results.push_back(std::make_pair("", row));
-
-  pt::ptree query_result;
-  query_result.put_child("request", request);
-  query_result.put_child("results", results);
-
-  DistributedQueryResult r;
-  auto s = deserializeDistributedQueryResult(query_result, r);
-  EXPECT_TRUE(s.ok());
-  EXPECT_EQ(r.request.id, "foo");
-  EXPECT_EQ(r.request.query, "bar");
-  EXPECT_EQ(r.results[0]["foo"], "bar");
-}
-
-TEST_F(ResultsTests, test_deserialize_distributed_query_result_json) {
-  auto json =
-      "{"
-      "  \"request\": {"
-      "    \"id\": \"foo\","
-      "    \"query\": \"bar\""
-      "  },"
-      "  \"results\": ["
-      "    {"
-      "      \"foo\": \"bar\""
-      "    }"
-      "  ]"
-      "}";
-
-  DistributedQueryResult r;
-  auto s = deserializeDistributedQueryResultJSON(json, r);
-  EXPECT_TRUE(s.ok());
-  EXPECT_EQ(r.request.id, "foo");
-  EXPECT_EQ(r.request.query, "bar");
-  EXPECT_EQ(r.results[0]["foo"], "bar");
-}
-
 TEST_F(ResultsTests, test_adding_duplicate_rows_to_query_data) {
   Row r1, r2, r3;
   r1["foo"] = "bar";

osquery/devtools/shell.cpp

@@ -15,11 +15,11 @@
 
 #include <signal.h>
 #include <stdio.h>
-#include <sys/time.h>
 #include <sys/resource.h>
+#include <sys/time.h>
 
-#include <readline/readline.h>
 #include <readline/history.h>
+#include <readline/readline.h>
 
 #include <sqlite3.h>
 

osquery/dispatcher/scheduler.cpp

@@ -47,8 +47,8 @@ inline SQL monitor(const std::string& name, const ScheduledQuery& query) {
       }
     }
     // Always called while processes table is working.
-    Config::getInstance().recordQueryPerformance(
+    Config::getInstance().recordQueryPerformance(name, t1 - t0, size, r0[0],
-        name, t1 - t0, size, r0[0], r1[0]);
+                                                 r1[0]);
   }
   return sql;
 }

osquery/distributed/distributed.cpp

@@ -18,7 +18,6 @@
 #include <osquery/logger.h>
 #include <osquery/sql.h>
 
-
 namespace pt = boost::property_tree;
 
 namespace osquery {
@@ -187,4 +186,124 @@ DistributedQueryRequest Distributed::popRequest() {
   queries_.erase(queries_.begin());
   return q;
 }
+
+Status serializeDistributedQueryRequest(const DistributedQueryRequest& r,
+                                        pt::ptree& tree) {
+  tree.put("query", r.query);
+  tree.put("id", r.id);
+  return Status(0, "OK");
+}
+
+Status serializeDistributedQueryRequestJSON(const DistributedQueryRequest& r,
+                                            std::string& json) {
+  pt::ptree tree;
+  auto s = serializeDistributedQueryRequest(r, tree);
+  if (!s.ok()) {
+    return s;
+  }
+  std::stringstream ss;
+  try {
+    pt::write_json(ss, tree, false);
+  } catch (const pt::ptree_error& e) {
+    return Status(1, "Error serializing JSON: " + std::string(e.what()));
+  }
+  json = ss.str();
+
+  return Status(0, "OK");
+}
+
+Status deserializeDistributedQueryRequest(const pt::ptree& tree,
+                                          DistributedQueryRequest& r) {
+  r.query = tree.get<std::string>("query", "");
+  r.id = tree.get<std::string>("id", "");
+  return Status(0, "OK");
+}
+
+Status deserializeDistributedQueryRequestJSON(const std::string& json,
+                                              DistributedQueryRequest& r) {
+  std::stringstream ss(json);
+  pt::ptree tree;
+  try {
+    pt::read_json(ss, tree);
+  } catch (const pt::ptree_error& e) {
+    return Status(1, "Error serializing JSON: " + std::string(e.what()));
+  }
+  return deserializeDistributedQueryRequest(tree, r);
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// DistributedQueryResult - small struct containing the results of a
+// distributed query
+/////////////////////////////////////////////////////////////////////////////
+
+Status serializeDistributedQueryResult(const DistributedQueryResult& r,
+                                       pt::ptree& tree) {
+  pt::ptree request;
+  auto s = serializeDistributedQueryRequest(r.request, request);
+  if (!s.ok()) {
+    return s;
+  }
+
+  pt::ptree results;
+  s = serializeQueryData(r.results, results);
+  if (!s.ok()) {
+    return s;
+  }
+
+  tree.add_child("request", request);
+  tree.add_child("results", results);
+
+  return Status(0, "OK");
+}
+
+Status serializeDistributedQueryResultJSON(const DistributedQueryResult& r,
+                                           std::string& json) {
+  pt::ptree tree;
+  auto s = serializeDistributedQueryResult(r, tree);
+  if (!s.ok()) {
+    return s;
+  }
+  std::stringstream ss;
+  try {
+    pt::write_json(ss, tree, false);
+  } catch (const pt::ptree_error& e) {
+    return Status(1, "Error serializing JSON: " + std::string(e.what()));
+  }
+  json = ss.str();
+
+  return Status(0, "OK");
+}
+
+Status deserializeDistributedQueryResult(const pt::ptree& tree,
+                                         DistributedQueryResult& r) {
+  DistributedQueryRequest request;
+  auto s =
+      deserializeDistributedQueryRequest(tree.get_child("request"), request);
+  if (!s.ok()) {
+    return s;
+  }
+
+  QueryData results;
+  s = deserializeQueryData(tree.get_child("results"), results);
+  if (!s.ok()) {
+    return s;
+  }
+
+  r.request = request;
+  r.results = results;
+
+  return Status(0, "OK");
+}
+
+Status deserializeDistributedQueryResultJSON(const std::string& json,
+                                             DistributedQueryResult& r) {
+  std::stringstream ss(json);
+  pt::ptree tree;
+  try {
+    pt::read_json(ss, tree);
+  } catch (const pt::ptree_error& e) {
+    return Status(1, "Error serializing JSON: " + std::string(e.what()));
+  }
+  return deserializeDistributedQueryResult(tree, r);
+}
 }

osquery/distributed/tests/distributed_tests.cpp

@@ -65,6 +65,111 @@ class DistributedTests : public testing::Test {
   std::string distributed_tls_write_endpoint_;
 };
 
+TEST_F(DistributedTests, test_serialize_distributed_query_request) {
+  DistributedQueryRequest r;
+  r.query = "foo";
+  r.id = "bar";
+
+  pt::ptree tree;
+  auto s = serializeDistributedQueryRequest(r, tree);
+  EXPECT_TRUE(s.ok());
+  EXPECT_EQ(tree.get<std::string>("query"), "foo");
+  EXPECT_EQ(tree.get<std::string>("id"), "bar");
+}
+
+TEST_F(DistributedTests, test_deserialize_distributed_query_request) {
+  pt::ptree tree;
+  tree.put<std::string>("query", "foo");
+  tree.put<std::string>("id", "bar");
+
+  DistributedQueryRequest r;
+  auto s = deserializeDistributedQueryRequest(tree, r);
+  EXPECT_TRUE(s.ok());
+  EXPECT_EQ(r.query, "foo");
+  EXPECT_EQ(r.id, "bar");
+}
+
+TEST_F(DistributedTests, test_deserialize_distributed_query_request_json) {
+  auto json =
+      "{"
+      "  \"query\": \"foo\","
+      "  \"id\": \"bar\""
+      "}";
+
+  DistributedQueryRequest r;
+  auto s = deserializeDistributedQueryRequestJSON(json, r);
+  EXPECT_TRUE(s.ok());
+  EXPECT_EQ(r.query, "foo");
+  EXPECT_EQ(r.id, "bar");
+}
+
+TEST_F(DistributedTests, test_serialize_distributed_query_result) {
+  DistributedQueryResult r;
+  r.request.query = "foo";
+  r.request.id = "bar";
+
+  Row r1;
+  r1["foo"] = "bar";
+  r.results = {r1};
+
+  pt::ptree tree;
+  auto s = serializeDistributedQueryResult(r, tree);
+  EXPECT_TRUE(s.ok());
+  EXPECT_EQ(tree.get<std::string>("request.query"), "foo");
+  EXPECT_EQ(tree.get<std::string>("request.id"), "bar");
+  auto& results = tree.get_child("results");
+  for (const auto& q : results) {
+    for (const auto& row : q.second) {
+      EXPECT_EQ(row.first, "foo");
+      EXPECT_EQ(q.second.get<std::string>(row.first), "bar");
+    }
+  }
+}
+
+TEST_F(DistributedTests, test_deserialize_distributed_query_result) {
+  pt::ptree request;
+  request.put<std::string>("id", "foo");
+  request.put<std::string>("query", "bar");
+
+  pt::ptree row;
+  row.put<std::string>("foo", "bar");
+  pt::ptree results;
+  results.push_back(std::make_pair("", row));
+
+  pt::ptree query_result;
+  query_result.put_child("request", request);
+  query_result.put_child("results", results);
+
+  DistributedQueryResult r;
+  auto s = deserializeDistributedQueryResult(query_result, r);
+  EXPECT_TRUE(s.ok());
+  EXPECT_EQ(r.request.id, "foo");
+  EXPECT_EQ(r.request.query, "bar");
+  EXPECT_EQ(r.results[0]["foo"], "bar");
+}
+
+TEST_F(DistributedTests, test_deserialize_distributed_query_result_json) {
+  auto json =
+      "{"
+      "  \"request\": {"
+      "    \"id\": \"foo\","
+      "    \"query\": \"bar\""
+      "  },"
+      "  \"results\": ["
+      "    {"
+      "      \"foo\": \"bar\""
+      "    }"
+      "  ]"
+      "}";
+
+  DistributedQueryResult r;
+  auto s = deserializeDistributedQueryResultJSON(json, r);
+  EXPECT_TRUE(s.ok());
+  EXPECT_EQ(r.request.id, "foo");
+  EXPECT_EQ(r.request.query, "bar");
+  EXPECT_EQ(r.results[0]["foo"], "bar");
+}
+
 TEST_F(DistributedTests, test_workflow) {
   auto dist = Distributed();
   auto s = dist.pullUpdates();

osquery/events/benchmarks/events_benchmarks.cpp

@@ -58,9 +58,13 @@ class BenchmarkEventSubscriber
   }
 
   void clearRows() {
+    auto ee = expire_events_;
+    auto et = expire_time_;
     expire_events_ = true;
     expire_time_ = -1;
     getIndexes(0, 0);
+    expire_events_ = ee;
+    expire_time_ = et;
   }
 
   void benchmarkGet(int low, int high) { auto results = get(low, high); }
@@ -113,7 +117,7 @@ BENCHMARK(EVENTS_add_events);
 static void EVENTS_retrieve_events(benchmark::State& state) {
   auto sub = std::make_shared<BenchmarkEventSubscriber>();
 
-  for (int i = 0; i < 10000; i++) {
+  for (int i = 0; i < 1000; i++) {
     sub->benchmarkAdd(i++);
   }
 
@@ -125,8 +129,8 @@ static void EVENTS_retrieve_events(benchmark::State& state) {
 }
 
 BENCHMARK(EVENTS_retrieve_events)
+    ->ArgPair(0, 10)
+    ->ArgPair(0, 50)
     ->ArgPair(0, 100)
-    ->ArgPair(0, 500)
+    ->ArgPair(0, 1000);
-    ->ArgPair(0, 1000)
-    ->ArgPair(0, 10000);
 }

osquery/events/events.cpp

@@ -21,7 +21,6 @@
 #include <osquery/logger.h>
 
 #include "osquery/core/conversions.h"
-#include "osquery/database/db_handle.h"
 
 namespace osquery {
 
@@ -92,9 +91,8 @@ QueryData EventSubscriberPlugin::genTable(QueryContext& context) {
 
     // Store the optimize time such that it can be restored if the daemon is
     // restarted.
-    auto db = DBHandle::getInstance();
     auto index_key = "optimize." + dbNamespace();
-    db->Put(kEvents, index_key, std::to_string(optimize_time_));
+    setDatabaseValue(kEvents, index_key, std::to_string(optimize_time_));
   }
 
   return get(start, stop);
@@ -135,7 +133,6 @@ void EventPublisherPlugin::fire(const EventContextRef& ec, EventTime time) {
 std::set<std::string> EventSubscriberPlugin::getIndexes(EventTime start,
                                                         EventTime stop,
                                                         size_t list_key) {
-  auto db = DBHandle::getInstance();
   auto index_key = "indexes." + dbNamespace();
   std::set<std::string> indexes;
 
@@ -152,7 +149,7 @@ std::set<std::string> EventSubscriberPlugin::getIndexes(EventTime start,
 
     std::string time_list;
     auto list_type = boost::lexical_cast<std::string>(size);
-    db->Get(kEvents, index_key + "." + list_type, time_list);
+    getDatabaseValue(kEvents, index_key + "." + list_type, time_list);
     if (time_list.empty()) {
       // No events in this binning size.
       return indexes;
@@ -240,7 +237,6 @@ std::set<std::string> EventSubscriberPlugin::getIndexes(EventTime start,
 void EventSubscriberPlugin::expireRecords(const std::string& list_type,
                                           const std::string& index,
                                           bool all) {
-  auto db = DBHandle::getInstance();
   auto record_key = "records." + dbNamespace();
   auto data_key = "data." + dbNamespace();
 
@@ -250,7 +246,7 @@ void EventSubscriberPlugin::expireRecords(const std::string& list_type,
   auto expired_records = getRecords({list_type + "." + index});
   for (const auto& record : expired_records) {
     if (all) {
-      db->Delete(kEvents, data_key + "." + record.first);
+      deleteDatabaseValue(kEvents, data_key + "." + record.first);
     } else if (record.second > expire_time_) {
       persisting_records.push_back(record.first + ":" +
                                    std::to_string(record.second));
@@ -259,10 +255,11 @@ void EventSubscriberPlugin::expireRecords(const std::string& list_type,
 
   // Either drop or overwrite the record list.
   if (all) {
-    db->Delete(kEvents, record_key + "." + list_type + "." + index);
+    deleteDatabaseValue(kEvents, record_key + "." + list_type + "." + index);
   } else {
     auto new_records = boost::algorithm::join(persisting_records, ",");
-    db->Put(kEvents, record_key + "." + list_type + "." + index, new_records);
+    setDatabaseValue(
+        kEvents, record_key + "." + list_type + "." + index, new_records);
   }
 }
 
@@ -270,7 +267,6 @@ void EventSubscriberPlugin::expireIndexes(
     const std::string& list_type,
     const std::vector<std::string>& indexes,
     const std::vector<std::string>& expirations) {
-  auto db = DBHandle::getInstance();
   auto index_key = "indexes." + dbNamespace();
 
   // Construct a mutable list of persisting indexes to rewrite as records.
@@ -285,16 +281,15 @@ void EventSubscriberPlugin::expireIndexes(
 
   // Update the list of indexes with the non-expired indexes.
   auto new_indexes = boost::algorithm::join(persisting_indexes, ",");
-  db->Put(kEvents, index_key + "." + list_type, new_indexes);
+  setDatabaseValue(kEvents, index_key + "." + list_type, new_indexes);
 }
 
 void EventSubscriberPlugin::expireCheck() {
-  auto db = DBHandle::getInstance();
   auto data_key = "data." + dbNamespace();
   auto eid_key = "eid." + dbNamespace();
 
   std::vector<std::string> keys;
-  db->ScanPrefix(kEvents, keys, data_key);
+  scanDatabaseKeys(kEvents, keys, data_key);
   if (keys.size() <= FLAGS_events_max) {
     return;
   }
@@ -305,13 +300,13 @@ void EventSubscriberPlugin::expireCheck() {
   // Inspect the N-FLAGS_events_max -th event's value and expire before the
   // time within the content.
   std::string last_key;
-  db->Get(kEvents, eid_key, last_key);
+  getDatabaseValue(kEvents, eid_key, last_key);
   // The EID is the next-index.
   size_t max_key = boost::lexical_cast<size_t>(last_key) - FLAGS_events_max - 1;
 
   // Convert the key index into a time using the content.
   std::string content;
-  db->Get(kEvents, data_key + "." + std::to_string(max_key), content);
+  getDatabaseValue(kEvents, data_key + "." + std::to_string(max_key), content);
 
   // Decode the value into a row structure to extract the time.
   Row r;
@@ -332,7 +327,6 @@ void EventSubscriberPlugin::expireCheck() {
 
 std::vector<EventRecord> EventSubscriberPlugin::getRecords(
     const std::set<std::string>& indexes) {
-  auto db = DBHandle::getInstance();
   auto record_key = "records." + dbNamespace();
 
   std::vector<EventRecord> records;
@@ -340,7 +334,7 @@ std::vector<EventRecord> EventSubscriberPlugin::getRecords(
     std::vector<std::string> bin_records;
     {
       std::string record_value;
-      db->Get(kEvents, record_key + "." + index, record_value);
+      getDatabaseValue(kEvents, record_key + "." + index, record_value);
       if (record_value.empty()) {
         // There are actually no events in this bin, interesting error case.
         continue;
@@ -364,7 +358,6 @@ std::vector<EventRecord> EventSubscriberPlugin::getRecords(
 
 Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime time) {
   Status status;
-  auto db = DBHandle::getInstance();
   std::string time_value = boost::lexical_cast<std::string>(time);
 
   // The record is identified by the event type then module name.
@@ -386,27 +379,29 @@ Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime time) {
       boost::lock_guard<boost::mutex> lock(event_record_lock_);
       // Append the record (eid, unix_time) to the list bin.
       std::string record_value;
-      status = db->Get(
+      status = getDatabaseValue(
           kEvents, record_key + "." + list_key + "." + list_id, record_value);
 
       if (record_value.length() == 0) {
         // This is a new list_id for list_key, append the ID to the indirect
         // lookup for this list_key.
         std::string index_value;
-        status = db->Get(kEvents, index_key + "." + list_key, index_value);
+        status =
+            getDatabaseValue(kEvents, index_key + "." + list_key, index_value);
         if (index_value.length() == 0) {
           // A new index.
           index_value = list_id;
         } else {
           index_value += "," + list_id;
         }
-        status = db->Put(kEvents, index_key + "." + list_key, index_value);
+        status =
+            setDatabaseValue(kEvents, index_key + "." + list_key, index_value);
         record_value = eid + ":" + time_value;
       } else {
         // Tokenize a record using ',' and the EID/time using ':'.
         record_value += "," + eid + ":" + time_value;
       }
-      status = db->Put(
+      status = setDatabaseValue(
           kEvents, record_key + "." + list_key + "." + list_id, record_value);
       if (!status.ok()) {
         LOG(ERROR) << "Could not put Event Record key: " << record_key;
@@ -419,7 +414,6 @@ Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime time) {
 
 EventID EventSubscriberPlugin::getEventID() {
   Status status;
-  auto db = DBHandle::getInstance();
   // First get an event ID from the meta key.
   std::string eid_key = "eid." + dbNamespace();
   std::string last_eid_value;
@@ -427,14 +421,14 @@ EventID EventSubscriberPlugin::getEventID() {
 
   {
     boost::lock_guard<boost::mutex> lock(event_id_lock_);
-    status = db->Get(kEvents, eid_key, last_eid_value);
+    status = getDatabaseValue(kEvents, eid_key, last_eid_value);
     if (!status.ok() || last_eid_value.empty()) {
       last_eid_value = "0";
     }
 
     last_eid_ = boost::lexical_cast<size_t>(last_eid_value) + 1;
     eid_value = boost::lexical_cast<std::string>(last_eid_);
-    status = db->Put(kEvents, eid_key, eid_value);
+    status = setDatabaseValue(kEvents, eid_key, eid_value);
   }
 
   if (!status.ok()) {
@@ -448,7 +442,6 @@ QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
   QueryData results;
 
   // Get the records for this time range.
-  auto db = DBHandle::getInstance();
   auto indexes = getIndexes(start, stop);
   auto records = getRecords(indexes);
   std::string events_key = "data." + dbNamespace();
@@ -464,7 +457,7 @@ QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
   std::string data_value;
   for (const auto& record : mapped_records) {
     Row r;
-    auto status = db->Get(kEvents, record, data_value);
+    auto status = getDatabaseValue(kEvents, record, data_value);
     if (data_value.length() == 0) {
       // There is no record here, interesting error case.
       continue;
@@ -485,7 +478,6 @@ QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
 }
 
 Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
-  auto db = DBHandle::getInstance();
   // Get and increment the EID for this module.
   EventID eid = getEventID();
   // Without encouraging a missing event time, do not support a 0-time.
@@ -506,7 +498,7 @@ Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
 
   // Store the event data.
   std::string event_key = "data." + dbNamespace() + "." + eid;
-  status = db->Put(kEvents, event_key, data);
+  status = setDatabaseValue(kEvents, event_key, data);
   // Record the event in the indexing bins, using the index time.
   recordEvent(eid, event_time);
   return status;
@@ -692,10 +684,9 @@ Status EventFactory::registerEventSubscriber(const PluginRef& sub) {
 
   // Restore optimize times for a daemon.
   if (kToolType == OSQUERY_TOOL_DAEMON && FLAGS_events_optimize) {
-    auto db = DBHandle::getInstance();
     auto index_key = "optimize." + specialized_sub->dbNamespace();
     std::string content;
-    if (db->Get(kEvents, index_key, content)) {
+    if (getDatabaseValue(kEvents, index_key, content)) {
       long long optimize_time = 0;
       safeStrtoll(content, 10, optimize_time);
       specialized_sub->optimize_time_ = static_cast<EventTime>(optimize_time);

osquery/events/tests/events_database_tests.cpp

@@ -19,8 +19,6 @@
 #include <osquery/flags.h>
 #include <osquery/tables.h>
 
-#include "osquery/database/db_handle.h"
-
 namespace osquery {
 
 DECLARE_uint64(events_expiry);
@@ -209,8 +207,8 @@ TEST_F(EventsDatabaseTests, test_gentable) {
   // The optimize time should have been written to the database.
   // It should be the same as the current (relative) optimize time.
   std::string content;
-  getDatabaseValue(
+  getDatabaseValue("events", "optimize.DBFakePublisher.DBFakeSubscriber",
-      "events", "optimize.DBFakePublisher.DBFakeSubscriber", content);
+                   content);
   EXPECT_EQ(std::to_string(sub->optimize_time_), content);
 
   keys.clear();

osquery/extensions/interface.cpp

@@ -44,7 +44,6 @@ void ExtensionHandler::call(ExtensionResponse& _return,
   _return.status.code = status.getCode();
   _return.status.message = status.getMessage();
   _return.status.uuid = uuid_;
-
   if (status.ok()) {
     for (const auto& response_item : response) {
       // Translate a PluginResponse to an ExtensionPluginResponse.
@@ -176,18 +175,18 @@ bool ExtensionManagerHandler::exists(const std::string& name) {
 ExtensionRunnerCore::~ExtensionRunnerCore() { remove(path_); }
 
 void ExtensionRunnerCore::stop() {
-  boost::lock_guard<boost::mutex> lock(service_start_);
+  {
-  service_stopping_ = true;
+    std::unique_lock<std::mutex> lock(service_start_);
-  if (transport_ != nullptr) {
+    service_stopping_ = true;
-    transport_->interrupt();
+    if (transport_ != nullptr) {
+      transport_->interrupt();
+      transport_->interruptChildren();
+    }
   }
 
-  {
+  // In most cases the service thread has started before the stop request.
-    // In most cases the service thread has started before the stop request.
+  if (server_ != nullptr) {
-    boost::lock_guard<boost::mutex> lock(service_run_);
+    server_->stop();
-    if (server_ != nullptr) {
-      server_->stop();
-    }
   }
 }
 
@@ -202,7 +201,7 @@ inline void removeStalePaths(const std::string& manager) {
 
 void ExtensionRunnerCore::startServer(TProcessorRef processor) {
   {
-    boost::lock_guard<boost::mutex> lock(service_start_);
+    std::unique_lock<std::mutex> lock(service_start_);
     // A request to stop the service may occur before the thread starts.
     if (service_stopping_) {
       return;
@@ -221,10 +220,7 @@ void ExtensionRunnerCore::startServer(TProcessorRef processor) {
         processor, transport_, transport_fac, protocol_fac));
   }
 
-  {
+  server_->serve();
-    boost::lock_guard<boost::mutex> lock(service_run_);
-    server_->serve();
-  }
 }
 
 void ExtensionRunner::start() {
@@ -243,7 +239,7 @@ void ExtensionRunner::start() {
 
 ExtensionManagerRunner::~ExtensionManagerRunner() {
   // Only attempt to remove stale paths if the server was started.
-  boost::lock_guard<boost::mutex> lock(service_start_);
+  std::unique_lock<std::mutex> lock(service_start_);
   if (server_ != nullptr) {
     removeStalePaths(path_);
   }

osquery/extensions/interface.h

@@ -261,11 +261,7 @@ class ExtensionRunnerCore : public InternalRunnable {
   TThreadedServerRef server_{nullptr};
 
   /// Protect the service start and stop, this mutex protects server creation.
-  boost::mutex service_start_;
+  std::mutex service_start_;
-
- private:
-  /// Protect the service start and stop, this mutex protects the transport.
-  boost::mutex service_run_;
 
   /// Record a dispatcher's request to stop the service.
   bool service_stopping_{false};

osquery/logger/CMakeLists.txt

@@ -1,19 +1,16 @@
-ADD_OSQUERY_LIBRARY(TRUE osquery_logger
+file(GLOB OSQUERY_LOGGER "*.cpp")
-  logger.cpp
+ADD_OSQUERY_LIBRARY(TRUE osquery_logger ${OSQUERY_LOGGER})
-)
+
 add_dependencies(osquery_logger libglog)
 
-ADD_OSQUERY_LIBRARY(FALSE osquery_logger_plugins
+file(GLOB OSQUERY_LOGGER_PLUGINS "plugins/*.cpp")
-  plugins/filesystem.cpp
+ADD_OSQUERY_LIBRARY(FALSE osquery_logger_plugins ${OSQUERY_LOGGER_PLUGINS})
-  plugins/tls.cpp
+
-  plugins/syslog.cpp
+file(GLOB OSQUERY_LOGGER_TESTS "tests/*.cpp")
-)
+ADD_OSQUERY_TEST(TRUE ${OSQUERY_LOGGER_TESTS})
 
 # Keep the logger testing in the additional to test filesystem logging.
 # There is a significant difference between the Glog-backed filesystem plugin
 # and other, which use a Glog sink. They must be tested in tandem.
-file(GLOB OSQUERY_LOGGER_TESTS "tests/*.cpp")
-ADD_OSQUERY_TEST(TRUE ${OSQUERY_LOGGER_TESTS})
-
 file(GLOB OSQUERY_LOGGER_PLUGIN_TESTS "plugins/tests/*.cpp")
 ADD_OSQUERY_TEST(FALSE ${OSQUERY_LOGGER_PLUGIN_TESTS})

osquery/logger/plugins/tls.cpp

@@ -8,21 +8,20 @@
  *
  */
 
-#include <vector>
 #include <string>
+#include <vector>
 
-#include <boost/property_tree/ptree.hpp>
 #include <boost/property_tree/json_parser.hpp>
+#include <boost/property_tree/ptree.hpp>
 
 #include <osquery/enroll.h>
 #include <osquery/flags.h>
 #include <osquery/registry.h>
 
 #include "osquery/remote/requests.h"
-#include "osquery/remote/transports/tls.h"
 #include "osquery/remote/serializers/json.h"
+#include "osquery/remote/transports/tls.h"
 #include "osquery/remote/utility.h"
-#include "osquery/database/db_handle.h"
 
 #include "osquery/logger/plugins/tls.h"
 
@@ -144,8 +143,7 @@ Status TLSLogForwarderRunner::send(std::vector<std::string>& log_data,
     // Read each logged line into JSON and populate a list of lines.
     // The result list will use the 'data' key.
     pt::ptree children;
-    iterate(log_data,
+    iterate(log_data, ([&children](std::string& item) {
-            ([&children](std::string& item) {
               pt::ptree child;
               try {
                 std::stringstream input;
@@ -169,21 +167,16 @@ Status TLSLogForwarderRunner::send(std::vector<std::string>& log_data,
 }
 
 void TLSLogForwarderRunner::check() {
-  // Instead of using the 'help' database API, prefer to interact with the
-  // DBHandle directly for additional performance.
-  auto handle = DBHandle::getInstance();
-
   // Get a list of all the buffered log items, with a max of 1024 lines.
   std::vector<std::string> indexes;
-  auto status = handle->Scan(kLogs, indexes, kTLSMaxLogLines);
+  auto status = scanDatabaseKeys(kLogs, indexes, kTLSMaxLogLines);
 
   // For each index, accumulate the log line into the result or status set.
   std::vector<std::string> results, statuses;
-  iterate(indexes,
+  iterate(indexes, ([&results, &statuses](std::string& index) {
-          ([&handle, &results, &statuses](std::string& index) {
             std::string value;
             auto& target = ((index.at(0) == 'r') ? results : statuses);
-            if (handle->Get(kLogs, index, value)) {
+            if (getDatabaseValue(kLogs, index, value)) {
               // Enforce a max log line size for TLS logging.
               if (value.size() > FLAGS_logger_tls_max) {
                 LOG(WARNING) << "Line exceeds TLS logger max: " << value.size();
@@ -201,8 +194,7 @@ void TLSLogForwarderRunner::check() {
               << status.getMessage() << ")";
     } else {
       // Clear the results logs once they were sent.
-      iterate(indexes,
+      iterate(indexes, ([&results](std::string& index) {
-              ([&results](std::string& index) {
                 if (index.at(0) != 'r') {
                   return;
                 }
@@ -218,8 +210,7 @@ void TLSLogForwarderRunner::check() {
               << status.getMessage() << ")";
     } else {
       // Clear the status logs once they were sent.
-      iterate(indexes,
+      iterate(indexes, ([&results](std::string& index) {
-              ([&results](std::string& index) {
                 if (index.at(0) != 's') {
                   return;
                 }

osquery/main/tests.cpp

@@ -19,5 +19,8 @@ int main(int argc, char* argv[]) {
   testing::InitGoogleTest(&argc, argv);
   // Optionally enable Goggle Logging
   // google::InitGoogleLogging(argv[0]);
-  return RUN_ALL_TESTS();
+  auto result = RUN_ALL_TESTS();
+
+  osquery::shutdownTesting();
+  return result;
 }

osquery/sql/benchmarks/sql_benchmarks.cpp

@@ -62,6 +62,40 @@ static void SQL_virtual_table_internal(benchmark::State& state) {
 
 BENCHMARK(SQL_virtual_table_internal);
 
+static void SQL_virtual_table_internal_global(benchmark::State& state) {
+  Registry::add<BenchmarkTablePlugin>("table", "benchmark");
+  PluginResponse res;
+  Registry::call("table", "benchmark", {{"action", "columns"}}, res);
+
+  while (state.KeepRunning()) {
+    // Get a connection to the persistent database.
+    auto dbc = SQLiteDBManager::get();
+    attachTableInternal("benchmark", columnDefinition(res), dbc);
+
+    QueryData results;
+    queryInternal("select * from benchmark", results, dbc->db());
+  }
+}
+
+BENCHMARK(SQL_virtual_table_internal_global);
+
+static void SQL_virtual_table_internal_unique(benchmark::State& state) {
+  Registry::add<BenchmarkTablePlugin>("table", "benchmark");
+  PluginResponse res;
+  Registry::call("table", "benchmark", {{"action", "columns"}}, res);
+
+  while (state.KeepRunning()) {
+    // Get a new database connection (to a unique database).
+    auto dbc = SQLiteDBManager::getUnique();
+    attachTableInternal("benchmark", columnDefinition(res), dbc);
+
+    QueryData results;
+    queryInternal("select * from benchmark", results, dbc->db());
+  }
+}
+
+BENCHMARK(SQL_virtual_table_internal_unique);
+
 class BenchmarkLongTablePlugin : public TablePlugin {
  private:
   TableColumns columns() const {
@@ -83,7 +117,7 @@ static void SQL_virtual_table_internal_long(benchmark::State& state) {
   Registry::call("table", "long_benchmark", {{"action", "columns"}}, res);
 
   // Attach a sample virtual table.
-  auto dbc = SQLiteDBManager::get();
+  auto dbc = SQLiteDBManager::getUnique();
   attachTableInternal("long_benchmark", columnDefinition(res), dbc);
 
   while (state.KeepRunning()) {
@@ -123,7 +157,7 @@ static void SQL_virtual_table_internal_wide(benchmark::State& state) {
   Registry::call("table", "wide_benchmark", {{"action", "columns"}}, res);
 
   // Attach a sample virtual table.
-  auto dbc = SQLiteDBManager::get();
+  auto dbc = SQLiteDBManager::getUnique();
   attachTableInternal("wide_benchmark", columnDefinition(res), dbc);
 
   while (state.KeepRunning()) {

osquery/sql/sqlite_util.cpp

@@ -9,8 +9,8 @@
  */
 
 #include <osquery/core.h>
-#include <osquery/logger.h>
 #include <osquery/flags.h>
+#include <osquery/logger.h>
 #include <osquery/sql.h>
 
 #include "osquery/sql/sqlite_util.h"
@@ -49,6 +49,13 @@ const std::map<int, std::string> kSQLiteReturnCodes = {
     {101, "SQLITE_DONE"},
 };
 
+const std::map<std::string, std::string> kMemoryDBSettings = {
+    {"synchronous", "OFF"},      {"count_changes", "OFF"},
+    {"default_temp_store", "0"}, {"auto_vacuum", "FULL"},
+    {"journal_mode", "OFF"},     {"cache_size", "0"},
+    {"page_count", "0"},
+};
+
 #define OpComparator(x) \
   { x, QueryPlanner::Opcode(OpReg::P2, INTEGER_TYPE) }
 #define Arithmetic(x) \
@@ -140,9 +147,19 @@ SQLiteDBInstance::SQLiteDBInstance(sqlite3*& db, std::mutex& mtx)
   }
 }
 
+static inline void openOptimized(sqlite3*& db) {
+  sqlite3_open(":memory:", &db);
+
+  std::string settings;
+  for (const auto& setting : kMemoryDBSettings) {
+    settings += "PRAGMA " + setting.first + "=" + setting.second + "; ";
+  }
+  sqlite3_exec(db, settings.c_str(), nullptr, nullptr, nullptr);
+}
+
 void SQLiteDBInstance::init() {
   primary_ = false;
-  sqlite3_open(":memory:", &db_);
+  openOptimized(db_);
 }
 
 void SQLiteDBInstance::addAffectedTable(VirtualTableContent* table) {
@@ -175,7 +192,7 @@ SQLiteDBInstance::~SQLiteDBInstance() {
 }
 
 SQLiteDBManager::SQLiteDBManager() : db_(nullptr) {
-  sqlite3_soft_heap_limit64(SQLITE_SOFT_HEAP_LIMIT);
+  sqlite3_soft_heap_limit64(1);
   setDisabledTables(Flag::getValue("disable_tables"));
 }
 
@@ -202,7 +219,7 @@ SQLiteDBInstanceRef SQLiteDBManager::getConnection(bool primary) {
 
   if (self.db_ == nullptr) {
     // Create primary SQLite DB instance.
-    sqlite3_open(":memory:", &self.db_);
+    openOptimized(self.db_);
     self.connection_ = SQLiteDBInstanceRef(new SQLiteDBInstance(self.db_));
     attachVirtualTables(self.connection_);
   }
@@ -301,7 +318,6 @@ Status queryInternal(const std::string& q, QueryData& results, sqlite3* db) {
     sqlite3_free(err);
     return Status(1, "Error running query: " + error_string);
   }
-
   return Status(0, "OK");
 }
 

osquery/sql/virtual_table.h

@@ -10,6 +10,8 @@
 
 #pragma once
 
+#include <boost/noncopyable.hpp>
+
 #include <osquery/tables.h>
 
 #include "osquery/core/conversions.h"
@@ -22,7 +24,7 @@ namespace osquery {
  *
  * Only used in the SQLite virtual table module methods.
  */
-struct BaseCursor {
+struct BaseCursor : private boost::noncopyable {
   /// SQLite virtual table cursor.
   sqlite3_vtab_cursor base;
   /// Track cursors for optional planner output.
@@ -41,7 +43,7 @@ struct BaseCursor {
  * Only used in the SQLite virtual table module methods.
  * This adds each table plugin class to the state tracking in SQLite.
  */
-struct VirtualTable {
+struct VirtualTable : private boost::noncopyable {
   /// The SQLite-provided virtual table structure.
   sqlite3_vtab base;
 

osquery/tables/system/linux/processes.cpp

@@ -8,8 +8,8 @@
  *
  */
 
-#include <string>
 #include <map>
+#include <string>
 
 #include <stdlib.h>
 #include <unistd.h>
@@ -17,9 +17,9 @@
 #include <boost/algorithm/string/trim.hpp>
 
 #include <osquery/core.h>
-#include <osquery/tables.h>
 #include <osquery/filesystem.h>
 #include <osquery/logger.h>
+#include <osquery/tables.h>
 
 #include "osquery/core/conversions.h"
 
@@ -143,7 +143,7 @@ void genProcessMap(const std::string& pid, QueryData& results) {
 
     // BSS with name in pathname.
     r["pseudo"] = (fields[4] == "0" && !r["path"].empty()) ? "1" : "0";
-    results.push_back(r);
+    results.push_back(std::move(r));
   }
 }
 

third-party

@@ -1 +1 @@
-Subproject commit bd9d658ff4fbc642384376663c1f9ea4f66be7be
+Subproject commit 467917469b31f567f5f6a755641cfe8179a68394

tools/tests/stress.py

@@ -87,15 +87,21 @@ def audit(args):
 
 def single(args):
     start_time = time.time()
-    proc = subprocess.Popen(args,
+    if ARGS.verbose:
-                            shell=True,
+        proc = subprocess.Popen(args, shell=True)
-                            stderr=subprocess.PIPE,
+    else:
-                            stdout=subprocess.PIPE)
+        proc = subprocess.Popen(args,
+                                shell=True,
+                                stderr=subprocess.PIPE,
+                                stdout=subprocess.PIPE)
+    if ARGS.verbose:
+        print("PID: %d" % (proc.pid))
     stdout, stderr = proc.communicate()
     end_time = time.time() - start_time
     if proc.returncode is not 0:
-        print(stdout)
+        if not ARGS.verbose:
-        print(stderr)
+            print(stdout)
+            print(stderr)
         print("%s Test failed. (total %6.4fs)" % (
             red("FAILED"), end_time))
         sys.exit(proc.returncode)
@@ -129,14 +135,16 @@ if __name__ == "__main__":
                         help="Arguments to pass to test binary")
     parser.add_argument("--stat", action="store_true", default=False,
                         help="Only print numerical values")
+    parser.add_argument("--verbose", action="store_true", default=False,
+                        help="Do not consume stderr/stdout")
     parser.add_argument("run", nargs="?", help="Run specific test binary")
-    args = parser.parse_args()
+    ARGS = parser.parse_args()
 
     # A baseline was requested, first run baselines then normal.
-    if args.baseline:
+    if ARGS.baseline:
         print("Running baseline tests...")
-        stress(vars(args))
+        stress(vars(ARGS))
-        args.baseline = False
+        ARGS.baseline = False
         print("Finished. Running tests...")
 
-    stress(vars(args))
+    stress(vars(ARGS))

tools/tests/test_base.py

@@ -165,8 +165,10 @@ class ProcRunner(object):
         pid = 0
         try:
             if self.silent:
-                self.proc = subprocess.Popen([self.path] + self.args,
+                self.proc = subprocess.Popen(
-                                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                    [self.path] + self.args,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE)
             else:
                 self.proc = subprocess.Popen([self.path] + self.args)
             pid = self.proc.pid

tools/tests/test_extensions.py

@@ -303,6 +303,7 @@ class ExtensionTests(test_base.ProcessGenerator, unittest.TestCase):
         # Now start a daemon
         daemon = self._run_daemon({
             "disable_watchdog": True,
+            "verbose": True,
             "extensions_timeout": EXTENSION_TIMEOUT,
             "extensions_socket": extension.options["extensions_socket"],
         })

tools/tests/test_osqueryi.py

@@ -69,7 +69,7 @@ class OsqueryiTest(unittest.TestCase):
         self.assertEqual(proc.proc.poll(), 0)
 
     @test_base.flaky
-    def test_config_check_failure(self):
+    def test_config_check_failure_invalid_path(self):
         '''Test that a missing config fails'''
         proc = test_base.TimeoutRunner([
             self.binary,
@@ -83,6 +83,8 @@ class OsqueryiTest(unittest.TestCase):
         print(proc.stderr)
         self.assertEqual(proc.proc.poll(), 1)
 
+    @test_base.flaky
+    def test_config_check_failure_valid_path(self):
         # Now with a valid path, but invalid content.
         proc = test_base.TimeoutRunner([
             self.binary,
@@ -94,6 +96,8 @@ class OsqueryiTest(unittest.TestCase):
         self.assertEqual(proc.proc.poll(), 1)
         self.assertNotEqual(proc.stderr, "")
 
+    @test_base.flaky
+    def test_config_check_failure_missing_plugin(self):
         # Finally with a missing config plugin
         proc = test_base.TimeoutRunner([
             self.binary,