database: Move initialization retry logic into DB API (#6633)

osquery/core/init.cpp

@@ -125,8 +125,6 @@ DWORD kLegacyThreadId;
 /// When no flagfile is provided via CLI, attempt to read flag 'defaults'.
 const std::string kBackupDefaultFlagfile{OSQUERY_HOME "osquery.flags.default"};
 
-const size_t kDatabaseMaxRetryCount{25};
-const size_t kDatabaseRetryDelay{200};
 bool Initializer::isWorker_{false};
 
 namespace {
@@ -526,20 +524,11 @@ void Initializer::start() const {
     // A daemon must always have R/W access to the database.
     setDatabaseRequireWrite(isDaemon());
 
-    for (size_t i = 1; i <= kDatabaseMaxRetryCount; i++) {
-      if (initDatabasePlugin().ok()) {
-        break;
-      }
-
-      if (i == kDatabaseMaxRetryCount) {
-        auto message = std::string(RLOG(1629)) + binary_ +
-                       " initialize failed: Could not initialize database";
-        auto retcode = (isWorker()) ? EXIT_CATASTROPHIC : EXIT_FAILURE;
-        requestShutdown(retcode, message);
-        return;
-      }
-
-      sleepFor(kDatabaseRetryDelay);
+    auto status = initDatabasePlugin();
+    if (!status.ok()) {
+      auto retcode = (isWorker()) ? EXIT_CATASTROPHIC : EXIT_FAILURE;
+      requestShutdown(retcode, status.getMessage());
+      return;
     }
 
     // Ensure the database results version is up to date before proceeding

osquery/database/database.cpp

@@ -15,6 +15,7 @@
 #include <osquery/core/flags.h>
 #include <osquery/database/database.h>
 #include <osquery/logger/logger.h>
+#include <osquery/process/process.h>
 #include <osquery/registry/registry.h>
 #include <osquery/utils/config/default_paths.h>
 #include <osquery/utils/conversions/tryto.h>
@@ -66,6 +67,17 @@ std::atomic<bool> kDBChecking(false);
  */
 Mutex kDatabaseReset;
 
+/**
+ * @brief Try multiple times to initialize persistent storage.
+ *
+ * It might be the case that other processes are stopping and have not released
+ * their whole-process lock on the database.
+ */
+const size_t kDatabaseMaxRetryCount{25};
+
+/// Number of millisecons to pause between database initialize retries.
+const size_t kDatabaseRetryDelay{200};
+
 Status DatabasePlugin::reset() {
   // Keep this simple, scope the critical section to the broader methods.
   tearDown();
@@ -465,24 +477,42 @@ void setDatabaseAllowOpen(bool allow_open) {
 }
 
 Status initDatabasePlugin() {
+  if (kDBInitialized) {
+    return Status::success();
+  }
+
   // Initialize the database plugin using the flag.
   auto plugin = (FLAGS_disable_database) ? "ephemeral" : kInternalDatabase;
   {
-    auto const status = RegistryFactory::get().setActive("database", plugin);
-    if (status.ok()) {
-      kDBInitialized = true;
-      return status;
+    Status status;
+    for (size_t i = 0; i < kDatabaseMaxRetryCount; i++) {
+      status = RegistryFactory::get().setActive("database", plugin);
+      if (status.ok()) {
+        kDBInitialized = true;
+        return status;
+      }
+
+      if (FLAGS_disable_database) {
+        // Do not try multiple times to initialize the emphemeral plugin.
+        break;
+      }
+      sleepFor(kDatabaseRetryDelay);
     }
     LOG(WARNING) << "Failed to activate database plugin "
                  << boost::io::quoted(plugin) << ": " << status.what();
   }
 
   // If the database did not setUp override the active plugin.
+  if (FLAGS_disable_database) {
+    return Status::failure("Could not activate any database plugin");
+  }
+
   auto const status = RegistryFactory::get().setActive("database", "ephemeral");
   if (!status.ok()) {
     LOG(ERROR) << "Failed to activate database plugin \"ephemeral\": "
                << status.what();
   }
+
   kDBInitialized = status.ok();
   return status;
 }

osquery/database/database.h

@@ -254,7 +254,15 @@ void dumpDatabase();
 /// Require all database accesses to open a read and write handle.
 void setDatabaseRequireWrite(bool require_write = true);
 
-/// Allow database usage creations.
+/**
+ * @brief Allow database usage.
+ *
+ * We want to prevent implicit calls to database APIs before the application
+ * starts. To do this we require a call to setDatabaseAllowOpen.
+ *
+ * It is possible to "flip" this to not allow opening the database for testing
+ * purposes.
+ */
 void setDatabaseAllowOpen(bool allow_open = true);
 
 /**