diff --git a/include/osquery/core.h b/include/osquery/core.h index 81cc6254..045c49ae 100644 --- a/include/osquery/core.h +++ b/include/osquery/core.h @@ -117,7 +117,7 @@ class Initializer : private boost::noncopyable { * * @param name The name of the worker process. */ - void initWorkerWatcher(const std::string& name) const; + void initWorkerWatcher(const std::string& name = "") const; /// Assume initialization finished, start work. void start() const; diff --git a/include/osquery/registry.h b/include/osquery/registry.h index b6ff406e..f8dbccdc 100644 --- a/include/osquery/registry.h +++ b/include/osquery/registry.h @@ -272,6 +272,44 @@ class RegistryHelperCore : private boost::noncopyable { /// Allow a registry type to react to configuration updates. virtual void configure(); + /** + * @brief Add a set of item names broadcasted by an extension uuid. + * + * When an extension is registered the RegistryFactory will receive a + * RegistryBroadcast containing a all of the extension's registry names and + * the set of items with their optional route info. The factory depends on + * each registry to manage calls/requests to these external plugins. + * + * @param uuid The uuid chosen for the extension. + * @param routes The plugin name and optional route info list. + * @return Success if all routes were added, failure if any failed. + */ + Status addExternal(const RouteUUID& uuid, const RegistryRoutes& routes); + + /** + * @brief Each RegistryType will include a trampoline into the PluginType. + * + * A PluginType may act on registry modifications. Each specialized registry + * will include a trampoline method to call the plugin type's addExternal. + * + * @param name Plugin name (not the extension UUID). + * @param info The route information broadcasted. + */ + virtual Status addExternalPlugin(const std::string& name, + const PluginResponse& info) const = 0; + + /// Remove all the routes for a given uuid. + void removeExternal(const RouteUUID& uuid); + + /** + * @brief Each RegistryType will include a trampoline into the PluginType. + * + * A PluginType may act on registry modifications. Each specialized registry + * will include a trampoline method to call the plugin type's removeExternal. + * @param name Plugin name (not the extension UUID). + */ + virtual void removeExternalPlugin(const std::string& name) const = 0; + /// Facility method to check if a registry item exists. bool exists(const std::string& item_name, bool local = false) const; @@ -358,49 +396,6 @@ class RegistryHelper : public RegistryHelperCore { remove_(&RegistryType::removeExternal){}; virtual ~RegistryHelper() {} - /** - * @brief Add a set of item names broadcasted by an extension uuid. - * - * When an extension is registered the RegistryFactory will receive a - * RegistryBroadcast containing a all of the extension's registry names and - * the set of items with their optional route info. The factory depends on - * each registry to manage calls/requests to these external plugins. - * - * @param uuid The uuid chosen for the extension. - * @param routes The plugin name and optional route info list. - * @return Success if all routes were added, failure if any failed. - */ - Status addExternal(const RouteUUID& uuid, const RegistryRoutes& routes) { - // Add each route name (item name) to the tracking. - for (const auto& route : routes) { - // Keep the routes info assigned to the registry. - routes_[route.first] = route.second; - auto status = add_(route.first, route.second); - external_[route.first] = uuid; - if (!status.ok()) { - return status; - } - } - return Status(0, "OK"); - } - - /// Remove all the routes for a given uuid. - void removeExternal(const RouteUUID& uuid) { - std::vector removed_items; - for (const auto& item : external_) { - if (item.second == uuid) { - remove_(item.first); - removed_items.push_back(item.first); - } - } - - // Remove items belonging to the external uuid. - for (const auto& item : removed_items) { - external_.erase(item); - routes_.erase(item); - } - } - /** * @brief Add a plugin to this registry by allocating and indexing * a type Item and a key identifier. @@ -440,6 +435,18 @@ class RegistryHelper : public RegistryHelperCore { return std::dynamic_pointer_cast(items_.at(item_name)); } + /// Trampoline function for calling the PluginType's addExternal. + Status addExternalPlugin(const std::string& name, + const PluginResponse& info) const override { + return add_(name, info); + } + + /// Trampoline function for calling the PluginType's removeExternal. + void removeExternalPlugin(const std::string& name) const override { + remove_(name); + } + + /// Construct and return a map of plugin names to their implementation. const std::map all() const { std::map ditems; for (const auto& item : items_) { @@ -458,10 +465,12 @@ class RegistryHelper : public RegistryHelperCore { RemoveExternalCallback remove_; }; -/// Helper defintion for a shared pointer to a Plugin. +/// Helper definition for a shared pointer to a Plugin. using PluginRef = std::shared_ptr; + /// Helper definition for a basic-templated Registry type using a base Plugin. using PluginRegistryHelper = RegistryHelper; + /// Helper definitions for a shared pointer to the basic Registry type. using PluginRegistryHelperRef = std::shared_ptr; diff --git a/include/osquery/tables.h b/include/osquery/tables.h index 236e077a..d46576f0 100644 --- a/include/osquery/tables.h +++ b/include/osquery/tables.h @@ -231,7 +231,7 @@ struct ConstraintList { std::set getAll(ConstraintOperator op) const; /// See ConstraintList::getAll, but as a selected literal type. - template + template std::set getAll(ConstraintOperator op) const { std::set literal_matches; auto matches = getAll(op); @@ -399,7 +399,7 @@ class TablePlugin : public Plugin { std::string columnDefinition() const; /// Return the name and column pairs for attaching virtual tables. - PluginResponse routeInfo() const; + PluginResponse routeInfo() const override; /** * @brief Check if there are fresh cache results for this table. @@ -471,7 +471,7 @@ class TablePlugin : public Plugin { * @param request The plugin request, must include an action key. * @param response A plugin response, for generation this contains the rows. */ - Status call(const PluginRequest& request, PluginResponse& response); + Status call(const PluginRequest& request, PluginResponse& response) override; public: /// Helper data structure transformation methods. diff --git a/osquery/core/flags.cpp b/osquery/core/flags.cpp index e1474a2d..82619c12 100644 --- a/osquery/core/flags.cpp +++ b/osquery/core/flags.cpp @@ -132,6 +132,15 @@ void Flag::printFlags(bool shell, bool external, bool cli) { // Additional index for flag values. max += 6; + // Show a reference to the flags documentation. + + // Show the Gflags-specific 'flagfile'. + if (!shell && cli) { + fprintf(stdout, " --flagfile PATH"); + fprintf(stdout, "%s", std::string(max - 8 - 5, ' ').c_str()); + fprintf(stdout, " Line-delimited file of additional flags\n"); + } + auto& aliases = instance().aliases_; for (const auto& flag : info) { if (details.count(flag.name) > 0) { diff --git a/osquery/core/init.cpp b/osquery/core/init.cpp index 1827e040..fdefb60e 100644 --- a/osquery/core/init.cpp +++ b/osquery/core/init.cpp @@ -65,6 +65,12 @@ enum { }; #endif +#ifdef __linux__ +#define OSQUERY_HOME "/etc/osquery" +#else +#define OSQUERY_HOME "/var/osquery" +#endif + #define DESCRIPTION \ "osquery %s, your OS as a high-performance relational database\n" #define EPILOG "\nosquery project page .\n" @@ -77,7 +83,8 @@ enum { "You are using default configurations for osqueryd for one or more of the " \ "following\n" \ "flags: pidfile, db_path.\n\n" \ - "These options create files in /var/osquery but it looks like that path " \ + "These options create files in " OSQUERY_HOME \ + " but it looks like that path " \ "has not\n" \ "been created. Please consider explicitly defining those " \ "options as a different \n" \ @@ -242,6 +249,8 @@ Initializer::Initializer(int& argc, char**& argv, ToolType tool) if (tool == OSQUERY_TOOL_SHELL) { // The shell is transient, rewrite config-loaded paths. FLAGS_disable_logging = true; + // The shell never will not fork a worker. + FLAGS_disable_watchdog = true; // Get the caller's home dir for temporary storage/state management. auto homedir = osqueryHomeDirectory(); boost::system::error_code ec; @@ -309,7 +318,7 @@ void Initializer::initDaemon() const { // Check if /var/osquery exists if ((Flag::isDefault("pidfile") || Flag::isDefault("database_path")) && - !isDirectory("/var/osquery")) { + !isDirectory(OSQUERY_HOME)) { std::cerr << CONFIG_ERROR; } @@ -337,10 +346,11 @@ void Initializer::initDaemon() const { void Initializer::initWatcher() const { // The watcher takes a list of paths to autoload extensions from. + // The loadExtensions call will populate the watcher's list of extensions. osquery::loadExtensions(); - // Add a watcher service thread to start/watch an optional worker and set - // of optional extensions in the autoload paths. + // Add a watcher service thread to start/watch an optional worker and list + // of optional extensions from the autoload paths. if (Watcher::hasManagedExtensions() || !FLAGS_disable_watchdog) { Dispatcher::addService(std::make_shared( *argc_, *argv_, !FLAGS_disable_watchdog)); @@ -385,7 +395,8 @@ void Initializer::initWorker(const std::string& name) const { (*argv_)[0][original_name.size()] = '\0'; } - // Start a watcher watcher thread to exit the process if the watcher exits. + // Start a 'watcher watcher' thread to exit the process if the watcher exits. + // In this case the parent process is called the 'watcher' process. Dispatcher::addService(std::make_shared(getppid())); } @@ -409,6 +420,7 @@ void Initializer::initActivePlugin(const std::string& type, if (timeout < kExtensionInitializeLatencyUS * 10) { timeout = kExtensionInitializeLatencyUS * 10; } + while (!Registry::setActive(type, name)) { if (!Watcher::hasManagedExtensions() || delay > timeout) { LOG(ERROR) << "Active " << type << " plugin not found: " << name; @@ -476,7 +488,9 @@ void Initializer::start() const { } // Initialize the status and result plugin logger. - initActivePlugin("logger", FLAGS_logger_plugin); + if (!FLAGS_disable_logging) { + initActivePlugin("logger", FLAGS_logger_plugin); + } initLogger(binary_); // Initialize the distributed plugin, if necessary diff --git a/osquery/extensions/extensions.cpp b/osquery/extensions/extensions.cpp index cf700e07..3397a6f2 100644 --- a/osquery/extensions/extensions.cpp +++ b/osquery/extensions/extensions.cpp @@ -77,6 +77,8 @@ EXTENSION_FLAG_ALIAS(interval, extensions_interval); void ExtensionWatcher::start() { // Watch the manager, if the socket is removed then the extension will die. + // A check for sane paths and activity is applied before the watcher + // service is added and started. while (true) { watch(); interruptableSleep(interval_); @@ -89,17 +91,30 @@ void ExtensionWatcher::exitFatal(int return_code) { } void ExtensionWatcher::watch() { + // Attempt to ping the extension core. + // This does NOT use pingExtension to avoid the latency checks applied. ExtensionStatus status; - try { - auto client = EXManagerClient(path_); - // Ping the extension manager until it goes down. - client.get()->ping(status); - } catch (const std::exception& e) { - LOG(WARNING) << "Extension watcher ending: osquery core has gone away"; + bool core_sane = true; + if (isWritable(path_)) { + try { + auto client = EXManagerClient(path_); + // Ping the extension manager until it goes down. + client.get()->ping(status); + } catch (const std::exception& e) { + core_sane = false; + } + } else { + // The previously-writable extension socket is not usable. + core_sane = false; + } + + if (!core_sane) { + LOG(INFO) << "Extension watcher ending: osquery core has gone away"; exitFatal(0); } if (status.code != ExtensionCode::EXT_SUCCESS && fatal_) { + // The core may be healthy but return a failed ping status. exitFatal(); } } @@ -164,7 +179,8 @@ void loadExtensions() { return; } - // Optionally autoload extensions + // Optionally autoload extensions, sanitize the binary path and inform + // the osquery watcher to execute the extension when started. auto status = loadExtensions(FLAGS_extensions_autoload); if (!status.ok()) { VLOG(1) << "Could not autoload extensions: " << status.what(); @@ -184,6 +200,8 @@ Status loadExtensions(const std::string& loadfile) { for (auto& path : osquery::split(autoload_paths, "\n")) { boost::trim(path); if (path.size() > 0 && path[0] != '#' && path[0] != ';') { + // After the path is sanitized the watcher becomes responsible for + // forking and executing the extension binary. Watcher::addExtensionPath(path); } } diff --git a/osquery/main/shell.cpp b/osquery/main/shell.cpp index 795c25b7..1ed283f4 100644 --- a/osquery/main/shell.cpp +++ b/osquery/main/shell.cpp @@ -69,6 +69,12 @@ int profile(int argc, char *argv[]) { int main(int argc, char *argv[]) { // Parse/apply flags, start registry, load logger/config plugins. osquery::Initializer runner(argc, argv, osquery::OSQUERY_TOOL_SHELL); + + // The shell will not use a worker process. + // It will initialize a watcher thread for potential auto-loaded extensions. + runner.initWorkerWatcher(); + + // Check for shell-specific switches and positional arguments. if (argc > 1 || !isatty(fileno(stdin)) || osquery::FLAGS_A.size() > 0 || osquery::FLAGS_L || osquery::FLAGS_profile > 0) { // A query was set as a positional argument, via stdin, or profiling is on. diff --git a/osquery/registry/registry.cpp b/osquery/registry/registry.cpp index 0dc9b49a..85e456cc 100644 --- a/osquery/registry/registry.cpp +++ b/osquery/registry/registry.cpp @@ -60,14 +60,21 @@ Status RegistryHelperCore::setActive(const std::string& item_name) { } } + Status status(0, "OK"); active_ = item_name; // The active plugin is setup when initialized. for (const auto& item : osquery::split(item_name, ",")) { if (exists(item, true)) { - Registry::get(name_, item)->setUp(); + status = Registry::get(name_, item)->setUp(); + } else if (exists(item, false) && !Registry::external()) { + // If the active plugin is within an extension we must wait. + // An extension will first broadcast the registry, then receive the list + // of active plugins, active them if they are extension-local, and finally + // start their extension socket. + status = pingExtension(getExtensionSocket(external_.at(item_name))); } } - return Status(0, "OK"); + return status; } const std::string& RegistryHelperCore::getActive() const { return active_; } @@ -139,6 +146,20 @@ const std::string& RegistryHelperCore::getAlias( return aliases_.at(alias); } +Status RegistryHelperCore::add(const std::string& item_name, bool internal) { + // The item can be listed as internal, meaning it does not broadcast. + if (internal) { + internal_.push_back(item_name); + } + + // The item may belong to a module. + if (RegistryFactory::usingModule()) { + modules_[item_name] = RegistryFactory::getModule(); + } + + return Status(0, "OK"); +} + void RegistryHelperCore::setUp() { // If this registry does not auto-setup do NOT setup the registry items. if (!auto_setup_) { @@ -176,6 +197,38 @@ void RegistryHelperCore::configure() { } } +Status RegistryHelperCore::addExternal(const RouteUUID& uuid, + const RegistryRoutes& routes) { + // Add each route name (item name) to the tracking. + for (const auto& route : routes) { + // Keep the routes info assigned to the registry. + routes_[route.first] = route.second; + auto status = addExternalPlugin(route.first, route.second); + external_[route.first] = uuid; + if (!status.ok()) { + return status; + } + } + return Status(0, "OK"); +} + +/// Remove all the routes for a given uuid. +void RegistryHelperCore::removeExternal(const RouteUUID& uuid) { + std::vector removed_items; + for (const auto& item : external_) { + if (item.second == uuid) { + removeExternalPlugin(item.first); + removed_items.push_back(item.first); + } + } + + // Remove items belonging to the external uuid. + for (const auto& item : removed_items) { + external_.erase(item); + routes_.erase(item); + } +} + /// Facility method to check if a registry item exists. bool RegistryHelperCore::exists(const std::string& item_name, bool local) const { @@ -424,20 +477,6 @@ size_t RegistryFactory::count(const std::string& registry_name) { return instance().registry(registry_name)->count(); } -Status RegistryHelperCore::add(const std::string& item_name, bool internal) { - // The item can be listed as internal, meaning it does not broadcast. - if (internal) { - internal_.push_back(item_name); - } - - // The item may belong to a module. - if (RegistryFactory::usingModule()) { - modules_[item_name] = RegistryFactory::getModule(); - } - - return Status(0, "OK"); -} - const std::map& RegistryFactory::getModules() { return instance().modules_; } diff --git a/osquery/tables/utility/osquery.cpp b/osquery/tables/utility/osquery.cpp index eb7e9948..e3117dd3 100644 --- a/osquery/tables/utility/osquery.cpp +++ b/osquery/tables/utility/osquery.cpp @@ -156,14 +156,14 @@ QueryData genOsqueryExtensions(QueryContext& context) { ExtensionList extensions; if (getExtensions(extensions).ok()) { - for (const auto& extenion : extensions) { + for (const auto& extension : extensions) { Row r; - r["uuid"] = TEXT(extenion.first); - r["name"] = extenion.second.name; - r["version"] = extenion.second.version; - r["sdk_version"] = extenion.second.sdk_version; - r["path"] = getExtensionSocket(extenion.first); - r["type"] = "extension"; + r["uuid"] = TEXT(extension.first); + r["name"] = extension.second.name; + r["version"] = extension.second.version; + r["sdk_version"] = extension.second.sdk_version; + r["path"] = getExtensionSocket(extension.first); + r["type"] = (extension.first == 0) ? "core" : "extension"; results.push_back(r); } }