valitydev/osquery-1
14
0
Fork 0
mirror of https://github.com/valitydev/osquery-1.git synced 2024-11-06 09:35:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Building example extension with SDK

Browse Source
This commit is contained in:
Teddy Reed 2015-02-18 17:19:45 -08:00
parent 472c6052ef
commit 451ef686ed
26 changed files with 487 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Makefile
View File

@ -45,6 +45,14 @@ test_sdk: .setup
cd build/$(BUILD_DIR)/sdk && SDK=True cmake ../../../ && \
$(MAKE) test --no-print-directory $(MAKEFLAGS)
debug_sdk: .setup
cd build/$(BUILD_DIR)/sdk && SDK=True DEBUG=True cmake ../../../ && \
$(MAKE) --no-print-directory $(MAKEFLAGS)
test_debug_sdk: .setup
cd build/$(BUILD_DIR)/sdk && SDK=True DEBUG=True cmake ../../../ && \
$(MAKE) test --no-print-directory $(MAKEFLAGS)
deps: .setup
./tools/provision.sh build build/$(BUILD_DIR)

4
include/osquery/core.h
View File

@ -31,9 +31,11 @@ namespace osquery {
* @brief The version of osquery
*/
extern const std::string kVersion;
extern const std::string kSDKVersion;
/// Use a macro for the version literal, set the kVersion symbol in the library.
#define OSQUERY_VERSION STR(OSQUERY_BUILD_VERSION)
#define OSQUERY_SDK_VERSION STR(OSQUERY_BUILD_SDK_VERSION)
/**
* @brief A helpful tool type to report when logging, print help, or debugging.
@ -42,7 +44,7 @@ enum osqueryTool {
OSQUERY_TOOL_SHELL,
OSQUERY_TOOL_DAEMON,
OSQUERY_TOOL_TEST,
OSQUERY_TOOL_EXTENSION,
OSQUERY_EXTENSION,
};
/**

5
include/osquery/dispatcher.h
View File

@ -28,9 +28,12 @@
#endif
#include <osquery/status.h>
#include <osquery/flags.h>
namespace osquery {
DECLARE_int32(worker_threads);
typedef apache::thrift::concurrency::ThreadManager InternalThreadManager;
typedef std::shared_ptr<InternalThreadManager> InternalThreadManagerRef;
@ -152,7 +155,7 @@ class Dispatcher {
void join();
/// See `join`, but applied to osquery services.
void joinServices();
static void joinServices();
/// Destroy and stop all osquery service threads and service objects.
void removeServices();

65
include/osquery/extensions.h
View File

@ -14,6 +14,7 @@
#include <osquery/dispatcher.h>
#include <osquery/flags.h>
#include <osquery/registry.h>
#include <osquery/sql.h>
#ifdef FBOSQUERY
#include "osquery/gen-cpp/Extension.h"
@ -143,8 +144,30 @@ class ExtensionManagerHandler : virtual public ExtensionManagerIf,
void deregisterExtension(ExtensionStatus& _return,
const ExtensionRouteUUID uuid);
/**
* @brief Execute an SQL statement in osquery core.
*
* Extensions do not have access to the internal SQLite implementation.
* For complex queries (beyond select all from a table) the statement must
* be passed into SQLite.
*
* @param _return The output Status and QueryData (as response).
* @param sql The sql statement.
*/
void query(ExtensionResponse& _return, const std::string& sql);
/**
* @brief Get SQL column information for SQL statements in osquery core.
*
* Extensions do not have access to the internal SQLite implementation.
* For complex queries (beyond metadata for a table) the statement must
* be passed into SQLite.
*
* @param _return The output Status and TableColumns (as response).
* @param sql The sql statement.
*/
void getQueryColumns(ExtensionResponse& _return, const std::string& sql);
private:
/// Check if an extension exists by the name it registered.
bool exists(const std::string& name);
@ -158,31 +181,36 @@ class ExtensionManagerHandler : virtual public ExtensionManagerIf,
class ExtensionWatcher : public InternalRunnable {
public:
virtual ~ExtensionWatcher() {}
ExtensionWatcher(const std::string& manager_path,
size_t interval,
bool fatal) {
manager_path_ = manager_path;
interval_ = interval;
fatal_ = fatal;
}
ExtensionWatcher(const std::string& path, size_t interval, bool fatal)
: path_(path), interval_(interval), fatal_(fatal) {}
public:
/// The Dispatcher thread entry point.
void enter();
/// Perform health checks.
virtual void watch();
private:
protected:
/// Exit the extension process with a fatal if the ExtensionManager dies.
void exitFatal();
void exitFatal(int return_code = 1);
private:
protected:
/// The UNIX domain socket path for the ExtensionManager.
std::string manager_path_;
std::string path_;
/// The internal in milliseconds to ping the ExtensionManager.
size_t interval_;
/// If the ExtensionManager socket is closed, should the extension exit.
bool fatal_;
};
class ExtensionManagerWatcher : public ExtensionWatcher {
public:
ExtensionManagerWatcher(const std::string& path, size_t interval)
: ExtensionWatcher(path, interval, false) {}
void watch();
};
/// A Dispatcher service thread that starts ExtensionHandler.
class ExtensionRunner : public InternalRunnable {
public:
@ -221,6 +249,13 @@ class ExtensionManagerRunner : public InternalRunnable {
std::string path_;
};
/// External (extensions) SQL implementation of the osquery query API.
Status queryExternal(const std::string& query, QueryData& results);
/// External (extensions) SQL implementation of the osquery getQueryColumns API.
Status getQueryColumnsExternal(const std::string& q,
tables::TableColumns& columns);
/// Status get a list of active extenions.
Status getExtensions(ExtensionList& extensions);
@ -259,12 +294,18 @@ Status callExtension(const std::string& extension_path,
PluginResponse& response);
/// The main runloop entered by an Extension, start an ExtensionRunner thread.
Status startExtension();
Status startExtension(const std::string& name, const std::string& version);
/// The main runloop entered by an Extension, start an ExtensionRunner thread.
Status startExtension(const std::string& name,
const std::string& version,
const std::string& min_sdk_version);
/// Internal startExtension implementation using a UNIX domain socket path.
Status startExtension(const std::string& manager_path,
const std::string& name,
const std::string& version,
const std::string& min_sdk_version,
const std::string& sdk_version);
/// Start an ExtensionWatcher thread.

21
include/osquery/registry.h
View File

@ -73,6 +73,10 @@ namespace osquery {
#define REGISTER(type, registry, name) \
const auto type##RegistryItem = Registry::add<type>(registry, name);
/// The same as REGISTER but prevents the plugin item from being broadcasted.
#define REGISTER_INTERNAL(type, registry, name) \
const auto type##RegistryItem = Registry::add<type>(registry, name, true);
/// A plugin (registry item) may return a custom key value map with its Route.
typedef std::map<std::string, std::string> RouteInfo;
/// Registry routes are a map of item name to each optional RouteInfo.
@ -220,6 +224,7 @@ class RegistryHelperCore {
/// A map of registered plugin instances to their registered identifier.
std::map<std::string, std::shared_ptr<Plugin> > items_;
std::map<std::string, std::string> aliases_;
std::vector<std::string> internal_;
};
/**
@ -251,7 +256,7 @@ class RegistryHelper : public RegistryHelperCore {
* @return A success/failure status.
*/
template <class Item>
Status add(const std::string& item_name) {
Status add(const std::string& item_name, bool internal = false) {
if (items_.count(item_name) > 0) {
return Status(1, "Duplicate registry item exists: " + item_name);
}
@ -261,6 +266,12 @@ class RegistryHelper : public RegistryHelperCore {
std::shared_ptr<RegistryType> item((RegistryType*)new Item());
item->setName(item_name);
items_[item_name] = item;
// The item can be listed as internal, meaning it does not broadcast.
if (internal) {
internal_.push_back(item_name);
}
return Status(0, "OK");
}
@ -335,9 +346,10 @@ class RegistryFactory : private boost::noncopyable {
template <class Item>
static Status add(const std::string& registry_name,
const std::string& item_name) {
const std::string& item_name,
bool internal = false) {
auto registry = instance().registry(registry_name);
return registry->template add<Item>(item_name);
return registry->template add<Item>(item_name, internal);
}
static const std::map<std::string, PluginRegistryHelperRef>& all();
@ -381,6 +393,9 @@ class RegistryFactory : private boost::noncopyable {
static std::vector<std::string> names(const std::string& registry_name);
/// Get a list of the registered extension UUIDs.
static std::vector<RouteUUID> routeUUIDs();
static size_t count();
static size_t count(const std::string& registry_name);

29
include/osquery/sdk.h Normal file
View File

@ -0,0 +1,29 @@
/*
* Copyright (c) 2014, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#pragma once
#ifndef OSQUERY_BUILD_SDK
#define OSQUERY_BUILD_SDK
#endif
#include <osquery/config.h>
#include <osquery/core.h>
#include <osquery/database.h>
#include <osquery/events.h>
#include <osquery/extensions.h>
#include <osquery/filesystem.h>
#include <osquery/flags.h>
#include <osquery/hash.h>
#include <osquery/logger.h>
#include <osquery/registry.h>
#include <osquery/sql.h>
#include <osquery/status.h>
#include <osquery/tables.h>

5
osquery.thrift
View File

@ -9,6 +9,7 @@ struct InternalExtensionInfo {
1:string name,
2:string version,
3:string sdk_version,
4:string min_sdk_version,
}
/// Unique ID for each extension.
@ -75,4 +76,8 @@ service ExtensionManager extends Extension {
ExtensionResponse query(
1:string sql,
),
/// Allow an extension to introspect into SQL used in a parsed query.
ExtensionResponse getQueryColumns(
1:string sql,
),
}

22
osquery/core/init.cpp
View File

@ -94,19 +94,21 @@ void initOsquery(int argc, char* argv[], int tool) {
// Initialize the status and results logger.
initStatusLogger(binary);
VLOG(1) << "osquery initializing [version=" OSQUERY_VERSION "]";
if (tool != OSQUERY_EXTENSION) {
VLOG(1) << "osquery initializing [version=" OSQUERY_VERSION "]";
// Load the osquery config using the default/active config plugin.
Config::getInstance().load();
// Load the osquery config using the default/active config plugin.
Config::getInstance().load();
if (FLAGS_config_check) {
// The initiator requested an initialization and config check.
auto s = Config::checkConfig();
if (!s.ok()) {
std::cerr << "Error reading config: " << s.toString() << "\n";
if (FLAGS_config_check) {
// The initiator requested an initialization and config check.
auto s = Config::checkConfig();
if (!s.ok()) {
std::cerr << "Error reading config: " << s.toString() << "\n";
}
// A configuration check exits the application.
::exit(s.getCode());
}
// A configuration check exits the application.
::exit(s.getCode());
}
// Run the setup for all non-lazy registries.

5
osquery/core/test_util.cpp
View File

@ -24,7 +24,12 @@ namespace pt = boost::property_tree;
namespace osquery {
const std::string kTestQuery = "SELECT * FROM test_table";
#ifndef OSQUERY_BUILD_SDK
const std::string kTestDataPath = "../../../../tools/tests/";
#else
const std::string kTestDataPath = "../../../../../tools/tests/";
#endif
QueryData getTestDBExpectedResults() {
QueryData d;

2
osquery/dispatcher/dispatcher.cpp
View File

@ -81,7 +81,7 @@ InternalThreadManagerRef Dispatcher::getThreadManager() {
void Dispatcher::join() { thread_manager_->join(); }
void Dispatcher::joinServices() {
for (auto& thread : service_threads_) {
for (auto& thread : getInstance().service_threads_) {
thread->join();
}
}

9
osquery/distributed/distributed_tests.cpp
View File

@ -58,6 +58,8 @@ TEST_F(DistributedTests, test_parse_query_json) {
}
TEST_F(DistributedTests, test_handle_query) {
// Access to the internal SQL implementation is only available in core.
#ifndef OSQUERY_BUILD_SDK
SQL query = DistributedQueryHandler::handleQuery("SELECT hour from time");
ASSERT_TRUE(query.ok());
QueryData rows = query.rows();
@ -68,6 +70,7 @@ TEST_F(DistributedTests, test_handle_query) {
ASSERT_FALSE(query.ok());
rows = query.rows();
ASSERT_EQ(0, rows.size());
#endif
}
TEST_F(DistributedTests, test_serialize_results_empty) {
@ -132,6 +135,8 @@ TEST_F(DistributedTests, test_serialize_results_multiple) {
}
TEST_F(DistributedTests, test_do_queries) {
// Access to the internal SQL implementation is only available in core.
#ifndef OSQUERY_BUILD_SDK
auto provider_raw = new MockDistributedProvider();
provider_raw->queriesJSON_ =
R"([
@ -176,9 +181,12 @@ TEST_F(DistributedTests, test_do_queries) {
EXPECT_LE(row->second.get<int>("minutes"), 60);
EXPECT_EQ(getHostname(), row->second.get<std::string>("_source_host"));
}
#endif
}
TEST_F(DistributedTests, test_duplicate_request) {
// Access to the internal SQL implementation is only available in core.
#ifndef OSQUERY_BUILD_SDK
auto provider_raw = new MockDistributedProvider();
provider_raw->queriesJSON_ =
R"([
@ -209,6 +217,7 @@ TEST_F(DistributedTests, test_duplicate_request) {
json_stream.str(provider_raw->resultsJSON_);
ASSERT_NO_THROW(pt::read_json(json_stream, tree));
EXPECT_EQ(0, tree.get_child("results").size());
#endif
}
}

17
osquery/examples/example_extension.cpp
View File

@ -8,9 +8,9 @@
*
*/
#include <osquery/tables.h>
#include <osquery/sdk.h>
namespace osquery {
using namespace osquery;
class ExampleTable : public tables::TablePlugin {
private:
@ -31,9 +31,16 @@ class ExampleTable : public tables::TablePlugin {
};
REGISTER(ExampleTable, "table", "example");
}
int main(int argc, char* argv[]) {
// Do some broadcast of the registry.
auto example = std::make_shared<osquery::ExampleTable>();
initOsquery(argc, argv, OSQUERY_EXTENSION);
auto status = startExtension("example", "0.0.1");
if (!status.ok()) {
LOG(ERROR) << status.getMessage();
}
// Finally shutdown.
shutdownOsquery();
return 0;
}

368
osquery/extensions/extensions.cpp
View File

@ -11,19 +11,23 @@
#include <csignal>
#ifdef FBOSQUERY
#include <thrift/lib/cpp/concurrency/ThreadManager.h>
#include <thrift/lib/cpp/concurrency/PosixThreadFactory.h>
#include <thrift/lib/cpp/server/example/TThreadPoolServer.h>
#include <thrift/lib/cpp/protocol/TBinaryProtocol.h>
#include <thrift/lib/cpp/server/example/TSimpleServer.h>
#include <thrift/lib/cpp/transport/TServerSocket.h>
#include <thrift/lib/cpp/transport/TBufferTransports.h>
#include <thrift/lib/cpp/transport/TSocket.h>
#define _SHARED_PTR std::shared_ptr
#define _SHARED_PTR std
#else
#include <thrift/concurrency/ThreadManager.h>
#include <thrift/concurrency/PosixThreadFactory.h>
#include <thrift/server/TThreadPoolServer.h>
#include <thrift/protocol/TBinaryProtocol.h>
#include <thrift/server/TSimpleServer.h>
#include <thrift/transport/TServerSocket.h>
#include <thrift/transport/TBufferTransports.h>
#include <thrift/transport/TSocket.h>
#define _SHARED_PTR boost::shared_ptr
#define _SHARED_PTR boost
#endif
#include <osquery/extensions.h>
@ -35,11 +39,14 @@ using namespace apache::thrift;
using namespace apache::thrift::protocol;
using namespace apache::thrift::transport;
using namespace apache::thrift::server;
using namespace apache::thrift::concurrency;
using namespace osquery::extensions;
namespace osquery {
const int kWatcherMLatency = 3000;
FLAG(bool, disable_extensions, false, "Disable extension API");
FLAG(string,
@ -47,6 +54,50 @@ FLAG(string,
"/var/osquery/osquery.em",
"Path to the extensions UNIX domain socket")
/// Internal accessor for extension clients.
class EXInternal {
public:
EXInternal(const std::string& path)
: socket_(new TSocket(path)),
transport_(new TBufferedTransport(socket_)),
protocol_(new TBinaryProtocol(transport_)) {}
virtual ~EXInternal() { transport_->close(); }
protected:
_SHARED_PTR::shared_ptr<TSocket> socket_;
_SHARED_PTR::shared_ptr<TTransport> transport_;
_SHARED_PTR::shared_ptr<TProtocol> protocol_;
};
/// Internal accessor for a client to an extension (from an extension manager).
class EXClient : public EXInternal {
public:
EXClient(const std::string& path) : EXInternal(path) {
client_ = std::make_shared<ExtensionClient>(protocol_);
transport_->open();
}
const std::shared_ptr<ExtensionClient>& get() { return client_; }
private:
std::shared_ptr<ExtensionClient> client_;
};
/// Internal accessor for a client to an extension manager (from an extension).
class EXManagerClient : public EXInternal {
public:
EXManagerClient(const std::string& manager_path) : EXInternal(manager_path) {
client_ = std::make_shared<ExtensionManagerClient>(protocol_);
transport_->open();
}
const std::shared_ptr<ExtensionManagerClient>& get() { return client_; }
private:
std::shared_ptr<ExtensionManagerClient> client_;
};
namespace extensions {
void ExtensionHandler::ping(ExtensionStatus& _return) {
@ -98,10 +149,10 @@ void ExtensionManagerHandler::registerExtension(
<< ")";
if (!Registry::addBroadcast(uuid, registry).ok()) {
LOG(WARNING) << "Could not add extension (" << uuid
LOG(WARNING) << "Could not add extension (" << info.name << ", " << uuid
<< ") broadcast to registry";
_return.code = ExtensionCode::EXT_FAILED;
_return.message = "Failed adding registry broadcase";
_return.message = "Failed adding registry broadcast";
return;
}
@ -137,7 +188,36 @@ void ExtensionManagerHandler::query(ExtensionResponse& _return,
}
}
void ExtensionManagerHandler::getQueryColumns(ExtensionResponse& _return,
const std::string& sql) {
tables::TableColumns columns;
auto status = osquery::getQueryColumns(sql, columns);
_return.status.code = status.getCode();
_return.status.message = status.getMessage();
if (status.ok()) {
for (const auto& column : columns) {
_return.response.push_back({{column.first, column.second}});
}
}
}
bool ExtensionManagerHandler::exists(const std::string& name) {
std::vector<RouteUUID> removed_routes;
const auto uuids = Registry::routeUUIDs();
for (const auto& ext : extensions_) {
// Find extension UUIDs that have gone away.
if (std::find(uuids.begin(), uuids.end(), ext.first) == uuids.end()) {
removed_routes.push_back(ext.first);
}
}
// Remove each from the manager's list of extenion metadata.
for (const auto& uuid : removed_routes) {
extensions_.erase(uuid);
}
// Search the remaining extension list for duplicates.
for (const auto& extension : extensions_) {
if (extension.second.name == name) {
return true;
@ -154,24 +234,34 @@ void ExtensionRunner::enter() {
auto socket_path = path_;
// Create the thrift instances.
_SHARED_PTR<ExtensionHandler> handler(new ExtensionHandler());
_SHARED_PTR<TProcessor> processor(new ExtensionProcessor(handler));
_SHARED_PTR<TServerTransport> serverTransport(
_SHARED_PTR::shared_ptr<ExtensionHandler> handler(new ExtensionHandler());
_SHARED_PTR::shared_ptr<TProcessor> processor(new ExtensionProcessor(handler));
_SHARED_PTR::shared_ptr<TServerTransport> serverTransport(
new TServerSocket(socket_path));
_SHARED_PTR<TTransportFactory> transportFactory(
_SHARED_PTR::shared_ptr<TTransportFactory> transportFactory(
new TBufferedTransportFactory());
_SHARED_PTR<TProtocolFactory> protocolFactory(
_SHARED_PTR::shared_ptr<TProtocolFactory> protocolFactory(
new TBinaryProtocolFactory());
_SHARED_PTR::shared_ptr<ThreadManager> threadManager =
ThreadManager::newSimpleThreadManager(FLAGS_worker_threads);
_SHARED_PTR::shared_ptr<PosixThreadFactory> threadFactory =
_SHARED_PTR::shared_ptr<PosixThreadFactory>(new PosixThreadFactory());
threadManager->threadFactory(threadFactory);
threadManager->start();
// Start the Thrift server's run loop.
try {
TSimpleServer server(
processor, serverTransport, transportFactory, protocolFactory);
TThreadPoolServer server(processor,
serverTransport,
transportFactory,
protocolFactory,
threadManager);
server.serve();
} catch (const std::exception& e) {
LOG(ERROR) << "Cannot start extension handler (" << socket_path << ") ("
LOG(ERROR) << "Cannot start extension handler: " << socket_path << " ("
<< e.what() << ")";
throw e;
return;
}
}
@ -185,21 +275,31 @@ void ExtensionManagerRunner::enter() {
auto socket_path = path_;
// Create the thrift instances.
_SHARED_PTR<ExtensionManagerHandler> handler(
_SHARED_PTR::shared_ptr<ExtensionManagerHandler> handler(
new ExtensionManagerHandler());
_SHARED_PTR<TProcessor> processor(
_SHARED_PTR::shared_ptr<TProcessor> processor(
new ExtensionManagerProcessor(handler));
_SHARED_PTR<TServerTransport> serverTransport(
_SHARED_PTR::shared_ptr<TServerTransport> serverTransport(
new TServerSocket(socket_path));
_SHARED_PTR<TTransportFactory> transportFactory(
_SHARED_PTR::shared_ptr<TTransportFactory> transportFactory(
new TBufferedTransportFactory());
_SHARED_PTR<TProtocolFactory> protocolFactory(
_SHARED_PTR::shared_ptr<TProtocolFactory> protocolFactory(
new TBinaryProtocolFactory());
_SHARED_PTR::shared_ptr<ThreadManager> threadManager =
ThreadManager::newSimpleThreadManager(FLAGS_worker_threads);
_SHARED_PTR::shared_ptr<PosixThreadFactory> threadFactory =
_SHARED_PTR::shared_ptr<PosixThreadFactory>(new PosixThreadFactory());
threadManager->threadFactory(threadFactory);
threadManager->start();
// Start the Thrift server's run loop.
try {
TSimpleServer server(
processor, serverTransport, transportFactory, protocolFactory);
TThreadPoolServer server(processor,
serverTransport,
transportFactory,
protocolFactory,
threadManager);
server.serve();
} catch (const std::exception& e) {
LOG(WARNING) << "Extensions disabled: cannot start extension manager ("
@ -209,65 +309,102 @@ void ExtensionManagerRunner::enter() {
void ExtensionWatcher::enter() {
// Watch the manager, if the socket is removed then the extension will die.
_SHARED_PTR<TSocket> socket(new TSocket(manager_path_));
_SHARED_PTR<TTransport> transport(new TBufferedTransport(socket));
_SHARED_PTR<TProtocol> protocol(new TBinaryProtocol(transport));
// Open a long-lived client to the extension manager.
ExtensionManagerClient client(protocol);
transport->open();
ExtensionStatus status;
while (true) {
// Ping the extension manager until it goes down.
client.ping(status);
if (status.code != ExtensionCode::EXT_SUCCESS && fatal_) {
transport->close();
exitFatal();
}
watch();
interruptableSleep(interval_);
}
// Code will never reach this socket close.
transport->close();
}
void ExtensionWatcher::exitFatal() {
void ExtensionWatcher::exitFatal(int return_code) {
// Exit the extension.
// Not yet implemented.
::exit(return_code);
}
#ifdef OSQUERY_EXTENSION_NAME
Status startExtension() {
void ExtensionWatcher::watch() {
ExtensionStatus status;
try {
auto client = EXManagerClient(path_);
// Ping the extension manager until it goes down.
client.get()->ping(status);
} catch (const std::exception& e) {
LOG(WARNING) << "Extension watcher ending: osquery core has gone away";
exitFatal(0);
}
if (status.code != ExtensionCode::EXT_SUCCESS && fatal_) {
exitFatal();
}
}
void ExtensionManagerWatcher::watch() {
// Watch the set of extensions, if the socket is removed then the extension
// will be deregistered.
const auto uuids = Registry::routeUUIDs();
ExtensionStatus status;
for (const auto& uuid : uuids) {
try {
auto client = EXClient(getExtensionSocket(uuid));
// Ping the extension until it goes down.
client.get()->ping(status);
} catch (const std::exception& e) {
LOG(INFO) << "Extension UUID " << uuid << " has gone away";
Registry::removeBroadcast(uuid);
}
if (status.code != ExtensionCode::EXT_SUCCESS && fatal_) {
Registry::removeBroadcast(uuid);
}
}
}
Status startExtension(const std::string& name, const std::string& version) {
return startExtension(name, version, "0.0.0");
}
Status startExtension(const std::string& name,
const std::string& version,
const std::string& min_sdk_version) {
// No assumptions about how the extensions logs, the first action is to
// start the extension's registry.
Registry::setUp();
auto status = startExtensionWatcher(FLAGS_extensions_socket, 3000, true);
if (status.ok()) {
status = startExtension(FLAGS_extensions_socket,
OSQUERY_EXTENSION_NAME,
OSQUERY_EXTENSION_VERSION,
OSQUERY_SDK_VERSION);
auto status =
startExtensionWatcher(FLAGS_extensions_socket, kWatcherMLatency, true);
if (!status.ok()) {
// If the threaded watcher fails to start, fail the extension.
return status;
}
return status;
status = startExtension(
FLAGS_extensions_socket, name, version, min_sdk_version, kSDKVersion);
if (!status.ok()) {
// If the extension failed to start then the EM is most likely unavailable.
return status;
}
try {
Dispatcher::joinServices();
} catch (const std::exception& e) {
// The extension manager may shutdown without notifying the extension.
return Status(0, e.what());
}
// An extension will only return on failure.
return Status(0, "OK");
}
#endif
Status startExtension(const std::string& manager_path,
const std::string& name,
const std::string& version,
const std::string& min_sdk_version,
const std::string& sdk_version) {
// Make sure the extension manager path exists, and is writable.
if (!pathExists(manager_path) || !isWritable(manager_path)) {
return Status(1, "Extension manager socket not availabe: " + manager_path);
}
// Open a socket to the extension manager to register.
_SHARED_PTR<TSocket> socket(new TSocket(manager_path));
_SHARED_PTR<TTransport> transport(new TBufferedTransport(socket));
_SHARED_PTR<TProtocol> protocol(new TBinaryProtocol(transport));
// The Registry broadcast is used as the ExtensionRegistry.
auto broadcast = Registry::getBroadcast();
@ -275,14 +412,13 @@ Status startExtension(const std::string& manager_path,
info.name = name;
info.version = version;
info.sdk_version = sdk_version;
info.min_sdk_version = min_sdk_version;
// Register the extension's registry broadcast with the manager.
ExtensionManagerClient client(protocol);
ExtensionStatus status;
try {
transport->open();
client.registerExtension(status, info, broadcast);
transport->close();
auto client = EXManagerClient(manager_path);
client.get()->registerExtension(status, info, broadcast);
}
catch (const std::exception& e) {
return Status(1, "Extension register failed: " + std::string(e.what()));
@ -292,12 +428,84 @@ Status startExtension(const std::string& manager_path,
return Status(status.code, status.message);
}
// Now that the uuid is known, try to clean up stale socket paths.
auto extension_path = getExtensionSocket(status.uuid, manager_path);
if (pathExists(extension_path).ok()) {
if (!isWritable(extension_path).ok()) {
return Status(1, "Cannot write extension socket: " + extension_path);
}
if (!remove(extension_path).ok()) {
return Status(1, "Cannot remove extension socket: " + extension_path);
}
}
// Start the extension's Thrift server
Dispatcher::getInstance().addService(
std::make_shared<ExtensionRunner>(manager_path, status.uuid));
VLOG(1) << "Extension (" << name << ", " << status.uuid << ", " << version
<< ", " << sdk_version << ") registered";
return Status(0, std::to_string(status.uuid));
}
Status queryExternal(const std::string& manager_path,
const std::string& query,
QueryData& results) {
// Make sure the extension path exists, and is writable.
if (!pathExists(manager_path) || !isWritable(manager_path)) {
return Status(1, "Extension manager socket not availabe: " + manager_path);
}
ExtensionResponse response;
try {
auto client = EXManagerClient(manager_path);
client.get()->query(response, query);
} catch (const std::exception& e) {
return Status(1, "Extension call failed: " + std::string(e.what()));
}
for (const auto& row : response.response) {
results.push_back(row);
}
return Status(response.status.code, response.status.message);
}
Status queryExternal(const std::string& query, QueryData& results) {
return queryExternal(FLAGS_extensions_socket, query, results);
}
Status getQueryColumnsExternal(const std::string& manager_path,
const std::string& query,
tables::TableColumns& columns) {
// Make sure the extension path exists, and is writable.
if (!pathExists(manager_path) || !isWritable(manager_path)) {
return Status(1, "Extension manager socket not availabe: " + manager_path);
}
ExtensionResponse response;
try {
auto client = EXManagerClient(manager_path);
client.get()->getQueryColumns(response, query);
} catch (const std::exception& e) {
return Status(1, "Extension call failed: " + std::string(e.what()));
}
// Translate response map: {string: string} to a vector: pair(name, type).
for (const auto& column : response.response) {
for (const auto& column_detail : column) {
columns.push_back(make_pair(column_detail.first, column_detail.second));
}
}
return Status(response.status.code, response.status.message);
}
Status getQueryColumnsExternal(const std::string& query,
tables::TableColumns& columns) {
return getQueryColumnsExternal(FLAGS_extensions_socket, query, columns);
}
Status pingExtension(const std::string& path) {
if (FLAGS_disable_extensions) {
return Status(1, "Extensions disabled");
@ -308,17 +516,10 @@ Status pingExtension(const std::string& path) {
return Status(1, "Extension socket not availabe: " + path);
}
// Open a socket to the extension.
_SHARED_PTR<TSocket> socket(new TSocket(path));
_SHARED_PTR<TTransport> transport(new TBufferedTransport(socket));
_SHARED_PTR<TProtocol> protocol(new TBinaryProtocol(transport));
ExtensionClient client(protocol);
ExtensionStatus ext_status;
try {
transport->open();
client.ping(ext_status);
transport->close();
auto client = EXClient(path);
client.get()->ping(ext_status);
} catch (const std::exception& e) {
return Status(1, "Extension call failed: " + std::string(e.what()));
}
@ -340,17 +541,10 @@ Status getExtensions(const std::string& manager_path,
return Status(1, "Extension manager socket not availabe: " + manager_path);
}
// Open a socket to the extension.
_SHARED_PTR<TSocket> socket(new TSocket(manager_path));
_SHARED_PTR<TTransport> transport(new TBufferedTransport(socket));
_SHARED_PTR<TProtocol> protocol(new TBinaryProtocol(transport));
ExtensionManagerClient client(protocol);
InternalExtensionList ext_list;
try {
transport->open();
client.extensions(ext_list);
transport->close();
auto client = EXManagerClient(manager_path);
client.get()->extensions(ext_list);
} catch (const std::exception& e) {
return Status(1, "Extension call failed: " + std::string(e.what()));
}
@ -388,17 +582,10 @@ Status callExtension(const std::string& extension_path,
return Status(1, "Extension socket not availabe: " + extension_path);
}
// Open a socket to the extension manager to register.
_SHARED_PTR<TSocket> socket(new TSocket(extension_path));
_SHARED_PTR<TTransport> transport(new TBufferedTransport(socket));
_SHARED_PTR<TProtocol> protocol(new TBinaryProtocol(transport));
ExtensionClient client(protocol);
ExtensionResponse ext_response;
try {
transport->open();
client.call(ext_response, registry, item, request);
transport->close();
auto client = EXClient(extension_path);
client.get()->call(ext_response, registry, item, request);
}
catch (const std::exception& e) {
return Status(1, "Extension call failed: " + std::string(e.what()));
@ -446,6 +633,11 @@ Status startExtensionManager(const std::string& manager_path) {
}
}
// Start a extension manager watcher, if the manager dies, so should we.
Dispatcher::getInstance().addService(
std::make_shared<ExtensionManagerWatcher>(manager_path,
kWatcherMLatency));
// Start the extension manager thread.
Dispatcher::getInstance().addService(
std::make_shared<ExtensionManagerRunner>(manager_path));

23
osquery/extensions/extensions_tests.cpp
View File

@ -149,18 +149,6 @@ TEST_F(ExtensionsTest, test_extension_runnable) {
EXPECT_TRUE(ping());
}
TEST_F(ExtensionsTest, test_extension_start_failed) {
auto status = startExtensionManager(kTestManagerSocket);
EXPECT_TRUE(status.ok());
// Wait for the extension manager to start.
EXPECT_TRUE(socketExists(kTestManagerSocket));
// Start an extension that does NOT fatal if the extension manager dies.
status = startExtension(kTestManagerSocket, "test", "0.1", "0.0.1");
// This will be false since we are registering duplicate items
EXPECT_FALSE(status.ok());
}
TEST_F(ExtensionsTest, test_extension_start) {
auto status = startExtensionManager(kTestManagerSocket);
EXPECT_TRUE(status.ok());
@ -168,7 +156,7 @@ TEST_F(ExtensionsTest, test_extension_start) {
// Now allow duplicates (for testing, since EM/E are the same).
Registry::allowDuplicates(true);
status = startExtension(kTestManagerSocket, "test", "0.1", "0.0.1");
status = startExtension(kTestManagerSocket, "test", "0.1", "0.0.0", "0.0.1");
// This will be false since we are registering duplicate items
EXPECT_TRUE(status.ok());
@ -192,6 +180,8 @@ TEST_F(ExtensionsTest, test_extension_start) {
}
TEST_F(ExtensionsTest, test_extension_query) {
// Only test calling query when the extension manager is core.
#ifndef OSQUERY_BUILD_SDK
auto status = startExtensionManager(kTestManagerSocket);
EXPECT_TRUE(status.ok());
// Wait for the extension manager to start.
@ -199,7 +189,10 @@ TEST_F(ExtensionsTest, test_extension_query) {
auto qd = query("select seconds from time");
EXPECT_EQ(qd.size(), 1);
EXPECT_EQ(qd[0].count("seconds"), 1);
if (qd.size() > 0) {
EXPECT_EQ(qd[0].count("seconds"), 1);
}
#endif
}
class ExtensionPlugin : public Plugin {
@ -237,7 +230,7 @@ TEST_F(ExtensionsTest, test_extension_broadcast) {
EXPECT_TRUE(Registry::exists("extension_test", "test_item"));
EXPECT_FALSE(Registry::exists("extension_test", "test_alias"));
status = startExtension(kTestManagerSocket, "test", "0.1", "0.0.1");
status = startExtension(kTestManagerSocket, "test", "0.1", "0.0.0", "0.0.1");
EXPECT_TRUE(status.ok());
RouteUUID uuid;

6
osquery/filesystem/darwin/plist_tests.cpp
View File

@ -79,11 +79,9 @@ TEST_F(PlistTests, test_parse_plist_array) {
TEST_F(PlistTests, test_parse_plist_content_with_blobs) {
pt::ptree tree;
fs::path bin_path(argv0);
fs::path test_root(kTestDataPath);
auto s = parsePlist((bin_path.parent_path() /
"../../../../tools/tests/test_binary.plist").string(),
tree);
auto s = parsePlist((test_root / "test_binary.plist").string(), tree);
EXPECT_TRUE(s.ok());
EXPECT_EQ(s.toString(), "OK");
EXPECT_THROW(tree.get<bool>("foobar"), pt::ptree_bad_path);

1
osquery/main/lib.cpp
View File

@ -15,4 +15,5 @@
namespace osquery {
const std::string kVersion = OSQUERY_VERSION;
const std::string kSDKVersion = OSQUERY_SDK_VERSION;
}

18
osquery/registry/registry.cpp
View File

@ -13,6 +13,7 @@
#include <boost/property_tree/json_parser.hpp>
#include <osquery/logger.h>
#include <osquery/registry.h>
namespace osquery {
@ -39,6 +40,12 @@ void RegistryHelperCore::remove(const std::string& item_name) {
RegistryRoutes RegistryHelperCore::getRoutes() const {
RegistryRoutes route_table;
for (const auto& item : items_) {
if (std::find(internal_.begin(), internal_.end(), item.first) !=
internal_.end()) {
// This is an internal plugin, do not include the route.
continue;
}
bool has_alias = false;
for (const auto& alias : aliases_) {
if (alias.second == item.first) {
@ -160,6 +167,9 @@ Status RegistryFactory::addBroadcast(const RouteUUID& uuid,
for (const auto& registry : broadcast) {
for (const auto& item : registry.second) {
if (Registry::exists(registry.first, item.first)) {
VLOG(1) << "Extension " << uuid
<< " has duplicate plugin name: " << item.first
<< " in registry: " << registry.first;
return Status(1, "Duplicate registry item: " + item.first);
}
}
@ -255,6 +265,14 @@ std::vector<std::string> RegistryFactory::names(
return instance().registry(registry_name)->names();
}
std::vector<RouteUUID> RegistryFactory::routeUUIDs() {
std::vector<RouteUUID> uuids;
for (const auto& extension : instance().extensions_) {
uuids.push_back(extension.first);
}
return uuids;
}
size_t RegistryFactory::count() { return instance().registries_.size(); }
size_t RegistryFactory::count(const std::string& registry_name) {

8
osquery/sql/sql_tests.cpp
View File

@ -20,17 +20,25 @@ namespace osquery {
class SQLTests : public testing::Test {};
TEST_F(SQLTests, test_simple_query_execution) {
// Access to the internal SQL implementation is only available in core.
#ifndef OSQUERY_BUILD_SDK
auto sql = SQL("SELECT * FROM time");
EXPECT_TRUE(sql.ok());
EXPECT_EQ(sql.rows().size(), 1);
#endif
}
TEST_F(SQLTests, test_get_tables) {
// Access to the internal SQL implementation is only available in core.
#ifndef OSQUERY_BUILD_SDK
auto tables = SQL::getTableNames();
EXPECT_TRUE(tables.size() > 0);
#endif
}
TEST_F(SQLTests, test_raw_access) {
// Access to the table plugins (no SQL parsing required) works in both
// extensions and core, though with limitations on available tables.
auto results = SQL::selectAllFrom("time");
EXPECT_EQ(results.size(), 1);
}

1
osquery/tables/specs/x/file.table
View File

@ -21,4 +21,5 @@ schema([
Column("is_char", INTEGER, "1 if a character special device else 0"),
Column("is_block", INTEGER, "1 if a block special device else 0"),
])
attributes(utility=True)
implementation("utility/file@genFile")

1
osquery/tables/specs/x/hash.table
View File

@ -7,4 +7,5 @@ schema([
Column("sha1", TEXT, "SHA1 hash of provided filesystem data"),
Column("sha256", TEXT, "SHA256 hash of provided filesystem data"),
])
attributes(utility=True)
implementation("utility/hash@genHash")

1
osquery/tables/specs/x/osquery_extensions.table
View File

@ -7,4 +7,5 @@ schema([
Column("sdk_version", TEXT),
Column("socket", TEXT),
])
attributes(utility=True)
implementation("osquery@genOsqueryExtensions")

1
osquery/tables/specs/x/osquery_flags.table
View File

@ -8,4 +8,5 @@ schema([
Column("value", TEXT),
Column("shell_only", INTEGER),
])
attributes(utility=True)
implementation("osquery@genOsqueryFlags")

1
osquery/tables/specs/x/osquery_info.table
View File

@ -7,4 +7,5 @@ schema([
Column("pid", INTEGER, "Process (or thread) ID"),
Column("extensions", TEXT),
])
attributes(utility=True)
implementation("osquery@genOsqueryInfo")

1
osquery/tables/specs/x/time.table
View File

@ -5,4 +5,5 @@ schema([
Column("minutes", INTEGER),
Column("seconds", INTEGER),
])
attributes(utility=True)
implementation("time@genTime")

4
osquery/tables/templates/default.cpp.in
View File

@ -48,7 +48,11 @@ class {{table_name_cc}}TablePlugin : public TablePlugin {
}
};
{% if attributes.utility %}
REGISTER_INTERNAL({{table_name_cc}}TablePlugin, "table", "{{table_name}}");
{% else %}
REGISTER({{table_name_cc}}TablePlugin, "table", "{{table_name}}");
{% endif %}
/// END[GENTABLE]
}}

1
tools/codegen/gentable.py
View File

@ -150,6 +150,7 @@ class TableState(Singleton):
impl=self.impl,
function=self.function,
class_name=self.class_name,
attributes=self.attributes,
)
# Check for reserved column names