From ed338e8356dc3b4f360b3a6b3562379feaf59f11 Mon Sep 17 00:00:00 2001 From: Teddy Reed Date: Thu, 25 Sep 2014 10:17:32 -0700 Subject: [PATCH] [events] Events lifecycle complete, passwd_changes vtable --- include/osquery/database/results.h | 5 +- include/osquery/events.h | 164 ++++++++++++++--- osquery/database/results.cpp | 28 ++- osquery/events/events.cpp | 171 ++++++++++++++---- osquery/events/events_database_tests.cpp | 8 +- osquery/events/events_tests.cpp | 11 +- osquery/events/linux/inotify.cpp | 5 +- osquery/events/linux/inotify.h | 9 + osquery/events/linux/inotify_tests.cpp | 32 ++-- osquery/main/shell.cpp | 1 - osquery/tables/CMakeLists.txt | 1 + .../tables/events/linux/passwd_changes.cpp | 69 +++++++ .../tables/specs/linux/passwd_changes.table | 8 + tools/gentable.py | 18 ++ 14 files changed, 433 insertions(+), 97 deletions(-) create mode 100644 osquery/tables/events/linux/passwd_changes.cpp create mode 100644 osquery/tables/specs/linux/passwd_changes.table diff --git a/include/osquery/database/results.h b/include/osquery/database/results.h index 85851d88..9cdae3cd 100644 --- a/include/osquery/database/results.h +++ b/include/osquery/database/results.h @@ -47,7 +47,10 @@ osquery::Status serializeRow(const Row& r, boost::property_tree::ptree& tree); * @return an instance of osquery::Status, indicating the success or failure * of the operation */ -osquery::Status serializeRowJSON(const Row& r, std::string json); +osquery::Status serializeRowJSON(const Row& r, std::string& json); + +osquery::Status deserializeRow(const boost::property_tree::ptree& tree, Row& r); +osquery::Status deserializeRowJSON(const std::string& json, Row& r); ///////////////////////////////////////////////////////////////////////////// // QueryData diff --git a/include/osquery/events.h b/include/osquery/events.h index 05be467e..06a01833 100644 --- a/include/osquery/events.h +++ b/include/osquery/events.h @@ -26,9 +26,14 @@ typedef const std::string EventTypeID; typedef const std::string EventID; typedef uint32_t EventContextID; typedef uint32_t EventTime; +typedef std::pair EventRecord; struct MonitorContext {}; -struct EventContext {}; +struct EventContext { + EventContextID id; + EventTime time; + std::string time_string; +}; typedef std::shared_ptr MonitorRef; typedef std::shared_ptr EventTypeRef; @@ -36,8 +41,7 @@ typedef std::shared_ptr MonitorContextRef; typedef std::shared_ptr EventContextRef; typedef std::shared_ptr EventModuleRef; -typedef std::function -EventCallback; +typedef std::function EventCallback; /// An EventType must track every monitor added. typedef std::vector MonitorVector; @@ -69,6 +73,11 @@ extern const std::vector kEventTimeLists; #define DECLARE_EVENTTYPE(TYPE, MONITOR, EVENT) \ public: \ EventTypeID type() const { return #TYPE; } \ + bool shouldFire(const MonitorContextRef mc, const EventContextRef ec) { \ + if (#MONITOR == "MonitorContext" && #EVENT == "EventContext") \ + return true; \ + return shouldFire(getMonitorContext(mc), getEventContext(ec)); \ + } \ static std::shared_ptr getEventContext(EventContextRef context) { \ return std::static_pointer_cast(context); \ } \ @@ -78,6 +87,9 @@ extern const std::vector kEventTimeLists; } \ static std::shared_ptr createEventContext() { \ return std::make_shared(); \ + } \ + static std::shared_ptr createMonitorContext() { \ + return std::make_shared(); \ } /** @@ -97,14 +109,18 @@ extern const std::vector kEventTimeLists; */ #define DECLARE_EVENTMODULE(NAME, TYPE) \ public: \ - static std::shared_ptr get() { \ + static std::shared_ptr getInstance() { \ static auto q = std::shared_ptr(new NAME()); \ return q; \ + } \ + static QueryData genTable() __attribute__((used)) { \ + return getInstance()->get(0, 0); \ } \ \ private: \ EventTypeID name() const { return #NAME; } \ - EventTypeID type() const { return #TYPE; } + EventTypeID type() const { return #TYPE; } \ + NAME() {} /** * @brief Required callin EventModule method declaration helper. @@ -138,17 +154,53 @@ extern const std::vector kEventTimeLists; * instance boilerplate code is added automatically. * Note: The macro will append `Module` to `MyCallback`. */ -#define FUNC_NAME(__NAME__) __NAME__ -#define DECLARE_CALLBACK(__NAME__, EVENT) \ - public: \ - static Status __NAME__(EventContextID ec_id, \ - EventTime time, \ - const EventContextRef ec, \ - bool reserved) { \ - auto ec_ = std::static_pointer_cast(ec); \ - return get()->Module##__NAME__(ec_id, time, ec_); \ +#define DECLARE_CALLBACK(NAME, EVENT) \ + public: \ + static Status Event##NAME(const EventContextRef ec, bool reserved) { \ + auto ec_ = std::static_pointer_cast(ec); \ + return getInstance()->NAME(ec_); \ + } \ + \ + private: \ + void BindTo##NAME(const MonitorContextRef mc) { \ + EventFactory::addMonitor(type(), mc, Event##NAME); \ } +/** + * @brief Bind a monitor context to a declared EventCallback for this module. + * + * Binding refers to the association of a callback for this EventModule to + * a configured MonitorContext. Under the hood "binding" creates a factory + * Monitor for the EventType used by the EventModule. Such that when an event + * of the EventType is fired, if the event details match the specifics of the + * MonitorContext the EventMonitor%'s EventCallback will be called. + * + * @code{.cpp} + * #include "osquery/events.h" + * + * class MyEventModule: public EventModule { + * DECLARE_EVENTMODULE(MyEventModule, MyEventType); + * DECLARE_CALLBACK(MyCallback, MyEventContext); + * + * public: + * void init() { + * auto mc = MyEventType::createMonitorContext(); + * mc->requirement = "SOME_SPECIFIC_DETAIL"; + * BIND_CALLBACK(MyCallback, mc); + * } + * Status MyCallback(const MyEventContextRef ec) {} + * } + * @endcode + * + * The symbol `MyCallback` must match in `DECLARE_CALLBACK`, `BIND_CALLBACK` and + * as a member of this EventModule. + * + * @param NAME The symbol for the EventCallback method used in DECLARE_CALLBACK. + * @param MC The MonitorContext to bind. + */ +#define BIND_CALLBACK(NAME, MC) \ + EventFactory::addMonitor(type(), MC, Event##NAME); + /** * @brief A Monitor is used to configure an EventType and bind a callback. * @@ -260,6 +312,14 @@ class EventType { /// Return a string identifier associated with this EventType. virtual EventTypeID type() const = 0; + template + static EventTypeID type() { + const auto& event_type = new T(); + auto type_id = event_type->type(); + delete event_type; + return type_id; + } + protected: /** * @brief The generic check loop to call MonitorContext callback methods. @@ -268,10 +328,11 @@ class EventType { * the Monitor%s and using `shouldFire` is more appropraite. * * @param ec The EventContext created and fired by the EventType. - * @param event_time The most accurate time associated with the event. + * @param time The most accurate time associated with the event. */ - void fire(const EventContextRef ec, EventTime event_time = 0); + void fire(const EventContextRef ec, EventTime time = 0); + protected: /** * @brief The generic `fire` will call `shouldFire` for each Monitor. * @@ -314,16 +375,50 @@ class EventModule { /// Called after EventType `setUp`. Add all Monitor%s here. virtual void init() {} + /// Suggested entrypoint for table generation. + static QueryData genTable(); + protected: /// Store an event for table-access into the underlying backing store. - virtual Status add(const osquery::Row& r, int event_time) final; + virtual Status add(const osquery::Row& r, EventTime time) final; + + /** + * @brief Return all events added by this EventModule within start, stop. + * + * This is used internally (for the most part) by EventModule::genTable. + * + * @param start Inclusive lower bound time limit. + * @param stop Inclusive upper bound time limit. + * @return Set of event rows matching time limits. + */ + virtual QueryData get(EventTime start, EventTime stop); + + /* + * @brief When `get`ting event results, return EventID%s from time indexes. + * + * Used by EventModule::get to retrieve EventID, EventTime indexes. This + * applies the lookup-efficiency checks for time list appropriate bins. + * If the time range in 24 hours and there is a 24-hour list bin it will + * be queried using a single backing store `Get` followed by two `Get`s of + * the most-specific boundary lists. + * + * @return List of EventID, EventTime%s + */ + std::vector getRecords(EventTime start, EventTime stop); private: /// Returns a new EventID for this module, increments to the current EID. EventID getEventID(); - /// Records an added EventID, which contains row data, and a time for lookups. - Status recordEvent(EventID eid, int event_time); + /* + * @brief Add an EventID, EventTime pair to all matching list types. + * + * The list types are defined by time size. Based on the EventTime this pair + * is added to the list bin for each list type. If there are two list types: + * 60 seconds and 3600 seconds and `time` is 92, this pair will be added to + * list type 1 bin 4 and list type 2 bin 1. + */ + Status recordEvent(EventID eid, EventTime time); protected: /** @@ -336,6 +431,7 @@ class EventModule { EventModule() {} /// Database namespace definition methods. + EventTypeID dbNamespace() { return type() + "." + name(); } virtual EventTypeID type() const = 0; virtual EventTypeID name() const = 0; @@ -364,7 +460,7 @@ class EventModule { class EventFactory { public: /// Access to the EventFactory instance. - static std::shared_ptr get(); + static std::shared_ptr getInstance(); /** * @brief Add an EventType to the factory. @@ -397,7 +493,7 @@ class EventFactory { */ template static Status registerEventModule() { - auto event_module = T::get(); + auto event_module = T::getInstance(); return EventFactory::registerEventModule(event_module); } @@ -430,17 +526,29 @@ class EventFactory { */ static Status addMonitor(EventTypeID type_id, const MonitorContextRef mc, - EventCallback callback = 0); + EventCallback cb = 0); /// Add a Monitor using a caller Monitor instance. static Status addMonitor(EventTypeID type_id, const MonitorRef monitor); + /// Add a Monitor by templating the EventType, using a MonitorContext. + template + static Status addMonitor(const MonitorContextRef mc, EventCallback cb = 0) { + return addMonitor(EventType::type(), mc, cb); + } + + /// Add a Monitor by templating the EventType, using a Monitor instance. + template + static Status addMonitor(const MonitorRef monitor) { + return addMonitor(EventType::type(), monitor); + } + /// Get the total number of Monitor%s across ALL EventType%s. static size_t numMonitors(EventTypeID type_id); /// Get the number of EventTypes. static size_t numEventTypes() { - return EventFactory::get()->event_types_.size(); + return EventFactory::getInstance()->event_types_.size(); } /** @@ -456,8 +564,10 @@ class EventFactory { */ static Status deregisterEventType(const EventTypeRef event_type); + /// Deregister an EventType by EventTypeID. static Status deregisterEventType(EventTypeID type_id); + /// Deregister all EventType%s. static Status deregisterEventTypes(); /// Return an instance to a registered EventType. @@ -500,8 +610,8 @@ class EventFactory { /// Expose a Plugin-like Registry for EventType instances. DECLARE_REGISTRY(EventTypes, std::string, EventTypeRef); #define REGISTERED_EVENTTYPES REGISTRY(EventTypes) -#define REGISTER_EVENTTYPE(name, decorator) \ - REGISTER(EventTypes, name, std::make_shared()); +#define REGISTER_EVENTTYPE(decorator) \ + REGISTER(EventTypes, #decorator, std::make_shared()); /** * @brief Expose a Plugin-link Registry for EventModule instances. @@ -511,8 +621,8 @@ DECLARE_REGISTRY(EventTypes, std::string, EventTypeRef); */ DECLARE_REGISTRY(EventModules, std::string, EventModuleRef); #define REGISTERED_EVENTMODULES REGISTRY(EventModules) -#define REGISTER_EVENTMODULE(name, decorator) \ - REGISTER(EventModules, name, std::make_shared()); +#define REGISTER_EVENTMODULE(decorator) \ + REGISTER(EventModules, #decorator, decorator::getInstance()); namespace osquery { namespace registries { diff --git a/osquery/database/results.cpp b/osquery/database/results.cpp index 0f64f5ee..3c3db3fd 100644 --- a/osquery/database/results.cpp +++ b/osquery/database/results.cpp @@ -36,7 +36,7 @@ Status serializeRow(const Row& r, pt::ptree& tree) { return Status(0, "OK"); } -Status serializeRowJSON(const Row& r, std::string json) { +Status serializeRowJSON(const Row& r, std::string& json) { pt::ptree tree; try { auto status = serializeRow(r, tree); @@ -52,6 +52,32 @@ Status serializeRowJSON(const Row& r, std::string json) { return Status(0, "OK"); } +Status deserializeRow(const pt::ptree& tree, Row& r) { + try { + for (auto& i : tree) { + if (i.first.length() > 0) { + r[i.first] = i.second.data(); + } + } + return Status(0, "OK"); + } catch (const std::exception& e) { + LOG(ERROR) << e.what(); + return Status(1, e.what()); + } +} + +Status deserializeRowJSON(const std::string& json, Row& r) { + pt::ptree tree; + try { + std::stringstream j; + j << json; + pt::read_json(j, tree); + } catch (const std::exception& e) { + return Status(1, e.what()); + } + return deserializeRow(tree, r); +} + ///////////////////////////////////////////////////////////////////////////// // QueryData - the representation of a database query result set. It's a // vector of rows diff --git a/osquery/events/events.cpp b/osquery/events/events.cpp index b0e1146f..706c9d21 100644 --- a/osquery/events/events.cpp +++ b/osquery/events/events.cpp @@ -1,9 +1,12 @@ // Copyright 2004-present Facebook. All Rights Reserved. +#include +#include #include #include +#include "osquery/core.h" #include "osquery/core/conversions.h" #include "osquery/events.h" #include "osquery/dispatcher.h" @@ -15,7 +18,7 @@ const std::vector kEventTimeLists = {1 * 60, // 1 minute 12 * 60 * 60, // half-day }; -void EventType::fire(const EventContextRef ec, EventTime event_time) { +void EventType::fire(const EventContextRef ec, EventTime time) { EventContextID ec_id; { @@ -23,10 +26,25 @@ void EventType::fire(const EventContextRef ec, EventTime event_time) { ec_id = next_ec_id_++; } + // Fill in EventContext ID and time if needed. + if (ec != nullptr) { + ec->id = ec_id; + if (ec->time == 0) { + if (time == 0) { + time = getUnixTime(); + } + ec->time = time; + } + + // Set the optional string-verion of the time for DB columns. + ec->time_string = boost::lexical_cast(ec->time); + } + for (const auto& monitor : monitors_) { auto callback = monitor->callback; if (shouldFire(monitor->context, ec) && callback != nullptr) { - callback(ec_id, event_time, ec, false); + callback(ec, false); + } else { } } } @@ -41,41 +59,101 @@ Status EventType::run() { return Status(1, "No runloop required"); } -Status EventModule::recordEvent(EventID eid, int event_time) { +std::vector EventModule::getRecords(EventTime start, + EventTime stop) { + Status status; + std::vector records; + auto db = DBHandle::getInstance(); + + std::string index_key = "indexes." + dbNamespace(); + std::string record_key = "records." + dbNamespace(); + + // For now, cheat and use the first list type. + std::string list_key = boost::lexical_cast(kEventTimeLists[0]); + std::string index_value; + + // Get all bins for this list type. + status = db->Get(kEvents, index_key + "." + list_key, index_value); + if (index_value.length() == 0) { + // There are no events in this time range. + return records; + } + // Tokenize the value into our bins of the list type. + std::vector lists; + boost::split(lists, index_value, boost::is_any_of(",")); + std::string record_value; + for (const auto& list_id : lists) { + status = db->Get( + kEvents, record_key + "." + list_key + "." + list_id, record_value); + if (record_value.length() == 0) { + // There are actually no events in this bin, interesting error case. + continue; + } + std::vector bin_records; + boost::split(bin_records, record_value, boost::is_any_of(",:")); + auto bin_it = bin_records.begin(); + for (; bin_it != bin_records.end(); bin_it++) { + std::string eid = *bin_it; + EventTime time = boost::lexical_cast(*(++bin_it)); + records.push_back(std::make_pair(eid, time)); + } + } + + // Now all the event_ids/event_times within the binned range exist. + // Select further on the EXACT time range. + + return records; +} + +Status EventModule::recordEvent(EventID eid, EventTime time) { Status status; auto db = DBHandle::getInstance(); - std::string time_value = boost::lexical_cast(event_time); + std::string time_value = boost::lexical_cast(time); // The record is identified by the event type then module name. - std::string record_key = "records." + type() + "." + name(); + std::string index_key = "indexes." + dbNamespace(); + std::string record_key = "records." + dbNamespace(); // The list key includes the list type (bin size) and the list ID (bin). std::string list_key; std::string list_id; - // This is an append operation, the record value is tokenized with this event. - std::string record_value; for (auto time_list : kEventTimeLists) { // The list_id is the MOST-Specific key ID, the bin for this list. // If the event time was 13 and the time_list is 5 seconds, lid = 2. - list_id = boost::lexical_cast(event_time % time_list); + list_id = boost::lexical_cast(time / time_list); // The list name identifies the 'type' of list. list_key = boost::lexical_cast(time_list); - list_key = record_key + "." + list_key + "." + list_id; + // list_key = list_key + "." + list_id; { boost::lock_guard lock(event_record_lock_); // Append the record (eid, unix_time) to the list bin. - status = db->Get(kEvents, list_key, record_value); + std::string record_value; + status = db->Get( + kEvents, record_key + "." + list_key + "." + list_id, record_value); if (record_value.length() == 0) { + // This is a new list_id for list_key, append the ID to the indirect + // lookup for this list_key. + std::string index_value; + status = db->Get(kEvents, index_key + "." + list_key, index_value); + if (index_value.length() == 0) { + // A new index. + index_value = list_id; + } else { + index_value += "," + list_id; + } + status = db->Put(kEvents, index_key + "." + list_key, index_value); record_value = eid + ":" + time_value; } else { // Tokenize a record using ',' and the EID/time using ':'. record_value += "," + eid + ":" + time_value; } - status = db->Put(kEvents, list_key, record_value); + status = db->Put( + kEvents, record_key + "." + list_key + "." + list_id, record_value); if (!status.ok()) { - LOG(ERROR) << "Could not put Event Record key: " << list_key; + LOG(ERROR) << "Could not put Event Record key: " << record_key << "." + << list_key << "." << list_id; } } } @@ -87,7 +165,7 @@ EventID EventModule::getEventID() { Status status; auto db = DBHandle::getInstance(); // First get an event ID from the meta key. - std::string eid_key = "eid." + type() + "." + name(); + std::string eid_key = "eid." + dbNamespace(); std::string last_eid_value; std::string eid_value; @@ -110,32 +188,58 @@ EventID EventModule::getEventID() { return eid_value; } -Status EventModule::add(const Row& r, int event_time) { +QueryData EventModule::get(EventTime start, EventTime stop) { + QueryData results; + Status status; + auto db = DBHandle::getInstance(); + + // Get the records for this time range. + auto records = getRecords(start, stop); + + std::string events_key = "data." + dbNamespace(); + + // Select records using event_ids as keys. + std::string data_value; + for (const auto& record : records) { + Row r; + status = db->Get(kEvents, events_key + "." + record.first, data_value); + if (data_value.length() == 0) { + // THere is no record here, interesting error case. + continue; + } + status = deserializeRowJSON(data_value, r); + if (status.ok()) { + results.push_back(r); + } + } + return results; +} + +Status EventModule::add(const Row& r, EventTime time) { Status status; auto db = DBHandle::getInstance(); // Get and increment the EID for this module. EventID eid = getEventID(); - std::string event_key = "data." + type() + "." + name() + "." + eid; + std::string event_key = "data." + dbNamespace() + "." + eid; std::string data; status = serializeRowJSON(r, data); if (!status.ok()) { - printf("could not serialize json\n"); return status; } // Store the event data. status = db->Put(kEvents, event_key, data); // Record the event in the indexing bins. - recordEvent(eid, event_time); + recordEvent(eid, time); return status; } void EventFactory::delay() { - auto ef = EventFactory::get(); - for (const auto& eventtype : EventFactory::get()->event_types_) { + auto ef = EventFactory::getInstance(); + for (const auto& eventtype : EventFactory::getInstance()->event_types_) { auto thread_ = std::make_shared( boost::bind(&EventFactory::run, eventtype.first)); ef->threads_.push_back(thread_); @@ -148,13 +252,13 @@ Status EventFactory::run(EventTypeID type_id) { // Assume it can either make use of an entrypoint poller/selector or // take care of async callback registrations in setUp/configure/run // only once and handle event queueing/firing in callbacks. - auto event_type = EventFactory::get()->getEventType(type_id); + auto event_type = EventFactory::getInstance()->getEventType(type_id); if (event_type == nullptr) { return Status(1, "No Event Type"); } Status status = Status(0, "OK"); - while (!EventFactory::get()->ending_ && status.ok()) { + while (!EventFactory::getInstance()->ending_ && status.ok()) { // Can optionally implement a global cooloff latency here. status = event_type->run(); } @@ -164,19 +268,20 @@ Status EventFactory::run(EventTypeID type_id) { } void EventFactory::end(bool should_end) { - EventFactory::get()->ending_ = should_end; + EventFactory::getInstance()->ending_ = should_end; // Join on the thread group. + ::usleep(400); } // There's no reason for the event factory to keep multiple instances. -std::shared_ptr EventFactory::get() { +std::shared_ptr EventFactory::getInstance() { static auto q = std::shared_ptr(new EventFactory()); return q; } Status EventFactory::registerEventType(const EventTypeRef event_type) { EventTypeID type_id = event_type->type(); - auto ef = EventFactory::get(); + auto ef = EventFactory::getInstance(); if (ef->getEventType(type_id) != nullptr) { // This is a duplicate type id? @@ -189,7 +294,7 @@ Status EventFactory::registerEventType(const EventTypeRef event_type) { } Status EventFactory::registerEventModule(const EventModuleRef event_module) { - auto ef = EventFactory::get(); + auto ef = EventFactory::getInstance(); // Let the module initialize any Monitors. event_module->init(); ef->event_modules_.push_back(event_module); @@ -197,7 +302,7 @@ Status EventFactory::registerEventModule(const EventModuleRef event_module) { } Status EventFactory::addMonitor(EventTypeID type_id, const MonitorRef monitor) { - auto event_type = EventFactory::get()->getEventType(type_id); + auto event_type = EventFactory::getInstance()->getEventType(type_id); if (event_type == nullptr) { // Cannot create a Monitor for a missing type_id. return Status(1, "No Event Type"); @@ -211,13 +316,13 @@ Status EventFactory::addMonitor(EventTypeID type_id, const MonitorRef monitor) { Status EventFactory::addMonitor(EventTypeID type_id, const MonitorContextRef mc, - EventCallback callback) { - auto monitor = Monitor::create(mc, callback); + EventCallback cb) { + auto monitor = Monitor::create(mc, cb); return EventFactory::addMonitor(type_id, monitor); } size_t EventFactory::numMonitors(EventTypeID type_id) { - const auto& event_type = EventFactory::get()->getEventType(type_id); + const auto& event_type = EventFactory::getInstance()->getEventType(type_id); if (event_type != nullptr) { return event_type->numMonitors(); } @@ -225,7 +330,7 @@ size_t EventFactory::numMonitors(EventTypeID type_id) { } std::shared_ptr EventFactory::getEventType(EventTypeID type_id) { - const auto& ef = EventFactory::get(); + const auto& ef = EventFactory::getInstance(); const auto& it = ef->event_types_.find(type_id); if (it != ef->event_types_.end()) { return ef->event_types_[type_id]; @@ -238,7 +343,7 @@ Status EventFactory::deregisterEventType(const EventTypeRef event_type) { } Status EventFactory::deregisterEventType(EventTypeID type_id) { - auto ef = EventFactory::get(); + auto ef = EventFactory::getInstance(); const auto& it = ef->event_types_.find(type_id); if (it == ef->event_types_.end()) { return Status(1, "No Event Type registered"); @@ -250,7 +355,7 @@ Status EventFactory::deregisterEventType(EventTypeID type_id) { } Status EventFactory::deregisterEventTypes() { - auto ef = EventFactory::get(); + auto ef = EventFactory::getInstance(); auto it = ef->event_types_.begin(); for (; it != ef->event_types_.end(); it++) { it->second->tearDown(); @@ -264,7 +369,7 @@ Status EventFactory::deregisterEventTypes() { namespace osquery { namespace registries { void faucet(EventTypes ets, EventModules ems) { - auto ef = osquery::EventFactory::get(); + auto ef = osquery::EventFactory::getInstance(); for (const auto& event_type : ets) { ef->registerEventType(event_type.second); } diff --git a/osquery/events/events_database_tests.cpp b/osquery/events/events_database_tests.cpp index cd64bda6..d475d5dc 100644 --- a/osquery/events/events_database_tests.cpp +++ b/osquery/events/events_database_tests.cpp @@ -40,7 +40,7 @@ class AnotherFakeEventModule : public EventModule { }; TEST_F(EventsDatabaseTests, test_event_module_id) { - auto fake_event_module = FakeEventModule::get(); + auto fake_event_module = FakeEventModule::getInstance(); // Not normally available outside of EventModule->Add(). auto event_id1 = fake_event_module->getEventID(); EXPECT_EQ(event_id1, "1"); @@ -49,8 +49,8 @@ TEST_F(EventsDatabaseTests, test_event_module_id) { } TEST_F(EventsDatabaseTests, test_unique_event_module_id) { - auto fake_event_module = FakeEventModule::get(); - auto another_fake_event_module = AnotherFakeEventModule::get(); + auto fake_event_module = FakeEventModule::getInstance(); + auto another_fake_event_module = AnotherFakeEventModule::getInstance(); // Not normally available outside of EventModule->Add(). auto event_id1 = fake_event_module->getEventID(); EXPECT_EQ(event_id1, "3"); @@ -63,7 +63,7 @@ TEST_F(EventsDatabaseTests, test_event_add) { r["testing"] = std::string("hello from space"); size_t event_time = 10; - auto fake_event_module = FakeEventModule::get(); + auto fake_event_module = FakeEventModule::getInstance(); auto status = fake_event_module->testAdd(1); EXPECT_TRUE(status.ok()); } diff --git a/osquery/events/events_tests.cpp b/osquery/events/events_tests.cpp index 8138e479..7396148b 100644 --- a/osquery/events/events_tests.cpp +++ b/osquery/events/events_tests.cpp @@ -8,7 +8,7 @@ namespace osquery { class EventsTests : public testing::Test { protected: - virtual void SetUp() { ef = EventFactory::get(); } + virtual void SetUp() { ef = EventFactory::getInstance(); } virtual void TearDown() { ef->deregisterEventTypes(); } @@ -16,8 +16,8 @@ class EventsTests : public testing::Test { }; TEST_F(EventsTests, test_singleton) { - auto one = EventFactory::get(); - auto two = EventFactory::get(); + auto one = EventFactory::getInstance(); + auto two = EventFactory::getInstance(); EXPECT_EQ(one, two); } @@ -196,10 +196,7 @@ TEST_F(EventsTests, test_tear_down) { static int kBellHathTolled = 0; -Status TestTheeCallback(EventContextID ec_id, - EventTime time, - EventContextRef context, - bool reserved) { +Status TestTheeCallback(EventContextRef context, bool reserved) { kBellHathTolled += 1; return Status(0, "OK"); } diff --git a/osquery/events/linux/inotify.cpp b/osquery/events/linux/inotify.cpp index 2d617ee0..79c0ddd2 100644 --- a/osquery/events/linux/inotify.cpp +++ b/osquery/events/linux/inotify.cpp @@ -11,7 +11,7 @@ namespace osquery { -REGISTER_EVENTTYPE("INotifyEventType", INotifyEventType); +REGISTER_EVENTTYPE(INotifyEventType); int kINotifyULatency = 200; static const uint32_t BUFFER_SIZE = @@ -63,9 +63,7 @@ Status INotifyEventType::run() { // Read timeout. return Status(0, "Continue"); } - ssize_t record_num = ::read(getHandle(), buffer, BUFFER_SIZE); - LOG(INFO) << "INotify read " << record_num << " event records"; if (record_num == 0 || record_num == -1) { return Status(1, "INotify read failed"); } @@ -144,7 +142,6 @@ Status INotifyEventType::addMonitor(const MonitorRef monitor) { LOG(ERROR) << "Could not add inotify watch on: " << mc->path; return Status(1, "Add Watch Failed"); } - descriptors_.push_back(watch); path_descriptors_[mc->path] = watch; descriptor_paths_[watch] = mc->path; diff --git a/osquery/events/linux/inotify.h b/osquery/events/linux/inotify.h index 589e933c..073883db 100644 --- a/osquery/events/linux/inotify.h +++ b/osquery/events/linux/inotify.h @@ -16,6 +16,8 @@ namespace osquery { +extern std::map kMaskActions; + struct INotifyMonitorContext : public MonitorContext { /// Monitor the following filesystem path. std::string path; @@ -25,6 +27,13 @@ struct INotifyMonitorContext : public MonitorContext { bool recursive; INotifyMonitorContext() : mask(0), recursive(false) {} + void requireAction(std::string action) { + for (const auto& bit : kMaskActions) { + if (action == bit.second) { + mask = mask | bit.first; + } + } + } }; struct INotifyEventContext : public EventContext { diff --git a/osquery/events/linux/inotify_tests.cpp b/osquery/events/linux/inotify_tests.cpp index 75cbc69f..d03e67c4 100644 --- a/osquery/events/linux/inotify_tests.cpp +++ b/osquery/events/linux/inotify_tests.cpp @@ -15,7 +15,7 @@ const std::string kRealTestPath = "/tmp/osquery-inotify-trigger"; class INotifyTests : public testing::Test { protected: - virtual void SetUp() { ef = EventFactory::get(); } + virtual void SetUp() { ef = EventFactory::getInstance(); } virtual void TearDown() { EventFactory::deregisterEventTypes(); } @@ -147,16 +147,13 @@ class TestINotifyEventModule : public EventModule { DECLARE_CALLBACK(Callback, INotifyEventContext); public: - Status ModuleSimpleCallback(EventContextID ec_id, - EventTime time, - const INotifyEventContextRef ec) { + void init() { callback_count_ = 0; } + Status SimpleCallback(const INotifyEventContextRef ec) { callback_count_ += 1; return Status(0, "OK"); } - Status ModuleCallback(EventContextID ec_id, - EventTime time, - const INotifyEventContextRef ec) { + Status Callback(const INotifyEventContextRef ec) { Row r; r["action"] = ec->action; r["path"] = ec->path; @@ -166,9 +163,6 @@ class TestINotifyEventModule : public EventModule { return Status(0, "OK"); } - private: - TestINotifyEventModule() : callback_count_(0) {} - public: int callback_count_; std::vector actions_; @@ -178,8 +172,8 @@ TEST_F(INotifyTests, test_inotify_fire_event) { // Assume event type is registered. StartEventLoop(); - // Create a monitoring context (with callback) - MonitorAction(0, TestINotifyEventModule::SimpleCallback); + // Create a monitoring context, note the added Event to the symbol + MonitorAction(0, TestINotifyEventModule::EventSimpleCallback); FILE* fd = fopen(kRealTestPath.c_str(), "w"); fputs("inotify", fd); @@ -187,7 +181,7 @@ TEST_F(INotifyTests, test_inotify_fire_event) { waitForEvent(2000); // Make sure our expected event fired (aka monitor callback was called). - EXPECT_TRUE(TestINotifyEventModule::get()->callback_count_ > 0); + EXPECT_TRUE(TestINotifyEventModule::getInstance()->callback_count_ > 0); // Cause the thread to tear down. EndEventLoop(); @@ -196,7 +190,7 @@ TEST_F(INotifyTests, test_inotify_fire_event) { TEST_F(INotifyTests, test_inotify_event_action) { // Assume event type is registered. StartEventLoop(); - MonitorAction(0, TestINotifyEventModule::Callback); + MonitorAction(0, TestINotifyEventModule::EventCallback); FILE* fd = fopen(kRealTestPath.c_str(), "w"); fputs("inotify", fd); @@ -204,11 +198,11 @@ TEST_F(INotifyTests, test_inotify_event_action) { waitForEvent(2000, 4); // Make sure the inotify action was expected. - EXPECT_EQ(TestINotifyEventModule::get()->actions_.size(), 4); - EXPECT_EQ(TestINotifyEventModule::get()->actions_[0], "UPDATED"); - EXPECT_EQ(TestINotifyEventModule::get()->actions_[1], "OPENED"); - EXPECT_EQ(TestINotifyEventModule::get()->actions_[2], "UPDATED"); - EXPECT_EQ(TestINotifyEventModule::get()->actions_[3], "UPDATED"); + EXPECT_EQ(TestINotifyEventModule::getInstance()->actions_.size(), 4); + EXPECT_EQ(TestINotifyEventModule::getInstance()->actions_[0], "UPDATED"); + EXPECT_EQ(TestINotifyEventModule::getInstance()->actions_[1], "OPENED"); + EXPECT_EQ(TestINotifyEventModule::getInstance()->actions_[2], "UPDATED"); + EXPECT_EQ(TestINotifyEventModule::getInstance()->actions_[3], "UPDATED"); // Cause the thread to tear down. EndEventLoop(); diff --git a/osquery/main/shell.cpp b/osquery/main/shell.cpp index 1ce41e87..1be14428 100644 --- a/osquery/main/shell.cpp +++ b/osquery/main/shell.cpp @@ -15,6 +15,5 @@ int main(int argc, char *argv[]) { // End any event type threads. osquery::EventFactory::end(); - return retcode; } diff --git a/osquery/tables/CMakeLists.txt b/osquery/tables/CMakeLists.txt index e450a756..1be2dd73 100644 --- a/osquery/tables/CMakeLists.txt +++ b/osquery/tables/CMakeLists.txt @@ -27,6 +27,7 @@ if(APPLE) ADD_OSQUERY_LINK("-framework Security") else() ADD_OSQUERY_LIBRARY(osquery_tables_linux + events/linux/passwd_changes.cpp networking/linux/routes.cpp system/linux/kernel_modules.cpp system/linux/processes.cpp diff --git a/osquery/tables/events/linux/passwd_changes.cpp b/osquery/tables/events/linux/passwd_changes.cpp new file mode 100644 index 00000000..9d7a02d8 --- /dev/null +++ b/osquery/tables/events/linux/passwd_changes.cpp @@ -0,0 +1,69 @@ +// Copyright 2004-present Facebook. All Rights Reserved. + +#include +#include + +#include + +#include + +#include "osquery/core.h" +#include "osquery/database.h" +#include "osquery/events/linux/inotify.h" + +namespace osquery { +namespace tables { + +/** + * @brief Track time, action changes to /etc/passwd + * + * This is mostly an example EventModule implementation. + */ +class PasswdChangesEventModule : public EventModule { + DECLARE_EVENTMODULE(PasswdChangesEventModule, INotifyEventType); + DECLARE_CALLBACK(Callback, INotifyEventContext); + + public: + void init(); + + /** + * @brief This exports a single Callback for INotifyEventType events. + * + * @param ec The EventCallback type receives an EventContextRef substruct + * for the INotifyEventType declared in this EventModule subclass. + * + * @return Was the callback successfull. + */ + Status Callback(const INotifyEventContextRef ec); +}; + +/** + * @brief Each EventModule must register itself so the init method is called. + * + * This registers PasswdChangesEventModule into the osquery EventModule + * pseudo-plugin registry. + */ +REGISTER_EVENTMODULE(PasswdChangesEventModule); + +void PasswdChangesEventModule::init() { + auto mc = INotifyEventType::createMonitorContext(); + mc->path = "/etc/passwd"; + mc->mask = IN_ATTRIB | IN_MODIFY | IN_DELETE | IN_CREATE; + BIND_CALLBACK(Callback, mc); +} + +Status PasswdChangesEventModule::Callback(const INotifyEventContextRef ec) { + Row r; + r["action"] = ec->action; + r["time"] = ec->time_string; + r["target_path"] = ec->path; + r["transaction_id"] = boost::lexical_cast(ec->event->cookie); + if (ec->action != "" && ec->action != "OPENED") { + // A callback is somewhat useless unless it changes the EventModule state + // or calls `add` to store a marked up event. + add(r, ec->time); + } + return Status(0, "OK"); +} +} +} diff --git a/osquery/tables/specs/linux/passwd_changes.table b/osquery/tables/specs/linux/passwd_changes.table new file mode 100644 index 00000000..7ce13b2f --- /dev/null +++ b/osquery/tables/specs/linux/passwd_changes.table @@ -0,0 +1,8 @@ +table_name("passwd_changes") +schema([ + Column(name="target_path", type="std::string"), + Column(name="time", type="std::string"), + Column(name="action", type="std::string"), + Column(name="transaction_id", type="std::string"), +]) +implementation("events/linux/passwd_changes@PasswdChangesEventModule::genTable") diff --git a/tools/gentable.py b/tools/gentable.py index b84a32bc..ea3b3a58 100755 --- a/tools/gentable.py +++ b/tools/gentable.py @@ -38,7 +38,14 @@ IMPL_TEMPLATE = """// Copyright 2004-present Facebook. All Rights Reserved. namespace osquery { namespace tables { +{% if class_name == "" %} osquery::QueryData {{function}}(); +{% else %} +class {{class_name}} { + public: + static osquery::QueryData {{function}}(); +}; +{% endif %} struct sqlite3_{{table_name}} { int n; @@ -126,7 +133,11 @@ int {{table_name_cc}}Filter( pVtab->pContent->{{col.name}}.clear(); {% endfor %}\ +{% if class_name != "" %} + for (auto& row : osquery::tables::{{class_name}}::{{function}}()) { +{% else %} for (auto& row : osquery::tables::{{function}}()) { +{% endif %} {% for col in schema %}\ {% if col.type == "std::string" %}\ pVtab->pContent->{{col.name}}.push_back(row["{{col.name}}"]); @@ -227,6 +238,7 @@ class TableState(Singleton): self.header = "" self.impl = "" self.function = "" + self.class_name = "" def generate(self, path): """Generate the virtual table files""" @@ -238,6 +250,7 @@ class TableState(Singleton): header=self.header, impl=self.impl, function=self.function, + class_name=self.class_name ) path_bits = path.split("/") @@ -289,11 +302,16 @@ def implementation(impl_string): """ logging.debug("- implementation") path, function = impl_string.split("@") + class_parts = function.split("::")[::-1] + function = class_parts[0] + class_name = class_parts[1] if len(class_parts) > 1 else "" impl = "%s.cpp" % path logging.debug(" - impl => %s" % impl) logging.debug(" - function => %s" % function) + logging.debug(" - class_name => %s" % class_name) table.impl = impl table.function = function + table.class_name = class_name def main(argc, argv): if DEVELOPING: