diff --git a/include/osquery/tables.h b/include/osquery/tables.h index 3be44d2f..fe741ee4 100644 --- a/include/osquery/tables.h +++ b/include/osquery/tables.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -391,6 +392,29 @@ struct QueryContext : private boost::noncopyable { typedef struct QueryContext QueryContext; typedef struct Constraint Constraint; +/** + * @brief osquery table content descriptor. + * + * This object is the abstracted SQLite database's virtual table descriptor. + * When the virtual table is created/connected the name and columns are + * retrieved via the TablePlugin call API. The details are kept in this context + * so column parsing and row walking does not require additional Registry calls. + * + * When tables are accessed as the result of an SQL statement a QueryContext is + * created to represent metadata that can be used by the virtual table + * implementation code. Thus the code that generates rows can choose to emit + * additional data, restrict based on constraints, or potentially yield from + * a cache or choose not to generate certain columns. + */ +struct VirtualTableContent { + /// Friendly name for the table. + TableName name; + /// Table column structure, retrieved once via the TablePlugin call API. + TableColumns columns; + /// Transient set of virtual table access constraints. + std::unordered_map constraints; +}; + /** * @brief The TablePlugin defines the name, types, and column information. * diff --git a/osquery/devtools/shell.cpp b/osquery/devtools/shell.cpp index f63cad7d..00cce1cd 100644 --- a/osquery/devtools/shell.cpp +++ b/osquery/devtools/shell.cpp @@ -1060,6 +1060,7 @@ static int shell_exec( } } } /* end while */ + dbc->clearAffectedTables(); if (pArg && pArg->mode == MODE_Pretty) { if (osquery::FLAGS_json) { @@ -1147,16 +1148,16 @@ static sqlite3_int64 integerValue(const char *zArg) { char *zSuffix; int iMult; } aMult[] = { - {(char *)"KiB", 1024}, - {(char *)"MiB", 1024 * 1024}, - {(char *)"GiB", 1024 * 1024 * 1024}, - {(char *)"KB", 1000}, - {(char *)"MB", 1000000}, - {(char *)"GB", 1000000000}, - {(char *)"K", 1000}, - {(char *)"M", 1000000}, - {(char *)"G", 1000000000}, - }; + {(char *)"KiB", 1024}, + {(char *)"MiB", 1024 * 1024}, + {(char *)"GiB", 1024 * 1024 * 1024}, + {(char *)"KB", 1000}, + {(char *)"MB", 1000000}, + {(char *)"GB", 1000000000}, + {(char *)"K", 1000}, + {(char *)"M", 1000000}, + {(char *)"G", 1000000000}, + }; int i; int isNeg = 0; if (zArg[0] == '-') { diff --git a/osquery/main/shell.cpp b/osquery/main/shell.cpp index 0ebadb54..d0e8d2ee 100644 --- a/osquery/main/shell.cpp +++ b/osquery/main/shell.cpp @@ -50,6 +50,7 @@ int profile(int argc, char *argv[]) { for (size_t i = 0; i < static_cast(osquery::FLAGS_profile); ++i) { osquery::QueryData results; auto status = osquery::queryInternal(query, results, dbc->db()); + dbc->clearAffectedTables(); if (!status) { fprintf(stderr, "Query failed (%d): %s\n", diff --git a/osquery/sql/benchmarks/sql_benchmarks.cpp b/osquery/sql/benchmarks/sql_benchmarks.cpp index 830a529b..84783068 100644 --- a/osquery/sql/benchmarks/sql_benchmarks.cpp +++ b/osquery/sql/benchmarks/sql_benchmarks.cpp @@ -52,7 +52,7 @@ static void SQL_virtual_table_internal(benchmark::State& state) { // Attach a sample virtual table. auto dbc = SQLiteDBManager::get(); - attachTableInternal("benchmark", columnDefinition(res), dbc->db()); + attachTableInternal("benchmark", columnDefinition(res), dbc); while (state.KeepRunning()) { QueryData results; @@ -84,7 +84,7 @@ static void SQL_virtual_table_internal_long(benchmark::State& state) { // Attach a sample virtual table. auto dbc = SQLiteDBManager::get(); - attachTableInternal("long_benchmark", columnDefinition(res), dbc->db()); + attachTableInternal("long_benchmark", columnDefinition(res), dbc); while (state.KeepRunning()) { QueryData results; @@ -124,7 +124,7 @@ static void SQL_virtual_table_internal_wide(benchmark::State& state) { // Attach a sample virtual table. auto dbc = SQLiteDBManager::get(); - attachTableInternal("wide_benchmark", columnDefinition(res), dbc->db()); + attachTableInternal("wide_benchmark", columnDefinition(res), dbc); while (state.KeepRunning()) { QueryData results; diff --git a/osquery/sql/sql.cpp b/osquery/sql/sql.cpp index 426206f3..070a52d9 100644 --- a/osquery/sql/sql.cpp +++ b/osquery/sql/sql.cpp @@ -20,16 +20,6 @@ namespace osquery { FLAG(int32, value_max, 512, "Maximum returned row value size"); -const std::map kSQLOperatorRepr = { - {EQUALS, "="}, - {GREATER_THAN, ">"}, - {LESS_THAN_OR_EQUALS, "<="}, - {LESS_THAN, "<"}, - {GREATER_THAN_OR_EQUALS, ">="}, -}; - -typedef unsigned char byte; - SQL::SQL(const std::string& q) { status_ = query(q, results_); } const QueryData& SQL::rows() const { return results_; } @@ -40,7 +30,7 @@ const Status& SQL::getStatus() const { return status_; } std::string SQL::getMessageString() { return status_.toString(); } -void escapeNonPrintableBytes(std::string& data) { +static inline void escapeNonPrintableBytes(std::string& data) { std::string escaped; // clang-format off char const hex_chars[16] = { @@ -65,11 +55,11 @@ void escapeNonPrintableBytes(std::string& data) { bool needs_replacement = false; for (size_t i = 0; i < data.length(); i++) { - if (((byte)data[i]) < 0x20 || ((byte)data[i]) >= 0x80) { + if (((unsigned char)data[i]) < 0x20 || ((unsigned char)data[i]) >= 0x80) { needs_replacement = true; escaped += "\\x"; - escaped += hex_chars[(((byte)data[i])) >> 4]; - escaped += hex_chars[((byte)data[i] & 0x0F) >> 0]; + escaped += hex_chars[(((unsigned char)data[i])) >> 4]; + escaped += hex_chars[((unsigned char)data[i] & 0x0F) >> 0]; } else { escaped += data[i]; } @@ -77,10 +67,14 @@ void escapeNonPrintableBytes(std::string& data) { // Only replace if any escapes were made. if (needs_replacement) { - data = escaped; + data = std::move(escaped); } } +void escapeNonPrintableBytesEx(std::string& data) { + return escapeNonPrintableBytes(data); +} + void SQL::escapeResults() { for (auto& row : results_) { for (auto& column : row) { @@ -91,8 +85,7 @@ void SQL::escapeResults() { QueryData SQL::selectAllFrom(const std::string& table) { PluginResponse response; - PluginRequest request = {{"action", "generate"}}; - Registry::call("table", table, request, response); + Registry::call("table", table, {{"action", "generate"}}, response); return response; } @@ -100,12 +93,14 @@ QueryData SQL::selectAllFrom(const std::string& table, const std::string& column, ConstraintOperator op, const std::string& expr) { - PluginResponse response; PluginRequest request = {{"action", "generate"}}; - QueryContext ctx; - ctx.constraints[column].add(Constraint(op, expr)); + { + QueryContext ctx; + ctx.constraints[column].add(Constraint(op, expr)); + TablePlugin::setRequestFromContext(ctx, request); + } - TablePlugin::setRequestFromContext(ctx, request); + PluginResponse response; Registry::call("table", table, request, response); return response; } diff --git a/osquery/sql/sqlite_util.cpp b/osquery/sql/sqlite_util.cpp index 866c31a1..be1b3ac0 100644 --- a/osquery/sql/sqlite_util.cpp +++ b/osquery/sql/sqlite_util.cpp @@ -36,36 +36,16 @@ using SQLiteDBInstanceRef = std::shared_ptr; * Details of this map are defined at: http://www.sqlite.org/c3ref/c_abort.html */ const std::map kSQLiteReturnCodes = { - {0, "SQLITE_OK"}, - {1, "SQLITE_ERROR"}, - {2, "SQLITE_INTERNAL"}, - {3, "SQLITE_PERM"}, - {4, "SQLITE_ABORT"}, - {5, "SQLITE_BUSY"}, - {6, "SQLITE_LOCKED"}, - {7, "SQLITE_NOMEM"}, - {8, "SQLITE_READONLY"}, - {9, "SQLITE_INTERRUPT"}, - {10, "SQLITE_IOERR"}, - {11, "SQLITE_CORRUPT"}, - {12, "SQLITE_NOTFOUND"}, - {13, "SQLITE_FULL"}, - {14, "SQLITE_CANTOPEN"}, - {15, "SQLITE_PROTOCOL"}, - {16, "SQLITE_EMPTY"}, - {17, "SQLITE_SCHEMA"}, - {18, "SQLITE_TOOBIG"}, - {19, "SQLITE_CONSTRAINT"}, - {20, "SQLITE_MISMATCH"}, - {21, "SQLITE_MISUSE"}, - {22, "SQLITE_NOLFS"}, - {23, "SQLITE_AUTH"}, - {24, "SQLITE_FORMAT"}, - {25, "SQLITE_RANGE"}, - {26, "SQLITE_NOTADB"}, - {27, "SQLITE_NOTICE"}, - {28, "SQLITE_WARNING"}, - {100, "SQLITE_ROW"}, + {0, "SQLITE_OK"}, {1, "SQLITE_ERROR"}, {2, "SQLITE_INTERNAL"}, + {3, "SQLITE_PERM"}, {4, "SQLITE_ABORT"}, {5, "SQLITE_BUSY"}, + {6, "SQLITE_LOCKED"}, {7, "SQLITE_NOMEM"}, {8, "SQLITE_READONLY"}, + {9, "SQLITE_INTERRUPT"}, {10, "SQLITE_IOERR"}, {11, "SQLITE_CORRUPT"}, + {12, "SQLITE_NOTFOUND"}, {13, "SQLITE_FULL"}, {14, "SQLITE_CANTOPEN"}, + {15, "SQLITE_PROTOCOL"}, {16, "SQLITE_EMPTY"}, {17, "SQLITE_SCHEMA"}, + {18, "SQLITE_TOOBIG"}, {19, "SQLITE_CONSTRAINT"}, {20, "SQLITE_MISMATCH"}, + {21, "SQLITE_MISUSE"}, {22, "SQLITE_NOLFS"}, {23, "SQLITE_AUTH"}, + {24, "SQLITE_FORMAT"}, {25, "SQLITE_RANGE"}, {26, "SQLITE_NOTADB"}, + {27, "SQLITE_NOTICE"}, {28, "SQLITE_WARNING"}, {100, "SQLITE_ROW"}, {101, "SQLITE_DONE"}, }; @@ -129,13 +109,6 @@ std::string getStringForSQLiteReturnCode(int code) { } Status SQLiteSQLPlugin::attach(const std::string& name) { - // This may be the managed DB, or a transient. - auto dbc = SQLiteDBManager::get(); - if (!dbc->isPrimary()) { - // Do not "reattach" to transient instance. - return Status(0, "OK"); - } - PluginResponse response; auto status = Registry::call("table", name, {{"action", "columns"}}, response); @@ -144,7 +117,8 @@ Status SQLiteSQLPlugin::attach(const std::string& name) { } auto statement = columnDefinition(response); - return attachTableInternal(name, statement, dbc->db()); + auto dbc = SQLiteDBManager::getConnection(true); + return attachTableInternal(name, statement, dbc); } void SQLiteSQLPlugin::detach(const std::string& name) { @@ -169,46 +143,85 @@ SQLiteDBInstance::SQLiteDBInstance(sqlite3*& db, std::mutex& mtx) void SQLiteDBInstance::init() { primary_ = false; sqlite3_open(":memory:", &db_); - attachVirtualTables(db_); +} + +void SQLiteDBInstance::addAffectedTable(VirtualTableContent* table) { + // An xFilter/scan was requested for this virtual table. + affected_tables_.insert(std::make_pair(table->name, table)); +} + +void SQLiteDBInstance::clearAffectedTables() { + if (isPrimary() && !managed_) { + // A primary instance must forward clear requests to the DB manager's + // 'connection' instance. This is a temporary primary instance. + SQLiteDBManager::getConnection(true)->clearAffectedTables(); + return; + } + + for (const auto& table : affected_tables_) { + table.second->constraints.clear(); + } + // Since the affected tables are cleared, there are no more affected tables. + // There is no concept of compounding tables between queries. + affected_tables_.clear(); } SQLiteDBInstance::~SQLiteDBInstance() { - if (!primary_) { + if (!isPrimary()) { sqlite3_close(db_); } else { db_ = nullptr; } } +SQLiteDBManager::SQLiteDBManager() : db_(nullptr) { + sqlite3_soft_heap_limit64(SQLITE_SOFT_HEAP_LIMIT); + setDisabledTables(Flag::getValue("disable_tables")); +} + bool SQLiteDBManager::isDisabled(const std::string& table_name) { const auto& element = instance().disabled_tables_.find(table_name); return (element != instance().disabled_tables_.end()); } -std::unordered_set SQLiteDBManager::parseDisableTablesFlag( - const std::string& list) { +void SQLiteDBManager::setDisabledTables(const std::string& list) { const auto& tables = split(list, ","); - return std::unordered_set(tables.begin(), tables.end()); + disabled_tables_ = + std::unordered_set(tables.begin(), tables.end()); } SQLiteDBInstanceRef SQLiteDBManager::getUnique() { - return std::make_shared(); + auto instance = std::make_shared(); + attachVirtualTables(instance); + return instance; } -SQLiteDBInstanceRef SQLiteDBManager::get() { +SQLiteDBInstanceRef SQLiteDBManager::getConnection(bool primary) { auto& self = instance(); std::unique_lock lock(self.create_mutex_); if (self.db_ == nullptr) { // Create primary SQLite DB instance. sqlite3_open(":memory:", &self.db_); - attachVirtualTables(self.db_); + self.connection_ = SQLiteDBInstanceRef(new SQLiteDBInstance(self.db_)); + attachVirtualTables(self.connection_); } - return std::make_shared(self.db_, self.mutex_); + // Internal usage may request the primary connection explicitly. + if (primary) { + return self.connection_; + } + + // Create a 'database connection' for the managed database instance. + auto instance = std::make_shared(self.db_, self.mutex_); + if (!instance->isPrimary()) { + attachVirtualTables(instance); + } + return instance; } SQLiteDBManager::~SQLiteDBManager() { + connection_ = nullptr; if (db_ != nullptr) { sqlite3_close(db_); db_ = nullptr; diff --git a/osquery/sql/sqlite_util.h b/osquery/sql/sqlite_util.h index 947ec4f0..f9e63c01 100644 --- a/osquery/sql/sqlite_util.h +++ b/osquery/sql/sqlite_util.h @@ -25,6 +25,8 @@ namespace osquery { +class SQLiteDBManager; + /** * @brief An RAII wrapper around an `sqlite3` object. * @@ -54,17 +56,41 @@ class SQLiteDBInstance : private boost::noncopyable { */ sqlite3* db() const { return db_; } + /// Allow a virtual table implementation to record use/access of a table. + void addAffectedTable(VirtualTableContent* table); + + /// Clear per-query state of a table affected by the use of this instance. + void clearAffectedTables(); + + private: + /// An opaque constructor only used by the DBManager. + SQLiteDBInstance(sqlite3* db) : primary_(true), managed_(true), db_(db) {} + private: /// Introspection into the database pointer, primary means managed. bool primary_{false}; + /// Track whether this instance is managed internally by the DB manager. + bool managed_{false}; + /// Either the managed primary database or an ephemeral instance. sqlite3* db_{nullptr}; /// An attempted unique lock on the manager's primary database access mutex. std::unique_lock lock_; + + /// Vector of tables that need their constraints cleared after execution. + std::map affected_tables_; + + private: + friend class SQLiteDBManager; + + private: + FRIEND_TEST(SQLiteUtilTests, test_affected_tables); }; +using SQLiteDBInstanceRef = std::shared_ptr; + /** * @brief osquery internal SQLite DB abstraction resource management. * @@ -97,10 +123,10 @@ class SQLiteDBManager : private boost::noncopyable { * * @return a SQLiteDBInstance with all virtual tables attached. */ - static std::shared_ptr get(); + static SQLiteDBInstanceRef get() { return getConnection(); } /// See `get` but always return a transient DB connection (for testing). - static std::shared_ptr getUnique(); + static SQLiteDBInstanceRef getUnique(); /** * @brief Check if `table_name` is disabled. @@ -113,22 +139,21 @@ class SQLiteDBManager : private boost::noncopyable { */ static bool isDisabled(const std::string& table_name); - /// When the primary SQLiteDBInstance is destructed it will unlock. - static void unlock(); - protected: - SQLiteDBManager() : db_(nullptr) { - sqlite3_soft_heap_limit64(SQLITE_SOFT_HEAP_LIMIT); - disabled_tables_ = parseDisableTablesFlag(Flag::getValue("disable_tables")); - } - SQLiteDBManager(SQLiteDBManager const&); - SQLiteDBManager& operator=(SQLiteDBManager const&); + SQLiteDBManager(); virtual ~SQLiteDBManager(); + public: + SQLiteDBManager(SQLiteDBManager const&) = delete; + SQLiteDBManager& operator=(SQLiteDBManager const&) = delete; + private: /// Primary (managed) sqlite3 database. sqlite3* db_{nullptr}; + /// The primary connection maintains an opaque instance. + SQLiteDBInstanceRef connection_{nullptr}; + /// Mutex and lock around sqlite3 access. std::mutex mutex_; @@ -139,7 +164,14 @@ class SQLiteDBManager : private boost::noncopyable { std::unordered_set disabled_tables_; /// Parse a comma-delimited set of tables names, passed in as a flag. - std::unordered_set parseDisableTablesFlag(const std::string& s); + void setDisabledTables(const std::string& s); + + /// Request a connection, optionally request the primary connection. + static SQLiteDBInstanceRef getConnection(bool primary = false); + + private: + friend class SQLiteDBInstance; + friend class SQLiteSQLPlugin; }; /** @@ -247,7 +279,9 @@ class SQLiteSQLPlugin : SQLPlugin { public: Status query(const std::string& q, QueryData& results) const { auto dbc = SQLiteDBManager::get(); - return queryInternal(q, results, dbc->db()); + auto result = queryInternal(q, results, dbc->db()); + dbc->clearAffectedTables(); + return result; } Status getQueryColumns(const std::string& q, TableColumns& columns) const { @@ -274,6 +308,7 @@ class SQLInternal : public SQL { explicit SQLInternal(const std::string& q) { auto dbc = SQLiteDBManager::get(); status_ = queryInternal(q, results_, dbc->db()); + dbc->clearAffectedTables(); } }; diff --git a/osquery/sql/tests/sql_tests.cpp b/osquery/sql/tests/sql_tests.cpp index 21cabefc..5915e95e 100644 --- a/osquery/sql/tests/sql_tests.cpp +++ b/osquery/sql/tests/sql_tests.cpp @@ -17,7 +17,7 @@ namespace osquery { -extern void escapeNonPrintableBytes(std::string& data); +extern void escapeNonPrintableBytesEx(std::string& data); class SQLTests : public testing::Test {}; @@ -68,25 +68,25 @@ TEST_F(SQLTests, test_raw_access_context) { TEST_F(SQLTests, test_sql_escape) { std::string input = "しかたがない"; - escapeNonPrintableBytes(input); + escapeNonPrintableBytesEx(input); EXPECT_EQ(input, "\\xE3\\x81\\x97\\xE3\\x81\\x8B\\xE3\\x81\\x9F\\xE3\\x81\\x8C\\xE3" "\\x81\\xAA\\xE3\\x81\\x84"); input = "悪因悪果"; - escapeNonPrintableBytes(input); + escapeNonPrintableBytesEx(input); EXPECT_EQ(input, "\\xE6\\x82\\xAA\\xE5\\x9B\\xA0\\xE6\\x82\\xAA\\xE6\\x9E\\x9C"); input = "モンスターハンター"; - escapeNonPrintableBytes(input); + escapeNonPrintableBytesEx(input); EXPECT_EQ(input, "\\xE3\\x83\\xA2\\xE3\\x83\\xB3\\xE3\\x82\\xB9\\xE3\\x82\\xBF\\xE3" "\\x83\\xBC\\xE3\\x83\\x8F\\xE3\\x83\\xB3\\xE3\\x82\\xBF\\xE3\\x83" "\\xBC"); input = "съешь же ещё этих мягких французских булок, да выпей чаю"; - escapeNonPrintableBytes(input); + escapeNonPrintableBytesEx(input); EXPECT_EQ( input, "\\xD1\\x81\\xD1\\x8A\\xD0\\xB5\\xD1\\x88\\xD1\\x8C \\xD0\\xB6\\xD0\\xB5 " @@ -99,7 +99,7 @@ TEST_F(SQLTests, test_sql_escape) { "\\xD1\\x87\\xD0\\xB0\\xD1\\x8E"); input = "The quick brown fox jumps over the lazy dog."; - escapeNonPrintableBytes(input); + escapeNonPrintableBytesEx(input); EXPECT_EQ(input, "The quick brown fox jumps over the lazy dog."); } } diff --git a/osquery/sql/tests/sqlite_util_tests.cpp b/osquery/sql/tests/sqlite_util_tests.cpp index 7621348a..bd69f5bb 100644 --- a/osquery/sql/tests/sqlite_util_tests.cpp +++ b/osquery/sql/tests/sqlite_util_tests.cpp @@ -108,6 +108,17 @@ TEST_F(SQLiteUtilTests, test_get_test_db_result_stream) { } } +TEST_F(SQLiteUtilTests, test_affected_tables) { + auto dbc = getTestDBC(); + QueryData results; + auto status = queryInternal("SELECT * FROM time", results, dbc->db()); + + // Since the table scanned from "time", it should be recorded as affected. + EXPECT_EQ(dbc->affected_tables_.count("time"), 1U); + dbc->clearAffectedTables(); + EXPECT_EQ(dbc->affected_tables_.size(), 0U); +} + TEST_F(SQLiteUtilTests, test_get_query_columns) { auto dbc = getTestDBC(); TableColumns results; diff --git a/osquery/sql/tests/virtual_table_tests.cpp b/osquery/sql/tests/virtual_table_tests.cpp index b85c84dd..63dd750f 100644 --- a/osquery/sql/tests/virtual_table_tests.cpp +++ b/osquery/sql/tests/virtual_table_tests.cpp @@ -46,8 +46,7 @@ TEST_F(VirtualTableTests, test_sqlite3_attach_vtable) { auto dbc = SQLiteDBManager::get(); // Virtual tables require the registry/plugin API to query tables. - auto status = - attachTableInternal("failed_sample", "(foo INTEGER)", dbc->db()); + auto status = attachTableInternal("failed_sample", "(foo INTEGER)", dbc); EXPECT_EQ(status.getCode(), SQLITE_ERROR); // The table attach will complete only when the table name is registered. @@ -57,7 +56,7 @@ TEST_F(VirtualTableTests, test_sqlite3_attach_vtable) { EXPECT_TRUE(status.ok()); // Use the table name, plugin-generated schema to attach. - status = attachTableInternal("sample", columnDefinition(response), dbc->db()); + status = attachTableInternal("sample", columnDefinition(response), dbc); EXPECT_EQ(status.getCode(), SQLITE_OK); std::string q = "SELECT sql FROM sqlite_temp_master WHERE tbl_name='sample';"; @@ -141,9 +140,9 @@ TEST_F(VirtualTableTests, test_constraints_stacking) { { // To simplify the attach, just access the column definition directly. auto p = std::make_shared(); - attachTableInternal("p", p->columnDefinition(), dbc->db()); + attachTableInternal("p", p->columnDefinition(), dbc); auto k = std::make_shared(); - attachTableInternal("k", k->columnDefinition(), dbc->db()); + attachTableInternal("k", k->columnDefinition(), dbc); } QueryData results; @@ -225,7 +224,7 @@ TEST_F(VirtualTableTests, test_json_extract) { { auto json = std::make_shared(); - attachTableInternal("json", json->columnDefinition(), dbc->db()); + attachTableInternal("json", json->columnDefinition(), dbc); } QueryData results; diff --git a/osquery/sql/virtual_table.cpp b/osquery/sql/virtual_table.cpp index a69b2158..91e0bbfd 100644 --- a/osquery/sql/virtual_table.cpp +++ b/osquery/sql/virtual_table.cpp @@ -8,6 +8,8 @@ * */ +#include + #include #include @@ -20,10 +22,18 @@ SHELL_FLAG(bool, planner, false, "Enable osquery runtime planner output"); namespace tables { namespace sqlite { -static size_t kPlannerCursorID = 0; -static size_t kConstraintIndexID = 0; +/// For planner and debugging an incrementing cursor ID is used. +static std::atomic kPlannerCursorID{0}; -static std::string opString(unsigned char op) { +/** + * @brief A next-ID for within-query constraints stacking. + * + * As constraints are evaluated within xBestIndex, an IDX is assigned for + * operator and operand retrieval during xFilter/scanning. + */ +static std::atomic kConstraintIndexID{0}; + +static inline std::string opString(unsigned char op) { switch (op) { case EQUALS: return "="; @@ -73,12 +83,7 @@ int xOpen(sqlite3_vtab *tab, sqlite3_vtab_cursor **ppCursor) { int xClose(sqlite3_vtab_cursor *cur) { BaseCursor *pCur = (BaseCursor *)cur; - const auto *pVtab = (VirtualTable *)cur->pVtab; plan("Closing cursor (" + std::to_string(pCur->id) + ")"); - if (pVtab != nullptr) { - // Reset all constraints for the virtual table content. - pVtab->content->constraints.clear(); - } delete pCur; return SQLITE_OK; } @@ -126,6 +131,7 @@ int xCreate(sqlite3 *db, memset(pVtab, 0, sizeof(VirtualTable)); pVtab->content = new VirtualTableContent; + pVtab->instance = (SQLiteDBInstance *)pAux; // Create a TablePlugin Registry call, expect column details as the response. PluginResponse response; @@ -283,6 +289,7 @@ static int xFilter(sqlite3_vtab_cursor *pVtabCursor, BaseCursor *pCur = (BaseCursor *)pVtabCursor; auto *pVtab = (VirtualTable *)pVtabCursor->pVtab; auto *content = pVtab->content; + pVtab->instance->addAffectedTable(content); pCur->row = 0; pCur->n = 0; @@ -344,7 +351,7 @@ static int xFilter(sqlite3_vtab_cursor *pVtabCursor, Status attachTableInternal(const std::string &name, const std::string &statement, - sqlite3 *db) { + const SQLiteDBInstanceRef &instance) { if (SQLiteDBManager::isDisabled(name)) { VLOG(1) << "Table " << name << " is disabled, not attaching"; return Status(0, getStringForSQLiteReturnCode(0)); @@ -381,11 +388,12 @@ Status attachTableInternal(const std::string &name, // Note, if the clientData API is used then this will save a registry call // within xCreate. - int rc = sqlite3_create_module(db, name.c_str(), &module, 0); + int rc = sqlite3_create_module( + instance->db(), name.c_str(), &module, (void *)&(*instance)); if (rc == SQLITE_OK || rc == SQLITE_MISUSE) { auto format = "CREATE VIRTUAL TABLE temp." + name + " USING " + name + statement; - rc = sqlite3_exec(db, format.c_str(), nullptr, nullptr, 0); + rc = sqlite3_exec(instance->db(), format.c_str(), nullptr, nullptr, 0); } else { LOG(ERROR) << "Error attaching table: " << name << " (" << rc << ")"; } @@ -402,7 +410,7 @@ Status detachTableInternal(const std::string &name, sqlite3 *db) { return Status(rc, getStringForSQLiteReturnCode(rc)); } -void attachVirtualTables(sqlite3 *db) { +void attachVirtualTables(const SQLiteDBInstanceRef &instance) { PluginResponse response; for (const auto &name : Registry::names("table")) { // Column information is nice for virtual table create call. @@ -410,7 +418,7 @@ void attachVirtualTables(sqlite3 *db) { Registry::call("table", name, {{"action", "columns"}}, response); if (status.ok()) { auto statement = columnDefinition(response); - attachTableInternal(name, statement, db); + attachTableInternal(name, statement, instance); } } } diff --git a/osquery/sql/virtual_table.h b/osquery/sql/virtual_table.h index d333ffd1..48824b3d 100644 --- a/osquery/sql/virtual_table.h +++ b/osquery/sql/virtual_table.h @@ -10,8 +10,6 @@ #pragma once -#include - #include #include "osquery/core/conversions.h" @@ -19,29 +17,6 @@ namespace osquery { -/** - * @brief osquery virtual table connection. - * - * This object is the SQLite database's virtual table context. - * When the virtual table is created/connected the name and columns are - * retrieved via the TablePlugin call API. The details are kept in this context - * so column parsing and row walking does not require additional Registry calls. - * - * When tables are accessed as the result of an SQL statement a QueryContext is - * created to represent metadata that can be used by the virtual table - * implementation code. Thus the code that generates rows can choose to emit - * additional data, restrict based on constraints, or potentially yield from - * a cache or choose not to generate certain columns. - */ -struct VirtualTableContent { - /// Friendly name for the table. - TableName name; - /// Table column structure, retrieved once via the TablePlugin call API. - TableColumns columns; - /// Transient set of virtual table access constraints. - std::unordered_map constraints; -}; - /** * @brief osquery cursor object. * @@ -67,18 +42,24 @@ struct BaseCursor { * This adds each table plugin class to the state tracking in SQLite. */ struct VirtualTable { + /// The SQLite-provided virtual table structure. sqlite3_vtab base; + + /// Added structure: A content structure with metadata about the table. VirtualTableContent *content{nullptr}; + + /// Added structure: The thread-local DB instance associated with the query. + SQLiteDBInstance *instance{nullptr}; }; /// Attach a table plugin name to an in-memory SQLite database. Status attachTableInternal(const std::string &name, const std::string &statement, - sqlite3 *db); + const SQLiteDBInstanceRef &instance); /// Detach (drop) a table. Status detachTableInternal(const std::string &name, sqlite3 *db); /// Attach all table plugins to an in-memory SQLite database. -void attachVirtualTables(sqlite3 *db); +void attachVirtualTables(const SQLiteDBInstanceRef &instance); }