Some tweaks to estimated scratch/heap for SQLite and RocksDB

include/osquery/database.h

@@ -263,7 +263,7 @@ struct ScheduledQuery {
   unsigned long long int system_time;
 
   /// Average memory differentials. This should be near 0.
-  unsigned long long int memory;
+  unsigned long long int average_memory;
 
   /// Total characters, bytes, generated by query.
   unsigned long long int output_size;
@@ -278,7 +278,7 @@ struct ScheduledQuery {
         wall_time(0),
         user_time(0),
         system_time(0),
-        memory(0),
+        average_memory(0),
         output_size(0) {}
 
   /// equals operator

osquery/config/config.cpp

@@ -360,8 +360,8 @@ void Config::recordQueryPerformance(const std::string& name,
          AS_LITERAL(BIGINT_LITERAL, r0.at("resident_size"));
   if (diff > 0) {
     // Memory is stored as an average of RSS changes between query executions.
-    query.memory = (query.memory * query.executions) + diff;
-    query.memory = (query.memory / (query.executions + 1));
+    query.average_memory = (query.average_memory * query.executions) + diff;
+    query.average_memory = (query.average_memory / (query.executions + 1));
   }
 
   query.wall_time += delay;

osquery/database/db_handle.cpp

@@ -94,6 +94,10 @@ DBHandle::DBHandle(const std::string& path, bool in_memory) {
   options_.log_file_time_to_roll = 0;
   options_.keep_log_file_num = 10;
   options_.max_log_file_size = 1024 * 1024 * 1;
+  options_.compaction_style = rocksdb::kCompactionStyleLevel;
+  options_.write_buffer_size = 1 * 1024 * 1024;
+  options_.max_write_buffer_number = 2;
+  options_.max_background_compactions = 1;
 
   if (in_memory) {
     // Remove when MemEnv is included in librocksdb

osquery/sql/sqlite_util.cpp

@@ -173,13 +173,14 @@ int queryDataCallback(void* argument, int argc, char* argv[], char* column[]) {
       r[column[i]] = (argv[i] != nullptr) ? argv[i] : "";
     }
   }
-  (*qData).push_back(r);
+  (*qData).push_back(std::move(r));
   return 0;
 }
 
 Status queryInternal(const std::string& q, QueryData& results, sqlite3* db) {
   char* err = nullptr;
   sqlite3_exec(db, q.c_str(), queryDataCallback, &results, &err);
+  sqlite3_db_release_memory(db);
   if (err != nullptr) {
     sqlite3_free(err);
     return Status(1, "Error running query: " + q);

osquery/sql/sqlite_util.h

@@ -21,6 +21,8 @@
 
 #include <osquery/sql.h>
 
+#define SQLITE_SOFT_HEAP_LIMIT (5 * 1024 * 1024)
+
 namespace osquery {
 
 /**
@@ -107,6 +109,7 @@ class SQLiteDBManager : private boost::noncopyable {
 
  protected:
   SQLiteDBManager() : db_(nullptr), lock_(mutex_, boost::defer_lock) {
+    sqlite3_soft_heap_limit64(SQLITE_SOFT_HEAP_LIMIT);
     disabled_tables_ = parseDisableTablesFlag(Flag::getValue("disable_tables"));
   }
   SQLiteDBManager(SQLiteDBManager const&);

osquery/tables/utility/osquery.cpp

@@ -221,7 +221,7 @@ QueryData genOsquerySchedule(QueryContext& context) {
     r["wall_time"] = BIGINT(query.second.wall_time);
     r["user_time"] = BIGINT(query.second.user_time);
     r["system_time"] = BIGINT(query.second.system_time);
-    r["average_memory"] = BIGINT(query.second.memory);
+    r["average_memory"] = BIGINT(query.second.average_memory);
     results.push_back(r);
   }