Adding a config block to create views (#3306)

docs/wiki/deployment/configuration.md

@@ -411,6 +411,23 @@ Example:
 }
 ```
 
+### Views
+
+Views are saved queries expressed as tables. Large subqueries or complex joining logic can often be moved into views allowing you to make your queries more concise.
+
+Example:
+```json
+{
+  "views": {
+    "kernel_hashses" : "select hash.path as kernel_binary, version, hash.sha256 as sha256, hash.sha1 as sha1, hash.md5 as md5 from (select path || '/Contents/MacOS/' as directory, name, version from kernel_extensions) join hash using (directory)"
+  }
+}
+```
+
+```SQL
+select * from kernel_hashes where kernel_binary not like "%apple%"
+```
+
 ### Decorator queries
 
 Decorator queries exist in osquery versions 1.7.3+ and are used to add additional "decorations" to results and snapshot logs. There are three types of decorator queries based on when and how you want the decoration data.

include/osquery/config.h

@@ -337,6 +337,9 @@ class Config : private boost::noncopyable {
   friend class DecoratorsConfigParserPluginTests;
   friend class SchedulerTests;
   FRIEND_TEST(OptionsConfigParserPluginTests, test_get_option);
+  FRIEND_TEST(ViewsConfigParserPluginTests, test_add_view);
+  FRIEND_TEST(ViewsConfigParserPluginTests, test_swap_view);
+  FRIEND_TEST(ViewsConfigParserPluginTests, test_update_view);
   FRIEND_TEST(EventsConfigParserPluginTests, test_get_event);
   FRIEND_TEST(PacksTests, test_discovery_cache);
   FRIEND_TEST(PacksTests, test_multi_pack);

osquery/config/parsers/tests/views_tests.cpp

@@ -0,0 +1,69 @@
+/*
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <gtest/gtest.h>
+
+#include <osquery/config.h>
+#include <osquery/database.h>
+#include <osquery/registry.h>
+
+#include "osquery/tests/test_util.h"
+
+namespace osquery {
+
+class ViewsConfigParserPluginTests : public testing::Test {};
+
+TEST_F(ViewsConfigParserPluginTests, test_add_view) {
+  Config c;
+  auto s = c.update(getTestConfigMap());
+  EXPECT_TRUE(s.ok());
+
+  std::vector<std::string> old_views_vec;
+  scanDatabaseKeys(kQueries, old_views_vec, "config_views.");
+  EXPECT_EQ(old_views_vec.size(), 1U);
+  c.reset();
+}
+
+TEST_F(ViewsConfigParserPluginTests, test_swap_view) {
+  Config c;
+  std::vector<std::string> old_views_vec;
+  scanDatabaseKeys(kQueries, old_views_vec, "config_views.");
+  EXPECT_EQ(old_views_vec.size(), 1U);
+  old_views_vec.clear();
+  auto s = c.update(getTestConfigMap("view_test.conf"));
+  EXPECT_TRUE(s.ok());
+  scanDatabaseKeys(kQueries, old_views_vec, "config_views.");
+  EXPECT_EQ(old_views_vec.size(), 1U);
+  EXPECT_EQ(old_views_vec[0], "config_views.kernel_hashes_new");
+
+  c.reset();
+}
+
+TEST_F(ViewsConfigParserPluginTests, test_update_view) {
+  Config c;
+  std::vector<std::string> old_views_vec;
+  scanDatabaseKeys(kQueries, old_views_vec, "config_views.");
+  EXPECT_EQ(old_views_vec.size(), 1U);
+  old_views_vec.clear();
+  auto s = c.update(getTestConfigMap("view_test2.conf"));
+  EXPECT_TRUE(s.ok());
+  scanDatabaseKeys(kQueries, old_views_vec, "config_views.");
+  EXPECT_EQ(old_views_vec.size(), 1U);
+  std::string query;
+  getDatabaseValue(kQueries, "config_views.kernel_hashes_new", query);
+  EXPECT_EQ(query,
+            "select hash.path as binary, version, hash.sha256 as SHA256, "
+            "hash.sha1 as SHA1, hash.md5 as MD5 from (select path || "
+            "'/Contents/MacOS/' as directory, name, version from "
+            "kernel_extensions) join hash using (directory)");
+
+  c.reset();
+}
+}

osquery/config/parsers/views.cpp

@@ -0,0 +1,98 @@
+/*
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <set>
+
+#include <osquery/config.h>
+#include <osquery/database.h>
+#include <osquery/logger.h>
+#include <osquery/sql.h>
+
+#include "osquery/core/conversions.h"
+
+namespace pt = boost::property_tree;
+
+namespace osquery {
+
+/**
+ * @brief A simple ConfigParserPlugin for a "views" dictionary key.
+ */
+class ViewsConfigParserPlugin : public ConfigParserPlugin {
+ public:
+  std::vector<std::string> keys() const override {
+    return {"views"};
+  }
+
+  Status setUp() override;
+
+  Status update(const std::string& source, const ParserConfig& config) override;
+
+ private:
+  const std::string kConfigViews = "config_views.";
+};
+
+Status ViewsConfigParserPlugin::setUp() {
+  data_.put_child("views", pt::ptree());
+  return Status(0, "OK");
+}
+
+Status ViewsConfigParserPlugin::update(const std::string& source,
+                                       const ParserConfig& config) {
+  if (config.count("views") > 0) {
+    data_ = pt::ptree();
+    data_.put_child("views", config.at("views"));
+  }
+  const auto& views = data_.get_child("views");
+
+  // We use a restricted scope below to change the data structure from
+  // an array to a set. This lets us do deletes much more efficiently
+  std::vector<std::string> created_views;
+  std::set<std::string> erase_views;
+  {
+    std::vector<std::string> old_views_vec;
+    scanDatabaseKeys(kQueries, old_views_vec, kConfigViews);
+    for (const auto& view : old_views_vec) {
+      erase_views.insert(view.substr(kConfigViews.size()));
+    }
+  }
+  QueryData r;
+  for (const auto& view : views) {
+    const auto& name = view.first;
+    std::string query = views.get<std::string>(view.first, "");
+    if (query.empty()) {
+      continue;
+    }
+    std::string old_query = "";
+    getDatabaseValue(kQueries, kConfigViews + name, old_query);
+    erase_views.erase(name);
+    if (old_query == query) {
+      continue;
+    }
+
+    // View has been updated
+    osquery::query("DROP VIEW " + name, r);
+    auto s = osquery::query("CREATE VIEW " + name + " AS " + query, r);
+    if (s.ok()) {
+      setDatabaseValue(kQueries, kConfigViews + name, query);
+    } else {
+      LOG(INFO) << "Error creating view (" << name << "): " << s.getMessage();
+    }
+  }
+  // Any views left are views that don't exist in the new configuration file
+  // so we tear them down and remove them from the database.
+  for (const auto& old_view : erase_views) {
+    osquery::query("DROP VIEW " + old_view, r);
+    deleteDatabaseValue(kQueries, kConfigViews + old_view);
+  }
+  return Status(0, "OK");
+}
+
+REGISTER_INTERNAL(ViewsConfigParserPlugin, "config_parser", "views");
+}

osquery/tests/test_util.cpp

@@ -103,9 +103,9 @@ void shutdownTesting() {
   Initializer::platformTeardown();
 }
 
-std::map<std::string, std::string> getTestConfigMap() {
+std::map<std::string, std::string> getTestConfigMap(const std::string& file) {
   std::string content;
-  readFile(fs::path(kTestDataPath) / "test_parse_items.conf", content);
+  readFile(fs::path(kTestDataPath) / file, content);
   std::map<std::string, std::string> config;
   config["awesome"] = content;
   return config;

osquery/tests/test_util.h

@@ -76,7 +76,8 @@ extern const char* kExpectedExtensionArgs[];
 extern const size_t kExpectedExtensionArgsCount;
 
 // Get an example generate config with one static source name to JSON content.
-std::map<std::string, std::string> getTestConfigMap();
+std::map<std::string, std::string> getTestConfigMap(
+    const std::string& file = "test_parse_items.conf");
 
 pt::ptree getExamplePacksConfig();
 pt::ptree getUnrestrictedPack();

tools/tests/test_parse_items.conf

@@ -77,5 +77,9 @@
         "select 'invalid' as invalid_interval_test"
       ]
     }
+  },
+
+  "views" : {
+    "kernel_hashes" : "select hash.path as kernel_binary, version, hash.sha256 as sha256, hash.sha1 as sha1, hash.md5 as md5 from (select path || '/Contents/MacOS/' as directory, name, version from kernel_extensions) join hash using (directory)"
   }
 }

tools/tests/view_test.conf

@@ -0,0 +1,5 @@
+{
+  "views" : {
+    "kernel_hashes_new" : "select hash.path as kernel_binary, version, hash.sha256 as sha256, hash.sha1 as sha1, hash.md5 as md5 from (select path || '/Contents/MacOS/' as directory, name, version from kernel_extensions) join hash using (directory)"
+  }
+}

tools/tests/view_test2.conf

@@ -0,0 +1,5 @@
+{
+  "views" : {
+    "kernel_hashes_new" : "select hash.path as binary, version, hash.sha256 as SHA256, hash.sha1 as SHA1, hash.md5 as MD5 from (select path || '/Contents/MacOS/' as directory, name, version from kernel_extensions) join hash using (directory)"
+  }
+}