Use 'denylist' instead of 'blacklist' in query scheduling (#6487)

docs/wiki/deployment/configuration.md

@@ -312,7 +312,7 @@ The basic scheduled query specification includes:
 - `platform`: restrict this query to a given platform, default is 'all' platforms; you may use commas to set multiple platforms
 - `version`: only run on osquery versions greater than or equal-to this version string
 - `shard`: restrict this query to a percentage (1-100) of target hosts
-- `blacklist`: a boolean to determine if this query may be blacklisted, default true
+- `denylist`: a boolean to determine if this query may be denylisted (when stopped for excessive resource consumption), default true
 
 The `platform` key can be:
 
@@ -330,7 +330,7 @@ The schedule and associated queries generate a timeline of events through the de
 Snapshot queries, those with `snapshot: true` will not store differentials and will not emulate an event stream. Snapshots always return the entire results from the query on the given interval. See
 the next section on [logging](../deployment/logging.md) for examples of each log output.
 
-Queries may be "blacklisted" if they cause osquery to take too many system resources. A blacklisted query returns to the schedule after a cool-down period of 1 day. Some queries may be very important and you may request that they continue to run even if they are latent. Set the `blacklist: false` to prevent a query from being blacklisted.
+Queries may be "denylisted" if they cause osquery to use excessive system resources. A denylisted query returns to the schedule after a cool-down period of 1 day. Some queries may be very important and you may request that they continue to run even if they are latent. Set the `denylist: false` to prevent a query from being denylisted.
 
 ### Packs
 

docs/wiki/deployment/debugging.md

@@ -96,7 +96,7 @@ If the worker finds itself in a re-occurring error state or the watchdog continu
 osqueryd worker respawning too quickly: 1 times
 ```
 
-The watchdog implements an exponential backoff when respawning workers and the associated 'dirty' query is blacklisted from running for 24 hours.
+The watchdog implements an exponential backoff when respawning workers and the associated 'dirty' query is denylisted from running for 24 hours.
 
 ### Checking the database sanity
 

docs/wiki/development/building.md

@@ -305,6 +305,6 @@ Generating a virtual table should *not* impact system performance. This is easie
 Some quick features include:
 
 - Performance regression and leak detection CI guards.
-- Blacklisting performance-impacting virtual tables.
+- Denylisting performance-impacting virtual tables.
 - Scheduled query optimization and profiling.
 - Query implementation isolation options.

osquery/config/config.cpp

@@ -145,7 +145,7 @@ class Schedule : private boost::noncopyable {
    *
    * This will check for previously executing queries. If any query was
    * executing it is considered in a 'dirty' state and should generate logs.
-   * The schedule may also choose to blacklist this query.
+   * The schedule may also choose to denylist this query.
    */
   Schedule();
 
@@ -194,13 +194,13 @@ class Schedule : private boost::noncopyable {
   std::string failed_query_;
 
   /**
-   * @brief List of blacklisted queries.
+   * @brief List of denylisted queries.
    *
-   * A list of queries that are blacklisted from executing due to prior
+   * A list of queries that are denylisted from executing due to prior
    * failures. If a query caused a worker to fail it will be recorded during
-   * the next execution and saved to the blacklist.
+   * the next execution and saved to the denylist.
    */
-  std::map<std::string, size_t> blacklist_;
+  std::map<std::string, size_t> denylist_;
 
  private:
   friend class Config;
@@ -281,29 +281,28 @@ class ConfigRefreshRunner : public InternalRunnable {
   friend class Config;
 };
 
-void restoreScheduleBlacklist(std::map<std::string, size_t>& blacklist) {
+void restoreScheduleDenylist(std::map<std::string, size_t>& denylist) {
   std::string content;
   getDatabaseValue(kPersistentSettings, kFailedQueries, content);
-  auto blacklist_pairs = osquery::split(content, ":");
-  if (blacklist_pairs.size() == 0 || blacklist_pairs.size() % 2 != 0) {
-    // Nothing in the blacklist, or malformed data.
+  auto denylist_pairs = osquery::split(content, ":");
+  if (denylist_pairs.size() == 0 || denylist_pairs.size() % 2 != 0) {
+    // Nothing in the denylist, or malformed data.
     return;
   }
 
   size_t current_time = getUnixTime();
-  for (size_t i = 0; i < blacklist_pairs.size() / 2; i++) {
-    // Fill in a mapping of query name to time the blacklist expires.
-    auto expire =
-        tryTo<long long>(blacklist_pairs[(i * 2) + 1], 10).takeOr(0ll);
+  for (size_t i = 0; i < denylist_pairs.size() / 2; i++) {
+    // Fill in a mapping of query name to time the denylist expires.
+    auto expire = tryTo<long long>(denylist_pairs[(i * 2) + 1], 10).takeOr(0ll);
     if (expire > 0 && current_time < (size_t)expire) {
-      blacklist[blacklist_pairs[(i * 2)]] = (size_t)expire;
+      denylist[denylist_pairs[(i * 2)]] = (size_t)expire;
     }
   }
 }
 
-void saveScheduleBlacklist(const std::map<std::string, size_t>& blacklist) {
+void saveScheduleDenylist(const std::map<std::string, size_t>& denylist) {
   std::string content;
-  for (const auto& query : blacklist) {
+  for (const auto& query : denylist) {
     if (!content.empty()) {
       content += ":";
     }
@@ -317,17 +316,17 @@ Schedule::Schedule() {
     // Extensions should not restore or save schedule details.
     return;
   }
-  // Parse the schedule's query blacklist from backing storage.
-  restoreScheduleBlacklist(blacklist_);
+  // Parse the schedule's query denylist from backing storage.
+  restoreScheduleDenylist(denylist_);
 
   // Check if any queries were executing when the tool last stopped.
   getDatabaseValue(kPersistentSettings, kExecutingQuery, failed_query_);
   if (!failed_query_.empty()) {
     LOG(WARNING) << "Scheduled query may have failed: " << failed_query_;
     setDatabaseValue(kPersistentSettings, kExecutingQuery, "");
-    // Add this query name to the blacklist and save the blacklist.
-    blacklist_[failed_query_] = getUnixTime() + 86400;
-    saveScheduleBlacklist(blacklist_);
+    // Add this query name to the denylist and save the denylist.
+    denylist_[failed_query_] = getUnixTime() + 86400;
+    saveScheduleDenylist(denylist_);
   }
 }
 
@@ -404,24 +403,24 @@ void Config::removeFiles(const std::string& source) {
 }
 
 /**
- * @brief Return true if the failed query is no longer blacklisted.
+ * @brief Return true if the failed query is no longer denylisted.
  *
- * There are two scenarios where a blacklisted query becomes 'unblacklisted'.
- * The first is simple, the amount of time it was blacklisted for has expired.
+ * There are two scenarios where a denylisted query becomes 'undenylisted'.
+ * The first is simple, the amount of time it was denylisted for has expired.
  * The second is more complex, the query failed but the schedule has requested
- * that the query should not be blacklisted.
+ * that the query should not be denylisted.
  *
- * @param blt The time the query was originally blacklisted.
+ * @param blt The time the query was originally denylisted.
  * @param query The scheduled query and its options.
  */
-static inline bool blacklistExpired(size_t blt, const ScheduledQuery& query) {
+static inline bool denylistExpired(size_t blt, const ScheduledQuery& query) {
   if (getUnixTime() > blt) {
     return true;
   }
 
-  auto blo = query.options.find("blacklist");
+  auto blo = query.options.find("denylist");
   if (blo != query.options.end() && blo->second == false) {
-    // The schedule requested that we do not blacklist this query.
+    // The schedule requested that we do not denylist this query.
     return true;
   }
   return false;
@@ -430,7 +429,7 @@ static inline bool blacklistExpired(size_t blt, const ScheduledQuery& query) {
 void Config::scheduledQueries(
     std::function<void(std::string name, const ScheduledQuery& query)>
         predicate,
-    bool blacklisted) const {
+    bool denylisted) const {
   RecursiveLock lock(config_schedule_mutex_);
   for (PackRef& pack : *schedule_) {
     for (auto& it : pack->getSchedule()) {
@@ -441,19 +440,19 @@ void Config::scheduledQueries(
                FLAGS_pack_delimiter + it.first;
       }
 
-      // They query may have failed and been added to the schedule's blacklist.
-      auto blacklisted_query = schedule_->blacklist_.find(name);
-      if (blacklisted_query != schedule_->blacklist_.end()) {
-        if (blacklistExpired(blacklisted_query->second, it.second)) {
-          // The blacklisted query passed the expiration time (remove).
-          schedule_->blacklist_.erase(blacklisted_query);
-          saveScheduleBlacklist(schedule_->blacklist_);
-          it.second.blacklisted = false;
+      // They query may have failed and been added to the schedule's denylist.
+      auto denylisted_query = schedule_->denylist_.find(name);
+      if (denylisted_query != schedule_->denylist_.end()) {
+        if (denylistExpired(denylisted_query->second, it.second)) {
+          // The denylisted query passed the expiration time (remove).
+          schedule_->denylist_.erase(denylisted_query);
+          saveScheduleDenylist(schedule_->denylist_);
+          it.second.denylisted = false;
         } else {
-          // The query is still blacklisted.
-          it.second.blacklisted = true;
-          if (!blacklisted) {
-            // The caller does not want blacklisted queries.
+          // The query is still denylisted.
+          it.second.denylisted = true;
+          if (!denylisted) {
+            // The caller does not want denylisted queries.
             continue;
           }
         }

osquery/config/config.h

@@ -175,7 +175,7 @@ class Config : private boost::noncopyable {
    * the query and the ScheduledQuery struct of the queries data. predicate
    * will be called on each currently scheduled query.
    *
-   * @param blacklisted [optional] return blacklisted queries if true.
+   * @param denylisted [optional] return denylisted queries if true.
    *
    * @code{.cpp}
    *   std::map<std::string, ScheduledQuery> queries;
@@ -188,7 +188,7 @@ class Config : private boost::noncopyable {
   void scheduledQueries(
       std::function<void(std::string name, const ScheduledQuery& query)>
           predicate,
-      bool blacklisted = false) const;
+      bool denylisted = false) const;
 
   /**
    * @brief Map a function across the set of configured files
@@ -304,7 +304,7 @@ class Config : private boost::noncopyable {
    * associated value is passed to the parser.
    *
    * Use this utility method for both the top-level configuration JSON and
-   * the content of each configuration pack. There is an optional black list
+   * the content of each configuration pack. There is an optional denylist
    * parameter to differentiate pack content.
    *
    * @param source The input configuration source name.
@@ -378,7 +378,7 @@ class Config : private boost::noncopyable {
   FRIEND_TEST(ConfigTests, test_config_backup_integrate);
   FRIEND_TEST(ConfigTests, test_config_refresh);
   FRIEND_TEST(ConfigTests, test_get_scheduled_queries);
-  FRIEND_TEST(ConfigTests, test_nonblacklist_query);
+  FRIEND_TEST(ConfigTests, test_nondenylist_query);
   FRIEND_TEST(OptionsConfigParserPluginTests, test_get_option);
   FRIEND_TEST(OptionsConfigParserPluginTests, test_get_option_first);
   FRIEND_TEST(ViewsConfigParserPluginTests, test_add_view);

osquery/config/packs.cpp

@@ -239,9 +239,14 @@ void Pack::initialize(const std::string& name,
       query.options["removed"] = JSON::valueToBool(q.value["removed"]);
     }
 
-    query.options["blacklist"] = true;
-    if (q.value.HasMember("blacklist")) {
-      query.options["blacklist"] = JSON::valueToBool(q.value["blacklist"]);
+    query.options["denylist"] = true;
+    if (q.value.HasMember("denylist")) {
+      query.options["denylist"] = JSON::valueToBool(q.value["denylist"]);
+    } else if (q.value.HasMember("blacklist")) {
+      query.options["denylist"] = JSON::valueToBool(q.value["blacklist"]);
+      LOG(WARNING)
+          << "Query uses deprecated 'blacklist' option. Use 'denylist': "
+          << q.name.GetString();
     }
 
     schedule_.emplace(std::make_pair(q.name.GetString(), std::move(query)));

osquery/config/tests/config_tests.cpp

@@ -47,10 +47,9 @@ DECLARE_bool(disable_database);
 
 namespace fs = boost::filesystem;
 
-// Blacklist testing methods, internal to config implementations.
-extern void restoreScheduleBlacklist(std::map<std::string, size_t>& blacklist);
-extern void saveScheduleBlacklist(
-    const std::map<std::string, size_t>& blacklist);
+// Denylist testing methods, internal to config implementations.
+extern void restoreScheduleDenylist(std::map<std::string, size_t>& denylist);
+extern void saveScheduleDenylist(const std::map<std::string, size_t>& denylist);
 
 class ConfigTests : public testing::Test {
  public:
@@ -272,32 +271,32 @@ TEST_F(ConfigTests, test_strip_comments) {
   EXPECT_TRUE(get().update({{"data", json_comments}}));
 }
 
-TEST_F(ConfigTests, test_schedule_blacklist) {
+TEST_F(ConfigTests, test_schedule_denylist) {
   auto current_time = getUnixTime();
-  std::map<std::string, size_t> blacklist;
-  saveScheduleBlacklist(blacklist);
-  restoreScheduleBlacklist(blacklist);
-  EXPECT_EQ(blacklist.size(), 0U);
+  std::map<std::string, size_t> denylist;
+  saveScheduleDenylist(denylist);
+  restoreScheduleDenylist(denylist);
+  EXPECT_EQ(denylist.size(), 0U);
 
   // Create some entries.
-  blacklist["test_1"] = current_time * 2;
-  blacklist["test_2"] = current_time * 3;
-  saveScheduleBlacklist(blacklist);
-  blacklist.clear();
-  restoreScheduleBlacklist(blacklist);
-  ASSERT_EQ(blacklist.count("test_1"), 1U);
-  ASSERT_EQ(blacklist.count("test_2"), 1U);
-  EXPECT_EQ(blacklist.at("test_1"), current_time * 2);
-  EXPECT_EQ(blacklist.at("test_2"), current_time * 3);
+  denylist["test_1"] = current_time * 2;
+  denylist["test_2"] = current_time * 3;
+  saveScheduleDenylist(denylist);
+  denylist.clear();
+  restoreScheduleDenylist(denylist);
+  ASSERT_EQ(denylist.count("test_1"), 1U);
+  ASSERT_EQ(denylist.count("test_2"), 1U);
+  EXPECT_EQ(denylist.at("test_1"), current_time * 2);
+  EXPECT_EQ(denylist.at("test_2"), current_time * 3);
 
   // Now save an expired query.
-  blacklist["test_1"] = 1;
-  saveScheduleBlacklist(blacklist);
-  blacklist.clear();
+  denylist["test_1"] = 1;
+  saveScheduleDenylist(denylist);
+  denylist.clear();
 
   // When restoring, the values below the current time will not be included.
-  restoreScheduleBlacklist(blacklist);
-  EXPECT_EQ(blacklist.size(), 1U);
+  restoreScheduleDenylist(denylist);
+  EXPECT_EQ(denylist.size(), 1U);
 }
 
 TEST_F(ConfigTests, test_pack_noninline) {
@@ -407,14 +406,14 @@ TEST_F(ConfigTests, test_get_scheduled_queries) {
       << ") should equal " << expected_size;
   ASSERT_FALSE(query_names.empty());
 
-  // Construct a schedule blacklist and place the first scheduled query.
-  std::map<std::string, size_t> blacklist;
+  // Construct a schedule denylist and place the first scheduled query.
+  std::map<std::string, size_t> denylist;
   std::string query_name = query_names[0];
-  blacklist[query_name] = getUnixTime() * 2;
-  saveScheduleBlacklist(blacklist);
-  blacklist.clear();
+  denylist[query_name] = getUnixTime() * 2;
+  saveScheduleDenylist(denylist);
+  denylist.clear();
 
-  // When the blacklist is edited externally, the config must re-read.
+  // When the denylist is edited externally, the config must re-read.
   get().reset();
   get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc());
 
@@ -430,42 +429,42 @@ TEST_F(ConfigTests, test_get_scheduled_queries) {
 
   // Try again, this time requesting scheduled queries.
   query_names.clear();
-  bool blacklisted = false;
-  get().scheduledQueries(([&blacklisted, &query_names, &query_name](
+  bool denylisted = false;
+  get().scheduledQueries(([&denylisted, &query_names, &query_name](
                               std::string name, const ScheduledQuery& query) {
                            if (name == query_name) {
-                             // Only populate the query we've blacklisted.
+                             // Only populate the query we've denylisted.
                              query_names.push_back(std::move(name));
-                             blacklisted = query.blacklisted;
+                             denylisted = query.denylisted;
                            }
                          }),
                          true);
   ASSERT_EQ(query_names.size(), std::size_t{1});
   EXPECT_EQ(query_names[0], query_name);
-  EXPECT_TRUE(blacklisted);
+  EXPECT_TRUE(denylisted);
 }
 
-TEST_F(ConfigTests, test_nonblacklist_query) {
-  std::map<std::string, size_t> blacklist;
+TEST_F(ConfigTests, test_nondenylist_query) {
+  std::map<std::string, size_t> denylist;
 
-  const std::string kConfigTestNonBlacklistQuery{
+  const std::string kConfigTestNonDenylistQuery{
       "pack_unrestricted_pack_process_heartbeat"};
 
-  blacklist[kConfigTestNonBlacklistQuery] = getUnixTime() * 2;
-  saveScheduleBlacklist(blacklist);
+  denylist[kConfigTestNonDenylistQuery] = getUnixTime() * 2;
+  saveScheduleDenylist(denylist);
 
   get().reset();
   get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc());
 
-  std::map<std::string, bool> blacklisted;
+  std::map<std::string, bool> denylisted;
   get().scheduledQueries(
-      ([&blacklisted](std::string name, const ScheduledQuery& query) {
-        blacklisted[name] = query.blacklisted;
+      ([&denylisted](std::string name, const ScheduledQuery& query) {
+        denylisted[name] = query.denylisted;
       }));
 
-  // This query cannot be blacklisted.
-  auto query = blacklisted.find(kConfigTestNonBlacklistQuery);
-  ASSERT_NE(query, blacklisted.end());
+  // This query cannot be denylisted.
+  auto query = denylisted.find(kConfigTestNonDenylistQuery);
+  ASSERT_NE(query, denylisted.end());
   EXPECT_FALSE(query->second);
 }
 

osquery/core/sql/scheduled_query.h

@@ -8,8 +8,8 @@
 
 #pragma once
 
-#include <string>
 #include <map>
+#include <string>
 
 #include <osquery/utils/only_movable.h>
 
@@ -41,13 +41,13 @@ struct ScheduledQuery : private only_movable {
   size_t splayed_interval{0};
 
   /**
-   * @brief Queries are blacklisted based on logic in the configuration.
+   * @brief Queries are denylisted based on logic in the configuration.
    *
-   * Most calls to inspect scheduled queries will abstract away the blacklisting
-   * concept and only return non-blacklisted queries. The config may be asked
+   * Most calls to inspect scheduled queries will abstract away the denylisting
+   * concept and only return non-denylisted queries. The config may be asked
    * to return all queries, thus it is important to capture this optional data.
    */
-  bool blacklisted{false};
+  bool denylisted{false};
 
   /// Set of query options.
   std::map<std::string, bool> options;

osquery/tables/utility/osquery.cpp

@@ -237,7 +237,7 @@ QueryData genOsquerySchedule(QueryContext& context) {
         r["name"] = name;
         r["query"] = query.query;
         r["interval"] = INTEGER(query.interval);
-        r["blacklisted"] = (query.blacklisted) ? "1" : "0";
+        r["denylisted"] = (query.denylisted) ? "1" : "0";
         // Set default (0) values for each query if it has not yet executed.
         r["executions"] = "0";
         r["wall_time"] = "0";

packs/osquery-monitoring.conf

@@ -1,10 +1,10 @@
 {
   "queries": {
     "schedule": {
-      "query": "select name, interval, executions, output_size, wall_time, (user_time/executions) as avg_user_time, (system_time/executions) as avg_system_time, average_memory, last_executed, blacklisted from osquery_schedule;",
+      "query": "select name, interval, executions, output_size, wall_time, (user_time/executions) as avg_user_time, (system_time/executions) as avg_system_time, average_memory, last_executed, denylisted from osquery_schedule;",
       "interval": 7200,
       "removed": false,
-      "blacklist": false,
+      "denylist": false,
       "version": "2.11.0",
       "description": "Report performance for every query within packs and the general schedule."
     },
@@ -12,7 +12,7 @@
       "query": "select name, publisher, type, subscriptions, events, active from osquery_events;",
       "interval": 86400,
       "removed": false,
-      "blacklist": false,
+      "denylist": false,
       "version": "1.5.3",
       "description": "Report event publisher health and track event counters."
     },
@@ -20,7 +20,7 @@
       "query": "select i.*, p.resident_size, p.user_time, p.system_time, time.minutes as counter from osquery_info i, processes p, time where p.pid = i.pid;",
       "interval": 600,
       "removed": false,
-      "blacklist": false,
+      "denylist": false,
       "version": "1.2.2",
       "description": "A heartbeat counter that reports general performance (CPU, memory) and version."
     }

specs/utility/osquery_schedule.table

@@ -8,7 +8,8 @@ schema([
     Column("executions", BIGINT, "Number of times the query was executed"),
     Column("last_executed", BIGINT,
       "UNIX time stamp in seconds of the last completed execution"),
-    Column("blacklisted", INTEGER, "1 if the query is blacklisted else 0"),
+    Column("denylisted", INTEGER, "1 if the query is denylisted else 0",
+        aliases=["blacklisted"]), # 'blacklist' now deprecated
     Column("output_size", BIGINT,
       "Total number of bytes generated by the query"),
     Column("wall_time", BIGINT, "Total wall time spent executing"),

tests/integration/tables/osquery_schedule.cpp

@@ -15,10 +15,10 @@ namespace osquery {
 namespace table_tests {
 
 class osquerySchedule : public testing::Test {
-  protected:
-    void SetUp() override {
-      setUpEnvironment();
-    }
+ protected:
+  void SetUp() override {
+    setUpEnvironment();
+  }
 };
 
 TEST_F(osquerySchedule, test_sanity) {
@@ -37,7 +37,7 @@ TEST_F(osquerySchedule, test_sanity) {
   //      {"interval", IntType}
   //      {"executions", IntType}
   //      {"last_executed", IntType}
-  //      {"blacklisted", IntType}
+  //      {"denylisted", IntType}
   //      {"output_size", IntType}
   //      {"wall_time", IntType}
   //      {"user_time", IntType}

tools/tests/configs/test_inline_pack.conf

@@ -12,7 +12,7 @@
         "process_heartbeat": {
           "query": "select * from osquery_info",
           "interval": 3600,
-          "blacklist": false
+          "denylist": false
         }
       },
       "file_paths": {