Implement azure metadata and tags tables (#5434)

docs/wiki/deployment/configuration.md

@@ -445,6 +445,10 @@ SELECT * FROM kernel_hashes WHERE kernel_binary NOT LIKE "%apple%";
 
 There are two tables that provide EC2 instance related information. On non-EC2 instances these tables return empty results. `ec2_instance_metadata` table contains instance meta data information. `ec2_instance_tags` returns tags for the EC2 instance osquery is running on. Retrieving tags for EC2 instance requires authentication and appropriate permission. There are multiple ways credentials can be provided to osquery. See [AWS logging configuration](../deployment/aws-logging.md#configuration) for configuring credentials. AWS region (`--aws_region`) argument is not required and will be ignored by `ec2_instance_tags` implementation. The credentials configured should have permission to perform `ec2:DescribeTags` action.
 
+### Azure
+
+Like EC2, there are two tables that provide Azure instance related information. These tables query a REST endpoint that may or may not exist outside of Azure, so querying them outside of Azure is not recommended. The `azure_instance_metadata` table contains general metadata for the instance. The `azure_instance_tags` table contains tags for the Azure instance that osquery is running on. These tables don't require any special Azure permissions or credentials.
+
 ### Decorator queries
 
 Decorator queries exist in osquery versions 1.7.3+ and are used to add additional "decorations" to results and snapshot logs. There are three types of decorator queries based on when and how you want the decoration data.

osquery/tables/CMakeLists.txt

@@ -39,7 +39,13 @@ function(generateOsqueryTablesTableimplementations)
 
   if(DEFINED PLATFORM_LINUX)
     target_link_libraries(osquery_tables_tableimplementations INTERFACE
-      osquery_tables_cloud
+      osquery_tables_cloud_aws
+    )
+  endif()
+
+  if(DEFINED PLATFORM_LINUX OR DEFINED PLATFORM_WINDOWS)
+    target_link_libraries(osquery_tables_tableimplementations INTERFACE
+      osquery_tables_cloud_azure
     )
   endif()
 

osquery/tables/cloud/BUCK

@@ -11,14 +11,17 @@ load("//tools/build_defs/oss/osquery:third_party.bzl", "osquery_tp_target")
 osquery_cxx_library(
     name = "cloud",
     srcs = [
-        "ec2_instance_metadata.cpp",
-        "ec2_instance_tags.cpp",
+        "aws/ec2_instance_metadata.cpp",
+        "aws/ec2_instance_tags.cpp",
+        "azure/azure_instance_metadata.cpp",
+        "azure/azure_instance_tags.cpp",
     ],
     visibility = ["PUBLIC"],
     deps = [
         osquery_target("osquery:headers"),
         osquery_target("osquery/logger:logger"),
         osquery_target("osquery/utils/aws:aws"),
+        osquery_target("osquery/utils/azure:azure"),
         osquery_tp_target("boost"),
     ],
 )

osquery/tables/cloud/CMakeLists.txt

@@ -6,17 +6,21 @@
 
 function(osqueryTablesCloudMain)
   if(DEFINED PLATFORM_LINUX)
-    generateOsqueryTablesCloud()
+    generateOsqueryTablesCloudAws()
+  endif()
+
+  if(DEFINED PLATFORM_LINUX OR DEFINED PLATFORM_WINDOWS)
+    generateOsqueryTablesCloudAzure()
   endif()
 endfunction()
 
-function(generateOsqueryTablesCloud)
-  add_osquery_library(osquery_tables_cloud EXCLUDE_FROM_ALL
-    ec2_instance_metadata.cpp
-    ec2_instance_tags.cpp
+function(generateOsqueryTablesCloudAws)
+  add_osquery_library(osquery_tables_cloud_aws EXCLUDE_FROM_ALL
+    aws/ec2_instance_metadata.cpp
+    aws/ec2_instance_tags.cpp
   )
 
-  target_link_libraries(osquery_tables_cloud PUBLIC
+  target_link_libraries(osquery_tables_cloud_aws PUBLIC
     osquery_cxx_settings
     osquery_headers
     osquery_logger
@@ -27,4 +31,19 @@ function(generateOsqueryTablesCloud)
   )
 endfunction()
 
+function(generateOsqueryTablesCloudAzure)
+  add_osquery_library(osquery_tables_cloud_azure EXCLUDE_FROM_ALL
+    azure/azure_instance_metadata.cpp
+    azure/azure_instance_tags.cpp
+  )
+
+  target_link_libraries(osquery_tables_cloud_azure PUBLIC
+    osquery_cxx_settings
+    osquery_headers
+    osquery_logger
+    osquery_utils_azure
+    thirdparty_boost
+  )
+endfunction()
+
 osqueryTablesCloudMain()

osquery/tables/cloud/aws/ec2_instance_metadata.cpp

@@ -254,5 +254,5 @@ QueryData genEc2Metadata(QueryContext& context) {
   results.push_back(r);
   return results;
 }
-}
-}
+} // namespace tables
+} // namespace osquery

osquery/tables/cloud/aws/ec2_instance_tags.cpp

@@ -62,5 +62,5 @@ QueryData genEc2InstanceTags(QueryContext& context) {
 
   return results;
 }
-}
-}
+} // namespace tables
+} // namespace osquery

osquery/tables/cloud/azure/azure_instance_metadata.cpp

@@ -0,0 +1,54 @@
+/**
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under both the Apache 2.0 license (found in the
+ *  LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ *  in the COPYING file in the root directory of this source tree).
+ *  You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <osquery/core.h>
+#include <osquery/logger.h>
+#include <osquery/tables.h>
+#include <osquery/utils/azure/azure_util.h>
+
+namespace osquery {
+namespace tables {
+
+QueryData genAzureMetadata(QueryContext& context) {
+  QueryData results;
+  Row r;
+
+  JSON doc;
+
+  Status s = fetchAzureMetadata(doc);
+
+  if (!s.ok()) {
+    TLOG << "Couldn't fetch metadata: " << s.what();
+    return results;
+  }
+
+  r["vm_id"] = getAzureKey(doc, "vmId");
+  r["location"] = getAzureKey(doc, "location");
+  r["name"] = getAzureKey(doc, "name");
+  r["offer"] = getAzureKey(doc, "offer");
+  r["publisher"] = getAzureKey(doc, "publisher");
+  r["sku"] = getAzureKey(doc, "sku");
+  r["version"] = getAzureKey(doc, "version");
+  r["os_type"] = getAzureKey(doc, "osType");
+  r["platform_update_domain"] = getAzureKey(doc, "platformUpdateDomain");
+  r["platform_fault_domain"] = getAzureKey(doc, "platformFaultDomain");
+  r["vm_size"] = getAzureKey(doc, "vmSize");
+  r["subscription_id"] = getAzureKey(doc, "subscriptionId");
+  r["resource_group_name"] = getAzureKey(doc, "resourceGroupName");
+  r["placement_group_id"] = getAzureKey(doc, "placementGroupId");
+  r["vm_scale_set_name"] = getAzureKey(doc, "vmScaleSetName");
+  r["zone"] = getAzureKey(doc, "zone");
+
+  results.push_back(r);
+
+  return results;
+}
+} // namespace tables
+} // namespace osquery

osquery/tables/cloud/azure/azure_instance_tags.cpp

@@ -0,0 +1,61 @@
+/**
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under both the Apache 2.0 license (found in the
+ *  LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ *  in the COPYING file in the root directory of this source tree).
+ *  You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <boost/algorithm/string.hpp>
+
+#include <osquery/core.h>
+#include <osquery/logger.h>
+#include <osquery/tables.h>
+#include <osquery/utils/azure/azure_util.h>
+
+namespace osquery {
+namespace tables {
+
+QueryData genAzureTags(QueryContext& context) {
+  QueryData results;
+  JSON doc;
+
+  Status s = fetchAzureMetadata(doc);
+
+  if (!s.ok()) {
+    TLOG << "Couldn't fetch metadata: " << s.what();
+    return results;
+  }
+
+  auto tags_str = getAzureKey(doc, "tags");
+  auto vm_id = getAzureKey(doc, "vmId");
+  std::vector<std::string> tags;
+
+  boost::split(tags, tags_str, boost::is_any_of(";"));
+
+  for (auto& tag : tags) {
+    Row r;
+
+    auto colon = tag.find_first_of(':');
+
+    // This shouldn't ever happen, but it doesn't hurt to be safe.
+    if (colon == std::string::npos) {
+      continue;
+    }
+
+    auto key = tag.substr(0, colon);
+    auto value = tag.substr(colon + 1);
+
+    r["vm_id"] = vm_id;
+    r["key"] = key;
+    r["value"] = value;
+    results.push_back(r);
+  }
+
+  return results;
+}
+
+} // namespace tables
+} // namespace osquery

osquery/utils/CMakeLists.txt

@@ -6,6 +6,7 @@
 
 function(osqueryUtilsMain)
   add_subdirectory("aws")
+  add_subdirectory("azure")
   add_subdirectory("caches")
   add_subdirectory("system")
   add_subdirectory("status")

osquery/utils/azure/BUCK

@@ -0,0 +1,29 @@
+#  Copyright (c) 2014-present, Facebook, Inc.
+#  All rights reserved.
+#
+#  This source code is licensed under both the Apache 2.0 license (found in the
+#  LICENSE file in the root directory of this source tree) and the GPLv2 (found
+#  in the COPYING file in the root directory of this source tree).
+#  You may select, at your option, one of the above-listed licenses.
+
+load("//tools/build_defs/oss/osquery:cxx.bzl", "osquery_cxx_library")
+load("//tools/build_defs/oss/osquery:native.bzl", "osquery_target")
+load("//tools/build_defs/oss/osquery:third_party.bzl", "osquery_tp_target")
+
+osquery_cxx_library(
+    name = "azure",
+    srcs = [
+        "azure_util.cpp",
+    ],
+    header_namespace = "osquery/utils/azure",
+    exported_headers = [
+        "azure_util.h",
+    ],
+    visibility = ["PUBLIC"],
+    deps = [
+        osquery_target("osquery/remote:http_client"),
+        osquery_target("osquery/remote/transports:transports_tls"),
+        osquery_target("osquery/utils/json:json"),
+        osquery_target("osquery/utils/status:status"),
+    ],
+)

osquery/utils/azure/CMakeLists.txt

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+
+function(osqueryUtilsAzureMain)
+  generateOsqueryUtilsAzure()
+endfunction()
+
+function(generateOsqueryUtilsAzure)
+  add_osquery_library(osquery_utils_azure EXCLUDE_FROM_ALL
+    azure_util.cpp
+  )
+
+  target_link_libraries(osquery_utils_azure PUBLIC
+    osquery_cxx_settings
+    osquery_remote_httpclient
+    osquery_remote_transports_transportstls
+    osquery_utils_json
+    osquery_utils_status
+  )
+
+  set(public_header_files
+    azure_util.h
+  )
+
+  generateIncludeNamespace(osquery_utils_azure "osquery/utils/azure" "FILE_ONLY" ${public_header_files})
+
+  add_test(NAME osquery_utils_aws_tests-test COMMAND osquery_utils_aws_tests-test)
+
+  set_tests_properties(
+    osquery_utils_aws_tests-test
+    PROPERTIES ENVIRONMENT "TEST_CONF_FILES_DIR=${TEST_CONFIGS_DIR}"
+  )
+endfunction()
+
+osqueryUtilsAzureMain()

osquery/utils/azure/azure_util.cpp

@@ -0,0 +1,114 @@
+/**
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under both the Apache 2.0 license (found in the
+ *  LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ *  in the COPYING file in the root directory of this source tree).
+ *  You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <osquery/core.h>
+#include <osquery/logger.h>
+
+#include <osquery/filesystem/filesystem.h>
+#include <osquery/remote/http_client.h>
+#include <osquery/utils/azure/azure_util.h>
+#include <osquery/utils/info/platform_type.h>
+#include <osquery/utils/json/json.h>
+
+namespace http = osquery::http;
+namespace fs = boost::filesystem;
+
+namespace osquery {
+
+// 2018-02-01 is supported across all Azure regions, according to MS.
+const std::string kAzureMetadataEndpoint =
+    "http://169.254.169.254/metadata/instance/compute?api-version=2018-02-01";
+
+// 3 seconds should be more than enough time for the metadata endpoint to
+// respond.
+const int kAzureMetadataTimeout = 3;
+
+static bool isAzureInstance() {
+  static std::atomic<bool> checked(false);
+  static std::atomic<bool> is_azure_instance(false);
+
+  if (checked) {
+    return is_azure_instance;
+  }
+
+  static std::once_flag once_flag;
+  std::call_once(once_flag, []() {
+    if (checked) {
+      return;
+    }
+
+    checked = true;
+
+    if (isPlatform(PlatformType::TYPE_WINDOWS)) {
+      is_azure_instance = pathExists(fs::path("C:\\WindowsAzure")).ok();
+    } else if (isPlatform(PlatformType::TYPE_POSIX)) {
+      is_azure_instance = pathExists(fs::path("/var/log/waagent.log")).ok();
+    } else {
+      TLOG << "Unsupported Azure platform: " << OSQUERY_PLATFORM;
+      is_azure_instance = false;
+    }
+  });
+
+  return is_azure_instance;
+}
+
+std::string getAzureKey(JSON& doc, const std::string& key) {
+  if (!doc.doc().HasMember(key)) {
+    return {};
+  }
+
+  if (!doc.doc()[key].IsString()) {
+    return {};
+  }
+
+  return doc.doc()[key].GetString();
+}
+
+Status fetchAzureMetadata(JSON& doc) {
+  if (!isAzureInstance()) {
+    return Status(1, "Not an Azure instance");
+  }
+
+  http::Request request(kAzureMetadataEndpoint);
+  http::Client::Options opts;
+  http::Response response;
+
+  opts.timeout(kAzureMetadataTimeout);
+  http::Client client(opts);
+
+  request << http::Request::Header("Metadata", "true");
+
+  try {
+    response = client.get(request);
+  } catch (const std::system_error& e) {
+    return Status(
+        1, "Couldn't request " + kAzureMetadataEndpoint + ": " + e.what());
+  }
+
+  // Non-200s can indicate a variety of conditions, so report them.
+  if (response.result_int() != 200) {
+    return Status(1,
+                  std::string("Azure metadata service responded with ") +
+                      std::to_string(response.result_int()));
+  }
+
+  auto s = doc.fromString(response.body());
+  if (!s.ok()) {
+    return s;
+  }
+
+  if (!doc.doc().IsObject()) {
+    return Status(1, "Azure metadata service response isn't a JSON object");
+  }
+
+  return s;
+}
+
+} // namespace osquery

osquery/utils/azure/azure_util.h

@@ -0,0 +1,22 @@
+/**
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under both the Apache 2.0 license (found in the
+ *  LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ *  in the COPYING file in the root directory of this source tree).
+ *  You may select, at your option, one of the above-listed licenses.
+ */
+
+#pragma once
+
+#include <osquery/utils/json/json.h>
+#include <osquery/utils/status/status.h>
+
+namespace osquery {
+
+std::string getAzureKey(JSON& doc, const std::string& key);
+
+Status fetchAzureMetadata(JSON& doc);
+
+} // namespace osquery

specs/BUCK

@@ -814,6 +814,8 @@ osquery_gentable_cxx_library(
         ),
     ],
     spec_files = [
+        "azure_instance_metadata.table",
+        "azure_instance_tags.table",
         "carbon_black_info.table",
         "carves.table",
         "chrome_extensions.table",

specs/azure_instance_metadata.table

@@ -0,0 +1,25 @@
+table_name("azure_instance_metadata")
+description("Azure instance metadata.")
+schema([
+    Column("location", TEXT, "Azure Region the VM is running in"),
+    Column("name", TEXT, "Name of the VM"),
+    Column("offer", TEXT, "Offer information for the VM image (Azure image gallery VMs only)"),
+    Column("publisher", TEXT, "Publisher of the VM image"),
+    Column("sku", TEXT, "SKU for the VM image"),
+    Column("version", TEXT, "Version of the VM image"),
+    Column("os_type", TEXT, "Linux or Windows"),
+    Column("platform_update_domain", TEXT, "Update domain the VM is running in"),
+    Column("platform_fault_domain", TEXT, "Fault domain the VM is running in"),
+    Column("vm_id", TEXT, "Unique identifier for the VM", index=True),
+    Column("vm_size", TEXT, "VM size"),
+    Column("subscription_id", TEXT, "Azure subscription for the VM"),
+    Column("resource_group_name", TEXT, "Resource group for the VM"),
+    Column("placement_group_id", TEXT, "Placement group for the VM scale set"),
+    Column("vm_scale_set_name", TEXT, "VM scale set name"),
+    Column("zone", TEXT, "Availability zone of the VM"),
+])
+attributes(cacheable=True)
+implementation("cloud/azure_metadata@genAzureMetadata")
+examples([
+    "select * from ec2_instance_metadata"
+])

specs/azure_instance_tags.table

@@ -0,0 +1,12 @@
+table_name("azure_instance_tags")
+description("Azure instance tags.")
+schema([
+    Column("vm_id", TEXT, "Unique identifier for the VM"),
+    Column("key", TEXT, "The tag key"),
+    Column("value", TEXT, "The tag value"),
+])
+attributes(cacheable=True)
+implementation("cloud/azure_metadata@genAzureTags")
+examples([
+    "select * from ec2_instance_tags"
+])