logging: Add Firehose/Kinesis support to Windows (#3641)

osquery/logger/CMakeLists.txt

@@ -36,7 +36,7 @@ set(OSQUERY_LOGGER_PLUGIN_TESTS
 # and others, which use a Glog sink. They must be tested in tandem.
 ADD_OSQUERY_TEST_ADDITIONAL(${OSQUERY_LOGGER_PLUGIN_TESTS})
 
-if(NOT SKIP_AWS AND NOT WINDOWS)
+if(NOT SKIP_AWS)
   set(OSQUERY_LOGGER_AWS_PLUGINS
     "plugins/aws_firehose.cpp"
     "plugins/aws_kinesis.cpp"
@@ -53,6 +53,11 @@ if(NOT SKIP_AWS AND NOT WINDOWS)
 
   ADD_OSQUERY_TEST_ADDITIONAL(${OSQUERY_LOGGER_AWS_PLUGIN_TESTS})
 
+  if(WINDOWS)
+    ADD_OSQUERY_LINK_CORE("UserEnv.lib")
+    ADD_OSQUERY_LINK_CORE("bcrypt.lib")
+  endif()
+
   ADD_OSQUERY_LINK_ADDITIONAL("aws-cpp-sdk-kinesis")
   ADD_OSQUERY_LINK_ADDITIONAL("aws-cpp-sdk-firehose")
 endif()

osquery/logger/plugins/aws_firehose.cpp

@@ -12,6 +12,7 @@
 
 #include <boost/algorithm/string/join.hpp>
 
+#include <aws/core/client/AWSError.h>
 #include <aws/core/utils/Outcome.h>
 #include <aws/firehose/model/PutRecordBatchRequest.h>
 #include <aws/firehose/model/PutRecordBatchResult.h>

osquery/logger/plugins/aws_kinesis.cpp

@@ -13,6 +13,7 @@
 #include <iterator>
 #include <thread>
 
+#include <aws/core/client/AWSError.h>
 #include <aws/core/utils/Outcome.h>
 #include <aws/kinesis/model/PutRecordsRequest.h>
 #include <aws/kinesis/model/PutRecordsResult.h>

osquery/logger/plugins/aws_log_forwarder.h

@@ -21,6 +21,14 @@
 #include "osquery/logger/plugins/buffered.h"
 #include "osquery/utils/aws_util.h"
 
+// This macro from the Windows headers is used to map the GetMessage
+// name to either GetMessageW or GetMessageA depending on the UNICODE
+// define. We have to undefine this because it causes a method in the
+// AWS sdk to be renamed, causing a compilation error.
+#if defined(WINDOWS) && defined(GetMessage)
+#undef GetMessage
+#endif
+
 namespace osquery {
 template <typename RecordType,
           typename ClientType,

osquery/utils/CMakeLists.txt

@@ -5,11 +5,16 @@
 #  LICENSE file in the root directory of this source tree. An additional grant
 #  of patent rights can be found in the PATENTS file in the same directory.
 
-if(NOT SKIP_AWS AND NOT WINDOWS)
+if(NOT SKIP_AWS)
   set(OSQUERY_AWS_UTIL
     "aws_util.cpp"
   )
 
+  if(WINDOWS)
+    # This is a fix for a conversion error in the uri.hpp file of cpp-netlib
+    set_source_files_properties("aws_util.cpp" PROPERTIES COMPILE_FLAGS "/wd4244")
+  endif()
+
   ADD_OSQUERY_LIBRARY_ADDITIONAL(
     osquery_aws_util
     ${OSQUERY_AWS_UTIL}

osquery/utils/aws_util.cpp

@@ -231,7 +231,7 @@ OsquerySTSAWSCredentialsProvider::GetAWSCredentials() {
     Model::AssumeRoleRequest sts_r;
     sts_r.SetRoleArn(FLAGS_aws_sts_arn_role);
     sts_r.SetRoleSessionName(FLAGS_aws_sts_session_name);
-    sts_r.SetDurationSeconds(FLAGS_aws_sts_timeout);
+    sts_r.SetDurationSeconds(static_cast<int>(FLAGS_aws_sts_timeout));
 
     // Pull our STS credentials.
     Model::AssumeRoleOutcome sts_outcome = client_->AssumeRole(sts_r);
@@ -270,8 +270,12 @@ OsqueryAWSCredentialsProviderChain::OsqueryAWSCredentialsProviderChain(bool sts)
   AddProvider(std::make_shared<Aws::Auth::EnvironmentAWSCredentialsProvider>());
   AddProvider(
       std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>());
+
+// This is disabled on Windows because it causes a crash
+#if !defined(WINDOWS)
   AddProvider(
       std::make_shared<Aws::Auth::InstanceProfileCredentialsProvider>());
+#endif
 }
 
 Status getAWSRegionFromProfile(std::string& region) {
@@ -324,7 +328,7 @@ void initAwsSdk() {
       };
       Aws::InitAPI(options);
     });
-  } catch (const std::system_error& e) {
+  } catch (const std::system_error&) {
     LOG(ERROR) << "call_once was not executed for initAwsSdk";
   }
 }

osquery/utils/tests/aws_util_tests.cpp

@@ -12,6 +12,7 @@
 
 #include <gtest/gtest.h>
 
+#include <osquery/core/process.h>
 #include <osquery/logger.h>
 
 #include "osquery/tests/test_util.h"
@@ -39,11 +40,11 @@ class AwsUtilTests : public testing::Test {
 TEST_F(AwsUtilTests, test_get_credentials) {
   // Set a good path for the credentials file
   std::string profile_path = kTestDataPath + "/aws/credentials";
-  setenv(kAwsProfileFileEnvVar, profile_path.c_str(), true);
+  setEnvVar(kAwsProfileFileEnvVar, profile_path.c_str());
 
   // Clear any values for the other AWS env vars
-  unsetenv(kAwsAccessKeyEnvVar);
+  unsetEnvVar(kAwsAccessKeyEnvVar);
-  unsetenv(kAwsSecretKeyEnvVar);
+  unsetEnvVar(kAwsSecretKeyEnvVar);
 
   OsqueryAWSCredentialsProviderChain provider;
   Aws::Auth::AWSCredentials credentials("", "");
@@ -64,40 +65,44 @@ TEST_F(AwsUtilTests, test_get_credentials) {
   ASSERT_EQ("FLAG_ACCESS_KEY_ID", credentials.GetAWSAccessKeyId());
   ASSERT_EQ("flag_secret_key", credentials.GetAWSSecretKey());
 
-  FLAGS_aws_access_key_id = "";
+  // Profiles are not working on Windows; see the constructor of
-  FLAGS_aws_secret_access_key = "flag_secret_key";
+  // OsqueryAWSCredentialsProviderChain for more information
-  // With the flags set improperly, the profile should be used
+  if (!isPlatform(PlatformType::TYPE_WINDOWS)) {
-  provider = OsqueryAWSCredentialsProviderChain();
+    FLAGS_aws_access_key_id = "";
-  credentials = provider.GetAWSCredentials();
+    FLAGS_aws_secret_access_key = "flag_secret_key";
-  ASSERT_EQ("DEFAULT_ACCESS_KEY_ID", credentials.GetAWSAccessKeyId());
+    // With the flags set improperly, the profile should be used
-  ASSERT_EQ("default_secret_key", credentials.GetAWSSecretKey());
+    provider = OsqueryAWSCredentialsProviderChain();
+    credentials = provider.GetAWSCredentials();
+    ASSERT_EQ("DEFAULT_ACCESS_KEY_ID", credentials.GetAWSAccessKeyId());
+    ASSERT_EQ("default_secret_key", credentials.GetAWSSecretKey());
 
-  FLAGS_aws_access_key_id = "FLAG_ACCESS_KEY_ID";
+    FLAGS_aws_access_key_id = "FLAG_ACCESS_KEY_ID";
-  FLAGS_aws_secret_access_key = "";
+    FLAGS_aws_secret_access_key = "";
-  // With the flags set improperly, the profile should be used
+    // With the flags set improperly, the profile should be used
-  provider = OsqueryAWSCredentialsProviderChain();
+    provider = OsqueryAWSCredentialsProviderChain();
-  credentials = provider.GetAWSCredentials();
+    credentials = provider.GetAWSCredentials();
-  ASSERT_EQ("DEFAULT_ACCESS_KEY_ID", credentials.GetAWSAccessKeyId());
+    ASSERT_EQ("DEFAULT_ACCESS_KEY_ID", credentials.GetAWSAccessKeyId());
-  ASSERT_EQ("default_secret_key", credentials.GetAWSSecretKey());
+    ASSERT_EQ("default_secret_key", credentials.GetAWSSecretKey());
 
-  // Clear flags
+    // Clear flags
-  FLAGS_aws_access_key_id = "";
+    FLAGS_aws_access_key_id = "";
-  FLAGS_aws_secret_access_key = "";
+    FLAGS_aws_secret_access_key = "";
 
-  setenv(kAwsAccessKeyEnvVar, "ENV_ACCESS_KEY_ID", true);
+    setEnvVar(kAwsAccessKeyEnvVar, "ENV_ACCESS_KEY_ID");
-  setenv(kAwsSecretKeyEnvVar, "env_secret_key", true);
+    setEnvVar(kAwsSecretKeyEnvVar, "env_secret_key");
-  // Now env variables should be the primary source
+    // Now env variables should be the primary source
-  provider = OsqueryAWSCredentialsProviderChain();
+    provider = OsqueryAWSCredentialsProviderChain();
-  credentials = provider.GetAWSCredentials();
+    credentials = provider.GetAWSCredentials();
-  ASSERT_EQ("ENV_ACCESS_KEY_ID", credentials.GetAWSAccessKeyId());
+    ASSERT_EQ("ENV_ACCESS_KEY_ID", credentials.GetAWSAccessKeyId());
-  ASSERT_EQ("env_secret_key", credentials.GetAWSSecretKey());
+    ASSERT_EQ("env_secret_key", credentials.GetAWSSecretKey());
 
-  FLAGS_aws_profile_name = "test";
+    FLAGS_aws_profile_name = "test";
-  provider = OsqueryAWSCredentialsProviderChain();
+    provider = OsqueryAWSCredentialsProviderChain();
-  credentials = provider.GetAWSCredentials();
+    credentials = provider.GetAWSCredentials();
-  // Now the "test" profile should take precedence
+    // Now the "test" profile should take precedence
-  ASSERT_EQ("TEST_ACCESS_KEY_ID", credentials.GetAWSAccessKeyId());
+    ASSERT_EQ("TEST_ACCESS_KEY_ID", credentials.GetAWSAccessKeyId());
-  ASSERT_EQ("test_secret_key", credentials.GetAWSSecretKey());
+    ASSERT_EQ("test_secret_key", credentials.GetAWSSecretKey());
+  }
 }
 
 TEST_F(AwsUtilTests, test_get_region) {
@@ -132,31 +137,36 @@ TEST_F(AwsUtilTests, test_get_region) {
 
   // Test no credential file, should default to us-east-1
   std::string profile_path = kTestDataPath + "credentials";
-  setenv(kAwsProfileFileEnvVar, profile_path.c_str(), true);
+  setEnvVar(kAwsProfileFileEnvVar, profile_path.c_str());
   ASSERT_EQ(Status(0), getAWSRegion(region));
   ASSERT_EQ(std::string(Aws::Region::US_EAST_1), region);
 
-  // Set an invalid path for the credentials file with a profile name provided,
+  // Profiles are not working on Windows; see the constructor of
-  profile_path = kTestDataPath + "credentials";
+  // OsqueryAWSCredentialsProviderChain for more information
-  setenv(kAwsProfileFileEnvVar, profile_path.c_str(), true);
+  if (!isPlatform(PlatformType::TYPE_WINDOWS)) {
-  FLAGS_aws_profile_name = "test";
+    // Set an invalid path for the credentials file with a profile name
-  ASSERT_FALSE(getAWSRegion(region).ok());
+    // provided,
+    profile_path = kTestDataPath + "credentials";
+    setEnvVar(kAwsProfileFileEnvVar, profile_path.c_str());
+    FLAGS_aws_profile_name = "test";
+    ASSERT_FALSE(getAWSRegion(region).ok());
 
-  // Set a valid path for the credentials file with profile name.
+    // Set a valid path for the credentials file with profile name.
-  profile_path = kTestDataPath + "aws/credentials";
+    profile_path = kTestDataPath + "aws/credentials";
-  setenv(kAwsProfileFileEnvVar, profile_path.c_str(), true);
+    setEnvVar(kAwsProfileFileEnvVar, profile_path.c_str());
-  FLAGS_aws_profile_name = "test";
+    FLAGS_aws_profile_name = "test";
-  ASSERT_EQ(Status(0), getAWSRegion(region));
+    ASSERT_EQ(Status(0), getAWSRegion(region));
-  ASSERT_EQ(std::string(Aws::Region::EU_CENTRAL_1), region);
+    ASSERT_EQ(std::string(Aws::Region::EU_CENTRAL_1), region);
 
-  FLAGS_aws_profile_name = "default";
+    FLAGS_aws_profile_name = "default";
-  ASSERT_EQ(Status(0), getAWSRegion(region));
+    ASSERT_EQ(Status(0), getAWSRegion(region));
-  ASSERT_EQ(std::string(Aws::Region::US_WEST_2), region);
+    ASSERT_EQ(std::string(Aws::Region::US_WEST_2), region);
 
-  // Should default to "default" and give same result as just above
+    // Should default to "default" and give same result as just above
-  FLAGS_aws_profile_name = "";
+    FLAGS_aws_profile_name = "";
-  ASSERT_EQ(Status(0), getAWSRegion(region));
+    ASSERT_EQ(Status(0), getAWSRegion(region));
-  ASSERT_EQ(std::string(Aws::Region::US_WEST_2), region);
+    ASSERT_EQ(std::string(Aws::Region::US_WEST_2), region);
+  }
 }
 
 TEST_F(AwsUtilTests, test_append_log_type_to_json) {

tools/provision/chocolatey/aws-sdk-cpp.ps1

@@ -6,16 +6,16 @@
 #  of patent rights can be found in the PATENTS file in the same directory.
 
 # Update-able metadata
-$version = '1.0.107'
+$version = '1.1.44'
-$chocoVersion = '1.0.107-r1'
+$chocoVersion = '1.1.44'
 $packageName = 'aws-sdk-cpp'
 $projectSource = 'https://github.com/aws/aws-sdk-cpp'
-$packageSourceUrl = 'https://github.com/apache/thrift'
+$packageSourceUrl = 'https://github.com/aws/aws-sdk-cpp/archive/$version.zip'
 $authors = 'Amazon'
 $owners = 'Amazon'
 $copyright = 'https://github.com/aws/aws-sdk-cpp/blob/master/LICENSE'
 $license = 'https://github.com/aws/aws-sdk-cpp/blob/master/LICENSE'
-$url = "https://github.com/aws/aws-sdk-cpp/archive/$version.zip"
+$url = "$packageSourceUrl"
 
 $libs = @(
   'aws-cpp-sdk-core',
@@ -49,7 +49,7 @@ if (-not (Test-Path "$chocoBuildPath")) {
 }
 Set-Location $chocoBuildPath
 
-# Retreive the source
+# Retrieve the source
 Invoke-WebRequest $url -OutFile "$packageName-$version.zip"
 
 # Extract the source
@@ -70,8 +70,8 @@ cmake -G 'Visual Studio 14 2015 Win64' -DSTATIC_LINKING=1 -DNO_HTTP_CLIENT=1 -DM
 
 # Build the libraries
 $libs | Foreach-Object {
-  msbuild 'aws-cpp-sdk-all.sln' /p:Configuration=Release /m /t:$_ /v:m
+  msbuild 'AWSSDK.sln' /p:Configuration=Release /m /t:$_ /v:m
-  msbuild 'aws-cpp-sdk-all.sln' /p:Configuration=Debug /m /t:$_ /v:m
+  msbuild 'AWSSDK.sln' /p:Configuration=Debug /m /t:$_ /v:m
 }
 
 # Construct the Chocolatey Package