Bundle C++ extensions into a single executable (#4335)

CMake/CMakeLibs.cmake

@@ -283,6 +283,180 @@ macro(ADD_OSQUERY_EXTENSION TARGET)
   set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME "${TARGET}.ext")
 endmacro(ADD_OSQUERY_EXTENSION)
 
+function(add_osquery_extension_ex class_name extension_type extension_name ${ARGN})
+  # Make sure the extension type is valid
+  if(NOT "${extension_type}" STREQUAL "config" AND NOT "${extension_type}" STREQUAL "table")
+    message(FATAL_ERROR "Invalid extension type specified")
+  endif()
+
+  # Update the initializer list; this will be added to the main.cpp file of the extension
+  # group
+  set_property(GLOBAL APPEND_STRING
+    PROPERTY OSQUERY_EXTENSION_GROUP_INITIALIZERS
+    "REGISTER_EXTERNAL(${class_name}, \"${extension_type}\", \"${extension_name}\");\n"
+  )
+
+  # Loop through each argument
+  foreach(argument ${ARGN})
+    if("${argument}" STREQUAL "SOURCES" OR "${argument}" STREQUAL "LIBRARIES" OR
+      "${argument}" STREQUAL "INCLUDEDIRS" OR "${argument}" STREQUAL "MAININCLUDES")
+
+      set(current_scope "${argument}")
+      continue()
+    endif()
+
+    if("${current_scope}" STREQUAL "SOURCES")
+      if(NOT IS_ABSOLUTE "${argument}")
+        set(argument "${CMAKE_CURRENT_SOURCE_DIR}/${argument}")
+      endif()
+
+      list(APPEND source_file_list "${argument}")
+
+    elseif("${current_scope}" STREQUAL "INCLUDEDIRS")
+      if(NOT IS_ABSOLUTE "${argument}")
+        set(argument "${CMAKE_CURRENT_SOURCE_DIR}/${argument}")
+      endif()
+
+      list(APPEND include_folder_list "${argument}")
+
+    elseif("${current_scope}" STREQUAL "LIBRARIES")
+      list(APPEND library_list "${argument}")
+    elseif("${current_scope}" STREQUAL "MAININCLUDES")
+      list(APPEND main_include_list "${argument}")
+    else()
+      message(FATAL_ERROR "Invalid scope")
+    endif()
+  endforeach()
+
+  # Validate the arguments
+  if("${source_file_list}" STREQUAL "")
+    message(FATAL_ERROR "Source files are missing")
+  endif()
+
+  if("${main_include_list}" STREQUAL "")
+    message(FATAL_ERROR "The main include list is missing")
+  endif()
+
+  # Update the global properties
+  set_property(GLOBAL APPEND
+    PROPERTY OSQUERY_EXTENSION_GROUP_SOURCES
+    ${source_file_list}
+  )
+
+  set_property(GLOBAL APPEND
+    PROPERTY OSQUERY_EXTENSION_GROUP_MAIN_INCLUDES
+    ${main_include_list}
+  )
+
+  if(NOT "${library_list}" STREQUAL "")
+    set_property(GLOBAL APPEND
+      PROPERTY OSQUERY_EXTENSION_GROUP_LIBRARIES
+      ${library_list}
+    )
+  endif()
+
+  if(NOT "${include_folder_list}" STREQUAL "")
+    set_property(GLOBAL APPEND
+      PROPERTY OSQUERY_EXTENSION_GROUP_INCLUDE_FOLDERS
+      ${include_folder_list}
+    )
+  endif()
+endfunction()
+
+# This function takes the global properties saved by add_osquery_extension_ex and generates
+# a single extenion executable containing all the user code
+function(generate_osquery_extension_group)
+  get_property(extension_source_files GLOBAL PROPERTY OSQUERY_EXTENSION_GROUP_SOURCES)
+  if("${extension_source_files}" STREQUAL "")
+    return()
+  endif()
+
+  # Allow the user to customize the extension name and version using
+  # environment variables
+  if(DEFINED ENV{OSQUERY_EXTENSION_GROUP_NAME})
+    set(OSQUERY_EXTENSION_GROUP_NAME $ENV{OSQUERY_EXTENSION_GROUP_NAME})
+  else()
+    set(OSQUERY_EXTENSION_GROUP_NAME "osquery_extension_group")
+  endif()
+
+  if(DEFINED ENV{OSQUERY_EXTENSION_GROUP_VERSION})
+    set(OSQUERY_EXTENSION_GROUP_VERSION $ENV{OSQUERY_EXTENSION_GROUP_VERSION})
+  else()
+    set(OSQUERY_EXTENSION_GROUP_VERSION "1.0")
+  endif()
+
+  # Build the include list; this contains the files required to declare
+  # the classes used in the REGISTER_EXTERNAL directives
+  #
+  # Note: The variables in uppercase are used by the template
+  get_property(main_include_list GLOBAL PROPERTY OSQUERY_EXTENSION_GROUP_MAIN_INCLUDES)
+  foreach(include_file ${main_include_list})
+    set(OSQUERY_EXTENSION_GROUP_INCLUDES "${OSQUERY_EXTENSION_GROUP_INCLUDES}\n#include <${include_file}>")
+  endforeach()
+
+  # We need to generate the main.cpp file, containing all the required
+  # REGISTER_EXTERNAL directives
+  get_property(OSQUERY_EXTENSION_GROUP_INITIALIZERS GLOBAL PROPERTY OSQUERY_EXTENSION_GROUP_INITIALIZERS)
+  configure_file(
+    "${CMAKE_SOURCE_DIR}/tools/codegen/templates/osquery_extension_group_main.cpp.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/osquery_extension_group_main.cpp"
+  )
+
+  # Extensions can no longer control which compilation flags to use here (as they are shared) so
+  # we are going to enforce sane defaults
+  if(UNIX)
+    set(extension_cxx_flags
+      -pedantic -Wall -Wcast-align -Wcast-qual -Wctor-dtor-privacy -Wdisabled-optimization
+      -Wformat=2 -Winit-self -Wlong-long -Wmissing-declarations -Wmissing-include-dirs -Wcomment
+      -Wold-style-cast -Woverloaded-virtual -Wredundant-decls -Wshadow -Wsign-conversion
+      -Wsign-promo -Wstrict-overflow=5 -Wswitch-default -Wundef -Werror -Wunused -Wuninitialized
+      -Wconversion
+    )
+
+    if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+      list(APPEND extension_cxx_flags -g3 --gdwarf-2)
+    endif()
+  else()
+    set(extension_cxx_flags /W4)
+  endif()
+
+  # Generate the extension target
+  add_executable("${OSQUERY_EXTENSION_GROUP_NAME}"
+    "${CMAKE_CURRENT_BINARY_DIR}/osquery_extension_group_main.cpp"
+    ${extension_source_files}
+  )
+
+  set_property(TARGET "${OSQUERY_EXTENSION_GROUP_NAME}" PROPERTY INCLUDE_DIRECTORIES "")
+  target_compile_features("${OSQUERY_EXTENSION_GROUP_NAME}" PUBLIC cxx_std_14)
+  target_compile_options("${OSQUERY_EXTENSION_GROUP_NAME}" PRIVATE ${extension_cxx_flags})
+
+  set_target_properties("${OSQUERY_EXTENSION_GROUP_NAME}" PROPERTIES
+    OUTPUT_NAME "${OSQUERY_EXTENSION_GROUP_NAME}.ext"
+  )
+
+  # Import the core libraries; note that we are going to inherit include directories
+  # with the wrong scope, so we'll have to fix it
+  set_property(TARGET "${OSQUERY_EXTENSION_GROUP_NAME}" PROPERTY INCLUDE_DIRECTORIES "")
+
+  get_property(include_folder_list TARGET libosquery PROPERTY INCLUDE_DIRECTORIES)
+  target_include_directories("${OSQUERY_EXTENSION_GROUP_NAME}" SYSTEM PRIVATE ${include_folder_list})
+
+  TARGET_OSQUERY_LINK_WHOLE("${OSQUERY_EXTENSION_GROUP_NAME}" libosquery)
+
+  # Apply the user (extension) settings
+  get_property(library_list GLOBAL PROPERTY OSQUERY_EXTENSION_GROUP_LIBRARIES)
+  if(NOT "${library_list}" STREQUAL "")
+    target_link_libraries("${OSQUERY_EXTENSION_GROUP_NAME}" ${library_list})
+  endif()
+
+  get_property(include_folder_list GLOBAL PROPERTY OSQUERY_EXTENSION_GROUP_INCLUDE_FOLDERS)
+  if(NOT "${include_folder_list}" STREQUAL "")
+    target_include_directories("${OSQUERY_EXTENSION_GROUP_NAME}" PRIVATE
+      ${include_folder_list}
+    )
+  endif()
+endfunction()
+
 # Helper to abstract OS/Compiler whole linking.
 macro(TARGET_OSQUERY_LINK_WHOLE TARGET OSQUERY_LIB)
   if(WINDOWS)

docs/wiki/development/osquery-sdk.md

@@ -148,6 +148,20 @@ Scanning dependencies of target external_extension_awesome
 [100%] Built target externals
 ```
 
+## Bundling extensions into a single executable
+All the extensions declared with the **add_osquery_extension_ex()** CMake function will be automatically bundled into a single executable.
+
+The executable name and version can be changed using the following two environment variables:
+
+1. OSQUERY_EXTENSION_GROUP_NAME (default: osquery_extension_group)
+2. OSQUERY_EXTENSION_GROUP_VERSION (default: 1.0)
+
+It is important to provide a header file that can be included by the generated main.cpp file; its purpose is to define the types used by the **REGISTER_EXTERNAL** directive.
+
+An example is included in the `osquery/examples/extension_group_example`.
+
+Please note that when using bundling the source directory of each extension is added to the include folder list; developers should always use uniquely named include files.
+
 ## Thrift API
 
 [Thrift](https://thrift.apache.org/) is a code-generation/cross-language service development framework. osquery uses Thrift to allow plugin extensions for config retrieval, log export, table implementations, event subscribers, and event publishers. We also use Thrift to wrap our SQL implementation using SQLite.

external/CMakeLists.txt

@@ -53,3 +53,7 @@ foreach(external_project ${EXTERNAL_PROJECTS})
     endif()
   endif()
 endforeach()
+
+# If the user has generated extensions using the new generate_osquery_extension_group
+# function, then this call will generate the bundle
+generate_osquery_extension_group()

include/osquery/extensions.h

@@ -56,11 +56,14 @@ class ExternalSQLPlugin : public SQLPlugin {
   Status query(const std::string& query,
                QueryData& results,
                bool use_cache = false) const override {
+    static_cast<void>(use_cache);
     return queryExternal(query, results);
   }
 
   Status getQueryTables(const std::string& query,
                         std::vector<std::string>& tables) const override {
+    static_cast<void>(query);
+    static_cast<void>(tables);
     return Status(0, "Not used");
   }
 

osquery/examples/extension_group_example/CMakeLists.txt

@@ -0,0 +1,16 @@
+project(ext_example)
+
+function(main)
+  set(PROJECT_SOURCEFILES
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/example.h"
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/example.cpp"
+  )
+
+  add_osquery_extension_ex("ExampleTable" "table" "example"
+    SOURCES ${PROJECT_SOURCEFILES}
+    INCLUDEDIRS "${CMAKE_CURRENT_SOURCE_DIR}/src"
+    MAININCLUDES "example.h"
+  )
+endfunction()
+
+main()

osquery/examples/extension_group_example/src/example.cpp

@@ -0,0 +1,30 @@
+/**
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under both the Apache 2.0 license (found in the
+ *  LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ *  in the COPYING file in the root directory of this source tree).
+ *  You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "example.h"
+
+namespace osquery {
+TableColumns ExampleTable::columns() const {
+  return {
+      std::make_tuple("example_text", TEXT_TYPE, ColumnOptions::DEFAULT),
+      std::make_tuple("example_integer", INTEGER_TYPE, ColumnOptions::DEFAULT),
+  };
+}
+
+QueryData ExampleTable::generate(QueryContext& request) {
+  static_cast<void>(request);
+
+  Row r;
+  r["example_text"] = "example";
+  r["example_integer"] = INTEGER(1);
+
+  return {r};
+}
+} // namespace osquery

osquery/examples/extension_group_example/src/example.h

@@ -0,0 +1,22 @@
+/**
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under both the Apache 2.0 license (found in the
+ *  LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ *  in the COPYING file in the root directory of this source tree).
+ *  You may select, at your option, one of the above-listed licenses.
+ */
+
+#pragma once
+
+#include <osquery/sdk.h>
+#include <osquery/system.h>
+
+namespace osquery {
+class ExampleTable : public TablePlugin {
+ private:
+  TableColumns columns() const;
+  QueryData generate(QueryContext& request);
+};
+} // namespace osquery

tools/codegen/templates/osquery_extension_group_main.cpp.in

@@ -0,0 +1,32 @@
+/**
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under both the Apache 2.0 license (found in the
+ *  LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ *  in the COPYING file in the root directory of this source tree).
+ *  You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <osquery/sdk.h>
+#include <osquery/system.h>
+
+using namespace osquery;
+
+@OSQUERY_EXTENSION_GROUP_INCLUDES@
+
+@OSQUERY_EXTENSION_GROUP_INITIALIZERS@
+
+int main(int argc, char* argv[]) {
+  osquery::Initializer runner(argc, argv, ToolType::EXTENSION);
+
+  auto status = startExtension("@OSQUERY_EXTENSION_GROUP_NAME@", "@OSQUERY_EXTENSION_GROUP_VERSION@");
+  if (!status.ok()) {
+    LOG(ERROR) << status.getMessage();
+    runner.requestShutdown(status.getCode());
+  }
+
+  // Finally wait for a signal / interrupt to shutdown.
+  runner.waitForShutdown();
+  return 0;
+}