valitydev/osquery-1
14
0
Fork 0
mirror of https://github.com/valitydev/osquery-1.git synced 2024-11-06 09:35:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Simplify how third party libraries formula work (#6303)

Browse Source 
Remove a level of indirection when configuring and building formulas.
This should simplify working with them and also remove some issues
encountered when trying to build on Windows.
This commit is contained in:
Stefano Bonicatti 2020-03-31 15:45:20 +02:00 committed by GitHub
parent c3e052c8be
commit c22ab5c404
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 91 additions and 265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
azure-pipelines.yml
View File

@ -80,10 +80,6 @@ jobs:
workingDirectory: $(BUILD_DIR)
cmakeArgs: --build . -j 3
- script: cat $(BUILD_DIR)/formula_logs/openssl.txt
displayName: "OpenSSL formula output"
condition: always()
- task: CMake@1
displayName: "Run cppcheck"
inputs:
@ -245,10 +241,6 @@ jobs:
workingDirectory: $(Build.BinariesDirectory)/build
cmakeArgs: --build . -j 3
- script: cat $(Build.BinariesDirectory)/build/formula_logs/openssl.txt
displayName: "OpenSSL formula output"
condition: always()
- script: |
ctest --build-nocmake -V
displayName: "Run tests"
@ -428,10 +420,6 @@ jobs:
displayName: "Build osquery"
workingDirectory: $(Build.BinariesDirectory)\build
- script: cmd /C type $(Build.BinariesDirectory)\build\formula_logs\openssl.txt
displayName: "OpenSSL formula output"
condition: always()
- powershell: |
ctest --build-nocmake -C Release -V
displayName: "Run tests"

194
libraries/cmake/formula/modules/utils.cmake
View File

@ -11,6 +11,7 @@ set(OSQUERY_FORMULA_INSTALL_DIRECTORY "${CMAKE_BINARY_DIR}/installed_formulas")
set(OSQUERY_FORMULA_BUILD_JOBS 4 CACHE STRING "Numer of parallel jobs that will be used for each third party library which uses the formula system to build")
function(importFormula library_name)
if("${library_name}" STREQUAL "modules")
message(FATAL_ERROR "Invalid library name specified: ${library_name}")
@ -22,197 +23,10 @@ function(importFormula library_name)
set(build_directory_path "${CMAKE_CURRENT_BINARY_DIR}/${library_name}")
set(install_prefix "${OSQUERY_FORMULA_INSTALL_DIRECTORY}/${library_name}")
if(NOT ${library_name}_formula_initialized)
# Make sure the source folder exists
if(NOT EXISTS "${project_directory_path}")
message(FATAL_ERROR "The formula was not found: ${project_directory_path}")
endif()
getCompilationFlags(c OSQUERY_FORMULA_CFLAGS)
getCompilationFlags(cxx OSQUERY_FORMULA_CXXFLAGS)
# Create the build folder in advance
execute_process(
COMMAND "${CMAKE_COMMAND}" -E make_directory "${build_directory_path}"
RESULT_VARIABLE error
)
if(NOT ${error} EQUAL 0)
message(FATAL_ERROR "The formula build folder could not be created: ${build_directory_path}")
endif()
# We need to configure the project to capture its metadata
if(NOT "${CMAKE_GENERATOR_TOOLSET}" STREQUAL "")
set(toolset_option -T "${CMAKE_GENERATOR_TOOLSET}")
endif()
if(DEFINED PLATFORM_POSIX)
# Most of the third party libraries will use makefiles and won't support Ninja
set(generator_option -G "Unix Makefiles")
elseif(NOT "${CMAKE_GENERATOR}" STREQUAL "")
set(generator_option -G "${CMAKE_GENERATOR}")
endif()
if(DEFINED PLATFORM_MACOS)
set(optional_osx_sysroot_setting
"-DCMAKE_OSX_SYSROOT:PATH=${CMAKE_OSX_SYSROOT}"
)
endif()
execute_process(
COMMAND "${CMAKE_COMMAND}"
${generator_option}
${toolset_option}
${optional_osx_sysroot_setting}
"-DCMAKE_SYSROOT:PATH=${CMAKE_SYSROOT}"
"-DCMAKE_C_COMPILER:PATH=${CMAKE_C_COMPILER}"
"-DCMAKE_CXX_COMPILER:PATH=${CMAKE_CXX_COMPILER}"
"-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}"
"-DCMAKE_INSTALL_PREFIX:PATH=${install_prefix}"
"${project_directory_path}"
WORKING_DIRECTORY "${build_directory_path}"
RESULT_VARIABLE error
OUTPUT_VARIABLE std_output
ERROR_VARIABLE std_error
)
if(NOT ${error} EQUAL 0)
message(FATAL_ERROR "The ${library_name} formula could not be configured\nstdout\n===\n${std_output}\n\n\nstderr\n===\n${std_error}")
endif()
# Get the formula metadata
set(metadata_field_list
metadata_version
metadata_revision
metadata_dependencies
metadata_libraries
)
foreach(metadata_field ${metadata_field_list})
execute_process(
COMMAND "${CMAKE_COMMAND}" --build . --config "${CMAKE_BUILD_TYPE}" --target "${metadata_field}"
WORKING_DIRECTORY "${build_directory_path}"
RESULT_VARIABLE error
OUTPUT_VARIABLE std_output
ERROR_VARIABLE std_error
)
if(NOT ${error} EQUAL 0)
message(FATAL_ERROR "Failed to acquire the metadata field ${metadata_field} for formula ${library_name}\nstdout\n===\n${std_output}\n\n\nstderr\n===\n${std_error}")
endif()
string(FIND "${std_output}" "[" value_start_index REVERSE)
if(${value_start_index} EQUAL -1)
message(FATAL_ERROR "Malformed metadata field ${metadata_field} for formula ${library_name}\nstdout\n===\n${std_output}\n\n\nstderr\n===\n${std_error}")
endif()
string(FIND "${std_output}" "]" value_end_index REVERSE)
if(${value_end_index} EQUAL -1)
message(FATAL_ERROR "Malformed metadata field ${metadata_field} for formula ${library_name}\nstdout\n===\n${std_output}\n\n\nstderr\n===\n${std_error}")
endif()
math(EXPR value_start_index "${value_start_index} + 1")
math(EXPR value_char_length "${value_end_index} - ${value_start_index}")
if(${value_char_length} EQUAL 0)
if("${metadata_field}" STREQUAL "metadata_dependencies")
continue()
else()
message(FATAL_ERROR "Malformed metadata field ${metadata_field} for formula ${library_name}\nstdout\n===\n${std_output}\n\n\nstderr\n===\n${std_error}")
endif()
endif()
string(SUBSTRING "${std_output}" ${value_start_index} ${value_char_length} metadata_field_value)
if("${metadata_field_value}" STREQUAL "")
message(FATAL_ERROR "Malformed metadata field ${metadata_field} for formula ${library_name}\nstdout\n===\n${std_output}\n\n\nstderr\n===\n${std_error}")
endif()
set("${metadata_field}" "${metadata_field_value}")
endforeach()
# Generate the options necessary to pass the dependencies to the formula
foreach(dependency ${metadata_dependencies})
if(NOT TARGET "${dependency}")
message(FATAL_ERROR "Invalid dependency specified: ${dependency}")
endif()
get_target_property(dependency_include_dirs "${dependency}" INTERFACE_INCLUDE_DIRECTORIES)
if("${dependency_include_dirs}" STREQUAL "dependency_include_dirs-NOTFOUND")
message(FATAL_ERROR "Failed to acquire the interface include directory for library ${dependency}")
endif()
list(APPEND formula_dependency_settings
"-D${dependency}_INCLUDE_DIRS:STRING=${dependency_include_dirs}"
"-D${dependency}_LIBRARIES:STRING=$<TARGET_FILE:${dependency}>"
)
endforeach()
# Generate the formula runner that will actually build the libraries
# TODO(alessandro): Change it so we use a completely new environment
foreach(output_lib ${metadata_libraries})
list(APPEND output_file_list "${install_prefix}/${output_lib}")
endforeach()
set(${library_name}_output_file_list "${output_file_list}" CACHE STRING "${library_name} output files")
set(${library_name}_formula_dependency_settings "${formula_dependency_settings}" CACHE STRING "${library_name} dependency settings")
set(${library_name}_metadata_dependencies "${metadata_dependencies}" CACHE STRING "${library_name} dependencies")
set(${library_name}_metadata_version "${metadata_version}" CACHE STRING "${library_name} version")
set(${library_name}_metadata_revision "${metadata_revision}" CACHE STRING "${library_name} revision")
set(${library_name}_formula_initialized true CACHE BOOL "${library_name} initialized")
endif()
set(log_folder_path
"${CMAKE_BINARY_DIR}/formula_logs"
)
set(log_file_path
"${log_folder_path}/${library_name}.txt"
)
getCompilationFlags(c c_compilation_flags)
getCompilationFlags(cxx cxx_compilation_flags)
add_custom_command(
OUTPUT ${${library_name}_output_file_list}
COMMAND "${CMAKE_COMMAND}" -E make_directory "${log_folder_path}"
COMMAND "${CMAKE_COMMAND}" -E remove -f "${log_file_path}"
COMMAND "${CMAKE_COMMAND}" "-DC_FLAGS:STRING=${c_compilation_flags}" "-DCXX_FLAGS:STRING=${cxx_compilation_flags}" ${${library_name}_formula_dependency_settings} "${project_directory_path}" > "${log_file_path}" 2>&1
COMMAND "${CMAKE_COMMAND}" --build . --config "${CMAKE_BUILD_TYPE}" -j ${OSQUERY_FORMULA_BUILD_JOBS} >> "${log_file_path}" 2>&1
WORKING_DIRECTORY "${build_directory_path}"
COMMENT "Running formula: ${library_name} (${log_file_path})"
VERBATIM
)
add_custom_target("${library_name}_formula_runner"
DEPENDS ${${library_name}_output_file_list}
)
add_dependencies("${library_name}_formula_runner"
${${library_name}_metadata_dependencies}
)
# Generate the imported library
add_library("thirdparty_${library_name}" INTERFACE)
foreach(output_lib ${${library_name}_output_file_list})
get_filename_component(output_lib_name "${output_lib}" NAME_WE)
set(intermediate_target_name "thirdparty_intermediate_${output_lib_name}")
add_library("${intermediate_target_name}" STATIC IMPORTED GLOBAL)
set_target_properties("${intermediate_target_name}" PROPERTIES IMPORTED_LOCATION
"${output_lib}"
)
target_include_directories("thirdparty_${library_name}" INTERFACE "${install_prefix}/include")
target_link_libraries("thirdparty_${library_name}" INTERFACE "${intermediate_target_name}")
endforeach()
add_dependencies("thirdparty_${library_name}"
"${library_name}_formula_runner"
)
message(STATUS " Version: ${${library_name}_metadata_version}")
message(STATUS " Revision: ${${library_name}_metadata_revision}")
message(STATUS " Dependencies: ${${library_name}_metadata_dependencies}")
add_subdirectory("${project_directory_path}" "${build_directory_path}")
endfunction()
function(getCompilationFlags language output_variable)

150
libraries/cmake/formula/openssl/CMakeLists.txt
View File

@ -6,13 +6,10 @@ set(OPENSSL_VERSION "1.1.1d")
include(ExternalProject)
function(opensslMain)
generateMetadataTargets()
if(NOT DEFINED thirdparty_zlib_INCLUDE_DIRS OR
NOT DEFINED thirdparty_zlib_LIBRARIES)
message(WARNING "The zlib dependency was not specified correctly")
return()
endif()
get_target_property(zlib_library_dir_path thirdparty_zlib BINARY_DIR)
get_target_property(zlib_library_lib_name thirdparty_zlib OUTPUT_NAME)
get_target_property(zlib_library_source_dir thirdparty_zlib SOURCE_DIR)
set(common_options
no-ssl2
@ -22,19 +19,22 @@ function(opensslMain)
no-weak-ssl-ciphers
zlib-dynamic
enable-cms
"--with-zlib-include=${thirdparty_zlib_INCLUDE_DIRS}"
"--with-zlib-lib=${thirdparty_zlib_LIBRARIES}"
"--with-zlib-include=${zlib_library_source_dir}/src"
"--with-zlib-lib=${zlib_library_dir_path}/${zlib_library_lib_name}"
)
add_library(thirdparty_openssl_ssl STATIC IMPORTED GLOBAL)
add_library(thirdparty_openssl_crypto STATIC IMPORTED GLOBAL)
if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
set(configure_command
"${CMAKE_COMMAND}" -E env CC="${CMAKE_C_COMPILER}" AR="${CMAKE_AR}"
"${CMAKE_COMMAND}" -E env "CC=${CMAKE_C_COMPILER}" "AR=${CMAKE_AR}"
perl ./Configure linux-x86_64
${common_options}
${C_FLAGS}
${OSQUERY_FORMULA_CFLAGS}
"--prefix=${CMAKE_INSTALL_PREFIX}"
"--openssldir=${CMAKE_INSTALL_PREFIX}/etc/openssl"
"--prefix=${install_prefix}"
"--openssldir=${install_prefix}/etc/openssl"
-fPIC
--sysroot=${CMAKE_SYSROOT}
@ -43,24 +43,36 @@ function(opensslMain)
)
set(build_command
"${CMAKE_COMMAND}" -E make_directory "${CMAKE_INSTALL_PREFIX}/etc/openssl" &&
"${CMAKE_COMMAND}" -E make_directory "${install_prefix}/etc/openssl" &&
make depend &&
make
)
set(optional_install_command
set(install_command
make install
)
set(openssl_libs
"${install_prefix}/lib/libssl.a"
"${install_prefix}/lib/libcrypto.a"
)
set_target_properties(thirdparty_openssl_ssl PROPERTIES IMPORTED_LOCATION
"${install_prefix}/lib/libssl.a"
)
set_target_properties(thirdparty_openssl_crypto PROPERTIES IMPORTED_LOCATION
"${install_prefix}/lib/libcrypto.a"
)
elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
set(configure_command
"${CMAKE_COMMAND}" -E env CC="${CMAKE_C_COMPILER}" AR="${CMAKE_AR}"
"${CMAKE_COMMAND}" -E env "CC=${CMAKE_C_COMPILER}" "AR=${CMAKE_AR}"
perl ./Configure darwin64-x86_64-cc
${common_options}
${C_FLAGS}
${OSQUERY_FORMULA_CFLAGS}
"--prefix=${CMAKE_INSTALL_PREFIX}"
"--openssldir=${CMAKE_INSTALL_PREFIX}/etc/openssl"
"--prefix=${install_prefix}"
"--openssldir=${install_prefix}/etc/openssl"
enable-ec_nistp_64_gcc_128
-fPIC
@ -71,15 +83,27 @@ function(opensslMain)
# see https://wiki.openssl.org/index.php/Compilation_and_Installation#Modifying_Build_Settings
set(build_command
/usr/bin/sed -i ".bak" "s+^CFLAGS=+CFLAGS=-isysroot ${CMAKE_OSX_SYSROOT} +g" "Makefile" &&
"${CMAKE_COMMAND}" -E make_directory "${CMAKE_INSTALL_PREFIX}/etc/openssl" &&
"${CMAKE_COMMAND}" -E make_directory "${install_prefix}/etc/openssl" &&
make depend &&
make
)
set(optional_install_command
set(install_command
make install
)
set(openssl_libs
"${install_prefix}/lib/libssl.a"
"${install_prefix}/lib/libcrypto.a"
)
set_target_properties(thirdparty_openssl_ssl PROPERTIES IMPORTED_LOCATION
"${install_prefix}/lib/libssl.a"
)
set_target_properties(thirdparty_openssl_crypto PROPERTIES IMPORTED_LOCATION
"${install_prefix}/lib/libcrypto.a"
)
elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Windows")
set(CMAKE_PREFIX_PATH "C:\\Strawberry\\perl\\bin")
find_package(Perl REQUIRED)
@ -95,67 +119,67 @@ function(opensslMain)
string(REPLACE "/" "\\\\" perl_executable_path "${PERL_EXECUTABLE}")
set(build_command
cmake -E env "cmd.exe" "/C" "nmake PERL=${perl_executable_path}"
"${CMAKE_COMMAND}" -E env "cmd.exe" "/C" "nmake PERL=${perl_executable_path}"
)
set(install_command
"${CMAKE_COMMAND}" -E make_directory "${CMAKE_INSTALL_PREFIX}/lib" &&
"${CMAKE_COMMAND}" -E copy "./libssl.lib" "${CMAKE_INSTALL_PREFIX}/lib" &&
"${CMAKE_COMMAND}" -E copy "./libcrypto.lib" "${CMAKE_INSTALL_PREFIX}/lib" &&
"${CMAKE_COMMAND}" -E make_directory "${CMAKE_INSTALL_PREFIX}/include" &&
"${CMAKE_COMMAND}" -E copy_directory "./include/openssl" "${CMAKE_INSTALL_PREFIX}/include/openssl" &&
"${CMAKE_COMMAND}" -E remove "${CMAKE_INSTALL_PREFIX}/include/openssl/__DECC_INCLUDE_PROLOGUE.h"
"${CMAKE_INSTALL_PREFIX}/include/openssl/__DECC_INCLUDE_EPILOGUE.h"
"${CMAKE_INSTALL_PREFIX}/include/openssl/opensslconf.h.in"
"${CMAKE_COMMAND}" -E make_directory "${install_prefix}/lib" &&
"${CMAKE_COMMAND}" -E copy "./libssl.lib" "${install_prefix}/lib" &&
"${CMAKE_COMMAND}" -E copy "./libcrypto.lib" "${install_prefix}/lib" &&
"${CMAKE_COMMAND}" -E make_directory "${install_prefix}/include" &&
"${CMAKE_COMMAND}" -E copy_directory "./include/openssl" "${install_prefix}/include/openssl" &&
"${CMAKE_COMMAND}" -E remove "${install_prefix}/include/openssl/__DECC_INCLUDE_PROLOGUE.h"
"${install_prefix}/include/openssl/__DECC_INCLUDE_EPILOGUE.h"
"${install_prefix}/include/openssl/opensslconf.h.in"
)
set(openssl_libs
"${install_prefix}/lib/libssl.lib"
"${install_prefix}/lib/libcrypto.lib"
)
set_target_properties(thirdparty_openssl_ssl PROPERTIES IMPORTED_LOCATION
"${install_prefix}/lib/libssl.lib"
)
set_target_properties(thirdparty_openssl_crypto PROPERTIES IMPORTED_LOCATION
"${install_prefix}/lib/libcrypto.lib"
)
else()
message(FATAL_ERROR "Unsupported system")
endif()
list(APPEND openssl_c_flags ${C_FLAGS})
list(APPEND openssl_c_flags ${OSQUERY_FORMULA_CFLAGS})
string(REPLACE ";" " " openssl_c_flags "${openssl_c_flags}")
ExternalProject_Add(openssl
URL "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"
CONFIGURE_COMMAND ${configure_command}
BUILD_COMMAND ${build_command}
INSTALL_COMMAND ${install_command}
INSTALL_COMMAND ""
BUILD_IN_SOURCE true
)
endfunction()
function(generateMetadataTargets)
set(metadata_version "${OPENSSL_VERSION}")
set(metadata_revision "1")
set(metadata_dependencies "thirdparty_zlib")
if("${CMAKE_SYSTEM_NAME}" STREQUAL "Windows")
set(metadata_libraries
"lib/libssl.lib"
"lib/libcrypto.lib"
)
else()
set(metadata_libraries
"lib/libssl.a"
"lib/libcrypto.a"
)
endif()
set(metadata_field_list
metadata_version
metadata_revision
metadata_dependencies
metadata_libraries
EXCLUDE_FROM_ALL true
)
foreach(metadata_field ${metadata_field_list})
add_custom_target(
"${metadata_field}"
COMMAND "${CMAKE_COMMAND}" -E echo "${metadata_field}:[${${metadata_field}}]"
VERBATIM
)
endforeach()
ExternalProject_Get_property(openssl SOURCE_DIR)
ExternalProject_Add_Step(openssl custom_install
COMMAND ${install_command}
DEPENDEES build
BYPRODUCTS ${openssl_libs}
WORKING_DIRECTORY "${SOURCE_DIR}"
)
add_library(thirdparty_openssl INTERFACE)
add_dependencies(thirdparty_openssl_ssl openssl)
add_dependencies(thirdparty_openssl_crypto openssl)
target_include_directories(thirdparty_openssl INTERFACE "${install_prefix}/include")
target_link_libraries(thirdparty_openssl INTERFACE
thirdparty_openssl_ssl
thirdparty_openssl_crypto
)
endfunction()
opensslMain()