2019-06-25 19:28:04 +00:00
|
|
|
trigger:
|
|
|
|
|
- master
|
|
|
|
|
|
|
|
|
|
jobs:
|
|
|
|
|
|
|
|
|
|
# LINUX
|
|
|
|
|
|
2019-07-12 00:12:54 +00:00
|
|
|
- job: LinuxCMake
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
strategy:
|
|
|
|
|
matrix:
|
|
|
|
|
Release:
|
|
|
|
|
BUILD_TYPE: Release
|
|
|
|
|
EXTRA_CMAKE_ARGS:
|
|
|
|
|
Debug:
|
|
|
|
|
BUILD_TYPE: Debug
|
|
|
|
|
EXTRA_CMAKE_ARGS: -DOSQUERY_NO_DEBUG_SYMBOLS=ON
|
2019-06-29 10:48:08 +00:00
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
pool:
|
|
|
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
|
|
2019-06-28 15:51:12 +00:00
|
|
|
container:
|
2019-09-09 23:19:20 +00:00
|
|
|
image: trailofbits/osquery:ubuntu-18.04-toolchain-v2
|
2019-06-28 15:51:12 +00:00
|
|
|
options: --privileged
|
2019-06-25 19:28:04 +00:00
|
|
|
|
Refactor third-party libraries to build from source on Linux (#5706)
Add a way to compile third-party libraries from source instead of downloading prebuilt ones.
Each library source code is downloaded with git into a submodule at configure time,
in response to the find_package(library_name) CMake call,
except for OpenSSL where the official source archive is used.
Each submodule is attached to a release tag on its own upstream repository.
All the libraries are built using CMake directly, except for OpenSSL which uses a formula system,
which permits to build libraries with a separate build system
when there's no easy way to integrate it directly with CMake.
This new dependency system determines which library is fetched from where using the concept of "layers".
Currently we have three of them: source, formula, facebook,
where the last layer represents the pre-built libraries.
The provided order will be used when looking for libraries.
A system to patch submodule source code has been added and it's currently used with googletest, libudev and util-linux.
Patches should be put under libraries/cmake/source/<library name>/patches/<submodule>,
where <submodule> is often one and is "src", but in other cases, like AWS,
there are multiple with a more specific name.
If for whatever reason the submodule cloning or the patching fails,
the submodule has to be unregistered and its folder should be cleared.
This should be achievable with "git submodule deinit -f <submodule path>"
Following some other changes on existing functionality:
- Changed the CMake variable BUILD_TESTING to OSQUERY_BUILD_TESTS
to avoid enabling tests on third party libraries.
Due to an issue with glog the BUILD_TESTING variable
will be always forced to OFF.
- Moved compiler and linker flags to their own file cmake/flags.cmake
- Moved all the third-party CMakeLists.txt used for pre-built libraries under libraries/cmake/facebook
- Added the --exclude-folders option to tools/format-check.py and tools/git-clang-format.py,
so that it's possible to ignore any third party library source code.
- The format and format_check target use the new --exclude-folders option
to exclude libraries/cmake/source from formatting.
- The test and osquery binaries are properly compiled with PIE (osquery/osquery#5611)
Co-authored-by: Stefano Bonicatti <stefano.bonicatti@gmail.com>
Co-authored-by: Teddy Reed <teddy@casualhacking.io>
2019-08-30 14:25:19 +00:00
|
|
|
timeoutInMinutes: 120
|
|
|
|
|
|
2019-09-06 20:11:45 +00:00
|
|
|
variables:
|
|
|
|
|
CCACHE_DIR: $(Pipeline.Workspace)/ccache
|
|
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
steps:
|
|
|
|
|
- script: mkdir $(Build.BinariesDirectory)/build
|
|
|
|
|
displayName: "Create build folder"
|
|
|
|
|
|
|
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Configure osquery"
|
|
|
|
|
inputs:
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
2019-07-09 22:32:26 +00:00
|
|
|
cmakeArgs:
|
|
|
|
|
-DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
|
2019-09-06 23:51:25 +00:00
|
|
|
-DOSQUERY_TOOLCHAIN_SYSROOT=/usr/local/osquery-toolchain
|
Refactor third-party libraries to build from source on Linux (#5706)
Add a way to compile third-party libraries from source instead of downloading prebuilt ones.
Each library source code is downloaded with git into a submodule at configure time,
in response to the find_package(library_name) CMake call,
except for OpenSSL where the official source archive is used.
Each submodule is attached to a release tag on its own upstream repository.
All the libraries are built using CMake directly, except for OpenSSL which uses a formula system,
which permits to build libraries with a separate build system
when there's no easy way to integrate it directly with CMake.
This new dependency system determines which library is fetched from where using the concept of "layers".
Currently we have three of them: source, formula, facebook,
where the last layer represents the pre-built libraries.
The provided order will be used when looking for libraries.
A system to patch submodule source code has been added and it's currently used with googletest, libudev and util-linux.
Patches should be put under libraries/cmake/source/<library name>/patches/<submodule>,
where <submodule> is often one and is "src", but in other cases, like AWS,
there are multiple with a more specific name.
If for whatever reason the submodule cloning or the patching fails,
the submodule has to be unregistered and its folder should be cleared.
This should be achievable with "git submodule deinit -f <submodule path>"
Following some other changes on existing functionality:
- Changed the CMake variable BUILD_TESTING to OSQUERY_BUILD_TESTS
to avoid enabling tests on third party libraries.
Due to an issue with glog the BUILD_TESTING variable
will be always forced to OFF.
- Moved compiler and linker flags to their own file cmake/flags.cmake
- Moved all the third-party CMakeLists.txt used for pre-built libraries under libraries/cmake/facebook
- Added the --exclude-folders option to tools/format-check.py and tools/git-clang-format.py,
so that it's possible to ignore any third party library source code.
- The format and format_check target use the new --exclude-folders option
to exclude libraries/cmake/source from formatting.
- The test and osquery binaries are properly compiled with PIE (osquery/osquery#5611)
Co-authored-by: Stefano Bonicatti <stefano.bonicatti@gmail.com>
Co-authored-by: Teddy Reed <teddy@casualhacking.io>
2019-08-30 14:25:19 +00:00
|
|
|
-DOSQUERY_BUILD_TESTS=ON
|
2019-07-09 22:32:26 +00:00
|
|
|
$(EXTRA_CMAKE_ARGS)
|
|
|
|
|
$(Build.SourcesDirectory)
|
2019-06-25 19:28:04 +00:00
|
|
|
|
|
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Check code formatting"
|
|
|
|
|
inputs:
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
|
|
|
cmakeArgs: --build . --target format_check
|
|
|
|
|
|
2019-09-06 20:11:45 +00:00
|
|
|
- task: CacheBeta@0
|
|
|
|
|
inputs:
|
|
|
|
|
key: ccache | Linux$(BUILD_TYPE)CMake | $(CacheVersion) | $(Build.SourceVersion)
|
|
|
|
|
restoreKeys: ccache | Linux$(BUILD_TYPE)CMake | $(CacheVersion)
|
|
|
|
|
path: $(CCACHE_DIR)
|
|
|
|
|
displayName: ccache
|
|
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Build osquery"
|
|
|
|
|
inputs:
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
|
|
|
cmakeArgs: --build . -j 3
|
|
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
ctest --build-nocmake -V
|
|
|
|
|
displayName: "Run tests"
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
2019-06-29 10:48:08 +00:00
|
|
|
|
2019-07-09 22:32:26 +00:00
|
|
|
- script: |
|
|
|
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
|
|
|
name: JobResult
|
|
|
|
|
|
|
|
|
|
|
2019-07-12 00:12:54 +00:00
|
|
|
- job: LinuxBuck
|
|
|
|
|
displayName: "LinuxBuck Release"
|
|
|
|
|
|
|
|
|
|
pool:
|
|
|
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
|
|
|
|
|
|
container:
|
|
|
|
|
image: trailofbits/osql-experimental:ubuntu-18.04
|
|
|
|
|
options: --privileged
|
|
|
|
|
|
|
|
|
|
steps:
|
|
|
|
|
- script: |
|
|
|
|
|
wget https://github.com/facebook/buck/releases/download/v2019.06.17.01/buck.2019.06.17.01_all.deb
|
|
|
|
|
sudo apt update
|
|
|
|
|
sudo apt install -q -y --no-install-recommends ./buck.2019.06.17.01_all.deb openjdk-8-jre python3-distutils
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
|
|
|
|
|
buck build @mode/linux-x86_64/release osquery:osqueryd
|
|
|
|
|
displayName: "Build osquery"
|
|
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
|
2019-09-05 23:32:58 +00:00
|
|
|
buck test @mode/linux-x86_64/release osquery/... tests/... plugins/...
|
2019-07-12 00:12:54 +00:00
|
|
|
displayName: "Run tests"
|
|
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
|
|
|
name: JobResult
|
|
|
|
|
|
|
|
|
|
|
2019-07-09 22:32:26 +00:00
|
|
|
- job: Linux
|
|
|
|
|
|
|
|
|
|
pool:
|
|
|
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
|
|
|
|
|
|
condition: succeededOrFailed()
|
|
|
|
|
|
|
|
|
|
dependsOn:
|
2019-07-12 00:12:54 +00:00
|
|
|
- LinuxCMake
|
|
|
|
|
- LinuxBuck
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
variables:
|
2019-07-12 00:12:54 +00:00
|
|
|
LinuxCMakeReleaseStatus: $[ dependencies.LinuxCMake.outputs['Release.JobResult.Status'] ]
|
|
|
|
|
LinuxCMakeDebugStatus: $[ dependencies.LinuxCMake.outputs['Debug.JobResult.Status'] ]
|
|
|
|
|
LinuxBuckReleaseStatus: $[ dependencies.LinuxBuck.outputs['JobResult.Status'] ]
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
steps:
|
|
|
|
|
- checkout: none
|
|
|
|
|
|
|
|
|
|
- script: |
|
2019-07-12 00:12:54 +00:00
|
|
|
if [ -z "$(LinuxCMakeReleaseStatus)" ] || [ -z "$(LinuxCMakeDebugStatus)" ] || \
|
|
|
|
|
[ -z "$(LinuxBuckReleaseStatus)" ]
|
|
|
|
|
then
|
2019-07-09 22:32:26 +00:00
|
|
|
exit 1
|
|
|
|
|
fi
|
2019-07-12 00:12:54 +00:00
|
|
|
displayName: "Detect Linux jobs build statuses"
|
2019-07-09 22:32:26 +00:00
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
# LINUX
|
|
|
|
|
|
|
|
|
|
# MACOS
|
|
|
|
|
|
2019-07-12 00:12:54 +00:00
|
|
|
- job: macOSCMake
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
strategy:
|
|
|
|
|
matrix:
|
|
|
|
|
Release:
|
|
|
|
|
BUILD_TYPE: Release
|
|
|
|
|
EXTRA_CMAKE_ARGS:
|
|
|
|
|
Debug:
|
|
|
|
|
BUILD_TYPE: Debug
|
|
|
|
|
EXTRA_CMAKE_ARGS: -DOSQUERY_NO_DEBUG_SYMBOLS=ON
|
2019-06-25 19:28:04 +00:00
|
|
|
|
|
|
|
|
pool:
|
|
|
|
|
vmImage: macos-10.14
|
|
|
|
|
|
2019-09-06 20:11:45 +00:00
|
|
|
variables:
|
|
|
|
|
CCACHE_DIR: $(Pipeline.Workspace)/ccache
|
|
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
steps:
|
2019-06-29 10:48:08 +00:00
|
|
|
- script: |
|
|
|
|
|
brew upgrade
|
Refactor third-party libraries to build from source on Linux (#5706)
Add a way to compile third-party libraries from source instead of downloading prebuilt ones.
Each library source code is downloaded with git into a submodule at configure time,
in response to the find_package(library_name) CMake call,
except for OpenSSL where the official source archive is used.
Each submodule is attached to a release tag on its own upstream repository.
All the libraries are built using CMake directly, except for OpenSSL which uses a formula system,
which permits to build libraries with a separate build system
when there's no easy way to integrate it directly with CMake.
This new dependency system determines which library is fetched from where using the concept of "layers".
Currently we have three of them: source, formula, facebook,
where the last layer represents the pre-built libraries.
The provided order will be used when looking for libraries.
A system to patch submodule source code has been added and it's currently used with googletest, libudev and util-linux.
Patches should be put under libraries/cmake/source/<library name>/patches/<submodule>,
where <submodule> is often one and is "src", but in other cases, like AWS,
there are multiple with a more specific name.
If for whatever reason the submodule cloning or the patching fails,
the submodule has to be unregistered and its folder should be cleared.
This should be achievable with "git submodule deinit -f <submodule path>"
Following some other changes on existing functionality:
- Changed the CMake variable BUILD_TESTING to OSQUERY_BUILD_TESTS
to avoid enabling tests on third party libraries.
Due to an issue with glog the BUILD_TESTING variable
will be always forced to OFF.
- Moved compiler and linker flags to their own file cmake/flags.cmake
- Moved all the third-party CMakeLists.txt used for pre-built libraries under libraries/cmake/facebook
- Added the --exclude-folders option to tools/format-check.py and tools/git-clang-format.py,
so that it's possible to ignore any third party library source code.
- The format and format_check target use the new --exclude-folders option
to exclude libraries/cmake/source from formatting.
- The test and osquery binaries are properly compiled with PIE (osquery/osquery#5611)
Co-authored-by: Stefano Bonicatti <stefano.bonicatti@gmail.com>
Co-authored-by: Teddy Reed <teddy@casualhacking.io>
2019-08-30 14:25:19 +00:00
|
|
|
brew install ccache flex bison
|
2019-06-29 10:48:08 +00:00
|
|
|
displayName: "Install Homebrew and prerequisites"
|
|
|
|
|
timeoutInMinutes: 20
|
|
|
|
|
|
|
|
|
|
- script: mkdir $(Build.BinariesDirectory)/build
|
|
|
|
|
displayName: "Create build folder"
|
|
|
|
|
|
|
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Configure osquery"
|
|
|
|
|
inputs:
|
2019-06-28 12:59:14 +00:00
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
Refactor third-party libraries to build from source on Linux (#5706)
Add a way to compile third-party libraries from source instead of downloading prebuilt ones.
Each library source code is downloaded with git into a submodule at configure time,
in response to the find_package(library_name) CMake call,
except for OpenSSL where the official source archive is used.
Each submodule is attached to a release tag on its own upstream repository.
All the libraries are built using CMake directly, except for OpenSSL which uses a formula system,
which permits to build libraries with a separate build system
when there's no easy way to integrate it directly with CMake.
This new dependency system determines which library is fetched from where using the concept of "layers".
Currently we have three of them: source, formula, facebook,
where the last layer represents the pre-built libraries.
The provided order will be used when looking for libraries.
A system to patch submodule source code has been added and it's currently used with googletest, libudev and util-linux.
Patches should be put under libraries/cmake/source/<library name>/patches/<submodule>,
where <submodule> is often one and is "src", but in other cases, like AWS,
there are multiple with a more specific name.
If for whatever reason the submodule cloning or the patching fails,
the submodule has to be unregistered and its folder should be cleared.
This should be achievable with "git submodule deinit -f <submodule path>"
Following some other changes on existing functionality:
- Changed the CMake variable BUILD_TESTING to OSQUERY_BUILD_TESTS
to avoid enabling tests on third party libraries.
Due to an issue with glog the BUILD_TESTING variable
will be always forced to OFF.
- Moved compiler and linker flags to their own file cmake/flags.cmake
- Moved all the third-party CMakeLists.txt used for pre-built libraries under libraries/cmake/facebook
- Added the --exclude-folders option to tools/format-check.py and tools/git-clang-format.py,
so that it's possible to ignore any third party library source code.
- The format and format_check target use the new --exclude-folders option
to exclude libraries/cmake/source from formatting.
- The test and osquery binaries are properly compiled with PIE (osquery/osquery#5611)
Co-authored-by: Stefano Bonicatti <stefano.bonicatti@gmail.com>
Co-authored-by: Teddy Reed <teddy@casualhacking.io>
2019-08-30 14:25:19 +00:00
|
|
|
cmakeArgs: -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DOSQUERY_BUILD_TESTS=ON $(EXTRA_CMAKE_ARGS) $(Build.SourcesDirectory)
|
2019-06-29 10:48:08 +00:00
|
|
|
|
2019-09-06 20:11:45 +00:00
|
|
|
- task: CacheBeta@0
|
|
|
|
|
inputs:
|
|
|
|
|
key: ccache | macOS$(BUILD_TYPE)CMake | $(CacheVersion) | $(Build.SourceVersion)
|
|
|
|
|
restoreKeys: ccache | macOS$(BUILD_TYPE)CMake | $(CacheVersion)
|
|
|
|
|
path: $(CCACHE_DIR)
|
|
|
|
|
displayName: ccache
|
|
|
|
|
|
2019-06-29 10:48:08 +00:00
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Build osquery"
|
|
|
|
|
inputs:
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
|
|
|
cmakeArgs: --build . -j 3
|
|
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
ctest --build-nocmake -V
|
|
|
|
|
displayName: "Run tests"
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
|
|
|
|
2019-07-09 22:32:26 +00:00
|
|
|
- script: |
|
|
|
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
|
|
|
name: JobResult
|
|
|
|
|
|
|
|
|
|
|
2019-07-12 00:12:54 +00:00
|
|
|
- job: macOSBuck
|
|
|
|
|
displayName: "macOSBuck Release"
|
|
|
|
|
|
|
|
|
|
pool:
|
|
|
|
|
vmImage: macos-10.14
|
|
|
|
|
|
|
|
|
|
steps:
|
|
|
|
|
- script: |
|
|
|
|
|
brew tap facebook/fb
|
|
|
|
|
brew upgrade
|
|
|
|
|
brew cask install adoptopenjdk8
|
|
|
|
|
brew install buck watchman
|
|
|
|
|
displayName: "Install Homebrew and prerequisites"
|
|
|
|
|
timeoutInMinutes: 20
|
|
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
export JAVA_HOME="$(/usr/libexec/java_home -v 1.8)"
|
|
|
|
|
buck build @mode/macos-x86_64/release osquery:osqueryd
|
|
|
|
|
displayName: "Build osquery"
|
|
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
export JAVA_HOME="$(/usr/libexec/java_home -v 1.8)"
|
2019-09-05 23:32:58 +00:00
|
|
|
buck test @mode/macos-x86_64/release osquery/... tests/... plugins/...
|
2019-07-12 00:12:54 +00:00
|
|
|
displayName: "Run tests"
|
|
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
|
|
|
name: JobResult
|
|
|
|
|
|
|
|
|
|
|
2019-07-09 22:32:26 +00:00
|
|
|
- job: macOS
|
|
|
|
|
|
|
|
|
|
pool:
|
|
|
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
|
|
|
|
|
|
condition: succeededOrFailed()
|
|
|
|
|
|
|
|
|
|
dependsOn:
|
2019-07-12 00:12:54 +00:00
|
|
|
- macOSCMake
|
|
|
|
|
- macOSBuck
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
variables:
|
2019-07-12 00:12:54 +00:00
|
|
|
macOSCMakeReleaseStatus: $[ dependencies.macOSCMake.outputs['Release.JobResult.Status'] ]
|
|
|
|
|
macOSCMakeDebugStatus: $[ dependencies.macOSCMake.outputs['Debug.JobResult.Status'] ]
|
|
|
|
|
macOSBuckReleaseStatus: $[ dependencies.macOSBuck.outputs['JobResult.Status'] ]
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
steps:
|
|
|
|
|
- checkout: none
|
|
|
|
|
|
|
|
|
|
- script: |
|
2019-07-12 00:12:54 +00:00
|
|
|
if [ -z "$(macOSCMakeReleaseStatus)" ] || [ -z "$(macOSCMakeDebugStatus)" ] || \
|
|
|
|
|
[ -z "$(macOSBuckReleaseStatus)" ]
|
|
|
|
|
then
|
2019-07-09 22:32:26 +00:00
|
|
|
exit 1
|
|
|
|
|
fi
|
2019-07-12 00:12:54 +00:00
|
|
|
displayName: "Detect macOS jobs build statuses"
|
2019-07-09 22:32:26 +00:00
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
# MACOS
|
|
|
|
|
|
|
|
|
|
# WINDOWS
|
|
|
|
|
|
2019-07-12 00:12:54 +00:00
|
|
|
- job: WindowsCMake
|
|
|
|
|
displayName: "WindowsCMake Release"
|
2019-06-25 19:28:04 +00:00
|
|
|
|
|
|
|
|
pool:
|
|
|
|
|
vmImage: vs2017-win2016
|
|
|
|
|
|
|
|
|
|
steps:
|
2019-06-28 23:48:50 +00:00
|
|
|
- powershell: |
|
|
|
|
|
git config --global core.autocrlf false
|
|
|
|
|
|
|
|
|
|
- checkout: self
|
|
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
- powershell: |
|
|
|
|
|
mkdir $(Build.BinariesDirectory)\build
|
|
|
|
|
displayName: "Create build folder"
|
|
|
|
|
|
|
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Configure osquery"
|
|
|
|
|
inputs:
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)\build
|
Refactor third-party libraries to build from source on Linux (#5706)
Add a way to compile third-party libraries from source instead of downloading prebuilt ones.
Each library source code is downloaded with git into a submodule at configure time,
in response to the find_package(library_name) CMake call,
except for OpenSSL where the official source archive is used.
Each submodule is attached to a release tag on its own upstream repository.
All the libraries are built using CMake directly, except for OpenSSL which uses a formula system,
which permits to build libraries with a separate build system
when there's no easy way to integrate it directly with CMake.
This new dependency system determines which library is fetched from where using the concept of "layers".
Currently we have three of them: source, formula, facebook,
where the last layer represents the pre-built libraries.
The provided order will be used when looking for libraries.
A system to patch submodule source code has been added and it's currently used with googletest, libudev and util-linux.
Patches should be put under libraries/cmake/source/<library name>/patches/<submodule>,
where <submodule> is often one and is "src", but in other cases, like AWS,
there are multiple with a more specific name.
If for whatever reason the submodule cloning or the patching fails,
the submodule has to be unregistered and its folder should be cleared.
This should be achievable with "git submodule deinit -f <submodule path>"
Following some other changes on existing functionality:
- Changed the CMake variable BUILD_TESTING to OSQUERY_BUILD_TESTS
to avoid enabling tests on third party libraries.
Due to an issue with glog the BUILD_TESTING variable
will be always forced to OFF.
- Moved compiler and linker flags to their own file cmake/flags.cmake
- Moved all the third-party CMakeLists.txt used for pre-built libraries under libraries/cmake/facebook
- Added the --exclude-folders option to tools/format-check.py and tools/git-clang-format.py,
so that it's possible to ignore any third party library source code.
- The format and format_check target use the new --exclude-folders option
to exclude libraries/cmake/source from formatting.
- The test and osquery binaries are properly compiled with PIE (osquery/osquery#5611)
Co-authored-by: Stefano Bonicatti <stefano.bonicatti@gmail.com>
Co-authored-by: Teddy Reed <teddy@casualhacking.io>
2019-08-30 14:25:19 +00:00
|
|
|
cmakeArgs: -G "Visual Studio 15 2017 Win64" -T host=x64 -DOSQUERY_BUILD_TESTS=ON $(Build.SourcesDirectory)
|
2019-06-25 19:28:04 +00:00
|
|
|
|
|
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Build osquery"
|
|
|
|
|
inputs:
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)\build
|
|
|
|
|
cmakeArgs: --build . -j 3 --config Release
|
|
|
|
|
|
|
|
|
|
- powershell: |
|
|
|
|
|
ctest --build-nocmake -C Release -V
|
|
|
|
|
displayName: "Run tests"
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
2019-06-29 10:48:08 +00:00
|
|
|
|
2019-07-12 00:12:54 +00:00
|
|
|
- powershell: |
|
|
|
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
|
|
|
name: JobResult
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
- job: WindowsBuck
|
|
|
|
|
displayName: "WindowsBuck Release"
|
|
|
|
|
|
|
|
|
|
pool:
|
|
|
|
|
vmImage: vs2017-win2016
|
|
|
|
|
|
|
|
|
|
steps:
|
|
|
|
|
- powershell: |
|
|
|
|
|
git config --global core.autocrlf false
|
|
|
|
|
|
|
|
|
|
- checkout: self
|
|
|
|
|
|
|
|
|
|
- powershell: |
|
|
|
|
|
choco uninstall mingw -y
|
|
|
|
|
(New-Object System.Net.WebClient).DownloadFile("https://github.com/facebook/buck/releases/download/v2019.06.17.01/buck.2019.06.17.01.nupkg", `
|
|
|
|
|
"buck.2019.06.17.01.nupkg")
|
|
|
|
|
Install-Package buck -Source . -SkipDependencies
|
|
|
|
|
displayName: "Prepare Buck environment"
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
|
|
2019-07-29 08:26:23 +00:00
|
|
|
- powershell: |
|
|
|
|
|
$python2_path = (Get-Item C:\Python27amd64).FullName
|
|
|
|
|
|
|
|
|
|
if (-not $python2_path) {
|
|
|
|
|
$python2_path = ((Get-Item C:\hostedtoolcache\windows\Python\2*\x64) | Sort-Object -Descending)[0].FullName
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$python3_path = ((Get-Item C:\hostedtoolcache\windows\Python\3*\x64) | Sort-Object -Descending)[0].FullName
|
|
|
|
|
|
|
|
|
|
echo "##vso[task.setvariable variable=python2]$python2_path"
|
|
|
|
|
echo "##vso[task.setvariable variable=python3]$python3_path"
|
|
|
|
|
|
|
|
|
|
displayName: "Detect python 2 and 3 installation folders"
|
|
|
|
|
|
2019-07-12 00:12:54 +00:00
|
|
|
- script: |
|
|
|
|
|
mklink /J $(Build.BinariesDirectory)\vsinstall "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise"
|
|
|
|
|
|
|
|
|
|
- powershell: |
|
|
|
|
|
$vcvers = cat $(Build.BinariesDirectory)\vsinstall\VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt
|
|
|
|
|
.\tools\generate_buck_config.ps1 -VsInstall '$(Build.BinariesDirectory)\vsinstall' `
|
|
|
|
|
-VcToolsVersion "$vcvers" `
|
|
|
|
|
-SdkInstall '' `
|
|
|
|
|
-SdkVersion '' `
|
2019-07-29 08:26:23 +00:00
|
|
|
-Python3Path '$(python3)\python.exe' `
|
2019-07-12 00:12:54 +00:00
|
|
|
-BuckConfigRoot .\tools\buckconfigs\
|
|
|
|
|
displayName: "Generate Buck config"
|
|
|
|
|
|
|
|
|
|
- script: |
|
2019-07-29 08:26:23 +00:00
|
|
|
set PATH=$(python2);%PATH%
|
|
|
|
|
$(python2)\python.exe "C:\Program Files\PackageManagement\NuGet\Packages\buck.2019.06.17.01\tools\buck.pex" build @mode/windows-x86_64/release osquery:osqueryd
|
2019-07-12 00:12:54 +00:00
|
|
|
displayName: "Build osquery"
|
|
|
|
|
|
|
|
|
|
- script: |
|
2019-07-29 08:26:23 +00:00
|
|
|
set PATH=$(python2);%PATH%
|
2019-09-05 23:32:58 +00:00
|
|
|
$(python2)\python.exe "C:\Program Files\PackageManagement\NuGet\Packages\buck.2019.06.17.01\tools\buck.pex" test @mode/windows-x86_64/release osquery/... tests/... plugins/...
|
2019-07-12 00:12:54 +00:00
|
|
|
displayName: "Run tests"
|
|
|
|
|
|
|
|
|
|
- powershell: |
|
|
|
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
|
|
|
name: JobResult
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
- job: Windows
|
|
|
|
|
|
|
|
|
|
pool:
|
|
|
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
|
|
|
|
|
|
condition: succeededOrFailed()
|
|
|
|
|
|
|
|
|
|
dependsOn:
|
|
|
|
|
- WindowsCMake
|
|
|
|
|
- WindowsBuck
|
|
|
|
|
|
|
|
|
|
variables:
|
|
|
|
|
WindowsCMakeReleaseStatus: $[ dependencies.WindowsCMake.outputs['JobResult.Status'] ]
|
|
|
|
|
WindowsBuckReleaseStatus: $[ dependencies.WindowsBuck.outputs['JobResult.Status'] ]
|
|
|
|
|
|
|
|
|
|
steps:
|
|
|
|
|
- checkout: none
|
|
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
if [ -z "$(WindowsCMakeReleaseStatus)" ] || [ -z "$(WindowsBuckReleaseStatus)" ]; then
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
displayName: "Detect Windows CMake and Buck build status"
|
|
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
# WINDOWS
|
