2019-06-25 19:28:04 +00:00
|
|
|
trigger:
|
|
|
|
|
- master
|
|
|
|
|
|
|
|
|
|
jobs:
|
|
|
|
|
|
|
|
|
|
# LINUX
|
|
|
|
|
|
2020-04-08 23:56:21 +00:00
|
|
|
- job: LinuxBuild
|
|
|
|
|
displayName: "Linux"
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
strategy:
|
|
|
|
|
matrix:
|
|
|
|
|
Release:
|
|
|
|
|
BUILD_TYPE: Release
|
|
|
|
|
EXTRA_CMAKE_ARGS:
|
|
|
|
|
Debug:
|
|
|
|
|
BUILD_TYPE: Debug
|
|
|
|
|
EXTRA_CMAKE_ARGS: -DOSQUERY_NO_DEBUG_SYMBOLS=ON
|
2019-06-29 10:48:08 +00:00
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
pool:
|
|
|
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
|
|
2019-06-28 15:51:12 +00:00
|
|
|
container:
|
2020-03-24 15:19:32 +00:00
|
|
|
image: trailofbits/osquery:ubuntu-18.04-toolchain-v8
|
2020-02-24 00:00:38 +00:00
|
|
|
options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock
|
2019-06-25 19:28:04 +00:00
|
|
|
|
Refactor third-party libraries to build from source on Linux (#5706)
Add a way to compile third-party libraries from source instead of downloading prebuilt ones.
Each library source code is downloaded with git into a submodule at configure time,
in response to the find_package(library_name) CMake call,
except for OpenSSL where the official source archive is used.
Each submodule is attached to a release tag on its own upstream repository.
All the libraries are built using CMake directly, except for OpenSSL which uses a formula system,
which permits to build libraries with a separate build system
when there's no easy way to integrate it directly with CMake.
This new dependency system determines which library is fetched from where using the concept of "layers".
Currently we have three of them: source, formula, facebook,
where the last layer represents the pre-built libraries.
The provided order will be used when looking for libraries.
A system to patch submodule source code has been added and it's currently used with googletest, libudev and util-linux.
Patches should be put under libraries/cmake/source/<library name>/patches/<submodule>,
where <submodule> is often one and is "src", but in other cases, like AWS,
there are multiple with a more specific name.
If for whatever reason the submodule cloning or the patching fails,
the submodule has to be unregistered and its folder should be cleared.
This should be achievable with "git submodule deinit -f <submodule path>"
Following some other changes on existing functionality:
- Changed the CMake variable BUILD_TESTING to OSQUERY_BUILD_TESTS
to avoid enabling tests on third party libraries.
Due to an issue with glog the BUILD_TESTING variable
will be always forced to OFF.
- Moved compiler and linker flags to their own file cmake/flags.cmake
- Moved all the third-party CMakeLists.txt used for pre-built libraries under libraries/cmake/facebook
- Added the --exclude-folders option to tools/format-check.py and tools/git-clang-format.py,
so that it's possible to ignore any third party library source code.
- The format and format_check target use the new --exclude-folders option
to exclude libraries/cmake/source from formatting.
- The test and osquery binaries are properly compiled with PIE (osquery/osquery#5611)
Co-authored-by: Stefano Bonicatti <stefano.bonicatti@gmail.com>
Co-authored-by: Teddy Reed <teddy@casualhacking.io>
2019-08-30 14:25:19 +00:00
|
|
|
timeoutInMinutes: 120
|
|
|
|
|
|
2019-09-06 20:11:45 +00:00
|
|
|
variables:
|
|
|
|
|
CCACHE_DIR: $(Pipeline.Workspace)/ccache
|
2019-10-31 15:39:46 +00:00
|
|
|
# Debug packages require padded source prefixes (#5936).
|
|
|
|
|
BUILD_DIR: $(Build.BinariesDirectory)/usr/src/debug/osquery/build
|
2019-09-06 20:11:45 +00:00
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
steps:
|
2019-10-31 15:39:46 +00:00
|
|
|
- checkout: self
|
|
|
|
|
# See BUILD_DIR.
|
|
|
|
|
path: s/usr/src/debug/osquery
|
|
|
|
|
|
|
|
|
|
- script: mkdir -p $(BUILD_DIR)
|
2019-06-25 19:28:04 +00:00
|
|
|
displayName: "Create build folder"
|
|
|
|
|
|
2020-02-05 18:42:37 +00:00
|
|
|
- task: CacheBeta@2
|
|
|
|
|
inputs:
|
|
|
|
|
key: submodules | Linux | $(SubmoduleCacheVersion) | $(Build.SourceVersion)
|
|
|
|
|
restoreKeys: submodules | Linux | $(SubmoduleCacheVersion)
|
|
|
|
|
path: $(Build.SourcesDirectory)/.git/modules
|
2020-02-06 15:07:58 +00:00
|
|
|
displayName: Submodule cache
|
2020-02-05 18:42:37 +00:00
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Configure osquery"
|
|
|
|
|
inputs:
|
2019-10-31 15:39:46 +00:00
|
|
|
workingDirectory: $(BUILD_DIR)
|
2019-07-09 22:32:26 +00:00
|
|
|
cmakeArgs:
|
|
|
|
|
-DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
|
2019-09-06 23:51:25 +00:00
|
|
|
-DOSQUERY_TOOLCHAIN_SYSROOT=/usr/local/osquery-toolchain
|
Refactor third-party libraries to build from source on Linux (#5706)
Add a way to compile third-party libraries from source instead of downloading prebuilt ones.
Each library source code is downloaded with git into a submodule at configure time,
in response to the find_package(library_name) CMake call,
except for OpenSSL where the official source archive is used.
Each submodule is attached to a release tag on its own upstream repository.
All the libraries are built using CMake directly, except for OpenSSL which uses a formula system,
which permits to build libraries with a separate build system
when there's no easy way to integrate it directly with CMake.
This new dependency system determines which library is fetched from where using the concept of "layers".
Currently we have three of them: source, formula, facebook,
where the last layer represents the pre-built libraries.
The provided order will be used when looking for libraries.
A system to patch submodule source code has been added and it's currently used with googletest, libudev and util-linux.
Patches should be put under libraries/cmake/source/<library name>/patches/<submodule>,
where <submodule> is often one and is "src", but in other cases, like AWS,
there are multiple with a more specific name.
If for whatever reason the submodule cloning or the patching fails,
the submodule has to be unregistered and its folder should be cleared.
This should be achievable with "git submodule deinit -f <submodule path>"
Following some other changes on existing functionality:
- Changed the CMake variable BUILD_TESTING to OSQUERY_BUILD_TESTS
to avoid enabling tests on third party libraries.
Due to an issue with glog the BUILD_TESTING variable
will be always forced to OFF.
- Moved compiler and linker flags to their own file cmake/flags.cmake
- Moved all the third-party CMakeLists.txt used for pre-built libraries under libraries/cmake/facebook
- Added the --exclude-folders option to tools/format-check.py and tools/git-clang-format.py,
so that it's possible to ignore any third party library source code.
- The format and format_check target use the new --exclude-folders option
to exclude libraries/cmake/source from formatting.
- The test and osquery binaries are properly compiled with PIE (osquery/osquery#5611)
Co-authored-by: Stefano Bonicatti <stefano.bonicatti@gmail.com>
Co-authored-by: Teddy Reed <teddy@casualhacking.io>
2019-08-30 14:25:19 +00:00
|
|
|
-DOSQUERY_BUILD_TESTS=ON
|
Implement container access from tables on Linux
- Add the possibility of running table logic inside a container
namespace, so that's possible to query it instead of the host.
Needs minor modifications to each table logic and how they use logging.
In practice it works by having a pid_with_namespace column, which should
contain pids that are in the same mount namespace of the container one
wants to query.
The worker receives that column as a constraint, prepares two unnamed
pipes for read/write communications with the future child, then forks
into a new process.
While the parent sends a query job to the just created child and then waits
for results, the child receives the job, takes all the values given in the
pid_with_namespace constraint, retrieves the fd of the mount namespace
under "/proc/<constraint pid>/ns/mnt", then switches to it.
Finally it runs the table logic, sending the results back to the parent
through the pipe with a JSON message.
Important to note that the logging in the table logic is not GLOG
directly, because in the child this is in an unknown state; a custom
logging system that resembles glog and that takes advantage of the
existing communication channel is used to send the messages in JSON
format to the parent, which will take care to forward to GLOG.
- Add FLAGS_keep_container_worker_open so that the process used for
accessing a container is kept open, until the queries are for the same
table; when the table changes, the process will be closed
and a new one created.
This is off by default, which means that a new process will be always
created.
- Implemented a way to run tests that require root separated
from the others.
The OSQUERY_BUILD_ROOT_TESTS has been added to requests such tests to
be built.
To run only tests which require a normal user, one has to use
`ctest -LE "root-required"`, while `sudo ctest -L "root-required"`
to run those who need root.
PR: osquery/osquery#6209
2019-11-21 13:13:03 +00:00
|
|
|
-DOSQUERY_BUILD_ROOT_TESTS=ON
|
2019-07-09 22:32:26 +00:00
|
|
|
$(EXTRA_CMAKE_ARGS)
|
|
|
|
|
$(Build.SourcesDirectory)
|
2019-06-25 19:28:04 +00:00
|
|
|
|
2019-09-26 03:35:43 +00:00
|
|
|
- script: |
|
2019-10-31 15:39:46 +00:00
|
|
|
./tools/formatting/format-test.sh --build $(BUILD_DIR)
|
2019-09-26 03:35:43 +00:00
|
|
|
displayName: "format_check.py test"
|
|
|
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Check code formatting"
|
|
|
|
|
inputs:
|
2019-10-31 15:39:46 +00:00
|
|
|
workingDirectory: $(BUILD_DIR)
|
2019-06-25 19:28:04 +00:00
|
|
|
cmakeArgs: --build . --target format_check
|
|
|
|
|
|
2020-02-05 18:42:37 +00:00
|
|
|
- task: CacheBeta@2
|
2019-09-06 20:11:45 +00:00
|
|
|
inputs:
|
|
|
|
|
key: ccache | Linux$(BUILD_TYPE)CMake | $(CacheVersion) | $(Build.SourceVersion)
|
|
|
|
|
restoreKeys: ccache | Linux$(BUILD_TYPE)CMake | $(CacheVersion)
|
|
|
|
|
path: $(CCACHE_DIR)
|
|
|
|
|
displayName: ccache
|
|
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Build osquery"
|
|
|
|
|
inputs:
|
2019-10-31 15:39:46 +00:00
|
|
|
workingDirectory: $(BUILD_DIR)
|
2019-06-25 19:28:04 +00:00
|
|
|
cmakeArgs: --build . -j 3
|
|
|
|
|
|
2019-10-08 16:17:11 +00:00
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Run cppcheck"
|
|
|
|
|
inputs:
|
2019-10-31 15:39:46 +00:00
|
|
|
workingDirectory: $(BUILD_DIR)
|
2019-10-08 16:17:11 +00:00
|
|
|
cmakeArgs: --build . --target cppcheck
|
|
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
- script: |
|
Implement container access from tables on Linux
- Add the possibility of running table logic inside a container
namespace, so that's possible to query it instead of the host.
Needs minor modifications to each table logic and how they use logging.
In practice it works by having a pid_with_namespace column, which should
contain pids that are in the same mount namespace of the container one
wants to query.
The worker receives that column as a constraint, prepares two unnamed
pipes for read/write communications with the future child, then forks
into a new process.
While the parent sends a query job to the just created child and then waits
for results, the child receives the job, takes all the values given in the
pid_with_namespace constraint, retrieves the fd of the mount namespace
under "/proc/<constraint pid>/ns/mnt", then switches to it.
Finally it runs the table logic, sending the results back to the parent
through the pipe with a JSON message.
Important to note that the logging in the table logic is not GLOG
directly, because in the child this is in an unknown state; a custom
logging system that resembles glog and that takes advantage of the
existing communication channel is used to send the messages in JSON
format to the parent, which will take care to forward to GLOG.
- Add FLAGS_keep_container_worker_open so that the process used for
accessing a container is kept open, until the queries are for the same
table; when the table changes, the process will be closed
and a new one created.
This is off by default, which means that a new process will be always
created.
- Implemented a way to run tests that require root separated
from the others.
The OSQUERY_BUILD_ROOT_TESTS has been added to requests such tests to
be built.
To run only tests which require a normal user, one has to use
`ctest -LE "root-required"`, while `sudo ctest -L "root-required"`
to run those who need root.
PR: osquery/osquery#6209
2019-11-21 13:13:03 +00:00
|
|
|
ctest --build-nocmake -LE "root-required" -V
|
|
|
|
|
displayName: "Run tests with a normal user"
|
|
|
|
|
workingDirectory: $(BUILD_DIR)
|
|
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
sudo ctest --build-nocmake -L "root-required" -V
|
|
|
|
|
displayName: "Run tests which requires root"
|
2019-10-31 15:39:46 +00:00
|
|
|
workingDirectory: $(BUILD_DIR)
|
2019-06-29 10:48:08 +00:00
|
|
|
|
2019-10-13 10:53:16 +00:00
|
|
|
- script: |
|
|
|
|
|
cmake -DPACKAGING_SYSTEM=DEB $(Build.SourcesDirectory)
|
|
|
|
|
cmake --build . --target package -j 3
|
|
|
|
|
displayName: "Run DEB packaging"
|
2019-10-31 15:39:46 +00:00
|
|
|
workingDirectory: $(BUILD_DIR)
|
2019-10-13 10:53:16 +00:00
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
cmake -DPACKAGING_SYSTEM=RPM $(Build.SourcesDirectory)
|
|
|
|
|
cmake --build . --target package -j 3
|
|
|
|
|
displayName: "Run RPM packaging"
|
2019-10-31 15:39:46 +00:00
|
|
|
workingDirectory: $(BUILD_DIR)
|
2019-10-13 10:53:16 +00:00
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
cmake -DPACKAGING_SYSTEM=TGZ $(Build.SourcesDirectory)
|
|
|
|
|
cmake --build . --target package -j 3
|
|
|
|
|
displayName: "Run TGZ packaging"
|
2019-10-31 15:39:46 +00:00
|
|
|
workingDirectory: $(BUILD_DIR)
|
2019-10-13 10:53:16 +00:00
|
|
|
|
2019-07-09 22:32:26 +00:00
|
|
|
- script: |
|
|
|
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
|
|
|
name: JobResult
|
|
|
|
|
|
2020-03-18 14:06:29 +00:00
|
|
|
- script: |
|
|
|
|
|
rm -rf $(BUILD_DIR)
|
|
|
|
|
displayName: "Reclaim disk space"
|
|
|
|
|
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
- job: Linux
|
|
|
|
|
|
|
|
|
|
pool:
|
|
|
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
|
|
|
|
|
|
condition: succeededOrFailed()
|
|
|
|
|
|
|
|
|
|
dependsOn:
|
2020-04-08 23:56:21 +00:00
|
|
|
- LinuxBuild
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
variables:
|
2020-04-08 23:56:21 +00:00
|
|
|
LinuxReleaseStatus: $[ dependencies.LinuxBuild.outputs['Release.JobResult.Status'] ]
|
|
|
|
|
LinuxDebugStatus: $[ dependencies.LinuxBuild.outputs['Debug.JobResult.Status'] ]
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
steps:
|
|
|
|
|
- checkout: none
|
|
|
|
|
|
|
|
|
|
- script: |
|
2020-04-08 23:56:21 +00:00
|
|
|
if [ -z "$(LinuxReleaseStatus)" ] || [ -z "$(LinuxDebugStatus)" ]; then
|
2019-07-09 22:32:26 +00:00
|
|
|
exit 1
|
|
|
|
|
fi
|
2019-07-12 00:12:54 +00:00
|
|
|
displayName: "Detect Linux jobs build statuses"
|
2019-07-09 22:32:26 +00:00
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
# LINUX
|
|
|
|
|
|
|
|
|
|
# MACOS
|
|
|
|
|
|
2020-04-08 23:56:21 +00:00
|
|
|
- job: macOSBuild
|
|
|
|
|
displayName: "macOS"
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
strategy:
|
|
|
|
|
matrix:
|
|
|
|
|
Release:
|
|
|
|
|
BUILD_TYPE: Release
|
|
|
|
|
EXTRA_CMAKE_ARGS:
|
|
|
|
|
Debug:
|
|
|
|
|
BUILD_TYPE: Debug
|
|
|
|
|
EXTRA_CMAKE_ARGS: -DOSQUERY_NO_DEBUG_SYMBOLS=ON
|
2019-06-25 19:28:04 +00:00
|
|
|
|
|
|
|
|
pool:
|
|
|
|
|
vmImage: macos-10.14
|
|
|
|
|
|
2019-10-26 12:03:31 +00:00
|
|
|
timeoutInMinutes: 120
|
|
|
|
|
|
2019-09-06 20:11:45 +00:00
|
|
|
variables:
|
|
|
|
|
CCACHE_DIR: $(Pipeline.Workspace)/ccache
|
|
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
steps:
|
2019-06-29 10:48:08 +00:00
|
|
|
- script: |
|
2019-12-13 01:33:47 +00:00
|
|
|
cmake --version
|
2019-06-29 10:48:08 +00:00
|
|
|
brew upgrade
|
Refactor third-party libraries to build from source on Linux (#5706)
Add a way to compile third-party libraries from source instead of downloading prebuilt ones.
Each library source code is downloaded with git into a submodule at configure time,
in response to the find_package(library_name) CMake call,
except for OpenSSL where the official source archive is used.
Each submodule is attached to a release tag on its own upstream repository.
All the libraries are built using CMake directly, except for OpenSSL which uses a formula system,
which permits to build libraries with a separate build system
when there's no easy way to integrate it directly with CMake.
This new dependency system determines which library is fetched from where using the concept of "layers".
Currently we have three of them: source, formula, facebook,
where the last layer represents the pre-built libraries.
The provided order will be used when looking for libraries.
A system to patch submodule source code has been added and it's currently used with googletest, libudev and util-linux.
Patches should be put under libraries/cmake/source/<library name>/patches/<submodule>,
where <submodule> is often one and is "src", but in other cases, like AWS,
there are multiple with a more specific name.
If for whatever reason the submodule cloning or the patching fails,
the submodule has to be unregistered and its folder should be cleared.
This should be achievable with "git submodule deinit -f <submodule path>"
Following some other changes on existing functionality:
- Changed the CMake variable BUILD_TESTING to OSQUERY_BUILD_TESTS
to avoid enabling tests on third party libraries.
Due to an issue with glog the BUILD_TESTING variable
will be always forced to OFF.
- Moved compiler and linker flags to their own file cmake/flags.cmake
- Moved all the third-party CMakeLists.txt used for pre-built libraries under libraries/cmake/facebook
- Added the --exclude-folders option to tools/format-check.py and tools/git-clang-format.py,
so that it's possible to ignore any third party library source code.
- The format and format_check target use the new --exclude-folders option
to exclude libraries/cmake/source from formatting.
- The test and osquery binaries are properly compiled with PIE (osquery/osquery#5611)
Co-authored-by: Stefano Bonicatti <stefano.bonicatti@gmail.com>
Co-authored-by: Teddy Reed <teddy@casualhacking.io>
2019-08-30 14:25:19 +00:00
|
|
|
brew install ccache flex bison
|
2019-09-25 12:19:49 +00:00
|
|
|
pip3 install setuptools pexpect==3.3 psutil timeout_decorator six thrift==0.11.0 osquery
|
2019-10-30 21:46:36 +00:00
|
|
|
sudo xcode-select -s /Applications/Xcode_10.3.app/Contents/Developer
|
2019-06-29 10:48:08 +00:00
|
|
|
displayName: "Install Homebrew and prerequisites"
|
|
|
|
|
timeoutInMinutes: 20
|
|
|
|
|
|
|
|
|
|
- script: mkdir $(Build.BinariesDirectory)/build
|
|
|
|
|
displayName: "Create build folder"
|
|
|
|
|
|
2020-02-05 18:42:37 +00:00
|
|
|
- task: CacheBeta@2
|
|
|
|
|
inputs:
|
|
|
|
|
key: submodules | macOS | $(SubmoduleCacheVersion) | $(Build.SourceVersion)
|
|
|
|
|
restoreKeys: submodules | macOS | $(SubmoduleCacheVersion)
|
|
|
|
|
path: $(Build.SourcesDirectory)/.git/modules
|
2020-02-06 15:07:58 +00:00
|
|
|
displayName: Submodule cache
|
2020-02-05 18:42:37 +00:00
|
|
|
|
2019-06-29 10:48:08 +00:00
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Configure osquery"
|
|
|
|
|
inputs:
|
2019-06-28 12:59:14 +00:00
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
2019-11-15 20:20:34 +00:00
|
|
|
cmakeArgs: -DCMAKE_OSX_DEPLOYMENT_TARGET=10.11 -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DOSQUERY_BUILD_TESTS=ON $(EXTRA_CMAKE_ARGS) $(Build.SourcesDirectory)
|
2019-06-29 10:48:08 +00:00
|
|
|
|
2020-02-05 18:42:37 +00:00
|
|
|
- task: CacheBeta@2
|
2019-09-06 20:11:45 +00:00
|
|
|
inputs:
|
|
|
|
|
key: ccache | macOS$(BUILD_TYPE)CMake | $(CacheVersion) | $(Build.SourceVersion)
|
|
|
|
|
restoreKeys: ccache | macOS$(BUILD_TYPE)CMake | $(CacheVersion)
|
|
|
|
|
path: $(CCACHE_DIR)
|
|
|
|
|
displayName: ccache
|
|
|
|
|
|
2019-06-29 10:48:08 +00:00
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Build osquery"
|
|
|
|
|
inputs:
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
|
|
|
cmakeArgs: --build . -j 3
|
|
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
ctest --build-nocmake -V
|
|
|
|
|
displayName: "Run tests"
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
|
|
|
|
2019-10-13 10:53:16 +00:00
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Run productbuild packaging"
|
|
|
|
|
inputs:
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
|
|
|
cmakeArgs: --build . --target package -j 3
|
|
|
|
|
|
|
|
|
|
- script: |
|
|
|
|
|
cmake -DPACKAGING_SYSTEM=TGZ $(Build.SourcesDirectory)
|
|
|
|
|
cmake --build . --target package -j 3
|
|
|
|
|
displayName: "Run TGZ packaging"
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
|
|
|
|
2019-07-09 22:32:26 +00:00
|
|
|
- script: |
|
|
|
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
|
|
|
name: JobResult
|
|
|
|
|
|
2020-03-18 14:06:29 +00:00
|
|
|
- script: |
|
|
|
|
|
rm -rf $(Build.BinariesDirectory)/build
|
|
|
|
|
displayName: "Reclaim disk space"
|
|
|
|
|
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
- job: macOS
|
|
|
|
|
|
|
|
|
|
pool:
|
|
|
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
|
|
|
|
|
|
condition: succeededOrFailed()
|
|
|
|
|
|
|
|
|
|
dependsOn:
|
2020-04-08 23:56:21 +00:00
|
|
|
- macOSBuild
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
variables:
|
2020-04-08 23:56:21 +00:00
|
|
|
macOSReleaseStatus: $[ dependencies.macOSBuild.outputs['Release.JobResult.Status'] ]
|
|
|
|
|
macOSDebugStatus: $[ dependencies.macOSBuild.outputs['Debug.JobResult.Status'] ]
|
2019-07-09 22:32:26 +00:00
|
|
|
|
|
|
|
|
steps:
|
|
|
|
|
- checkout: none
|
|
|
|
|
|
|
|
|
|
- script: |
|
2020-04-08 23:56:21 +00:00
|
|
|
if [ -z "$(macOSReleaseStatus)" ] || [ -z "$(macOSDebugStatus)" ]; then
|
2019-07-09 22:32:26 +00:00
|
|
|
exit 1
|
|
|
|
|
fi
|
2019-07-12 00:12:54 +00:00
|
|
|
displayName: "Detect macOS jobs build statuses"
|
2019-07-09 22:32:26 +00:00
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
# MACOS
|
|
|
|
|
|
|
|
|
|
# WINDOWS
|
|
|
|
|
|
2020-04-08 23:56:21 +00:00
|
|
|
- job: WindowsBuild
|
|
|
|
|
displayName: "Windows Release"
|
2019-06-25 19:28:04 +00:00
|
|
|
|
2020-09-02 03:29:26 +00:00
|
|
|
strategy:
|
|
|
|
|
matrix:
|
|
|
|
|
x64:
|
|
|
|
|
VC_VARS_FILE: vcvars64.bat
|
|
|
|
|
PYTHON_FOLDER: x64
|
|
|
|
|
win32:
|
|
|
|
|
VC_VARS_FILE: vcvars32.bat
|
|
|
|
|
PYTHON_FOLDER: x86
|
|
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
pool:
|
2020-06-27 14:17:23 +00:00
|
|
|
vmImage: windows-2019
|
2019-06-25 19:28:04 +00:00
|
|
|
|
2019-10-26 12:03:31 +00:00
|
|
|
timeoutInMinutes: 120
|
|
|
|
|
|
2020-02-06 15:07:58 +00:00
|
|
|
variables:
|
|
|
|
|
SCCACHE_DIR: $(Pipeline.Workspace)\sccache
|
|
|
|
|
SCCACHE_CACHE_SIZE: "5G"
|
|
|
|
|
AZP_CACHING_CONTENT_FORMAT: Files
|
|
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
steps:
|
2019-06-28 23:48:50 +00:00
|
|
|
- powershell: |
|
|
|
|
|
git config --global core.autocrlf false
|
2019-11-21 02:36:23 +00:00
|
|
|
git config --global core.symlinks true
|
2019-06-28 23:48:50 +00:00
|
|
|
|
|
|
|
|
- checkout: self
|
|
|
|
|
|
2019-09-25 12:19:49 +00:00
|
|
|
- powershell: |
|
2019-12-13 01:33:47 +00:00
|
|
|
cmake --version
|
2020-09-02 03:29:26 +00:00
|
|
|
$python3_path = ((Get-Item C:\hostedtoolcache\windows\Python\3*\$(PYTHON_FOLDER)) | Sort-Object -Descending)[0].FullName
|
2020-08-03 14:40:03 +00:00
|
|
|
& $python3_path\python -m pip install --upgrade pip --no-warn-script-location
|
2019-09-25 12:19:49 +00:00
|
|
|
& $python3_path\python -m pip install setuptools psutil timeout_decorator thrift==0.11.0 osquery pywin32
|
|
|
|
|
displayName: Install tests prerequisites
|
|
|
|
|
|
2019-06-25 19:28:04 +00:00
|
|
|
- powershell: |
|
|
|
|
|
mkdir $(Build.BinariesDirectory)\build
|
|
|
|
|
displayName: "Create build folder"
|
|
|
|
|
|
2019-10-26 12:03:31 +00:00
|
|
|
- powershell: |
|
|
|
|
|
tools\ci\scripts\install_openssl_formula_dependencies.ps1
|
2019-11-02 01:25:14 +00:00
|
|
|
displayName: "Installing: Strawberry Perl"
|
2019-10-26 12:03:31 +00:00
|
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
|
|
2020-02-05 18:42:37 +00:00
|
|
|
- task: CacheBeta@2
|
|
|
|
|
inputs:
|
|
|
|
|
key: submodules | Windows | $(SubmoduleCacheVersion) | $(Build.SourceVersion)
|
|
|
|
|
restoreKeys: submodules | Windows | $(SubmoduleCacheVersion)
|
|
|
|
|
path: $(Build.SourcesDirectory)/.git/modules
|
2020-02-06 15:07:58 +00:00
|
|
|
displayName: Submodule cache
|
2020-02-05 18:42:37 +00:00
|
|
|
|
2020-02-06 15:07:58 +00:00
|
|
|
- powershell: |
|
|
|
|
|
(New-Object System.Net.WebClient).DownloadFile(`
|
|
|
|
|
"https://github.com/osquery/sccache/releases/download/0.0.1-osquery/sccache-0.0.1-windows.7z",`
|
|
|
|
|
"$env:TEMP\sccache.7z")
|
|
|
|
|
|
|
|
|
|
mkdir "C:\Program Files\sccache"
|
|
|
|
|
7z x -o"C:\Program Files\sccache" -y "$env:TEMP\sccache.7z"
|
|
|
|
|
Write-Host "##vso[task.prependpath]C:\Program Files\sccache"
|
|
|
|
|
displayName: "Install sccache"
|
|
|
|
|
|
|
|
|
|
- powershell: |
|
|
|
|
|
(New-Object System.Net.WebClient).DownloadFile(`
|
2020-06-27 14:17:23 +00:00
|
|
|
"https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-win.zip",`
|
2020-02-06 15:07:58 +00:00
|
|
|
"$env:TEMP\ninja-win.zip")
|
|
|
|
|
|
|
|
|
|
mkdir "C:\Program Files\Ninja"
|
|
|
|
|
7z x -o"C:\Program Files\Ninja" -y "$env:TEMP\ninja-win.zip"
|
|
|
|
|
Write-Host "##vso[task.prependpath]C:\Program Files\Ninja"
|
|
|
|
|
displayName: "Install Ninja"
|
|
|
|
|
|
|
|
|
|
- script: |
|
2020-09-02 03:29:26 +00:00
|
|
|
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\$(VC_VARS_FILE)"
|
2020-02-06 15:07:58 +00:00
|
|
|
cmake -G Ninja ^
|
|
|
|
|
-DCMAKE_C_COMPILER=cl.exe ^
|
|
|
|
|
-DCMAKE_CXX_COMPILER=cl.exe ^
|
|
|
|
|
-DCMAKE_BUILD_TYPE=Release ^
|
|
|
|
|
-DOSQUERY_BUILD_TESTS=ON ^
|
|
|
|
|
-DCMAKE_C_COMPILER_LAUNCHER="sccache.exe" ^
|
|
|
|
|
-DCMAKE_CXX_COMPILER_LAUNCHER="sccache.exe" ^
|
|
|
|
|
$(Build.SourcesDirectory)
|
2019-06-25 19:28:04 +00:00
|
|
|
displayName: "Configure osquery"
|
2020-02-06 15:07:58 +00:00
|
|
|
workingDirectory: $(Build.BinariesDirectory)\build
|
|
|
|
|
|
|
|
|
|
- task: CacheBeta@2
|
2019-06-25 19:28:04 +00:00
|
|
|
inputs:
|
2020-09-02 03:29:26 +00:00
|
|
|
key: sccache | Windows$(System.JobName) | $(CacheVersion) | $(Build.SourceVersion)
|
|
|
|
|
restoreKeys: sccache | Windows$(System.JobName) | $(CacheVersion)
|
2020-02-06 15:07:58 +00:00
|
|
|
path: $(SCCACHE_DIR)
|
|
|
|
|
displayName: sccache
|
2019-06-25 19:28:04 +00:00
|
|
|
|
2020-02-06 15:07:58 +00:00
|
|
|
- script: |
|
2020-09-02 03:29:26 +00:00
|
|
|
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\$(VC_VARS_FILE)"
|
2020-02-06 15:07:58 +00:00
|
|
|
cmake --build . -j 3
|
2020-02-27 15:02:37 +00:00
|
|
|
if %errorlevel% neq 0 exit /b %errorlevel%
|
2020-02-06 15:07:58 +00:00
|
|
|
sccache.exe --stop-server
|
2019-06-25 19:28:04 +00:00
|
|
|
displayName: "Build osquery"
|
2020-02-06 15:07:58 +00:00
|
|
|
workingDirectory: $(Build.BinariesDirectory)\build
|
2019-06-25 19:28:04 +00:00
|
|
|
|
|
|
|
|
- powershell: |
|
|
|
|
|
ctest --build-nocmake -C Release -V
|
|
|
|
|
displayName: "Run tests"
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
2019-06-29 10:48:08 +00:00
|
|
|
|
2019-10-13 10:53:16 +00:00
|
|
|
- task: CMake@1
|
|
|
|
|
displayName: "Run WIX packaging"
|
|
|
|
|
inputs:
|
|
|
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
|
|
|
cmakeArgs: --build . --target package --config Release -j 3
|
|
|
|
|
|
2019-07-12 00:12:54 +00:00
|
|
|
- powershell: |
|
2020-02-06 15:07:58 +00:00
|
|
|
# .artifactignore has to be copied in the cached folder, otherwise the CacheBeta task won't see it
|
|
|
|
|
cp $(Build.SourcesDirectory)\.artifactignore $(Build.SourcesDirectory)\.git\modules
|
|
|
|
|
|
2019-07-12 00:12:54 +00:00
|
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
|
|
|
name: JobResult
|
|
|
|
|
|
2020-03-18 14:06:29 +00:00
|
|
|
- powershell: |
|
|
|
|
|
rm -r -Force $(Build.BinariesDirectory)/build
|
|
|
|
|
displayName: "Reclaim disk space"
|
|
|
|
|
|
2019-07-12 00:12:54 +00:00
|
|
|
- job: Windows
|
|
|
|
|
|
|
|
|
|
pool:
|
|
|
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
|
|
|
|
|
|
condition: succeededOrFailed()
|
|
|
|
|
|
|
|
|
|
dependsOn:
|
2020-04-08 23:56:21 +00:00
|
|
|
- WindowsBuild
|
2019-07-12 00:12:54 +00:00
|
|
|
|
|
|
|
|
variables:
|
2020-09-02 03:29:26 +00:00
|
|
|
WindowsRelease64Status: $[ dependencies.WindowsBuild.outputs['x64.JobResult.Status'] ]
|
|
|
|
|
WindowsRelease32Status: $[ dependencies.WindowsBuild.outputs['win32.JobResult.Status'] ]
|
2019-07-12 00:12:54 +00:00
|
|
|
|
|
|
|
|
steps:
|
|
|
|
|
- checkout: none
|
|
|
|
|
|
|
|
|
|
- script: |
|
2020-09-02 03:29:26 +00:00
|
|
|
if [ -z "$(WindowsRelease64Status)" ] || [ -z "$(WindowsRelease32Status)" ]; then
|
2019-07-12 00:12:54 +00:00
|
|
|
exit 1
|
|
|
|
|
fi
|
2020-09-02 03:29:26 +00:00
|
|
|
displayName: "Detect Windows jobs build statuses"
|
2019-06-25 19:28:04 +00:00
|
|
|
# WINDOWS
|
