mirror of
https://github.com/valitydev/osquery-1.git
synced 2024-11-07 01:55:20 +00:00
9a99c6b4c0
* Add sccache to the Windows job Also fix debug symbols on googletest, so that it doesn't try to create a separate pdb. * Fix uploading the submodule cache sccache needs its cache to be uploaded as normal files, not with a tar, otherwise the cache when extracted will contain extraneous files and folders (PaxHeader). The Files mode though by default ignores the .git folder, so we have to use the .artifactignore file to unignore it, and we have to copy in the cached path for the cache task to see it.
532 lines
16 KiB
YAML
532 lines
16 KiB
YAML
trigger:
|
|
- master
|
|
|
|
jobs:
|
|
|
|
# LINUX
|
|
|
|
- job: LinuxCMake
|
|
|
|
strategy:
|
|
matrix:
|
|
Release:
|
|
BUILD_TYPE: Release
|
|
EXTRA_CMAKE_ARGS:
|
|
Debug:
|
|
BUILD_TYPE: Debug
|
|
EXTRA_CMAKE_ARGS: -DOSQUERY_NO_DEBUG_SYMBOLS=ON
|
|
|
|
pool:
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
container:
|
|
image: trailofbits/osquery:ubuntu-18.04-toolchain-v6
|
|
options: --privileged --init
|
|
|
|
timeoutInMinutes: 120
|
|
|
|
variables:
|
|
CCACHE_DIR: $(Pipeline.Workspace)/ccache
|
|
# Debug packages require padded source prefixes (#5936).
|
|
BUILD_DIR: $(Build.BinariesDirectory)/usr/src/debug/osquery/build
|
|
|
|
steps:
|
|
- checkout: self
|
|
# See BUILD_DIR.
|
|
path: s/usr/src/debug/osquery
|
|
|
|
- script: mkdir -p $(BUILD_DIR)
|
|
displayName: "Create build folder"
|
|
|
|
- task: CacheBeta@2
|
|
inputs:
|
|
key: submodules | Linux | $(SubmoduleCacheVersion) | $(Build.SourceVersion)
|
|
restoreKeys: submodules | Linux | $(SubmoduleCacheVersion)
|
|
path: $(Build.SourcesDirectory)/.git/modules
|
|
displayName: Submodule cache
|
|
|
|
- task: CMake@1
|
|
displayName: "Configure osquery"
|
|
inputs:
|
|
workingDirectory: $(BUILD_DIR)
|
|
cmakeArgs:
|
|
-DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
|
|
-DOSQUERY_TOOLCHAIN_SYSROOT=/usr/local/osquery-toolchain
|
|
-DOSQUERY_BUILD_TESTS=ON
|
|
$(EXTRA_CMAKE_ARGS)
|
|
$(Build.SourcesDirectory)
|
|
|
|
- script: |
|
|
./tools/formatting/format-test.sh --build $(BUILD_DIR)
|
|
displayName: "format_check.py test"
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CMake@1
|
|
displayName: "Check code formatting"
|
|
inputs:
|
|
workingDirectory: $(BUILD_DIR)
|
|
cmakeArgs: --build . --target format_check
|
|
|
|
- task: CacheBeta@2
|
|
inputs:
|
|
key: ccache | Linux$(BUILD_TYPE)CMake | $(CacheVersion) | $(Build.SourceVersion)
|
|
restoreKeys: ccache | Linux$(BUILD_TYPE)CMake | $(CacheVersion)
|
|
path: $(CCACHE_DIR)
|
|
displayName: ccache
|
|
|
|
- task: CMake@1
|
|
displayName: "Build osquery"
|
|
inputs:
|
|
workingDirectory: $(BUILD_DIR)
|
|
cmakeArgs: --build . -j 3
|
|
|
|
- script: cat $(BUILD_DIR)/formula_logs/openssl.txt
|
|
displayName: "OpenSSL formula output"
|
|
condition: always()
|
|
|
|
- task: CMake@1
|
|
displayName: "Run cppcheck"
|
|
inputs:
|
|
workingDirectory: $(BUILD_DIR)
|
|
cmakeArgs: --build . --target cppcheck
|
|
|
|
- script: |
|
|
ctest --build-nocmake -V
|
|
displayName: "Run tests"
|
|
workingDirectory: $(BUILD_DIR)
|
|
|
|
- script: |
|
|
cmake -DPACKAGING_SYSTEM=DEB $(Build.SourcesDirectory)
|
|
cmake --build . --target package -j 3
|
|
displayName: "Run DEB packaging"
|
|
workingDirectory: $(BUILD_DIR)
|
|
|
|
- script: |
|
|
cmake -DPACKAGING_SYSTEM=RPM $(Build.SourcesDirectory)
|
|
cmake --build . --target package -j 3
|
|
displayName: "Run RPM packaging"
|
|
workingDirectory: $(BUILD_DIR)
|
|
|
|
- script: |
|
|
cmake -DPACKAGING_SYSTEM=TGZ $(Build.SourcesDirectory)
|
|
cmake --build . --target package -j 3
|
|
displayName: "Run TGZ packaging"
|
|
workingDirectory: $(BUILD_DIR)
|
|
|
|
- script: |
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
name: JobResult
|
|
|
|
|
|
- job: LinuxBuck
|
|
displayName: "LinuxBuck Release"
|
|
|
|
pool:
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
container:
|
|
image: trailofbits/osql-experimental:ubuntu-18.04
|
|
options: --privileged
|
|
|
|
steps:
|
|
- script: |
|
|
wget https://github.com/facebook/buck/releases/download/v2019.06.17.01/buck.2019.06.17.01_all.deb
|
|
sudo apt update
|
|
sudo apt install -q -y --no-install-recommends ./buck.2019.06.17.01_all.deb openjdk-8-jre python3-distutils
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- script: |
|
|
export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
|
|
buck build @mode/linux-x86_64/release osquery:osqueryd
|
|
displayName: "Build osquery"
|
|
|
|
- script: |
|
|
# Buck compiles these tests in Debug mode, exhausting disk space.
|
|
# Until we find a solution, we'll disable the tests.
|
|
#export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
|
|
#buck test @mode/linux-x86_64/release osquery/... tests/... plugins/...
|
|
displayName: "Run tests (DISABLED)"
|
|
|
|
- script: |
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
name: JobResult
|
|
|
|
|
|
- job: Linux
|
|
|
|
pool:
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
condition: succeededOrFailed()
|
|
|
|
dependsOn:
|
|
- LinuxCMake
|
|
- LinuxBuck
|
|
|
|
variables:
|
|
LinuxCMakeReleaseStatus: $[ dependencies.LinuxCMake.outputs['Release.JobResult.Status'] ]
|
|
LinuxCMakeDebugStatus: $[ dependencies.LinuxCMake.outputs['Debug.JobResult.Status'] ]
|
|
LinuxBuckReleaseStatus: $[ dependencies.LinuxBuck.outputs['JobResult.Status'] ]
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
- script: |
|
|
if [ -z "$(LinuxCMakeReleaseStatus)" ] || [ -z "$(LinuxCMakeDebugStatus)" ] || \
|
|
[ -z "$(LinuxBuckReleaseStatus)" ]
|
|
then
|
|
exit 1
|
|
fi
|
|
displayName: "Detect Linux jobs build statuses"
|
|
|
|
# LINUX
|
|
|
|
# MACOS
|
|
|
|
- job: macOSCMake
|
|
|
|
strategy:
|
|
matrix:
|
|
Release:
|
|
BUILD_TYPE: Release
|
|
EXTRA_CMAKE_ARGS:
|
|
Debug:
|
|
BUILD_TYPE: Debug
|
|
EXTRA_CMAKE_ARGS: -DOSQUERY_NO_DEBUG_SYMBOLS=ON
|
|
|
|
pool:
|
|
vmImage: macos-10.14
|
|
|
|
timeoutInMinutes: 120
|
|
|
|
variables:
|
|
CCACHE_DIR: $(Pipeline.Workspace)/ccache
|
|
|
|
steps:
|
|
- script: |
|
|
cmake --version
|
|
brew upgrade
|
|
brew install ccache flex bison
|
|
pip3 install setuptools pexpect==3.3 psutil timeout_decorator six thrift==0.11.0 osquery
|
|
sudo xcode-select -s /Applications/Xcode_10.3.app/Contents/Developer
|
|
displayName: "Install Homebrew and prerequisites"
|
|
timeoutInMinutes: 20
|
|
|
|
- script: mkdir $(Build.BinariesDirectory)/build
|
|
displayName: "Create build folder"
|
|
|
|
- task: CacheBeta@2
|
|
inputs:
|
|
key: submodules | macOS | $(SubmoduleCacheVersion) | $(Build.SourceVersion)
|
|
restoreKeys: submodules | macOS | $(SubmoduleCacheVersion)
|
|
path: $(Build.SourcesDirectory)/.git/modules
|
|
displayName: Submodule cache
|
|
|
|
- task: CMake@1
|
|
displayName: "Configure osquery"
|
|
inputs:
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
cmakeArgs: -DCMAKE_OSX_DEPLOYMENT_TARGET=10.11 -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DOSQUERY_BUILD_TESTS=ON $(EXTRA_CMAKE_ARGS) $(Build.SourcesDirectory)
|
|
|
|
- task: CacheBeta@2
|
|
inputs:
|
|
key: ccache | macOS$(BUILD_TYPE)CMake | $(CacheVersion) | $(Build.SourceVersion)
|
|
restoreKeys: ccache | macOS$(BUILD_TYPE)CMake | $(CacheVersion)
|
|
path: $(CCACHE_DIR)
|
|
displayName: ccache
|
|
|
|
- task: CMake@1
|
|
displayName: "Build osquery"
|
|
inputs:
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
cmakeArgs: --build . -j 3
|
|
|
|
- script: cat $(Build.BinariesDirectory)/build/formula_logs/openssl.txt
|
|
displayName: "OpenSSL formula output"
|
|
condition: always()
|
|
|
|
- script: |
|
|
ctest --build-nocmake -V
|
|
displayName: "Run tests"
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
|
|
- task: CMake@1
|
|
displayName: "Run productbuild packaging"
|
|
inputs:
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
cmakeArgs: --build . --target package -j 3
|
|
|
|
- script: |
|
|
cmake -DPACKAGING_SYSTEM=TGZ $(Build.SourcesDirectory)
|
|
cmake --build . --target package -j 3
|
|
displayName: "Run TGZ packaging"
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
|
|
- script: |
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
name: JobResult
|
|
|
|
|
|
- job: macOSBuck
|
|
displayName: "macOSBuck Release"
|
|
|
|
pool:
|
|
vmImage: macos-10.14
|
|
|
|
steps:
|
|
- script: |
|
|
brew update
|
|
brew tap facebook/fb
|
|
brew upgrade
|
|
brew cask install adoptopenjdk
|
|
brew install buck watchman
|
|
sudo xcode-select -s /Applications/Xcode_10.3.app/Contents/Developer
|
|
displayName: "Install Homebrew and prerequisites"
|
|
timeoutInMinutes: 20
|
|
|
|
- script: |
|
|
export JAVA_HOME="$(/usr/libexec/java_home -v 1.8)"
|
|
buck build @mode/macos-x86_64/release osquery:osqueryd
|
|
displayName: "Build osquery"
|
|
|
|
- script: |
|
|
export JAVA_HOME="$(/usr/libexec/java_home -v 1.8)"
|
|
buck test @mode/macos-x86_64/release osquery/... tests/... plugins/...
|
|
displayName: "Run tests"
|
|
|
|
- script: |
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
name: JobResult
|
|
|
|
|
|
- job: macOS
|
|
|
|
pool:
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
condition: succeededOrFailed()
|
|
|
|
dependsOn:
|
|
- macOSCMake
|
|
- macOSBuck
|
|
|
|
variables:
|
|
macOSCMakeReleaseStatus: $[ dependencies.macOSCMake.outputs['Release.JobResult.Status'] ]
|
|
macOSCMakeDebugStatus: $[ dependencies.macOSCMake.outputs['Debug.JobResult.Status'] ]
|
|
macOSBuckReleaseStatus: $[ dependencies.macOSBuck.outputs['JobResult.Status'] ]
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
- script: |
|
|
if [ -z "$(macOSCMakeReleaseStatus)" ] || [ -z "$(macOSCMakeDebugStatus)" ] || \
|
|
[ -z "$(macOSBuckReleaseStatus)" ]
|
|
then
|
|
exit 1
|
|
fi
|
|
displayName: "Detect macOS jobs build statuses"
|
|
|
|
# MACOS
|
|
|
|
# WINDOWS
|
|
|
|
- job: WindowsCMake
|
|
displayName: "WindowsCMake Release"
|
|
|
|
pool:
|
|
vmImage: vs2017-win2016
|
|
|
|
timeoutInMinutes: 120
|
|
|
|
variables:
|
|
SCCACHE_DIR: $(Pipeline.Workspace)\sccache
|
|
SCCACHE_CACHE_SIZE: "5G"
|
|
AZP_CACHING_CONTENT_FORMAT: Files
|
|
|
|
steps:
|
|
- powershell: |
|
|
git config --global core.autocrlf false
|
|
git config --global core.symlinks true
|
|
|
|
- checkout: self
|
|
|
|
- powershell: |
|
|
cmake --version
|
|
$python3_path = ((Get-Item C:\hostedtoolcache\windows\Python\3*\x64) | Sort-Object -Descending)[0].FullName
|
|
& $python3_path\python -m pip install setuptools psutil timeout_decorator thrift==0.11.0 osquery pywin32
|
|
displayName: Install tests prerequisites
|
|
|
|
- powershell: |
|
|
mkdir $(Build.BinariesDirectory)\build
|
|
displayName: "Create build folder"
|
|
|
|
- powershell: |
|
|
tools\ci\scripts\install_openssl_formula_dependencies.ps1
|
|
displayName: "Installing: Strawberry Perl"
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CacheBeta@2
|
|
inputs:
|
|
key: submodules | Windows | $(SubmoduleCacheVersion) | $(Build.SourceVersion)
|
|
restoreKeys: submodules | Windows | $(SubmoduleCacheVersion)
|
|
path: $(Build.SourcesDirectory)/.git/modules
|
|
displayName: Submodule cache
|
|
|
|
- powershell: |
|
|
(New-Object System.Net.WebClient).DownloadFile(`
|
|
"https://github.com/osquery/sccache/releases/download/0.0.1-osquery/sccache-0.0.1-windows.7z",`
|
|
"$env:TEMP\sccache.7z")
|
|
|
|
mkdir "C:\Program Files\sccache"
|
|
7z x -o"C:\Program Files\sccache" -y "$env:TEMP\sccache.7z"
|
|
Write-Host "##vso[task.prependpath]C:\Program Files\sccache"
|
|
displayName: "Install sccache"
|
|
|
|
- powershell: |
|
|
(New-Object System.Net.WebClient).DownloadFile(`
|
|
"https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip",`
|
|
"$env:TEMP\ninja-win.zip")
|
|
|
|
mkdir "C:\Program Files\Ninja"
|
|
7z x -o"C:\Program Files\Ninja" -y "$env:TEMP\ninja-win.zip"
|
|
Write-Host "##vso[task.prependpath]C:\Program Files\Ninja"
|
|
displayName: "Install Ninja"
|
|
|
|
- script: |
|
|
call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars64.bat" -vcvars_ver=14.1
|
|
cmake -G Ninja ^
|
|
-DCMAKE_C_COMPILER=cl.exe ^
|
|
-DCMAKE_CXX_COMPILER=cl.exe ^
|
|
-DCMAKE_BUILD_TYPE=Release ^
|
|
-DOSQUERY_BUILD_TESTS=ON ^
|
|
-DCMAKE_C_COMPILER_LAUNCHER="sccache.exe" ^
|
|
-DCMAKE_CXX_COMPILER_LAUNCHER="sccache.exe" ^
|
|
$(Build.SourcesDirectory)
|
|
displayName: "Configure osquery"
|
|
workingDirectory: $(Build.BinariesDirectory)\build
|
|
|
|
- task: CacheBeta@2
|
|
inputs:
|
|
key: sccache | Windows | $(CacheVersion) | $(Build.SourceVersion)
|
|
restoreKeys: sccache | Windows | $(CacheVersion)
|
|
path: $(SCCACHE_DIR)
|
|
displayName: sccache
|
|
|
|
- script: |
|
|
call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars64.bat" -vcvars_ver=14.1
|
|
cmake --build . -j 3
|
|
sccache.exe --stop-server
|
|
displayName: "Build osquery"
|
|
workingDirectory: $(Build.BinariesDirectory)\build
|
|
|
|
- script: cmd /C type $(Build.BinariesDirectory)\build\formula_logs\openssl.txt
|
|
displayName: "OpenSSL formula output"
|
|
condition: always()
|
|
|
|
- powershell: |
|
|
ctest --build-nocmake -C Release -V
|
|
displayName: "Run tests"
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
|
|
- task: CMake@1
|
|
displayName: "Run WIX packaging"
|
|
inputs:
|
|
workingDirectory: $(Build.BinariesDirectory)/build
|
|
cmakeArgs: --build . --target package --config Release -j 3
|
|
|
|
- powershell: |
|
|
# .artifactignore has to be copied in the cached folder, otherwise the CacheBeta task won't see it
|
|
cp $(Build.SourcesDirectory)\.artifactignore $(Build.SourcesDirectory)\.git\modules
|
|
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
name: JobResult
|
|
|
|
|
|
- job: WindowsBuck
|
|
displayName: "WindowsBuck Release"
|
|
|
|
pool:
|
|
vmImage: vs2017-win2016
|
|
|
|
steps:
|
|
- powershell: |
|
|
git config --global core.autocrlf false
|
|
|
|
- checkout: self
|
|
|
|
- powershell: |
|
|
choco uninstall mingw -y
|
|
(New-Object System.Net.WebClient).DownloadFile("https://github.com/facebook/buck/releases/download/v2019.06.17.01/buck.2019.06.17.01.nupkg", `
|
|
"buck.2019.06.17.01.nupkg")
|
|
Install-Package buck -Source . -SkipDependencies
|
|
displayName: "Prepare Buck environment"
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- powershell: |
|
|
$python2_path = (Get-Item C:\Python27amd64).FullName
|
|
|
|
if (-not $python2_path) {
|
|
$python2_path = ((Get-Item C:\hostedtoolcache\windows\Python\2*\x64) | Sort-Object -Descending)[0].FullName
|
|
}
|
|
|
|
$python3_path = ((Get-Item C:\hostedtoolcache\windows\Python\3*\x64) | Sort-Object -Descending)[0].FullName
|
|
|
|
echo "##vso[task.setvariable variable=python2]$python2_path"
|
|
echo "##vso[task.setvariable variable=python3]$python3_path"
|
|
|
|
displayName: "Detect python 2 and 3 installation folders"
|
|
|
|
- script: |
|
|
mklink /J $(Build.BinariesDirectory)\vsinstall "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise"
|
|
|
|
- powershell: |
|
|
$vcvers = cat $(Build.BinariesDirectory)\vsinstall\VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt
|
|
.\tools\generate_buck_config.ps1 -VsInstall '$(Build.BinariesDirectory)\vsinstall' `
|
|
-VcToolsVersion "$vcvers" `
|
|
-SdkInstall '' `
|
|
-SdkVersion '' `
|
|
-Python3Path '$(python3)\python.exe' `
|
|
-BuckConfigRoot .\tools\buckconfigs\
|
|
displayName: "Generate Buck config"
|
|
|
|
- script: |
|
|
set PATH=$(python2);%PATH%
|
|
$(python2)\python.exe "C:\Program Files\PackageManagement\NuGet\Packages\buck.2019.06.17.01\tools\buck.pex" build @mode/windows-x86_64/release osquery:osqueryd
|
|
displayName: "Build osquery"
|
|
|
|
- script: |
|
|
set PATH=$(python2);%PATH%
|
|
$(python2)\python.exe "C:\Program Files\PackageManagement\NuGet\Packages\buck.2019.06.17.01\tools\buck.pex" test @mode/windows-x86_64/release osquery/... tests/... plugins/...
|
|
displayName: "Run tests"
|
|
|
|
- powershell: |
|
|
echo "##vso[task.setvariable variable=Status;isOutput=true]1"
|
|
name: JobResult
|
|
|
|
|
|
- job: Windows
|
|
|
|
pool:
|
|
vmImage: 'Ubuntu-16.04'
|
|
|
|
condition: succeededOrFailed()
|
|
|
|
dependsOn:
|
|
- WindowsCMake
|
|
- WindowsBuck
|
|
|
|
variables:
|
|
WindowsCMakeReleaseStatus: $[ dependencies.WindowsCMake.outputs['JobResult.Status'] ]
|
|
WindowsBuckReleaseStatus: $[ dependencies.WindowsBuck.outputs['JobResult.Status'] ]
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
- script: |
|
|
if [ -z "$(WindowsCMakeReleaseStatus)" ] || [ -z "$(WindowsBuckReleaseStatus)" ]; then
|
|
exit 1
|
|
fi
|
|
displayName: "Detect Windows CMake and Buck build status"
|
|
|
|
# WINDOWS
|