osquery-1/azure-pipelines.yml

trigger:
- master

jobs:

# LINUX

  - job: LinuxBuild
    displayName: "Linux"

    strategy:
      matrix:
        Release:
          BUILD_TYPE: Release
          EXTRA_CMAKE_ARGS:
          BUILD_TESTS: ON
        RelWithDebInfo:
          BUILD_TYPE: RelWithDebInfo
          EXTRA_CMAKE_ARGS:
          BUILD_TESTS: OFF
        Debug:
          BUILD_TYPE: Debug
          EXTRA_CMAKE_ARGS: -DOSQUERY_NO_DEBUG_SYMBOLS=ON
          BUILD_TESTS: ON

    pool:
      vmImage: 'Ubuntu-16.04'

    container:
      image: trailofbits/osquery:ubuntu-18.04-toolchain-v9
      options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock

    timeoutInMinutes: 120

    variables:
      CCACHE_DIR: $(Pipeline.Workspace)/ccache
      # Debug packages require padded source prefixes (#5936).
      BUILD_DIR: $(Build.BinariesDirectory)/usr/src/debug/osquery/build

    steps:
    - checkout: self
      # See BUILD_DIR.
      path: s/usr/src/debug/osquery

    - script: mkdir -p $(BUILD_DIR)
      displayName: "Create build folder"

    - task: CMake@1
      displayName: "Configure osquery for formatting"
      condition: and(succeeded(), ne(variables['Agent.JobName'], 'Linux RelWithDebInfo'))
      inputs:
        workingDirectory: $(BUILD_DIR)
        cmakeArgs:
          -DOSQUERY_TOOLCHAIN_SYSROOT=/usr/local/osquery-toolchain
          -DOSQUERY_ENABLE_FORMAT_ONLY=ON
          $(Build.SourcesDirectory)

    - script: |
        ./tools/formatting/format-test.sh --build $(BUILD_DIR)        
      displayName: "format_check.py test"
      condition: and(succeeded(), ne(variables['Agent.JobName'], 'Linux RelWithDebInfo'))
      workingDirectory: $(Build.SourcesDirectory)

    - task: CMake@1
      displayName: "Check code formatting"
      condition: and(succeeded(), ne(variables['Agent.JobName'], 'Linux RelWithDebInfo'))
      inputs:
        workingDirectory: $(BUILD_DIR)
        cmakeArgs: --build . --target format_check

    - task: CacheBeta@2
      inputs:
        key: submodules | Linux | $(SubmoduleCacheVersion) | $(Build.SourceVersion)
        restoreKeys: submodules | Linux | $(SubmoduleCacheVersion)
        path: $(Build.SourcesDirectory)/.git/modules
      displayName: Submodule cache

    - script: |
        cmake --version
        cmake -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
              -DOSQUERY_TOOLCHAIN_SYSROOT=/usr/local/osquery-toolchain \
              -DOSQUERY_BUILD_TESTS=$(BUILD_TESTS) \
              -DOSQUERY_BUILD_ROOT_TESTS=$(BUILD_TESTS) \
              -UOSQUERY_ENABLE_FORMAT_ONLY \
              $(EXTRA_CMAKE_ARGS) \
              $(Build.SourcesDirectory)        
      displayName: "Configure osquery"
      workingDirectory: $(BUILD_DIR)

    - task: CacheBeta@2
      inputs:
        key: ccache | Linux$(BUILD_TYPE)CMake | $(CacheVersion) | $(Build.SourceVersion)
        restoreKeys: ccache | Linux$(BUILD_TYPE)CMake | $(CacheVersion)
        path: $(CCACHE_DIR)
      displayName: ccache

    - task: CMake@1
      displayName: "Build osquery"
      inputs:
        workingDirectory: $(BUILD_DIR)
        cmakeArgs: --build . -j 3

    - task: CMake@1
      condition: and(succeeded(), ne(variables['Agent.JobName'], 'Linux RelWithDebInfo'))
      displayName: "Run cppcheck"
      inputs:
        workingDirectory: $(BUILD_DIR)
        cmakeArgs: --build . --target cppcheck

    - script: |
        ctest --build-nocmake -LE "root-required" -V        
      condition: and(succeeded(), ne(variables['Agent.JobName'], 'Linux RelWithDebInfo'))
      displayName: "Run tests with a normal user"
      workingDirectory: $(BUILD_DIR)

    - script: |
        sudo ctest --build-nocmake -L "root-required" -V        
      condition: and(succeeded(), ne(variables['Agent.JobName'], 'Linux RelWithDebInfo'))
      displayName: "Run tests which requires root"
      workingDirectory: $(BUILD_DIR)

    - script: |
        cmake -DPACKAGING_SYSTEM=DEB $(Build.SourcesDirectory)
        cmake --build . --target package -j 3
        cp *.deb $(Build.ArtifactStagingDirectory)
        cp *.ddeb $(Build.ArtifactStagingDirectory)        
      condition: and(succeeded(), ne(variables['Agent.JobName'], 'Linux Release'))
      displayName: "Run DEB packaging"
      workingDirectory: $(BUILD_DIR)

    - script: |
        cmake -DPACKAGING_SYSTEM=RPM $(Build.SourcesDirectory)
        cmake --build . --target package -j 3
        cp *.rpm $(Build.ArtifactStagingDirectory)        
      condition: and(succeeded(), ne(variables['Agent.JobName'], 'Linux Release'))
      displayName: "Run RPM packaging"
      workingDirectory: $(BUILD_DIR)

    - script: |
        cmake -DPACKAGING_SYSTEM=TGZ $(Build.SourcesDirectory)
        cmake --build . --target package -j 3
        cp *.tar.gz $(Build.ArtifactStagingDirectory)        
      condition: and(succeeded(), ne(variables['Agent.JobName'], 'Linux Release'))
      displayName: "Run TGZ packaging"
      workingDirectory: $(BUILD_DIR)

    - task: PublishBuildArtifacts@1
      condition: and(succeeded(), ne(variables['Agent.JobName'], 'Linux Release'))
      inputs:
        pathToPublish: "$(Build.ArtifactStagingDirectory)"
        artifactName: "Linux $(BUILD_TYPE) Packages"
      displayName: Publish Linux packages

    - script: |
        echo "##vso[task.setvariable variable=Status;isOutput=true]1"        
      name: JobResult

    - script: |
        rm -rf $(BUILD_DIR)        
      displayName: "Reclaim disk space"


  - job: Linux

    pool:
      vmImage: 'Ubuntu-16.04'

    condition: succeededOrFailed()

    dependsOn:
      - LinuxBuild

    variables:
      LinuxReleaseStatus: $[ dependencies.LinuxBuild.outputs['Release.JobResult.Status'] ]
      LinuxRelWithDebInfoStatus: $[ dependencies.LinuxBuild.outputs['RelWithDebInfo.JobResult.Status'] ]
      LinuxDebugStatus: $[ dependencies.LinuxBuild.outputs['Debug.JobResult.Status'] ]

    steps:
    - checkout: none

    - script: |
        if [ -z "$(LinuxReleaseStatus)" ] || [ -z "$(LinuxDebugStatus)" ] || [ -z "$(LinuxRelWithDebInfoStatus)" ]; then
          exit 1
        fi        
      displayName: "Detect Linux jobs build statuses"

# LINUX

# MACOS

  - job: macOSBuild
    displayName: "macOS"

    strategy:
      matrix:
        Release:
          BUILD_TYPE: Release
          EXTRA_CMAKE_ARGS:
        Debug:
          BUILD_TYPE: Debug
          EXTRA_CMAKE_ARGS: -DOSQUERY_NO_DEBUG_SYMBOLS=ON

    pool:
      vmImage: macos-10.15

    timeoutInMinutes: 120

    variables:
      CCACHE_DIR: $(Pipeline.Workspace)/ccache

    steps:
    - script: |
        rm -f /usr/local/bin/2to3
        brew update
        brew upgrade
        brew install ccache flex bison
        sudo xcode-select -s /Applications/Xcode_11.7.app/Contents/Developer        
      displayName: "Upgrade Homebrew and install build prerequisites"
      timeoutInMinutes: 20

    - script: |
        wget https://cmake.org/files/v3.17/cmake-3.17.5-Darwin-x86_64.tar.gz -O /tmp/cmake-3.17.5-Darwin-x86_64.tar.gz
        tar xf /tmp/cmake-3.17.5-Darwin-x86_64.tar.gz -C $HOME
        echo "##vso[task.prependpath]$HOME/cmake-3.17.5-Darwin-x86_64/CMake.app/Contents/bin"        
      displayName: "Install CMake"

    - script: |
        pip3 install setuptools pexpect==3.3 psutil timeout_decorator six thrift==0.11.0 osquery        
      displayName: "Install tests prerequisites"

    - script: mkdir $(Build.BinariesDirectory)/build
      displayName: "Create build folder"

    - task: CacheBeta@2
      inputs:
        key: submodules | macOS | $(SubmoduleCacheVersion) | $(Build.SourceVersion)
        restoreKeys: submodules | macOS | $(SubmoduleCacheVersion)
        path: $(Build.SourcesDirectory)/.git/modules
      displayName: Submodule cache

    - script: |
        cmake --version
        cmake -DCMAKE_OSX_DEPLOYMENT_TARGET=10.11 \
              -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
              -DOSQUERY_BUILD_TESTS=ON \
              $(EXTRA_CMAKE_ARGS) \
              $(Build.SourcesDirectory)        
      displayName: "Configure osquery"
      workingDirectory: $(Build.BinariesDirectory)/build

    - task: CacheBeta@2
      inputs:
        key: ccache | macOS$(BUILD_TYPE)CMake | $(CacheVersion) | $(Build.SourceVersion)
        restoreKeys: ccache | macOS$(BUILD_TYPE)CMake | $(CacheVersion)
        path: $(CCACHE_DIR)
      displayName: ccache

    - script: |
        cmake --build . -j 3        
      displayName: "Build osquery"
      workingDirectory: $(Build.BinariesDirectory)/build

    - script: |
        ctest --build-nocmake -V        
      displayName: "Run tests"
      workingDirectory: $(Build.BinariesDirectory)/build

    - script: |
        cmake --build . --target package -j 3
        cp *.pkg $(Build.ArtifactStagingDirectory)        
      displayName: "Run productbuild packaging"
      workingDirectory: $(Build.BinariesDirectory)/build

    - script: |
        cmake -DPACKAGING_SYSTEM=TGZ $(Build.SourcesDirectory)
        cmake --build . --target package -j 3
        cp *.tar.gz $(Build.ArtifactStagingDirectory)        
      displayName: "Run TGZ packaging"
      workingDirectory: $(Build.BinariesDirectory)/build

    - task: PublishBuildArtifacts@1
      inputs:
        pathToPublish: "$(Build.ArtifactStagingDirectory)"
        artifactName: "macOS $(BUILD_TYPE) Packages"
      displayName: Publish macOS packages

    - script: |
        echo "##vso[task.setvariable variable=Status;isOutput=true]1"        
      name: JobResult

    - script: |
        rm -rf $(Build.BinariesDirectory)/build        
      displayName: "Reclaim disk space"


  - job: macOS

    pool:
      vmImage: 'Ubuntu-16.04'

    condition: succeededOrFailed()

    dependsOn:
      - macOSBuild

    variables:
      macOSReleaseStatus: $[ dependencies.macOSBuild.outputs['Release.JobResult.Status'] ]
      macOSDebugStatus: $[ dependencies.macOSBuild.outputs['Debug.JobResult.Status'] ]

    steps:
    - checkout: none

    - script: |
        if [ -z "$(macOSReleaseStatus)" ] || [ -z "$(macOSDebugStatus)" ]; then
          exit 1
        fi        
      displayName: "Detect macOS jobs build statuses"

# MACOS

# WINDOWS

  - job: WindowsBuild
    displayName: "Windows Release"

    strategy:
      matrix:
        x64:
          VC_VARS_FILE: vcvars64.bat
        win32:
          VC_VARS_FILE: vcvars32.bat

    pool:
      vmImage: windows-2019

    timeoutInMinutes: 240

    variables:
      SCCACHE_DIR: $(Pipeline.Workspace)\sccache
      SCCACHE_CACHE_SIZE: "5G"
      AZP_CACHING_CONTENT_FORMAT: Files

    steps:
    - powershell: |
        git config --global core.autocrlf false
        git config --global core.symlinks true        

    - checkout: self

    # This needs to be done before trying to install packages
    # because the python paths added in the PATH will only be visible in successive steps
    - powershell: |
        $python3_path = (Get-Item C:\hostedtoolcache\windows\Python\3*\x64 | Sort-Object -Descending)[0].FullName
        echo "Found python installation at $python3_path"
        Write-Host "##vso[task.prependpath]$python3_path"
        Write-Host "##vso[task.prependpath]$python3_path\Scripts"
        Write-Host "##vso[task.setvariable variable=PYTHON_ROOT]$python3_path"        
      displayName: Find newest version of python

    - powershell: |
        python -m pip install --upgrade pip
        python -m pip install wheel
        python -m pip install setuptools psutil timeout_decorator thrift==0.11.0 osquery pywin32        
      displayName: Install tests prerequisites

    - powershell: |
        (New-Object System.Net.WebClient).DownloadFile("https://cmake.org/files/v3.17/cmake-3.17.5-win64-x64.zip", "$env:TEMP\cmake-3.17.5-win64-x64.zip")
        7z x -oC:\ -y "$env:TEMP\cmake-3.17.5-win64-x64.zip"
        Write-Host "##vso[task.prependpath]C:\cmake-3.17.5-win64-x64\bin"        
      displayName: Install CMake

    - powershell: |
        tools\ci\scripts\install_openssl_formula_dependencies.ps1        
      displayName: "Installing: Strawberry Perl"
      workingDirectory: $(Build.SourcesDirectory)

    - powershell: |
        (New-Object System.Net.WebClient).DownloadFile(`
        "https://github.com/osquery/sccache/releases/download/0.0.1-osquery/sccache-0.0.1-windows.7z",`
        "$env:TEMP\sccache.7z")

        mkdir "C:\Program Files\sccache"
        7z x -o"C:\Program Files\sccache" -y "$env:TEMP\sccache.7z"
        Write-Host "##vso[task.prependpath]C:\Program Files\sccache"        
      displayName: "Install sccache"

    - powershell: |
        (New-Object System.Net.WebClient).DownloadFile(`
        "https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-win.zip",`
        "$env:TEMP\ninja-win.zip")

        mkdir "C:\Program Files\Ninja"
        7z x -o"C:\Program Files\Ninja" -y "$env:TEMP\ninja-win.zip"
        Write-Host "##vso[task.prependpath]C:\Program Files\Ninja"        
      displayName: "Install Ninja"

    - powershell: |
        mkdir $(Build.BinariesDirectory)\build        
      displayName: "Create build folder"

    - task: CacheBeta@2
      inputs:
        key: submodules | Windows | $(SubmoduleCacheVersion) | $(Build.SourceVersion)
        restoreKeys: submodules | Windows | $(SubmoduleCacheVersion)
        path: $(Build.SourcesDirectory)/.git/modules
      displayName: Submodule cache

    - script: |
        cmake --version
        call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\$(VC_VARS_FILE)"
        cmake -G Ninja ^
        -DCMAKE_C_COMPILER=cl.exe ^
        -DCMAKE_CXX_COMPILER=cl.exe ^
        -DCMAKE_BUILD_TYPE=Release ^
        -DOSQUERY_BUILD_TESTS=ON ^
        -DCMAKE_C_COMPILER_LAUNCHER="sccache.exe" ^
        -DCMAKE_CXX_COMPILER_LAUNCHER="sccache.exe" ^
        -DPython3_ROOT_DIR="$(PYTHON_ROOT)" ^
        $(Build.SourcesDirectory)        
      displayName: "Configure osquery"
      workingDirectory: $(Build.BinariesDirectory)\build

    - task: CacheBeta@2
      inputs:
        key: sccache | Windows$(System.JobName) | $(CacheVersion) | $(Build.SourceVersion)
        restoreKeys: sccache | Windows$(System.JobName)  | $(CacheVersion)
        path: $(SCCACHE_DIR)
      displayName: sccache

    - script: |
        call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\$(VC_VARS_FILE)"
        cmake --build . -j 3
        if %errorlevel% neq 0 exit /b %errorlevel%
        sccache.exe --stop-server        
      displayName: "Build osquery"
      workingDirectory: $(Build.BinariesDirectory)\build

    - script: |
        ctest --build-nocmake -C Release -V        
      displayName: "Run tests"
      workingDirectory: $(Build.BinariesDirectory)/build

    - script: |
        cmake --build . --target package --config Release -j 3
        cp *.msi $(Build.ArtifactStagingDirectory)        
      displayName: "Run WIX packaging"
      workingDirectory: $(Build.BinariesDirectory)/build

    - task: PublishBuildArtifacts@1
      inputs:
        pathToPublish: "$(Build.ArtifactStagingDirectory)"
        artifactName: "Windows-$(System.JobName) Packages"
      displayName: Publish Windows packages

    - powershell: |
        # .artifactignore has to be copied in the cached folder, otherwise the CacheBeta task won't see it
        cp $(Build.SourcesDirectory)\.artifactignore $(Build.SourcesDirectory)\.git\modules

        echo "##vso[task.setvariable variable=Status;isOutput=true]1"        
      name: JobResult

    - powershell: |
        rm -r -Force $(Build.BinariesDirectory)/build        
      displayName: "Reclaim disk space"

  - job: Windows

    pool:
      vmImage: 'Ubuntu-16.04'

    condition: succeededOrFailed()

    dependsOn:
      - WindowsBuild

    variables:
      WindowsRelease64Status: $[ dependencies.WindowsBuild.outputs['x64.JobResult.Status'] ]
      WindowsRelease32Status: $[ dependencies.WindowsBuild.outputs['win32.JobResult.Status'] ]

    steps:
    - checkout: none

    - script: |
        if [ -z "$(WindowsRelease64Status)" ] || [ -z "$(WindowsRelease32Status)" ]; then
          exit 1
        fi        
      displayName: "Detect Windows jobs build statuses"
# WINDOWS