osquery-1/azure-pipelines.yml

trigger:
- master

jobs:

# LINUX

  - job: LinuxCMake

    strategy:
      matrix:
        Release:
          BUILD_TYPE: Release
          EXTRA_CMAKE_ARGS:
        Debug:
          BUILD_TYPE: Debug
          EXTRA_CMAKE_ARGS: -DOSQUERY_NO_DEBUG_SYMBOLS=ON

    pool:
      vmImage: 'Ubuntu-16.04'

    container:
      image: trailofbits/osquery:ubuntu-18.04-toolchain-v7
      options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock

    timeoutInMinutes: 120

    variables:
      CCACHE_DIR: $(Pipeline.Workspace)/ccache
      # Debug packages require padded source prefixes (#5936).
      BUILD_DIR: $(Build.BinariesDirectory)/usr/src/debug/osquery/build

    steps:
    - checkout: self
      # See BUILD_DIR.
      path: s/usr/src/debug/osquery

    - script: mkdir -p $(BUILD_DIR)
      displayName: "Create build folder"

    - task: CacheBeta@2
      inputs:
        key: submodules | Linux | $(SubmoduleCacheVersion) | $(Build.SourceVersion)
        restoreKeys: submodules | Linux | $(SubmoduleCacheVersion)
        path: $(Build.SourcesDirectory)/.git/modules
      displayName: Submodule cache

    - task: CMake@1
      displayName: "Configure osquery"
      inputs:
        workingDirectory: $(BUILD_DIR)
        cmakeArgs:
          -DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
          -DOSQUERY_TOOLCHAIN_SYSROOT=/usr/local/osquery-toolchain
          -DOSQUERY_BUILD_TESTS=ON
          $(EXTRA_CMAKE_ARGS)
          $(Build.SourcesDirectory)

    - script: |
        ./tools/formatting/format-test.sh --build $(BUILD_DIR)
      displayName: "format_check.py test"
      workingDirectory: $(Build.SourcesDirectory)

    - task: CMake@1
      displayName: "Check code formatting"
      inputs:
        workingDirectory: $(BUILD_DIR)
        cmakeArgs: --build . --target format_check

    - task: CacheBeta@2
      inputs:
        key: ccache | Linux$(BUILD_TYPE)CMake | $(CacheVersion) | $(Build.SourceVersion)
        restoreKeys: ccache | Linux$(BUILD_TYPE)CMake | $(CacheVersion)
        path: $(CCACHE_DIR)
      displayName: ccache

    - task: CMake@1
      displayName: "Build osquery"
      inputs:
        workingDirectory: $(BUILD_DIR)
        cmakeArgs: --build . -j 3

    - script: cat $(BUILD_DIR)/formula_logs/openssl.txt
      displayName: "OpenSSL formula output"
      condition: always()

    - task: CMake@1
      displayName: "Run cppcheck"
      inputs:
        workingDirectory: $(BUILD_DIR)
        cmakeArgs: --build . --target cppcheck

    - script: |
        ctest --build-nocmake -V
      displayName: "Run tests"
      workingDirectory: $(BUILD_DIR)

    - script: |
        cmake -DPACKAGING_SYSTEM=DEB $(Build.SourcesDirectory)
        cmake --build . --target package -j 3
      displayName: "Run DEB packaging"
      workingDirectory: $(BUILD_DIR)

    - script: |
        cmake -DPACKAGING_SYSTEM=RPM $(Build.SourcesDirectory)
        cmake --build . --target package -j 3
      displayName: "Run RPM packaging"
      workingDirectory: $(BUILD_DIR)

    - script: |
        cmake -DPACKAGING_SYSTEM=TGZ $(Build.SourcesDirectory)
        cmake --build . --target package -j 3
      displayName: "Run TGZ packaging"
      workingDirectory: $(BUILD_DIR)

    - script: |
        echo "##vso[task.setvariable variable=Status;isOutput=true]1"
      name: JobResult

    - script: |
        rm -rf $(BUILD_DIR)
      displayName: "Reclaim disk space"


  - job: LinuxBuck
    displayName: "LinuxBuck Release"

    pool:
      vmImage: 'Ubuntu-16.04'

    container:
      image: trailofbits/osql-experimental:ubuntu-18.04
      options: --privileged

    steps:
    - script: |
        wget https://github.com/facebook/buck/releases/download/v2019.06.17.01/buck.2019.06.17.01_all.deb
        sudo apt update
        sudo apt install -q -y --no-install-recommends ./buck.2019.06.17.01_all.deb openjdk-8-jre python3-distutils
      workingDirectory: $(Build.BinariesDirectory)

    - script: |
        export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
        buck build @mode/linux-x86_64/release osquery:osqueryd
      displayName: "Build osquery"

    - script: |
        # Buck compiles these tests in Debug mode, exhausting disk space.
        # Until we find a solution, we'll disable the tests.
        #export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
        #buck test @mode/linux-x86_64/release osquery/... tests/... plugins/...
      displayName: "Run tests (DISABLED)"

    - script: |
        echo "##vso[task.setvariable variable=Status;isOutput=true]1"
      name: JobResult


  - job: Linux

    pool:
      vmImage: 'Ubuntu-16.04'

    condition: succeededOrFailed()

    dependsOn:
      - LinuxCMake
      - LinuxBuck

    variables:
      LinuxCMakeReleaseStatus: $[ dependencies.LinuxCMake.outputs['Release.JobResult.Status'] ]
      LinuxCMakeDebugStatus: $[ dependencies.LinuxCMake.outputs['Debug.JobResult.Status'] ]
      LinuxBuckReleaseStatus: $[ dependencies.LinuxBuck.outputs['JobResult.Status'] ]

    steps:
    - checkout: none

    - script: |
        if [ -z "$(LinuxCMakeReleaseStatus)" ] || [ -z "$(LinuxCMakeDebugStatus)" ] || \
           [ -z "$(LinuxBuckReleaseStatus)" ]
        then
          exit 1
        fi
      displayName: "Detect Linux jobs build statuses"

# LINUX

# MACOS

  - job: macOSCMake

    strategy:
      matrix:
        Release:
          BUILD_TYPE: Release
          EXTRA_CMAKE_ARGS:
        Debug:
          BUILD_TYPE: Debug
          EXTRA_CMAKE_ARGS: -DOSQUERY_NO_DEBUG_SYMBOLS=ON

    pool:
      vmImage: macos-10.14

    timeoutInMinutes: 120

    variables:
      CCACHE_DIR: $(Pipeline.Workspace)/ccache

    steps:
    - script: |
        cmake --version
        brew upgrade
        brew install ccache flex bison
        pip3 install setuptools pexpect==3.3 psutil timeout_decorator six thrift==0.11.0 osquery
        sudo xcode-select -s /Applications/Xcode_10.3.app/Contents/Developer
      displayName: "Install Homebrew and prerequisites"
      timeoutInMinutes: 20

    - script: mkdir $(Build.BinariesDirectory)/build
      displayName: "Create build folder"

    - task: CacheBeta@2
      inputs:
        key: submodules | macOS | $(SubmoduleCacheVersion) | $(Build.SourceVersion)
        restoreKeys: submodules | macOS | $(SubmoduleCacheVersion)
        path: $(Build.SourcesDirectory)/.git/modules
      displayName: Submodule cache

    - task: CMake@1
      displayName: "Configure osquery"
      inputs:
        workingDirectory: $(Build.BinariesDirectory)/build
        cmakeArgs: -DCMAKE_OSX_DEPLOYMENT_TARGET=10.11 -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DOSQUERY_BUILD_TESTS=ON $(EXTRA_CMAKE_ARGS) $(Build.SourcesDirectory)

    - task: CacheBeta@2
      inputs:
        key: ccache | macOS$(BUILD_TYPE)CMake | $(CacheVersion) | $(Build.SourceVersion)
        restoreKeys: ccache | macOS$(BUILD_TYPE)CMake | $(CacheVersion)
        path: $(CCACHE_DIR)
      displayName: ccache

    - task: CMake@1
      displayName: "Build osquery"
      inputs:
        workingDirectory: $(Build.BinariesDirectory)/build
        cmakeArgs: --build . -j 3

    - script: cat $(Build.BinariesDirectory)/build/formula_logs/openssl.txt
      displayName: "OpenSSL formula output"
      condition: always()

    - script: |
        ctest --build-nocmake -V
      displayName: "Run tests"
      workingDirectory: $(Build.BinariesDirectory)/build

    - task: CMake@1
      displayName: "Run productbuild packaging"
      inputs:
        workingDirectory: $(Build.BinariesDirectory)/build
        cmakeArgs: --build . --target package -j 3

    - script: |
        cmake -DPACKAGING_SYSTEM=TGZ $(Build.SourcesDirectory)
        cmake --build . --target package -j 3
      displayName: "Run TGZ packaging"
      workingDirectory: $(Build.BinariesDirectory)/build

    - script: |
        echo "##vso[task.setvariable variable=Status;isOutput=true]1"
      name: JobResult

    - script: |
        rm -rf $(Build.BinariesDirectory)/build
      displayName: "Reclaim disk space"


  - job: macOSBuck
    displayName: "macOSBuck Release"

    pool:
      vmImage: macos-10.14

    steps:
    - script: |
        brew update
        brew tap facebook/fb
        brew upgrade
        brew cask install adoptopenjdk
        brew install buck watchman
        sudo xcode-select -s /Applications/Xcode_10.3.app/Contents/Developer
      displayName: "Install Homebrew and prerequisites"
      timeoutInMinutes: 20

    - script: |
        export JAVA_HOME="$(/usr/libexec/java_home -v 1.8)"
        buck build @mode/macos-x86_64/release osquery:osqueryd
      displayName: "Build osquery"

    - script: |
        export JAVA_HOME="$(/usr/libexec/java_home -v 1.8)"
        buck test @mode/macos-x86_64/release osquery/... tests/... plugins/...
      displayName: "Run tests"

    - script: |
        echo "##vso[task.setvariable variable=Status;isOutput=true]1"
      name: JobResult


  - job: macOS

    pool:
      vmImage: 'Ubuntu-16.04'

    condition: succeededOrFailed()

    dependsOn:
      - macOSCMake
      - macOSBuck

    variables:
      macOSCMakeReleaseStatus: $[ dependencies.macOSCMake.outputs['Release.JobResult.Status'] ]
      macOSCMakeDebugStatus: $[ dependencies.macOSCMake.outputs['Debug.JobResult.Status'] ]
      macOSBuckReleaseStatus: $[ dependencies.macOSBuck.outputs['JobResult.Status'] ]

    steps:
    - checkout: none

    - script: |
        if [ -z "$(macOSCMakeReleaseStatus)" ] || [ -z "$(macOSCMakeDebugStatus)" ] || \
           [ -z "$(macOSBuckReleaseStatus)" ]
        then
          exit 1
        fi
      displayName: "Detect macOS jobs build statuses"

# MACOS

# WINDOWS

  - job: WindowsCMake
    displayName: "WindowsCMake Release"

    pool:
      vmImage: vs2017-win2016

    timeoutInMinutes: 120

    variables:
      SCCACHE_DIR: $(Pipeline.Workspace)\sccache
      SCCACHE_CACHE_SIZE: "5G"
      AZP_CACHING_CONTENT_FORMAT: Files

    steps:
    - powershell: |
        git config --global core.autocrlf false
        git config --global core.symlinks true

    - checkout: self

    - powershell: |
        cmake --version
        $python3_path = ((Get-Item C:\hostedtoolcache\windows\Python\3*\x64) | Sort-Object -Descending)[0].FullName
        & $python3_path\python -m pip install setuptools psutil timeout_decorator thrift==0.11.0 osquery pywin32
      displayName: Install tests prerequisites

    - powershell: |
        mkdir $(Build.BinariesDirectory)\build
      displayName: "Create build folder"

    - powershell: |
        tools\ci\scripts\install_openssl_formula_dependencies.ps1
      displayName: "Installing: Strawberry Perl"
      workingDirectory: $(Build.SourcesDirectory)

    - task: CacheBeta@2
      inputs:
        key: submodules | Windows | $(SubmoduleCacheVersion) | $(Build.SourceVersion)
        restoreKeys: submodules | Windows | $(SubmoduleCacheVersion)
        path: $(Build.SourcesDirectory)/.git/modules
      displayName: Submodule cache

    - powershell: |
        (New-Object System.Net.WebClient).DownloadFile(`
        "https://github.com/osquery/sccache/releases/download/0.0.1-osquery/sccache-0.0.1-windows.7z",`
        "$env:TEMP\sccache.7z")

        mkdir "C:\Program Files\sccache"
        7z x -o"C:\Program Files\sccache" -y "$env:TEMP\sccache.7z"
        Write-Host "##vso[task.prependpath]C:\Program Files\sccache"
      displayName: "Install sccache"

    - powershell: |
        (New-Object System.Net.WebClient).DownloadFile(`
        "https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip",`
        "$env:TEMP\ninja-win.zip")

        mkdir "C:\Program Files\Ninja"
        7z x -o"C:\Program Files\Ninja" -y "$env:TEMP\ninja-win.zip"
        Write-Host "##vso[task.prependpath]C:\Program Files\Ninja"
      displayName: "Install Ninja"

    - script: |
        call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars64.bat" -vcvars_ver=14.1
        cmake -G Ninja ^
        -DCMAKE_C_COMPILER=cl.exe ^
        -DCMAKE_CXX_COMPILER=cl.exe ^
        -DCMAKE_BUILD_TYPE=Release ^
        -DOSQUERY_BUILD_TESTS=ON ^
        -DCMAKE_C_COMPILER_LAUNCHER="sccache.exe" ^
        -DCMAKE_CXX_COMPILER_LAUNCHER="sccache.exe" ^
        $(Build.SourcesDirectory)
      displayName: "Configure osquery"
      workingDirectory: $(Build.BinariesDirectory)\build

    - task: CacheBeta@2
      inputs:
        key: sccache | Windows | $(CacheVersion) | $(Build.SourceVersion)
        restoreKeys: sccache | Windows | $(CacheVersion)
        path: $(SCCACHE_DIR)
      displayName: sccache

    - script: |
        call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars64.bat" -vcvars_ver=14.1
        cmake --build . -j 3
        if %errorlevel% neq 0 exit /b %errorlevel%
        sccache.exe --stop-server
      displayName: "Build osquery"
      workingDirectory: $(Build.BinariesDirectory)\build

    - script: cmd /C type $(Build.BinariesDirectory)\build\formula_logs\openssl.txt
      displayName: "OpenSSL formula output"
      condition: always()

    - powershell: |
        ctest --build-nocmake -C Release -V
      displayName: "Run tests"
      workingDirectory: $(Build.BinariesDirectory)/build

    - task: CMake@1
      displayName: "Run WIX packaging"
      inputs:
        workingDirectory: $(Build.BinariesDirectory)/build
        cmakeArgs: --build . --target package --config Release -j 3

    - powershell: |
        # .artifactignore has to be copied in the cached folder, otherwise the CacheBeta task won't see it
        cp $(Build.SourcesDirectory)\.artifactignore $(Build.SourcesDirectory)\.git\modules

        echo "##vso[task.setvariable variable=Status;isOutput=true]1"
      name: JobResult

    - powershell: |
        rm -r -Force $(Build.BinariesDirectory)/build
      displayName: "Reclaim disk space"


  - job: WindowsBuck
    displayName: "WindowsBuck Release"

    pool:
      vmImage: vs2017-win2016

    steps:
    - powershell: |
        git config --global core.autocrlf false

    - checkout: self

    - powershell: |
        choco uninstall mingw -y
        (New-Object System.Net.WebClient).DownloadFile("https://github.com/facebook/buck/releases/download/v2019.06.17.01/buck.2019.06.17.01.nupkg", `
        "buck.2019.06.17.01.nupkg")
        Install-Package buck -Source . -SkipDependencies
      displayName: "Prepare Buck environment"
      workingDirectory: $(Build.BinariesDirectory)

    - powershell: |
        $python2_path = ((Get-Item C:\hostedtoolcache\windows\Python\2*\x64) | Sort-Object -Descending)[0].FullName
        $python3_path = ((Get-Item C:\hostedtoolcache\windows\Python\3*\x64) | Sort-Object -Descending)[0].FullName

        echo "##vso[task.setvariable variable=python2]$python2_path"
        echo "##vso[task.setvariable variable=python3]$python3_path"

      displayName: "Detect python 2 and 3 installation folders"

    - script: |
        mklink /J $(Build.BinariesDirectory)\vsinstall "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise"

    - powershell: |
        $vcvers = cat $(Build.BinariesDirectory)\vsinstall\VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt
        .\tools\generate_buck_config.ps1 -VsInstall '$(Build.BinariesDirectory)\vsinstall' `
        -VcToolsVersion "$vcvers" `
        -SdkInstall '' `
        -SdkVersion '' `
        -Python3Path '$(python3)\python.exe' `
        -BuckConfigRoot .\tools\buckconfigs\
      displayName: "Generate Buck config"

    - script: |
        set PATH=$(python2);%PATH%
        $(python2)\python.exe "C:\Program Files\PackageManagement\NuGet\Packages\buck.2019.06.17.01\tools\buck.pex" build @mode/windows-x86_64/release osquery:osqueryd
      displayName: "Build osquery"

    - script: |
        set PATH=$(python2);%PATH%
        $(python2)\python.exe "C:\Program Files\PackageManagement\NuGet\Packages\buck.2019.06.17.01\tools\buck.pex" test @mode/windows-x86_64/release osquery/... tests/... plugins/...
      displayName: "Run tests"

    - powershell: |
        echo "##vso[task.setvariable variable=Status;isOutput=true]1"
      name: JobResult


  - job: Windows

    pool:
      vmImage: 'Ubuntu-16.04'

    condition: succeededOrFailed()

    dependsOn:
      - WindowsCMake
      - WindowsBuck

    variables:
      WindowsCMakeReleaseStatus: $[ dependencies.WindowsCMake.outputs['JobResult.Status'] ]
      WindowsBuckReleaseStatus: $[ dependencies.WindowsBuck.outputs['JobResult.Status'] ]

    steps:
    - checkout: none

    - script: |
        if [ -z "$(WindowsCMakeReleaseStatus)" ] || [ -z "$(WindowsBuckReleaseStatus)" ]; then
          exit 1
        fi
      displayName: "Detect Windows CMake and Buck build status"

# WINDOWS