From c269bbeaf354a682ddac3aa69c5f6284540d0278 Mon Sep 17 00:00:00 2001 From: Teddy Reed Date: Mon, 13 Jul 2015 00:47:20 -0700 Subject: [PATCH] Rollup of build changes --- CMake/FindLZMA.cmake | 48 ------------------------- CMake/FindRocksDB.cmake | 47 +++++++++++++++++------- CMake/FindSnappy.cmake | 31 ---------------- CMake/FindThrift.cmake | 25 ++++--------- CMakeLists.txt | 8 +++-- Vagrantfile | 4 +-- docs/wiki/development/building.md | 14 ++++++++ osquery/CMakeLists.txt | 6 ++-- osquery/remote/transports/tls.cpp | 2 +- osquery/tables/system/darwin/keychain.h | 7 ++-- tools/provision/lib.sh | 20 ++++++----- tools/tests/test_example_queries.py | 4 ++- 12 files changed, 86 insertions(+), 130 deletions(-) delete mode 100644 CMake/FindLZMA.cmake delete mode 100644 CMake/FindSnappy.cmake diff --git a/CMake/FindLZMA.cmake b/CMake/FindLZMA.cmake deleted file mode 100644 index 63bc5d97..00000000 --- a/CMake/FindLZMA.cmake +++ /dev/null @@ -1,48 +0,0 @@ -# - Find lzma and lzmadec -# Find the native LZMA includes and library -# -# LZMA_INCLUDE_DIR - where to find lzma.h, etc. -# LZMA_LIBRARIES - List of libraries when using liblzma. -# LZMA_FOUND - True if liblzma found. -# LZMADEC_INCLUDE_DIR - where to find lzmadec.h, etc. -# LZMADEC_LIBRARIES - List of libraries when using liblzmadec. -# LZMADEC_FOUND - True if liblzmadec found. - -IF (LZMA_INCLUDE_DIR) - # Already in cache, be silent - SET(LZMA_FIND_QUIETLY TRUE) -ENDIF (LZMA_INCLUDE_DIR) - -FIND_PATH(LZMA_INCLUDE_DIR lzma.h) -FIND_LIBRARY(LZMA_LIBRARY NAMES lzma ) - -# handle the QUIETLY and REQUIRED arguments and set LZMA_FOUND to TRUE if -# all listed variables are TRUE -INCLUDE(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(LZMA DEFAULT_MSG LZMA_LIBRARY LZMA_INCLUDE_DIR) - -IF(LZMA_FOUND) - SET( LZMA_LIBRARIES ${LZMA_LIBRARY} ) -ELSE(LZMA_FOUND) - SET( LZMA_LIBRARIES ) - - IF (LZMADEC_INCLUDE_DIR) - # Already in cache, be silent - SET(LZMADEC_FIND_QUIETLY TRUE) - ENDIF (LZMADEC_INCLUDE_DIR) - - FIND_PATH(LZMADEC_INCLUDE_DIR lzmadec.h) - FIND_LIBRARY(LZMADEC_LIBRARY NAMES lzmadec ) - - # handle the QUIETLY and REQUIRED arguments and set LZMADEC_FOUND to TRUE if - # all listed variables are TRUE - INCLUDE(FindPackageHandleStandardArgs) - FIND_PACKAGE_HANDLE_STANDARD_ARGS(LZMADEC DEFAULT_MSG LZMADEC_LIBRARY - LZMADEC_INCLUDE_DIR) - - IF(LZMADEC_FOUND) - SET( LZMADEC_LIBRARIES ${LZMADEC_LIBRARY} ) - ELSE(LZMADEC_FOUND) - SET( LZMADEC_LIBRARIES ) - ENDIF(LZMADEC_FOUND) -ENDIF(LZMA_FOUND) diff --git a/CMake/FindRocksDB.cmake b/CMake/FindRocksDB.cmake index 0c01383f..b3a1a170 100644 --- a/CMake/FindRocksDB.cmake +++ b/CMake/FindRocksDB.cmake @@ -69,42 +69,63 @@ FIND_PATH(ROCKSDB_INCLUDE_DIR rocksdb/db.h # locate the library if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") # On MacOS - set(ROCKSDB_LIBRARY_NAMES ${ROCKSDB_LIBRARY_NAMES} librocksdb.dylib) + set(ROCKSDB_LIBRARY_NAMES librocksdb.dylib) + set(ROCKSDB_LITE_LIBRARY_NAMES librocksdb_lite.dylib) elseif(${CMAKE_SYSTEM_NAME} MATCHES "Linux") # On Linux - set(ROCKSDB_LIBRARY_NAMES ${ROCKSDB_LIBRARY_NAMES} librocksdb.so) + set(ROCKSDB_LIBRARY_NAMES librocksdb.so) + set(ROCKSDB_LITE_LIBRARY_NAMES librocksdb_lite.so) else() - set(ROCKSDB_LIBRARY_NAMES ${ROCKSDB_LIBRARY_NAMES} librocksdb.a) + set(ROCKSDB_LIBRARY_NAMES librocksdb.a) + set(ROCKSDB_LITE_LIBRARY_NAMES librocksdb_lite.a) endif() -set(ROCKSDB_STATIC_LIBRARY_NAMES ${ROCKSDB_STATIC_LIBRARY_NAMES} librocksdb.a) +set(ROCKSDB_STATIC_LIBRARY_NAMES librocksdb.a) +set(ROCKSDB_LITE_STATIC_LIBRARY_NAMES librocksdb_lite.a) find_library(ROCKSDB_LIBRARIES NAMES ${ROCKSDB_LIBRARY_NAMES} HINTS ${_rocksdb_LIBRARIES_SEARCH_DIRS} ) +find_library(ROCKSDB_LITE_LIBRARIES NAMES ${ROCKSDB_LITE_LIBRARY_NAMES} + HINTS ${_rocksdb_LIBRARIES_SEARCH_DIRS} +) + find_library(ROCKSDB_STATIC_LIBRARY NAMES ${ROCKSDB_STATIC_LIBRARY_NAMES} HINTS ${_rocksdb_LIBRARIES_SEARCH_DIRS} ) +find_library(ROCKSDB_LITE_STATIC_LIBRARY NAMES ${ROCKSDB_LITE_STATIC_LIBRARY_NAMES} + HINTS ${_rocksdb_LIBRARIES_SEARCH_DIRS} +) + find_library(ROCKSDB_SNAPPY_LIBRARY NAMES libsnappy.a HINTS ${_rocksdb_LIBRARIES_SEARCH_DIRS} ) -if(NOT ${ROCKSDB_SNAPPY_LIBRARY} STREQUAL "ROCKSDB_SNAPPY_LIBRARY-NOTFOUND") - set(ROCKSDB_STATIC_LIBRARIES - ${ROCKSDB_STATIC_LIBRARY} - ${ROCKSDB_SNAPPY_LIBRARY} - ) +# If the lite library was found, override and prefer LITE. +if(NOT ${ROCKSDB_LITE_LIBRARIES} STREQUAL "ROCKSDB_LITE_LIBRARIES-NOTFOUND") + set(ROCKSDB_LIBRARIES ${ROCKSDB_LITE_LIBRARIES}) + set(ROCKSDB_LITE_FOUND "YES") endif() -if( ${ROCKSDB_LIBRARIES} STREQUAL "ROCKSDB_LIBRARIES-NOTFOUND" ) - set(ROCKSDB_LIBRARIES ${ROCKSDB_STATIC_LIBRARIES}) +if(NOT ${ROCKSDB_LITE_STATIC_LIBRARY} STREQUAL "ROCKSDB_LITE_STATIC_LIBRARY-NOTFOUND") + set(ROCKSDB_STATIC_LIBRARY ${ROCKSDB_LITE_STATIC_LIBRARY}) + set(ROCKSDB_LITE_FOUND "YES") +endif() + +# If shared libraries are not found, fall back to static. +# If not explicitly building using shared libraries, prefer static libraries. +if(${ROCKSDB_LIBRARIES} STREQUAL "ROCKSDB_LIBRARIES-NOTFOUND" + OR NOT DEFINED $ENV{BUILD_LINK_SHARED}) + set(ROCKSDB_LIBRARIES ${ROCKSDB_STATIC_LIBRARY} ${ROCKSDB_SNAPPY_LIBRARY}) + LOG_LIBRARY(rocksdb "${ROCKSDB_STATIC_LIBRARY}") + LOG_LIBRARY(snappy "${ROCKSDB_SNAPPY_LIBRARY}") +else() + LOG_LIBRARY(rocksdb "${ROCKSDB_LIBRARIES}") endif() # if the include and the program are found then we have it if(ROCKSDB_INCLUDE_DIR AND ROCKSDB_LIBRARIES) set(ROCKSDB_FOUND "YES") - LOG_LIBRARY(rocksdb "${ROCKSDB_STATIC_LIBRARY}") - LOG_LIBRARY(snappy "${ROCKSDB_SNAPPY_LIBRARY}") endif() diff --git a/CMake/FindSnappy.cmake b/CMake/FindSnappy.cmake deleted file mode 100644 index 28293733..00000000 --- a/CMake/FindSnappy.cmake +++ /dev/null @@ -1,31 +0,0 @@ -# This code is released under the -# Apache License Version 2.0 http://www.apache.org/licenses/. -# -# Copyright (c) 2012 Louis Dionne -# -# Find snappy compression library and includes. This module defines: -# snappy_INCLUDE_DIRS - The directories containing snappy's headers. -# snappy_LIBRARIES - A list of snappy's libraries. -# snappy_FOUND - Whether snappy was found. -# -# This module can be controlled by setting the following variables: -# snappy_ROOT - The root directory where to find snappy. If this is not -# set, the default paths are searched. - -if(NOT snappy_ROOT) - find_path(snappy_INCLUDE_DIRS snappy.h) - find_library(snappy_LIBRARIES NAMES snappy) -else() - find_path(snappy_INCLUDE_DIRS snappy.h NO_DEFAULT_PATH PATHS ${snappy_ROOT}) - find_library(snappy_LIBRARIES NAMES snappy NO_DEFAULT_PATH PATHS ${snappy_ROOT}) -endif() - -if(snappy_INCLUDE_DIRS AND snappy_LIBRARIES) - set(snappy_FOUND TRUE) -else() - set(snappy_FOUND FALSE) - set(snappy_INCLUDE_DIR) - set(snappy_LIBRARIES) -endif() - -mark_as_advanced(snappy_LIBRARIES snappy_INCLUDE_DIRS) diff --git a/CMake/FindThrift.cmake b/CMake/FindThrift.cmake index ff230522..ec77feb4 100644 --- a/CMake/FindThrift.cmake +++ b/CMake/FindThrift.cmake @@ -23,10 +23,9 @@ set(THRIFT_LIB_PATHS /usr/local/lib /opt/local/lib) -find_path(THRIFT_STATIC_LIB_PATH libthrift.a PATHS ${THRIFT_LIB_PATHS}) - # prefer the thrift version supplied in THRIFT_HOME -find_library(THRIFT_LIB NAMES thrift HINTS ${THRIFT_LIB_PATHS}) +find_library(THRIFT_LIBRARY NAMES thrift HINTS ${THRIFT_LIB_PATHS}) +#find_library(THRIFT_STATIC_LIBRARY NAMES libthrift.a HINTS ${THRIFT_LIB_PATHS}) find_program(THRIFT_COMPILER thrift $ENV{THRIFT_HOME}/bin @@ -35,27 +34,17 @@ find_program(THRIFT_COMPILER thrift NO_DEFAULT_PATH ) -if (THRIFT_LIB) +if (THRIFT_LIBRARY) set(THRIFT_FOUND TRUE) - set(THRIFT_LIBS ${THRIFT_LIB}) - set(THRIFT_STATIC_LIBRARY ${THRIFT_STATIC_LIB_PATH}/libthrift.a) + LOG_LIBRARY(thrift "${THRIFT_LIBRARY}") exec_program(${THRIFT_COMPILER} ARGS -version OUTPUT_VARIABLE THRIFT_VERSION RETURN_VALUE THRIFT_RETURN) -else () - set(THRIFT_FOUND FALSE) -endif () - -if (THRIFT_FOUND) - if (NOT THRIFT_FIND_QUIETLY) - LOG_LIBRARY(thrift "${THRIFT_STATIC_LIBRARY}") - endif () -else () +else() message(FATAL_ERROR "Thrift compiler/libraries NOT found.") -endif () - +endif() mark_as_advanced( - THRIFT_LIB + THRIFT_LIBRARY THRIFT_COMPILER THRIFT_INCLUDE_DIR ) diff --git a/CMakeLists.txt b/CMakeLists.txt index f185cc3f..1dccd647 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -229,8 +229,6 @@ add_definitions( -D${OSQUERY_BUILD_PLATFORM_DEFINE}_${OSQUERY_BUILD_DISTRO_DEFINE} -DSTRIP_FLAG_HELP=1 -DBOOST_NETWORK_ENABLE_HTTPS - # There are platform incompatibilities on OS X with lite and construction. - #-DROCKSDB_LITE=1 ) if(APPLE) @@ -303,10 +301,14 @@ find_package(Glog REQUIRED) find_package(Gflags REQUIRED) find_package(Gtest REQUIRED) find_package(RocksDB REQUIRED) -find_package(Snappy REQUIRED) find_package(Sqlite3 REQUIRED) find_package(Thrift 0.9.1 REQUIRED) +# If using the RocksDB LITE version our code must also define ROCKSDB_LITE=1 +if(ROCKSDB_LITE_FOUND) + add_definitions(-DROCKSDB_LITE=1) +endif() + # Python is used for table spec generation and formating. if(PYTHON_VERSION_MAJOR STREQUAL "2" AND PYTHON_VERSION_MINOR STREQUAL "4") WARNING_LOG("Found python 2.4, overriding to /usr/bin/python2.6") diff --git a/Vagrantfile b/Vagrantfile index a3f89af3..cc0a43fb 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,8 +2,8 @@ targets = { "centos6.5" => { "box" => "chef/centos-6.5" }, - "centos7" => { - "box" => "chef/centos-7.0" + "centos7.1" => { + "box" => "chef/centos-7.1" }, "ubuntu14" => { "box" => "ubuntu/trusty64" diff --git a/docs/wiki/development/building.md b/docs/wiki/development/building.md index 83012792..0165f4df 100644 --- a/docs/wiki/development/building.md +++ b/docs/wiki/development/building.md @@ -148,6 +148,20 @@ make sdk # Build only the osquery SDK (libosquery.a) make sync # Create a tarball for building the SDK externally ``` +Finally, subtle changes to the build are mostly controlled through environment +variables. When making these changes it is best to removed your build cache +by removing the `./build/` or `./build/{platform}/` directory. + +```sh +OSQUERY_PLATFORM=custom_linux;1.0 # Set a wacky platform/distro name +OSQUERY_BUILD_VERSION=9.9.9 # Set a wacky version string +BUILD_LINK_SHARED=True # Set CMake library discovery to prefer shared libraries +SDK_VERSION=9.9.9 # Set a wacky SDK-version string +SANITIZE_THREAD=True # Add -fsanitize=thread when using "make sanitize" +OPTIMIZED=True # Disable generic CPU optimizations +SKIP_TESTS=True # Skip unit test building (very very not recommended!) +``` + ## Custom Packages Building osquery on OS X or Linux requires a significant number of dependencies, which are not needed when deploying. It does not make sense to install osquery on your build hosts. See the [Custom Packages](../installation/custom-packages) guide for generating pkgs, debs or rpms. diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt index becffa0e..2eb2935b 100644 --- a/osquery/CMakeLists.txt +++ b/osquery/CMakeLists.txt @@ -12,9 +12,9 @@ set(OSQUERY_TABLES_TESTS "") # The core set of osquery libraries most discovered with find_package. set(OSQUERY_LIBS - ${THRIFT_STATIC_LIBRARY} - ${ROCKSDB_STATIC_LIBRARY} - ${ROCKSDB_SNAPPY_LIBRARY} + # This includes librocksdb[_lite] and libsnappy. + ${ROCKSDB_LIBRARIES} + ${THRIFT_LIBRARY} ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY} diff --git a/osquery/remote/transports/tls.cpp b/osquery/remote/transports/tls.cpp index ba259f31..a048e47b 100644 --- a/osquery/remote/transports/tls.cpp +++ b/osquery/remote/transports/tls.cpp @@ -80,7 +80,7 @@ http::client TLSTransport::getClient() { std::string ciphers = kTLSCiphers; // Some Ubuntu 12.04 clients exhaust their cipher suites without SHA. -#if defined(SSL_TXT_TLSV1_2) && !defined(UBUNTU_PRECISE) +#if defined(SSL_TXT_TLSV1_2) && !defined(UBUNTU_PRECISE) && !defined(DARWIN) // Otherwise we prefer GCM and SHA256+ ciphers += ":!CBC:!SHA"; #endif diff --git a/osquery/tables/system/darwin/keychain.h b/osquery/tables/system/darwin/keychain.h index 7f2e17a4..fc86b52d 100644 --- a/osquery/tables/system/darwin/keychain.h +++ b/osquery/tables/system/darwin/keychain.h @@ -29,9 +29,12 @@ // If using the system-provided OpenSSL on 10.10, mark x509 methods deprecated. #ifdef SSL_TXT_TLSV1_2 -#define OSX_OPENSSL(x) (x) +#define OSX_OPENSSL(expr) \ + do { \ + expr; \ + } while (0) #else -#define OSX_OPENSSL(x) OSQUERY_USE_DEPRECATED(x) +#define OSX_OPENSSL(expr) OSQUERY_USE_DEPRECATED(expr) #endif namespace osquery { diff --git a/tools/provision/lib.sh b/tools/provision/lib.sh index 5054e2e9..0dfb7381 100755 --- a/tools/provision/lib.sh +++ b/tools/provision/lib.sh @@ -114,8 +114,8 @@ function install_rocksdb() { URL=$DEPS_URL/rocksdb-3.10.2.tar.gz SOURCE=rocksdb-rocksdb-3.10.2 - if provision rocksdb /usr/local/lib/librocksdb.a; then - if [[ ! -f rocksdb-rocksdb-3.10.2/librocksdb.a ]]; then + if provision rocksdb /usr/local/lib/librocksdb_lite.a; then + if [[ ! -f rocksdb-rocksdb-3.10.2/librocksdb_lite.a ]]; then if [[ $FAMILY = "debian" ]]; then CLANG_INCLUDE="-I/usr/include/clang/3.4/include" elif [[ $FAMILY = "redhat" ]]; then @@ -124,15 +124,17 @@ function install_rocksdb() { fi pushd $SOURCE if [[ $OS = "freebsd" ]]; then - PORTABLE=1 CC=cc CXX=c++ gmake -j $THREADS static_lib \ - CFLAGS="$CLANG_INCLUDE $CFLAGS" + CC=cc + CXX=c++ + MAKE=gmake else - PORTABLE=1 CC="$CC" CXX="$CXX" make -j $THREADS static_lib \ - CFLAGS="$CLANG_INCLUDE $CFLAGS" + MAKE=make fi + PORTABLE=1 OPT="-DROCKSDB_LITE=1" LIBNAME=librocksdb_lite CC="$CC" CXX="$CXX" \ + $MAKE -j $THREADS static_lib CFLAGS="$CLANG_INCLUDE $CFLAGS" popd fi - sudo cp rocksdb-rocksdb-3.10.2/librocksdb.a /usr/local/lib + sudo cp rocksdb-rocksdb-3.10.2/librocksdb_lite.a /usr/local/lib sudo cp -R rocksdb-rocksdb-3.10.2/include/rocksdb /usr/local/include fi } @@ -390,9 +392,11 @@ function package() { export HOMEBREW_NO_EMOJI=1 if [[ $1 = "rocksdb" ]]; then # Build RocksDB from source in brew + export LIBNAME=librocksdb_lite export HOMEBREW_BUILD_FROM_SOURCE=1 - HOMEBREW_ARGS=--build-bottle + HOMEBREW_ARGS="--build-bottle --with-lite" else + unset LIBNAME unset HOMEBREW_BUILD_FROM_SOURCE fi brew install -v $HOMEBREW_ARGS $1 || brew upgrade -v $HOMEBREW_ARGS $@ diff --git a/tools/tests/test_example_queries.py b/tools/tests/test_example_queries.py index 60f6a383..0f762a94 100755 --- a/tools/tests/test_example_queries.py +++ b/tools/tests/test_example_queries.py @@ -27,10 +27,12 @@ class ExampleQueryTests(test_base.ProcessGenerator, unittest.TestCase): self.daemon = self._run_daemon({ # The set of queries will hammer the daemon process. "disable_watchdog": True, + # Enable the 'hidden' flag "registry_exceptions" to prevent catching. + "registry_exceptions": True, }) self.assertTrue(self.daemon.isAlive()) - # The sets of example tests will use the extensions API.s + # The sets of example tests will use the extensions APIs. self.client = test_base.EXClient(self.daemon.options["extensions_socket"]) test_base.expectTrue(self.client.open) self.assertTrue(self.client.open())