Merge pull request #1331 from theopolis/rocksdb_lite2

Rollup of build changes

CMake/FindLZMA.cmake

@@ -1,48 +0,0 @@
-# - Find lzma and lzmadec
-# Find the native LZMA includes and library
-#
-#  LZMA_INCLUDE_DIR    - where to find lzma.h, etc.
-#  LZMA_LIBRARIES      - List of libraries when using liblzma.
-#  LZMA_FOUND          - True if liblzma found.
-#  LZMADEC_INCLUDE_DIR - where to find lzmadec.h, etc.
-#  LZMADEC_LIBRARIES   - List of libraries when using liblzmadec.
-#  LZMADEC_FOUND       - True if liblzmadec found.
-
-IF (LZMA_INCLUDE_DIR)
-  # Already in cache, be silent
-  SET(LZMA_FIND_QUIETLY TRUE)
-ENDIF (LZMA_INCLUDE_DIR)
-
-FIND_PATH(LZMA_INCLUDE_DIR lzma.h)
-FIND_LIBRARY(LZMA_LIBRARY NAMES lzma )
-
-# handle the QUIETLY and REQUIRED arguments and set LZMA_FOUND to TRUE if
-# all listed variables are TRUE
-INCLUDE(FindPackageHandleStandardArgs)
-FIND_PACKAGE_HANDLE_STANDARD_ARGS(LZMA DEFAULT_MSG LZMA_LIBRARY LZMA_INCLUDE_DIR)
-
-IF(LZMA_FOUND)
-  SET( LZMA_LIBRARIES ${LZMA_LIBRARY} )
-ELSE(LZMA_FOUND)
-  SET( LZMA_LIBRARIES )
-
-  IF (LZMADEC_INCLUDE_DIR)
-    # Already in cache, be silent
-    SET(LZMADEC_FIND_QUIETLY TRUE)
-  ENDIF (LZMADEC_INCLUDE_DIR)
-
-  FIND_PATH(LZMADEC_INCLUDE_DIR lzmadec.h)
-  FIND_LIBRARY(LZMADEC_LIBRARY NAMES lzmadec )
-
-  # handle the QUIETLY and REQUIRED arguments and set LZMADEC_FOUND to TRUE if
-  # all listed variables are TRUE
-  INCLUDE(FindPackageHandleStandardArgs)
-  FIND_PACKAGE_HANDLE_STANDARD_ARGS(LZMADEC DEFAULT_MSG LZMADEC_LIBRARY
-    LZMADEC_INCLUDE_DIR)
-
-  IF(LZMADEC_FOUND)
-    SET( LZMADEC_LIBRARIES ${LZMADEC_LIBRARY} )
-  ELSE(LZMADEC_FOUND)
-    SET( LZMADEC_LIBRARIES )
-  ENDIF(LZMADEC_FOUND)
-ENDIF(LZMA_FOUND)

CMake/FindRocksDB.cmake

@@ -69,42 +69,63 @@ FIND_PATH(ROCKSDB_INCLUDE_DIR rocksdb/db.h
 # locate the library
 if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
   # On MacOS
-  set(ROCKSDB_LIBRARY_NAMES ${ROCKSDB_LIBRARY_NAMES} librocksdb.dylib)
+  set(ROCKSDB_LIBRARY_NAMES librocksdb.dylib)
+  set(ROCKSDB_LITE_LIBRARY_NAMES librocksdb_lite.dylib)
 elseif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
   # On Linux
-  set(ROCKSDB_LIBRARY_NAMES ${ROCKSDB_LIBRARY_NAMES} librocksdb.so)
+  set(ROCKSDB_LIBRARY_NAMES librocksdb.so)
+  set(ROCKSDB_LITE_LIBRARY_NAMES librocksdb_lite.so)
 else()
-  set(ROCKSDB_LIBRARY_NAMES ${ROCKSDB_LIBRARY_NAMES} librocksdb.a)
+  set(ROCKSDB_LIBRARY_NAMES librocksdb.a)
+  set(ROCKSDB_LITE_LIBRARY_NAMES librocksdb_lite.a)
 endif()
 
-set(ROCKSDB_STATIC_LIBRARY_NAMES ${ROCKSDB_STATIC_LIBRARY_NAMES} librocksdb.a)
+set(ROCKSDB_STATIC_LIBRARY_NAMES librocksdb.a)
+set(ROCKSDB_LITE_STATIC_LIBRARY_NAMES librocksdb_lite.a)
 
 find_library(ROCKSDB_LIBRARIES NAMES ${ROCKSDB_LIBRARY_NAMES}
   HINTS ${_rocksdb_LIBRARIES_SEARCH_DIRS}
 )
 
+find_library(ROCKSDB_LITE_LIBRARIES NAMES ${ROCKSDB_LITE_LIBRARY_NAMES}
+  HINTS ${_rocksdb_LIBRARIES_SEARCH_DIRS}
+)
+
 find_library(ROCKSDB_STATIC_LIBRARY NAMES ${ROCKSDB_STATIC_LIBRARY_NAMES}
   HINTS ${_rocksdb_LIBRARIES_SEARCH_DIRS}
 )
 
+find_library(ROCKSDB_LITE_STATIC_LIBRARY NAMES ${ROCKSDB_LITE_STATIC_LIBRARY_NAMES}
+  HINTS ${_rocksdb_LIBRARIES_SEARCH_DIRS}
+)
+
 find_library(ROCKSDB_SNAPPY_LIBRARY NAMES libsnappy.a
   HINTS ${_rocksdb_LIBRARIES_SEARCH_DIRS}
 )
 
-if(NOT ${ROCKSDB_SNAPPY_LIBRARY} STREQUAL "ROCKSDB_SNAPPY_LIBRARY-NOTFOUND")
-  set(ROCKSDB_STATIC_LIBRARIES
-    ${ROCKSDB_STATIC_LIBRARY}
-    ${ROCKSDB_SNAPPY_LIBRARY}
-  )
+# If the lite library was found, override and prefer LITE.
+if(NOT ${ROCKSDB_LITE_LIBRARIES} STREQUAL "ROCKSDB_LITE_LIBRARIES-NOTFOUND")
+  set(ROCKSDB_LIBRARIES ${ROCKSDB_LITE_LIBRARIES})
+  set(ROCKSDB_LITE_FOUND "YES")
 endif()
 
-if( ${ROCKSDB_LIBRARIES} STREQUAL "ROCKSDB_LIBRARIES-NOTFOUND" )
-  set(ROCKSDB_LIBRARIES ${ROCKSDB_STATIC_LIBRARIES})
+if(NOT ${ROCKSDB_LITE_STATIC_LIBRARY} STREQUAL "ROCKSDB_LITE_STATIC_LIBRARY-NOTFOUND")
+  set(ROCKSDB_STATIC_LIBRARY ${ROCKSDB_LITE_STATIC_LIBRARY})
+  set(ROCKSDB_LITE_FOUND "YES")
+endif()
+
+# If shared libraries are not found, fall back to static.
+# If not explicitly building using shared libraries, prefer static libraries.
+if(${ROCKSDB_LIBRARIES} STREQUAL "ROCKSDB_LIBRARIES-NOTFOUND"
+    OR NOT DEFINED $ENV{BUILD_LINK_SHARED})
+  set(ROCKSDB_LIBRARIES ${ROCKSDB_STATIC_LIBRARY} ${ROCKSDB_SNAPPY_LIBRARY})
+  LOG_LIBRARY(rocksdb "${ROCKSDB_STATIC_LIBRARY}")
+  LOG_LIBRARY(snappy "${ROCKSDB_SNAPPY_LIBRARY}")
+else()
+  LOG_LIBRARY(rocksdb "${ROCKSDB_LIBRARIES}")
 endif()
 
 # if the include and the program are found then we have it
 if(ROCKSDB_INCLUDE_DIR AND ROCKSDB_LIBRARIES)
   set(ROCKSDB_FOUND "YES")
-  LOG_LIBRARY(rocksdb "${ROCKSDB_STATIC_LIBRARY}")
-  LOG_LIBRARY(snappy "${ROCKSDB_SNAPPY_LIBRARY}")
 endif()

CMake/FindSnappy.cmake

@@ -1,31 +0,0 @@
-# This code is released under the
-# Apache License Version 2.0 http://www.apache.org/licenses/.
-#
-# Copyright (c) 2012 Louis Dionne
-#
-# Find snappy compression library and includes. This module defines:
-#   snappy_INCLUDE_DIRS - The directories containing snappy's headers.
-#   snappy_LIBRARIES    - A list of snappy's libraries.
-#   snappy_FOUND        - Whether snappy was found.
-#
-# This module can be controlled by setting the following variables:
-#   snappy_ROOT - The root directory where to find snappy. If this is not
-#                 set, the default paths are searched.
-
-if(NOT snappy_ROOT)
-    find_path(snappy_INCLUDE_DIRS snappy.h)
-    find_library(snappy_LIBRARIES NAMES snappy)
-else()
-    find_path(snappy_INCLUDE_DIRS snappy.h NO_DEFAULT_PATH PATHS ${snappy_ROOT})
-    find_library(snappy_LIBRARIES NAMES snappy NO_DEFAULT_PATH PATHS ${snappy_ROOT})
-endif()
-
-if(snappy_INCLUDE_DIRS AND snappy_LIBRARIES)
-    set(snappy_FOUND TRUE)
-else()
-    set(snappy_FOUND FALSE)
-    set(snappy_INCLUDE_DIR)
-    set(snappy_LIBRARIES)
-endif()
-
-mark_as_advanced(snappy_LIBRARIES snappy_INCLUDE_DIRS)

CMake/FindThrift.cmake

@@ -23,10 +23,9 @@ set(THRIFT_LIB_PATHS
   /usr/local/lib
   /opt/local/lib)
 
-find_path(THRIFT_STATIC_LIB_PATH libthrift.a PATHS ${THRIFT_LIB_PATHS})
-
 # prefer the thrift version supplied in THRIFT_HOME
-find_library(THRIFT_LIB NAMES thrift HINTS ${THRIFT_LIB_PATHS})
+find_library(THRIFT_LIBRARY NAMES thrift HINTS ${THRIFT_LIB_PATHS})
+#find_library(THRIFT_STATIC_LIBRARY NAMES libthrift.a HINTS ${THRIFT_LIB_PATHS})
 
 find_program(THRIFT_COMPILER thrift
   $ENV{THRIFT_HOME}/bin
@@ -35,27 +34,17 @@ find_program(THRIFT_COMPILER thrift
   NO_DEFAULT_PATH
 )
 
-if (THRIFT_LIB)
+if (THRIFT_LIBRARY)
   set(THRIFT_FOUND TRUE)
-  set(THRIFT_LIBS ${THRIFT_LIB})
-  set(THRIFT_STATIC_LIBRARY ${THRIFT_STATIC_LIB_PATH}/libthrift.a)
+  LOG_LIBRARY(thrift "${THRIFT_LIBRARY}")
   exec_program(${THRIFT_COMPILER}
     ARGS -version OUTPUT_VARIABLE THRIFT_VERSION RETURN_VALUE THRIFT_RETURN)
-else ()
-  set(THRIFT_FOUND FALSE)
-endif ()
-
-if (THRIFT_FOUND)
-  if (NOT THRIFT_FIND_QUIETLY)
-    LOG_LIBRARY(thrift "${THRIFT_STATIC_LIBRARY}")
-  endif ()
-else ()
+else()
   message(FATAL_ERROR "Thrift compiler/libraries NOT found.")
-endif ()
-
+endif()
 
 mark_as_advanced(
-  THRIFT_LIB
+  THRIFT_LIBRARY
   THRIFT_COMPILER
   THRIFT_INCLUDE_DIR
 )

CMakeLists.txt

@@ -229,8 +229,6 @@ add_definitions(
   -D${OSQUERY_BUILD_PLATFORM_DEFINE}_${OSQUERY_BUILD_DISTRO_DEFINE}
   -DSTRIP_FLAG_HELP=1
   -DBOOST_NETWORK_ENABLE_HTTPS
-  # There are platform incompatibilities on OS X with lite and construction.
-  #-DROCKSDB_LITE=1
 )
 
 if(APPLE)
@@ -303,10 +301,14 @@ find_package(Glog REQUIRED)
 find_package(Gflags REQUIRED)
 find_package(Gtest REQUIRED)
 find_package(RocksDB REQUIRED)
-find_package(Snappy REQUIRED)
 find_package(Sqlite3 REQUIRED)
 find_package(Thrift 0.9.1 REQUIRED)
 
+# If using the RocksDB LITE version our code must also define ROCKSDB_LITE=1
+if(ROCKSDB_LITE_FOUND)
+  add_definitions(-DROCKSDB_LITE=1)
+endif()
+
 # Python is used for table spec generation and formating.
 if(PYTHON_VERSION_MAJOR STREQUAL "2" AND PYTHON_VERSION_MINOR STREQUAL "4")
   WARNING_LOG("Found python 2.4, overriding to /usr/bin/python2.6")

Vagrantfile

@@ -2,8 +2,8 @@ targets = {
   "centos6.5" => {
     "box" => "chef/centos-6.5"
   },
-  "centos7"   => {
-    "box" => "chef/centos-7.0"
+  "centos7.1"   => {
+    "box" => "chef/centos-7.1"
   },
   "ubuntu14"  => {
     "box" => "ubuntu/trusty64"

docs/wiki/development/building.md

@@ -148,6 +148,20 @@ make sdk # Build only the osquery SDK (libosquery.a)
 make sync # Create a tarball for building the SDK externally
 ```
 
+Finally, subtle changes to the build are mostly controlled through environment
+variables. When making these changes it is best to removed your build cache
+by removing the `./build/` or `./build/{platform}/` directory.
+
+```sh
+OSQUERY_PLATFORM=custom_linux;1.0 # Set a wacky platform/distro name
+OSQUERY_BUILD_VERSION=9.9.9 # Set a wacky version string
+BUILD_LINK_SHARED=True # Set CMake library discovery to prefer shared libraries
+SDK_VERSION=9.9.9 # Set a wacky SDK-version string
+SANITIZE_THREAD=True # Add -fsanitize=thread when using "make sanitize"
+OPTIMIZED=True # Disable generic CPU optimizations
+SKIP_TESTS=True # Skip unit test building (very very not recommended!)
+```
+
 ## Custom Packages
 
 Building osquery on OS X or Linux requires a significant number of dependencies, which are not needed when deploying. It does not make sense to install osquery on your build hosts. See the [Custom Packages](../installation/custom-packages) guide for generating pkgs, debs or rpms.

osquery/CMakeLists.txt

@@ -12,9 +12,9 @@ set(OSQUERY_TABLES_TESTS "")
 
 # The core set of osquery libraries most discovered with find_package.
 set(OSQUERY_LIBS
-  ${THRIFT_STATIC_LIBRARY}
-  ${ROCKSDB_STATIC_LIBRARY}
-  ${ROCKSDB_SNAPPY_LIBRARY}
+  # This includes librocksdb[_lite] and libsnappy.
+  ${ROCKSDB_LIBRARIES}
+  ${THRIFT_LIBRARY}
   ${GLOG_LIBRARY}
   ${GFLAGS_LIBRARY}
   ${OPENSSL_CRYPTO_LIBRARY}

osquery/remote/transports/tls.cpp

@@ -80,7 +80,7 @@ http::client TLSTransport::getClient() {
 
   std::string ciphers = kTLSCiphers;
   // Some Ubuntu 12.04 clients exhaust their cipher suites without SHA.
-#if defined(SSL_TXT_TLSV1_2) && !defined(UBUNTU_PRECISE)
+#if defined(SSL_TXT_TLSV1_2) && !defined(UBUNTU_PRECISE) && !defined(DARWIN)
   // Otherwise we prefer GCM and SHA256+
   ciphers += ":!CBC:!SHA";
 #endif

osquery/tables/system/darwin/keychain.h

@@ -29,9 +29,12 @@
 
 // If using the system-provided OpenSSL on 10.10, mark x509 methods deprecated.
 #ifdef SSL_TXT_TLSV1_2
-#define OSX_OPENSSL(x) (x)
+#define OSX_OPENSSL(expr) \
+  do {                    \
+    expr;                 \
+  } while (0)
 #else
-#define OSX_OPENSSL(x) OSQUERY_USE_DEPRECATED(x)
+#define OSX_OPENSSL(expr) OSQUERY_USE_DEPRECATED(expr)
 #endif
 
 namespace osquery {

tools/provision/lib.sh

@@ -114,8 +114,8 @@ function install_rocksdb() {
   URL=$DEPS_URL/rocksdb-3.10.2.tar.gz
   SOURCE=rocksdb-rocksdb-3.10.2
 
-  if provision rocksdb /usr/local/lib/librocksdb.a; then
-    if [[ ! -f rocksdb-rocksdb-3.10.2/librocksdb.a ]]; then
+  if provision rocksdb /usr/local/lib/librocksdb_lite.a; then
+    if [[ ! -f rocksdb-rocksdb-3.10.2/librocksdb_lite.a ]]; then
       if [[ $FAMILY = "debian" ]]; then
         CLANG_INCLUDE="-I/usr/include/clang/3.4/include"
       elif [[ $FAMILY = "redhat" ]]; then
@@ -124,15 +124,17 @@ function install_rocksdb() {
       fi
       pushd $SOURCE
       if [[ $OS = "freebsd" ]]; then
-        PORTABLE=1 CC=cc CXX=c++ gmake -j $THREADS static_lib \
-          CFLAGS="$CLANG_INCLUDE $CFLAGS"
+        CC=cc
+        CXX=c++
+        MAKE=gmake
       else
-        PORTABLE=1 CC="$CC" CXX="$CXX" make -j $THREADS static_lib \
-          CFLAGS="$CLANG_INCLUDE $CFLAGS"
+        MAKE=make
       fi
+      PORTABLE=1 OPT="-DROCKSDB_LITE=1" LIBNAME=librocksdb_lite CC="$CC" CXX="$CXX" \
+        $MAKE -j $THREADS static_lib CFLAGS="$CLANG_INCLUDE $CFLAGS"
       popd
     fi
-    sudo cp rocksdb-rocksdb-3.10.2/librocksdb.a /usr/local/lib
+    sudo cp rocksdb-rocksdb-3.10.2/librocksdb_lite.a /usr/local/lib
     sudo cp -R rocksdb-rocksdb-3.10.2/include/rocksdb /usr/local/include
   fi
 }
@@ -390,9 +392,11 @@ function package() {
       export HOMEBREW_NO_EMOJI=1
       if [[ $1 = "rocksdb" ]]; then
         # Build RocksDB from source in brew
+        export LIBNAME=librocksdb_lite
         export HOMEBREW_BUILD_FROM_SOURCE=1
-        HOMEBREW_ARGS=--build-bottle
+        HOMEBREW_ARGS="--build-bottle --with-lite"
       else
+        unset LIBNAME
         unset HOMEBREW_BUILD_FROM_SOURCE
       fi
       brew install -v $HOMEBREW_ARGS $1 || brew upgrade -v $HOMEBREW_ARGS $@

tools/tests/test_example_queries.py

@@ -27,10 +27,12 @@ class ExampleQueryTests(test_base.ProcessGenerator, unittest.TestCase):
         self.daemon = self._run_daemon({
             # The set of queries will hammer the daemon process.
             "disable_watchdog": True,
+            # Enable the 'hidden' flag "registry_exceptions" to prevent catching.
+            "registry_exceptions": True,
         })
         self.assertTrue(self.daemon.isAlive())
 
-        # The sets of example tests will use the extensions API.s
+        # The sets of example tests will use the extensions APIs.
         self.client = test_base.EXClient(self.daemon.options["extensions_socket"])
         test_base.expectTrue(self.client.open)
         self.assertTrue(self.client.open())