Build Glog with OS X ABI, add SKIP_BENCHMARK

CMake/FindGlog.cmake

@@ -7,9 +7,18 @@ set(GLOG_ROOT_DIR "${CMAKE_BINARY_DIR}/third-party/glog")
 set(GLOG_SOURCE_DIR "${CMAKE_SOURCE_DIR}/third-party/glog")
 
 set(GLOG_C_FLAGS "${CMAKE_C_FLAGS} ${C_COMPILE_FLAGS}")
-set(GLOG_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_COMPILE_FLAGS}")
+if(APPLE)
+  set(GLOG_C_FLAGS "${GLOG_C_FLAGS} -mmacosx-version-min=${APPLE_MIN_ABI}")
+endif()
+set(GLOG_CXX_FLAGS "${GLOG_C_FLAGS} ${CMAKE_CXX_FLAGS} ${CXX_COMPILE_FLAGS}")
 set(GLOG_CXX_FLAGS "${GLOG_CXX_FLAGS} -Wno-deprecated-register -Wno-unnamed-type-template-args -Wno-deprecated -Wno-error")
 
+set(GLOG_INCLUDE_DIR "${GLOG_ROOT_DIR}/include")
+set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR})
+
+set(GLOG_LIBRARY "${GLOG_ROOT_DIR}/lib/libglog.a")
+set(GLOG_LIBRARIES ${GLOG_LIBRARY})
+
 INCLUDE(ExternalProject)
 ExternalProject_Add(
   libglog
@@ -26,10 +35,4 @@ ExternalProject_Add(
   LOG_BUILD ON
 )
 
-set(GLOG_INCLUDE_DIR "${GLOG_ROOT_DIR}/include")
-set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR})
-
-set(GLOG_LIBRARY "${GLOG_ROOT_DIR}/lib/libglog.a")
-set(GLOG_LIBRARIES ${GLOG_LIBRARY})
-
 LOG_LIBRARY(glog "${GLOG_LIBRARY}")

CMakeLists.txt

@@ -80,13 +80,16 @@ list(GET PLATFORM 1 OSQUERY_BUILD_DISTRO_DEFINE)
 # late-loading modules and SQLite introspection utilities.
 if(APPLE)
   set(APPLE_MIN_ABI "10.10")
+  add_compile_options(
+    -mmacosx-version-min=${APPLE_MIN_ABI}
+  )
+
   set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++11 -stdlib=libc++")
-  set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -mmacosx-version-min=${APPLE_MIN_ABI}")
   set(OS_WHOLELINK_PRE "-Wl,-force_load")
   set(OS_WHOLELINK_POST "")
   # Special compile flags for Objective-C++
   set(OBJCXX_COMPILE_FLAGS
-    "-x objective-c++ -fobjc-arc -Wno-c++11-extensions -mmacosx-version-min=${APPLE_MIN_ABI}")
+    "-x objective-c++ -fobjc-arc -Wno-c++11-extensions")
 else()
   set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++11")
   set(OS_WHOLELINK_PRE "-Wl,-whole-archive")

docs/README.md

@@ -0,0 +1,45 @@
+osquery documentation
+=====================
+
+The osquery documentation includes both a ReadTheDocs wiki and Doxygen-generated set of developer APIs.
+
+## Wiki
+
+The ReadTheDocs wiki (https://osquery.readthedocs.org/en/stable) is generated using a RTD-configured osquery project and associated GitHub Service. This Service is documented by RTD and more-or-less setup automatically with the project. RTD generates documentation for every version (git tag). It calls the most recent tag 'stable', the most recent commit to master 'devel', and includes links to every past version. The project settings and sidebar for RTD is kept in the root as [mkdocs.yml](https://github.com/facebook/osquery/blob/master/mkdocs.yml).
+
+### Adding a new page
+
+New wiki pages should be organized into one of the following categories:
+
+- **Introduction**: Overview of the project or a tool.
+- **Installation**: Deep dives into OS-specifics, packaging, and switches that control starting tools.
+- **Deployment**: Tool concepts and all the wonderful goodies of making osquery useful.
+- **Development**: Help and guides for starting with osquery development and build.
+
+Make a new "filename.md" within the category folder withing `/docs/wiki/CATEGORY/`. Then add the friendly page title and path to [mkdocs.yml](https://github.com/facebook/osquery/blob/master/mkdocs.yml), in the order the page should appear within the wiki sidebar.
+
+### Wiki style tips
+
+- Inline code highlighing (`$ echo 'this is inline'`) does not look the best in RTD, try to have as little inline syntax highlighting as possible.
+- **osqueryd**, **osqueryi** and other tool names should be in bold. Use `inline highligh` when a tool or script is mentioned for the first time.
+- Filesystem paths and non-clickable URI examples should also be bold.
+- Flag names are usually in quotes, `inline highlight` when introduced for the first time or used as an example.
+
+## Doxygen
+
+The Doxygen documentation is not hosted anywhere, each developer must build and view-locally. To build the docs use `make docs`.
+
+The output HTML documentation is written to `./build/docs/html/`. Use `index.html` to begin exploring.
+
+## Tables and Packs
+
+Table schema, the osquery user API, is created using the Python-based ".spec" files in [`./specs`](https://github.com/facebook/osquery/tree/master/specs). More documentation on how specs work can be found in the [Creating New Tables](http://osquery.readthedocs.org/en/stable/development/creating-tables/) developer documentation. These files are used to build osquery, but can be parsed to create JSON-based API schema. This JSON is published to the homepage at [https://osquery.io/docs/tables].
+
+Use: `./tools/codegen/genapi.py` to generate the amalgamated schema. To generate a "change log" between tags, use the same script but use `--diff` and supply the two JSON imputs.
+
+```python
+./tools/codegen/genapi.py > ./build/docs/CURRENT.json
+./tools/codegen/genapi.py --diff ./build/docs/OLD.json ./build/docs/CURRENT.json
+```
+
+Packs do not require any processing, they can be copied to the homepage's data folder.

kernel/CMakeLists.txt

@@ -166,13 +166,15 @@ if(APPLE)
       COMMAND sudo $<TARGET_FILE:osquery_kernel_tests>
     )
 
-    add_custom_target(
-      run-kernel-benchmark
-      COMMAND echo ""
-      COMMAND echo "Running kernel benchmarks requires root."
-      COMMAND sudo bash -c "$<TARGET_FILE:osquery_kernel_benchmarks> $ENV{BENCHMARK_TO_FILE}"
-      WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
-    )
+    if(NOT DEFINED ENV{SKIP_BENCHMARKS})
+      add_custom_target(
+        run-kernel-benchmark
+        COMMAND echo ""
+        COMMAND echo "Running kernel benchmarks requires root."
+        COMMAND sudo bash -c "$<TARGET_FILE:osquery_kernel_benchmarks> $ENV{BENCHMARK_TO_FILE}"
+        WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+      )
+    endif()
   endif()
 elseif(LINUX)
   add_custom_target(
@@ -206,10 +208,12 @@ elseif(LINUX)
       COMMAND echo "-- No kernel test is run for Linux"
     )
 
-    add_custom_target(
-      run-kernel-benchmark
-      COMMAND echo "-- No kernel benchmark is run for Linux"
-    )
+    if(NOT DEFINED ENV{SKIP_BENCHMARKS})
+      add_custom_target(
+        run-kernel-benchmark
+        COMMAND echo "-- No kernel benchmark is run for Linux"
+      )
+    endif()
   endif()
 else()
   add_custom_target(
@@ -243,10 +247,12 @@ else()
       COMMAND echo "-- No kernel test is run for unsupported platform"
     )
 
-    add_custom_target(
-      run-kernel-benchmark
-      COMMAND echo "-- No kernel benchmark is run for unsupported platform"
-    )
+    if(NOT DEFINED ENV{SKIP_BENCHMARKS})
+      add_custom_target(
+        run-kernel-benchmark
+        COMMAND echo "-- No kernel benchmark is run for unsupported platform"
+      )
+    endif()
   endif()
 endif()
 

osquery/CMakeLists.txt

@@ -209,27 +209,29 @@ if(NOT DEFINED ENV{SKIP_TESTS})
   SET_OSQUERY_COMPILE(osquery_kernel_tests "${CXX_COMPILE_FLAGS} -DKERNEL_TEST=1 -DGTEST_HAS_TR1_TUPLE=0")
 
   # osquery benchmarks.
-  add_executable(osquery_benchmarks main/benchmarks.cpp ${OSQUERY_BENCHMARKS})
-  TARGET_OSQUERY_LINK_WHOLE(osquery_benchmarks libosquery)
-  TARGET_OSQUERY_LINK_WHOLE(osquery_benchmarks libosquery_additional)
-  target_link_libraries(osquery_benchmarks benchmark libosquery_testing)
-  SET_OSQUERY_COMPILE(osquery_benchmarks "${CXX_COMPILE_FLAGS}")
-  set(BENCHMARK_TARGET "$<TARGET_FILE:osquery_benchmarks>")
-  
-  # osquery kernel benchmarks.
-  add_executable(osquery_kernel_benchmarks main/benchmarks.cpp ${OSQUERY_KERNEL_BENCHMARKS})
-  TARGET_OSQUERY_LINK_WHOLE(osquery_kernel_benchmarks libosquery)
-  TARGET_OSQUERY_LINK_WHOLE(osquery_kernel_benchmarks libosquery_additional)
-  target_link_libraries(osquery_kernel_benchmarks benchmark libosquery_testing)
-  SET_OSQUERY_COMPILE(osquery_kernel_benchmarks "${CXX_COMPILE_FLAGS} -DKERNEL_TEST=1")
+  if(NOT DEFINED ENV{SKIP_BENCHMARKS})
+    add_executable(osquery_benchmarks main/benchmarks.cpp ${OSQUERY_BENCHMARKS})
+    TARGET_OSQUERY_LINK_WHOLE(osquery_benchmarks libosquery)
+    TARGET_OSQUERY_LINK_WHOLE(osquery_benchmarks libosquery_additional)
+    target_link_libraries(osquery_benchmarks benchmark libosquery_testing)
+    SET_OSQUERY_COMPILE(osquery_benchmarks "${CXX_COMPILE_FLAGS}")
+    set(BENCHMARK_TARGET "$<TARGET_FILE:osquery_benchmarks>")
 
-  # make benchmark
-  add_custom_target(
-    run-benchmark
-    COMMAND bash -c "${BENCHMARK_TARGET} $ENV{BENCHMARK_TO_FILE}"
-    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
-    DEPENDS osquery_benchmarks
-  )
+    # osquery kernel benchmarks.
+    add_executable(osquery_kernel_benchmarks main/benchmarks.cpp ${OSQUERY_KERNEL_BENCHMARKS})
+    TARGET_OSQUERY_LINK_WHOLE(osquery_kernel_benchmarks libosquery)
+    TARGET_OSQUERY_LINK_WHOLE(osquery_kernel_benchmarks libosquery_additional)
+    target_link_libraries(osquery_kernel_benchmarks benchmark libosquery_testing)
+    SET_OSQUERY_COMPILE(osquery_kernel_benchmarks "${CXX_COMPILE_FLAGS} -DKERNEL_TEST=1")
+
+    # make benchmark
+    add_custom_target(
+      run-benchmark
+      COMMAND bash -c "${BENCHMARK_TARGET} $ENV{BENCHMARK_TO_FILE}"
+      WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+      DEPENDS osquery_benchmarks
+    )
+  endif()
 
   if(NOT OSQUERY_BUILD_SDK_ONLY)
     # osquery core (additional) set of unit tests built outside of SDK.

osquery/logger/logger.cpp

@@ -195,6 +195,7 @@ void setVerboseLevel() {
 
 void initStatusLogger(const std::string& name) {
   FLAGS_alsologtostderr = false;
+  FLAGS_colorlogtostderr = true;
   FLAGS_logbufsecs = 0; // flush the log buffer immediately
   FLAGS_stop_logging_if_full_disk = true;
   FLAGS_max_log_size = 10; // max size for individual log file is 10MB