Merge pull request #1708 from theopolis/tsk_more

Additional TSK table: device_hash

osquery/core/hash.cpp

@@ -86,7 +86,9 @@ std::string Hash::digest() {
   return digest.str();
 }
 
-std::string hashFromBuffer(HashType hash_type, const void* buffer, size_t size) {
+std::string hashFromBuffer(HashType hash_type,
+                           const void* buffer,
+                           size_t size) {
   Hash hash(hash_type);
   hash.update(buffer, size);
   return hash.digest();

osquery/tables/forensic/sleuthkit.cpp

@@ -17,6 +17,7 @@
 #include <tsk/libtsk.h>
 
 #include <osquery/filesystem.h>
+#include <osquery/hash.h>
 #include <osquery/logger.h>
 #include <osquery/tables.h>
 
@@ -58,6 +59,12 @@ class DeviceHelper : private boost::noncopyable {
     }
   }
 
+  void inodes(
+      const std::set<std::string>& inodes,
+      TskFsInfo* fs,
+      std::function<void(const std::string&, TskFsFile*, const std::string&)>
+          predicate);
+
   /// Provide a partition description for context and iterate from path.
   void generateFiles(const std::string& partition,
                      TskFsInfo* fs,
@@ -68,6 +75,7 @@ class DeviceHelper : private boost::noncopyable {
   /// Similar to generateFiles but only yield a row to results.
   void generateFile(const std::string& partition,
                     TskFsFile* file,
+                    TskFsInfo* fs,
                     const std::string& path,
                     QueryData& results);
 
@@ -82,24 +90,7 @@ class DeviceHelper : private boost::noncopyable {
 
  private:
   /// Attempt to open the provided device image and volume.
-  bool open() {
+  bool open();
-    if (opened_) {
-      return opened_result_;
-    }
-
-    // Attempt to open the device image.
-    opened_ = true;
-    auto status = image_->open(device_path_.c_str(), TSK_IMG_TYPE_DETECT, 0);
-    if (status) {
-      opened_result_ = false;
-      return opened_result_;
-    }
-
-    // Attempt to open the device image volumn.
-    status = volume_->open(&*image_, 0, TSK_VS_TYPE_DETECT);
-    opened_result_ = (status == 0);
-    return opened_result_;
-  }
 
  private:
   /// Has the device open been attempted.
@@ -121,59 +112,56 @@ class DeviceHelper : private boost::noncopyable {
   std::set<std::string> loops_;
 };
 
-QueryData genDevicePartitions(QueryContext& context) {
+bool DeviceHelper::open() {
-  QueryData results;
+  if (opened_) {
-
+    return opened_result_;
-  auto devices = context.constraints["device"].getAll(EQUALS);
-  for (const auto& dev : devices) {
-    DeviceHelper dh(dev);
-    dh.partitions(([&results, &dev, &dh](const TskVsPartInfo* part) {
-      Row r;
-      r["device"] = dev;
-      r["partition"] = std::to_string(part->getAddr());
-
-      const auto* desc = part->getDesc();
-      if (desc != nullptr) {
-        r["label"] = desc;
   }
 
-      r["flags"] = "0";
+  // Attempt to open the device image.
-      if (part->getFlags() & TSK_VS_PART_FLAG_META) {
+  opened_ = true;
-        r["type"] = "meta";
+  auto status = image_->open(device_path_.c_str(), TSK_IMG_TYPE_DETECT, 0);
-      } else if (part->getFlags() & TSK_VS_PART_FLAG_UNALLOC) {
-        r["type"] = "unallocated";
-      } else {
-        r["type"] = "normal";
-      }
-
-      auto* fs = new TskFsInfo();
-      auto status = fs->open(part, TSK_FS_TYPE_DETECT);
   if (status) {
-        r["offset"] = BIGINT(part->getStart() * dh.getVolume()->getBlockSize());
+    opened_result_ = false;
-        r["blocks_size"] = BIGINT(dh.getVolume()->getBlockSize());
+    return opened_result_;
-        r["blocks"] = BIGINT(part->getLen());
-        r["inodes"] = "-1";
-        r["flags"] = INTEGER(part->getFlags());
-      } else {
-        // If there is a filesystem in this partition we can use the name/type
-        // of the filesystem as the "type".
-        r["type"] = TskFsInfo::typeToName(fs->getFsType());
-        r["flags"] = INTEGER(fs->getFlags());
-        r["offset"] = BIGINT(fs->getOffset());
-        r["blocks_size"] = BIGINT(fs->getBlockSize());
-        r["blocks"] = BIGINT(fs->getBlockCount());
-        r["inodes"] = BIGINT(fs->getINumCount());
-      }
-      delete fs;
-      results.push_back(r);
-    }));
   }
 
-  return results;
+  // Attempt to open the device image volumn.
+  status = volume_->open(&*image_, 0, TSK_VS_TYPE_DETECT);
+  opened_result_ = (status == 0);
+  return opened_result_;
+}
+
+void DeviceHelper::inodes(
+    const std::set<std::string>& inodes,
+    TskFsInfo* fs,
+    std::function<void(const std::string&, TskFsFile*, const std::string&)>
+        predicate) {
+  // Given a set of constraint inodes, convert each to an INUM.
+  for (const auto& inode : inodes) {
+    long int meta = 0;
+    safeStrtol(inode, 10, meta);
+    auto* file = new TskFsFile();
+    if (file->open(fs, file, static_cast<TSK_INUM_T>(meta)) == 0) {
+      // Attempt to get the meta and filesystem name for the inode.
+      // If this inode is a file a valid meta/name structures are parsed.
+      auto* meta = file->getMeta();
+      if (meta != nullptr) {
+        const auto* name = meta->getName2(0);
+        if (name != nullptr) {
+          auto path = std::string(name->getName());
+          predicate(inode, file, path);
+          delete name;
+        }
+        delete meta;
+      }
+    }
+    delete file;
+  }
 }
 
 void DeviceHelper::generateFile(const std::string& partition,
                                 TskFsFile* file,
+                                TskFsInfo* fs,
                                 const std::string& path,
                                 QueryData& results) {
   Row r;
@@ -182,10 +170,8 @@ void DeviceHelper::generateFile(const std::string& partition,
   r["path"] = path;
   r["filename"] = fs::path(path).leaf().string();
 
-  const auto* fs = file->getFsInfo();
   if (fs != nullptr) {
     r["block_size"] = BIGINT(fs->getBlockSize());
-    delete fs;
   }
 
   const auto* meta = file->getMeta();
@@ -246,7 +232,7 @@ void DeviceHelper::generateFiles(const std::string& partition,
     }
 
     if (meta->getType() == TSK_FS_META_TYPE_REG) {
-      generateFile(partition, file, leaf, results);
+      generateFile(partition, file, fs, leaf, results);
     } else if (meta->getType() == TSK_FS_META_TYPE_DIR) {
       if (name != nullptr && !TSK_FS_ISDOT(name->getName())) {
         additional[meta->getAddr()] = leaf;
@@ -270,6 +256,111 @@ void DeviceHelper::generateFiles(const std::string& partition,
   }
 }
 
+struct DeviceHashes {
+  std::string md5;
+  std::string sha1;
+  std::string sha256;
+};
+
+DeviceHashes hashInode(TskFsFile* file) {
+  Hash md5(HASH_TYPE_MD5);
+  Hash sha1(HASH_TYPE_SHA1);
+  Hash sha256(HASH_TYPE_SHA256);
+
+  // We are guaranteed by the expected callsite to have a valid meta.
+  auto* meta = file->getMeta();
+  if (meta == nullptr) {
+    return DeviceHashes();
+  }
+
+  // Set a maximum 'chunk' or block size to 1 page or the file size.
+  TSK_OFF_T size = meta->getSize();
+  auto buffer_size = (size < 4096) ? size : 4096;
+
+  // Allocate some heap memory and iterate over reading a chunk and updating.
+  auto* buffer = (char*)malloc(buffer_size * sizeof(char*));
+  if (buffer != nullptr) {
+    ssize_t chunk_size = 0;
+    for (ssize_t offset = 0; offset < size; offset += chunk_size) {
+      // Here max represents the local max requested bytes.
+      auto max = (size - offset < buffer_size) ? (size - offset) : buffer_size;
+      chunk_size =
+          file->read(offset, buffer, max, (TSK_FS_FILE_READ_FLAG_ENUM)0U);
+      if (chunk_size == -1 || chunk_size != max) {
+        // Huge problem, either a read failed or didn't read the max size.
+        free(buffer);
+        delete meta;
+        return DeviceHashes();
+      }
+
+      md5.update(buffer, chunk_size);
+      sha1.update(buffer, chunk_size);
+      sha256.update(buffer, chunk_size);
+    }
+    free(buffer);
+  }
+  delete meta;
+
+  // Convert the set of hashes into a device hashes transport.
+  DeviceHashes dhs;
+  dhs.md5 = md5.digest();
+  dhs.sha1 = sha1.digest();
+  dhs.sha256 = sha256.digest();
+  return dhs;
+}
+
+QueryData genDeviceHash(QueryContext& context) {
+  QueryData results;
+
+  auto devices = context.constraints["device"].getAll(EQUALS);
+  // This table requires three columns to determine an action.
+  auto parts = context.constraints["partition"].getAll(EQUALS);
+  auto inodes = context.constraints["inode"].getAll(EQUALS);
+
+  for (const auto& dev : devices) {
+    // For each require device path, open a device helper that checks the
+    // image, checks the volume, and allows partition iteration.
+    DeviceHelper dh(dev);
+    dh.partitions(([&results, &dev, &dh, &parts, &inodes](
+        const TskVsPartInfo* part) {
+      // The table also requires a partition for searching.
+      auto address = std::to_string(part->getAddr());
+      if (address != *parts.begin()) {
+        // If this partition does not match the requested, continue.
+        return;
+      }
+
+      auto* fs = new TskFsInfo();
+      auto status = fs->open(part, TSK_FS_TYPE_DETECT);
+      // Cannot retrieve file information without accessing the filesystem.
+      if (status) {
+        delete fs;
+        return;
+      }
+
+      dh.inodes(inodes,
+                fs,
+                ([&results, &address, &dev, &dh, &fs](const std::string& inode,
+                                                      TskFsFile* file,
+                                                      const std::string& path) {
+                  Row r;
+                  r["device"] = dev;
+                  r["partition"] = address;
+                  r["inode"] = inode;
+
+                  auto hashes = hashInode(file);
+                  r["md5"] = std::move(hashes.md5);
+                  r["sha1"] = std::move(hashes.sha1);
+                  r["sha256"] = std::move(hashes.sha256);
+                  results.push_back(r);
+                }));
+      delete fs;
+    }));
+  }
+
+  return results;
+}
+
 QueryData genDeviceFile(QueryContext& context) {
   QueryData results;
 
@@ -300,7 +391,7 @@ QueryData genDeviceFile(QueryContext& context) {
 
       auto* fs = new TskFsInfo();
       auto status = fs->open(part, TSK_FS_TYPE_DETECT);
-      // Cannot retrieve fail information without accessing the filesystem.
+      // Cannot retrieve file information without accessing the filesystem.
       if (status) {
         delete fs;
         return;
@@ -317,27 +408,70 @@ QueryData genDeviceFile(QueryContext& context) {
       for (const auto& path : paths) {
         auto* file = new TskFsFile();
         if (file->open(fs, file, path.c_str()) == 0) {
-          dh.generateFile(address, file, path, results);
+          dh.generateFile(address, file, fs, path, results);
         }
         delete file;
       }
 
-      for (const auto& inode : inodes) {
+      dh.inodes(inodes,
-        long int meta = 0;
+                fs,
-        safeStrtol(inode, 10, meta);
+                ([&results, &address, &dh, &fs](const std::string& inode,
-        auto* file = new TskFsFile();
+                                                TskFsFile* file,
-        if (file->open(fs, file, static_cast<TSK_INUM_T>(meta)) == 0) {
+                                                const std::string& path) {
-          std::string path;
+                  dh.generateFile(address, file, fs, path, results);
-          auto* meta = file->getMeta();
+                }));
-          if (meta != nullptr) {
+      delete fs;
-            path = std::string(meta->getName2(0)->getName());
+    }));
   }
-          delete meta;
+
-          dh.generateFile(address, file, path, results);
+  return results;
 }
-        delete file;
+
+QueryData genDevicePartitions(QueryContext& context) {
+  QueryData results;
+
+  auto devices = context.constraints["device"].getAll(EQUALS);
+  for (const auto& dev : devices) {
+    DeviceHelper dh(dev);
+    dh.partitions(([&results, &dev, &dh](const TskVsPartInfo* part) {
+      Row r;
+      r["device"] = dev;
+      r["partition"] = std::to_string(part->getAddr());
+
+      const auto* desc = part->getDesc();
+      if (desc != nullptr) {
+        r["label"] = desc;
+      }
+
+      r["flags"] = "0";
+      if (part->getFlags() & TSK_VS_PART_FLAG_META) {
+        r["type"] = "meta";
+      } else if (part->getFlags() & TSK_VS_PART_FLAG_UNALLOC) {
+        r["type"] = "unallocated";
+      } else {
+        r["type"] = "normal";
+      }
+
+      auto* fs = new TskFsInfo();
+      auto status = fs->open(part, TSK_FS_TYPE_DETECT);
+      if (status) {
+        r["offset"] = BIGINT(part->getStart() * dh.getVolume()->getBlockSize());
+        r["blocks_size"] = BIGINT(dh.getVolume()->getBlockSize());
+        r["blocks"] = BIGINT(part->getLen());
+        r["inodes"] = "-1";
+        r["flags"] = INTEGER(part->getFlags());
+      } else {
+        // If there is a filesystem in this partition we can use the name/type
+        // of the filesystem as the "type".
+        r["type"] = TskFsInfo::typeToName(fs->getFsType());
+        r["flags"] = INTEGER(fs->getFlags());
+        r["offset"] = BIGINT(fs->getOffset());
+        r["blocks_size"] = BIGINT(fs->getBlockSize());
+        r["blocks"] = BIGINT(fs->getBlockCount());
+        r["inodes"] = BIGINT(fs->getINumCount());
       }
       delete fs;
+      results.push_back(r);
     }));
   }
 

specs/device_hash.table

@@ -0,0 +1,11 @@
+table_name("device_hash")
+description("Similar to the hash table, but use TSK and allow block address access.")
+schema([
+    Column("device", TEXT, "Absolute file path to device node", required=True),
+    Column("partition", TEXT, "A partition number", required=True),
+    Column("inode", BIGINT, "Filesystem inode number", required=True),
+    Column("md5", TEXT, "MD5 hash of provided inode data"),
+    Column("sha1", TEXT, "SHA1 hash of provided inode data"),
+    Column("sha256", TEXT, "SHA256 hash of provided inode data"),
+])
+implementation("forensic/sleuthkit@genDeviceHash")

tools/profile.py

@@ -17,7 +17,6 @@ import os
 import subprocess
 import sys
 import time
-import utils
 
 try:
     import argparse
@@ -28,6 +27,8 @@ except ImportError:
 # Import the testing utils
 sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/tests/")
 
+import utils
+
 KB = 1024 * 1024
 RANGES = {
     "colors": (utils.blue, utils.green, utils.yellow, utils.red),
@@ -306,7 +307,7 @@ if __name__ == "__main__":
             profile1 = json.loads(fh.read())
 
     if not os.path.exists(args.shell):
-        print("Cannot find --daemon: %s" % (args.shell))
+        print("Cannot find --shell: %s" % (args.shell))
         exit(1)
     if args.config is None and not os.path.exists(args.tables):
         print("Cannot find --tables: %s" % (args.tables))