empayre/fleet
14
0
Fork 0
mirror of https://github.com/empayre/fleet.git synced 2024-11-06 08:55:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add CentOS parsing+post-processing to reduce false positives in vulnerability processing (#4037)

Browse Source 
* Add CentOS parsing and post-processing in fleet

* Add tests and amend SyncCPEDatabase

* Add test for centosPostProcessing

* Changes from PR comments

* Amend software test

* Fix sync test

* Add index to source and vendor

* Use os.MkdirTemp

* Rearrange migrations

* Regenerate test schema

* Add support for testing migrations (#4112)

* Add support for testing migrations

* Rename migration in tests

* Changes suggested in PR

* Go mod tidy
This commit is contained in:
Lucas Manuel Rodriguez 2022-02-14 15:13:44 -03:00 committed by GitHub
parent 57d9546081
commit be72dc356c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 1599 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -57,3 +57,8 @@ terraform.tfstate*
# generated installers # generated installers
fleet-osquery* fleet-osquery*
# residual files when running the cpe command
cmd/cpe/etagenv
cmd/cpe/cpe*.sqlite
cmd/cpe/cpe*.sqlite.gz

1
changes/issue-3081-parse-centos-repository Normal file
View File

@ -0,0 +1 @@
* Add parsing of the CentOS repository as well as the loading of the results in fleet vulnerability post-processing (to reduce false positives on RPM packages).

103
cmd/cpe/generate.go
View File

@ -2,6 +2,8 @@ package main
import ( import (
"compress/gzip" "compress/gzip"
"database/sql"
"flag"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@ -11,6 +13,7 @@ import (
"github.com/facebookincubator/nvdtools/cpedict" "github.com/facebookincubator/nvdtools/cpedict"
"github.com/fleetdm/fleet/v4/server/vulnerabilities" "github.com/fleetdm/fleet/v4/server/vulnerabilities"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/vuln_centos"
) )
func panicif(err error) { func panicif(err error) {
@ -20,11 +23,43 @@ func panicif(err error) {
} }
func main() { func main() {
fmt.Println("Starting CPE sqlite generation") var (
runCentOS bool
verbose bool
)
flag.BoolVar(&runCentOS, "centos", true, "Sets whether to run the CentOS sqlite generation")
flag.BoolVar(&verbose, "verbose", false, "Sets verbose mode")
flag.Parse()
dbPath := cpe()
fmt.Printf("Sqlite file %s size: %.2f MB\n", dbPath, getSizeMB(dbPath))
// The CentOS repository data is added to the CPE database.
if runCentOS {
centos(dbPath, verbose)
fmt.Printf("Sqlite file %s size with CentOS data: %.2f MB\n", dbPath, getSizeMB(dbPath))
}
fmt.Println("Compressing DB...")
compressedPath, err := compress(dbPath)
panicif(err)
fmt.Printf("Final compressed file %s size: %.2f MB\n", compressedPath, getSizeMB(compressedPath))
fmt.Println("Done.")
}
func getSizeMB(path string) float64 {
info, err := os.Stat(path)
panicif(err)
return float64(info.Size()) / 1024.0 / 1024.0
}
func cpe() string {
fmt.Println("Starting CPE sqlite generation...")
cwd, err := os.Getwd() cwd, err := os.Getwd()
panicif(err) panicif(err)
fmt.Println("CWD:", cwd) fmt.Println("CWD:", cwd)
resp, err := http.Get("https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz") resp, err := http.Get("https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz")
@ -34,16 +69,6 @@ func main() {
remoteEtag := getSanitizedEtag(resp) remoteEtag := getSanitizedEtag(resp)
fmt.Println("Got ETag:", remoteEtag) fmt.Println("Got ETag:", remoteEtag)
nvdRelease, err := vulnerabilities.GetLatestNVDRelease(nil)
panicif(err)
if nvdRelease != nil && nvdRelease.Etag == remoteEtag {
fmt.Println("No updates. Exiting...")
return
}
fmt.Println("Needs updating. Generating...")
gr, err := gzip.NewReader(resp.Body) gr, err := gzip.NewReader(resp.Body)
panicif(err) panicif(err)
defer gr.Close() defer gr.Close()
@ -56,25 +81,51 @@ func main() {
err = vulnerabilities.GenerateCPEDB(dbPath, cpeDict) err = vulnerabilities.GenerateCPEDB(dbPath, cpeDict)
panicif(err) panicif(err)
fmt.Println("Compressing db...")
compressedDB, err := os.Create(fmt.Sprintf("%s.gz", dbPath))
panicif(err)
db, err := os.Open(dbPath)
panicif(err)
w := gzip.NewWriter(compressedDB)
_, err = io.Copy(w, db)
panicif(err)
w.Close()
compressedDB.Close()
file, err := os.Create(path.Join(cwd, "etagenv")) file, err := os.Create(path.Join(cwd, "etagenv"))
panicif(err) panicif(err)
file.WriteString(fmt.Sprintf(`ETAG=%s`, remoteEtag)) file.WriteString(fmt.Sprintf(`ETAG=%s`, remoteEtag))
file.Close() file.Close()
fmt.Println("Done.") return dbPath
}
func compress(path string) (string, error) {
compressedPath := fmt.Sprintf("%s.gz", path)
compressedDB, err := os.Create(compressedPath)
if err != nil {
return "", err
}
defer compressedDB.Close()
db, err := os.Open(path)
if err != nil {
return "", err
}
defer db.Close()
w := gzip.NewWriter(compressedDB)
defer w.Close()
_, err = io.Copy(w, db)
if err != nil {
return "", err
}
return compressedPath, nil
}
func centos(dbPath string, verbose bool) {
fmt.Println("Starting CentOS sqlite generation...")
db, err := sql.Open("sqlite3", dbPath)
panicif(err)
defer db.Close()
pkgs, err := vuln_centos.ParseCentOSRepository(vuln_centos.WithVerbose(verbose))
panicif(err)
fmt.Printf("Storing CVE info for %d CentOS packages...\n", len(pkgs))
err = vuln_centos.GenCentOSSqlite(db, pkgs)
panicif(err)
} }
func getSanitizedEtag(resp *http.Response) string { func getSanitizedEtag(resp *http.Response) string {

10
cmd/fleet/serve.go
View File

@ -722,6 +722,16 @@ func cronVulnerabilities(
sentry.CaptureException(err) sentry.CaptureException(err)
} }
// It's important vulnerabilities.PostProcess runs after ds.CalculateHostsPerSoftware
// because it cleans up any software that's not installed on the fleet (e.g. hosts removal,
// or software being uninstalled on hosts).
if !vulnDisabled {
if err := vulnerabilities.PostProcess(ctx, ds, vulnPath, logger, config); err != nil {
level.Error(logger).Log("msg", "post processing CVEs", "err", err)
sentry.CaptureException(err)
}
}
level.Debug(logger).Log("loop", "done") level.Debug(logger).Log("loop", "done")
} }
} }

2
cmd/fleetctl/vulnerability_data_stream.go
View File

@ -47,7 +47,7 @@ Downloads (if needed) the data streams that can be used by the Fleet server to p
dbPath := path.Join(dir, "cpe.sqlite") dbPath := path.Join(dir, "cpe.sqlite")
client := fleethttp.NewClient() client := fleethttp.NewClient()
err = vulnerabilities.SyncCPEDatabase(client, dbPath, config.FleetConfig{}) err = vulnerabilities.SyncCPEDatabase(client, dbPath)
if err != nil { if err != nil {
return err return err
} }

2
docs/03-Contributing/02-Testing.md
View File

@ -226,7 +226,7 @@ To intercept sent emails while running a Fleet development environment, first, i
Then, in the "SMTP options" section, enter any email address in the "Sender address" field, set the "SMTP server" to `localhost` on port `1025`, and set "Authentication type" to `None`. Note that you may use any active or inactive sender address. Then, in the "SMTP options" section, enter any email address in the "Sender address" field, set the "SMTP server" to `localhost` on port `1025`, and set "Authentication type" to `None`. Note that you may use any active or inactive sender address.
Visit [locahost:8025](http://localhost:8025) to view Mailhog's admin interface which will display all emails sent using the simulated mail server. Visit [localhost:8025](http://localhost:8025) to view Mailhog's admin interface which will display all emails sent using the simulated mail server.
## Development database management ## Development database management

9
go.mod
View File

@ -7,9 +7,12 @@ require (
github.com/AbGuthrie/goquery/v2 v2.0.1 github.com/AbGuthrie/goquery/v2 v2.0.1
github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect
github.com/PuerkitoBio/goquery v1.8.0 // indirect
github.com/VividCortex/gohistogram v1.0.0 // indirect github.com/VividCortex/gohistogram v1.0.0 // indirect
github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f
github.com/WatchBeam/clock v0.0.0-20170901150240-b08e6b4da7ea github.com/WatchBeam/clock v0.0.0-20170901150240-b08e6b4da7ea
github.com/antchfx/htmlquery v1.2.4 // indirect
github.com/antchfx/xmlquery v1.3.9 // indirect
github.com/aws/aws-sdk-go v1.40.34 github.com/aws/aws-sdk-go v1.40.34
github.com/beevik/etree v1.1.0 github.com/beevik/etree v1.1.0
github.com/briandowns/spinner v1.13.0 github.com/briandowns/spinner v1.13.0
@ -29,6 +32,7 @@ require (
github.com/ghodss/yaml v1.0.0 github.com/ghodss/yaml v1.0.0
github.com/go-kit/kit v0.9.0 github.com/go-kit/kit v0.9.0
github.com/go-sql-driver/mysql v1.6.0 github.com/go-sql-driver/mysql v1.6.0
github.com/gocolly/colly v1.2.0
github.com/golang-jwt/jwt/v4 v4.0.0 github.com/golang-jwt/jwt/v4 v4.0.0
github.com/gomodule/redigo v1.8.5 github.com/gomodule/redigo v1.8.5
github.com/google/go-cmp v0.5.6 github.com/google/go-cmp v0.5.6
@ -45,6 +49,7 @@ require (
github.com/jinzhu/copier v0.3.2 github.com/jinzhu/copier v0.3.2
github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5
github.com/jonboulle/clockwork v0.2.2 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
github.com/kevinburke/go-bindata v3.22.0+incompatible github.com/kevinburke/go-bindata v3.22.0+incompatible
github.com/kolide/kit v0.0.0-20180421083548-36eb8dc43916 github.com/kolide/kit v0.0.0-20180421083548-36eb8dc43916
github.com/kolide/launcher v0.0.0-20180427153757-cb412b945cf7 github.com/kolide/launcher v0.0.0-20180427153757-cb412b945cf7
@ -68,14 +73,18 @@ require (
github.com/rotisserie/eris v0.5.1 github.com/rotisserie/eris v0.5.1
github.com/rs/zerolog v1.20.0 github.com/rs/zerolog v1.20.0
github.com/russellhaering/goxmldsig v1.1.0 github.com/russellhaering/goxmldsig v1.1.0
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
github.com/spf13/cast v1.3.1 github.com/spf13/cast v1.3.1
github.com/spf13/cobra v1.2.1 github.com/spf13/cobra v1.2.1
github.com/spf13/viper v1.8.1 github.com/spf13/viper v1.8.1
github.com/stretchr/objx v0.3.0 // indirect github.com/stretchr/objx v0.3.0 // indirect
github.com/stretchr/testify v1.7.0 github.com/stretchr/testify v1.7.0
github.com/temoto/robotstxt v1.1.2 // indirect
github.com/theupdateframework/go-tuf v0.0.0-20220121203041-e3557e322879 github.com/theupdateframework/go-tuf v0.0.0-20220121203041-e3557e322879
github.com/throttled/throttled/v2 v2.8.0 github.com/throttled/throttled/v2 v2.8.0
github.com/tj/assert v0.0.3
github.com/ulikunitz/xz v0.5.10
github.com/urfave/cli/v2 v2.3.0 github.com/urfave/cli/v2 v2.3.0
github.com/valyala/fasthttp v1.31.0 github.com/valyala/fasthttp v1.31.0
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce

22
go.sum
View File

@ -164,6 +164,8 @@ github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a h1:W6RrgN/sTxg1
github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a/go.mod h1:NYt+V3/4rEeDuaev/zw1zCq8uqVEuPHzDPo3OZrlGJ4= github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a/go.mod h1:NYt+V3/4rEeDuaev/zw1zCq8uqVEuPHzDPo3OZrlGJ4=
github.com/ProtonMail/gopenpgp/v2 v2.2.2 h1:u2m7xt+CZWj88qK1UUNBoXeJCFJwJCZ/Ff4ymGoxEXs= github.com/ProtonMail/gopenpgp/v2 v2.2.2 h1:u2m7xt+CZWj88qK1UUNBoXeJCFJwJCZ/Ff4ymGoxEXs=
github.com/ProtonMail/gopenpgp/v2 v2.2.2/go.mod h1:ajUlBGvxMH1UBZnaYO3d1FSVzjiC6kK9XlZYGiDCvpM= github.com/ProtonMail/gopenpgp/v2 v2.2.2/go.mod h1:ajUlBGvxMH1UBZnaYO3d1FSVzjiC6kK9XlZYGiDCvpM=
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE=
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
@ -189,8 +191,16 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/andybalholm/brotli v1.0.2 h1:JKnhI/XQ75uFBTiuzXpzFrUriDPiZjlOSzh6wXogP0E= github.com/andybalholm/brotli v1.0.2 h1:JKnhI/XQ75uFBTiuzXpzFrUriDPiZjlOSzh6wXogP0E=
github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494=
github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc=
github.com/antchfx/xmlquery v1.3.9 h1:Y+zyMdiUZ4fasTQTkDb3DflOXP7+obcYEh80SISBmnQ=
github.com/antchfx/xmlquery v1.3.9/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks=
github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/apache/thrift v0.13.1-0.20200603211036-eac4d0c79a5f h1:33BV5v3u8I6dA2dEoPuXWCsAaHHOJfPtdxZhAMQV4uo= github.com/apache/thrift v0.13.1-0.20200603211036-eac4d0c79a5f h1:33BV5v3u8I6dA2dEoPuXWCsAaHHOJfPtdxZhAMQV4uo=
github.com/apache/thrift v0.13.1-0.20200603211036-eac4d0c79a5f/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.1-0.20200603211036-eac4d0c79a5f/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@ -436,6 +446,8 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@ -682,6 +694,8 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb
github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE=
github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro=
github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
github.com/kevinburke/go-bindata v3.22.0+incompatible h1:/JmqEhIWQ7GRScV0WjX/0tqBrC5D21ALg0H0U/KZ/ts= github.com/kevinburke/go-bindata v3.22.0+incompatible h1:/JmqEhIWQ7GRScV0WjX/0tqBrC5D21ALg0H0U/KZ/ts=
github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM=
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
@ -914,6 +928,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc= github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc=
@ -990,6 +1006,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM= github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM=
github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog= github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog=
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
github.com/theupdateframework/go-tuf v0.0.0-20220121203041-e3557e322879 h1:UeDpdrX16scCvbdgdMsrztZsQLDofld/Zo+WGDe/PBE= github.com/theupdateframework/go-tuf v0.0.0-20220121203041-e3557e322879 h1:UeDpdrX16scCvbdgdMsrztZsQLDofld/Zo+WGDe/PBE=
github.com/theupdateframework/go-tuf v0.0.0-20220121203041-e3557e322879/go.mod h1:I0Gs4Tev4hYQ5wiNqN8VJ7qS0gw7KOZNQuckC624RmE= github.com/theupdateframework/go-tuf v0.0.0-20220121203041-e3557e322879/go.mod h1:I0Gs4Tev4hYQ5wiNqN8VJ7qS0gw7KOZNQuckC624RmE=
github.com/throttled/throttled/v2 v2.8.0 h1:B5VfdM8BE+ClI2Ji238SbNOTWfYcocvuAhgT27lvwrE= github.com/throttled/throttled/v2 v2.8.0 h1:B5VfdM8BE+ClI2Ji238SbNOTWfYcocvuAhgT27lvwrE=
@ -1052,7 +1070,6 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/zclconf/go-cty v1.1.0 h1:uJwc9HiBOCpoKIObTQaLR+tsEXx1HBHnOsOOpcdhZgw= github.com/zclconf/go-cty v1.1.0 h1:uJwc9HiBOCpoKIObTQaLR+tsEXx1HBHnOsOOpcdhZgw=
github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c h1:TWQ2UvXPkhPxI2KmApKBOCaV6yD2N4mlvqFQ/DlPtpQ= github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c h1:TWQ2UvXPkhPxI2KmApKBOCaV6yD2N4mlvqFQ/DlPtpQ=
github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c/go.mod h1:OYYulo9tUqRadRLwB0+LE914sa1ui2yL7OrcU3Q/1XY= github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c/go.mod h1:OYYulo9tUqRadRLwB0+LE914sa1ui2yL7OrcU3Q/1XY=
@ -1183,6 +1200,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
@ -1190,6 +1208,7 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@ -1209,6 +1228,7 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211007125505-59d4e928ea9d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211007125505-59d4e928ea9d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=

2
pkg/README.md
View File

@ -1,3 +1,3 @@
# pkg directory # pkg directory
This top-level `pkg` directory contains packages that may be shared between `fleet` and `orbit`. This top-level `pkg` directory contains packages that may be shared between all `fleet` backend components.

70
pkg/download/download.go Normal file
View File

@ -0,0 +1,70 @@
// Package download has utilities to download resources from URLs.
package download
import (
"compress/bzip2"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"github.com/ulikunitz/xz"
)
// Decompressed downloads and decompresses a file from a URL to a local path.
//
// It supports gz, bz2 and gz compressed files.
func Decompressed(client *http.Client, u url.URL, path string) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
tmpFile, err := ioutil.TempFile("", fmt.Sprintf("%s*", filepath.Base(path)))
if err != nil {
return err
}
defer tmpFile.Close()
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return err
}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
var decompressor io.Reader
switch {
case strings.HasSuffix(u.Path, "gz"):
decompressor, err = gzip.NewReader(resp.Body)
if err != nil {
return err
}
case strings.HasSuffix(u.Path, "bz2"):
decompressor = bzip2.NewReader(resp.Body)
case strings.HasSuffix(u.Path, "xz"):
decompressor, err = xz.NewReader(resp.Body)
if err != nil {
return err
}
default:
return fmt.Errorf("unknown extension: %s", u.Path)
}
if _, err := io.Copy(tmpFile, decompressor); err != nil {
return err
}
if err := tmpFile.Close(); err != nil {
return err
}
if err := os.Rename(tmpFile.Name(), path); err != nil {
return err
}
return nil
}

66
server/datastore/mysql/migrations/tables/20220208144831_AddSoftwareReleaseArchVendorColumns.go Normal file
View File

@ -0,0 +1,66 @@
package tables
import (
"database/sql"
"fmt"
"github.com/pkg/errors"
)
func init() {
MigrationClient.AddMigration(Up_20220208144831, Down_20220208144831)
}
func Up_20220208144831(tx *sql.Tx) error {
// NOTE(lucas): I'm using short lengths for the new varchar columns
// due to constraints on the size of the key added below. Using 255
// for the three new fields would fail with:
// "Error 1071: Specified key was too long; max key length is 3072 bytes".
//
// We need to use "NOT NULL" because these new columns are to be included in the KEY.
if _, err := tx.Exec("ALTER TABLE software " +
"ADD COLUMN `release` VARCHAR(64) NOT NULL DEFAULT '', " +
"ADD COLUMN vendor VARCHAR(32) NOT NULL DEFAULT '', " +
"ADD COLUMN arch VARCHAR(16) NOT NULL DEFAULT ''"); err != nil {
return errors.Wrap(err, "add new software columns")
}
// Delete current identifier for a software.
currIndexName, err := indexNameByColumnName(tx, "software", "name")
if err != nil {
return errors.Wrap(err, "fetch current software index")
}
if _, err := tx.Exec(fmt.Sprintf("ALTER TABLE software DROP KEY %s", currIndexName)); err != nil {
return errors.Wrap(err, "add new software columns")
}
// A software piece was originally identified by name, version and source.
//
// We now add "vendor", "release" and "arch":
// - release is the version of the OS this software was released on (e.g. "30.el7" for a CentOS package).
// - vendor is the supplier of the software (e.g. "CentOS").
// - arch is the target architecture of the software (e.g. "x86_64").
if _, err := tx.Exec("ALTER TABLE software ADD UNIQUE KEY (name, version, source, `release`, vendor, arch)"); err != nil {
return errors.Wrap(err, "add new index")
}
// Remove all software with source rpm_packages, as we will be ingesting them with new osquery
// fields.
//
// Due to foreign keys, the following statement also deletes the corresponding
// entries in `software_cpe` and `software_cve`.
if _, err := tx.Exec("DELETE FROM software WHERE source = 'rpm_packages'"); err != nil {
return errors.Wrap(err, "delete existing software for rpm_packages")
}
// Adding index to optimize software listing by source and vendor for vulnerability post-processing.
if _, err := tx.Exec("CREATE INDEX software_source_vendor_idx ON software (source, vendor)"); err != nil {
return errors.Wrap(err, "creating source+vendor index")
}
return nil
}
func Down_20220208144831(tx *sql.Tx) error {
return nil
}

35
server/datastore/mysql/migrations/tables/20220208144831_AddSoftwareReleaseArchVendorColumns_test.go Normal file
View File

@ -0,0 +1,35 @@
package tables
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestUp_20220208144831(t *testing.T) {
db := applyUpToPrev(t)
_, err := db.Exec(`INSERT INTO software (name, version, source) VALUES ("authconfig", "6.2.8", "rpm_packages")`)
require.NoError(t, err)
_, err = db.Exec(`INSERT INTO software (name, version, source, bundle_identifier) VALUES ("iTerm.app", "3.4.14", "apps", "com.googlecode.iterm2")`)
require.NoError(t, err)
applyNext(t, db)
// Check migration removes rpm packages.
row := db.QueryRow(`SELECT COUNT(*) FROM software WHERE source = "rpm_packages"`)
var count int
require.NoError(t, row.Scan(&count))
require.Zero(t, count)
row = db.QueryRow(`SELECT COUNT(*) FROM software WHERE source = "apps"`)
require.NoError(t, row.Scan(&count))
require.Equal(t, 1, count)
// Check we can INSERT software with the new columns empty.
_, err = db.Exec(`INSERT INTO software (name, version, source, bundle_identifier) VALUES ("iCloud.app", "1.0", "apps", "com.apple.CloudKit.ShareBear")`)
require.NoError(t, err)
// Check we can INSERT software with the new columns set.
_, err = db.Exec(`INSERT INTO software (name, version, source, ` + "`release`" + `, vendor, arch) VALUES ("authconfig", "6.2.8", "rpm_packages", "30.el7", "CentOS", "x86_64")`)
require.NoError(t, err)
}

86
server/datastore/mysql/migrations/tables/migration_test.go Normal file
View File

@ -0,0 +1,86 @@
// Package tables holds fleet table migrations.
//
// Migrations can be tested with tests following the following format:
//
// $ cat 20220208144831_AddSoftwareReleaseArchVendorColumns_test.go
//
// [...]
// func TestUp_20220208144831(t *testing.T) {
// // Apply all migrations up to 20220208144831 (name of test), not included.
// db := applyUpToPrev(t)
//
// // insert testing data, etc.
//
// // The following will apply migration 20220208144831.
// applyNext(t, db)
//
// // insert testing data, verify migration.
// }
package tables
import (
"fmt"
"strconv"
"strings"
"testing"
"github.com/jmoiron/sqlx"
"github.com/stretchr/testify/require"
)
// TODO(lucas): I'm copy pasting some of the mysql functionality methods here
// otherwise we have import cycle errors.
//
// We need to decouple the server/datastore/mysql package,
// it contains both the implementation of the fleet.Datastore and
// MySQL functionality, and MySQL test functionality.
const (
testUsername = "root"
testPassword = "toor"
testAddress = "localhost:3307"
)
func newDBConnForTests(t *testing.T) *sqlx.DB {
db, err := sqlx.Open(
"mysql",
fmt.Sprintf("%s:%s@tcp(%s)/?multiStatements=true", testUsername, testPassword, testAddress),
)
require.NoError(t, err)
_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s; USE %s;", t.Name(), t.Name(), t.Name()))
require.NoError(t, err)
return db
}
func getMigrationVersion(t *testing.T) int64 {
v, err := strconv.Atoi(strings.TrimPrefix(t.Name(), "TestUp_"))
require.NoError(t, err)
return int64(v)
}
// applyUpToPrev will allocate a testing DB connection and apply
// migrations up to, not including, the migration specified in the test name.
//
// It returns the database connection to perform additional queries and migrations.
func applyUpToPrev(t *testing.T) *sqlx.DB {
db := newDBConnForTests(t)
v := getMigrationVersion(t)
for {
current, err := MigrationClient.GetDBVersion(db.DB)
require.NoError(t, err)
next, err := MigrationClient.Migrations.Next(current)
require.NoError(t, err)
if next.Version == v {
return db
}
applyNext(t, db)
}
}
// applyNext performs the next migration in the chain.
func applyNext(t *testing.T, db *sqlx.DB) {
// gooseNoDir is the value to not parse local files and instead use
// the migrations that were added manually via Add().
const gooseNoDir = ""
err := MigrationClient.UpByOne(db.DB, gooseNoDir)
require.NoError(t, err)
}

12
server/datastore/mysql/schema.sql
View File

File diff suppressed because one or more lines are too long

99
server/datastore/mysql/software.go
View File

@ -19,6 +19,10 @@ const (
maxSoftwareVersionLen = 255 maxSoftwareVersionLen = 255
maxSoftwareSourceLen = 64 maxSoftwareSourceLen = 64
maxSoftwareBundleIdentifierLen = 255 maxSoftwareBundleIdentifierLen = 255
maxSoftwareReleaseLen = 64
maxSoftwareVendorLen = 32
maxSoftwareArchLen = 16
) )
func truncateString(str string, length int) string { func truncateString(str string, length int) string {
@ -29,16 +33,36 @@ func truncateString(str string, length int) string {
} }
func softwareToUniqueString(s fleet.Software) string { func softwareToUniqueString(s fleet.Software) string {
return strings.Join([]string{s.Name, s.Version, s.Source, s.BundleIdentifier}, "\u0000") ss := []string{s.Name, s.Version, s.Source, s.BundleIdentifier}
// Release, Vendor and Arch fields were added on a migration,
// thus we only include them in the string if at least one of them is defined.
if s.Release != "" || s.Vendor != "" || s.Arch != "" {
ss = append(ss, s.Release, s.Vendor, s.Arch)
}
return strings.Join(ss, "\u0000")
} }
func uniqueStringToSoftware(s string) fleet.Software { func uniqueStringToSoftware(s string) fleet.Software {
parts := strings.Split(s, "\u0000") parts := strings.Split(s, "\u0000")
// Release, Vendor and Arch fields were added on a migration,
// If one of them is defined, then they are included in the string.
var release, vendor, arch string
if len(parts) > 4 {
release = truncateString(parts[4], maxSoftwareReleaseLen)
vendor = truncateString(parts[5], maxSoftwareVendorLen)
arch = truncateString(parts[6], maxSoftwareArchLen)
}
return fleet.Software{ return fleet.Software{
Name: truncateString(parts[0], maxSoftwareNameLen), Name: truncateString(parts[0], maxSoftwareNameLen),
Version: truncateString(parts[1], maxSoftwareVersionLen), Version: truncateString(parts[1], maxSoftwareVersionLen),
Source: truncateString(parts[2], maxSoftwareSourceLen), Source: truncateString(parts[2], maxSoftwareSourceLen),
BundleIdentifier: truncateString(parts[3], maxSoftwareBundleIdentifierLen), BundleIdentifier: truncateString(parts[3], maxSoftwareBundleIdentifierLen),
Release: release,
Vendor: vendor,
Arch: arch,
} }
} }
@ -151,8 +175,10 @@ func getOrGenerateSoftwareIdDB(ctx context.Context, tx sqlx.ExtContext, s fleet.
var existingId []int64 var existingId []int64
if err := sqlx.SelectContext(ctx, tx, if err := sqlx.SelectContext(ctx, tx,
&existingId, &existingId,
`SELECT id FROM software WHERE name = ? AND version = ? AND source = ? AND bundle_identifier = ?`, "SELECT id FROM software "+
s.Name, s.Version, s.Source, s.BundleIdentifier, "WHERE name = ? AND version = ? AND source = ? AND `release` = ? AND "+
"vendor = ? AND arch = ? AND bundle_identifier = ?",
s.Name, s.Version, s.Source, s.Release, s.Vendor, s.Arch, s.BundleIdentifier,
); err != nil { ); err != nil {
return 0, ctxerr.Wrap(ctx, err, "get software") return 0, ctxerr.Wrap(ctx, err, "get software")
} }
@ -161,9 +187,11 @@ func getOrGenerateSoftwareIdDB(ctx context.Context, tx sqlx.ExtContext, s fleet.
} }
result, err := tx.ExecContext(ctx, result, err := tx.ExecContext(ctx,
`INSERT INTO software (name, version, source, bundle_identifier) VALUES (?, ?, ?, ?) "INSERT INTO software "+
ON DUPLICATE KEY UPDATE bundle_identifier=VALUES(bundle_identifier)`, "(name, version, source, `release`, vendor, arch, bundle_identifier) "+
s.Name, s.Version, s.Source, s.BundleIdentifier, "VALUES (?, ?, ?, ?, ?, ?, ?) "+
"ON DUPLICATE KEY UPDATE bundle_identifier=VALUES(bundle_identifier)",
s.Name, s.Version, s.Source, s.Release, s.Vendor, s.Arch, s.BundleIdentifier,
) )
if err != nil { if err != nil {
return 0, ctxerr.Wrap(ctx, err, "insert software") return 0, ctxerr.Wrap(ctx, err, "insert software")
@ -503,6 +531,59 @@ func (ds *Datastore) CountSoftware(ctx context.Context, opt fleet.SoftwareListOp
return countSoftwareDB(ctx, ds.reader, nil, opt) return countSoftwareDB(ctx, ds.reader, nil, opt)
} }
// ListVulnerableSoftwareBySource lists all the vulnerable software that matches the given source.
func (ds *Datastore) ListVulnerableSoftwareBySource(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) {
var softwareCVEs []struct {
fleet.Software
CPE uint `db:"cpe_id"`
CVEs string `db:"cves"`
}
if err := sqlx.SelectContext(ctx, ds.reader, &softwareCVEs, `
SELECT s.*, scv.cpe_id, GROUP_CONCAT(scv.cve SEPARATOR ',') as cves
FROM software s
JOIN software_cpe scp ON (s.id=scp.software_id)
JOIN software_cve scv ON (scp.id=scv.cpe_id)
WHERE s.source = ?
GROUP BY scv.cpe_id
`, source); err != nil {
return nil, ctxerr.Wrapf(ctx, err, "listing vulnerable software by source")
}
software := make([]fleet.SoftwareWithCPE, 0, len(softwareCVEs))
for _, sc := range softwareCVEs {
for _, cve := range strings.Split(sc.CVEs, ",") {
sc.Software.Vulnerabilities = append(sc.Software.Vulnerabilities, fleet.SoftwareCVE{
CVE: cve,
DetailsLink: fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cve),
})
}
software = append(software, fleet.SoftwareWithCPE{
Software: sc.Software,
CPEID: sc.CPE,
})
}
return software, nil
}
// DeleteVulnerabilitiesByCPECVE deletes the given list of vulnerabilities identified by CPE+CVE.
func (ds *Datastore) DeleteVulnerabilitiesByCPECVE(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error {
if len(vulnerabilities) == 0 {
return nil
}
sql := fmt.Sprintf(
`DELETE FROM software_cve WHERE (cpe_id, cve) IN (%s)`,
strings.TrimSuffix(strings.Repeat("(?,?),", len(vulnerabilities)), ","),
)
var args []interface{}
for _, vulnerability := range vulnerabilities {
args = append(args, vulnerability.CPEID, vulnerability.CVE)
}
if _, err := ds.writer.ExecContext(ctx, sql, args...); err != nil {
return ctxerr.Wrapf(ctx, err, "deleting vulnerable software")
}
return nil
}
func (ds *Datastore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software, error) { func (ds *Datastore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software, error) {
software := fleet.Software{} software := fleet.Software{}
err := sqlx.GetContext(ctx, ds.reader, &software, `SELECT * FROM software WHERE id=?`, id) err := sqlx.GetContext(ctx, ds.reader, &software, `SELECT * FROM software WHERE id=?`, id)
@ -545,6 +626,9 @@ func (ds *Datastore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software
// CalculateHostsPerSoftware calculates the number of hosts having each // CalculateHostsPerSoftware calculates the number of hosts having each
// software installed and stores that information in the software_host_counts // software installed and stores that information in the software_host_counts
// table. // table.
//
// After aggregation, it cleans up unused software (e.g. software installed
// on removed hosts, software uninstalled on hosts, etc.)
func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error { func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error {
resetStmt := ` resetStmt := `
UPDATE software_host_counts UPDATE software_host_counts
@ -614,8 +698,6 @@ func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt ti
return ctxerr.Wrap(ctx, err, "iterate over host_software counts") return ctxerr.Wrap(ctx, err, "iterate over host_software counts")
} }
// delete any unused software from the software table (any that
// isn't in that list with a host count > 0).
cleanupStmt := ` cleanupStmt := `
DELETE FROM DELETE FROM
software software
@ -630,7 +712,6 @@ func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt ti
if _, err := ds.writer.ExecContext(ctx, cleanupStmt); err != nil { if _, err := ds.writer.ExecContext(ctx, cleanupStmt); err != nil {
return ctxerr.Wrap(ctx, err, "delete unused software") return ctxerr.Wrap(ctx, err, "delete unused software")
} }
return nil return nil
} }

154
server/datastore/mysql/software_test.go
View File

@ -33,6 +33,8 @@ func TestSoftware(t *testing.T) {
{"LoadSupportsTonsOfCVEs", testSoftwareLoadSupportsTonsOfCVEs}, {"LoadSupportsTonsOfCVEs", testSoftwareLoadSupportsTonsOfCVEs},
{"List", testSoftwareList}, {"List", testSoftwareList},
{"CalculateHostsPerSoftware", testSoftwareCalculateHostsPerSoftware}, {"CalculateHostsPerSoftware", testSoftwareCalculateHostsPerSoftware},
{"ListVulnerableSoftwareBySource", testListVulnerableSoftwareBySource},
{"DeleteVulnerabilitiesByCPECVE", testDeleteVulnerabilitiesByCPECVE},
} }
for _, c := range cases { for _, c := range cases {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
@ -426,7 +428,15 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
t.Run("paginates", func(t *testing.T) { t.Run("paginates", func(t *testing.T) {
software := listSoftwareCheckCount(t, ds, 1, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{Page: 1, PerPage: 1, OrderKey: "version"}}, true) software := listSoftwareCheckCount(t, ds, 1, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{Page: 1, PerPage: 1, OrderKey: "version"}}, true)
expected := []fleet.Software{foo003} require.Len(t, software, 1)
var expected []fleet.Software
// Both foo003 and bar003 have the same version, thus we check which one the database picked
// for the second page.
if software[0].Name == "foo" {
expected = []fleet.Software{foo003}
} else {
expected = []fleet.Software{bar003}
}
test.ElementsMatchSkipID(t, software, expected) test.ElementsMatchSkipID(t, software, expected)
}) })
@ -626,3 +636,145 @@ func testSoftwareCalculateHostsPerSoftware(t *testing.T, ds *Datastore) {
} }
cmpNameVersionCount(want, allSw) cmpNameVersionCount(want, allSw)
} }
func insertVulnSoftwareForTest(t *testing.T, ds *Datastore) {
host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now())
host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now())
software1 := []fleet.Software{
{
Name: "foo.rpm", Version: "0.0.1", Source: "rpm_packages", GenerateCPE: "cpe_foo_rpm",
},
{
Name: "foo.chrome", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "cpe_foo_chrome",
},
}
software2 := []fleet.Software{
{
Name: "foo.chrome", Version: "v0.0.2", Source: "chrome_extensions", GenerateCPE: "cpe_foo_chrome2",
},
{
Name: "foo.chrome", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "cpe_foo_chrome_3",
Vulnerabilities: fleet.VulnerabilitiesSlice{
{CVE: "cve-123-456-789", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-123-456-789"},
},
},
{
Name: "bar.rpm", Version: "0.0.3", Source: "rpm_packages", GenerateCPE: "cpe_bar_rpm",
Vulnerabilities: fleet.VulnerabilitiesSlice{
{CVE: "cve-321-432-543", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-321-432-543"},
{CVE: "cve-333-444-555", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-333-444-555"},
},
},
}
require.NoError(t, ds.UpdateHostSoftware(context.Background(), host1.ID, software1))
require.NoError(t, ds.UpdateHostSoftware(context.Background(), host2.ID, software2))
require.NoError(t, ds.LoadHostSoftware(context.Background(), host1))
require.NoError(t, ds.LoadHostSoftware(context.Background(), host2))
sort.Slice(host1.Software, func(i, j int) bool {
return host1.Software[i].Name+host1.Software[i].Version < host1.Software[j].Name+host1.Software[j].Version
})
sort.Slice(host2.Software, func(i, j int) bool {
return host2.Software[i].Name+host2.Software[i].Version < host2.Software[j].Name+host2.Software[j].Version
})
require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[0], "cpe_foo_chrome"))
require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[1], "cpe_foo_rpm"))
require.NoError(t, ds.AddCPEForSoftware(context.Background(), host2.Software[0], "cpe_bar_rpm"))
require.NoError(t, ds.AddCPEForSoftware(context.Background(), host2.Software[1], "cpe_foo_chrome_3"))
require.NoError(t, ds.AddCPEForSoftware(context.Background(), host2.Software[2], "cpe_foo_chrome_2"))
_, err := ds.InsertCVEForCPE(context.Background(), "cve-123-456-789", []string{"cpe_foo_chrome_3"})
require.NoError(t, err)
_, err = ds.InsertCVEForCPE(context.Background(), "cve-321-432-543", []string{"cpe_bar_rpm"})
require.NoError(t, err)
_, err = ds.InsertCVEForCPE(context.Background(), "cve-333-444-555", []string{"cpe_bar_rpm"})
require.NoError(t, err)
}
func testListVulnerableSoftwareBySource(t *testing.T, ds *Datastore) {
ctx := context.Background()
insertVulnSoftwareForTest(t, ds)
vulnerable, err := ds.ListVulnerableSoftwareBySource(ctx, "apps")
require.NoError(t, err)
require.Empty(t, vulnerable)
vulnerable, err = ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages")
require.NoError(t, err)
require.Len(t, vulnerable, 1)
require.Equal(t, vulnerable[0].Name, "bar.rpm")
require.Len(t, vulnerable[0].Vulnerabilities, 2)
sort.Slice(vulnerable[0].Vulnerabilities, func(i, j int) bool {
return vulnerable[0].Vulnerabilities[i].CVE < vulnerable[0].Vulnerabilities[j].CVE
})
require.Equal(t, "cve-321-432-543", vulnerable[0].Vulnerabilities[0].CVE)
require.Equal(t, "cve-333-444-555", vulnerable[0].Vulnerabilities[1].CVE)
}
func testDeleteVulnerabilitiesByCPECVE(t *testing.T, ds *Datastore) {
ctx := context.Background()
err := ds.DeleteVulnerabilitiesByCPECVE(ctx, nil)
require.NoError(t, err)
insertVulnSoftwareForTest(t, ds)
err = ds.DeleteVulnerabilitiesByCPECVE(ctx, []fleet.SoftwareVulnerability{
{
CPEID: 999, // unknown CPE
CVE: "cve-333-444-555",
},
})
require.NoError(t, err)
software, err := ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages")
require.NoError(t, err)
require.Len(t, software, 1)
barRPM := software[0]
require.Len(t, barRPM.Vulnerabilities, 2)
err = ds.DeleteVulnerabilitiesByCPECVE(ctx, []fleet.SoftwareVulnerability{
{
CPEID: barRPM.CPEID,
CVE: "unknown-cve",
},
})
require.NoError(t, err)
err = ds.DeleteVulnerabilitiesByCPECVE(ctx, []fleet.SoftwareVulnerability{
{
CPEID: barRPM.CPEID,
CVE: "cve-333-444-555",
},
})
require.NoError(t, err)
software, err = ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages")
require.NoError(t, err)
require.Len(t, software, 1)
barRPM = software[0]
require.Len(t, barRPM.Vulnerabilities, 1)
err = ds.DeleteVulnerabilitiesByCPECVE(ctx, []fleet.SoftwareVulnerability{
{
CPEID: barRPM.CPEID,
CVE: "cve-321-432-543",
},
})
require.NoError(t, err)
software, err = ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages")
require.NoError(t, err)
require.Len(t, software, 0)
software, err = ds.ListVulnerableSoftwareBySource(ctx, "chrome_extensions")
require.NoError(t, err)
require.Len(t, software, 1)
}

31
server/fleet/datastore.go
View File

@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
"time" "time"
) )
@ -330,6 +331,12 @@ type Datastore interface {
AllCPEs(ctx context.Context) ([]string, error) AllCPEs(ctx context.Context) ([]string, error)
InsertCVEForCPE(ctx context.Context, cve string, cpes []string) (int64, error) InsertCVEForCPE(ctx context.Context, cve string, cpes []string) (int64, error)
SoftwareByID(ctx context.Context, id uint) (*Software, error) SoftwareByID(ctx context.Context, id uint) (*Software, error)
// CalculateHostsPerSoftware calculates the number of hosts having each
// software installed and stores that information in the software_host_counts
// table.
//
// After aggregation, it cleans up unused software (e.g. software installed
// on removed hosts, software uninstalled on hosts, etc.)
CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error
HostsByCPEs(ctx context.Context, cpes []string) ([]*CPEHost, error) HostsByCPEs(ctx context.Context, cpes []string) ([]*CPEHost, error)
@ -374,6 +381,10 @@ type Datastore interface {
ListSoftware(ctx context.Context, opt SoftwareListOptions) ([]Software, error) ListSoftware(ctx context.Context, opt SoftwareListOptions) ([]Software, error)
CountSoftware(ctx context.Context, opt SoftwareListOptions) (int, error) CountSoftware(ctx context.Context, opt SoftwareListOptions) (int, error)
// ListVulnerableSoftwareBySource lists all the vulnerable software that matches the given source.
ListVulnerableSoftwareBySource(ctx context.Context, source string) ([]SoftwareWithCPE, error)
// DeleteVulnerabilities deletes the given list of vulnerabilities identified by CPE+CVE.
DeleteVulnerabilitiesByCPECVE(ctx context.Context, vulnerabilities []SoftwareVulnerability) error
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
// Team Policies // Team Policies
@ -549,6 +560,26 @@ const (
UnknownMigrations UnknownMigrations
) )
// SoftwareVulnerability identifies a vulnerability on a specific software (CPE).
type SoftwareVulnerability struct {
// CPEID is the ID of the software CPE in the system.
CPEID uint
CVE string
}
// String implements fmt.Stringer.
func (sv SoftwareVulnerability) String() string {
return fmt.Sprintf("{%d,%s}", sv.CPEID, sv.CVE)
}
// SoftwareWithCPE holds a software piece alongside its CPE ID.
type SoftwareWithCPE struct {
// Software holds the software data.
Software
// CPEID is the ID of the software CPE in the system.
CPEID uint
}
// NotFoundError is returned when the datastore resource cannot be found. // NotFoundError is returned when the datastore resource cannot be found.
type NotFoundError interface { type NotFoundError interface {
error error

8
server/fleet/software.go
View File

@ -19,6 +19,14 @@ type Software struct {
// Source is the source of the data (osquery table name). // Source is the source of the data (osquery table name).
Source string `json:"source" db:"source"` Source string `json:"source" db:"source"`
// Release is the version of the OS this software was released on
// (e.g. "30.el7" for a CentOS package).
Release string `json:"release,omitempty" db:"release"`
// Vendor is the supplier of the software (e.g. "CentOS").
Vendor string `json:"vendor,omitempty" db:"vendor"`
// Arch is the architecture of the software (e.g. "x86_64").
Arch string `json:"arch,omitempty" db:"arch"`
// GenerateCPE is the CPE23 string that corresponds to the current software // GenerateCPE is the CPE23 string that corresponds to the current software
GenerateCPE string `json:"generated_cpe" db:"generated_cpe"` GenerateCPE string `json:"generated_cpe" db:"generated_cpe"`
// Vulnerabilities lists all the found CVEs for the CPE // Vulnerabilities lists all the found CVEs for the CPE

20
server/mock/datastore_mock.go
View File

@ -312,6 +312,10 @@ type ListSoftwareFunc func(ctx context.Context, opt fleet.SoftwareListOptions) (
type CountSoftwareFunc func(ctx context.Context, opt fleet.SoftwareListOptions) (int, error) type CountSoftwareFunc func(ctx context.Context, opt fleet.SoftwareListOptions) (int, error)
type ListVulnerableSoftwareBySourceFunc func(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error)
type DeleteVulnerabilitiesByCPECVEFunc func(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error
type NewTeamPolicyFunc func(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) type NewTeamPolicyFunc func(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error)
type ListTeamPoliciesFunc func(ctx context.Context, teamID uint) ([]*fleet.Policy, error) type ListTeamPoliciesFunc func(ctx context.Context, teamID uint) ([]*fleet.Policy, error)
@ -825,6 +829,12 @@ type DataStore struct {
CountSoftwareFunc CountSoftwareFunc CountSoftwareFunc CountSoftwareFunc
CountSoftwareFuncInvoked bool CountSoftwareFuncInvoked bool
ListVulnerableSoftwareBySourceFunc ListVulnerableSoftwareBySourceFunc
ListVulnerableSoftwareBySourceFuncInvoked bool
DeleteVulnerabilitiesByCPECVEFunc DeleteVulnerabilitiesByCPECVEFunc
DeleteVulnerabilitiesByCPECVEFuncInvoked bool
NewTeamPolicyFunc NewTeamPolicyFunc NewTeamPolicyFunc NewTeamPolicyFunc
NewTeamPolicyFuncInvoked bool NewTeamPolicyFuncInvoked bool
@ -1669,6 +1679,16 @@ func (s *DataStore) CountSoftware(ctx context.Context, opt fleet.SoftwareListOpt
return s.CountSoftwareFunc(ctx, opt) return s.CountSoftwareFunc(ctx, opt)
} }
func (s *DataStore) ListVulnerableSoftwareBySource(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) {
s.ListVulnerableSoftwareBySourceFuncInvoked = true
return s.ListVulnerableSoftwareBySourceFunc(ctx, source)
}
func (s *DataStore) DeleteVulnerabilitiesByCPECVE(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error {
s.DeleteVulnerabilitiesByCPECVEFuncInvoked = true
return s.DeleteVulnerabilitiesByCPECVEFunc(ctx, vulnerabilities)
}
func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) { func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) {
s.NewTeamPolicyFuncInvoked = true s.NewTeamPolicyFuncInvoked = true
return s.NewTeamPolicyFunc(ctx, teamID, authorID, args) return s.NewTeamPolicyFunc(ctx, teamID, authorID, args)

2
server/ptr/ptr.go
View File

@ -11,6 +11,8 @@ func String(x string) *string {
return &x return &x
} }
// StringValueOrZero returns the string value.
// Returns empty string if x is nil.
func StringValueOrZero(x *string) string { func StringValueOrZero(x *string) string {
if x == nil { if x == nil {
return "" return ""

13
server/service/integration_core_test.go
View File

@ -84,6 +84,10 @@ func (s *integrationTestSuite) TearDownTest() {
_, err = s.ds.DeleteGlobalPolicies(ctx, globalPolicyIDs) _, err = s.ds.DeleteGlobalPolicies(ctx, globalPolicyIDs)
require.NoError(t, err) require.NoError(t, err)
} }
// CalculateHostsPerSoftware performs a cleanup.
err = s.ds.CalculateHostsPerSoftware(ctx, time.Now())
require.NoError(t, err)
} }
func TestIntegrations(t *testing.T) { func TestIntegrations(t *testing.T) {
@ -1742,13 +1746,15 @@ func (s *integrationTestSuite) TestScheduledQueries() {
// batch-delete by id, 3 ids, only one exists // batch-delete by id, 3 ids, only one exists
var delBatchResp deleteQueriesResponse var delBatchResp deleteQueriesResponse
s.DoJSON("POST", "/api/v1/fleet/queries/delete", map[string]interface{}{ s.DoJSON("POST", "/api/v1/fleet/queries/delete", map[string]interface{}{
"ids": []uint{query.ID, query2.ID, query3.ID}}, http.StatusOK, &delBatchResp) "ids": []uint{query.ID, query2.ID, query3.ID},
}, http.StatusOK, &delBatchResp)
assert.Equal(t, uint(1), delBatchResp.Deleted) assert.Equal(t, uint(1), delBatchResp.Deleted)
// batch-delete by id, none exist // batch-delete by id, none exist
delBatchResp.Deleted = 0 delBatchResp.Deleted = 0
s.DoJSON("POST", "/api/v1/fleet/queries/delete", map[string]interface{}{ s.DoJSON("POST", "/api/v1/fleet/queries/delete", map[string]interface{}{
"ids": []uint{query.ID, query2.ID, query3.ID}}, http.StatusNotFound, &delBatchResp) "ids": []uint{query.ID, query2.ID, query3.ID},
}, http.StatusNotFound, &delBatchResp)
assert.Equal(t, uint(0), delBatchResp.Deleted) assert.Equal(t, uint(0), delBatchResp.Deleted)
} }
@ -2615,7 +2621,8 @@ func (s *integrationTestSuite) TestQuerySpecs() {
// delete all queries created // delete all queries created
var delBatchResp deleteQueriesResponse var delBatchResp deleteQueriesResponse
s.DoJSON("POST", "/api/v1/fleet/queries/delete", map[string]interface{}{ s.DoJSON("POST", "/api/v1/fleet/queries/delete", map[string]interface{}{
"ids": []uint{q1ID, q2ID, q3ID}}, http.StatusOK, &delBatchResp) "ids": []uint{q1ID, q2ID, q3ID},
}, http.StatusOK, &delBatchResp)
assert.Equal(t, uint(3), delBatchResp.Deleted) assert.Equal(t, uint(3), delBatchResp.Deleted)
} }

45
server/service/osquery_utils/queries.go
View File

@ -395,56 +395,80 @@ SELECT
name AS name, name AS name,
version AS version, version AS version,
'Package (deb)' AS type, 'Package (deb)' AS type,
'deb_packages' AS source 'deb_packages' AS source,
'' AS release,
'' AS vendor,
'' AS arch
FROM deb_packages FROM deb_packages
UNION UNION
SELECT SELECT
package AS name, package AS name,
version AS version, version AS version,
'Package (Portage)' AS type, 'Package (Portage)' AS type,
'portage_packages' AS source 'portage_packages' AS source,
'' AS release,
'' AS vendor,
'' AS arch
FROM portage_packages FROM portage_packages
UNION UNION
SELECT SELECT
name AS name, name AS name,
version AS version, version AS version,
'Package (RPM)' AS type, 'Package (RPM)' AS type,
'rpm_packages' AS source 'rpm_packages' AS source,
release AS release,
vendor AS vendor,
arch AS arch
FROM rpm_packages FROM rpm_packages
UNION UNION
SELECT SELECT
name AS name, name AS name,
version AS version, version AS version,
'Package (NPM)' AS type, 'Package (NPM)' AS type,
'npm_packages' AS source 'npm_packages' AS source,
'' AS release,
'' AS vendor,
'' AS arch
FROM npm_packages FROM npm_packages
UNION UNION
SELECT SELECT
name AS name, name AS name,
version AS version, version AS version,
'Browser plugin (Chrome)' AS type, 'Browser plugin (Chrome)' AS type,
'chrome_extensions' AS source 'chrome_extensions' AS source,
'' AS release,
'' AS vendor,
'' AS arch
FROM cached_users CROSS JOIN chrome_extensions USING (uid) FROM cached_users CROSS JOIN chrome_extensions USING (uid)
UNION UNION
SELECT SELECT
name AS name, name AS name,
version AS version, version AS version,
'Browser plugin (Firefox)' AS type, 'Browser plugin (Firefox)' AS type,
'firefox_addons' AS source 'firefox_addons' AS source,
'' AS release,
'' AS vendor,
'' AS arch
FROM cached_users CROSS JOIN firefox_addons USING (uid) FROM cached_users CROSS JOIN firefox_addons USING (uid)
UNION UNION
SELECT SELECT
name AS name, name AS name,
version AS version, version AS version,
'Package (Atom)' AS type, 'Package (Atom)' AS type,
'atom_packages' AS source 'atom_packages' AS source,
'' AS release,
'' AS vendor,
'' AS arch
FROM cached_users CROSS JOIN atom_packages USING (uid) FROM cached_users CROSS JOIN atom_packages USING (uid)
UNION UNION
SELECT SELECT
name AS name, name AS name,
version AS version, version AS version,
'Package (Python)' AS type, 'Package (Python)' AS type,
'python_packages' AS source 'python_packages' AS source,
'' AS release,
'' AS vendor,
'' AS arch
FROM python_packages; FROM python_packages;
`, `,
Platforms: fleet.HostLinuxOSs, Platforms: fleet.HostLinuxOSs,
@ -649,11 +673,16 @@ func directIngestSoftware(ctx context.Context, logger log.Logger, host *fleet.Ho
) )
continue continue
} }
s := fleet.Software{ s := fleet.Software{
Name: name, Name: name,
Version: version, Version: version,
Source: source, Source: source,
BundleIdentifier: bundleIdentifier, BundleIdentifier: bundleIdentifier,
Release: row["release"],
Vendor: row["vendor"],
Arch: row["arch"],
} }
software = append(software, s) software = append(software, s)
} }

7
server/service/service_osquery.go
View File

@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"regexp"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
@ -17,7 +18,6 @@ import (
"github.com/fleetdm/fleet/v4/server/contexts/logging" "github.com/fleetdm/fleet/v4/server/contexts/logging"
"github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/ptr"
"github.com/fleetdm/fleet/v4/server/pubsub" "github.com/fleetdm/fleet/v4/server/pubsub"
"github.com/fleetdm/fleet/v4/server/service/osquery_utils" "github.com/fleetdm/fleet/v4/server/service/osquery_utils"
"github.com/go-kit/kit/log" "github.com/go-kit/kit/log"
@ -761,6 +761,8 @@ func (svc *Service) directIngestDetailQuery(ctx context.Context, host *fleet.Hos
return false, nil return false, nil
} }
var noSuchTableRegexp = regexp.MustCompile(`^no such table: \S+$`)
func (svc *Service) SubmitDistributedQueryResults( func (svc *Service) SubmitDistributedQueryResults(
ctx context.Context, ctx context.Context,
results fleet.OsqueryDistributedQueryResults, results fleet.OsqueryDistributedQueryResults,
@ -787,6 +789,9 @@ func (svc *Service) SubmitDistributedQueryResults(
// osquery docs say any nonzero (string) value for status indicates a query error // osquery docs say any nonzero (string) value for status indicates a query error
status, ok := statuses[query] status, ok := statuses[query]
failed := ok && status != fleet.StatusOK failed := ok && status != fleet.StatusOK
if failed && messages[query] != "" && !noSuchTableRegexp.MatchString(messages[query]) {
level.Debug(svc.logger).Log("query", query, "message", messages[query])
}
var err error var err error
switch { switch {
case strings.HasPrefix(query, hostDetailQueryPrefix): case strings.HasPrefix(query, hostDetailQueryPrefix):

92
server/vulnerabilities/centos.go Normal file
View File

@ -0,0 +1,92 @@
package vulnerabilities
import (
"context"
"database/sql"
"fmt"
"github.com/fleetdm/fleet/v4/server/config"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/vuln_centos"
kitlog "github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
)
// centosPostProcessing performs processing over the list of vulnerable rpm packages
// and removes the vulnerabilities where the CVEs are known to be fixed.
func centosPostProcessing(
ctx context.Context,
ds fleet.Datastore,
db *sql.DB,
logger kitlog.Logger,
config config.FleetConfig,
) error {
centOSPkgs, err := vuln_centos.LoadCentOSFixedCVEs(ctx, db, logger)
if err != nil {
return fmt.Errorf("failed to fetch CentOS packages: %w", err)
}
level.Info(logger).Log("centosPackages", len(centOSPkgs))
if len(centOSPkgs) == 0 {
return nil
}
rpmVulnerable, err := ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages")
if err != nil {
return fmt.Errorf("failed to list vulnerable software: %w", err)
}
level.Info(logger).Log("vulnerable rpm_packages", len(rpmVulnerable))
if len(rpmVulnerable) == 0 {
return nil
}
var fixedCVEs []fleet.SoftwareVulnerability
var softwareCount int
for _, software := range rpmVulnerable {
if software.Vendor != "CentOS" {
continue
}
pkgFixedCVEs, ok := centOSPkgs[vuln_centos.CentOSPkg{
Name: software.Name,
Version: software.Version,
Release: software.Release,
Arch: software.Arch,
}]
if !ok {
continue
}
var cves []string
for _, vulnerability := range software.Vulnerabilities {
if _, ok := pkgFixedCVEs[vulnerability.CVE]; ok {
cves = append(cves, vulnerability.CVE)
fixedCVEs = append(fixedCVEs, fleet.SoftwareVulnerability{
CPEID: software.CPEID,
CVE: vulnerability.CVE,
})
}
}
if len(cves) > 0 {
softwareCount++
level.Debug(logger).Log(
"msg", "fixedCVEs",
"software", fmt.Sprintf(
"%s-%s-%s.%s",
software.Name, software.Version, software.Release, software.Arch,
),
"softwareCPE", software.CPEID,
"cves", fmt.Sprintf("%v", cves),
)
}
}
level.Info(logger).Log(
"msg", "CentOS fixed CVEs",
"fixedCVEsCount", len(fixedCVEs),
"distinctSoftwareCount", softwareCount,
)
if err := ds.DeleteVulnerabilitiesByCPECVE(ctx, fixedCVEs); err != nil {
return fmt.Errorf("failed to delete fixed vulnerabilities: %w", err)
}
return nil
}

152
server/vulnerabilities/centos_test.go Normal file
View File

@ -0,0 +1,152 @@
package vulnerabilities
import (
"context"
"database/sql"
"testing"
"github.com/fleetdm/fleet/v4/server/config"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/mock"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/vuln_centos"
"github.com/go-kit/kit/log"
_ "github.com/mattn/go-sqlite3"
"github.com/stretchr/testify/require"
)
func TestCentOSPostProcessing(t *testing.T) {
ctx := context.Background()
ds := new(mock.Store)
db, err := sql.Open("sqlite3", ":memory:")
require.NoError(t, err)
pkgs := make(vuln_centos.CentOSPkgSet)
authConfigPkg := vuln_centos.CentOSPkg{
Name: "authconfig",
Version: "6.2.8",
Release: "30.el7",
Arch: "x86_64",
}
pkgs.Add(authConfigPkg, "CVE-2017-7488")
sqlitePkg := vuln_centos.CentOSPkg{
Name: "sqlite",
Version: "3.7.17",
Release: "8.el7_7",
Arch: "x86_64",
}
pkgs.Add(sqlitePkg, "CVE-2015-3415", "CVE-2015-3416", "CVE-2015-3414")
err = vuln_centos.GenCentOSSqlite(db, pkgs)
require.NoError(t, err)
vulnSoftware := []fleet.SoftwareWithCPE{
{
Software: fleet.Software{
Name: "authconfig",
Version: "6.2.8",
Release: "30.el7",
Arch: "x86_64",
Vendor: "CentOS",
Vulnerabilities: fleet.VulnerabilitiesSlice{
{
CVE: "CVE-2017-7488",
},
},
},
CPEID: 1,
},
{
Software: fleet.Software{
Name: "sqlite",
Version: "3.7.17",
Release: "8.el7_7",
Arch: "x86_64",
Vendor: "CentOS",
Vulnerabilities: fleet.VulnerabilitiesSlice{
{
CVE: "CVE-2015-3415",
},
{
CVE: "CVE-2015-3416",
},
{
CVE: "CVE-2022-9999",
},
},
},
CPEID: 2,
},
{
Software: fleet.Software{
Name: "ghostscript",
Version: "9.25",
Release: "5.el7",
Arch: "x86_64",
Vendor: "CentOS",
Vulnerabilities: fleet.VulnerabilitiesSlice{
{
CVE: "CVE-2019-3835",
},
},
},
CPEID: 3,
},
{
Software: fleet.Software{
Name: "gnutls",
Version: "3.3.29",
Release: "9.el7",
Arch: "x86_64",
Vendor: "",
Vulnerabilities: fleet.VulnerabilitiesSlice{
{
CVE: "CVE-8888-9999",
},
},
},
CPEID: 4,
},
}
ds.ListVulnerableSoftwareBySourceFunc = func(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) {
return vulnSoftware, nil
}
ds.DeleteVulnerabilitiesByCPECVEFunc = func(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error {
require.Equal(t, []fleet.SoftwareVulnerability{
{
CPEID: 1,
CVE: "CVE-2017-7488",
},
{
CPEID: 2,
CVE: "CVE-2015-3415",
},
{
CPEID: 2,
CVE: "CVE-2015-3416",
},
}, vulnerabilities)
return nil
}
err = centosPostProcessing(ctx, ds, db, log.NewNopLogger(), config.FleetConfig{})
require.NoError(t, err)
require.True(t, ds.ListVulnerableSoftwareBySourceFuncInvoked)
require.True(t, ds.DeleteVulnerabilitiesByCPECVEFuncInvoked)
}
func TestCentOSPostProcessingNoPkgs(t *testing.T) {
ctx := context.Background()
ds := new(mock.Store)
ds.ListVulnerableSoftwareBySourceFunc = func(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) {
t.Error("this method shouldn't be called if there are no pkgs in the CentOS table")
return nil, nil
}
db, err := sql.Open("sqlite3", ":memory:")
require.NoError(t, err)
err = centosPostProcessing(ctx, ds, db, log.NewNopLogger(), config.FleetConfig{})
require.Error(t, err)
}

63
server/vulnerabilities/cpe.go
View File

@ -1,18 +1,18 @@
package vulnerabilities package vulnerabilities
import ( import (
"compress/gzip"
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io"
"net/http" "net/http"
"net/url"
"os" "os"
"path" "path"
"regexp" "regexp"
"strings" "strings"
"time" "time"
"github.com/fleetdm/fleet/v4/pkg/download"
"github.com/fleetdm/fleet/v4/pkg/fleethttp" "github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/config"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
@ -69,22 +69,39 @@ func GetLatestNVDRelease(client *http.Client) (*NVDRelease, error) {
}, nil }, nil
} }
type syncOpts struct {
url string
}
type CPESyncOption func(*syncOpts)
func WithCPEURL(url string) CPESyncOption {
return func(o *syncOpts) {
o.url = url
}
}
// SyncCPEDatabase (by default) downloads the CPE database from the
// latest release of github.com/fleetdm/nvd to the given dbPath.
// An alternative URL can be set via the WithCPEURL option.
//
// It won't sync the database at dbPath has an mtime that happened after the
// available database release date.
func SyncCPEDatabase( func SyncCPEDatabase(
client *http.Client, client *http.Client,
dbPath string, dbPath string,
config config.FleetConfig, opts ...CPESyncOption,
) error { ) error {
if config.Vulnerabilities.DisableDataSync { var o syncOpts
return nil for _, fn := range opts {
fn(&o)
} }
url := config.Vulnerabilities.CPEDatabaseURL if o.url == "" {
if url == "" {
nvdRelease, err := GetLatestNVDRelease(client) nvdRelease, err := GetLatestNVDRelease(client)
if err != nil { if err != nil {
return err return err
} }
stat, err := os.Stat(dbPath) stat, err := os.Stat(dbPath)
if err != nil { if err != nil {
if !errors.Is(err, os.ErrNotExist) { if !errors.Is(err, os.ErrNotExist) {
@ -93,34 +110,14 @@ func SyncCPEDatabase(
} else if !nvdRelease.CreatedAt.After(stat.ModTime()) { } else if !nvdRelease.CreatedAt.After(stat.ModTime()) {
return nil return nil
} }
url = nvdRelease.CPEURL o.url = nvdRelease.CPEURL
} }
req, err := http.NewRequest(http.MethodGet, url, nil) u, err := url.Parse(o.url)
if err != nil { if err != nil {
return err return err
} }
if err := download.Decompressed(client, *u, dbPath); err != nil {
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
gr, err := gzip.NewReader(resp.Body)
if err != nil {
return err
}
defer gr.Close()
dbFile, err := os.Create(dbPath)
if err != nil {
return err
}
defer dbFile.Close()
_, err = io.Copy(dbFile, gr)
if err != nil {
return err return err
} }
@ -227,10 +224,12 @@ func TranslateSoftwareToCPE(
) error { ) error {
dbPath := path.Join(vulnPath, "cpe.sqlite") dbPath := path.Join(vulnPath, "cpe.sqlite")
if !config.Vulnerabilities.DisableDataSync {
client := fleethttp.NewClient() client := fleethttp.NewClient()
if err := SyncCPEDatabase(client, dbPath, config); err != nil { if err := SyncCPEDatabase(client, dbPath, WithCPEURL(config.Vulnerabilities.CPEDatabaseURL)); err != nil {
return ctxerr.Wrap(ctx, err, "sync cpe db") return ctxerr.Wrap(ctx, err, "sync cpe db")
} }
}
iterator, err := ds.AllSoftwareWithoutCPEIterator(ctx) iterator, err := ds.AllSoftwareWithoutCPEIterator(ctx)
if err != nil { if err != nil {

26
server/vulnerabilities/cpe_test.go
View File

@ -75,7 +75,7 @@ func TestSyncCPEDatabase(t *testing.T) {
} }
// first time, db doesn't exist, so it downloads // first time, db doesn't exist, so it downloads
err = SyncCPEDatabase(client, dbPath, config.FleetConfig{}) err = SyncCPEDatabase(client, dbPath)
require.NoError(t, err) require.NoError(t, err)
db, err := sqliteDB(dbPath) db, err := sqliteDB(dbPath)
@ -107,7 +107,7 @@ func TestSyncCPEDatabase(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
// then it will download // then it will download
err = SyncCPEDatabase(client, dbPath, config.FleetConfig{}) err = SyncCPEDatabase(client, dbPath)
require.NoError(t, err) require.NoError(t, err)
// let's register the mtime for the db // let's register the mtime for the db
@ -128,9 +128,8 @@ func TestSyncCPEDatabase(t *testing.T) {
time.Sleep(2 * time.Second) time.Sleep(2 * time.Second)
// let's check it doesn't download because it's new enough // let's check it doesn't download because it's new enough
err = SyncCPEDatabase(client, dbPath, config.FleetConfig{}) err = SyncCPEDatabase(client, dbPath)
require.NoError(t, err) require.NoError(t, err)
stat, err = os.Stat(dbPath) stat, err = os.Stat(dbPath)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, mtime, stat.ModTime()) require.Equal(t, mtime, stat.ModTime())
@ -226,27 +225,10 @@ func TestSyncsCPEFromURL(t *testing.T) {
dbPath := path.Join(tempDir, "cpe.sqlite") dbPath := path.Join(tempDir, "cpe.sqlite")
err := SyncCPEDatabase( err := SyncCPEDatabase(
client, dbPath, config.FleetConfig{Vulnerabilities: config.VulnerabilitiesConfig{CPEDatabaseURL: ts.URL}}) client, dbPath, WithCPEURL(ts.URL+"/hello-world.gz"))
require.NoError(t, err) require.NoError(t, err)
stored, err := ioutil.ReadFile(dbPath) stored, err := ioutil.ReadFile(dbPath)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "Hello world!", string(stored)) assert.Equal(t, "Hello world!", string(stored))
} }
func TestSyncsCPESkipsIfDisableSync(t *testing.T) {
client := fleethttp.NewClient()
tempDir := t.TempDir()
dbPath := path.Join(tempDir, "cpe.sqlite")
fleetConfig := config.FleetConfig{
Vulnerabilities: config.VulnerabilitiesConfig{
DisableDataSync: true,
},
}
err := SyncCPEDatabase(client, dbPath, fleetConfig)
require.NoError(t, err)
_, err = os.Stat(dbPath)
require.ErrorIs(t, err, os.ErrNotExist)
}

26
server/vulnerabilities/cve.go
View File

@ -2,9 +2,11 @@ package vulnerabilities
import ( import (
"context" "context"
"database/sql"
"fmt" "fmt"
"net/url" "net/url"
"os" "os"
"path"
"path/filepath" "path/filepath"
"regexp" "regexp"
"runtime" "runtime"
@ -140,7 +142,7 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger,
} }
cache := cvefeed.NewCache(dict).SetRequireVersion(true).SetMaxSize(-1) cache := cvefeed.NewCache(dict).SetRequireVersion(true).SetMaxSize(-1)
// This index consumes too much RAM // This index consumes too much RAM
//cache.Idx = cvefeed.NewIndex(dict) // cache.Idx = cvefeed.NewIndex(dict)
cpeCh := make(chan *wfn.Attributes) cpeCh := make(chan *wfn.Attributes)
collectVulns := recentVulns != nil collectVulns := recentVulns != nil
@ -243,3 +245,25 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger,
wg.Wait() wg.Wait()
return nil return nil
} }
// PostProcess performs additional processing over the results of
// the main vulnerability processing run (TranslateSoftwareToCPE+TranslateCPEToCVE).
func PostProcess(
ctx context.Context,
ds fleet.Datastore,
vulnPath string,
logger kitlog.Logger,
config config.FleetConfig,
) error {
dbPath := path.Join(vulnPath, "cpe.sqlite")
db, err := sql.Open("sqlite3", dbPath)
if err != nil {
return fmt.Errorf("failed to open cpe database: %w", err)
}
defer db.Close()
if err := centosPostProcessing(ctx, ds, db, logger, config); err != nil {
return err
}
return nil
}

451
server/vulnerabilities/vuln_centos/centos.go Normal file
View File

@ -0,0 +1,451 @@
// Package vuln_centos contains a ParseCentOSRepository method to parse the CentOS repository
// to look out for CentOS releases that patch CVEs. It parses the changelogs from the metadata.
//
// It also contains a LoadCentOSFixedCVEs to load the results of the parsing.
//
// Both the parsing and loading of results use sqlite3 as backend storage.
package vuln_centos
import (
"context"
"database/sql"
"encoding/xml"
"errors"
"fmt"
"io/fs"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"sort"
"strings"
"github.com/fleetdm/fleet/v4/pkg/download"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
kitlog "github.com/go-kit/kit/log"
"github.com/gocolly/colly"
_ "github.com/mattn/go-sqlite3"
)
// CentOSPkg holds data to identify a CentOS package.
type CentOSPkg struct {
Name string
Version string
Release string
Arch string
}
// String implements fmt.Stringer.
func (p CentOSPkg) String() string {
return p.Name + "-" + p.Version + "-" + p.Release + "." + p.Arch
}
// FixedCVESet is a set of fixed CVEs.
type FixedCVESet map[string]struct{}
// CentOSPkgSet is a set of CentOS packages and their fixed CVEs.
type CentOSPkgSet map[CentOSPkg]FixedCVESet
// Add adds the given package and CVE/s to the set.
func (p CentOSPkgSet) Add(pkg CentOSPkg, fixedCVEs ...string) {
s := p[pkg]
if s == nil {
s = make(FixedCVESet)
}
for _, fixedCVE := range fixedCVEs {
s[fixedCVE] = struct{}{}
}
p[pkg] = s
}
const centOSPkgsCVEsTable = "centos_pkgs_fixed_cves"
// LoadCentOSFixedCVEs loads the CentOS packages with known fixed CVEs from the given sqlite3 db.
func LoadCentOSFixedCVEs(ctx context.Context, db *sql.DB, logger kitlog.Logger) (CentOSPkgSet, error) {
rows, err := db.QueryContext(ctx, fmt.Sprintf(`SELECT name, version, release, arch, cves FROM %s`, centOSPkgsCVEsTable))
if err != nil {
return nil, fmt.Errorf("failed to fetch packages: %w", err)
}
defer rows.Close()
pkgs := make(CentOSPkgSet)
for rows.Next() {
var pkg CentOSPkg
var cves string
if err := rows.Scan(&pkg.Name, &pkg.Version, &pkg.Release, &pkg.Arch, &cves); err != nil {
return nil, err
}
for _, cve := range strings.Split(cves, ",") {
pkgs.Add(pkg, "CVE-"+cve)
}
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("failed to traverse packages: %w", err)
}
return pkgs, nil
}
type centOSOpts struct {
noCrawl bool
verbose bool
localDir string
root string
}
type CentOSOption func(*centOSOpts)
func WithLocalDir(dir string) CentOSOption {
return func(o *centOSOpts) {
o.localDir = dir
}
}
func NoCrawl() CentOSOption {
return func(o *centOSOpts) {
o.noCrawl = true
}
}
func WithVerbose(v bool) CentOSOption {
return func(o *centOSOpts) {
o.verbose = v
}
}
func WithRoot(root string) CentOSOption {
return func(o *centOSOpts) {
o.root = root
}
}
const (
repositoryDomain = "mirror.centos.org"
repositoryURL = "http://" + repositoryDomain
defaultRoot = "/centos/"
)
var (
// Only parse the repository metadata for CentOS 6, 7 and 8.
//
// CentOS 6 maintenance updates ended in 2020-11-30, but we will still
// fetch metadata for CentOS 6 because it's considered as of 2022-02-02
// a "recent" release.
//
// See https://en.wikipedia.org/wiki/CentOS#CentOS_releases.
recentCentOSPathRegex = regexp.MustCompile(`/centos/[678]\S*`)
// nonReleasePathRegex is used to skip non-package centos directories/files.
nonReleasePathRegex = regexp.MustCompile(`/centos/[^0-9]`)
)
// ParseCentOSRepository performs the following operations:
// - Crawls the CentOS repository website. To find all the sqlite3 files with
// the packages metadata.
// - Processes all the found sqlite3 files to find all fixed CVEs in each package version.
// It parses the changelogs for each package release and looks for the "CVE-" string.
//
// It writes progress messages to stdout.
func ParseCentOSRepository(opts ...CentOSOption) (CentOSPkgSet, error) {
var opts_ centOSOpts
for _, fn := range opts {
fn(&opts_)
}
if opts_.localDir == "" && opts_.noCrawl {
return nil, errors.New("invalid options: if no crawl is set, local dir must be set")
}
if opts_.localDir == "" {
localDir, err := os.MkdirTemp("", "centos*")
if err != nil {
return nil, err
}
opts_.localDir = localDir
}
fmt.Printf("Using local directory: %s\n", opts_.localDir)
if !opts_.noCrawl {
if err := crawl(opts_.root, opts_.localDir, opts_.verbose); err != nil {
return nil, err
}
}
pkgs, err := parse(opts_.localDir)
if err != nil {
return nil, err
}
if opts_.verbose {
for pkg, cves := range pkgs {
var cveList []string
for cve := range cves {
cveList = append(cveList, cve)
}
if opts_.verbose {
fmt.Printf("%s: %v\n", pkg, cveList)
}
}
}
return pkgs, nil
}
func crawl(root string, localDir string, verbose bool) error {
fmt.Println("Crawling CentOS repository...")
c := colly.NewCollector()
if err := os.MkdirAll(localDir, 0o755); err != nil {
return err
}
var repoMDs []url.URL
c.OnHTML("#indexlist .indexcolname a[href]", func(e *colly.HTMLElement) {
href := e.Attr("href")
// Skip going to parent directory.
if strings.HasPrefix(root, href) {
return
}
if nonReleasePathRegex.MatchString(path.Join(e.Request.URL.Path, href)) {
return
}
if !recentCentOSPathRegex.MatchString(path.Join(e.Request.URL.Path, href)) {
if verbose {
fmt.Printf("Ignoring old release: %s\n", path.Join(e.Request.URL.Path, href))
}
return
}
if href == "repomd.xml" {
u := *e.Request.URL
u.Path = path.Join(u.Path, href)
repoMDs = append(repoMDs, u)
if verbose {
fmt.Printf("%s\n", u.Path)
}
return
}
if !strings.Contains(href, "/") {
return
}
e.Request.Visit(href)
})
c.AllowedDomains = append(c.AllowedDomains, repositoryDomain)
if root == "" {
root = defaultRoot
}
if err := c.Visit(repositoryURL + root); err != nil {
return err
}
for _, u := range repoMDs {
if err := processRepoMD(u, localDir, verbose); err != nil {
return err
}
}
return nil
}
type dbs struct {
primary, other string
}
func parse(localDir string) (CentOSPkgSet, error) {
fmt.Println("Processing sqlite files...")
dbPaths := make(map[string]dbs)
if err := filepath.WalkDir(localDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if !strings.HasSuffix(path, ".sqlite") {
return nil
}
dbp := dbPaths[filepath.Dir(path)]
if strings.HasSuffix(path, "-primary.sqlite") {
dbp.primary = path
} else if strings.HasSuffix(path, "-other.sqlite") {
dbp.other = path
}
dbPaths[filepath.Dir(path)] = dbp
return nil
}); err != nil {
return nil, err
}
allPkgs := make(CentOSPkgSet)
for _, db := range dbPaths {
pkgs, err := processSqlites(db)
if err != nil {
return nil, err
}
for pkg, cves := range pkgs {
for cve := range cves {
allPkgs.Add(pkg, cve)
}
}
}
return allPkgs, nil
}
func processRepoMD(mdURL url.URL, localDir string, verbose bool) error {
resp, err := http.Get(mdURL.String())
if err != nil {
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
type location struct {
Href string `xml:"href,attr"`
}
type repoDataItem struct {
Type string `xml:"type,attr"`
Location location `xml:"location"`
}
type repoMetadata struct {
XMLName xml.Name `xml:"repomd"`
Datas []repoDataItem `xml:"data"`
}
var md repoMetadata
if err := xml.Unmarshal(b, &md); err != nil {
return err
}
for _, data := range md.Datas {
if data.Type != "primary_db" && data.Type != "other_db" {
continue
}
sqliteURL := mdURL
sqliteURL.Path = strings.TrimSuffix(sqliteURL.Path, "repomd.xml") + strings.TrimPrefix(data.Location.Href, "repodata/")
if verbose {
fmt.Printf("%s\n", sqliteURL.Path)
}
filePath := filePathfromURL(localDir, sqliteURL)
_, err := os.Stat(filePath)
switch {
case err == nil:
// File already exists, nothing to do.
case errors.Is(err, os.ErrNotExist):
if err := download.Decompressed(fleethttp.NewClient(), sqliteURL, filePath); err != nil {
return err
}
default:
return err
}
}
return nil
}
func filePathfromURL(dir string, url url.URL) string {
filePath := filepath.Join(dir, url.Path)
filePath = strings.TrimSuffix(filePath, ".bz2")
filePath = strings.TrimSuffix(filePath, ".xz")
filePath = strings.TrimSuffix(filePath, ".gz")
return filePath
}
func processSqlites(dbPaths dbs) (CentOSPkgSet, error) {
db, err := sql.Open("sqlite3", dbPaths.primary)
if err != nil {
return nil, err
}
defer db.Close()
if _, err := db.Exec(fmt.Sprintf("ATTACH DATABASE '%s' as other;", dbPaths.other)); err != nil {
return nil, err
}
db.SetMaxOpenConns(1)
rows, err := db.Query(`SELECT
p.name, p.version, p.release, p.arch, c.changelog
FROM packages p
JOIN other.changelog c ON (p.pkgKey=c.pkgKey)
WHERE c.changelog LIKE '%CVE-%-%';`)
if err != nil {
return nil, err
}
defer rows.Close()
pkgs := make(CentOSPkgSet)
for rows.Next() {
var p CentOSPkg
var changelog string
if err := rows.Scan(&p.Name, &p.Version, &p.Release, &p.Arch, &changelog); err != nil {
return nil, err
}
cves := parseCVEs(changelog)
for _, cve := range cves {
pkgs.Add(p, cve)
}
}
if err := rows.Err(); err != nil {
return nil, err
}
return pkgs, nil
}
var cveRegex = regexp.MustCompile(`CVE\-[0-9]+\-[0-9]+`)
func parseCVEs(changelog string) []string {
return cveRegex.FindAllString(changelog, -1)
}
// GenCentOSSqlite will store the CentOS package set in the given sqlite handle.
func GenCentOSSqlite(db *sql.DB, pkgs CentOSPkgSet) error {
if err := createTable(db); err != nil {
return err
}
type pkgWithCVEs struct {
pkg CentOSPkg
cves string
}
var pkgsWithCVEs []pkgWithCVEs
for pkg, cves := range pkgs {
var cveList []string
for cve := range cves {
cveList = append(cveList, strings.TrimPrefix(cve, "CVE-"))
}
sort.Slice(cveList, func(i, j int) bool {
return cveList[i] < cveList[j]
})
pkgsWithCVEs = append(pkgsWithCVEs, pkgWithCVEs{
pkg: pkg,
cves: strings.Join(cveList, ","),
})
}
for _, pkgWithCVEs := range pkgsWithCVEs {
if _, err := db.Exec(
fmt.Sprintf("REPLACE INTO %s (name, version, release, arch, cves) VALUES (?, ?, ?, ?, ?)", centOSPkgsCVEsTable),
pkgWithCVEs.pkg.Name,
pkgWithCVEs.pkg.Version,
pkgWithCVEs.pkg.Release,
pkgWithCVEs.pkg.Arch,
pkgWithCVEs.cves,
); err != nil {
return err
}
}
return nil
}
func createTable(db *sql.DB) error {
_, err := db.Exec(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
name TEXT,
version TEXT,
release TEXT,
arch TEXT,
cves TEXT,
UNIQUE (name, version, release, arch)
);`, centOSPkgsCVEsTable))
return err
}

94
server/vulnerabilities/vuln_centos/centos_test.go Normal file
View File

@ -0,0 +1,94 @@
package vuln_centos
import (
"context"
"database/sql"
"os"
"testing"
"github.com/go-kit/kit/log"
_ "github.com/mattn/go-sqlite3"
"github.com/stretchr/testify/require"
)
func TestCentOSPkgSetAdd(t *testing.T) {
pkgSet := make(CentOSPkgSet)
authConfig := CentOSPkg{
Name: "authconfig",
Version: "6.2.8",
Release: "30.el7",
Arch: "x86_64",
}
cve1 := "CVE-2017-7488"
pkgSet.Add(authConfig, cve1)
cve2 := "CVE-2017-7489"
pkgSet.Add(authConfig, cve2)
curl := CentOSPkg{
Name: "curl",
Version: "4.2",
Release: "30.el7",
Arch: "x86_64",
}
cve3 := "CVE-2017-7490"
pkgSet.Add(curl, cve1)
pkgSet.Add(curl, cve3)
require.Len(t, pkgSet, 2)
require.Len(t, pkgSet[authConfig], 2)
require.Contains(t, pkgSet[authConfig], cve1)
require.Contains(t, pkgSet[authConfig], cve2)
require.NotContains(t, pkgSet[authConfig], cve3)
require.Len(t, pkgSet[curl], 2)
require.Contains(t, pkgSet[curl], cve1)
require.NotContains(t, pkgSet[curl], cve2)
require.Contains(t, pkgSet[curl], cve3)
}
func TestParseCentOSRepository(t *testing.T) {
if os.Getenv("NETWORK_TEST") == "" {
t.Skip("set environment variable NETWORK_TEST=1 to run")
}
// Parse a subset of the CentOS repository.
pkgs, err := ParseCentOSRepository(WithRoot("/centos/7/os/x86_64/repodata/"))
require.NoError(t, err)
db, err := sql.Open("sqlite3", ":memory:")
require.NoError(t, err)
t.Cleanup(func() {
db.Close()
})
err = GenCentOSSqlite(db, pkgs)
require.NoError(t, err)
pkgSet, err := LoadCentOSFixedCVEs(context.Background(), db, log.NewNopLogger())
require.NoError(t, err)
// Shouldn't get _lower_ than what was reported during the development of this test (2221),
// as these are already published releases.
require.GreaterOrEqual(t, len(pkgSet), 2221)
for pkg, cveSet := range pkgSet {
require.NotEmpty(t, pkg.Name)
require.NotEmpty(t, pkg.Version)
require.NotEmpty(t, pkg.Release)
require.NotEmpty(t, pkg.Arch)
require.NotEmpty(t, cveSet)
}
// Check a known vulnerability fixed on a CentOS release.
authConfig := CentOSPkg{
Name: "authconfig",
Version: "6.2.8",
Release: "30.el7",
Arch: "x86_64",
}
cve := "CVE-2017-7488"
require.Contains(t, pkgSet, authConfig)
require.Len(t, pkgSet[authConfig], 1)
require.Contains(t, pkgSet[authConfig], cve)
}