mirror of
https://github.com/empayre/fleet.git
synced 2024-11-06 00:45:19 +00:00
Add CentOS parsing+post-processing to reduce false positives in vulnerability processing (#4037)
* Add CentOS parsing and post-processing in fleet * Add tests and amend SyncCPEDatabase * Add test for centosPostProcessing * Changes from PR comments * Amend software test * Fix sync test * Add index to source and vendor * Use os.MkdirTemp * Rearrange migrations * Regenerate test schema * Add support for testing migrations (#4112) * Add support for testing migrations * Rename migration in tests * Changes suggested in PR * Go mod tidy
This commit is contained in:
Lucas Manuel Rodriguez
GitHub
parent
57d9546081
commit
be72dc356c
5
.gitignore
vendored
5
.gitignore
vendored
@ -57,3 +57,8 @@ terraform.tfstate*
|
||||
|
||||
# generated installers
|
||||
fleet-osquery*
|
||||
|
||||
# residual files when running the cpe command
|
||||
cmd/cpe/etagenv
|
||||
cmd/cpe/cpe*.sqlite
|
||||
cmd/cpe/cpe*.sqlite.gz
|
||||
|
||||
1
changes/issue-3081-parse-centos-repository
Normal file
1
changes/issue-3081-parse-centos-repository
Normal file
@ -0,0 +1 @@
|
||||
* Add parsing of the CentOS repository as well as the loading of the results in fleet vulnerability post-processing (to reduce false positives on RPM packages).
|
||||
@ -2,6 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@ -11,6 +13,7 @@ import (
|
||||
|
||||
"github.com/facebookincubator/nvdtools/cpedict"
|
||||
"github.com/fleetdm/fleet/v4/server/vulnerabilities"
|
||||
"github.com/fleetdm/fleet/v4/server/vulnerabilities/vuln_centos"
|
||||
)
|
||||
|
||||
func panicif(err error) {
|
||||
@ -20,11 +23,43 @@ func panicif(err error) {
|
||||
}
|
||||
|
||||
func main() {
|
||||
fmt.Println("Starting CPE sqlite generation")
|
||||
var (
|
||||
runCentOS bool
|
||||
verbose bool
|
||||
)
|
||||
flag.BoolVar(&runCentOS, "centos", true, "Sets whether to run the CentOS sqlite generation")
|
||||
flag.BoolVar(&verbose, "verbose", false, "Sets verbose mode")
|
||||
flag.Parse()
|
||||
|
||||
dbPath := cpe()
|
||||
|
||||
fmt.Printf("Sqlite file %s size: %.2f MB\n", dbPath, getSizeMB(dbPath))
|
||||
|
||||
// The CentOS repository data is added to the CPE database.
|
||||
if runCentOS {
|
||||
centos(dbPath, verbose)
|
||||
fmt.Printf("Sqlite file %s size with CentOS data: %.2f MB\n", dbPath, getSizeMB(dbPath))
|
||||
}
|
||||
|
||||
fmt.Println("Compressing DB...")
|
||||
compressedPath, err := compress(dbPath)
|
||||
panicif(err)
|
||||
|
||||
fmt.Printf("Final compressed file %s size: %.2f MB\n", compressedPath, getSizeMB(compressedPath))
|
||||
fmt.Println("Done.")
|
||||
}
|
||||
|
||||
func getSizeMB(path string) float64 {
|
||||
info, err := os.Stat(path)
|
||||
panicif(err)
|
||||
return float64(info.Size()) / 1024.0 / 1024.0
|
||||
}
|
||||
|
||||
func cpe() string {
|
||||
fmt.Println("Starting CPE sqlite generation...")
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
panicif(err)
|
||||
|
||||
fmt.Println("CWD:", cwd)
|
||||
|
||||
resp, err := http.Get("https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz")
|
||||
@ -34,16 +69,6 @@ func main() {
|
||||
remoteEtag := getSanitizedEtag(resp)
|
||||
fmt.Println("Got ETag:", remoteEtag)
|
||||
|
||||
nvdRelease, err := vulnerabilities.GetLatestNVDRelease(nil)
|
||||
panicif(err)
|
||||
|
||||
if nvdRelease != nil && nvdRelease.Etag == remoteEtag {
|
||||
fmt.Println("No updates. Exiting...")
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("Needs updating. Generating...")
|
||||
|
||||
gr, err := gzip.NewReader(resp.Body)
|
||||
panicif(err)
|
||||
defer gr.Close()
|
||||
@ -56,25 +81,51 @@ func main() {
|
||||
err = vulnerabilities.GenerateCPEDB(dbPath, cpeDict)
|
||||
panicif(err)
|
||||
|
||||
fmt.Println("Compressing db...")
|
||||
compressedDB, err := os.Create(fmt.Sprintf("%s.gz", dbPath))
|
||||
panicif(err)
|
||||
|
||||
db, err := os.Open(dbPath)
|
||||
panicif(err)
|
||||
w := gzip.NewWriter(compressedDB)
|
||||
|
||||
_, err = io.Copy(w, db)
|
||||
panicif(err)
|
||||
w.Close()
|
||||
compressedDB.Close()
|
||||
|
||||
file, err := os.Create(path.Join(cwd, "etagenv"))
|
||||
panicif(err)
|
||||
file.WriteString(fmt.Sprintf(`ETAG=%s`, remoteEtag))
|
||||
file.Close()
|
||||
|
||||
fmt.Println("Done.")
|
||||
return dbPath
|
||||
}
|
||||
|
||||
func compress(path string) (string, error) {
|
||||
compressedPath := fmt.Sprintf("%s.gz", path)
|
||||
compressedDB, err := os.Create(compressedPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer compressedDB.Close()
|
||||
|
||||
db, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
w := gzip.NewWriter(compressedDB)
|
||||
defer w.Close()
|
||||
|
||||
_, err = io.Copy(w, db)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return compressedPath, nil
|
||||
}
|
||||
|
||||
func centos(dbPath string, verbose bool) {
|
||||
fmt.Println("Starting CentOS sqlite generation...")
|
||||
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
panicif(err)
|
||||
defer db.Close()
|
||||
|
||||
pkgs, err := vuln_centos.ParseCentOSRepository(vuln_centos.WithVerbose(verbose))
|
||||
panicif(err)
|
||||
|
||||
fmt.Printf("Storing CVE info for %d CentOS packages...\n", len(pkgs))
|
||||
err = vuln_centos.GenCentOSSqlite(db, pkgs)
|
||||
panicif(err)
|
||||
}
|
||||
|
||||
func getSanitizedEtag(resp *http.Response) string {
|
||||
|
||||
@ -722,6 +722,16 @@ func cronVulnerabilities(
|
||||
sentry.CaptureException(err)
|
||||
}
|
||||
|
||||
// It's important vulnerabilities.PostProcess runs after ds.CalculateHostsPerSoftware
|
||||
// because it cleans up any software that's not installed on the fleet (e.g. hosts removal,
|
||||
// or software being uninstalled on hosts).
|
||||
if !vulnDisabled {
|
||||
if err := vulnerabilities.PostProcess(ctx, ds, vulnPath, logger, config); err != nil {
|
||||
level.Error(logger).Log("msg", "post processing CVEs", "err", err)
|
||||
sentry.CaptureException(err)
|
||||
}
|
||||
}
|
||||
|
||||
level.Debug(logger).Log("loop", "done")
|
||||
}
|
||||
}
|
||||
|
||||
@ -47,7 +47,7 @@ Downloads (if needed) the data streams that can be used by the Fleet server to p
|
||||
|
||||
dbPath := path.Join(dir, "cpe.sqlite")
|
||||
client := fleethttp.NewClient()
|
||||
err = vulnerabilities.SyncCPEDatabase(client, dbPath, config.FleetConfig{})
|
||||
err = vulnerabilities.SyncCPEDatabase(client, dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -226,7 +226,7 @@ To intercept sent emails while running a Fleet development environment, first, i
|
||||
|
||||
Then, in the "SMTP options" section, enter any email address in the "Sender address" field, set the "SMTP server" to `localhost` on port `1025`, and set "Authentication type" to `None`. Note that you may use any active or inactive sender address.
|
||||
|
||||
Visit [locahost:8025](http://localhost:8025) to view Mailhog's admin interface which will display all emails sent using the simulated mail server.
|
||||
Visit [localhost:8025](http://localhost:8025) to view Mailhog's admin interface which will display all emails sent using the simulated mail server.
|
||||
|
||||
## Development database management
|
||||
|
||||
|
||||
9
go.mod
9
go.mod
@ -7,9 +7,12 @@ require (
|
||||
github.com/AbGuthrie/goquery/v2 v2.0.1
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.0
|
||||
github.com/OneOfOne/xxhash v1.2.8 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.8.0 // indirect
|
||||
github.com/VividCortex/gohistogram v1.0.0 // indirect
|
||||
github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f
|
||||
github.com/WatchBeam/clock v0.0.0-20170901150240-b08e6b4da7ea
|
||||
github.com/antchfx/htmlquery v1.2.4 // indirect
|
||||
github.com/antchfx/xmlquery v1.3.9 // indirect
|
||||
github.com/aws/aws-sdk-go v1.40.34
|
||||
github.com/beevik/etree v1.1.0
|
||||
github.com/briandowns/spinner v1.13.0
|
||||
@ -29,6 +32,7 @@ require (
|
||||
github.com/ghodss/yaml v1.0.0
|
||||
github.com/go-kit/kit v0.9.0
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/gocolly/colly v1.2.0
|
||||
github.com/golang-jwt/jwt/v4 v4.0.0
|
||||
github.com/gomodule/redigo v1.8.5
|
||||
github.com/google/go-cmp v0.5.6
|
||||
@ -45,6 +49,7 @@ require (
|
||||
github.com/jinzhu/copier v0.3.2
|
||||
github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5
|
||||
github.com/jonboulle/clockwork v0.2.2 // indirect
|
||||
github.com/kennygrant/sanitize v1.2.4 // indirect
|
||||
github.com/kevinburke/go-bindata v3.22.0+incompatible
|
||||
github.com/kolide/kit v0.0.0-20180421083548-36eb8dc43916
|
||||
github.com/kolide/launcher v0.0.0-20180427153757-cb412b945cf7
|
||||
@ -68,14 +73,18 @@ require (
|
||||
github.com/rotisserie/eris v0.5.1
|
||||
github.com/rs/zerolog v1.20.0
|
||||
github.com/russellhaering/goxmldsig v1.1.0
|
||||
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
|
||||
github.com/spf13/cast v1.3.1
|
||||
github.com/spf13/cobra v1.2.1
|
||||
github.com/spf13/viper v1.8.1
|
||||
github.com/stretchr/objx v0.3.0 // indirect
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/temoto/robotstxt v1.1.2 // indirect
|
||||
github.com/theupdateframework/go-tuf v0.0.0-20220121203041-e3557e322879
|
||||
github.com/throttled/throttled/v2 v2.8.0
|
||||
github.com/tj/assert v0.0.3
|
||||
github.com/ulikunitz/xz v0.5.10
|
||||
github.com/urfave/cli/v2 v2.3.0
|
||||
github.com/valyala/fasthttp v1.31.0
|
||||
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce
|
||||
|
||||
22
go.sum
22
go.sum
@ -164,6 +164,8 @@ github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a h1:W6RrgN/sTxg1
|
||||
github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a/go.mod h1:NYt+V3/4rEeDuaev/zw1zCq8uqVEuPHzDPo3OZrlGJ4=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.2.2 h1:u2m7xt+CZWj88qK1UUNBoXeJCFJwJCZ/Ff4ymGoxEXs=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.2.2/go.mod h1:ajUlBGvxMH1UBZnaYO3d1FSVzjiC6kK9XlZYGiDCvpM=
|
||||
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
|
||||
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
|
||||
github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE=
|
||||
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
|
||||
@ -189,8 +191,16 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/andybalholm/brotli v1.0.2 h1:JKnhI/XQ75uFBTiuzXpzFrUriDPiZjlOSzh6wXogP0E=
|
||||
github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494=
|
||||
github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc=
|
||||
github.com/antchfx/xmlquery v1.3.9 h1:Y+zyMdiUZ4fasTQTkDb3DflOXP7+obcYEh80SISBmnQ=
|
||||
github.com/antchfx/xmlquery v1.3.9/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks=
|
||||
github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
|
||||
github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/apache/thrift v0.13.1-0.20200603211036-eac4d0c79a5f h1:33BV5v3u8I6dA2dEoPuXWCsAaHHOJfPtdxZhAMQV4uo=
|
||||
github.com/apache/thrift v0.13.1-0.20200603211036-eac4d0c79a5f/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
@ -436,6 +446,8 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA
|
||||
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
|
||||
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
|
||||
github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
|
||||
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
@ -682,6 +694,8 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb
|
||||
github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE=
|
||||
github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro=
|
||||
github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=
|
||||
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
|
||||
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
|
||||
github.com/kevinburke/go-bindata v3.22.0+incompatible h1:/JmqEhIWQ7GRScV0WjX/0tqBrC5D21ALg0H0U/KZ/ts=
|
||||
github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
@ -914,6 +928,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
|
||||
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc=
|
||||
@ -990,6 +1006,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM=
|
||||
github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog=
|
||||
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
|
||||
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
github.com/theupdateframework/go-tuf v0.0.0-20220121203041-e3557e322879 h1:UeDpdrX16scCvbdgdMsrztZsQLDofld/Zo+WGDe/PBE=
|
||||
github.com/theupdateframework/go-tuf v0.0.0-20220121203041-e3557e322879/go.mod h1:I0Gs4Tev4hYQ5wiNqN8VJ7qS0gw7KOZNQuckC624RmE=
|
||||
github.com/throttled/throttled/v2 v2.8.0 h1:B5VfdM8BE+ClI2Ji238SbNOTWfYcocvuAhgT27lvwrE=
|
||||
@ -1052,7 +1070,6 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
|
||||
github.com/zclconf/go-cty v1.1.0 h1:uJwc9HiBOCpoKIObTQaLR+tsEXx1HBHnOsOOpcdhZgw=
|
||||
github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
|
||||
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
|
||||
github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
|
||||
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
|
||||
github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c h1:TWQ2UvXPkhPxI2KmApKBOCaV6yD2N4mlvqFQ/DlPtpQ=
|
||||
github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c/go.mod h1:OYYulo9tUqRadRLwB0+LE914sa1ui2yL7OrcU3Q/1XY=
|
||||
@ -1183,6 +1200,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
@ -1190,6 +1208,7 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/
|
||||
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
@ -1209,6 +1228,7 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211007125505-59d4e928ea9d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# pkg directory
|
||||
|
||||
This top-level `pkg` directory contains packages that may be shared between `fleet` and `orbit`.
|
||||
This top-level `pkg` directory contains packages that may be shared between all `fleet` backend components.
|
||||
|
||||
70
pkg/download/download.go
Normal file
70
pkg/download/download.go
Normal file
@ -0,0 +1,70 @@
|
||||
// Package download has utilities to download resources from URLs.
|
||||
package download
|
||||
|
||||
import (
|
||||
"compress/bzip2"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
// Decompressed downloads and decompresses a file from a URL to a local path.
|
||||
//
|
||||
// It supports gz, bz2 and gz compressed files.
|
||||
func Decompressed(client *http.Client, u url.URL, path string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmpFile, err := ioutil.TempFile("", fmt.Sprintf("%s*", filepath.Base(path)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tmpFile.Close()
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var decompressor io.Reader
|
||||
switch {
|
||||
case strings.HasSuffix(u.Path, "gz"):
|
||||
decompressor, err = gzip.NewReader(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case strings.HasSuffix(u.Path, "bz2"):
|
||||
decompressor = bzip2.NewReader(resp.Body)
|
||||
case strings.HasSuffix(u.Path, "xz"):
|
||||
decompressor, err = xz.NewReader(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unknown extension: %s", u.Path)
|
||||
}
|
||||
if _, err := io.Copy(tmpFile, decompressor); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tmpFile.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmpFile.Name(), path); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,66 @@
|
||||
package tables
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func init() {
|
||||
MigrationClient.AddMigration(Up_20220208144831, Down_20220208144831)
|
||||
}
|
||||
|
||||
func Up_20220208144831(tx *sql.Tx) error {
|
||||
// NOTE(lucas): I'm using short lengths for the new varchar columns
|
||||
// due to constraints on the size of the key added below. Using 255
|
||||
// for the three new fields would fail with:
|
||||
// "Error 1071: Specified key was too long; max key length is 3072 bytes".
|
||||
//
|
||||
// We need to use "NOT NULL" because these new columns are to be included in the KEY.
|
||||
if _, err := tx.Exec("ALTER TABLE software " +
|
||||
"ADD COLUMN `release` VARCHAR(64) NOT NULL DEFAULT '', " +
|
||||
"ADD COLUMN vendor VARCHAR(32) NOT NULL DEFAULT '', " +
|
||||
"ADD COLUMN arch VARCHAR(16) NOT NULL DEFAULT ''"); err != nil {
|
||||
return errors.Wrap(err, "add new software columns")
|
||||
}
|
||||
|
||||
// Delete current identifier for a software.
|
||||
currIndexName, err := indexNameByColumnName(tx, "software", "name")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "fetch current software index")
|
||||
}
|
||||
if _, err := tx.Exec(fmt.Sprintf("ALTER TABLE software DROP KEY %s", currIndexName)); err != nil {
|
||||
return errors.Wrap(err, "add new software columns")
|
||||
}
|
||||
|
||||
// A software piece was originally identified by name, version and source.
|
||||
//
|
||||
// We now add "vendor", "release" and "arch":
|
||||
// - release is the version of the OS this software was released on (e.g. "30.el7" for a CentOS package).
|
||||
// - vendor is the supplier of the software (e.g. "CentOS").
|
||||
// - arch is the target architecture of the software (e.g. "x86_64").
|
||||
if _, err := tx.Exec("ALTER TABLE software ADD UNIQUE KEY (name, version, source, `release`, vendor, arch)"); err != nil {
|
||||
return errors.Wrap(err, "add new index")
|
||||
}
|
||||
|
||||
// Remove all software with source rpm_packages, as we will be ingesting them with new osquery
|
||||
// fields.
|
||||
//
|
||||
// Due to foreign keys, the following statement also deletes the corresponding
|
||||
// entries in `software_cpe` and `software_cve`.
|
||||
if _, err := tx.Exec("DELETE FROM software WHERE source = 'rpm_packages'"); err != nil {
|
||||
return errors.Wrap(err, "delete existing software for rpm_packages")
|
||||
}
|
||||
|
||||
// Adding index to optimize software listing by source and vendor for vulnerability post-processing.
|
||||
if _, err := tx.Exec("CREATE INDEX software_source_vendor_idx ON software (source, vendor)"); err != nil {
|
||||
return errors.Wrap(err, "creating source+vendor index")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func Down_20220208144831(tx *sql.Tx) error {
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
package tables
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestUp_20220208144831(t *testing.T) {
|
||||
db := applyUpToPrev(t)
|
||||
|
||||
_, err := db.Exec(`INSERT INTO software (name, version, source) VALUES ("authconfig", "6.2.8", "rpm_packages")`)
|
||||
require.NoError(t, err)
|
||||
_, err = db.Exec(`INSERT INTO software (name, version, source, bundle_identifier) VALUES ("iTerm.app", "3.4.14", "apps", "com.googlecode.iterm2")`)
|
||||
require.NoError(t, err)
|
||||
|
||||
applyNext(t, db)
|
||||
|
||||
// Check migration removes rpm packages.
|
||||
row := db.QueryRow(`SELECT COUNT(*) FROM software WHERE source = "rpm_packages"`)
|
||||
var count int
|
||||
require.NoError(t, row.Scan(&count))
|
||||
require.Zero(t, count)
|
||||
row = db.QueryRow(`SELECT COUNT(*) FROM software WHERE source = "apps"`)
|
||||
require.NoError(t, row.Scan(&count))
|
||||
require.Equal(t, 1, count)
|
||||
|
||||
// Check we can INSERT software with the new columns empty.
|
||||
_, err = db.Exec(`INSERT INTO software (name, version, source, bundle_identifier) VALUES ("iCloud.app", "1.0", "apps", "com.apple.CloudKit.ShareBear")`)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check we can INSERT software with the new columns set.
|
||||
_, err = db.Exec(`INSERT INTO software (name, version, source, ` + "`release`" + `, vendor, arch) VALUES ("authconfig", "6.2.8", "rpm_packages", "30.el7", "CentOS", "x86_64")`)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
86
server/datastore/mysql/migrations/tables/migration_test.go
Normal file
86
server/datastore/mysql/migrations/tables/migration_test.go
Normal file
@ -0,0 +1,86 @@
|
||||
// Package tables holds fleet table migrations.
|
||||
//
|
||||
// Migrations can be tested with tests following the following format:
|
||||
//
|
||||
// $ cat 20220208144831_AddSoftwareReleaseArchVendorColumns_test.go
|
||||
//
|
||||
// [...]
|
||||
// func TestUp_20220208144831(t *testing.T) {
|
||||
// // Apply all migrations up to 20220208144831 (name of test), not included.
|
||||
// db := applyUpToPrev(t)
|
||||
//
|
||||
// // insert testing data, etc.
|
||||
//
|
||||
// // The following will apply migration 20220208144831.
|
||||
// applyNext(t, db)
|
||||
//
|
||||
// // insert testing data, verify migration.
|
||||
// }
|
||||
package tables
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TODO(lucas): I'm copy pasting some of the mysql functionality methods here
|
||||
// otherwise we have import cycle errors.
|
||||
//
|
||||
// We need to decouple the server/datastore/mysql package,
|
||||
// it contains both the implementation of the fleet.Datastore and
|
||||
// MySQL functionality, and MySQL test functionality.
|
||||
const (
|
||||
testUsername = "root"
|
||||
testPassword = "toor"
|
||||
testAddress = "localhost:3307"
|
||||
)
|
||||
|
||||
func newDBConnForTests(t *testing.T) *sqlx.DB {
|
||||
db, err := sqlx.Open(
|
||||
"mysql",
|
||||
fmt.Sprintf("%s:%s@tcp(%s)/?multiStatements=true", testUsername, testPassword, testAddress),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s; USE %s;", t.Name(), t.Name(), t.Name()))
|
||||
require.NoError(t, err)
|
||||
return db
|
||||
}
|
||||
|
||||
func getMigrationVersion(t *testing.T) int64 {
|
||||
v, err := strconv.Atoi(strings.TrimPrefix(t.Name(), "TestUp_"))
|
||||
require.NoError(t, err)
|
||||
return int64(v)
|
||||
}
|
||||
|
||||
// applyUpToPrev will allocate a testing DB connection and apply
|
||||
// migrations up to, not including, the migration specified in the test name.
|
||||
//
|
||||
// It returns the database connection to perform additional queries and migrations.
|
||||
func applyUpToPrev(t *testing.T) *sqlx.DB {
|
||||
db := newDBConnForTests(t)
|
||||
v := getMigrationVersion(t)
|
||||
for {
|
||||
current, err := MigrationClient.GetDBVersion(db.DB)
|
||||
require.NoError(t, err)
|
||||
next, err := MigrationClient.Migrations.Next(current)
|
||||
require.NoError(t, err)
|
||||
if next.Version == v {
|
||||
return db
|
||||
}
|
||||
applyNext(t, db)
|
||||
}
|
||||
}
|
||||
|
||||
// applyNext performs the next migration in the chain.
|
||||
func applyNext(t *testing.T, db *sqlx.DB) {
|
||||
// gooseNoDir is the value to not parse local files and instead use
|
||||
// the migrations that were added manually via Add().
|
||||
const gooseNoDir = ""
|
||||
err := MigrationClient.UpByOne(db.DB, gooseNoDir)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@ -19,6 +19,10 @@ const (
|
||||
maxSoftwareVersionLen = 255
|
||||
maxSoftwareSourceLen = 64
|
||||
maxSoftwareBundleIdentifierLen = 255
|
||||
|
||||
maxSoftwareReleaseLen = 64
|
||||
maxSoftwareVendorLen = 32
|
||||
maxSoftwareArchLen = 16
|
||||
)
|
||||
|
||||
func truncateString(str string, length int) string {
|
||||
@ -29,16 +33,36 @@ func truncateString(str string, length int) string {
|
||||
}
|
||||
|
||||
func softwareToUniqueString(s fleet.Software) string {
|
||||
return strings.Join([]string{s.Name, s.Version, s.Source, s.BundleIdentifier}, "\u0000")
|
||||
ss := []string{s.Name, s.Version, s.Source, s.BundleIdentifier}
|
||||
// Release, Vendor and Arch fields were added on a migration,
|
||||
// thus we only include them in the string if at least one of them is defined.
|
||||
if s.Release != "" || s.Vendor != "" || s.Arch != "" {
|
||||
ss = append(ss, s.Release, s.Vendor, s.Arch)
|
||||
}
|
||||
return strings.Join(ss, "\u0000")
|
||||
}
|
||||
|
||||
func uniqueStringToSoftware(s string) fleet.Software {
|
||||
parts := strings.Split(s, "\u0000")
|
||||
|
||||
// Release, Vendor and Arch fields were added on a migration,
|
||||
// If one of them is defined, then they are included in the string.
|
||||
var release, vendor, arch string
|
||||
if len(parts) > 4 {
|
||||
release = truncateString(parts[4], maxSoftwareReleaseLen)
|
||||
vendor = truncateString(parts[5], maxSoftwareVendorLen)
|
||||
arch = truncateString(parts[6], maxSoftwareArchLen)
|
||||
}
|
||||
|
||||
return fleet.Software{
|
||||
Name: truncateString(parts[0], maxSoftwareNameLen),
|
||||
Version: truncateString(parts[1], maxSoftwareVersionLen),
|
||||
Source: truncateString(parts[2], maxSoftwareSourceLen),
|
||||
BundleIdentifier: truncateString(parts[3], maxSoftwareBundleIdentifierLen),
|
||||
|
||||
Release: release,
|
||||
Vendor: vendor,
|
||||
Arch: arch,
|
||||
}
|
||||
}
|
||||
|
||||
@ -151,8 +175,10 @@ func getOrGenerateSoftwareIdDB(ctx context.Context, tx sqlx.ExtContext, s fleet.
|
||||
var existingId []int64
|
||||
if err := sqlx.SelectContext(ctx, tx,
|
||||
&existingId,
|
||||
`SELECT id FROM software WHERE name = ? AND version = ? AND source = ? AND bundle_identifier = ?`,
|
||||
s.Name, s.Version, s.Source, s.BundleIdentifier,
|
||||
"SELECT id FROM software "+
|
||||
"WHERE name = ? AND version = ? AND source = ? AND `release` = ? AND "+
|
||||
"vendor = ? AND arch = ? AND bundle_identifier = ?",
|
||||
s.Name, s.Version, s.Source, s.Release, s.Vendor, s.Arch, s.BundleIdentifier,
|
||||
); err != nil {
|
||||
return 0, ctxerr.Wrap(ctx, err, "get software")
|
||||
}
|
||||
@ -161,9 +187,11 @@ func getOrGenerateSoftwareIdDB(ctx context.Context, tx sqlx.ExtContext, s fleet.
|
||||
}
|
||||
|
||||
result, err := tx.ExecContext(ctx,
|
||||
`INSERT INTO software (name, version, source, bundle_identifier) VALUES (?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE bundle_identifier=VALUES(bundle_identifier)`,
|
||||
s.Name, s.Version, s.Source, s.BundleIdentifier,
|
||||
"INSERT INTO software "+
|
||||
"(name, version, source, `release`, vendor, arch, bundle_identifier) "+
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?) "+
|
||||
"ON DUPLICATE KEY UPDATE bundle_identifier=VALUES(bundle_identifier)",
|
||||
s.Name, s.Version, s.Source, s.Release, s.Vendor, s.Arch, s.BundleIdentifier,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, ctxerr.Wrap(ctx, err, "insert software")
|
||||
@ -503,6 +531,59 @@ func (ds *Datastore) CountSoftware(ctx context.Context, opt fleet.SoftwareListOp
|
||||
return countSoftwareDB(ctx, ds.reader, nil, opt)
|
||||
}
|
||||
|
||||
// ListVulnerableSoftwareBySource lists all the vulnerable software that matches the given source.
|
||||
func (ds *Datastore) ListVulnerableSoftwareBySource(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) {
|
||||
var softwareCVEs []struct {
|
||||
fleet.Software
|
||||
CPE uint `db:"cpe_id"`
|
||||
CVEs string `db:"cves"`
|
||||
}
|
||||
if err := sqlx.SelectContext(ctx, ds.reader, &softwareCVEs, `
|
||||
SELECT s.*, scv.cpe_id, GROUP_CONCAT(scv.cve SEPARATOR ',') as cves
|
||||
FROM software s
|
||||
JOIN software_cpe scp ON (s.id=scp.software_id)
|
||||
JOIN software_cve scv ON (scp.id=scv.cpe_id)
|
||||
WHERE s.source = ?
|
||||
GROUP BY scv.cpe_id
|
||||
`, source); err != nil {
|
||||
return nil, ctxerr.Wrapf(ctx, err, "listing vulnerable software by source")
|
||||
}
|
||||
software := make([]fleet.SoftwareWithCPE, 0, len(softwareCVEs))
|
||||
for _, sc := range softwareCVEs {
|
||||
for _, cve := range strings.Split(sc.CVEs, ",") {
|
||||
sc.Software.Vulnerabilities = append(sc.Software.Vulnerabilities, fleet.SoftwareCVE{
|
||||
CVE: cve,
|
||||
DetailsLink: fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cve),
|
||||
})
|
||||
}
|
||||
software = append(software, fleet.SoftwareWithCPE{
|
||||
Software: sc.Software,
|
||||
CPEID: sc.CPE,
|
||||
})
|
||||
}
|
||||
return software, nil
|
||||
}
|
||||
|
||||
// DeleteVulnerabilitiesByCPECVE deletes the given list of vulnerabilities identified by CPE+CVE.
|
||||
func (ds *Datastore) DeleteVulnerabilitiesByCPECVE(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error {
|
||||
if len(vulnerabilities) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
sql := fmt.Sprintf(
|
||||
`DELETE FROM software_cve WHERE (cpe_id, cve) IN (%s)`,
|
||||
strings.TrimSuffix(strings.Repeat("(?,?),", len(vulnerabilities)), ","),
|
||||
)
|
||||
var args []interface{}
|
||||
for _, vulnerability := range vulnerabilities {
|
||||
args = append(args, vulnerability.CPEID, vulnerability.CVE)
|
||||
}
|
||||
if _, err := ds.writer.ExecContext(ctx, sql, args...); err != nil {
|
||||
return ctxerr.Wrapf(ctx, err, "deleting vulnerable software")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ds *Datastore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software, error) {
|
||||
software := fleet.Software{}
|
||||
err := sqlx.GetContext(ctx, ds.reader, &software, `SELECT * FROM software WHERE id=?`, id)
|
||||
@ -545,6 +626,9 @@ func (ds *Datastore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software
|
||||
// CalculateHostsPerSoftware calculates the number of hosts having each
|
||||
// software installed and stores that information in the software_host_counts
|
||||
// table.
|
||||
//
|
||||
// After aggregation, it cleans up unused software (e.g. software installed
|
||||
// on removed hosts, software uninstalled on hosts, etc.)
|
||||
func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error {
|
||||
resetStmt := `
|
||||
UPDATE software_host_counts
|
||||
@ -614,8 +698,6 @@ func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt ti
|
||||
return ctxerr.Wrap(ctx, err, "iterate over host_software counts")
|
||||
}
|
||||
|
||||
// delete any unused software from the software table (any that
|
||||
// isn't in that list with a host count > 0).
|
||||
cleanupStmt := `
|
||||
DELETE FROM
|
||||
software
|
||||
@ -630,7 +712,6 @@ func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt ti
|
||||
if _, err := ds.writer.ExecContext(ctx, cleanupStmt); err != nil {
|
||||
return ctxerr.Wrap(ctx, err, "delete unused software")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -33,6 +33,8 @@ func TestSoftware(t *testing.T) {
|
||||
{"LoadSupportsTonsOfCVEs", testSoftwareLoadSupportsTonsOfCVEs},
|
||||
{"List", testSoftwareList},
|
||||
{"CalculateHostsPerSoftware", testSoftwareCalculateHostsPerSoftware},
|
||||
{"ListVulnerableSoftwareBySource", testListVulnerableSoftwareBySource},
|
||||
{"DeleteVulnerabilitiesByCPECVE", testDeleteVulnerabilitiesByCPECVE},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
@ -426,7 +428,15 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
|
||||
|
||||
t.Run("paginates", func(t *testing.T) {
|
||||
software := listSoftwareCheckCount(t, ds, 1, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{Page: 1, PerPage: 1, OrderKey: "version"}}, true)
|
||||
expected := []fleet.Software{foo003}
|
||||
require.Len(t, software, 1)
|
||||
var expected []fleet.Software
|
||||
// Both foo003 and bar003 have the same version, thus we check which one the database picked
|
||||
// for the second page.
|
||||
if software[0].Name == "foo" {
|
||||
expected = []fleet.Software{foo003}
|
||||
} else {
|
||||
expected = []fleet.Software{bar003}
|
||||
}
|
||||
test.ElementsMatchSkipID(t, software, expected)
|
||||
})
|
||||
|
||||
@ -626,3 +636,145 @@ func testSoftwareCalculateHostsPerSoftware(t *testing.T, ds *Datastore) {
|
||||
}
|
||||
cmpNameVersionCount(want, allSw)
|
||||
}
|
||||
|
||||
func insertVulnSoftwareForTest(t *testing.T, ds *Datastore) {
|
||||
host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now())
|
||||
host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now())
|
||||
|
||||
software1 := []fleet.Software{
|
||||
{
|
||||
Name: "foo.rpm", Version: "0.0.1", Source: "rpm_packages", GenerateCPE: "cpe_foo_rpm",
|
||||
},
|
||||
{
|
||||
Name: "foo.chrome", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "cpe_foo_chrome",
|
||||
},
|
||||
}
|
||||
software2 := []fleet.Software{
|
||||
{
|
||||
Name: "foo.chrome", Version: "v0.0.2", Source: "chrome_extensions", GenerateCPE: "cpe_foo_chrome2",
|
||||
},
|
||||
{
|
||||
Name: "foo.chrome", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "cpe_foo_chrome_3",
|
||||
Vulnerabilities: fleet.VulnerabilitiesSlice{
|
||||
{CVE: "cve-123-456-789", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-123-456-789"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "bar.rpm", Version: "0.0.3", Source: "rpm_packages", GenerateCPE: "cpe_bar_rpm",
|
||||
Vulnerabilities: fleet.VulnerabilitiesSlice{
|
||||
{CVE: "cve-321-432-543", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-321-432-543"},
|
||||
{CVE: "cve-333-444-555", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-333-444-555"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
require.NoError(t, ds.UpdateHostSoftware(context.Background(), host1.ID, software1))
|
||||
require.NoError(t, ds.UpdateHostSoftware(context.Background(), host2.ID, software2))
|
||||
require.NoError(t, ds.LoadHostSoftware(context.Background(), host1))
|
||||
require.NoError(t, ds.LoadHostSoftware(context.Background(), host2))
|
||||
|
||||
sort.Slice(host1.Software, func(i, j int) bool {
|
||||
return host1.Software[i].Name+host1.Software[i].Version < host1.Software[j].Name+host1.Software[j].Version
|
||||
})
|
||||
sort.Slice(host2.Software, func(i, j int) bool {
|
||||
return host2.Software[i].Name+host2.Software[i].Version < host2.Software[j].Name+host2.Software[j].Version
|
||||
})
|
||||
|
||||
require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[0], "cpe_foo_chrome"))
|
||||
require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[1], "cpe_foo_rpm"))
|
||||
|
||||
require.NoError(t, ds.AddCPEForSoftware(context.Background(), host2.Software[0], "cpe_bar_rpm"))
|
||||
require.NoError(t, ds.AddCPEForSoftware(context.Background(), host2.Software[1], "cpe_foo_chrome_3"))
|
||||
require.NoError(t, ds.AddCPEForSoftware(context.Background(), host2.Software[2], "cpe_foo_chrome_2"))
|
||||
|
||||
_, err := ds.InsertCVEForCPE(context.Background(), "cve-123-456-789", []string{"cpe_foo_chrome_3"})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = ds.InsertCVEForCPE(context.Background(), "cve-321-432-543", []string{"cpe_bar_rpm"})
|
||||
require.NoError(t, err)
|
||||
_, err = ds.InsertCVEForCPE(context.Background(), "cve-333-444-555", []string{"cpe_bar_rpm"})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func testListVulnerableSoftwareBySource(t *testing.T, ds *Datastore) {
|
||||
ctx := context.Background()
|
||||
|
||||
insertVulnSoftwareForTest(t, ds)
|
||||
|
||||
vulnerable, err := ds.ListVulnerableSoftwareBySource(ctx, "apps")
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, vulnerable)
|
||||
|
||||
vulnerable, err = ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, vulnerable, 1)
|
||||
require.Equal(t, vulnerable[0].Name, "bar.rpm")
|
||||
require.Len(t, vulnerable[0].Vulnerabilities, 2)
|
||||
sort.Slice(vulnerable[0].Vulnerabilities, func(i, j int) bool {
|
||||
return vulnerable[0].Vulnerabilities[i].CVE < vulnerable[0].Vulnerabilities[j].CVE
|
||||
})
|
||||
require.Equal(t, "cve-321-432-543", vulnerable[0].Vulnerabilities[0].CVE)
|
||||
require.Equal(t, "cve-333-444-555", vulnerable[0].Vulnerabilities[1].CVE)
|
||||
}
|
||||
|
||||
func testDeleteVulnerabilitiesByCPECVE(t *testing.T, ds *Datastore) {
|
||||
ctx := context.Background()
|
||||
|
||||
err := ds.DeleteVulnerabilitiesByCPECVE(ctx, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
insertVulnSoftwareForTest(t, ds)
|
||||
|
||||
err = ds.DeleteVulnerabilitiesByCPECVE(ctx, []fleet.SoftwareVulnerability{
|
||||
{
|
||||
CPEID: 999, // unknown CPE
|
||||
CVE: "cve-333-444-555",
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
software, err := ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, software, 1)
|
||||
barRPM := software[0]
|
||||
require.Len(t, barRPM.Vulnerabilities, 2)
|
||||
|
||||
err = ds.DeleteVulnerabilitiesByCPECVE(ctx, []fleet.SoftwareVulnerability{
|
||||
{
|
||||
CPEID: barRPM.CPEID,
|
||||
CVE: "unknown-cve",
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = ds.DeleteVulnerabilitiesByCPECVE(ctx, []fleet.SoftwareVulnerability{
|
||||
{
|
||||
CPEID: barRPM.CPEID,
|
||||
CVE: "cve-333-444-555",
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
software, err = ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, software, 1)
|
||||
barRPM = software[0]
|
||||
require.Len(t, barRPM.Vulnerabilities, 1)
|
||||
|
||||
err = ds.DeleteVulnerabilitiesByCPECVE(ctx, []fleet.SoftwareVulnerability{
|
||||
{
|
||||
CPEID: barRPM.CPEID,
|
||||
CVE: "cve-321-432-543",
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
software, err = ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, software, 0)
|
||||
|
||||
software, err = ds.ListVulnerableSoftwareBySource(ctx, "chrome_extensions")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, software, 1)
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
@ -330,6 +331,12 @@ type Datastore interface {
|
||||
AllCPEs(ctx context.Context) ([]string, error)
|
||||
InsertCVEForCPE(ctx context.Context, cve string, cpes []string) (int64, error)
|
||||
SoftwareByID(ctx context.Context, id uint) (*Software, error)
|
||||
// CalculateHostsPerSoftware calculates the number of hosts having each
|
||||
// software installed and stores that information in the software_host_counts
|
||||
// table.
|
||||
//
|
||||
// After aggregation, it cleans up unused software (e.g. software installed
|
||||
// on removed hosts, software uninstalled on hosts, etc.)
|
||||
CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error
|
||||
HostsByCPEs(ctx context.Context, cpes []string) ([]*CPEHost, error)
|
||||
|
||||
@ -374,6 +381,10 @@ type Datastore interface {
|
||||
|
||||
ListSoftware(ctx context.Context, opt SoftwareListOptions) ([]Software, error)
|
||||
CountSoftware(ctx context.Context, opt SoftwareListOptions) (int, error)
|
||||
// ListVulnerableSoftwareBySource lists all the vulnerable software that matches the given source.
|
||||
ListVulnerableSoftwareBySource(ctx context.Context, source string) ([]SoftwareWithCPE, error)
|
||||
// DeleteVulnerabilities deletes the given list of vulnerabilities identified by CPE+CVE.
|
||||
DeleteVulnerabilitiesByCPECVE(ctx context.Context, vulnerabilities []SoftwareVulnerability) error
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Team Policies
|
||||
@ -549,6 +560,26 @@ const (
|
||||
UnknownMigrations
|
||||
)
|
||||
|
||||
// SoftwareVulnerability identifies a vulnerability on a specific software (CPE).
|
||||
type SoftwareVulnerability struct {
|
||||
// CPEID is the ID of the software CPE in the system.
|
||||
CPEID uint
|
||||
CVE string
|
||||
}
|
||||
|
||||
// String implements fmt.Stringer.
|
||||
func (sv SoftwareVulnerability) String() string {
|
||||
return fmt.Sprintf("{%d,%s}", sv.CPEID, sv.CVE)
|
||||
}
|
||||
|
||||
// SoftwareWithCPE holds a software piece alongside its CPE ID.
|
||||
type SoftwareWithCPE struct {
|
||||
// Software holds the software data.
|
||||
Software
|
||||
// CPEID is the ID of the software CPE in the system.
|
||||
CPEID uint
|
||||
}
|
||||
|
||||
// NotFoundError is returned when the datastore resource cannot be found.
|
||||
type NotFoundError interface {
|
||||
error
|
||||
|
||||
@ -19,6 +19,14 @@ type Software struct {
|
||||
// Source is the source of the data (osquery table name).
|
||||
Source string `json:"source" db:"source"`
|
||||
|
||||
// Release is the version of the OS this software was released on
|
||||
// (e.g. "30.el7" for a CentOS package).
|
||||
Release string `json:"release,omitempty" db:"release"`
|
||||
// Vendor is the supplier of the software (e.g. "CentOS").
|
||||
Vendor string `json:"vendor,omitempty" db:"vendor"`
|
||||
// Arch is the architecture of the software (e.g. "x86_64").
|
||||
Arch string `json:"arch,omitempty" db:"arch"`
|
||||
|
||||
// GenerateCPE is the CPE23 string that corresponds to the current software
|
||||
GenerateCPE string `json:"generated_cpe" db:"generated_cpe"`
|
||||
// Vulnerabilities lists all the found CVEs for the CPE
|
||||
|
||||
@ -312,6 +312,10 @@ type ListSoftwareFunc func(ctx context.Context, opt fleet.SoftwareListOptions) (
|
||||
|
||||
type CountSoftwareFunc func(ctx context.Context, opt fleet.SoftwareListOptions) (int, error)
|
||||
|
||||
type ListVulnerableSoftwareBySourceFunc func(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error)
|
||||
|
||||
type DeleteVulnerabilitiesByCPECVEFunc func(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error
|
||||
|
||||
type NewTeamPolicyFunc func(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error)
|
||||
|
||||
type ListTeamPoliciesFunc func(ctx context.Context, teamID uint) ([]*fleet.Policy, error)
|
||||
@ -825,6 +829,12 @@ type DataStore struct {
|
||||
CountSoftwareFunc CountSoftwareFunc
|
||||
CountSoftwareFuncInvoked bool
|
||||
|
||||
ListVulnerableSoftwareBySourceFunc ListVulnerableSoftwareBySourceFunc
|
||||
ListVulnerableSoftwareBySourceFuncInvoked bool
|
||||
|
||||
DeleteVulnerabilitiesByCPECVEFunc DeleteVulnerabilitiesByCPECVEFunc
|
||||
DeleteVulnerabilitiesByCPECVEFuncInvoked bool
|
||||
|
||||
NewTeamPolicyFunc NewTeamPolicyFunc
|
||||
NewTeamPolicyFuncInvoked bool
|
||||
|
||||
@ -1669,6 +1679,16 @@ func (s *DataStore) CountSoftware(ctx context.Context, opt fleet.SoftwareListOpt
|
||||
return s.CountSoftwareFunc(ctx, opt)
|
||||
}
|
||||
|
||||
func (s *DataStore) ListVulnerableSoftwareBySource(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) {
|
||||
s.ListVulnerableSoftwareBySourceFuncInvoked = true
|
||||
return s.ListVulnerableSoftwareBySourceFunc(ctx, source)
|
||||
}
|
||||
|
||||
func (s *DataStore) DeleteVulnerabilitiesByCPECVE(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error {
|
||||
s.DeleteVulnerabilitiesByCPECVEFuncInvoked = true
|
||||
return s.DeleteVulnerabilitiesByCPECVEFunc(ctx, vulnerabilities)
|
||||
}
|
||||
|
||||
func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) {
|
||||
s.NewTeamPolicyFuncInvoked = true
|
||||
return s.NewTeamPolicyFunc(ctx, teamID, authorID, args)
|
||||
|
||||
@ -11,6 +11,8 @@ func String(x string) *string {
|
||||
return &x
|
||||
}
|
||||
|
||||
// StringValueOrZero returns the string value.
|
||||
// Returns empty string if x is nil.
|
||||
func StringValueOrZero(x *string) string {
|
||||
if x == nil {
|
||||
return ""
|
||||
|
||||
@ -84,6 +84,10 @@ func (s *integrationTestSuite) TearDownTest() {
|
||||
_, err = s.ds.DeleteGlobalPolicies(ctx, globalPolicyIDs)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// CalculateHostsPerSoftware performs a cleanup.
|
||||
err = s.ds.CalculateHostsPerSoftware(ctx, time.Now())
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestIntegrations(t *testing.T) {
|
||||
@ -1742,13 +1746,15 @@ func (s *integrationTestSuite) TestScheduledQueries() {
|
||||
// batch-delete by id, 3 ids, only one exists
|
||||
var delBatchResp deleteQueriesResponse
|
||||
s.DoJSON("POST", "/api/v1/fleet/queries/delete", map[string]interface{}{
|
||||
"ids": []uint{query.ID, query2.ID, query3.ID}}, http.StatusOK, &delBatchResp)
|
||||
"ids": []uint{query.ID, query2.ID, query3.ID},
|
||||
}, http.StatusOK, &delBatchResp)
|
||||
assert.Equal(t, uint(1), delBatchResp.Deleted)
|
||||
|
||||
// batch-delete by id, none exist
|
||||
delBatchResp.Deleted = 0
|
||||
s.DoJSON("POST", "/api/v1/fleet/queries/delete", map[string]interface{}{
|
||||
"ids": []uint{query.ID, query2.ID, query3.ID}}, http.StatusNotFound, &delBatchResp)
|
||||
"ids": []uint{query.ID, query2.ID, query3.ID},
|
||||
}, http.StatusNotFound, &delBatchResp)
|
||||
assert.Equal(t, uint(0), delBatchResp.Deleted)
|
||||
}
|
||||
|
||||
@ -2615,7 +2621,8 @@ func (s *integrationTestSuite) TestQuerySpecs() {
|
||||
// delete all queries created
|
||||
var delBatchResp deleteQueriesResponse
|
||||
s.DoJSON("POST", "/api/v1/fleet/queries/delete", map[string]interface{}{
|
||||
"ids": []uint{q1ID, q2ID, q3ID}}, http.StatusOK, &delBatchResp)
|
||||
"ids": []uint{q1ID, q2ID, q3ID},
|
||||
}, http.StatusOK, &delBatchResp)
|
||||
assert.Equal(t, uint(3), delBatchResp.Deleted)
|
||||
}
|
||||
|
||||
|
||||
@ -395,56 +395,80 @@ SELECT
|
||||
name AS name,
|
||||
version AS version,
|
||||
'Package (deb)' AS type,
|
||||
'deb_packages' AS source
|
||||
'deb_packages' AS source,
|
||||
'' AS release,
|
||||
'' AS vendor,
|
||||
'' AS arch
|
||||
FROM deb_packages
|
||||
UNION
|
||||
SELECT
|
||||
package AS name,
|
||||
version AS version,
|
||||
'Package (Portage)' AS type,
|
||||
'portage_packages' AS source
|
||||
'portage_packages' AS source,
|
||||
'' AS release,
|
||||
'' AS vendor,
|
||||
'' AS arch
|
||||
FROM portage_packages
|
||||
UNION
|
||||
SELECT
|
||||
name AS name,
|
||||
version AS version,
|
||||
'Package (RPM)' AS type,
|
||||
'rpm_packages' AS source
|
||||
'rpm_packages' AS source,
|
||||
release AS release,
|
||||
vendor AS vendor,
|
||||
arch AS arch
|
||||
FROM rpm_packages
|
||||
UNION
|
||||
SELECT
|
||||
name AS name,
|
||||
version AS version,
|
||||
'Package (NPM)' AS type,
|
||||
'npm_packages' AS source
|
||||
'npm_packages' AS source,
|
||||
'' AS release,
|
||||
'' AS vendor,
|
||||
'' AS arch
|
||||
FROM npm_packages
|
||||
UNION
|
||||
SELECT
|
||||
name AS name,
|
||||
version AS version,
|
||||
'Browser plugin (Chrome)' AS type,
|
||||
'chrome_extensions' AS source
|
||||
'chrome_extensions' AS source,
|
||||
'' AS release,
|
||||
'' AS vendor,
|
||||
'' AS arch
|
||||
FROM cached_users CROSS JOIN chrome_extensions USING (uid)
|
||||
UNION
|
||||
SELECT
|
||||
name AS name,
|
||||
version AS version,
|
||||
'Browser plugin (Firefox)' AS type,
|
||||
'firefox_addons' AS source
|
||||
'firefox_addons' AS source,
|
||||
'' AS release,
|
||||
'' AS vendor,
|
||||
'' AS arch
|
||||
FROM cached_users CROSS JOIN firefox_addons USING (uid)
|
||||
UNION
|
||||
SELECT
|
||||
name AS name,
|
||||
version AS version,
|
||||
'Package (Atom)' AS type,
|
||||
'atom_packages' AS source
|
||||
'atom_packages' AS source,
|
||||
'' AS release,
|
||||
'' AS vendor,
|
||||
'' AS arch
|
||||
FROM cached_users CROSS JOIN atom_packages USING (uid)
|
||||
UNION
|
||||
SELECT
|
||||
name AS name,
|
||||
version AS version,
|
||||
'Package (Python)' AS type,
|
||||
'python_packages' AS source
|
||||
'python_packages' AS source,
|
||||
'' AS release,
|
||||
'' AS vendor,
|
||||
'' AS arch
|
||||
FROM python_packages;
|
||||
`,
|
||||
Platforms: fleet.HostLinuxOSs,
|
||||
@ -649,11 +673,16 @@ func directIngestSoftware(ctx context.Context, logger log.Logger, host *fleet.Ho
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
s := fleet.Software{
|
||||
Name: name,
|
||||
Version: version,
|
||||
Source: source,
|
||||
BundleIdentifier: bundleIdentifier,
|
||||
|
||||
Release: row["release"],
|
||||
Vendor: row["vendor"],
|
||||
Arch: row["arch"],
|
||||
}
|
||||
software = append(software, s)
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@ -17,7 +18,6 @@ import (
|
||||
"github.com/fleetdm/fleet/v4/server/contexts/logging"
|
||||
"github.com/fleetdm/fleet/v4/server/fleet"
|
||||
"github.com/fleetdm/fleet/v4/server/ptr"
|
||||
|
||||
"github.com/fleetdm/fleet/v4/server/pubsub"
|
||||
"github.com/fleetdm/fleet/v4/server/service/osquery_utils"
|
||||
"github.com/go-kit/kit/log"
|
||||
@ -761,6 +761,8 @@ func (svc *Service) directIngestDetailQuery(ctx context.Context, host *fleet.Hos
|
||||
return false, nil
|
||||
}
|
||||
|
||||
var noSuchTableRegexp = regexp.MustCompile(`^no such table: \S+$`)
|
||||
|
||||
func (svc *Service) SubmitDistributedQueryResults(
|
||||
ctx context.Context,
|
||||
results fleet.OsqueryDistributedQueryResults,
|
||||
@ -787,6 +789,9 @@ func (svc *Service) SubmitDistributedQueryResults(
|
||||
// osquery docs say any nonzero (string) value for status indicates a query error
|
||||
status, ok := statuses[query]
|
||||
failed := ok && status != fleet.StatusOK
|
||||
if failed && messages[query] != "" && !noSuchTableRegexp.MatchString(messages[query]) {
|
||||
level.Debug(svc.logger).Log("query", query, "message", messages[query])
|
||||
}
|
||||
var err error
|
||||
switch {
|
||||
case strings.HasPrefix(query, hostDetailQueryPrefix):
|
||||
|
||||
92
server/vulnerabilities/centos.go
Normal file
92
server/vulnerabilities/centos.go
Normal file
@ -0,0 +1,92 @@
|
||||
package vulnerabilities
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"github.com/fleetdm/fleet/v4/server/config"
|
||||
"github.com/fleetdm/fleet/v4/server/fleet"
|
||||
"github.com/fleetdm/fleet/v4/server/vulnerabilities/vuln_centos"
|
||||
kitlog "github.com/go-kit/kit/log"
|
||||
"github.com/go-kit/kit/log/level"
|
||||
)
|
||||
|
||||
// centosPostProcessing performs processing over the list of vulnerable rpm packages
|
||||
// and removes the vulnerabilities where the CVEs are known to be fixed.
|
||||
func centosPostProcessing(
|
||||
ctx context.Context,
|
||||
ds fleet.Datastore,
|
||||
db *sql.DB,
|
||||
logger kitlog.Logger,
|
||||
config config.FleetConfig,
|
||||
) error {
|
||||
centOSPkgs, err := vuln_centos.LoadCentOSFixedCVEs(ctx, db, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch CentOS packages: %w", err)
|
||||
}
|
||||
level.Info(logger).Log("centosPackages", len(centOSPkgs))
|
||||
if len(centOSPkgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpmVulnerable, err := ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list vulnerable software: %w", err)
|
||||
}
|
||||
level.Info(logger).Log("vulnerable rpm_packages", len(rpmVulnerable))
|
||||
if len(rpmVulnerable) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var fixedCVEs []fleet.SoftwareVulnerability
|
||||
var softwareCount int
|
||||
for _, software := range rpmVulnerable {
|
||||
if software.Vendor != "CentOS" {
|
||||
continue
|
||||
}
|
||||
pkgFixedCVEs, ok := centOSPkgs[vuln_centos.CentOSPkg{
|
||||
Name: software.Name,
|
||||
Version: software.Version,
|
||||
Release: software.Release,
|
||||
Arch: software.Arch,
|
||||
}]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var cves []string
|
||||
for _, vulnerability := range software.Vulnerabilities {
|
||||
if _, ok := pkgFixedCVEs[vulnerability.CVE]; ok {
|
||||
cves = append(cves, vulnerability.CVE)
|
||||
fixedCVEs = append(fixedCVEs, fleet.SoftwareVulnerability{
|
||||
CPEID: software.CPEID,
|
||||
CVE: vulnerability.CVE,
|
||||
})
|
||||
}
|
||||
}
|
||||
if len(cves) > 0 {
|
||||
softwareCount++
|
||||
|
||||
level.Debug(logger).Log(
|
||||
"msg", "fixedCVEs",
|
||||
"software", fmt.Sprintf(
|
||||
"%s-%s-%s.%s",
|
||||
software.Name, software.Version, software.Release, software.Arch,
|
||||
),
|
||||
"softwareCPE", software.CPEID,
|
||||
"cves", fmt.Sprintf("%v", cves),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
level.Info(logger).Log(
|
||||
"msg", "CentOS fixed CVEs",
|
||||
"fixedCVEsCount", len(fixedCVEs),
|
||||
"distinctSoftwareCount", softwareCount,
|
||||
)
|
||||
|
||||
if err := ds.DeleteVulnerabilitiesByCPECVE(ctx, fixedCVEs); err != nil {
|
||||
return fmt.Errorf("failed to delete fixed vulnerabilities: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
152
server/vulnerabilities/centos_test.go
Normal file
152
server/vulnerabilities/centos_test.go
Normal file
@ -0,0 +1,152 @@
|
||||
package vulnerabilities
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
"github.com/fleetdm/fleet/v4/server/config"
|
||||
"github.com/fleetdm/fleet/v4/server/fleet"
|
||||
"github.com/fleetdm/fleet/v4/server/mock"
|
||||
"github.com/fleetdm/fleet/v4/server/vulnerabilities/vuln_centos"
|
||||
"github.com/go-kit/kit/log"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCentOSPostProcessing(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ds := new(mock.Store)
|
||||
|
||||
db, err := sql.Open("sqlite3", ":memory:")
|
||||
require.NoError(t, err)
|
||||
|
||||
pkgs := make(vuln_centos.CentOSPkgSet)
|
||||
authConfigPkg := vuln_centos.CentOSPkg{
|
||||
Name: "authconfig",
|
||||
Version: "6.2.8",
|
||||
Release: "30.el7",
|
||||
Arch: "x86_64",
|
||||
}
|
||||
pkgs.Add(authConfigPkg, "CVE-2017-7488")
|
||||
sqlitePkg := vuln_centos.CentOSPkg{
|
||||
Name: "sqlite",
|
||||
Version: "3.7.17",
|
||||
Release: "8.el7_7",
|
||||
Arch: "x86_64",
|
||||
}
|
||||
pkgs.Add(sqlitePkg, "CVE-2015-3415", "CVE-2015-3416", "CVE-2015-3414")
|
||||
|
||||
err = vuln_centos.GenCentOSSqlite(db, pkgs)
|
||||
require.NoError(t, err)
|
||||
|
||||
vulnSoftware := []fleet.SoftwareWithCPE{
|
||||
{
|
||||
Software: fleet.Software{
|
||||
Name: "authconfig",
|
||||
Version: "6.2.8",
|
||||
Release: "30.el7",
|
||||
Arch: "x86_64",
|
||||
Vendor: "CentOS",
|
||||
Vulnerabilities: fleet.VulnerabilitiesSlice{
|
||||
{
|
||||
CVE: "CVE-2017-7488",
|
||||
},
|
||||
},
|
||||
},
|
||||
CPEID: 1,
|
||||
},
|
||||
{
|
||||
Software: fleet.Software{
|
||||
Name: "sqlite",
|
||||
Version: "3.7.17",
|
||||
Release: "8.el7_7",
|
||||
Arch: "x86_64",
|
||||
Vendor: "CentOS",
|
||||
Vulnerabilities: fleet.VulnerabilitiesSlice{
|
||||
{
|
||||
CVE: "CVE-2015-3415",
|
||||
},
|
||||
{
|
||||
CVE: "CVE-2015-3416",
|
||||
},
|
||||
{
|
||||
CVE: "CVE-2022-9999",
|
||||
},
|
||||
},
|
||||
},
|
||||
CPEID: 2,
|
||||
},
|
||||
{
|
||||
Software: fleet.Software{
|
||||
Name: "ghostscript",
|
||||
Version: "9.25",
|
||||
Release: "5.el7",
|
||||
Arch: "x86_64",
|
||||
Vendor: "CentOS",
|
||||
Vulnerabilities: fleet.VulnerabilitiesSlice{
|
||||
{
|
||||
CVE: "CVE-2019-3835",
|
||||
},
|
||||
},
|
||||
},
|
||||
CPEID: 3,
|
||||
},
|
||||
{
|
||||
Software: fleet.Software{
|
||||
Name: "gnutls",
|
||||
Version: "3.3.29",
|
||||
Release: "9.el7",
|
||||
Arch: "x86_64",
|
||||
Vendor: "",
|
||||
Vulnerabilities: fleet.VulnerabilitiesSlice{
|
||||
{
|
||||
CVE: "CVE-8888-9999",
|
||||
},
|
||||
},
|
||||
},
|
||||
CPEID: 4,
|
||||
},
|
||||
}
|
||||
|
||||
ds.ListVulnerableSoftwareBySourceFunc = func(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) {
|
||||
return vulnSoftware, nil
|
||||
}
|
||||
|
||||
ds.DeleteVulnerabilitiesByCPECVEFunc = func(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error {
|
||||
require.Equal(t, []fleet.SoftwareVulnerability{
|
||||
{
|
||||
CPEID: 1,
|
||||
CVE: "CVE-2017-7488",
|
||||
},
|
||||
{
|
||||
CPEID: 2,
|
||||
CVE: "CVE-2015-3415",
|
||||
},
|
||||
{
|
||||
CPEID: 2,
|
||||
CVE: "CVE-2015-3416",
|
||||
},
|
||||
}, vulnerabilities)
|
||||
return nil
|
||||
}
|
||||
|
||||
err = centosPostProcessing(ctx, ds, db, log.NewNopLogger(), config.FleetConfig{})
|
||||
require.NoError(t, err)
|
||||
|
||||
require.True(t, ds.ListVulnerableSoftwareBySourceFuncInvoked)
|
||||
require.True(t, ds.DeleteVulnerabilitiesByCPECVEFuncInvoked)
|
||||
}
|
||||
|
||||
func TestCentOSPostProcessingNoPkgs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ds := new(mock.Store)
|
||||
ds.ListVulnerableSoftwareBySourceFunc = func(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) {
|
||||
t.Error("this method shouldn't be called if there are no pkgs in the CentOS table")
|
||||
return nil, nil
|
||||
}
|
||||
db, err := sql.Open("sqlite3", ":memory:")
|
||||
require.NoError(t, err)
|
||||
err = centosPostProcessing(ctx, ds, db, log.NewNopLogger(), config.FleetConfig{})
|
||||
require.Error(t, err)
|
||||
}
|
||||
@ -1,18 +1,18 @@
|
||||
package vulnerabilities
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/fleetdm/fleet/v4/pkg/download"
|
||||
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
|
||||
"github.com/fleetdm/fleet/v4/server/config"
|
||||
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
|
||||
@ -69,22 +69,39 @@ func GetLatestNVDRelease(client *http.Client) (*NVDRelease, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
type syncOpts struct {
|
||||
url string
|
||||
}
|
||||
|
||||
type CPESyncOption func(*syncOpts)
|
||||
|
||||
func WithCPEURL(url string) CPESyncOption {
|
||||
return func(o *syncOpts) {
|
||||
o.url = url
|
||||
}
|
||||
}
|
||||
|
||||
// SyncCPEDatabase (by default) downloads the CPE database from the
|
||||
// latest release of github.com/fleetdm/nvd to the given dbPath.
|
||||
// An alternative URL can be set via the WithCPEURL option.
|
||||
//
|
||||
// It won't sync the database at dbPath has an mtime that happened after the
|
||||
// available database release date.
|
||||
func SyncCPEDatabase(
|
||||
client *http.Client,
|
||||
dbPath string,
|
||||
config config.FleetConfig,
|
||||
opts ...CPESyncOption,
|
||||
) error {
|
||||
if config.Vulnerabilities.DisableDataSync {
|
||||
return nil
|
||||
var o syncOpts
|
||||
for _, fn := range opts {
|
||||
fn(&o)
|
||||
}
|
||||
|
||||
url := config.Vulnerabilities.CPEDatabaseURL
|
||||
if url == "" {
|
||||
if o.url == "" {
|
||||
nvdRelease, err := GetLatestNVDRelease(client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stat, err := os.Stat(dbPath)
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
@ -93,34 +110,14 @@ func SyncCPEDatabase(
|
||||
} else if !nvdRelease.CreatedAt.After(stat.ModTime()) {
|
||||
return nil
|
||||
}
|
||||
url = nvdRelease.CPEURL
|
||||
o.url = nvdRelease.CPEURL
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
u, err := url.Parse(o.url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
gr, err := gzip.NewReader(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gr.Close()
|
||||
|
||||
dbFile, err := os.Create(dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dbFile.Close()
|
||||
|
||||
_, err = io.Copy(dbFile, gr)
|
||||
if err != nil {
|
||||
if err := download.Decompressed(client, *u, dbPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -227,10 +224,12 @@ func TranslateSoftwareToCPE(
|
||||
) error {
|
||||
dbPath := path.Join(vulnPath, "cpe.sqlite")
|
||||
|
||||
if !config.Vulnerabilities.DisableDataSync {
|
||||
client := fleethttp.NewClient()
|
||||
if err := SyncCPEDatabase(client, dbPath, config); err != nil {
|
||||
if err := SyncCPEDatabase(client, dbPath, WithCPEURL(config.Vulnerabilities.CPEDatabaseURL)); err != nil {
|
||||
return ctxerr.Wrap(ctx, err, "sync cpe db")
|
||||
}
|
||||
}
|
||||
|
||||
iterator, err := ds.AllSoftwareWithoutCPEIterator(ctx)
|
||||
if err != nil {
|
||||
|
||||
@ -75,7 +75,7 @@ func TestSyncCPEDatabase(t *testing.T) {
|
||||
}
|
||||
|
||||
// first time, db doesn't exist, so it downloads
|
||||
err = SyncCPEDatabase(client, dbPath, config.FleetConfig{})
|
||||
err = SyncCPEDatabase(client, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := sqliteDB(dbPath)
|
||||
@ -107,7 +107,7 @@ func TestSyncCPEDatabase(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// then it will download
|
||||
err = SyncCPEDatabase(client, dbPath, config.FleetConfig{})
|
||||
err = SyncCPEDatabase(client, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// let's register the mtime for the db
|
||||
@ -128,9 +128,8 @@ func TestSyncCPEDatabase(t *testing.T) {
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
// let's check it doesn't download because it's new enough
|
||||
err = SyncCPEDatabase(client, dbPath, config.FleetConfig{})
|
||||
err = SyncCPEDatabase(client, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
stat, err = os.Stat(dbPath)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, mtime, stat.ModTime())
|
||||
@ -226,27 +225,10 @@ func TestSyncsCPEFromURL(t *testing.T) {
|
||||
dbPath := path.Join(tempDir, "cpe.sqlite")
|
||||
|
||||
err := SyncCPEDatabase(
|
||||
client, dbPath, config.FleetConfig{Vulnerabilities: config.VulnerabilitiesConfig{CPEDatabaseURL: ts.URL}})
|
||||
client, dbPath, WithCPEURL(ts.URL+"/hello-world.gz"))
|
||||
require.NoError(t, err)
|
||||
|
||||
stored, err := ioutil.ReadFile(dbPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Hello world!", string(stored))
|
||||
}
|
||||
|
||||
func TestSyncsCPESkipsIfDisableSync(t *testing.T) {
|
||||
client := fleethttp.NewClient()
|
||||
tempDir := t.TempDir()
|
||||
dbPath := path.Join(tempDir, "cpe.sqlite")
|
||||
|
||||
fleetConfig := config.FleetConfig{
|
||||
Vulnerabilities: config.VulnerabilitiesConfig{
|
||||
DisableDataSync: true,
|
||||
},
|
||||
}
|
||||
err := SyncCPEDatabase(client, dbPath, fleetConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = os.Stat(dbPath)
|
||||
require.ErrorIs(t, err, os.ErrNotExist)
|
||||
}
|
||||
|
||||
@ -2,9 +2,11 @@ package vulnerabilities
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
@ -243,3 +245,25 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger,
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// PostProcess performs additional processing over the results of
|
||||
// the main vulnerability processing run (TranslateSoftwareToCPE+TranslateCPEToCVE).
|
||||
func PostProcess(
|
||||
ctx context.Context,
|
||||
ds fleet.Datastore,
|
||||
vulnPath string,
|
||||
logger kitlog.Logger,
|
||||
config config.FleetConfig,
|
||||
) error {
|
||||
dbPath := path.Join(vulnPath, "cpe.sqlite")
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open cpe database: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := centosPostProcessing(ctx, ds, db, logger, config); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
451
server/vulnerabilities/vuln_centos/centos.go
Normal file
451
server/vulnerabilities/vuln_centos/centos.go
Normal file
@ -0,0 +1,451 @@
|
||||
// Package vuln_centos contains a ParseCentOSRepository method to parse the CentOS repository
|
||||
// to look out for CentOS releases that patch CVEs. It parses the changelogs from the metadata.
|
||||
//
|
||||
// It also contains a LoadCentOSFixedCVEs to load the results of the parsing.
|
||||
//
|
||||
// Both the parsing and loading of results use sqlite3 as backend storage.
|
||||
package vuln_centos
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/fleetdm/fleet/v4/pkg/download"
|
||||
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
|
||||
kitlog "github.com/go-kit/kit/log"
|
||||
"github.com/gocolly/colly"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// CentOSPkg holds data to identify a CentOS package.
|
||||
type CentOSPkg struct {
|
||||
Name string
|
||||
Version string
|
||||
Release string
|
||||
Arch string
|
||||
}
|
||||
|
||||
// String implements fmt.Stringer.
|
||||
func (p CentOSPkg) String() string {
|
||||
return p.Name + "-" + p.Version + "-" + p.Release + "." + p.Arch
|
||||
}
|
||||
|
||||
// FixedCVESet is a set of fixed CVEs.
|
||||
type FixedCVESet map[string]struct{}
|
||||
|
||||
// CentOSPkgSet is a set of CentOS packages and their fixed CVEs.
|
||||
type CentOSPkgSet map[CentOSPkg]FixedCVESet
|
||||
|
||||
// Add adds the given package and CVE/s to the set.
|
||||
func (p CentOSPkgSet) Add(pkg CentOSPkg, fixedCVEs ...string) {
|
||||
s := p[pkg]
|
||||
if s == nil {
|
||||
s = make(FixedCVESet)
|
||||
}
|
||||
for _, fixedCVE := range fixedCVEs {
|
||||
s[fixedCVE] = struct{}{}
|
||||
}
|
||||
p[pkg] = s
|
||||
}
|
||||
|
||||
const centOSPkgsCVEsTable = "centos_pkgs_fixed_cves"
|
||||
|
||||
// LoadCentOSFixedCVEs loads the CentOS packages with known fixed CVEs from the given sqlite3 db.
|
||||
func LoadCentOSFixedCVEs(ctx context.Context, db *sql.DB, logger kitlog.Logger) (CentOSPkgSet, error) {
|
||||
rows, err := db.QueryContext(ctx, fmt.Sprintf(`SELECT name, version, release, arch, cves FROM %s`, centOSPkgsCVEsTable))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch packages: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
pkgs := make(CentOSPkgSet)
|
||||
for rows.Next() {
|
||||
var pkg CentOSPkg
|
||||
var cves string
|
||||
if err := rows.Scan(&pkg.Name, &pkg.Version, &pkg.Release, &pkg.Arch, &cves); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, cve := range strings.Split(cves, ",") {
|
||||
pkgs.Add(pkg, "CVE-"+cve)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("failed to traverse packages: %w", err)
|
||||
}
|
||||
return pkgs, nil
|
||||
}
|
||||
|
||||
type centOSOpts struct {
|
||||
noCrawl bool
|
||||
verbose bool
|
||||
localDir string
|
||||
root string
|
||||
}
|
||||
|
||||
type CentOSOption func(*centOSOpts)
|
||||
|
||||
func WithLocalDir(dir string) CentOSOption {
|
||||
return func(o *centOSOpts) {
|
||||
o.localDir = dir
|
||||
}
|
||||
}
|
||||
|
||||
func NoCrawl() CentOSOption {
|
||||
return func(o *centOSOpts) {
|
||||
o.noCrawl = true
|
||||
}
|
||||
}
|
||||
|
||||
func WithVerbose(v bool) CentOSOption {
|
||||
return func(o *centOSOpts) {
|
||||
o.verbose = v
|
||||
}
|
||||
}
|
||||
|
||||
func WithRoot(root string) CentOSOption {
|
||||
return func(o *centOSOpts) {
|
||||
o.root = root
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
repositoryDomain = "mirror.centos.org"
|
||||
repositoryURL = "http://" + repositoryDomain
|
||||
defaultRoot = "/centos/"
|
||||
)
|
||||
|
||||
var (
|
||||
// Only parse the repository metadata for CentOS 6, 7 and 8.
|
||||
//
|
||||
// CentOS 6 maintenance updates ended in 2020-11-30, but we will still
|
||||
// fetch metadata for CentOS 6 because it's considered as of 2022-02-02
|
||||
// a "recent" release.
|
||||
//
|
||||
// See https://en.wikipedia.org/wiki/CentOS#CentOS_releases.
|
||||
recentCentOSPathRegex = regexp.MustCompile(`/centos/[678]\S*`)
|
||||
// nonReleasePathRegex is used to skip non-package centos directories/files.
|
||||
nonReleasePathRegex = regexp.MustCompile(`/centos/[^0-9]`)
|
||||
)
|
||||
|
||||
// ParseCentOSRepository performs the following operations:
|
||||
// - Crawls the CentOS repository website. To find all the sqlite3 files with
|
||||
// the packages metadata.
|
||||
// - Processes all the found sqlite3 files to find all fixed CVEs in each package version.
|
||||
// It parses the changelogs for each package release and looks for the "CVE-" string.
|
||||
//
|
||||
// It writes progress messages to stdout.
|
||||
func ParseCentOSRepository(opts ...CentOSOption) (CentOSPkgSet, error) {
|
||||
var opts_ centOSOpts
|
||||
for _, fn := range opts {
|
||||
fn(&opts_)
|
||||
}
|
||||
|
||||
if opts_.localDir == "" && opts_.noCrawl {
|
||||
return nil, errors.New("invalid options: if no crawl is set, local dir must be set")
|
||||
}
|
||||
|
||||
if opts_.localDir == "" {
|
||||
localDir, err := os.MkdirTemp("", "centos*")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts_.localDir = localDir
|
||||
}
|
||||
|
||||
fmt.Printf("Using local directory: %s\n", opts_.localDir)
|
||||
if !opts_.noCrawl {
|
||||
if err := crawl(opts_.root, opts_.localDir, opts_.verbose); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
pkgs, err := parse(opts_.localDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opts_.verbose {
|
||||
for pkg, cves := range pkgs {
|
||||
var cveList []string
|
||||
for cve := range cves {
|
||||
cveList = append(cveList, cve)
|
||||
}
|
||||
if opts_.verbose {
|
||||
fmt.Printf("%s: %v\n", pkg, cveList)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pkgs, nil
|
||||
}
|
||||
|
||||
func crawl(root string, localDir string, verbose bool) error {
|
||||
fmt.Println("Crawling CentOS repository...")
|
||||
c := colly.NewCollector()
|
||||
|
||||
if err := os.MkdirAll(localDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var repoMDs []url.URL
|
||||
c.OnHTML("#indexlist .indexcolname a[href]", func(e *colly.HTMLElement) {
|
||||
href := e.Attr("href")
|
||||
// Skip going to parent directory.
|
||||
if strings.HasPrefix(root, href) {
|
||||
return
|
||||
}
|
||||
if nonReleasePathRegex.MatchString(path.Join(e.Request.URL.Path, href)) {
|
||||
return
|
||||
}
|
||||
if !recentCentOSPathRegex.MatchString(path.Join(e.Request.URL.Path, href)) {
|
||||
if verbose {
|
||||
fmt.Printf("Ignoring old release: %s\n", path.Join(e.Request.URL.Path, href))
|
||||
}
|
||||
return
|
||||
}
|
||||
if href == "repomd.xml" {
|
||||
u := *e.Request.URL
|
||||
u.Path = path.Join(u.Path, href)
|
||||
repoMDs = append(repoMDs, u)
|
||||
if verbose {
|
||||
fmt.Printf("%s\n", u.Path)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !strings.Contains(href, "/") {
|
||||
return
|
||||
}
|
||||
e.Request.Visit(href)
|
||||
})
|
||||
|
||||
c.AllowedDomains = append(c.AllowedDomains, repositoryDomain)
|
||||
|
||||
if root == "" {
|
||||
root = defaultRoot
|
||||
}
|
||||
if err := c.Visit(repositoryURL + root); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, u := range repoMDs {
|
||||
if err := processRepoMD(u, localDir, verbose); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type dbs struct {
|
||||
primary, other string
|
||||
}
|
||||
|
||||
func parse(localDir string) (CentOSPkgSet, error) {
|
||||
fmt.Println("Processing sqlite files...")
|
||||
|
||||
dbPaths := make(map[string]dbs)
|
||||
if err := filepath.WalkDir(localDir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".sqlite") {
|
||||
return nil
|
||||
}
|
||||
dbp := dbPaths[filepath.Dir(path)]
|
||||
if strings.HasSuffix(path, "-primary.sqlite") {
|
||||
dbp.primary = path
|
||||
} else if strings.HasSuffix(path, "-other.sqlite") {
|
||||
dbp.other = path
|
||||
}
|
||||
dbPaths[filepath.Dir(path)] = dbp
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
allPkgs := make(CentOSPkgSet)
|
||||
for _, db := range dbPaths {
|
||||
pkgs, err := processSqlites(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for pkg, cves := range pkgs {
|
||||
for cve := range cves {
|
||||
allPkgs.Add(pkg, cve)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return allPkgs, nil
|
||||
}
|
||||
|
||||
func processRepoMD(mdURL url.URL, localDir string, verbose bool) error {
|
||||
resp, err := http.Get(mdURL.String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
b, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type location struct {
|
||||
Href string `xml:"href,attr"`
|
||||
}
|
||||
type repoDataItem struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Location location `xml:"location"`
|
||||
}
|
||||
type repoMetadata struct {
|
||||
XMLName xml.Name `xml:"repomd"`
|
||||
Datas []repoDataItem `xml:"data"`
|
||||
}
|
||||
var md repoMetadata
|
||||
if err := xml.Unmarshal(b, &md); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, data := range md.Datas {
|
||||
if data.Type != "primary_db" && data.Type != "other_db" {
|
||||
continue
|
||||
}
|
||||
sqliteURL := mdURL
|
||||
sqliteURL.Path = strings.TrimSuffix(sqliteURL.Path, "repomd.xml") + strings.TrimPrefix(data.Location.Href, "repodata/")
|
||||
if verbose {
|
||||
fmt.Printf("%s\n", sqliteURL.Path)
|
||||
}
|
||||
filePath := filePathfromURL(localDir, sqliteURL)
|
||||
_, err := os.Stat(filePath)
|
||||
switch {
|
||||
case err == nil:
|
||||
// File already exists, nothing to do.
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
if err := download.Decompressed(fleethttp.NewClient(), sqliteURL, filePath); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func filePathfromURL(dir string, url url.URL) string {
|
||||
filePath := filepath.Join(dir, url.Path)
|
||||
filePath = strings.TrimSuffix(filePath, ".bz2")
|
||||
filePath = strings.TrimSuffix(filePath, ".xz")
|
||||
filePath = strings.TrimSuffix(filePath, ".gz")
|
||||
return filePath
|
||||
}
|
||||
|
||||
func processSqlites(dbPaths dbs) (CentOSPkgSet, error) {
|
||||
db, err := sql.Open("sqlite3", dbPaths.primary)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if _, err := db.Exec(fmt.Sprintf("ATTACH DATABASE '%s' as other;", dbPaths.other)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
db.SetMaxOpenConns(1)
|
||||
rows, err := db.Query(`SELECT
|
||||
p.name, p.version, p.release, p.arch, c.changelog
|
||||
FROM packages p
|
||||
JOIN other.changelog c ON (p.pkgKey=c.pkgKey)
|
||||
WHERE c.changelog LIKE '%CVE-%-%';`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
pkgs := make(CentOSPkgSet)
|
||||
for rows.Next() {
|
||||
var p CentOSPkg
|
||||
var changelog string
|
||||
if err := rows.Scan(&p.Name, &p.Version, &p.Release, &p.Arch, &changelog); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cves := parseCVEs(changelog)
|
||||
for _, cve := range cves {
|
||||
pkgs.Add(p, cve)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pkgs, nil
|
||||
}
|
||||
|
||||
var cveRegex = regexp.MustCompile(`CVE\-[0-9]+\-[0-9]+`)
|
||||
|
||||
func parseCVEs(changelog string) []string {
|
||||
return cveRegex.FindAllString(changelog, -1)
|
||||
}
|
||||
|
||||
// GenCentOSSqlite will store the CentOS package set in the given sqlite handle.
|
||||
func GenCentOSSqlite(db *sql.DB, pkgs CentOSPkgSet) error {
|
||||
if err := createTable(db); err != nil {
|
||||
return err
|
||||
}
|
||||
type pkgWithCVEs struct {
|
||||
pkg CentOSPkg
|
||||
cves string
|
||||
}
|
||||
var pkgsWithCVEs []pkgWithCVEs
|
||||
for pkg, cves := range pkgs {
|
||||
var cveList []string
|
||||
for cve := range cves {
|
||||
cveList = append(cveList, strings.TrimPrefix(cve, "CVE-"))
|
||||
}
|
||||
sort.Slice(cveList, func(i, j int) bool {
|
||||
return cveList[i] < cveList[j]
|
||||
})
|
||||
pkgsWithCVEs = append(pkgsWithCVEs, pkgWithCVEs{
|
||||
pkg: pkg,
|
||||
cves: strings.Join(cveList, ","),
|
||||
})
|
||||
}
|
||||
for _, pkgWithCVEs := range pkgsWithCVEs {
|
||||
if _, err := db.Exec(
|
||||
fmt.Sprintf("REPLACE INTO %s (name, version, release, arch, cves) VALUES (?, ?, ?, ?, ?)", centOSPkgsCVEsTable),
|
||||
pkgWithCVEs.pkg.Name,
|
||||
pkgWithCVEs.pkg.Version,
|
||||
pkgWithCVEs.pkg.Release,
|
||||
pkgWithCVEs.pkg.Arch,
|
||||
pkgWithCVEs.cves,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func createTable(db *sql.DB) error {
|
||||
_, err := db.Exec(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
name TEXT,
|
||||
version TEXT,
|
||||
release TEXT,
|
||||
arch TEXT,
|
||||
cves TEXT,
|
||||
|
||||
UNIQUE (name, version, release, arch)
|
||||
);`, centOSPkgsCVEsTable))
|
||||
return err
|
||||
}
|
||||
94
server/vulnerabilities/vuln_centos/centos_test.go
Normal file
94
server/vulnerabilities/vuln_centos/centos_test.go
Normal file
@ -0,0 +1,94 @@
|
||||
package vuln_centos
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/go-kit/kit/log"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCentOSPkgSetAdd(t *testing.T) {
|
||||
pkgSet := make(CentOSPkgSet)
|
||||
authConfig := CentOSPkg{
|
||||
Name: "authconfig",
|
||||
Version: "6.2.8",
|
||||
Release: "30.el7",
|
||||
Arch: "x86_64",
|
||||
}
|
||||
cve1 := "CVE-2017-7488"
|
||||
pkgSet.Add(authConfig, cve1)
|
||||
|
||||
cve2 := "CVE-2017-7489"
|
||||
pkgSet.Add(authConfig, cve2)
|
||||
|
||||
curl := CentOSPkg{
|
||||
Name: "curl",
|
||||
Version: "4.2",
|
||||
Release: "30.el7",
|
||||
Arch: "x86_64",
|
||||
}
|
||||
cve3 := "CVE-2017-7490"
|
||||
pkgSet.Add(curl, cve1)
|
||||
pkgSet.Add(curl, cve3)
|
||||
|
||||
require.Len(t, pkgSet, 2)
|
||||
|
||||
require.Len(t, pkgSet[authConfig], 2)
|
||||
require.Contains(t, pkgSet[authConfig], cve1)
|
||||
require.Contains(t, pkgSet[authConfig], cve2)
|
||||
require.NotContains(t, pkgSet[authConfig], cve3)
|
||||
|
||||
require.Len(t, pkgSet[curl], 2)
|
||||
require.Contains(t, pkgSet[curl], cve1)
|
||||
require.NotContains(t, pkgSet[curl], cve2)
|
||||
require.Contains(t, pkgSet[curl], cve3)
|
||||
}
|
||||
|
||||
func TestParseCentOSRepository(t *testing.T) {
|
||||
if os.Getenv("NETWORK_TEST") == "" {
|
||||
t.Skip("set environment variable NETWORK_TEST=1 to run")
|
||||
}
|
||||
|
||||
// Parse a subset of the CentOS repository.
|
||||
pkgs, err := ParseCentOSRepository(WithRoot("/centos/7/os/x86_64/repodata/"))
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := sql.Open("sqlite3", ":memory:")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
db.Close()
|
||||
})
|
||||
|
||||
err = GenCentOSSqlite(db, pkgs)
|
||||
require.NoError(t, err)
|
||||
|
||||
pkgSet, err := LoadCentOSFixedCVEs(context.Background(), db, log.NewNopLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
// Shouldn't get _lower_ than what was reported during the development of this test (2221),
|
||||
// as these are already published releases.
|
||||
require.GreaterOrEqual(t, len(pkgSet), 2221)
|
||||
for pkg, cveSet := range pkgSet {
|
||||
require.NotEmpty(t, pkg.Name)
|
||||
require.NotEmpty(t, pkg.Version)
|
||||
require.NotEmpty(t, pkg.Release)
|
||||
require.NotEmpty(t, pkg.Arch)
|
||||
require.NotEmpty(t, cveSet)
|
||||
}
|
||||
|
||||
// Check a known vulnerability fixed on a CentOS release.
|
||||
authConfig := CentOSPkg{
|
||||
Name: "authconfig",
|
||||
Version: "6.2.8",
|
||||
Release: "30.el7",
|
||||
Arch: "x86_64",
|
||||
}
|
||||
cve := "CVE-2017-7488"
|
||||
require.Contains(t, pkgSet, authConfig)
|
||||
require.Len(t, pkgSet[authConfig], 1)
|
||||
require.Contains(t, pkgSet[authConfig], cve)
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user