diff --git a/changes/issue-4585-geolocation-support b/changes/issue-4585-geolocation-support new file mode 100644 index 000000000..c8652af35 --- /dev/null +++ b/changes/issue-4585-geolocation-support @@ -0,0 +1,3 @@ +* Add support for geolocation via public IP +* Add public_ip to host table (default empty string) +* Add public_ip to host(s) API response \ No newline at end of file diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 03d9f7671..5ef2ac91e 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -331,10 +331,21 @@ the way that the Fleet server works. defer sentry.Flush(2 * time.Second) } + var geoIP fleet.GeoIP + geoIP = &fleet.NoOpGeoIP{} + if config.GeoIP.DatabasePath != "" { + maxmind, err := fleet.NewMaxMindGeoIP(logger, config.GeoIP.DatabasePath) + if err != nil { + level.Error(logger).Log("msg", "failed to initialize maxmind geoip, check database path", "database_path", config.GeoIP.DatabasePath, "error", err) + } else { + geoIP = maxmind + } + } + // TODO: gather all the different contexts and use just one ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() - svc, err := service.NewService(ctx, ds, task, resultStore, logger, osqueryLogger, config, mailService, clock.C, ssoSessionStore, liveQueryStore, carveStore, *license, failingPolicySet) + svc, err := service.NewService(ctx, ds, task, resultStore, logger, osqueryLogger, config, mailService, clock.C, ssoSessionStore, liveQueryStore, carveStore, *license, failingPolicySet, geoIP) if err != nil { initFatal(err, "initializing service") } diff --git a/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json b/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json index cd17788d0..15fc6a499 100644 --- a/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json +++ b/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json @@ -31,6 +31,7 @@ "hardware_version":"", "hardware_serial":"", "computer_name":"test_host", + "public_ip": "", "primary_ip":"", "primary_mac":"", "distributed_interval":0, diff --git a/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml b/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml index c3f6e4adc..3877c95f2 100644 --- a/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml +++ b/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml @@ -63,6 +63,7 @@ spec: created_at: "0001-01-01T00:00:00Z" updated_at: "0001-01-01T00:00:00Z" policy_updated_at: "0001-01-01T00:00:00Z" + public_ip: "" primary_ip: "" primary_mac: "" refetch_requested: false diff --git a/cmd/fleetctl/testdata/expectedListHostsJson.json b/cmd/fleetctl/testdata/expectedListHostsJson.json index 4c2431176..57a938710 100644 --- a/cmd/fleetctl/testdata/expectedListHostsJson.json +++ b/cmd/fleetctl/testdata/expectedListHostsJson.json @@ -31,6 +31,7 @@ "hardware_version":"", "hardware_serial":"", "computer_name":"test_host", + "public_ip": "", "primary_ip":"", "primary_mac":"", "distributed_interval":0, @@ -90,6 +91,7 @@ "hardware_version":"", "hardware_serial":"", "computer_name":"test_host2", + "public_ip": "", "primary_ip":"", "primary_mac":"", "distributed_interval":0, diff --git a/cmd/fleetctl/testdata/expectedListHostsYaml.yml b/cmd/fleetctl/testdata/expectedListHostsYaml.yml index 6fa6cb502..72da4ba9b 100644 --- a/cmd/fleetctl/testdata/expectedListHostsYaml.yml +++ b/cmd/fleetctl/testdata/expectedListHostsYaml.yml @@ -40,6 +40,7 @@ spec: platform: "" platform_like: "" policy_updated_at: "0001-01-01T00:00:00Z" + public_ip: "" primary_ip: "" primary_mac: "" refetch_requested: false @@ -88,6 +89,7 @@ spec: platform: "" platform_like: "" policy_updated_at: "0001-01-01T00:00:00Z" + public_ip: "" primary_ip: "" primary_mac: "" refetch_requested: false diff --git a/docs/Deploying/Configuration.md b/docs/Deploying/Configuration.md index 1e2db92dd..71c455895 100644 --- a/docs/Deploying/Configuration.md +++ b/docs/Deploying/Configuration.md @@ -1942,6 +1942,25 @@ To download the data streams, you can use `fleetctl vulnerability-data-stream -- vulnerabilities: disable_data_sync: true ``` + +### GeoIP + +##### database_path + +The path to a valid Maxmind GeoIP database(mmdb). Support exists for the country & city versions of the database. If city database is supplied +then Fleet will attempt to resolve the location via the city lookup, otherwise it defaults to the country lookup. The IP address used +to determine location is extracted via HTTP headers in the following order: `True-Client-IP`, `X-Real-IP`, and finally `X-FORWARDED-FOR` [headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For) +on the Fleet web server. + +- Default value: none +- Environment variable: `FLEET_GEOIP_DATABASE_PATH` +- Config file format: + + ```yaml + geoip: + database_path: /some/path + ``` + ## Managing osquery configurations diff --git a/docs/Using-Fleet/REST-API.md b/docs/Using-Fleet/REST-API.md index cb2112fc2..4897ee609 100644 --- a/docs/Using-Fleet/REST-API.md +++ b/docs/Using-Fleet/REST-API.md @@ -536,6 +536,7 @@ If `additional_info_filters` is not specified, no `additional` information will "hardware_version": "", "hardware_serial": "", "computer_name": "2ceca32fe484", + "public_ip": "", "primary_ip": "", "primary_mac": "", "distributed_interval": 10, @@ -733,6 +734,7 @@ If the scheduled queries haven't run on the host yet, the stats have zero values "hardware_version": "", "hardware_serial": "", "computer_name": "23cfc9caacf0", + "public_ip": "", "primary_ip": "172.27.0.6", "primary_mac": "02:42:ac:1b:00:06", "distributed_interval": 10, diff --git a/frontend/interfaces/host.ts b/frontend/interfaces/host.ts index 9c663eaef..4343c76a2 100644 --- a/frontend/interfaces/host.ts +++ b/frontend/interfaces/host.ts @@ -106,6 +106,15 @@ export interface IHostPolicyQueryError { error: string; } +interface IGeoLocation { + country_iso: string; + city_name: string; + geometry?: { + type: string; + coordinates: number[]; + }; +} + export interface IHost { created_at: string; updated_at: string; @@ -134,6 +143,7 @@ export interface IHost { hardware_version: string; hardware_serial: string; computer_name: string; + public_ip: string; primary_ip: string; primary_mac: string; distributed_interval: number; @@ -161,4 +171,5 @@ export interface IHost { mdm?: IMDMData; policies: IHostPolicy[]; query_results?: []; + geolocation?: IGeoLocation; } diff --git a/frontend/pages/hosts/HostDetailsPage/HostDetailsPage.tsx b/frontend/pages/hosts/HostDetailsPage/HostDetailsPage.tsx index 71bd94d39..5a682a61b 100644 --- a/frontend/pages/hosts/HostDetailsPage/HostDetailsPage.tsx +++ b/frontend/pages/hosts/HostDetailsPage/HostDetailsPage.tsx @@ -395,6 +395,7 @@ const HostDetailsPage = ({ "hardware_model", "hardware_serial", "primary_ip", + "public_ip", ]) ); @@ -1122,6 +1123,22 @@ const HostDetailsPage = ({ ) : null; }; + const renderGeolocation = () => { + if (!host?.geolocation) { + return null; + } + const { geolocation } = host; + const location = [geolocation?.city_name, geolocation?.country_iso] + .filter(Boolean) + .join(", "); + return ( +
+ Location + {location} +
+ ); + }; + if (isLoadingHost) { return ; } @@ -1240,14 +1257,19 @@ const HostDetailsPage = ({
- IPv4 + Internal IP address {aboutData.primary_ip}
+
+ Public IP address + {aboutData.public_ip} +
{renderMunkiData()} {renderMdmData()} {renderDeviceUser()} + {renderGeolocation()}
diff --git a/go.mod b/go.mod index bb8835475..8b2341f60 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/ghodss/yaml v1.0.0 github.com/go-kit/kit v0.9.0 github.com/go-sql-driver/mysql v1.6.0 - github.com/gocarina/gocsv v0.0.0-20220310154401-d4df709ca055 // indirect + github.com/gocarina/gocsv v0.0.0-20220310154401-d4df709ca055 github.com/gocolly/colly v1.2.0 github.com/golang-jwt/jwt/v4 v4.0.0 github.com/gomodule/redigo v1.8.5 @@ -66,12 +66,12 @@ require ( github.com/oklog/run v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/open-policy-agent/opa v0.24.0 + github.com/oschwald/geoip2-golang v1.6.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect - github.com/quasilyte/go-ruleguard/dsl v0.3.17 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/rotisserie/eris v0.5.1 github.com/rs/zerolog v1.20.0 diff --git a/go.sum b/go.sum index 09c871af9..574d63860 100644 --- a/go.sum +++ b/go.sum @@ -391,8 +391,6 @@ github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fleetdm/goose v0.0.0-20210209032905-c3c01484bacb h1:p02npmJlTo+Px1s0VptKOJOJqH/rGlGBEVvLJRtzY3A= -github.com/fleetdm/goose v0.0.0-20210209032905-c3c01484bacb/go.mod h1:d7Q+0eCENnKQUhkfAUVLfGnD4QcgJMF/uB9WRTN9TDI= github.com/fleetdm/goose v0.0.0-20220214194029-91b5e5eb8e77 h1:oaRSVdXLGFxX0aQa5UI8GDr6+lRiscSM40B6zl8oUKI= github.com/fleetdm/goose v0.0.0-20220214194029-91b5e5eb8e77/go.mod h1:d7Q+0eCENnKQUhkfAUVLfGnD4QcgJMF/uB9WRTN9TDI= github.com/flynn/go-docopt v0.0.0-20140912013429-f6dd2ebbb31e/go.mod h1:HyVoz1Mz5Co8TFO8EupIdlcpwShBmY98dkT2xeHkvEI= @@ -938,6 +936,10 @@ github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je4 github.com/open-policy-agent/opa v0.24.0 h1:fnGOIux+TTGZsC0du1bRBtV8F+KPN55Hks12uE3Fq3E= github.com/open-policy-agent/opa v0.24.0/go.mod h1:qEyD/i8j+RQettHGp4f86yjrjvv+ZYia+JHCMv2G7wA= github.com/opencensus-integrations/ocsql v0.1.1/go.mod h1:ozPYpNVBHZsX33jfoQPO5TlI5lqh0/3R36kirEqJKAM= +github.com/oschwald/geoip2-golang v1.6.1 h1:GKxT3yaWWNXSb7vj6D7eoJBns+lGYgx08QO0UcNm0YY= +github.com/oschwald/geoip2-golang v1.6.1/go.mod h1:xdvYt5xQzB8ORWFqPnqMwZpCpgNagttWdoZLlJQzg7s= +github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph9p/UMXK/Hk= +github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= @@ -988,8 +990,6 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/quasilyte/go-ruleguard/dsl v0.3.17 h1:L5xf3nifnRIdYe9vyMuY2sDnZHIgQol/fDq74FQz7ZY= -github.com/quasilyte/go-ruleguard/dsl v0.3.17/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -1161,7 +1161,6 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/zclconf/go-cty v1.1.0 h1:uJwc9HiBOCpoKIObTQaLR+tsEXx1HBHnOsOOpcdhZgw= github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c h1:TWQ2UvXPkhPxI2KmApKBOCaV6yD2N4mlvqFQ/DlPtpQ= github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c/go.mod h1:OYYulo9tUqRadRLwB0+LE914sa1ui2yL7OrcU3Q/1XY= @@ -1431,6 +1430,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/server/config/config.go b/server/config/config.go index 79f717f68..8ac90bdaa 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -233,6 +233,10 @@ type SentryConfig struct { Dsn string `json:"dsn"` } +type GeoIPConfig struct { + DatabasePath string `json:"database_path" yaml:"database_path"` +} + // FleetConfig stores the application configuration. Each subcategory is // broken up into it's own struct, defined above. When editing any of these // structs, Manager.addConfigs and Manager.LoadConfig should be @@ -258,6 +262,7 @@ type FleetConfig struct { Vulnerabilities VulnerabilitiesConfig Upgrades UpgradesConfig Sentry SentryConfig + GeoIP GeoIPConfig } type TLS struct { @@ -554,6 +559,9 @@ func (man Manager) addConfigs() { // Sentry man.addConfigString("sentry.dsn", "", "DSN for Sentry") + + // GeoIP + man.addConfigString("geoip.database_path", "", "path to mmdb file") } // LoadConfig will load the config variables into a fully initialized @@ -734,6 +742,9 @@ func (man Manager) LoadConfig() FleetConfig { Sentry: SentryConfig{ Dsn: man.getConfigString("sentry.dsn"), }, + GeoIP: GeoIPConfig{ + DatabasePath: man.getConfigString("geoip.database_path"), + }, } } diff --git a/server/contexts/publicip/publicip.go b/server/contexts/publicip/publicip.go new file mode 100644 index 000000000..b7a1332ce --- /dev/null +++ b/server/contexts/publicip/publicip.go @@ -0,0 +1,23 @@ +package publicip + +import ( + "context" +) + +type key int + +const ipKey key = 0 + +// NewContext returns a new context carrying the current remote ip. +func NewContext(ctx context.Context, ip string) context.Context { + return context.WithValue(ctx, ipKey, ip) +} + +// FromContext extracts the remote ip from context if present. +func FromContext(ctx context.Context) string { + ip, ok := ctx.Value(ipKey).(string) + if !ok { + return "" + } + return ip +} diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 0476d2444..2aceee7f5 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -1567,6 +1567,7 @@ func (ds *Datastore) UpdateHost(ctx context.Context, host *fleet.Host) error { team_id = ?, primary_ip = ?, primary_mac = ?, + public_ip = ?, refetch_requested = ?, gigs_disk_space_available = ?, percent_disk_space_available = ? @@ -1603,6 +1604,7 @@ func (ds *Datastore) UpdateHost(ctx context.Context, host *fleet.Host) error { host.TeamID, host.PrimaryIP, host.PrimaryMac, + host.PublicIP, host.RefetchRequested, host.GigsDiskSpaceAvailable, host.PercentDiskSpaceAvailable, diff --git a/server/datastore/mysql/migrations/tables/20220316155700_AddPublicIPHosts.go b/server/datastore/mysql/migrations/tables/20220316155700_AddPublicIPHosts.go new file mode 100644 index 000000000..d88561479 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20220316155700_AddPublicIPHosts.go @@ -0,0 +1,25 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20220316155700, Down_20220316155700) +} + +func Up_20220316155700(tx *sql.Tx) error { + _, err := tx.Exec( + "ALTER TABLE `hosts` ADD COLUMN `public_ip` varchar(45) NOT NULL DEFAULT ''", + ) + if err != nil { + return errors.Wrap(err, "add public_ip column") + } + + return nil +} +func Down_20220316155700(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index e7e6a61dc..a884e3fad 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -240,6 +240,7 @@ CREATE TABLE `hosts` ( `gigs_disk_space_available` float NOT NULL DEFAULT '0', `percent_disk_space_available` float NOT NULL DEFAULT '0', `policy_updated_at` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', + `public_ip` varchar(45) NOT NULL DEFAULT '', PRIMARY KEY (`id`), UNIQUE KEY `idx_osquery_host_id` (`osquery_host_id`), UNIQUE KEY `idx_host_unique_nodekey` (`node_key`), @@ -327,9 +328,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=126 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=127 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220316155700,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `network_interfaces` ( diff --git a/server/fleet/geoip.go b/server/fleet/geoip.go new file mode 100644 index 000000000..ef93cf9cf --- /dev/null +++ b/server/fleet/geoip.go @@ -0,0 +1,89 @@ +package fleet + +import ( + "context" + "errors" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/oschwald/geoip2-golang" + "net" +) + +var notCityDBError = geoip2.InvalidMethodError{} + +type GeoLocation struct { + CountryISO string `json:"country_iso"` + CityName string `json:"city_name"` + Geometry *Geometry `json:"geometry,omitempty"` +} + +type Geometry struct { + Type string `json:"type"` + Coordinates []float64 `json:"coordinates"` +} + +type GeoIP interface { + Lookup(ctx context.Context, ip string) *GeoLocation +} + +type MaxMindGeoIP struct { + reader *geoip2.Reader + l log.Logger +} + +type NoOpGeoIP struct{} + +func (n *NoOpGeoIP) Lookup(ctx context.Context, ip string) *GeoLocation { + return nil +} + +func NewMaxMindGeoIP(logger log.Logger, path string) (*MaxMindGeoIP, error) { + r, err := geoip2.Open(path) + if err != nil { + return nil, err + } + return &MaxMindGeoIP{reader: r, l: logger}, nil +} + +func (m *MaxMindGeoIP) Lookup(ctx context.Context, ip string) *GeoLocation { + if ip == "" { + return nil + } + // City has location data, so we'll start there first + parseIP := net.ParseIP(ip) + if parseIP == nil { + return nil + } + resp, err := m.reader.City(parseIP) + if err != nil && errors.Is(err, notCityDBError) { + resp, err := m.reader.Country(parseIP) + if err != nil { + level.Debug(m.l).Log("err", err, "msg", "failed to lookup location from mmdb file") + return nil + } + if resp == nil { + return nil + } + // all we have is country iso, no geometry + return &GeoLocation{CountryISO: resp.Country.IsoCode} + } + if err != nil { + level.Debug(m.l).Log("err", err, "msg", "failed to lookup location from mmdb file") + return nil + } + return parseCity(resp) +} + +func parseCity(resp *geoip2.City) *GeoLocation { + if resp == nil { + return nil + } + return &GeoLocation{ + CountryISO: resp.Country.IsoCode, + CityName: resp.City.Names["en"], // names is a map of language to city name names["us"] = "New York" + Geometry: &Geometry{ + Type: "Point", + Coordinates: []float64{resp.Location.Latitude, resp.Location.Longitude}, + }, + } +} diff --git a/server/fleet/hosts.go b/server/fleet/hosts.go index b88352724..182f380ef 100644 --- a/server/fleet/hosts.go +++ b/server/fleet/hosts.go @@ -108,6 +108,7 @@ type Host struct { // can be found in the NetworkInterfaces element with the same ip_address. PrimaryNetworkInterfaceID *uint `json:"primary_ip_id,omitempty" db:"primary_ip_id" csv:"primary_ip_id"` NetworkInterfaces []*NetworkInterface `json:"-" db:"-" csv:"-"` + PublicIP string `json:"public_ip" db:"public_ip" csv:"public_ip"` PrimaryIP string `json:"primary_ip" db:"primary_ip" csv:"primary_ip"` PrimaryMac string `json:"primary_mac" db:"primary_mac" csv:"primary_mac"` DistributedInterval uint `json:"distributed_interval" db:"distributed_interval" csv:"distributed_interval"` diff --git a/server/fleet/service.go b/server/fleet/service.go index 34da61a14..fe536d0f2 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -449,4 +449,7 @@ type Service interface { DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) ModifyTeamPolicy(ctx context.Context, teamID uint, id uint, p ModifyPolicyPayload) (*Policy, error) GetTeamPolicyByIDQueries(ctx context.Context, teamID uint, policyID uint) (*Policy, error) + + /// Geolocation + LookupGeoIP(ctx context.Context, ip string) *GeoLocation } diff --git a/server/service/handler.go b/server/service/handler.go index 4c2dfef5a..df2099215 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -8,6 +8,7 @@ import ( "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/contexts/logging" + "github.com/fleetdm/fleet/v4/server/contexts/publicip" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/service/middleware/authzcheck" "github.com/fleetdm/fleet/v4/server/service/middleware/ratelimit" @@ -79,6 +80,7 @@ func MakeHandler(svc fleet.Service, config config.FleetConfig, logger kitlog.Log fleetAPIOptions := []kithttp.ServerOption{ kithttp.ServerBefore( kithttp.PopulateRequestContext, // populate the request context with common fields + setRequestsContexts(svc), ), kithttp.ServerErrorHandler(&errorHandler{logger}), @@ -95,6 +97,8 @@ func MakeHandler(svc fleet.Service, config config.FleetConfig, logger kitlog.Log r.Use(otmiddleware.Middleware("fleet")) } + r.Use(publicIP) + attachFleetAPIRoutes(r, svc, config, logger, limitStore, fleetAPIOptions) // Results endpoint is handled different due to websockets use @@ -109,6 +113,16 @@ func MakeHandler(svc fleet.Service, config config.FleetConfig, logger kitlog.Log return r } +func publicIP(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ip := extractIP(r) + if ip != "" { + r.RemoteAddr = ip + } + handler.ServeHTTP(w, r.WithContext(publicip.NewContext(r.Context(), ip))) + }) +} + // InstrumentHandler wraps the provided handler with prometheus metrics // middleware and returns the resulting handler that should be mounted for that // route. diff --git a/server/service/hosts.go b/server/service/hosts.go index dc179883e..2807f1f78 100644 --- a/server/service/hosts.go +++ b/server/service/hosts.go @@ -20,9 +20,10 @@ import ( // rendering in the UI. type HostResponse struct { *fleet.Host - Status fleet.HostStatus `json:"status"` - DisplayText string `json:"display_text"` - Labels []fleet.Label `json:"labels,omitempty"` + Status fleet.HostStatus `json:"status"` + DisplayText string `json:"display_text"` + Labels []fleet.Label `json:"labels,omitempty"` + Geolocation *fleet.GeoLocation `json:"geolocation,omitempty"` } func hostResponseForHost(ctx context.Context, svc fleet.Service, host *fleet.Host) (*HostResponse, error) { @@ -30,6 +31,7 @@ func hostResponseForHost(ctx context.Context, svc fleet.Service, host *fleet.Hos Host: host, Status: host.Status(time.Now()), DisplayText: host.Hostname, + Geolocation: svc.LookupGeoIP(ctx, host.PublicIP), }, nil } @@ -37,8 +39,9 @@ func hostResponseForHost(ctx context.Context, svc fleet.Service, host *fleet.Hos // with the HostDetail details. type HostDetailResponse struct { fleet.HostDetail - Status fleet.HostStatus `json:"status"` - DisplayText string `json:"display_text"` + Status fleet.HostStatus `json:"status"` + DisplayText string `json:"display_text"` + Geolocation *fleet.GeoLocation `json:"geolocation,omitempty"` } func hostDetailResponseForHost(ctx context.Context, svc fleet.Service, host *fleet.HostDetail) (*HostDetailResponse, error) { @@ -46,6 +49,7 @@ func hostDetailResponseForHost(ctx context.Context, svc fleet.Service, host *fle HostDetail: *host, Status: host.Status(time.Now()), DisplayText: host.Hostname, + Geolocation: svc.LookupGeoIP(ctx, host.PublicIP), }, nil } diff --git a/server/service/http_publicip.go b/server/service/http_publicip.go new file mode 100644 index 000000000..bba6eaa44 --- /dev/null +++ b/server/service/http_publicip.go @@ -0,0 +1,30 @@ +package service + +import ( + "net/http" + "strings" +) + +// copied from https://github.com/go-chi/chi/blob/c97bc988430d623a14f50b7019fb40529036a35a/middleware/realip.go#L42 + +var trueClientIP = http.CanonicalHeaderKey("True-Client-IP") +var xForwardedFor = http.CanonicalHeaderKey("X-Forwarded-For") +var xRealIP = http.CanonicalHeaderKey("X-Real-IP") + +func extractIP(r *http.Request) string { + var ip string + + if tcip := r.Header.Get(trueClientIP); tcip != "" { + ip = tcip + } else if xrip := r.Header.Get(xRealIP); xrip != "" { + ip = xrip + } else if xff := r.Header.Get(xForwardedFor); xff != "" { + i := strings.Index(xff, ",") + if i == -1 { + i = len(xff) + } + ip = xff[:i] + } + + return ip +} diff --git a/server/service/osquery.go b/server/service/osquery.go index 1d3d60e3a..39e459fb0 100644 --- a/server/service/osquery.go +++ b/server/service/osquery.go @@ -139,21 +139,21 @@ func (svc *Service) EnrollAgent(ctx context.Context, enrollSecret, hostIdentifie detailQueries := osquery_utils.GetDetailQueries(appConfig, svc.config) save := false if r, ok := hostDetails["os_version"]; ok { - err := detailQueries["os_version"].IngestFunc(svc.logger, host, []map[string]string{r}) + err := detailQueries["os_version"].IngestFunc(ctx, svc.logger, host, []map[string]string{r}) if err != nil { return "", ctxerr.Wrap(ctx, err, "Ingesting os_version") } save = true } if r, ok := hostDetails["osquery_info"]; ok { - err := detailQueries["osquery_info"].IngestFunc(svc.logger, host, []map[string]string{r}) + err := detailQueries["osquery_info"].IngestFunc(ctx, svc.logger, host, []map[string]string{r}) if err != nil { return "", ctxerr.Wrap(ctx, err, "Ingesting osquery_info") } save = true } if r, ok := hostDetails["system_info"]; ok { - err := detailQueries["system_info"].IngestFunc(svc.logger, host, []map[string]string{r}) + err := detailQueries["system_info"].IngestFunc(ctx, svc.logger, host, []map[string]string{r}) if err != nil { return "", ctxerr.Wrap(ctx, err, "Ingesting system_info") } @@ -1033,7 +1033,7 @@ func (svc *Service) ingestDetailQuery(ctx context.Context, host *fleet.Host, nam } if query.IngestFunc != nil { - err = query.IngestFunc(svc.logger, host, rows) + err = query.IngestFunc(ctx, svc.logger, host, rows) if err != nil { return osqueryError{ message: fmt.Sprintf("ingesting query %s: %s", name, err.Error()), diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go index c6acf8815..3d8ff2361 100644 --- a/server/service/osquery_utils/queries.go +++ b/server/service/osquery_utils/queries.go @@ -10,6 +10,7 @@ import ( "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/contexts/publicip" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -27,7 +28,7 @@ type DetailQuery struct { Platforms []string // IngestFunc translates a query result into an update to the host struct, // around data that lives on the hosts table. - IngestFunc func(logger log.Logger, host *fleet.Host, rows []map[string]string) error + IngestFunc func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error // DirectIngestFunc gathers results from a query and directly works with the datastore to // persist them. This is usually used for host data that is stored in a separate table. DirectIngestFunc func(ctx context.Context, logger log.Logger, host *fleet.Host, ds fleet.Datastore, rows []map[string]string, failed bool) error @@ -55,7 +56,7 @@ var detailQueries = map[string]DetailQuery{ from interface_details id join interface_addresses ia on ia.interface = id.interface where length(mac) > 0 order by (ibytes + obytes) desc`, - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) (err error) { + IngestFunc: func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) (err error) { if len(rows) == 0 { logger.Log("component", "service", "method", "IngestFunc", "err", "detail_query_network_interface expected 1 or more results") @@ -105,12 +106,13 @@ var detailQueries = map[string]DetailQuery{ host.PrimaryIP = selected["address"] host.PrimaryMac = selected["mac"] + host.PublicIP = publicip.FromContext(ctx) return nil }, }, "os_version": { Query: "select * from os_version limit 1", - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + IngestFunc: func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error { if len(rows) != 1 { logger.Log("component", "service", "method", "IngestFunc", "err", fmt.Sprintf("detail_query_os_version expected single result got %d", len(rows))) @@ -158,7 +160,7 @@ var detailQueries = map[string]DetailQuery{ // distributed_interval (but it's not required), and typically // do not control config_tls_refresh. Query: `select name, value from osquery_flags where name in ("distributed_interval", "config_tls_refresh", "config_refresh", "logger_tls_period")`, - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + IngestFunc: func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error { var configTLSRefresh, configRefresh uint var configRefreshSeen, configTLSRefreshSeen bool for _, row := range rows { @@ -215,7 +217,7 @@ var detailQueries = map[string]DetailQuery{ }, "osquery_info": { Query: "select * from osquery_info limit 1", - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + IngestFunc: func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error { if len(rows) != 1 { logger.Log("component", "service", "method", "IngestFunc", "err", fmt.Sprintf("detail_query_osquery_info expected single result got %d", len(rows))) @@ -229,7 +231,7 @@ var detailQueries = map[string]DetailQuery{ }, "system_info": { Query: "select * from system_info limit 1", - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + IngestFunc: func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error { if len(rows) != 1 { logger.Log("component", "service", "method", "IngestFunc", "err", fmt.Sprintf("detail_query_system_info expected single result got %d", len(rows))) @@ -264,7 +266,7 @@ var detailQueries = map[string]DetailQuery{ }, "uptime": { Query: "select * from uptime limit 1", - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + IngestFunc: func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error { if len(rows) != 1 { logger.Log("component", "service", "method", "IngestFunc", "err", fmt.Sprintf("detail_query_uptime expected single result got %d", len(rows))) @@ -745,7 +747,7 @@ func directIngestUsers(ctx context.Context, logger log.Logger, host *fleet.Host, return nil } -func ingestDiskSpace(logger log.Logger, host *fleet.Host, rows []map[string]string) error { +func ingestDiskSpace(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error { if len(rows) != 1 { logger.Log("component", "service", "method", "ingestDiskSpace", "err", fmt.Sprintf("detail_query_disk_space expected single result got %d", len(rows))) diff --git a/server/service/osquery_utils/queries_test.go b/server/service/osquery_utils/queries_test.go index 110093d1f..346be9063 100644 --- a/server/service/osquery_utils/queries_test.go +++ b/server/service/osquery_utils/queries_test.go @@ -22,7 +22,7 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) { ingest := GetDetailQueries(nil, config.FleetConfig{})["network_interface"].IngestFunc - assert.NoError(t, ingest(log.NewNopLogger(), &host, nil)) + assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, nil)) assert.Equal(t, initialHost, host) var rows []map[string]string @@ -39,7 +39,7 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) { &rows, )) - assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) + assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, rows)) assert.Equal(t, "192.168.1.3", host.PrimaryIP) assert.Equal(t, "f4:5d:79:93:58:5b", host.PrimaryMac) @@ -57,7 +57,7 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) { &rows, )) - assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) + assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, rows)) assert.Equal(t, "2604:3f08:1337:9411:cbe:814f:51a6:e4e3", host.PrimaryIP) assert.Equal(t, "27:1b:aa:60:e8:0a", host.PrimaryMac) @@ -76,7 +76,7 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) { &rows, )) - assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) + assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, rows)) assert.Equal(t, "205.111.43.79", host.PrimaryIP) assert.Equal(t, "ab:1b:aa:60:e8:0a", host.PrimaryMac) @@ -93,7 +93,7 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) { &rows, )) - assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) + assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, rows)) assert.Equal(t, "127.0.0.1", host.PrimaryIP) assert.Equal(t, "00:00:00:00:00:00", host.PrimaryMac) } @@ -323,7 +323,7 @@ func TestDetailQuerysOSVersion(t *testing.T) { ingest := GetDetailQueries(nil, config.FleetConfig{})["os_version"].IngestFunc - assert.NoError(t, ingest(log.NewNopLogger(), &host, nil)) + assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, nil)) assert.Equal(t, initialHost, host) // Rolling release for archlinux @@ -345,7 +345,7 @@ func TestDetailQuerysOSVersion(t *testing.T) { &rows, )) - assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) + assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, rows)) assert.Equal(t, "Arch Linux rolling", host.OSVersion) // Simulate a linux with a proper version @@ -366,7 +366,7 @@ func TestDetailQuerysOSVersion(t *testing.T) { &rows, )) - assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) + assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, rows)) assert.Equal(t, "Arch Linux 1.2.3", host.OSVersion) } diff --git a/server/service/service.go b/server/service/service.go index 895d66d37..a6c5ee3c1 100644 --- a/server/service/service.go +++ b/server/service/service.go @@ -19,6 +19,8 @@ import ( kitlog "github.com/go-kit/kit/log" ) +var _ fleet.Service = (*Service)(nil) + // Service is the struct implementing fleet.Service. Create a new one with NewService. type Service struct { ds fleet.Datastore @@ -44,6 +46,12 @@ type Service struct { jitterMu *sync.Mutex jitterH map[time.Duration]*jitterHashTable + + geoIP fleet.GeoIP +} + +func (s *Service) LookupGeoIP(ctx context.Context, ip string) *fleet.GeoLocation { + return s.geoIP.Lookup(ctx, ip) } // NewService creates a new service from the config struct @@ -62,6 +70,7 @@ func NewService( carveStore fleet.CarveStore, license fleet.LicenseInfo, failingPolicySet fleet.FailingPolicySet, + geoIP fleet.GeoIP, ) (fleet.Service, error) { authorizer, err := authz.NewAuthorizer() if err != nil { @@ -86,6 +95,7 @@ func NewService( authz: authorizer, jitterH: make(map[time.Duration]*jitterHashTable), jitterMu: new(sync.Mutex), + geoIP: geoIP, } return validationMiddleware{svc, ds, sso}, nil } diff --git a/server/service/testing_utils.go b/server/service/testing_utils.go index 1b02a8e26..6735f7951 100644 --- a/server/service/testing_utils.go +++ b/server/service/testing_utils.go @@ -69,7 +69,7 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf Datastore: ds, AsyncEnabled: false, } - svc, err := NewService(context.Background(), ds, task, rs, logger, osqlogger, fleetConfig, mailer, c, ssoStore, lq, ds, *license, failingPolicySet) + svc, err := NewService(context.Background(), ds, task, rs, logger, osqlogger, fleetConfig, mailer, c, ssoStore, lq, ds, *license, failingPolicySet, &fleet.NoOpGeoIP{}) if err != nil { panic(err) }