Add public ip to hosts & derive geolocation when rendering host (#4652)

* geoip wip
* return nil if ip is empty string or if ParseIP returns nil
* add ui component to render geolocation if available, address PR feedback
* render public ip if available
* add changes file, document geoip in deployment guide
* update rest-api docs
This commit is contained in:
Benjamin Edwards 2022-03-21 12:29:52 -04:00 committed by GitHub
parent 1164330bd4
commit 74bb559645
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 326 additions and 37 deletions

View File

@ -0,0 +1,3 @@
* Add support for geolocation via public IP
* Add public_ip to host table (default empty string)
* Add public_ip to host(s) API response

View File

@ -331,10 +331,21 @@ the way that the Fleet server works.
defer sentry.Flush(2 * time.Second)
}
var geoIP fleet.GeoIP
geoIP = &fleet.NoOpGeoIP{}
if config.GeoIP.DatabasePath != "" {
maxmind, err := fleet.NewMaxMindGeoIP(logger, config.GeoIP.DatabasePath)
if err != nil {
level.Error(logger).Log("msg", "failed to initialize maxmind geoip, check database path", "database_path", config.GeoIP.DatabasePath, "error", err)
} else {
geoIP = maxmind
}
}
// TODO: gather all the different contexts and use just one
ctx, cancelFunc := context.WithCancel(context.Background())
defer cancelFunc()
svc, err := service.NewService(ctx, ds, task, resultStore, logger, osqueryLogger, config, mailService, clock.C, ssoSessionStore, liveQueryStore, carveStore, *license, failingPolicySet)
svc, err := service.NewService(ctx, ds, task, resultStore, logger, osqueryLogger, config, mailService, clock.C, ssoSessionStore, liveQueryStore, carveStore, *license, failingPolicySet, geoIP)
if err != nil {
initFatal(err, "initializing service")
}

View File

@ -31,6 +31,7 @@
"hardware_version":"",
"hardware_serial":"",
"computer_name":"test_host",
"public_ip": "",
"primary_ip":"",
"primary_mac":"",
"distributed_interval":0,

View File

@ -63,6 +63,7 @@ spec:
created_at: "0001-01-01T00:00:00Z"
updated_at: "0001-01-01T00:00:00Z"
policy_updated_at: "0001-01-01T00:00:00Z"
public_ip: ""
primary_ip: ""
primary_mac: ""
refetch_requested: false

View File

@ -31,6 +31,7 @@
"hardware_version":"",
"hardware_serial":"",
"computer_name":"test_host",
"public_ip": "",
"primary_ip":"",
"primary_mac":"",
"distributed_interval":0,
@ -90,6 +91,7 @@
"hardware_version":"",
"hardware_serial":"",
"computer_name":"test_host2",
"public_ip": "",
"primary_ip":"",
"primary_mac":"",
"distributed_interval":0,

View File

@ -40,6 +40,7 @@ spec:
platform: ""
platform_like: ""
policy_updated_at: "0001-01-01T00:00:00Z"
public_ip: ""
primary_ip: ""
primary_mac: ""
refetch_requested: false
@ -88,6 +89,7 @@ spec:
platform: ""
platform_like: ""
policy_updated_at: "0001-01-01T00:00:00Z"
public_ip: ""
primary_ip: ""
primary_mac: ""
refetch_requested: false

View File

@ -1943,6 +1943,25 @@ To download the data streams, you can use `fleetctl vulnerability-data-stream --
disable_data_sync: true
```
### GeoIP
##### database_path
The path to a valid Maxmind GeoIP database(mmdb). Support exists for the country & city versions of the database. If city database is supplied
then Fleet will attempt to resolve the location via the city lookup, otherwise it defaults to the country lookup. The IP address used
to determine location is extracted via HTTP headers in the following order: `True-Client-IP`, `X-Real-IP`, and finally `X-FORWARDED-FOR` [headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For)
on the Fleet web server.
- Default value: none
- Environment variable: `FLEET_GEOIP_DATABASE_PATH`
- Config file format:
```yaml
geoip:
database_path: /some/path
```
## Managing osquery configurations
We recommend that you use an infrastructure configuration management tool to manage these osquery configurations consistently across your environment. If you're unsure about what configuration management tools your organization uses, contact your company's system administrators. If you are evaluating new solutions for this problem, the founders of Fleet have successfully managed configurations in large production environments using [Chef](https://www.chef.io/chef/) and [Puppet](https://puppet.com/).

View File

@ -536,6 +536,7 @@ If `additional_info_filters` is not specified, no `additional` information will
"hardware_version": "",
"hardware_serial": "",
"computer_name": "2ceca32fe484",
"public_ip": "",
"primary_ip": "",
"primary_mac": "",
"distributed_interval": 10,
@ -733,6 +734,7 @@ If the scheduled queries haven't run on the host yet, the stats have zero values
"hardware_version": "",
"hardware_serial": "",
"computer_name": "23cfc9caacf0",
"public_ip": "",
"primary_ip": "172.27.0.6",
"primary_mac": "02:42:ac:1b:00:06",
"distributed_interval": 10,

View File

@ -106,6 +106,15 @@ export interface IHostPolicyQueryError {
error: string;
}
interface IGeoLocation {
country_iso: string;
city_name: string;
geometry?: {
type: string;
coordinates: number[];
};
}
export interface IHost {
created_at: string;
updated_at: string;
@ -134,6 +143,7 @@ export interface IHost {
hardware_version: string;
hardware_serial: string;
computer_name: string;
public_ip: string;
primary_ip: string;
primary_mac: string;
distributed_interval: number;
@ -161,4 +171,5 @@ export interface IHost {
mdm?: IMDMData;
policies: IHostPolicy[];
query_results?: [];
geolocation?: IGeoLocation;
}

View File

@ -395,6 +395,7 @@ const HostDetailsPage = ({
"hardware_model",
"hardware_serial",
"primary_ip",
"public_ip",
])
);
@ -1122,6 +1123,22 @@ const HostDetailsPage = ({
) : null;
};
const renderGeolocation = () => {
if (!host?.geolocation) {
return null;
}
const { geolocation } = host;
const location = [geolocation?.city_name, geolocation?.country_iso]
.filter(Boolean)
.join(", ");
return (
<div className="info-grid__block">
<span className="info-grid__header">Location</span>
<span className="info-grid__data">{location}</span>
</div>
);
};
if (isLoadingHost) {
return <Spinner />;
}
@ -1240,14 +1257,19 @@ const HostDetailsPage = ({
</span>
</div>
<div className="info-grid__block">
<span className="info-grid__header">IPv4</span>
<span className="info-grid__header">Internal IP address</span>
<span className="info-grid__data">
{aboutData.primary_ip}
</span>
</div>
<div className="info-grid__block">
<span className="info-grid__header">Public IP address</span>
<span className="info-grid__data">{aboutData.public_ip}</span>
</div>
{renderMunkiData()}
{renderMdmData()}
{renderDeviceUser()}
{renderGeolocation()}
</div>
</div>
<div className="col-2">

4
go.mod
View File

@ -33,7 +33,7 @@ require (
github.com/ghodss/yaml v1.0.0
github.com/go-kit/kit v0.9.0
github.com/go-sql-driver/mysql v1.6.0
github.com/gocarina/gocsv v0.0.0-20220310154401-d4df709ca055 // indirect
github.com/gocarina/gocsv v0.0.0-20220310154401-d4df709ca055
github.com/gocolly/colly v1.2.0
github.com/golang-jwt/jwt/v4 v4.0.0
github.com/gomodule/redigo v1.8.5
@ -66,12 +66,12 @@ require (
github.com/oklog/run v1.1.0
github.com/olekukonko/tablewriter v0.0.5
github.com/open-policy-agent/opa v0.24.0
github.com/oschwald/geoip2-golang v1.6.1
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/quasilyte/go-ruleguard/dsl v0.3.17 // indirect
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
github.com/rotisserie/eris v0.5.1
github.com/rs/zerolog v1.20.0

10
go.sum
View File

@ -391,8 +391,6 @@ github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o=
github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/fleetdm/goose v0.0.0-20210209032905-c3c01484bacb h1:p02npmJlTo+Px1s0VptKOJOJqH/rGlGBEVvLJRtzY3A=
github.com/fleetdm/goose v0.0.0-20210209032905-c3c01484bacb/go.mod h1:d7Q+0eCENnKQUhkfAUVLfGnD4QcgJMF/uB9WRTN9TDI=
github.com/fleetdm/goose v0.0.0-20220214194029-91b5e5eb8e77 h1:oaRSVdXLGFxX0aQa5UI8GDr6+lRiscSM40B6zl8oUKI=
github.com/fleetdm/goose v0.0.0-20220214194029-91b5e5eb8e77/go.mod h1:d7Q+0eCENnKQUhkfAUVLfGnD4QcgJMF/uB9WRTN9TDI=
github.com/flynn/go-docopt v0.0.0-20140912013429-f6dd2ebbb31e/go.mod h1:HyVoz1Mz5Co8TFO8EupIdlcpwShBmY98dkT2xeHkvEI=
@ -938,6 +936,10 @@ github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je4
github.com/open-policy-agent/opa v0.24.0 h1:fnGOIux+TTGZsC0du1bRBtV8F+KPN55Hks12uE3Fq3E=
github.com/open-policy-agent/opa v0.24.0/go.mod h1:qEyD/i8j+RQettHGp4f86yjrjvv+ZYia+JHCMv2G7wA=
github.com/opencensus-integrations/ocsql v0.1.1/go.mod h1:ozPYpNVBHZsX33jfoQPO5TlI5lqh0/3R36kirEqJKAM=
github.com/oschwald/geoip2-golang v1.6.1 h1:GKxT3yaWWNXSb7vj6D7eoJBns+lGYgx08QO0UcNm0YY=
github.com/oschwald/geoip2-golang v1.6.1/go.mod h1:xdvYt5xQzB8ORWFqPnqMwZpCpgNagttWdoZLlJQzg7s=
github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph9p/UMXK/Hk=
github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
@ -988,8 +990,6 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/quasilyte/go-ruleguard/dsl v0.3.17 h1:L5xf3nifnRIdYe9vyMuY2sDnZHIgQol/fDq74FQz7ZY=
github.com/quasilyte/go-ruleguard/dsl v0.3.17/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
@ -1161,7 +1161,6 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/zclconf/go-cty v1.1.0 h1:uJwc9HiBOCpoKIObTQaLR+tsEXx1HBHnOsOOpcdhZgw=
github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c h1:TWQ2UvXPkhPxI2KmApKBOCaV6yD2N4mlvqFQ/DlPtpQ=
github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c/go.mod h1:OYYulo9tUqRadRLwB0+LE914sa1ui2yL7OrcU3Q/1XY=
@ -1431,6 +1430,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

View File

@ -233,6 +233,10 @@ type SentryConfig struct {
Dsn string `json:"dsn"`
}
type GeoIPConfig struct {
DatabasePath string `json:"database_path" yaml:"database_path"`
}
// FleetConfig stores the application configuration. Each subcategory is
// broken up into it's own struct, defined above. When editing any of these
// structs, Manager.addConfigs and Manager.LoadConfig should be
@ -258,6 +262,7 @@ type FleetConfig struct {
Vulnerabilities VulnerabilitiesConfig
Upgrades UpgradesConfig
Sentry SentryConfig
GeoIP GeoIPConfig
}
type TLS struct {
@ -554,6 +559,9 @@ func (man Manager) addConfigs() {
// Sentry
man.addConfigString("sentry.dsn", "", "DSN for Sentry")
// GeoIP
man.addConfigString("geoip.database_path", "", "path to mmdb file")
}
// LoadConfig will load the config variables into a fully initialized
@ -734,6 +742,9 @@ func (man Manager) LoadConfig() FleetConfig {
Sentry: SentryConfig{
Dsn: man.getConfigString("sentry.dsn"),
},
GeoIP: GeoIPConfig{
DatabasePath: man.getConfigString("geoip.database_path"),
},
}
}

View File

@ -0,0 +1,23 @@
package publicip
import (
"context"
)
type key int
const ipKey key = 0
// NewContext returns a new context carrying the current remote ip.
func NewContext(ctx context.Context, ip string) context.Context {
return context.WithValue(ctx, ipKey, ip)
}
// FromContext extracts the remote ip from context if present.
func FromContext(ctx context.Context) string {
ip, ok := ctx.Value(ipKey).(string)
if !ok {
return ""
}
return ip
}

View File

@ -1567,6 +1567,7 @@ func (ds *Datastore) UpdateHost(ctx context.Context, host *fleet.Host) error {
team_id = ?,
primary_ip = ?,
primary_mac = ?,
public_ip = ?,
refetch_requested = ?,
gigs_disk_space_available = ?,
percent_disk_space_available = ?
@ -1603,6 +1604,7 @@ func (ds *Datastore) UpdateHost(ctx context.Context, host *fleet.Host) error {
host.TeamID,
host.PrimaryIP,
host.PrimaryMac,
host.PublicIP,
host.RefetchRequested,
host.GigsDiskSpaceAvailable,
host.PercentDiskSpaceAvailable,

View File

@ -0,0 +1,25 @@
package tables
import (
"database/sql"
"github.com/pkg/errors"
)
func init() {
MigrationClient.AddMigration(Up_20220316155700, Down_20220316155700)
}
func Up_20220316155700(tx *sql.Tx) error {
_, err := tx.Exec(
"ALTER TABLE `hosts` ADD COLUMN `public_ip` varchar(45) NOT NULL DEFAULT ''",
)
if err != nil {
return errors.Wrap(err, "add public_ip column")
}
return nil
}
func Down_20220316155700(tx *sql.Tx) error {
return nil
}

File diff suppressed because one or more lines are too long

89
server/fleet/geoip.go Normal file
View File

@ -0,0 +1,89 @@
package fleet
import (
"context"
"errors"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/oschwald/geoip2-golang"
"net"
)
var notCityDBError = geoip2.InvalidMethodError{}
type GeoLocation struct {
CountryISO string `json:"country_iso"`
CityName string `json:"city_name"`
Geometry *Geometry `json:"geometry,omitempty"`
}
type Geometry struct {
Type string `json:"type"`
Coordinates []float64 `json:"coordinates"`
}
type GeoIP interface {
Lookup(ctx context.Context, ip string) *GeoLocation
}
type MaxMindGeoIP struct {
reader *geoip2.Reader
l log.Logger
}
type NoOpGeoIP struct{}
func (n *NoOpGeoIP) Lookup(ctx context.Context, ip string) *GeoLocation {
return nil
}
func NewMaxMindGeoIP(logger log.Logger, path string) (*MaxMindGeoIP, error) {
r, err := geoip2.Open(path)
if err != nil {
return nil, err
}
return &MaxMindGeoIP{reader: r, l: logger}, nil
}
func (m *MaxMindGeoIP) Lookup(ctx context.Context, ip string) *GeoLocation {
if ip == "" {
return nil
}
// City has location data, so we'll start there first
parseIP := net.ParseIP(ip)
if parseIP == nil {
return nil
}
resp, err := m.reader.City(parseIP)
if err != nil && errors.Is(err, notCityDBError) {
resp, err := m.reader.Country(parseIP)
if err != nil {
level.Debug(m.l).Log("err", err, "msg", "failed to lookup location from mmdb file")
return nil
}
if resp == nil {
return nil
}
// all we have is country iso, no geometry
return &GeoLocation{CountryISO: resp.Country.IsoCode}
}
if err != nil {
level.Debug(m.l).Log("err", err, "msg", "failed to lookup location from mmdb file")
return nil
}
return parseCity(resp)
}
func parseCity(resp *geoip2.City) *GeoLocation {
if resp == nil {
return nil
}
return &GeoLocation{
CountryISO: resp.Country.IsoCode,
CityName: resp.City.Names["en"], // names is a map of language to city name names["us"] = "New York"
Geometry: &Geometry{
Type: "Point",
Coordinates: []float64{resp.Location.Latitude, resp.Location.Longitude},
},
}
}

View File

@ -108,6 +108,7 @@ type Host struct {
// can be found in the NetworkInterfaces element with the same ip_address.
PrimaryNetworkInterfaceID *uint `json:"primary_ip_id,omitempty" db:"primary_ip_id" csv:"primary_ip_id"`
NetworkInterfaces []*NetworkInterface `json:"-" db:"-" csv:"-"`
PublicIP string `json:"public_ip" db:"public_ip" csv:"public_ip"`
PrimaryIP string `json:"primary_ip" db:"primary_ip" csv:"primary_ip"`
PrimaryMac string `json:"primary_mac" db:"primary_mac" csv:"primary_mac"`
DistributedInterval uint `json:"distributed_interval" db:"distributed_interval" csv:"distributed_interval"`

View File

@ -449,4 +449,7 @@ type Service interface {
DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error)
ModifyTeamPolicy(ctx context.Context, teamID uint, id uint, p ModifyPolicyPayload) (*Policy, error)
GetTeamPolicyByIDQueries(ctx context.Context, teamID uint, policyID uint) (*Policy, error)
/// Geolocation
LookupGeoIP(ctx context.Context, ip string) *GeoLocation
}

View File

@ -8,6 +8,7 @@ import (
"github.com/fleetdm/fleet/v4/server/config"
"github.com/fleetdm/fleet/v4/server/contexts/logging"
"github.com/fleetdm/fleet/v4/server/contexts/publicip"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/service/middleware/authzcheck"
"github.com/fleetdm/fleet/v4/server/service/middleware/ratelimit"
@ -79,6 +80,7 @@ func MakeHandler(svc fleet.Service, config config.FleetConfig, logger kitlog.Log
fleetAPIOptions := []kithttp.ServerOption{
kithttp.ServerBefore(
kithttp.PopulateRequestContext, // populate the request context with common fields
setRequestsContexts(svc),
),
kithttp.ServerErrorHandler(&errorHandler{logger}),
@ -95,6 +97,8 @@ func MakeHandler(svc fleet.Service, config config.FleetConfig, logger kitlog.Log
r.Use(otmiddleware.Middleware("fleet"))
}
r.Use(publicIP)
attachFleetAPIRoutes(r, svc, config, logger, limitStore, fleetAPIOptions)
// Results endpoint is handled different due to websockets use
@ -109,6 +113,16 @@ func MakeHandler(svc fleet.Service, config config.FleetConfig, logger kitlog.Log
return r
}
func publicIP(handler http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ip := extractIP(r)
if ip != "" {
r.RemoteAddr = ip
}
handler.ServeHTTP(w, r.WithContext(publicip.NewContext(r.Context(), ip)))
})
}
// InstrumentHandler wraps the provided handler with prometheus metrics
// middleware and returns the resulting handler that should be mounted for that
// route.

View File

@ -20,9 +20,10 @@ import (
// rendering in the UI.
type HostResponse struct {
*fleet.Host
Status fleet.HostStatus `json:"status"`
DisplayText string `json:"display_text"`
Labels []fleet.Label `json:"labels,omitempty"`
Status fleet.HostStatus `json:"status"`
DisplayText string `json:"display_text"`
Labels []fleet.Label `json:"labels,omitempty"`
Geolocation *fleet.GeoLocation `json:"geolocation,omitempty"`
}
func hostResponseForHost(ctx context.Context, svc fleet.Service, host *fleet.Host) (*HostResponse, error) {
@ -30,6 +31,7 @@ func hostResponseForHost(ctx context.Context, svc fleet.Service, host *fleet.Hos
Host: host,
Status: host.Status(time.Now()),
DisplayText: host.Hostname,
Geolocation: svc.LookupGeoIP(ctx, host.PublicIP),
}, nil
}
@ -37,8 +39,9 @@ func hostResponseForHost(ctx context.Context, svc fleet.Service, host *fleet.Hos
// with the HostDetail details.
type HostDetailResponse struct {
fleet.HostDetail
Status fleet.HostStatus `json:"status"`
DisplayText string `json:"display_text"`
Status fleet.HostStatus `json:"status"`
DisplayText string `json:"display_text"`
Geolocation *fleet.GeoLocation `json:"geolocation,omitempty"`
}
func hostDetailResponseForHost(ctx context.Context, svc fleet.Service, host *fleet.HostDetail) (*HostDetailResponse, error) {
@ -46,6 +49,7 @@ func hostDetailResponseForHost(ctx context.Context, svc fleet.Service, host *fle
HostDetail: *host,
Status: host.Status(time.Now()),
DisplayText: host.Hostname,
Geolocation: svc.LookupGeoIP(ctx, host.PublicIP),
}, nil
}

View File

@ -0,0 +1,30 @@
package service
import (
"net/http"
"strings"
)
// copied from https://github.com/go-chi/chi/blob/c97bc988430d623a14f50b7019fb40529036a35a/middleware/realip.go#L42
var trueClientIP = http.CanonicalHeaderKey("True-Client-IP")
var xForwardedFor = http.CanonicalHeaderKey("X-Forwarded-For")
var xRealIP = http.CanonicalHeaderKey("X-Real-IP")
func extractIP(r *http.Request) string {
var ip string
if tcip := r.Header.Get(trueClientIP); tcip != "" {
ip = tcip
} else if xrip := r.Header.Get(xRealIP); xrip != "" {
ip = xrip
} else if xff := r.Header.Get(xForwardedFor); xff != "" {
i := strings.Index(xff, ",")
if i == -1 {
i = len(xff)
}
ip = xff[:i]
}
return ip
}

View File

@ -139,21 +139,21 @@ func (svc *Service) EnrollAgent(ctx context.Context, enrollSecret, hostIdentifie
detailQueries := osquery_utils.GetDetailQueries(appConfig, svc.config)
save := false
if r, ok := hostDetails["os_version"]; ok {
err := detailQueries["os_version"].IngestFunc(svc.logger, host, []map[string]string{r})
err := detailQueries["os_version"].IngestFunc(ctx, svc.logger, host, []map[string]string{r})
if err != nil {
return "", ctxerr.Wrap(ctx, err, "Ingesting os_version")
}
save = true
}
if r, ok := hostDetails["osquery_info"]; ok {
err := detailQueries["osquery_info"].IngestFunc(svc.logger, host, []map[string]string{r})
err := detailQueries["osquery_info"].IngestFunc(ctx, svc.logger, host, []map[string]string{r})
if err != nil {
return "", ctxerr.Wrap(ctx, err, "Ingesting osquery_info")
}
save = true
}
if r, ok := hostDetails["system_info"]; ok {
err := detailQueries["system_info"].IngestFunc(svc.logger, host, []map[string]string{r})
err := detailQueries["system_info"].IngestFunc(ctx, svc.logger, host, []map[string]string{r})
if err != nil {
return "", ctxerr.Wrap(ctx, err, "Ingesting system_info")
}
@ -1033,7 +1033,7 @@ func (svc *Service) ingestDetailQuery(ctx context.Context, host *fleet.Host, nam
}
if query.IngestFunc != nil {
err = query.IngestFunc(svc.logger, host, rows)
err = query.IngestFunc(ctx, svc.logger, host, rows)
if err != nil {
return osqueryError{
message: fmt.Sprintf("ingesting query %s: %s", name, err.Error()),

View File

@ -10,6 +10,7 @@ import (
"github.com/fleetdm/fleet/v4/server/config"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/contexts/publicip"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
@ -27,7 +28,7 @@ type DetailQuery struct {
Platforms []string
// IngestFunc translates a query result into an update to the host struct,
// around data that lives on the hosts table.
IngestFunc func(logger log.Logger, host *fleet.Host, rows []map[string]string) error
IngestFunc func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error
// DirectIngestFunc gathers results from a query and directly works with the datastore to
// persist them. This is usually used for host data that is stored in a separate table.
DirectIngestFunc func(ctx context.Context, logger log.Logger, host *fleet.Host, ds fleet.Datastore, rows []map[string]string, failed bool) error
@ -55,7 +56,7 @@ var detailQueries = map[string]DetailQuery{
from interface_details id join interface_addresses ia
on ia.interface = id.interface where length(mac) > 0
order by (ibytes + obytes) desc`,
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) (err error) {
IngestFunc: func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) (err error) {
if len(rows) == 0 {
logger.Log("component", "service", "method", "IngestFunc", "err",
"detail_query_network_interface expected 1 or more results")
@ -105,12 +106,13 @@ var detailQueries = map[string]DetailQuery{
host.PrimaryIP = selected["address"]
host.PrimaryMac = selected["mac"]
host.PublicIP = publicip.FromContext(ctx)
return nil
},
},
"os_version": {
Query: "select * from os_version limit 1",
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
IngestFunc: func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "IngestFunc", "err",
fmt.Sprintf("detail_query_os_version expected single result got %d", len(rows)))
@ -158,7 +160,7 @@ var detailQueries = map[string]DetailQuery{
// distributed_interval (but it's not required), and typically
// do not control config_tls_refresh.
Query: `select name, value from osquery_flags where name in ("distributed_interval", "config_tls_refresh", "config_refresh", "logger_tls_period")`,
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
IngestFunc: func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error {
var configTLSRefresh, configRefresh uint
var configRefreshSeen, configTLSRefreshSeen bool
for _, row := range rows {
@ -215,7 +217,7 @@ var detailQueries = map[string]DetailQuery{
},
"osquery_info": {
Query: "select * from osquery_info limit 1",
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
IngestFunc: func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "IngestFunc", "err",
fmt.Sprintf("detail_query_osquery_info expected single result got %d", len(rows)))
@ -229,7 +231,7 @@ var detailQueries = map[string]DetailQuery{
},
"system_info": {
Query: "select * from system_info limit 1",
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
IngestFunc: func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "IngestFunc", "err",
fmt.Sprintf("detail_query_system_info expected single result got %d", len(rows)))
@ -264,7 +266,7 @@ var detailQueries = map[string]DetailQuery{
},
"uptime": {
Query: "select * from uptime limit 1",
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
IngestFunc: func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "IngestFunc", "err",
fmt.Sprintf("detail_query_uptime expected single result got %d", len(rows)))
@ -745,7 +747,7 @@ func directIngestUsers(ctx context.Context, logger log.Logger, host *fleet.Host,
return nil
}
func ingestDiskSpace(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
func ingestDiskSpace(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "ingestDiskSpace", "err",
fmt.Sprintf("detail_query_disk_space expected single result got %d", len(rows)))

View File

@ -22,7 +22,7 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) {
ingest := GetDetailQueries(nil, config.FleetConfig{})["network_interface"].IngestFunc
assert.NoError(t, ingest(log.NewNopLogger(), &host, nil))
assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, nil))
assert.Equal(t, initialHost, host)
var rows []map[string]string
@ -39,7 +39,7 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) {
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, rows))
assert.Equal(t, "192.168.1.3", host.PrimaryIP)
assert.Equal(t, "f4:5d:79:93:58:5b", host.PrimaryMac)
@ -57,7 +57,7 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) {
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, rows))
assert.Equal(t, "2604:3f08:1337:9411:cbe:814f:51a6:e4e3", host.PrimaryIP)
assert.Equal(t, "27:1b:aa:60:e8:0a", host.PrimaryMac)
@ -76,7 +76,7 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) {
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, rows))
assert.Equal(t, "205.111.43.79", host.PrimaryIP)
assert.Equal(t, "ab:1b:aa:60:e8:0a", host.PrimaryMac)
@ -93,7 +93,7 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) {
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, rows))
assert.Equal(t, "127.0.0.1", host.PrimaryIP)
assert.Equal(t, "00:00:00:00:00:00", host.PrimaryMac)
}
@ -323,7 +323,7 @@ func TestDetailQuerysOSVersion(t *testing.T) {
ingest := GetDetailQueries(nil, config.FleetConfig{})["os_version"].IngestFunc
assert.NoError(t, ingest(log.NewNopLogger(), &host, nil))
assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, nil))
assert.Equal(t, initialHost, host)
// Rolling release for archlinux
@ -345,7 +345,7 @@ func TestDetailQuerysOSVersion(t *testing.T) {
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, rows))
assert.Equal(t, "Arch Linux rolling", host.OSVersion)
// Simulate a linux with a proper version
@ -366,7 +366,7 @@ func TestDetailQuerysOSVersion(t *testing.T) {
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.NoError(t, ingest(context.Background(), log.NewNopLogger(), &host, rows))
assert.Equal(t, "Arch Linux 1.2.3", host.OSVersion)
}

View File

@ -19,6 +19,8 @@ import (
kitlog "github.com/go-kit/kit/log"
)
var _ fleet.Service = (*Service)(nil)
// Service is the struct implementing fleet.Service. Create a new one with NewService.
type Service struct {
ds fleet.Datastore
@ -44,6 +46,12 @@ type Service struct {
jitterMu *sync.Mutex
jitterH map[time.Duration]*jitterHashTable
geoIP fleet.GeoIP
}
func (s *Service) LookupGeoIP(ctx context.Context, ip string) *fleet.GeoLocation {
return s.geoIP.Lookup(ctx, ip)
}
// NewService creates a new service from the config struct
@ -62,6 +70,7 @@ func NewService(
carveStore fleet.CarveStore,
license fleet.LicenseInfo,
failingPolicySet fleet.FailingPolicySet,
geoIP fleet.GeoIP,
) (fleet.Service, error) {
authorizer, err := authz.NewAuthorizer()
if err != nil {
@ -86,6 +95,7 @@ func NewService(
authz: authorizer,
jitterH: make(map[time.Duration]*jitterHashTable),
jitterMu: new(sync.Mutex),
geoIP: geoIP,
}
return validationMiddleware{svc, ds, sso}, nil
}

View File

@ -69,7 +69,7 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf
Datastore: ds,
AsyncEnabled: false,
}
svc, err := NewService(context.Background(), ds, task, rs, logger, osqlogger, fleetConfig, mailer, c, ssoStore, lq, ds, *license, failingPolicySet)
svc, err := NewService(context.Background(), ds, task, rs, logger, osqlogger, fleetConfig, mailer, c, ssoStore, lq, ds, *license, failingPolicySet, &fleet.NoOpGeoIP{})
if err != nil {
panic(err)
}