add services to inspect traces and monitor a local server (#8597)

This adds tooling to debug and inspect traces locally, please refer to the README.md in this commit for more details.

docs/Contributing/Testing-and-local-development.md

@@ -13,6 +13,7 @@
 - [Testing SSO](#testing-sso)
 - [Testing Kinesis Logging](#testing-kinesis-logging)
 - [Testing pre-built installers](#testing-pre-built-installers)
+- [Telemetry](#telemetry)
 
 ## License key
 
@@ -421,3 +422,9 @@ Be sure to replace the `FLEET_PACKAGING_GLOBAL_ENROLL_SECRET` value above with t
 secret from the `fleetctl package` command used to build the installers. 
 
 MinIO also offers a web interface at http://localhost:9001. Credentials are `minio` / `minio123!`.
+
+## Telemetry
+
+You can configure the server to record and report trace data using OpenTelemetry or Elastic APM and use a tracing system like [Jaeger](https://www.jaegertracing.io/) to consume this data and inspect the traces locally.
+
+Please refer to [tools/telemetry](../../tools/telemetry/README.md) for instructions.

tools/telemetry/README.md

@@ -0,0 +1,22 @@
+### Telemetry tools
+
+Running the services specified in the `docker-compose.yml` file will give you access to:
+
+- The Jaeger UI with both the `/monitor` (latency, errors, req/sec) and `/search` (traces) tabs ready to use.
+- A Prometheus server used by Jaeger with enhanced monitoring data provided by OpenTelemetry.
+
+To get started:
+
+1. Start the necessary services by running `docker compose up` in this directory.
+2. Start the Fleet server with telemetry enabled and configured with this:
+
+```
+OTEL_SERVICE_NAME="fleet" \
+OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317" \
+./build/fleet serve \
+  --logging_tracing_enabled=true \
+  --logging_tracing_type=opentelemetry \
+  --dev --logging_debug
+``` 
+
+Afterward, you can navigate to http://localhost:16686/ to access the Jaeger UI.

tools/telemetry/docker-compose.yml

@@ -0,0 +1,37 @@
+version: "2"
+services:
+
+  # Jaeger
+  jaeger-all-in-one:
+    image: jaegertracing/all-in-one:latest
+    environment:
+      - COLLECTOR_OTLP_ENABLED=true
+      - METRICS_STORAGE_TYPE=prometheus
+      - PROMETHEUS_SERVER_URL=http://prometheus:9090
+    ports:
+      - "16686:16686"
+      - "14269:14269"
+
+  # Collector
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib-dev:latest
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "1888:1888"   # pprof extension
+      - "8888:8888"   # Prometheus metrics exposed by the collector
+      - "8889:8889"   # Prometheus exporter metrics
+      - "13133:13133" # health_check extension
+      - "4317:4317"   # OTLP gRPC receiver
+      - "55679:55679" # zpages extension
+    depends_on:
+      - jaeger-all-in-one
+
+  prometheus:
+    container_name: prometheus
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "18492:9090"

tools/telemetry/otel-collector-config.yaml

@@ -0,0 +1,53 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+
+  # Dummy receiver that's never used, because a pipeline is
+  # required to have one.
+  otlp/spanmetrics:
+    protocols:
+      grpc:
+        endpoint: "localhost:65535"
+
+exporters:
+  prometheus:
+    endpoint: "0.0.0.0:8889"
+
+  logging:
+
+  jaeger:
+    endpoint: jaeger-all-in-one:14250
+    tls:
+      insecure: true
+
+processors:
+  batch:
+  spanmetrics:
+    metrics_exporter: prometheus
+
+extensions:
+  health_check:
+  pprof:
+    endpoint: :1888
+  zpages:
+    endpoint: :55679
+
+service:
+  extensions: [zpages, pprof, health_check]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [spanmetrics, batch]
+      exporters: [logging, jaeger]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [logging, prometheus]
+    # The exporter name in this pipeline must match the
+    # spanmetrics.metrics_exporter name.  The receiver is just a
+    # dummy and never used; added to pass validation requiring at
+    # least one receiver in a pipeline.
+    metrics/spanmetrics:
+      receivers: [otlp/spanmetrics]
+      exporters: [prometheus]

tools/telemetry/prometheus.yml

@@ -0,0 +1,6 @@
+scrape_configs:
+  - job_name: 'otel-collector'
+    scrape_interval: 10s
+    static_configs:
+      - targets: ['otel-collector:8889']
+      - targets: ['otel-collector:8888']