Elk to monitoring (#152)

* clean grafana chart * grafana autosync dashboard * README update * change ns for elk * fix path * change namespace for elk * add grafana sync to prometheus chart * update README * fix error with secret * Update README.md Co-authored-by: vilorij <vilorij@ya.ru> * Update README.md Co-authored-by: vilorij <vilorij@ya.ru> * clean * fix * fix * fix * clean * resolve * README update Co-authored-by: ilya <Ilya Ivanov> Co-authored-by: vilorij <vilorij@ya.ru>
2024-11-06 00:45:18 +00:00 · 2020-12-14 11:54:20 +03:00 · 2020-12-14 11:54:20 +03:00 · a16237b954
commit a16237b954
parent 1c736f57f2
7 changed files with 140 additions and 16 deletions
--- a/README.md
+++ b/README.md
@ -168,3 +168,19 @@ Login as the elastic user. The password can be obtained with the following comma
 ```
 kubectl get secret rbk-es-elastic-user -o=jsonpath='{.data.elastic}' | base64 --decode; echo
 ```
+
+Доступ к grafana и синк dashboards
+-----------
+
+Используем kubectl port-forward
+
+```
+kubectl -n monitoring port-forward <grafana-pod> 3000
+```
+grafana доступна в браузере https://localhost:3000. Получить пароль для входа:
+
+```
+kubectl get secret --namespace monitoring prometheus-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
+```
+
+```
--- a/config/prometheus/values.yaml.gotmpl
+++ b/config/prometheus/values.yaml.gotmpl
@ -1,8 +1,17 @@
-# -*- mode: yaml -*-
-
 # Look for reference at https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml#L2008
 prometheus:
  additionalServiceMonitors:
+    - name: "rbk-erlang-service"
+      selector:
+        matchLabels:
+          prometheus.metrics.erlang.enabled: "true"
+      namespaceSelector:
+        matchNames:
+          - default
+      endpoints:
+        - port: "api"
+          path: /metrics
+          scheme: http
    - name: "rbk-java-service"
      selector:
        matchLabels:
@ -16,18 +25,97 @@ prometheus:
          scheme: http

 grafana:
+  enabled: true
+  replicas: 1
+  
+  create: true
+  ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true)
+  # useExistingRole: name-of-some-(cluster)role
+  rbac:
+    create: true
+    pspEnabled: true
+    pspUseAppArmor: true
+    namespaced: false
+    extraClusterRoleRules:
+    - apiGroups: [""]
+      resources: ["configmaps", "secrets"]
+      verbs: ["get", "watch", "list"]
+  
+  image:
+    repository: grafana/grafana
+    tag: 7.2.1
+    sha: ""
+    pullPolicy: IfNotPresent
+
+  extraEmptyDirMounts:
+    - name: dashboard-dir
+      mountPath: /var/lib/grafana/dashboards/general
+  
+  envValueFrom:
+    ELASTIC_PASS:
+      secretKeyRef:
+        name: rbkmoney-es-elastic-user
+        key: elastic
+
+  extraInitContainers:
+    - name: dashboard-autosync
+      image: alpine/git:v2.26.2
+      imagePullPolicy: IfNotPresent
+      args:
+      - clone
+      - -b
+      - dashboard/release
+      - https://github.com/rbkmoney/grafana-dashboards-common.git
+      - /git/dashboards
+      volumeMounts:
+        - name: dashboard-dir
+          mountPath: "/git/dashboards"
+      securityContext:
+       runAsUser: 0
+
+  extraContainerVolumes:
+    - name: sync-key
+      secret:
+        secretName: prometheus-grafana-env
+        items:
+        - key: synckey
+          path: synckey
+          mode: 0600
+
+  plugins: []
+  # - digrich-bubblechart-panel
+    # - grafana-clock-panel
+
+  datasources:
+    datasources.yaml:
+      apiVersion: 1
+      datasources:
+      - name: rbkm-elasticsearch
+        type: elasticsearch
+        database: "filebeat-rbkmoney-processing-*"
+        url: https://rbkmoney-es-http:9200
+        basicAuth: true
+        basicAuthUser: elastic
+        jsonData:
+          timeField: "@timestamp"
+          esVersion: 70
+          tlsSkipVerify: true
+        secureJsonData:
+          basicAuthPassword: $ELASTIC_PASS
+
  dashboardProviders:
    dashboardproviders.yaml:
      apiVersion: 1
      providers:
-      - name: 'rbk-dashboards'
+      - name: 'general'
        orgId: 1
-        folder: 'Processing'
+        folder: ''
        type: file
        disableDeletion: false
        editable: true
        options:
-          path: /var/lib/grafana/dashboards/rbk-dashboards
+          path: /var/lib/grafana/dashboards
+
  dashboards:
    rbk-dashboards:
      erlang-instance:
@ -36,3 +124,19 @@ grafana:
      machinegun-namespace:
        json: |
          {{- readFile "dashboards/result/machinegun-namespace.json" | nindent 10 }}
+
+  grafana.ini:
+    paths:
+      data: /var/lib/grafana/data
+      logs: /var/log/grafana
+      plugins: /var/lib/grafana/plugins
+      provisioning: /etc/grafana/provisioning
+    analytics:
+      check_for_updates: true
+    log:
+      mode: console
+    grafana_net:
+      url: https://grafana.net
+
+  revisionHistoryLimit: 10
+
--- a/helmfile-infra.lock
+++ b/helmfile-infra.lock
@ -1,7 +1,7 @@
-version: 0.131.0
+version: v0.132.0
 dependencies:
 - name: kube-prometheus-stack
  repository: https://prometheus-community.github.io/helm-charts
-  version: 10.1.0
-digest: sha256:681ca4d3f82dbb792bc805974a141b3d4082c69bb5021eed6fae3d700927d449
-generated: "2020-10-30T12:55:28.696343+05:00"
+  version: 12.1.0
+digest: sha256:f546085e97716d6a910025c43d416c86c8c95b584c92c946c3996b96f4524ac0
+generated: "2020-11-23T14:08:29.249338448+03:00"
--- a/helmfile-infra.yaml
+++ b/helmfile-infra.yaml
@ -8,10 +8,14 @@ releases:
 - name: prometheus
  <<: *default
  chart: prometheus-community/kube-prometheus-stack
-  version: 10.1.0
+  version: 12.1.0
  namespace: monitoring
+  needs:
+  - monitoring/logs
+#  - default/logs
 - name: logs
  # Change to installed: true if you need ECK installed
  installed: false
  <<: *default
+  namespace: monitoring
  chart: ./services/elk
--- a/helmfile.lock
+++ b/helmfile.lock
@ -1,4 +1,4 @@
-version: v0.130.1
+version: v0.132.0
 dependencies:
 - name: consul
  repository: https://charts.helm.sh/stable
--- a/services/elk/templates/filebeat.yaml
+++ b/services/elk/templates/filebeat.yaml
@ -106,7 +106,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: filebeat
-  namespace: default
+  namespace: {{ .Release.Namespace }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@ -115,7 +115,7 @@ metadata:
 subjects:
 - kind: ServiceAccount
  name: filebeat
-  namespace: default
+  namespace: {{ .Release.Namespace }}
 roleRef:
  kind: ClusterRole
  name: filebeat
--- a/services/elk/templates/hooks.yaml
+++ b/services/elk/templates/hooks.yaml
@ -29,17 +29,17 @@ spec:
          image: curlimages/curl
          env:
            - name: KB_USERNAME
-              value: default-filebeat-beat-kb-user
+              value: {{ .Release.Namespace }}-filebeat-beat-kb-user
            - name: KB_PASS
              valueFrom:
                secretKeyRef:
                  name: filebeat-beat-kb-user
-                  key: default-filebeat-beat-kb-user
+                  key: {{ .Release.Namespace }}-filebeat-beat-kb-user
          command:
            - /bin/sh
            - -c
            - |
-              curl -XPOST https://{{- .Values.kibana.nameref -}}-kb-http.default.svc:5601/api/saved_objects/index-pattern/filebeat-rbkmoney-processing?overwrite=true \
+              curl -XPOST https://{{- .Values.kibana.nameref -}}-kb-http.{{ .Release.Namespace }}.svc:5601/api/saved_objects/index-pattern/filebeat-rbkmoney-processing?overwrite=true \
              -H 'kbn-xsrf: true' -H 'Content-Type: application/json' \
              -u ${KB_USERNAME}:${KB_PASS} -k -d '
              {