Elk to monitoring (#152)

* clean grafana chart

* grafana autosync dashboard

* README update

* change ns for elk

* fix path

* change namespace for elk

* add grafana sync to prometheus chart

* update README

* fix error with secret

* Update README.md

Co-authored-by: vilorij <vilorij@ya.ru>

* Update README.md

Co-authored-by: vilorij <vilorij@ya.ru>

* clean

* fix

* fix

* fix

* clean

* resolve

* README update

Co-authored-by: ilya <Ilya Ivanov>
Co-authored-by: vilorij <vilorij@ya.ru>

README.md

@@ -168,3 +168,19 @@ Login as the elastic user. The password can be obtained with the following comma
 ```
 kubectl get secret rbk-es-elastic-user -o=jsonpath='{.data.elastic}' | base64 --decode; echo
 ```
+
+Доступ к grafana и синк dashboards
+-----------
+
+Используем kubectl port-forward
+
+```
+kubectl -n monitoring port-forward <grafana-pod> 3000
+```
+grafana доступна в браузере https://localhost:3000. Получить пароль для входа:
+
+```
+kubectl get secret --namespace monitoring prometheus-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
+```
+
+```

config/prometheus/values.yaml.gotmpl

@@ -1,8 +1,17 @@
-# -*- mode: yaml -*-
-
 # Look for reference at https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml#L2008
 prometheus:
   additionalServiceMonitors:
+    - name: "rbk-erlang-service"
+      selector:
+        matchLabels:
+          prometheus.metrics.erlang.enabled: "true"
+      namespaceSelector:
+        matchNames:
+          - default
+      endpoints:
+        - port: "api"
+          path: /metrics
+          scheme: http
     - name: "rbk-java-service"
       selector:
         matchLabels:
@@ -16,18 +25,97 @@ prometheus:
           scheme: http
 
 grafana:
+  enabled: true
+  replicas: 1
+  
+  create: true
+  ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true)
+  # useExistingRole: name-of-some-(cluster)role
+  rbac:
+    create: true
+    pspEnabled: true
+    pspUseAppArmor: true
+    namespaced: false
+    extraClusterRoleRules:
+    - apiGroups: [""]
+      resources: ["configmaps", "secrets"]
+      verbs: ["get", "watch", "list"]
+  
+  image:
+    repository: grafana/grafana
+    tag: 7.2.1
+    sha: ""
+    pullPolicy: IfNotPresent
+
+  extraEmptyDirMounts:
+    - name: dashboard-dir
+      mountPath: /var/lib/grafana/dashboards/general
+  
+  envValueFrom:
+    ELASTIC_PASS:
+      secretKeyRef:
+        name: rbkmoney-es-elastic-user
+        key: elastic
+
+  extraInitContainers:
+    - name: dashboard-autosync
+      image: alpine/git:v2.26.2
+      imagePullPolicy: IfNotPresent
+      args:
+      - clone
+      - -b
+      - dashboard/release
+      - https://github.com/rbkmoney/grafana-dashboards-common.git
+      - /git/dashboards
+      volumeMounts:
+        - name: dashboard-dir
+          mountPath: "/git/dashboards"
+      securityContext:
+       runAsUser: 0
+
+  extraContainerVolumes:
+    - name: sync-key
+      secret:
+        secretName: prometheus-grafana-env
+        items:
+        - key: synckey
+          path: synckey
+          mode: 0600
+
+  plugins: []
+  # - digrich-bubblechart-panel
+    # - grafana-clock-panel
+
+  datasources:
+    datasources.yaml:
+      apiVersion: 1
+      datasources:
+      - name: rbkm-elasticsearch
+        type: elasticsearch
+        database: "filebeat-rbkmoney-processing-*"
+        url: https://rbkmoney-es-http:9200
+        basicAuth: true
+        basicAuthUser: elastic
+        jsonData:
+          timeField: "@timestamp"
+          esVersion: 70
+          tlsSkipVerify: true
+        secureJsonData:
+          basicAuthPassword: $ELASTIC_PASS
+
   dashboardProviders:
     dashboardproviders.yaml:
       apiVersion: 1
       providers:
-      - name: 'rbk-dashboards'
+      - name: 'general'
         orgId: 1
-        folder: 'Processing'
+        folder: ''
         type: file
         disableDeletion: false
         editable: true
         options:
-          path: /var/lib/grafana/dashboards/rbk-dashboards
+          path: /var/lib/grafana/dashboards
+
   dashboards:
     rbk-dashboards:
       erlang-instance:
@@ -36,3 +124,19 @@ grafana:
       machinegun-namespace:
         json: |
           {{- readFile "dashboards/result/machinegun-namespace.json" | nindent 10 }}
+
+  grafana.ini:
+    paths:
+      data: /var/lib/grafana/data
+      logs: /var/log/grafana
+      plugins: /var/lib/grafana/plugins
+      provisioning: /etc/grafana/provisioning
+    analytics:
+      check_for_updates: true
+    log:
+      mode: console
+    grafana_net:
+      url: https://grafana.net
+
+  revisionHistoryLimit: 10
+

helmfile-infra.lock

@@ -1,7 +1,7 @@
-version: 0.131.0
+version: v0.132.0
 dependencies:
 - name: kube-prometheus-stack
   repository: https://prometheus-community.github.io/helm-charts
-  version: 10.1.0
-digest: sha256:681ca4d3f82dbb792bc805974a141b3d4082c69bb5021eed6fae3d700927d449
-generated: "2020-10-30T12:55:28.696343+05:00"
+  version: 12.1.0
+digest: sha256:f546085e97716d6a910025c43d416c86c8c95b584c92c946c3996b96f4524ac0
+generated: "2020-11-23T14:08:29.249338448+03:00"

helmfile-infra.yaml

@@ -8,10 +8,14 @@ releases:
 - name: prometheus
   <<: *default
   chart: prometheus-community/kube-prometheus-stack
-  version: 10.1.0
+  version: 12.1.0
   namespace: monitoring
+  needs:
+  - monitoring/logs
+#  - default/logs
 - name: logs
   # Change to installed: true if you need ECK installed
   installed: false
   <<: *default
+  namespace: monitoring
   chart: ./services/elk

helmfile.lock

@@ -1,4 +1,4 @@
-version: v0.130.1
+version: v0.132.0
 dependencies:
 - name: consul
   repository: https://charts.helm.sh/stable

services/elk/templates/filebeat.yaml

@@ -106,7 +106,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: filebeat
-  namespace: default
+  namespace: {{ .Release.Namespace }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -115,7 +115,7 @@ metadata:
 subjects:
 - kind: ServiceAccount
   name: filebeat
-  namespace: default
+  namespace: {{ .Release.Namespace }}
 roleRef:
   kind: ClusterRole
   name: filebeat

services/elk/templates/hooks.yaml

@@ -29,17 +29,17 @@ spec:
           image: curlimages/curl
           env:
             - name: KB_USERNAME
-              value: default-filebeat-beat-kb-user
+              value: {{ .Release.Namespace }}-filebeat-beat-kb-user
             - name: KB_PASS
               valueFrom:
                 secretKeyRef:
                   name: filebeat-beat-kb-user
-                  key: default-filebeat-beat-kb-user
+                  key: {{ .Release.Namespace }}-filebeat-beat-kb-user
           command:
             - /bin/sh
             - -c
             - |
-              curl -XPOST https://{{- .Values.kibana.nameref -}}-kb-http.default.svc:5601/api/saved_objects/index-pattern/filebeat-rbkmoney-processing?overwrite=true \
+              curl -XPOST https://{{- .Values.kibana.nameref -}}-kb-http.{{ .Release.Namespace }}.svc:5601/api/saved_objects/index-pattern/filebeat-rbkmoney-processing?overwrite=true \
               -H 'kbn-xsrf: true' -H 'Content-Type: application/json' \
               -u ${KB_USERNAME}:${KB_PASS} -k -d '
               {