Improve logging (#89)

* Fix ES index template dynamic mappings * Add basic common logging setting * Add stdout log configuration for shumway (with hardcode) * Fix hellgate mapping Co-authored-by: Sergey Yelin <elinsn@gmail.com>
2024-11-06 08:55:21 +00:00 · 2020-10-22 15:03:15 +03:00 · 2020-10-22 15:03:15 +03:00 · a979b58a3e
commit a979b58a3e
parent 51cdd7737a
21 changed files with 2620 additions and 67 deletions
--- a/config/_common/logging.yaml.gotmpl
+++ b/config/_common/logging.yaml.gotmpl
@ -0,0 +1,6 @@
+{{ if eq (index .Release.Labels "logfmt") "json" }}
+podAnnotations:
+  co.elastic.logs/json.keys_under_root: "true"
+  co.elastic.logs/json.overwrite_keys: "true"
+  co.elastic.logs/json.add_error_key: "true"
+{{ end }}
--- a/config/logs/template.json
+++ b/config/logs/template.json
--- a/helmfile-infra.yaml
+++ b/helmfile-infra.yaml
@ -1,18 +1,9 @@
+{{ readFile "hf-templates.yaml" }}
+
 repositories:
 - name: prometheus-community
  url: https://prometheus-community.github.io/helm-charts

-templates:
-  default: &default
-    chart: ./services/{{`{{ .Release.Name }}`}}
-    namespace: default
-    # This prevents helmfile exiting when it encounters a missing file
-    # Valid values are "Error", "Warn", "Info", "Debug". The default is "Error"
-    # Use "Debug" to make missing files errors invisible at the default log level(--log-level=INFO)
-    missingFileHandler: Warn
-    values:
-    - config/{{`{{ .Release.Name }}`}}/values.yaml
-
 releases:
 - name: prometheus
  <<: *default
@ -21,4 +12,6 @@ releases:
 - name: logs
  <<: *default
  chart: ./services/elk
-
+  set:
+    - name: filebeat.indextemplate
+      file: config/logs/template.json
--- a/helmfile.yaml
+++ b/helmfile.yaml
@ -1,3 +1,5 @@
+{{ readFile "hf-templates.yaml" }}
+
 repositories:
 - name: stable
  url: https://kubernetes-charts.storage.googleapis.com
@ -10,19 +12,6 @@ repositories:
 - name: codecentric
  url: https://codecentric.github.io/helm-charts

-templates:
-  default: &default
-    chart: ./services/{{`{{ .Release.Name }}`}}
-    namespace: default
-    # This prevents helmfile exiting when it encounters a missing file
-    # Valid values are "Error", "Warn", "Info", "Debug". The default is "Error"
-    # Use "Debug" to make missing files errors invisible at the default log level(--log-level=INFO)
-    missingFileHandler: Warn
-    values:
-    - config/{{`{{ .Release.Name }}`}}/values.yaml
-    timeout: 900
-
-
 helmfiles:
 - # Path to the helmfile state file being processed BEFORE releases in this state file
  path: helmfile-infra.yaml
@ -67,12 +56,16 @@ releases:
  <<: *default
 - name: riak
  <<: *default
+  labels:
+    logfmt: json
  chart: ./services/riak
  set:
  - name: config.user
    file: config/riak/user.yaml
 - name: machinegun
  <<: *default
+  labels:
+    logfmt: json
  needs:
  - default/consul
  - default/riak
@ -82,6 +75,8 @@ releases:
    file: config/machinegun/config.yaml
 - name: bender
  <<: *default
+  labels:
+    logfmt: json
  set:
  - name: appConfig
    file: config/bender/sys.config
@ -89,6 +84,8 @@ releases:
    file: config/vm/erl_inetrc
 - name: kds
  <<: *default
+  labels:
+    logfmt: json
  set:
  - name: appConfig
    file: config/kds/sys.config
@ -100,6 +97,8 @@ releases:
    file: config/kds/server.pem
 - name: cds
  <<: *default
+  labels:
+    logfmt: json
  needs:
    - default/kds
    - default/riak
@ -114,6 +113,8 @@ releases:
    file: config/cds/client.pem
 - name: shumway
  <<: *default
+  labels:
+    logfmt: json
  needs:
  - default/vault
  - default/postgres
@ -131,6 +132,8 @@ releases:
  - default/kafka
 - name: dominant
  <<: *default
+  labels:
+    logfmt: json
  needs:
  - default/shumway
  wait: true
@ -157,6 +160,8 @@ releases:
  <<: *default
 - name: hellgate
  <<: *default
+  labels:
+    logfmt: json
  set:
  - name: appConfig
    file: config/hellgate/sys.config
@ -164,6 +169,8 @@ releases:
    file: config/vm/erl_inetrc
 - name: capi-pcidss-v2
  <<: *default
+  labels:
+    logfmt: json
  set:
  - name: fetchKeycloakPubkey
    file: config/api-common/fetch-keycloak-pubkey.sh
@ -183,6 +190,8 @@ releases:
    - default/keycloak
 - name: capi-pcidss-v1
  <<: *default
+  labels:
+    logfmt: json
  set:
  - name: fetchKeycloakPubkey
    file: config/api-common/fetch-keycloak-pubkey.sh
@ -209,6 +218,8 @@ releases:
    file: config/vm/erl_inetrc
 - name: capi-v1
  <<: *default
+  labels:
+    logfmt: json
  set:
  - name: fetchKeycloakPubkey
    file: config/api-common/fetch-keycloak-pubkey.sh
@ -228,6 +239,8 @@ releases:
    - default/keycloak
 - name: capi-v2
  <<: *default
+  labels:
+    logfmt: json
  set:
  - name: fetchKeycloakPubkey
    file: config/api-common/fetch-keycloak-pubkey.sh
@ -247,6 +260,8 @@ releases:
    - default/keycloak
 - name: wapi-pcidss-v0
  <<: *default
+  labels:
+    logfmt: json
  set:
  - name: fetchKeycloakPubkey
    file: config/api-common/fetch-keycloak-pubkey.sh
@ -262,6 +277,8 @@ releases:
    - default/keycloak
 - name: wapi
  <<: *default
+  labels:
+    logfmt: json
  set:
  - name: fetchKeycloakPubkey
    file: config/api-common/fetch-keycloak-pubkey.sh
--- a/hf-templates.yaml
+++ b/hf-templates.yaml
@ -0,0 +1,9 @@
+templates:
+  default: &default
+    chart: ./services/{{ .Release.Name }}
+    namespace: default
+    missingFileHandler: Warn
+    values:
+      - config/_common/logging.yaml.gotmpl
+      - config/{{ .Release.Name }}/values.yaml
+    timeout: 900
--- a/services/bender/values.yaml
+++ b/services/bender/values.yaml
@ -2,7 +2,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.

-replicaCount: 3
+replicaCount: 1

 image:
  repository: docker.io/rbkmoney/bender
--- a/services/binbase/values.yaml
+++ b/services/binbase/values.yaml
@ -4,7 +4,7 @@ image:

 imagePullSecrets: []

-replicaCount: 3
+replicaCount: 1

 cdsUrl: http://cds.default.svc.cluster.local:8022/v2/storage

--- a/services/capi-pcidss-v1/values.yaml
+++ b/services/capi-pcidss-v1/values.yaml
@ -4,7 +4,7 @@ image:

 imagePullSecrets: []

-replicaCount: 3
+replicaCount: 1

 beam:
  disableSBWT: false
@ -59,4 +59,3 @@ affinity: {}
 ingress:
  host: api.rbk.dev
  path: /v1/processing/payment-resources
-
--- a/services/capi-pcidss-v2/values.yaml
+++ b/services/capi-pcidss-v2/values.yaml
@ -4,7 +4,7 @@ image:

 imagePullSecrets: []

-replicaCount: 3
+replicaCount: 1

 beam:
  disableSBWT: false
@ -59,4 +59,3 @@ affinity: {}
 ingress:
  host: api.rbk.dev
  path: /v2/processing/payment-resources
-
--- a/services/dominant/templates/deployment.yaml
+++ b/services/dominant/templates/deployment.yaml
@ -57,8 +57,6 @@ spec:
              mountPath: /opt/dominant/erl_inetrc
              subPath: erl_inetrc
              readOnly: true
-            - name: logs-volume
-              mountPath: /var/log
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      volumes:
@ -72,10 +70,6 @@ spec:
              path: vm.args
            - key: erl_inetrc
              path: erl_inetrc
-        - name: logs-volume
-          hostPath:
-            path: /datalog/rbk
-            type: DirectoryOrCreate
      {{- with .Values.nodeSelector }}
      nodeSelector:
        {{- toYaml . | nindent 8 }}
--- a/services/elk/templates/filebeat.yaml
+++ b/services/elk/templates/filebeat.yaml
@ -22,17 +22,27 @@ spec:
              type: container
              paths:
              - /var/log/containers/*${data.kubernetes.container.id}.log
-          templates:
-            - condition:
-                equals:
-                  kubernetes.namespace: default
-              config:
-                - type: log
-                  paths:
-                    - /datalog/rbk/${data.kubernetes.container.name}/*.json
    processors:
-    - add_cloud_metadata: {}
-    - add_host_metadata: {}
+      - decode_json_fields:
+          fields: ["log"]
+          max_depth: 5
+          target: ""
+          overwrite_keys: true
+          add_error_key: true
+    setup:
+      template:
+        enabled: true
+        name: filebeat-rbkm # could it conflict with .json.name?..
+        pattern: filebeat-*
+        overwrite: true
+        settings:
+          index:
+            number_of_shards: 1
+            number_of_replicas: 1
+        json:
+          enabled: true
+          path: "/etc/template.json"
+          name: filebeat-rbkm
  daemonSet:
    podTemplate:
      spec:
@ -54,8 +64,9 @@ spec:
            mountPath: /var/log/pods
          - name: varlibdockercontainers
            mountPath: /var/lib/docker/containers
-          - name: rbkdatalogfolder
-            mountPath: /datalog/rbk
+          - name: index-template
+            mountPath: /etc/template.json
+            subPath: template.json
          env:
            - name: NODE_NAME
              valueFrom:
@ -71,9 +82,9 @@ spec:
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
-        - name: rbkdatalogfolder
-          hostPath:
-            path: /datalog/rbk
+        - name: index-template
+          configMap:
+            name: filebeat-index-template
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
--- a/services/elk/templates/indextemplate.yaml
+++ b/services/elk/templates/indextemplate.yaml
@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat-index-template
+data:
+  template.json: |
+    {{- .Values.filebeat.indextemplate | nindent 4 }}
--- a/services/hellgate/templates/deployment.yaml
+++ b/services/hellgate/templates/deployment.yaml
@ -46,19 +46,17 @@ spec:
              port: api
          volumeMounts:
            - name: config-volume
-              mountPath: /opt/hellgate/releases/0.1.0/sys.config
+              mountPath: /opt/hellgate/releases/0.1/sys.config
              subPath: sys.config
              readOnly: true
            - name: config-volume
-              mountPath: /opt/hellgate/releases/0.1.0/vm.args
+              mountPath: /opt/hellgate/releases/0.1/vm.args
              subPath: vm.args
              readOnly: true
            - name: config-volume
              mountPath: /opt/hellgate/erl_inetrc
              subPath: erl_inetrc
              readOnly: true
-            - name: logs-volume
-              mountPath: /var/log
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      volumes:
@ -72,10 +70,6 @@ spec:
              path: vm.args
            - key: erl_inetrc
              path: erl_inetrc
-        - name: logs-volume
-          hostPath:
-            path: /datalog/rbk
-            type: DirectoryOrCreate
      {{- with .Values.nodeSelector }}
      nodeSelector:
        {{- toYaml . | nindent 8 }}
--- a/services/hooker/values.yaml
+++ b/services/hooker/values.yaml
@ -4,7 +4,7 @@ image:

 imagePullSecrets: []

-replicaCount: 3
+replicaCount: 1

 nameOverride: ""
 fullnameOverride: ""
--- a/services/proxy-mocket-inspector/values.yaml
+++ b/services/proxy-mocket-inspector/values.yaml
@ -4,7 +4,7 @@ image:

 imagePullSecrets: []

-replicaCount: 3
+replicaCount: 1

 entrypoint: |
  #!/bin/sh
--- a/services/proxy-mocketbank/values.yaml
+++ b/services/proxy-mocketbank/values.yaml
@ -4,7 +4,7 @@ image:

 imagePullSecrets: []

-replicaCount: 3
+replicaCount: 1

 cdsUrl: http://cds.default.svc.cluster.local:8022/v2/storage
 hellgateUrl: http://hellgate.default.svc.cluster.local:8022/v1/proxyhost/provider
--- a/services/shumway/templates/configmap.yaml
+++ b/services/shumway/templates/configmap.yaml
@ -8,3 +8,44 @@ metadata:
 data:
  entrypoint.sh: |
    {{- .Values.entrypoint | nindent 4 }}
+  # TODO: вынести в отдельный файл!
+  logback.xml: |
+    <?xml version="1.0" encoding="UTF-8"?>
+    <configuration>
+        <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+        <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+
+        <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+            <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
+                <providers>
+                    <mdc/>
+                    <threadName/>
+                    <message/>
+                    <version/>
+                    <loggerName/>
+                    <context/>
+                    <pattern>
+                        <pattern>
+                            {
+                            "@timestamp": "%date{yyy-MM-dd'T'HH:mm:ss.SSSXXX, UTC}",
+                            "@severity": "%level",
+                            "application": {{ include "shumway.fullname" . | quote }}
+                            }
+                        </pattern>
+                    </pattern>
+                    <stackTrace>
+                        <throwableConverter class="net.logstash.logback.stacktrace.ShortenedThrowableConverter">
+                            <shortenedClassNameLength>20</shortenedClassNameLength>
+                            <rootCauseFirst>true</rootCauseFirst>
+                        </throwableConverter>
+                    </stackTrace>
+                </providers>
+            </encoder>
+        </appender>
+
+        <root level="INFO">
+            <appender-ref ref="STDOUT"/>
+        </root>
+
+        <logger name="com.rbkmoney" level="INFO"/>
+    </configuration>
--- a/services/shumway/templates/deployment.yaml
+++ b/services/shumway/templates/deployment.yaml
@ -49,12 +49,17 @@ spec:
              mountPath: /opt/shumway/entrypoint.sh
              subPath: entrypoint.sh
              readOnly: true
+            - name: config-volume
+              mountPath: /opt/shumway/logback.xml
+              subPath: logback.xml
+              readOnly: true
          command: ["/opt/shumway/entrypoint.sh"]
          args:
            - -jar
            - /opt/shumway/shumway.jar
            - --spring.flyway.table=schema_version
            - --spring.flyway.schemas=shm
+            - --logging.config=/opt/shumway/logback.xml
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      volumes:
@ -65,6 +70,9 @@ spec:
            - key: entrypoint.sh
              path: entrypoint.sh
              mode: 0755
+            - key: logback.xml
+              path: logback.xml
+
      {{- with .Values.nodeSelector }}
      nodeSelector:
        {{- toYaml . | nindent 8 }}
--- a/services/shumway/values.yaml
+++ b/services/shumway/values.yaml
@ -4,7 +4,7 @@ image:

 imagePullSecrets: []

-replicaCount: 3
+replicaCount: 1

 nameOverride: ""
 fullnameOverride: ""
--- a/services/wapi-pcidss-v0/values.yaml
+++ b/services/wapi-pcidss-v0/values.yaml
@ -4,7 +4,7 @@ image:

 imagePullSecrets: []

-replicaCount: 3
+replicaCount: 1

 beam:
  disableSBWT: false
--- a/tools/quick_reset.sh
+++ b/tools/quick_reset.sh
@ -4,5 +4,6 @@ kubectl delete deploy,rs,pvc,svc,crd,ing,sts,job,cj,cm,secret,sa --all \
 && kubectl delete mutatingwebhookconfigurations,validatingwebhookconfigurations prometheus-prometheus-oper-admission || echo "prometheus webhooks already deleted" \
 && kubectl delete ns monitoring elastic-system || echo "namespaces not found" \
 && kubectl delete -f https://raw.githubusercontent.com/cilium/cilium/v1.8/install/kubernetes/quick-install.yaml || echo "cilium not found" \
+&& sleep 45 \
 && kubectl apply -f https://raw.githubusercontent.com/cilium/cilium/v1.8/install/kubernetes/quick-install.yaml \
 && kubectl apply -f config/vault/init-cm.yaml