filebeat-8.16.autodiscover-container

configmap

apiVersion: v1
kind: ConfigMap
metadata:
  namespace: logs
  name: filebeat-config
  labels:
    app: filebeat
data:
  filebeat.yml: |-
    filebeat.autodiscover:
      providers:
        - type: kubernetes
          hints.enabled: true
          hints.default_config:
            type: container
            paths:
              - /var/log/containers/*-${data.kubernetes.container.id}.log 
            multiline.pattern: '^((\d{4}-\d{2}-\d{2})|{).*'
            multiline.negate: true
            multiline.match: after
            ignore_older: 24h
            clean_removed: true
            close_inactive: 10m
            backoff: 1s
            max_backoff: 10s
            harvester_buffer_size: 262144  # 256KB
            tail_files: false
            fields:
              k8s-cluster: hw-bj-new
            fields_under_root: true
  
    processors:
      - drop_event:
          when:
            or:
              - contains:
                  kubernetes.namespace: "kube-"
              - contains:
                  kubernetes.namespace: "logs"
              - contains:
                  kubernetes.namespace: "ops"
              - contains:
                  kubernetes.namespace: "prometheus"
              - contains:
                  kubernetes.namespace: "gke-monitor"
      - add_cloud_metadata:
      - add_host_metadata:
      - add_kubernetes_metadata:
          in_cluster: true
      - drop_fields:
          fields:
            - "kubernetes.namespace_labels"
            - "ecs"
            - "agent"
            - "[@version]"
            - "host"
            - "stream"
            - "kubernetes.namespace_uid"
            - "kubernetes.node.uid"
            - "kubernetes.node.hostname"
            - "kubernetes.labels"
            - "kubernetes.node.name"
            - "kubernetes.node.labels"
            - "kubernetes.pod.uid"
            - "kubernetes.replicaset.name"
            - "container.id"
            - "kubernetes.container.image"
            - "container.image.name"
            - "container.runtime"
            - "input.type"
            - "log.file.path"
            - "log.offset"
      - decode_json_fields:
          fields: ["message"]
          target: ""
          overwrite_keys: true
          max_depth: 1
      - script:
          lang: javascript
          id: limit_message
          source: >
            function process(event) {
              var message = event.Get("message");
              if (message !== undefined && message.length > 3000) {
                event.Put("message", message.substring(0, 3000));
              }
                event.Put("message", event.Get("message").replace(/[0-9A-Z]{50,}/g, ""));
                event.Put("message", event.Get("message").replace(/[a-zA-Z0-9\/+=|]{800,}/g, ""));
              return event;
            }  
    output.kafka:
      hosts: ["10.187.0.68:9092","10.187.0.240:9092","10.187.0.95:9092"]
      topic: k8s-test-logs-new
      compression: snappy
      compression_level: 9
      required_acks: 1
      broker_timeout: 10s
      channel_buffer_size: 10240
      keep_alive: 120
      max_message_bytes: 10000000
      worker: 4
      bulk_max_size: 1024
      timeout: 30s
  
    http:
      enabled: true
      host: 0.0.0.0
      port: 5066

daemonset

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: filebeat
  namespace: logs
  labels:
    k8s-app: filebeat
spec:
  selector:
    matchLabels:
      k8s-app: filebeat
  template:
    metadata:
      labels:
        k8s-app: filebeat
    spec:
      serviceAccountName: filebeat
      terminationGracePeriodSeconds: 30
      containers:
        - name: filebeat
          image: harbor.com/elk/filebeat:8.16.2
          args: ["-c", "/etc/filebeat.yml", "-e"]
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          securityContext:
            runAsUser: 0
          ports:
            - name: http
              containerPort: 5066
          resources:
            limits:
              memory: 300Mi
            requests:
              cpu: 100m
              memory: 100Mi
          livenessProbe:
            httpGet:
              path: /
              port: 5066
            initialDelaySeconds: 20
            periodSeconds: 30
            failureThreshold: 3
          readinessProbe:
            httpGet:
              path: /
              port: 5066
            initialDelaySeconds: 10
            periodSeconds: 30
          volumeMounts:
            - name: config
              mountPath: /etc/filebeat.yml
              readOnly: true
              subPath: filebeat.yml
            - name: varlibdockercontainers
              mountPath: /var/lib/docker/containers
              readOnly: true
            - name: varlog
              mountPath: /var/log
              readOnly: true
            - name: varpods
              mountPath: /var/log/pods
              readOnly: true
            - name: data
              mountPath: /usr/share/filebeat/data
      tolerations:
        - effect: NoSchedule
          operator: Exists
      volumes:
        - name: config
          configMap:
            defaultMode: 0640
            name: filebeat-config
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
        - name: varlog
          hostPath:
            path: /var/log
        - name: varpods
          hostPath:
            path: /var/log/pods
        - name: data
          hostPath:
            path: /var/lib/filebeat-data
            type: DirectoryOrCreate

filebeat-7.10.autodiscover-docker

configmap

apiVersion: v1
kind: ConfigMap
metadata:
  namespace: logs
  name: filebeat-config
  labels:
    app: filebeat
data:
  filebeat.yml: |-
    filebeat.autodiscover:
      providers:
        - type: kubernetes
          hints.enabled: true
          templates:
            - condition:
              config:
                - type: docker
                  containers.ids:
                    - "${data.kubernetes.container.id}"
                  multiline:
                    #pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
                    pattern: '^((\d{4}-\d{2}-\d{2})|{).*'
                    negate: true
                    match: after
                    ignore_older: 24h
                    clean_removed: true
                    close_inactive: 10m
                    backoff: 1s
                    max_backoff: 10s
                    harvester_buffer_size: 262144  # 256KB
                    tail_files: false
    processors:
      - drop_event:
          when: 
            or:
            - contains: 
                kubernetes.namespace: "kube-"
            - contains: 
                kubernetes.namespace: "logs"
            - contains: 
                kubernetes.namespace: "ops"
            - contains: 
                kubernetes.namespace: "prometheus"
            - contains: 
                kubernetes.namespace: "cattle-system"
            - regexp: 
                message: '^(2020|2021|2022)'
      - drop_fields:
          fields: ["container","kubernetes.labels","input","ecs","kubernetes.node.name","kubernetes.node","kubernetes.pod.uid","kubernetes.replicaset","kubernetes.container.image","log","offset","prospector.type","source","stream","host","agent","ecs","version"]
      - decode_json_fields:
          fields: ["message"]
          target: ""
          overwrite_keys: true
          max_depth: 1
      - script:
          lang: javascript
          id: limit_message
          source: >
            function process(event) {
              var message = event.Get("message");
              if (message !== undefined && message.length > 6000) {
                event.Put("message", message.substring(0, 6000));
              }
              event.Put("message", event.Get("message").replace(/[a-zA-Z0-9\/+=|]{600,}/g, "elk delete"));
              event.Put("message", event.Get("message").replace(/[0-9A-Z]{50,}/g, ""));
              return event;
            }
    output.kafka:
      hosts: ["10.19.5.179:9092"]
      topic: k8s-perf-logs
      compression: snappy
      compression_level: 9
      required_acks: 1
      broker_timeout: 10s
      channel_buffer_size: 1024
      keep_alive: 120


    http:
      enabled: true
      host: 0.0.0.0
      port: 5066

daemonset

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  labels:
    k8s-app: filebeat
  name: filebeat
  namespace: logs
spec:
  selector:
    matchLabels:
      k8s-app: filebeat
  template:
    metadata:
      creationTimestamp: null
      labels:
        k8s-app: filebeat
    spec:
      containers:
      - args:
        - -c
        - /etc/filebeat.yml
        - -e
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        image: ccr.ccs.tencentyun.com/tke-market/filebeat:7.10.1 
        imagePullPolicy: IfNotPresent
        name: filebeat
        securityContext:
          runAsUser: 0 
        ports:
          - name: http
            containerPort: 5066
        resources:
          limits:
            memory: 512Mi
          requests:
            cpu: 100m
            memory: 100Mi
        livenessProbe:
          httpGet:
            path: /
            port: 5066
          initialDelaySeconds: 20
          periodSeconds: 30
          failureThreshold: 3
        readinessProbe:
          httpGet:
            path: /
            port: 5066
          initialDelaySeconds: 10
        volumeMounts:
        - mountPath: /etc/filebeat.yml
          name: config
          readOnly: true
          subPath: filebeat.yml
        - mountPath: /var/lib/docker/containers
          name: varlibdockercontainers
          readOnly: true
        - mountPath: /var/log
          name: varlog
          readOnly: true
        - mountPath: /var/log/pods
          name: varpods
          readOnly: true
        - mountPath: /usr/share/filebeat/data
          name: data
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: filebeat
      serviceAccountName: filebeat
      terminationGracePeriodSeconds: 30
      tolerations:
      - effect: NoSchedule
        operator: Exists
      volumes:
      - configMap:
          defaultMode: 416
          name: filebeat-config
        name: config
      - hostPath:
          path: /opt/docker/containers
          type: ""
        name: varlibdockercontainers
      - hostPath:
          path: /var/log
          type: ""
        name: varlog
      - hostPath:
          path: /var/log/pods
          type: ""
        name: varpods
      - hostPath:
          path: /var/lib/filebeat-data
          type: DirectoryOrCreate
        name: data

script 部分作用：只采集message字段的前3000个字节。

filebeat.rbac

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: filebeat
subjects:
- kind: ServiceAccount
  name: filebeat
  namespace: logs
roleRef:
  kind: ClusterRole
  name: filebeat
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: filebeat
  namespace: logs
subjects:
  - kind: ServiceAccount
    name: filebeat
    namespace: logs
roleRef:
  kind: Role
  name: filebeat
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: filebeat-kubeadm-config
  namespace: logs
subjects:
  - kind: ServiceAccount
    name: filebeat
    namespace: logs
roleRef:
  kind: Role
  name: filebeat-kubeadm-config
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: filebeat
  labels:
    k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
  resources:
  - namespaces
  - pods
  - nodes
  verbs:
  - get
  - watch
  - list
- apiGroups: ["apps"]
  resources:
    - replicasets
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: filebeat
  # should be the namespace where filebeat is running
  namespace: logs
  labels:
    k8s-app: filebeat
rules:
  - apiGroups:
      - coordination.k8s.io
    resources:
      - leases
    verbs: ["get", "create", "update"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: filebeat-kubeadm-config
  namespace: logs
  labels:
    k8s-app: filebeat
rules:
  - apiGroups: [""]
    resources:
      - configmaps
    resourceNames:
      - kubeadm-config
    verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: filebeat
  namespace: logs
  labels:
    k8s-app: filebeat