k8s日志收集配置

容器日志樣例

172.101.32.1 - - [03/Jun/2019:17:14:10 +0800] "POST /ajaxVideoQueues!queryAllUser.action?rnd=1559553110429 HTTP/1.0" 200 65 "http://www.wsjy.gszq.com:81/sysNotice!sysList.action" "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko" "192.168.200.252"

ELK配置

日志處理pipeline

# 注意 \\[ ,中括號前的兩個轉義反斜杠
[root@elk100 pipe]# cat nginx_pipeline.json
{
    "description": "Nginx log pipeline",
    "processors": [
        {
           "grok" :{
                "field": "message",
                "patterns" : ["%{IP:clientip} - - \\[%{HTTPDATE:timestamp}\\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) \"(?:%{URI:referrer}|-)\" %{QS:agent} %{QS:xforwardedfor}]
            }

        },
        {
            "date": {
            "field": "timestamp",
            "formats": ["dd/MMM/YYYY:HH:mm:ss Z"]
            }
        }
    ],
    "on_failure" : [{
      "set" : {
        "field" : "error.message",
        "value" : "{{ _ingest.on_failure_message }}"
      }
    }]
}  

[root@elk100 pipe]# curl -H 'Content-Type: application/json' -XPUT 'http://10.101.70.100:9200/_ingest/pipeline/nginx_pipeline' -d@nginx_pipeline.json
{"acknowledged":true}

模板配置

在Kibana的 Dev Tools中執行

PUT _template/nginx_log 
{
  "index_patterns": "nginx_log*",
    "settings": {
      "refresh_interval": "5s",
      "number_of_shards": 1
    },
    "mappings": {
      "_doc": {
        "properties": {
          "id": {"type": "integer"},
          "clientip": {"type": "ip"},
          "timestamp": {"type": "date",
            "format": "dd/MMM/yyyy:HH:mm:ss Z"
          },
          "method": {"type": "keyword"},
          "request": {"type": "text"},
          "httpversion": {"type": "integer"},
          "response": {"type": "integer"},
          "bytes": {"type": "integer"},
          "referrer": {"type": "text"},
          "xforwardedfor": {"type": "text"}
        }
      }
    },
    "aliases": {}
  }
}

k8s容器編排文件

采用每個POD應用啟動一個 filebeat 容器來收集應用日志的方案。

fiebeat 鏡像下載： https://cloud.docker.com/u/bugbeta/repository/list

[root@node1 filebeat]# cat filebeat-test.yaml 
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: filebeat-test
  namespace: default
spec:
  replicas: 1
  template:
    metadata:
      labels:
        k8s-app: filebeat-test
    spec:
      containers:
      - image: bugbeta/filebeat:6.8.0
        name: filebeat
        volumeMounts:
        - name: app-logs
          mountPath: /log
        - name: filebeat-config
          mountPath: /etc/filebeat/
      - image: nginx:1.7.9 
        name : app
        ports:
        - containerPort: 80
        volumeMounts:
        - name: app-logs
          mountPath: /var/log/nginx
      volumes:
      - name: app-logs
        emptyDir: {}
      - name: filebeat-config
        configMap:
          name: filebeat-config
      nodeSelector:
        name: "node1"
---
apiVersion: v1
kind: Service
metadata:
  name: filebeat-test
  labels:
    app: filebeat-test
spec:
  type: NodePort
  ports:
  - port: 80
    nodePort: 30085
    protocol: TCP
    name: http
  selector:
    k8s-app: filebeat-test
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
data:
  filebeat.yml: |
    filebeat.prospectors:
    - type: log
      paths:
        - "/log/*"
    setup.template.name: "nginx_log" 
    setup.template.pattern: "nginx_log*" 

    output.elasticsearch:
      hosts: ["10.101.70.100:9200"]
      index: "nginx_log"
      pipeline: "nginx_pipeline"