k8s實現elk日志收集系統
說明：es搭建集群，3台master，3台node，node角色分別為熱數據，溫數據，冷數據
kibana進行數據展示，同時連接es的工具使用es-head。
此處使用deployment+service結合搭建,logstash還未進行試驗，等我繼續更新。。。
master節點搭建
######### 創建名稱空間 ###########
kind: Namespace
apiVersion: v1
metadata:
name: elastic
---
######## service類型為nodeport用以暴露es端口供外部訪問 ########
kind: Service
apiVersion: v1
metadata:
name: es-cluster-nodeport
namespace: elastic
spec:
type: NodePort
selector:
app: es-cluster
ports:
- port: 9200
targetPort: 9200
nodePort: 32700
---
####### es集群間通訊端口，使用service進行暴露，但不像nodeport可對集群外服務器提供訪問 #############
kind: Service
apiVersion: v1
metadata:
name: es-svc
namespace: elastic
labels:
app: es-cluster
annotations:
es.kubernetes.io/es: es-cluster
spec:
selector:
app: es-cluster
role: master
ports:
- port: 9300
targetPort: 9300
---
###### es的master創建 #############################
kind: Deployment # 資源類型
apiVersion: apps/v1 # 版本
metadata: # 元數據
namespace: elastic # 名稱空間
name: es-cluster-master # deployment類型的元數據名
labels: # 標簽
app: es-cluster-master # 定義deployment的標簽
role: master
spec: # 說明
replicas: 3 # 副本數為3,表示創建3個pod
selector: # 選擇器
matchLabels: # 精確選擇
app: es-cluster # 選擇標簽為 app: es-cluster
role: master # 選擇標簽為 role: master
template: # 模板
metadata: # 元數據信息
labels: # 標簽
app: es-cluster # 此模板的信息
role: master
spec: # 模板對應的說明
containers: # 容器
- name: elasticsearch-master # 名稱
image: registry.cn-hangzhou.aliyuncs.com/k8sos/es:v1 # 鏡像
ports: # 端口
- containerPort: 9200 # 這里的端口不提供暴露，僅說明
protocol: TCP
- containerPort: 9300
protocol: TCP
lifecycle: # 生命周期，也是回調鈎子
postStart: # 當容器啟動后執行的操作
exec: # 執行命令
command:
- "/bin/sh"
- "-c"
- "sysctl -w vm.max_map_count=262144; ulimit -l unlimited;"
securityContext: # 安全上下文，指具備對系統的操作權限
privileged: true
readinessProbe: # 就緒性探測
httpGet: # 探測9200端口是否就緒
port: 9200
volumeMounts: # 存儲性掛載
- mountPath: /usr/share/elasticsearch/data # 容器中需要掛載的路徑
name: es-data # 掛載到的地方，這個名稱對應下方的掛載
env: # 此處定義的環境變量相當於在命令行執行命令，替代了配置文件的操作
- name: "cluster.name"
value: "es-cluster"
- name: "bootstrap.memory_lock"
value: "true"
- name: "discovery.zen.ping.unicast.hosts"
value: "es-svc"
- name: "discovery.zen.minimum_master_nodes"
value: "2"
- name: "discovery.zen.ping_timeout"
value: "5s"
- name: "node.master"
value: "true"
- name: "node.data"
value: "false"
- name: "ES_JAVA_OPTS"
value: "-Xms256m -Xmx256m"
- name: "http.cors.enabled"
value: "true"
- name: "http.cors.allow-origin"
value: "*"
volumes: # 對應上方的掛載名
- name: es-data
emptyDir: {}
head插件
kind: Deployment
apiVersion: apps/v1
metadata:
namespace: elastic
name: es-head
spec:
selector:
matchLabels: # deployment精確匹配選擇對應的模板
app: es-head
template:
metadata:
labels:
app: es-head # 模板標簽信息，供deployment選擇
spec:
containers:
- name: head
image: alvinos/elasticsearch-head
ports:
- containerPort: 9100
protocol: TCP
---
##### 使用nodeport型的service，將pod中的端口進行暴露 ##########
kind: Service
apiVersion: v1
metadata:
name: es-head-svc
namespace: elastic
spec:
type: NodePort
selector:
app: es-head
ports:
- port: 9100
targetPort: 9100
nodePort: 32701
node節點--hot
kind: Deployment
apiVersion: apps/v1
metadata:
name: es-cluster-data-hot
namespace: elastic
labels: # deployment的標簽
app: es-cluster-data-hot
role: data
spec:
selector:
matchLabels: # deployment選擇器精確匹配模板中的信息
app: es-cluster
role: data
datatype: hot
template:
metadata:
labels: # 供上方deployment選擇器選擇
app: es-cluster
role: data
datatype: hot
spec:
containers:
- name: es-cluster-hot
image: registry.cn-hangzhou.aliyuncs.com/k8sos/es:v1
ports:
- containerPort: 9200
protocol: TCP
- containerPort: 9300
protocol: TCP
env: # 環境變量
- name: "cluster.name"
value: "es-cluster"
- name: "bootstrap.memory_lock"
value: "true"
- name: "discovery.zen.ping.unicast.hosts"
value: "es-svc"
- name: "node.master"
value: "false"
- name: "node.data"
value: "true"
- name: "ES_JAVA_OPTS"
value: "-Xms256m -Xmx256m"
- name: "node.attr.box_type"
value: "hot" # 指定為hot數據類型
lifecycle:
postStart: # 容器啟動后的操作
exec:
command:
- "/bin/sh"
- "-c"
- "sysctl -w vm.max_map_count=262144; ulimit -l unlimited;"
securityContext:
privileged: true
volumeMounts:
- mountPath: /usr/share/elasticsearch/data
name: es-data
volumes:
- name: es-data
emptyDir: {}
node節點--warm
kind: Deployment
apiVersion: apps/v1
metadata:
name: es-cluster-data-warm
namespace: elastic
labels:
app: es-cluster-data-warm
role: data
spec:
selector:
matchLabels:
app: es-cluster
role: data
datatype: warm
template:
metadata:
labels:
app: es-cluster
role: data
datatype: warm
spec:
containers:
- name: es-cluster-warm
image: registry.cn-hangzhou.aliyuncs.com/k8sos/es:v1
ports:
- containerPort: 9200
protocol: TCP
- containerPort: 9300
protocol: TCP
env:
- name: "cluster.name"
value: "es-cluster"
- name: "bootstrap.memory_lock"
value: "true"
- name: "discovery.zen.ping.unicast.hosts"
value: "es-svc"
- name: "node.master"
value: "false"
- name: "node.data"
value: "true"
- name: "ES_JAVA_OPTS"
value: "-Xms256m -Xmx256m"
- name: "node.attr.box_type"
value: "warm"
lifecycle:
postStart:
exec:
command:
- "/bin/sh"
- "-c"
- "sysctl -w vm.max_map_count=262144; ulimit -l unlimited;"
securityContext:
privileged: true
readinessProbe:
httpGet:
port: 9200
volumeMounts:
- mountPath: /usr/share/elasticsearch/data
name: es-data
volumes:
- name: es-data
emptyDir: {}
node節點--cold
kind: Deployment
apiVersion: apps/v1
metadata:
name: es-cluster-data-cold
namespace: elastic
labels:
app: es-cluster-data-cold
role: data
spec:
selector:
matchLabels:
app: es-cluster
role: data
datatype: cold
template:
metadata:
labels:
app: es-cluster
role: data
datatype: cold
spec:
containers:
- name: es-cluster-cold
image: registry.cn-hangzhou.aliyuncs.com/k8sos/es:v1
ports:
- containerPort: 9200
protocol: TCP
- containerPort: 9300
protocol: TCP
env:
- name: "cluster.name"
value: "es-cluster"
- name: "bootstrap.memory_lock"
value: "true"
- name: "discovery.zen.ping.unicast.hosts"
value: "es-svc"
- name: "node.master"
value: "false"
- name: "node.data"
value: "true"
- name: "ES_JAVA_OPTS"
value: "-Xms256m -Xmx256m"
- name: "node.attr.box_type"
value: "cold"
lifecycle:
postStart:
exec:
command:
- "/bin/sh"
- "-c"
- "sysctl -w vm.max_map_count=262144; ulimit -l unlimited;"
securityContext:
privileged: true
readinessProbe:
httpGet:
port: 9200
volumeMounts:
- mountPath: /usr/share/elasticsearch/data
name: es-data
volumes:
- name: es-data
emptyDir: {}