kubernetes 安裝(全)

#http://blog.csdn.net/zhuchuangang/article/details/76572157
#https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/
#所有機器
#https://docs.docker.com/engine/installation/linux/docker-ce/centos/#install-using-the-repository
#安裝docker,安裝的版本取決於kubernetes支持docker的版本
#到這上面下載https://yum.dockerproject.org/repo/main/centos/7/Packages/
sudo systemctl start docker
#測試。出現Hello from Docker!
sudo docker run hello-world

#####################################

##################################
#https://coreos.com/etcd/docs/latest/
#https://github.com/coreos/etcd/
#安裝etcd
#准備
openssl genrsa -out IE.key 2048
openssl req -new -key IE.key  -out IE.csr
openssl x509 -req -in IE.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out IE.crt -days 5000
openssl pkcs12 -export -clcerts -in IE.crt -inkey IE.key -out IE.p12
openssl genrsa -out IE.key 2048
openssl req -new -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=system:masters/CN=admin"  -key IE.key  -out IE.csr
openssl x509 -req -in IE.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out IE.crt -days 5000
openssl pkcs12 -export -clcerts -in IE.crt -inkey IE.key -out IE.p12
#配置服務
tar xf etcd.tar.gz
cd etcd
cp etcd etcdctl /usr/bin
cat << EOF > /usr/lib/systemd/system/etcd.service
[unit]
Description=Etcd Server
After=network.target

[Service]
Type=simple
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=/etc/etcd/etcd.conf
ExecStart=/usr/bin/etcd $ETCD_CMD

[Install]
WantedBy=multi-user.target
EOF
cat << EOF > /etc/etcd/etcd.conf
ETCD_CMD="--listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://0.0.0.0:2379 "
EOF
systemctl daemon-reload
systemctl enable etcd.service
systemctl start etcd.service
systemctl status etcd.service
#創建網絡
etcdctl set /coreos.com/network/config '{ "Network":"10.1.0.0/16" }'
#驗證
etcdctl cluster-health







#member 8e9e05c52164694d is healthy: got healthy result from http://localhost:2379
###################################
#准備
#hosts
192.168.1.1 master
192.168.1.2 minion-1
192.168.1.3 minion-2
#下載,並解壓
https://github.com/kubernetes/kubernetes/releases/tag/v1.7.5
cd kubernetes
#下載kubernetes-server-linux-amd64.tar.gz
sh cluster/get-kube-binaries.sh
cd server
tar xf kubernetes-server-linux-amd64.tar.gz
cd server/bin/
#准備證書
mkdir ca
cd ca
openssl genrsa -out ca.key 2048
openssl req -x509 -new -nodes -key ca.key -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=system:masters/CN=test.com" -days 5000 -out ca.crt
openssl genrsa -out server.key 2048
#配置master_ssl.cnf
cat << EOF > master_ssl.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints= CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName= @alt_names
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.cluster.local
DNS.5 = master
IP.1 = 169.169.0.1
IP.2 = 192.168.1.1
EOF
openssl req -new -key server.key -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=system:masters/CN=master" -config master_ssl.cnf -out server.csr
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 5000 -extensions v3_req -extfile master_ssl.cnf -out server.crt
#全部執行完有6個文件復制到一個路徑(如：/etc/kubernetes/crt/)
#設置kube-controller-manager證書
openssl genrsa -out cs_client.key 2048
openssl req -new -key cs_client.key -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=system:masters/CN=master" -out cs_client.csr
openssl x509 -req -in cs_client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out cs_client.crt -days 5000
#創建kubeconfig文件
cat << EOF > /etc/kubernetes/kubeconfig
apiVersion: v1
kind: Config
users:
- name: controllermanager
  user:
    client-certificate: /etc/kubernetes/crt/cs_client.crt
    client-key: /etc/kubernetes/crt/cs_client.key
clusters:
- name: local
  cluster:
    certificate-authority: /etc/kubernetes/crt/ca.crt
contexts:
- context:
    cluster: local
    user: controllermanager
  name: my-context
current-context: my-context
EOF

###################################
#安裝kube-apiserver
cp ../kube-apiserver /usr/bin
cat << EOF > /usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=etcd.service
Wants=etcd.service

[Service]
EnvironmentFile=/etc/kubernetes/apiserver
ExecStart=/usr/bin/kube-apiserver $KUBE_API_ARGS
Restart=on-failure
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#配置文件
cat << EOF > /etc/kubernetes/apiserver
KUBE_API_ARGS="--etcd_servers=http://127.0.0.1:2379 --insecure-bind-address=0.0.0.0 \
--service-cluster-ip-range=169.169.0.0/16 \
--service-node-port-range=1-65535 \
--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota \
--logtostderr=false --log-dir=/var/log/kubernetes --v=2 \
--client_ca_file=/etc/kubernetes/crt/ca.crt \
--tls-private-key-file=/etc/kubernetes/crt/server.key \
--tls-cert-file=/etc/kubernetes/crt/server.crt \
--insecure-port=0 \
--secure-port=8080"
EOF

######################################
#安裝kube-controller-manager
cp ../kube-controller-manager /usr/bin
cat << EOF > /usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=kube-apiserver.service
Requires=kube-apiserver.service

[Service]
EnvironmentFile=/etc/kubernetes/controller-manager
ExecStart=/usr/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF
#配置
cat << EOF >/etc/kubernetes/controller-manager
KUBE_CONTROLLER_MANAGER_ARGS="--master=https://192.168.1.1:8080 --logtostderr=false --log-dir=/var/log/kubernetes --v=2 --service_account_private_key_file=/etc/kubernetes/crt/server.key --root-ca-file=/etc/kubernetes/crt/ca.crt --kubeconfig=/etc/kubernetes/kubeconfig"
EOF

##########################################
#安裝kube-scheduler
cp ../kube-scheduler /usr/bin
cat << EOF > /usr/lib/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=kube-apiserver.service
Requires=kube-apiserver.service

[Service]
EnvironmentFile=/etc/kubernetes/scheduler
ExecStart=/usr/bin/kube-scheduler $KUBE_SCHEDULER_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF
#配置
cat << EOF > /etc/kubernetes/scheduler
KUBE_SCHEDULER_ARGS="--master=https://192.168.1.1:8080 --logtostderr=false \
--log-dir=/var/log/kubernetes --v=2 \
--kubeconfig=/etc/kubernetes/kubeconfig \
--service-cluster-ip-range=169.169.0.0/16"
EOF

#啟動
systemctl daemon-reload
service="kube-apiserver kube-apiserver kube-scheduler"
for i in $service
do
    systemctl enable $i
    systemctl restart $i
    systemctl status $i
done

############################################
#配置節點
#准備
#https://github.com/kubernetes/kubernetes/releases/tag/v1.7.5
#下載kubernetes-server-linux-amd64.tar.gz
cd kubernetes
sh cluster/get-kube-binaries.sh
cd server
tar xf kubernetes-server-linux-amd64.tar.gz
cd server/bin/
#創建證書
mkdir ca
cd ca
#將master的ca.crt和ca.key復制到節點上
openssl genrsa -out kubelet_client.key 2048
openssl req -new -key kubelet_client.key -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=system:masters/CN=218.71.143.140" -out kubelet_client.csr
openssl x509 -req -in kubelet_client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out kubelet_client.crt -days 5000
#創建kubeconfig文件
cat << EOF > /etc/kubernetes/kubeconfig
apiVersion: v1
kind: Config
users:
- name: kubelet
  user:
    client-certificate: /etc/kubernetes/crt/kubelet_client.crt
    client-key: /etc/kubernetes/crt/kubelet_client.key
clusters:
- name: local
  cluster:
    certificate-authority: /etc/kubernetes/crt/ca.crt
contexts:
- context:
    cluster: local
    user: kubelet
  name: my-context
current-context: my-context
EOF

################################
#安裝kubelet
cp ../kubelet /usr/bin/
cat << EOF > /usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
EnvironmentFile=/etc/kubernetes/kubelet
ExecStart=/usr/bin/kubelet $KUBELET_ARGS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF
cat << EOF >/etc/kubernetes/kubelet
KUBELET_ARGS="--api-servers=https://192.168.1.1:8080 --hostname-override=minion-2 --logtostderr=false \
--log-dir=/var/log/kubernetes --v=2 \
--kubeconfig=/etc/kubernetes/kubeconfig \
--root-dir=/data/kubelet \
--cluster-dns=169.169.0.10"
EOF

#安裝kube-proxy
cp ../kube-proxy /usr/bin
cat << EOF >/usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
Requires=network.service

[Service]
EnvironmentFile=/etc/kubernetes/proxy
ExecStart=/usr/bin/kube-proxy $KUBE_PROXY_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF
cat << EOF >/etc/kubernetes/proxy
KUBE_PROXY_ARGS="--master=https://192.168.1.1:8080 \
--logtostderr=false --log-dir=/var/log/kubernetes --v=2 \
--kubeconfig=/etc/kubernetes/kubeconfig \
--cluster-cidr=169.169.0.0/16"
EOF
#啟動
systemctl daemon-reload
service="kubelet.service kube-proxy.service"
for i in $service
do
systemctl enable $i
systemctl restart $i
systemctl status $i
done

#flannel
#所有master和node都安裝
#下載https://github.com/coreos/flannel/releases
#解壓並把flanneld和mk-codker-opts.sh復制到/usr/bin
#配置服務
cp mk-docker-opts.sh flanneld /usr/bin/
cat << EOF > /usr/lib/systemd/system/flanneld.service
[Unit]
Description=flanneld overlay address etcd agent
After=network.target
Before=docker.service

[Service]
Type=notify
EnvironmentFile=/etc/sysconfig/flannel
ExecStart=/usr/bin/flanneld -etcd-endpoints=${FLANNEL_ETCD} $FLANNEL_OPTIONS

[Install]
RequiredBy=docker.service
WantedBy=multi-user.target
EOF
#創建文件
cat << EOF > /etc/sysconfig/flannel
FLANNEL_ETCD="http://192.168.1.1:2379"
FLANNEL_ETCD_KEY="/coreos.com/network"
EOF
#停止docker
systemctl daemon-reload
systemctl stop docker
systemctl start flanneld
#替換docker ip
mk-docker-opts.sh -i
source /run/flannel/subnet.env
ifconfig docker0 ${FLANNEL_SUBNET}
#修改docker
mk-docker-opts.sh -d /etc/docker/docker_opts.env -c
#/usr/lib/systemd/system/docker.service
    #修改，修改了存儲目錄和啟動方式
    ExecStart=/usr/bin/dockerd $DOCKER_OPTS --graph=/data/docker
    #新增
    EnvironmentFile=/etc/docker/docker_opts.env
#驗證
ip addr
#啟動docker
systemctl daemon-reload
systemctl restart docker
#etcd驗證
etcdctl ls /coreos.com/network/subnets
#ping驗證，通過etcd查看到的網段
ping 10.1.50.1
ping 10.1.46.1
#docker禁止了轉發，導致創建的pod無法跨主機互通
iptables -P FORWARD ACCEPT



######################################
#生成windows證書,將生成的證書IE.p12導入到IE個人證書
#http://www.jianshu.com/p/045f95c008a0
openssl genrsa -out IE.key 2048
openssl req -new -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=system:masters/CN=admin"  -key IE.key  -out IE.csr
openssl x509 -req -in IE.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out IE.crt -days 5000
openssl pkcs12 -export -clcerts -in IE.crt -inkey IE.key -out IE.p12


##################################
#UI
#下載配置
wget https://rawgit.com/kubernetes/dashboard/master/src/deploy/kubernetes-dashboard.yaml
kubectl create -f kubernetes-dashboard.yaml
#查看安裝狀態
kubectl get pods --all-namespaces
#訪問,將前面的IE證書導入
https://master.abc.com/ui

##################################
#DNS
#cd kubernetes/cluster/addons/dns
#修改transforms2sed.sed里的$DNS_SERVER_IP和$DNS_DOMAIN替換成169.169.0.10,和cluster.local
#生成yuml文件
sed -e 'a/$DNS_SERVER_IP/169.169.0.10/g' -e 's/$DNS_DOMAIN/cluster.local/g' transforms2sed.sed
sed -f transforms2sed.sed kubedns-svc.yaml.base > kubedns-svc.yaml
sed -f transforms2sed.sed kubedns-controller.yaml.base > kubedns-controller.yaml
kubectl create -f ../dns
#DNS自動擴容
kubectl create -f ../dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml
#驗證,可在UI上查看也可執行
kubectl get pods --all-namespaces

####################################
#監控工具
#當前最新版本 heapster https://github.com/kubernetes/heapster/archive/v1.5.0-beta.0.tar.gz
#修改 influxdb/grafana.yaml
#刪除
        - name: GF_AUTH_BASIC_ENABLED
          value: "false"
        - name: GF_AUTH_ANONYMOUS_ORG_ROLE
          value: Admin
#修改,開啟認證
        - name: GF_AUTH_ANONYMOUS_ENABLED
          value: "false"
#修改
  type: NodePort
  ports:
  - port: 80
    targetPort: 3000
    nodePort: 3001
#修改influxdb/heapster.yaml
- --source=kubernetes:https://kubernetes.default.svc.cluster.local
- --sink=influxdb:http://monitoring-influxdb.kube-system.svc.cluster.local:8086
#創建
kubectl create -f deploy/kube-config/influxdb/
kubectl create -f deploy/kube-config/rbac/heapster-rbac.yaml
#訪問
kubernetes-dashboard 也可以看到等一會出現圖形了
http://nodeip:3001 賬號密碼admin,也可以查看

############################################
#docker環境修改,由於被牆了所以無法下載谷歌的軟件,需要先下載到倉庫或翻牆
#或者直接到服務器上使用翻牆軟件
#將修改版的翻牆軟件上傳到服務器
#3128翻牆軟件的端口
iptables -A INPUT -s 127.0.0.1 -p tcp --dport '3128' -j ACCEPT
iptables -A INPUT -p tcp --dport '3128' -j DROP
#啟動
#修改啟動參數
vi /usr/lib/systemd/system/docker.service
[Service]
Environment="HTTP_PROXY=http://127.0.0.1:3128"
ExecStart=/usr/bin/dockerd --insecure-registry gcr.io
systemctl daemon-reload
systemctl start docker.service
systemctl status docker.service

##############################
#kubectl 使用,存在~/.kube/config里,刪除即可清空
kubectl config set-cluster default-cluster --server=https://master:8080 --certificate-authority=/etc/kubernetes/crt/ca.crt
kubectl config set-credentials default-admin --certificate-authority=/etc/kubernetes/crt/ca.crt --client-key=/etc/kubernetes/crt/cs_client.key --client-certificate=/etc/kubernetes/crt/cs_client.crt
kubectl config set-context default-system --cluster=default-cluster --user=default-admin
kubectl config use-context default-system
#kubectl --server https://master:443 --certificate-authority /etc/kubernetes/crt/ca.crt --client-certificate /etc/kubernetes/crt/cs_client.crt --client-key /etc/kubernetes/crt/cs_client.key get nodes
kubectl get nodes

############################################
#所有節點
#glusterfs
#http://www.cnblogs.com/jicki/p/5801712.html
#https://jimmysong.io/blogs/kubernetes-with-glusterfs/
yum install centos-release-gluster
yum install -y glusterfs glusterfs-server glusterfs-fuse glusterfs-rdma
# 創建 glusterfs 目錄
mkdir /data/glusterd
sed -i 's#var/lib#dara#g' /etc/glusterfs/glusterd.vol
# 啟動 glusterfs
systemctl start glusterd.service
# 設置開機啟動
systemctl enable glusterd.service
#查看狀態
systemctl status glusterd.service
#開放端口,只對節點IP開放
ip=192.168.1.1,192.168.1.2,192.168.1.3
for i in $ip
do
    iptables -I INPUT -s $i  -p tcp -m multiport --dport 24007,49152 -j ACCEPT
done

#創建存儲目錄
mkdir /data/gfs_data
#添加節點,在master主機上執行
gluster peer probe minion-1
gluster peer probe minion-2
#查看狀態
gluster peer status
#允許所有
#gluster volume reset disp_vol auth.allow
#限制IP
gluster volume set disp_vol auth.allow 192.168.1.1,192.168.1.2,192.168.1.3
#創建復制卷,副本數為3
gluster volume create test-volume replica 3 transport tcp master:/data/gfs_data minion-1:/data/gfs_data minion-2:/data/gfs_data
#調優，緩存過大可能突然重啟斷電等情況導致數據丟失
#啟動卷
gluster volume start test-volume
#查看卷狀態
gluster volume info
#設置配額
gluster volume quota test-volume enable
gluster volume quota test-volume limit-usage / 300GB
#設置緩存
gluster volume set test-volume performance.cache-size 2GB
#設置io線程
gluster volume set test-volume performance.io-thread-count 16
#設置網絡檢測時間
gluster volume set test-volume network.ping-timeout 10
#設置寫緩沖大小
gluster volume set test-volume performance.write-behind-window-size 512MB
#修改addresses ip每一個一組,port改為24007
#kubectl apply更新
#配置glusterfs節點ip和端口
kubectl apply -f ./kubernetes/examples/volumes/glusterfs/glusterfs-endpoints.json
#配置集群端口
kubectl apply -f ./kubernetes/examples/volumes/glusterfs/glusterfs-service.json
#kubectl apply -f demo.yum,添加了下面兩段,掛載到/data目錄,安裝好demo.yaml后可df -h查看data目錄是否掛載了300G
#在yaml的containers里添加
"volumes": [
            "volumeMounts": [
                {
                    "mountPath": "/data",
                    "name": "glusterfdata"
                }
            ]
          }
#在yaml的containers下面添加
"volumes": [
            {
                "name": "glusterfdata",
                "glusterfs": {
                    "endpoints": "glusterfs-cluster",
                    "path": "test-volume",
                    "readOnly": false
                }
            }
        ],
#也可直接掛載的物理主機
mount.glusterfs 192.168.1.1:/test-volume /data/mnt
#創建pv
cat << EOF > glusterfs-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: gluster-disk-1
spec:
  capacity:
    storage: 300Gi
  accessModes:
    - ReadWriteMany
  glusterfs:
    endpoints: "glusterfs-cluster"
    path: "test-volume"
    readOnly: false
EOF
kubectl apply -f glusterfs-pv.yaml
kubectl get pv
#創建PVC
cat << EOF>glusterfs-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: glusterfs-disk-1
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 300Gi
EOF
kubectl apply -f glusterfs-pvc.yaml
kubectl get pvc
kubect apply demo.yaml
################################
#運行centos鏡像
#運行命令填 /sbin/init,運行參數填2

#######################################

#錯誤
    error: failed to run Kubelet: failed to create kubelet: misconfiguration: kubelet cgroup driver: "systemd" is different from docker cgroup driver: "cgroupfs"
    #檢查docker
    docker info | grep Cgr
    #修改/etc/systemd/system/kubelet.service.d/10-kubeadm.conf里的cgroup與docker的cgroup一致
    #執行
    systemctl daemon-reload
    
    服務無法啟動但是檢查了配置都沒問題
    #手動啟動服務，檢查端口
    
可以使用如下的命令創建一個用於客戶端認證的證書

openssl req
-new
-nodes \ -x509
-subj "/C=CN/ST=GuangDong/L=ShenZhen/O=HuaWei/OU=PaaS/CN=batman"
-days 3650
-keyout 私鑰.key
-out 證書.crt

說明： /C 表示國家只能為兩個字母的國家縮寫，例如CN，US等 /ST 表示州或者省份 /L 表示城市或者地區 /O 表示組織機構名稱 /OU 表示組織機構內的部門或者項目名稱 /CN 表示公用名，如果用來作為SSL證書則應該填入域名或者子域名， 如果作為客戶端認證證書則可以填入期望的用戶名 為API Server指定要應用的客戶端認證證書 將上一步創建的證書文件拷貝到API Server所在的主機，然后通過啟動參數--client-ca-file將證書文件的路徑傳遞給API Server。