搭建docker私有倉庫，建立k8s集群

本文轉載自查看原文 2016-12-02 11:40 18003 docker k8s

服務器IP角色分布

192.168.5.2  etcd server
192.168.5.2  kubernetes master
192.168.5.3  kubernetes node
192.168.5.4  kubernetes node

確認環境

centos7
確認liunx內核版本  uname -a
yum update

systemctl start firewalld.service#啟動firewall
systemctl stop firewalld.service#停止firewall
systemctl disable firewalld.service#禁止firewall開機啟動


yum -y install ntp
systemctl start ntpd
systemctl enable ntpd

軟件安裝

etcd
yum install etcd -y

master
yum install kubernetes-master -y

node
yum install kubernetes-node flannel docker -y

各組件用途

kube master
    kube-apiserver
    kube-scheduer
    kube-controller-manager

etcd配置啟動

egrep -v “^#” /etc/etcd/etcd.conf
ETCD_NAME=default
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.5.2:2379"


[root@Control k8s]# cat etcd_start.sh
systemctl enable etcd
systemctl start etcd

ss -antl 檢查2379是否成功

master服務配置啟動

[root@Control k8s]# egrep -v '^#' /etc/kubernetes/apiserver | grep -v '^$'
KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"
KUBE_ETCD_SERVERS="--etcd-servers=http://192.168.5.2:2379"
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
KUBE_API_ARGS=""

[root@Control k8s]#  egrep -v '^#' /etc/kubernetes/controller-manager |grep -v '^$'
KUBE_CONTROLLER_MANAGER_ARGS="--node-monitor-grace-period=10s --pod-eviction-timeout=10s"

[root@Control k8s]#      egrep -v '^#' /etc/kubernetes/config | egrep -v '^$'
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=http://192.168.5.2:8080"
  
  
[root@Control k8s]# cat master_start.sh
 systemctl enable kube-apiserver kube-scheduler kube-controller-manager
 systemctl start kube-apiserver kube-scheduler kube-controller-manager

 ss -antl 檢查8080是否成功

node服務配置啟動

[root@Resources-s1 k8s]# egrep -v '^#' /etc/kubernetes/config | grep -v '^$'
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=http://192.168.5.2:8080"

[root@Resources-s1 k8s]# egrep -v '^#' /etc/kubernetes/kubelet | grep -v '^$'
KUBELET_ADDRESS="--address=127.0.0.1"
KUBELET_HOSTNAME="--hostname-override=192.168.5.3"
KUBELET_API_SERVER="--api-servers=http://192.168.5.2:8080"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"
KUBELET_ARGS=""

[root@Resources-s1 k8s]# cat node_start.sh
systemctl enable kubelet kube-proxy
systemctl start kubelet kube-proxy

網絡配置

etcd配置內網信息

etcdctl -C 192.168.5.2:2379 set /playcrab-inc.com/network/config '{ "Network": "10.1.0.0/16" }'

node配置flanneld

[root@Resources-s1 k8s]# egrep -v '^#' /etc/sysconfig/flanneld | grep -v '^$'
FLANNEL_ETCD="http://192.168.5.2:2379"
FLANNEL_ETCD_KEY="/playcrab-inc.com/network"

[root@Resources-s1 k8s]# cat flanneld_start.sh
#systemctl enable  flanenld
systemctl restart flanneld

[root@Resources-s1 k8s]# cat docker_start.sh
systemctl enable docker
systemctl restart docker

檢查集群啟動情況

[root@Control k8s]#  kubectl get nodes
NAME          STATUS    AGE
192.168.5.3   Ready     28m
192.168.5.4   Ready     25m

======================================================

docker私有倉庫建立

環境說明

我們選取192.168.5.2做私有倉庫地址
yum install docker -y

1.啟動docker倉庫端口服務

docker run -d -p 5000:5000 --privileged=true -v /data/history:/data/registry registry  

[root@Control docker_dw_images]# docker images
REPOSITORY           TAG                 IMAGE ID            CREATED             SIZE
docker.io/registry   latest              c9bd19d022f6        6 weeks ago         33.27 MB

2.查看docker倉庫端口服務

# curl -XGET http://192.168.5.2:5000/v2/_catalog
# curl -XGET http://192.168.5.2:5000/v2/image_name/tags/list

3.將自己的鏡像加到docker倉庫

1.1自己做基礎鏡像並加載到docker中
    cd centos6-image && tar -c .|docker import - centos6-base
1.2 創建一個帶ssh的基礎鏡像
    mkdir centos6-ssh
    cd centos6-ssh
    vim  Dockerfile
    輸入
    FROM centos6-base
    MAINTAINER wuqichao <wuqichao@playcrab.com>
    RUN ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
    RUN ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key
    RUN sed -ri 's/session    required     pam_loginuid.so/#session    required     pam_loginuid.so/g' /etc/pam.d/sshd
    RUN mkdir -p /root/.ssh && chown root.root /root && chmod 700 /root/.ssh
    EXPOSE 22
    RUN echo 'root:xxx.com.cn' | chpasswd
    ENV LANG en_US.UTF-8
    ENV LC_ALL en_US.UTF-8
    CMD /usr/sbin/sshd -D
    保存退出
    運行如下指令
    docker build -t centos6-ssh .
    不報錯的話，就完成本地鏡像
1.3 測試啟動ssh的基礎鏡像
    docker run -d -p 127.0.0.1:33333:22 centos6-ssh
1.4 登錄ssh的基礎鏡像實例
    ssh root@127.0.0.1 -p 33333



2.加載到自己的私有倉庫
    ###docker pull docker.io/nginx
    如果是本地建立docker不用執行上面的
    docker tag centos6-ssh 192.168.5.2:5000/centos6-ssh
    docker push 192.168.5.2:5000/centos6-ssh
    
3.檢查是否成功    
 [root@Control k8s]# curl -XGET http://192.168.5.2:5000/v2/_catalog
{"repositories":["centos6-ssh"]}

k8s中使用docker私有倉庫

環境設置

1.1.設置服務端

[root@Control k8s_yaml]# cat /etc/sysconfig/docker|grep 192.168.5.2
OPTIONS='--insecure-registry 192.168.5.2:5000 --log-driver=journald'
ADD_REGISTRY='--add-registry 192.168.5.2:5000'

1.2.設置客戶端

[root@Control k8s]# cat /etc/default/docker
DOCKER_OPTS="--insecure-registry 192.168.5.2:5000"

1.3.去掉權限驗證

在/etc/kubernetes/apiserver中
去除 KUBE_ADMISSION_CONTROL中的    SecurityContextDeny,ServiceAccount，
並重啟kube-apiserver.service服務
#systemctl restart kube-apiserver.service

1.4.加上DNS服務不然后報錯

KUBELET_ARGS="--cluster-dns=192.168.5.2 --cluster-domain=playcrab-inc.com"

配置YAML

2.0 常用指令

啟動指令    
 kubectl create -f centos6-ssh/centos6-ssh.yaml

刪除指令    
kubectl delete -f centos6-ssh/centos6-ssh.yaml

查看指令    
kubectl get pods

查看細節指令
 kubectl describe pod centos6-ssh

2.1啟動最簡單的pod

2.1.1 yaml配置

[root@Control k8s_yaml]# cat centos6-ssh/centos6-ssh.yaml
apiVersion: v1
kind: Pod
metadata:
  name: centos6-ssh
spec:
  containers:
  - name: centos6-ssh
    image: centos6-ssh

2.1.2 查看指令

[root@Control k8s_yaml]# kubectl get pods
NAME                READY     STATUS    RESTARTS   AGE
centos6-ssh-mucsv   1/1       Running   0          10m

2.1.3 查看細節指令

 kubectl describe pod centos6-ssh  

[root@Control k8s_yaml]# kubectl describe pod centos6-ssh
Name:        centos6-ssh
Namespace:    default
Node:        192.168.5.3/192.168.5.3
Start Time:    Wed, 30 Nov 2016 13:44:51 -0500
Labels:        <none>
Status:        Running
IP:        10.1.75.2
Controllers:    <none>
Containers:
  centos6-ssh:
    Container ID:    docker://7046491f05e3d549c198009f056b4e3e0508ad179712772bb296d0d08cc6ae29
    Image:        centos6-ssh
    Image ID:        docker://sha256:6525d364d418ae8dc854e6839dfaa653f2b6cd39c696a2f146bb918e69c20060
    Port:
    QoS Tier:
      cpu:        BestEffort
      memory:        BestEffort
    State:        Running
      Started:        Wed, 30 Nov 2016 13:44:52 -0500
    Ready:        True
    Restart Count:    0
    Environment Variables:
Conditions:
  Type        Status
  Ready     True
No volumes.
No events.

可以確認docker的實例跑在192.168.5.3這個NODE節點，分配到的集群內網IP為10.1.75.2
我們現在如果需要登錄10.1.75.2要到192.168.5.3這個服務，ssh root@10.1.75.2,才可以登錄

2.2啟動多份的pod

2.2.1 yaml配置

我們定義了一個centos6-ssh pod復制器，復制份數為2，使用centos6-ssh鏡像。

[root@Control k8s_yaml]# cat test/centos6-ssh-rc.yaml
apiVersion: v1
kind: ReplicationController
metadata:
  name: centos6-ssh
spec:
  replicas: 2
  selector:
    name: centos6-ssh
  template:
    metadata:
      labels:
        name: centos6-ssh
    spec:
      containers:
        - name: centos6-ssh
          image: centos6-ssh
          ports:
            - containerPort: 22

2.2.2 查看指令

[root@Control k8s_yaml]# kubectl get pods
NAME                READY     STATUS    RESTARTS   AGE
centos6-ssh-mucsv   1/1       Running   0          10m
centos6-ssh-yoghv   1/1       Running   0          10m

2.2.3 查看細節指令

[root@Control k8s_yaml]# kubectl describe pod centos6-ssh
Name:        centos6-ssh-mucsv
Namespace:    default
Node:        192.168.5.3/192.168.5.3
Start Time:    Thu, 01 Dec 2016 11:04:24 -0500
Labels:        name=centos6-ssh
Status:        Running
IP:        10.1.75.2
Controllers:    ReplicationController/centos6-ssh
Containers:
  centos6-ssh:
    Container ID:    docker://ba9327de6f067b46ce348f409e9efa2b44a9064c4f1ea508cf7d92ff9c450541
    Image:        centos6-ssh
    Image ID:        docker://sha256:6525d364d418ae8dc854e6839dfaa653f2b6cd39c696a2f146bb918e69c20060
    Port:        22/TCP
    QoS Tier:
      memory:        BestEffort
      cpu:        BestEffort
    State:        Running
      Started:        Thu, 01 Dec 2016 11:04:25 -0500
    Ready:        True
    Restart Count:    0
    Environment Variables:
Conditions:
  Type        Status
  Ready     True
No volumes.
Events:
  FirstSeen    LastSeen    Count    From            SubobjectPath            Type        Reason            Message
  ---------    --------    -----    ----            -------------            --------    ------            -------
  5h        5h        2    {kubelet 192.168.5.3}                    Warning        MissingClusterDNS    kubelet does not have ClusterDNS IP configured and cannot create Pod using "ClusterFirst" policy. Falling back to DNSDefault policy.
  5h        5h        1    {kubelet 192.168.5.3}    spec.containers{centos6-ssh}    Normal        Pulling            pulling image "centos6-ssh"
  5h        5h        1    {kubelet 192.168.5.3}    spec.containers{centos6-ssh}    Normal        Pulled            Successfully pulled image "centos6-ssh"
  5h        5h        1    {kubelet 192.168.5.3}    spec.containers{centos6-ssh}    Normal        Created            Created container with docker id ba9327de6f06
  5h        5h        1    {kubelet 192.168.5.3}    spec.containers{centos6-ssh}    Normal        Started            Started container with docker id ba9327de6f06
  3m        3m        1    {default-scheduler }                    Normal        Scheduled        Successfully assigned centos6-ssh-mucsv to 192.168.5.3

Name:        centos6-ssh-yoghv
Namespace:    default
Node:        192.168.5.4/192.168.5.4
Start Time:    Thu, 01 Dec 2016 11:04:37 -0500
Labels:        name=centos6-ssh
Status:        Running
IP:        10.1.68.2
Controllers:    ReplicationController/centos6-ssh
Containers:
  centos6-ssh:
    Container ID:    docker://221e4335774a8347a74fa7341f947954e3fb0eccff5fce7be427b532a4f5d31f
    Image:        centos6-ssh
    Image ID:        docker://sha256:6525d364d418ae8dc854e6839dfaa653f2b6cd39c696a2f146bb918e69c20060
    Port:        22/TCP
    QoS Tier:
      cpu:        BestEffort
      memory:        BestEffort
    State:        Running
      Started:        Thu, 01 Dec 2016 11:04:38 -0500
    Ready:        True
    Restart Count:    0
    Environment Variables:
Conditions:
  Type        Status
  Ready     False
No volumes.
Events:
  FirstSeen    LastSeen    Count    From            SubobjectPath            Type        Reason            Message
  ---------    --------    -----    ----            -------------            --------    ------            -------
  5h        5h        2    {kubelet 192.168.5.4}                    Warning        MissingClusterDNS    kubelet does not have ClusterDNS IP configured and cannot create Pod using "ClusterFirst" policy. Falling back to DNSDefault policy.
  5h        5h        1    {kubelet 192.168.5.4}    spec.containers{centos6-ssh}    Normal        Pulling            pulling image "centos6-ssh"
  5h        5h        1    {kubelet 192.168.5.4}    spec.containers{centos6-ssh}    Normal        Pulled            Successfully pulled image "centos6-ssh"
  5h        5h        1    {kubelet 192.168.5.4}    spec.containers{centos6-ssh}    Normal        Created            Created container with docker id 221e4335774a
  5h        5h        1    {kubelet 192.168.5.4}    spec.containers{centos6-ssh}    Normal        Started            Started container with docker id 221e4335774a
  3m        3m        1    {default-scheduler }                    Normal        Scheduled        Successfully assigned centos6-ssh-yoghv to 192.168.5.4

可以確認啟動了兩個實例
10.1.75.2實例在192.168.5.3上
10.1.68.2實例在192.168.5.4上
如果需要SSH連接上去操作還是需要登到各自的物理機上去才可操作

2.3啟動內網可訪問的services

2.3.1 yaml配置

[root@Control k8s_yaml]# cat test/centos6-ssh-clusterip.yaml
apiVersion: v1
kind: Service
metadata:
  name: centos6-ssh-clusterip
spec:
  ports:
    - port: 2222
      targetPort: 22
      protocol: TCP
  selector:
    name: centos6-ssh
  
selector中的name必須和rc或者pod保持一致

2.3.2 查看

[root@Control k8s_yaml]# kubectl get service
NAME                    CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
centos6-ssh-clusterip   10.254.155.14   <none>        2222/TCP   3s
kubernetes              10.254.0.1      <none>        443/TCP    1d
[root@Control k8s_yaml]# kubectl describe service centos6-ssh-clusterip
Name:            centos6-ssh-clusterip
Namespace:        default
Labels:            <none>
Selector:        name=centos6-ssh
Type:            ClusterIP
IP:            10.254.155.14
Port:            <unset>    2222/TCP
Endpoints:        10.1.68.2:22,10.1.75.2:22
Session Affinity:    None
No events.

上面可以確認centos6-ssh-clusterip已經啟動，分配到的IP為10.254.155.14，開啟2222端口
代理Endpoints:        10.1.68.2:22,10.1.75.2:22

2.3.3 登錄測試

[root@Resources-s1 ~]# telnet 10.254.155.14 2222
Trying 10.254.155.14...
Connected to 10.254.155.14.
Escape character is '^]'.
SSH-2.0-OpenSSH_5.3
^Cxx
Connection closed by foreign host.

QA:

1.解決https問題

[root@Control k8s]# docker push 192.168.5.2:5000/centos6-ssh
The push refers to a repository [192.168.5.2:5000/centos6-ssh]
unable to ping registry endpoint https://192.168.5.2:5000/v0/
v2 ping attempt failed with error: Get https://192.168.5.2:5000/v2/: http: server gave HTTP response to HTTPS client
 v1 ping attempt failed with error: Get https://192.168.5.2:5000/v1/_ping: http: server gave HTTP response to HTTPS client

要解決這個問題要在服務端和客戶端改配置

 服務端：
 [root@Control k8s]# cat /etc/sysconfig/docker|grep 192.168.5.2
 OPTIONS='--insecure-registry 192.168.5.2:5000 --log-driver=journald'
 ADD_REGISTRY='--add-registry 192.168.5.2:5000'

客戶端：
[root@Control k8s]# cat /etc/default/docker
DOCKER_OPTS="--insecure-registry 192.168.5.2:5000"

2.解決創建成功但是kubectl get pods 沒有的問題

  Error from server: error when creating "nginx.yaml": Pod "nginx" is forbidden: no API token found for service account default/default, retry after the token is automatically created and added to the service account

要解決這個問題如下：

創建pod:
# kubectl create -f nginx.yaml
此時有如下報錯：

Error from server: error when creating "nginx.yaml": Pod "nginx" is forbidden: no API token found for service account default/default, retry after the token is automatically created and added to the service account
解決辦法是編輯/etc/kubernetes/apiserver 去除 KUBE_ADMISSION_CONTROL中的SecurityContextDeny,ServiceAccount，並重啟kube-apiserver.service服務：

#vim /etc/kubernetes/apiserver
KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,ResourceQuota"

#systemctl restart kube-apiserver.service
之后重新創建pod:

# kubectl create -f nginx.yaml
pods/nginx

playcrab.com.cn

3. ClusterDNS 出問題，pod不成功

kubelet does not have ClusterDNS IP configured and cannot create Pod using "ClusterFirst" policy. Falling back to DNSDefault policy.

這樣解決
KUBELET_ARGS="--cluster-dns=192.168.5.2 --cluster-domain=playcrab-inc.com"

參考鏈接

k8s相關：
http://www.cnblogs.com/openxxs/p/5072865.html
http://www.dockone.io/article/578
http://www.dockone.io/article/1616
http://webpaas.com/index.php/archives/111/
https://mos.meituan.com/library/37/how-to-setup-k8s-cluster-on-CentOS7/
https://www.caicloud.io/article_detail/573d85d2824168110000001d
http://kubernetes.io/docs/user-guide/quick-start/
http://blog.csdn.net/qq1010885678/article/details/49405435
http://www.cnblogs.com/CraryPrimitiveMan/p/4657835.html
http://www.tuicool.com/articles/y26nyar
https://segmentfault.com/q/1010000006127473
http://www.cnblogs.com/stonehat/p/5148455.html
http://valleylord.github.io/post/201603-kubernetes-roll/

kubernetes入門之kube-proxy實現原理
http://www.cnblogs.com/xuxinkun/p/5799986.html
http://blog.coocla.org/kubernetes-storage-volumes-rbd-docker.html
http://www.fangyunlin.com/?p=54

docker相關：
http://dockone.io/article/783
http://dockone.io/article/372
http://dockone.io/article/259
http://blog.liuts.com/post/242/
http://www.infoq.com/cn/articles/docker-network-and-pipework-open-source-explanation-practice
http://note.youdao.com/share/?id=8387b9e886c84f413a97d678c3d01869&type=note#/
http://www.pangxie.space/docker/157
http://www.pangxie.space/docker/176
http://dockone.io/article/1264

docker打鏡像
http://my.oschina.net/feedao/blog
http://www.opstool.com/article/315
https://amao12580.github.io/post/2016/04/Nginx-with-docker-part-one/

刪除docker私有倉庫里的鏡像
https://www.v2ex.com/t/266876


微服務化相關：
http://www.infoq.com/cn/articles/micro-service-architecture-evolution-of-daocloud
http://www.infoq.com/cn/articles/enterprise-core-systems-transformation-practice
http://dockone.io/article/394
http://www.infoq.com/cn/articles/the-back-end-business-systems-service-transformation
http://www.infoq.com/cn/articles/ultimate-discussion-of-micro-service-architecture
http://martinfowler.com/articles/microservices.html


kube-ui:
http://blog.csdn.net/zczzsq/article/details/50787810


交換機：
http://blog.csdn.net/wylfengyujiancheng/article/details/51762169
http://blog.csdn.net/wylfengyujiancheng/article/details/51762792

排錯的用法
https://linfan1.gitbooks.io/kubernetes-chinese-docs/content/166-Applications.html


kube2sky
http://www.tuicool.com/articles/yeIJNjJ

手冊
https://linfan1.gitbooks.io/kubernetes-chinese-docs/content/170-Services_FAQ.html
http://www.widuu.com/chinese_docker/examples/nodejs_web_app.html
https://docs.docker.com/registry/spec/api/#pagination
http://kubernetes.io/docs/user-guide/configmap/





http://tonybai.com/2016/11/17/nginx-config-hot-reloading-approach-for-kubernetes-cluster/
http://tonybai.com/2016/11/21/kuberize-ceph-rbd-api-service/
http://tonybai.com/2016/11/22/deploy-nginx-service-for-the-services-in-kubernetes-cluster/
http://tonybai.com/2016/11/16/how-to-pull-images-from-private-registry-on-kubernetes-cluster/
http://tonybai.com/2016/11/07/integrate-kubernetes-with-ceph-rbd/
http://tonybai.com/2016/10/23/install-dns-addon-for-k8s/
        
http://www.pangxie.space/docker/735
http://www.csdn.net/article/2015-06-12/2824937
http://www.cnblogs.com/puroc/p/5764330.html
http://www.pangxie.space/docker/643
http://www.webpaas.com/index.php/archives/115/
http://zhjwpku.com/docker/2016/08/30/k8s-deploy-a-3-nodes-cluster.html
http://zhjwpku.com/docker/2016/09/01/cluster-addon.html
http://blog.csdn.net/dream_broken/article/details/53115770


http://blog.csdn.net/dc_726/article/details/46475633

請尊重本人勞動成功，可以隨意轉載但保留以下信息
作者：歲月經年
原站：www.djhull.com
時間：2016年11月

avatar

免責聲明！

本站轉載的文章為個人學習借鑒使用，本站對版權不負任何法律責任。如果侵犯了您的隱私權益，請聯系本站郵箱yoyou2525@163.com刪除。

猜您在找 k8s 結合docker搭建私有倉庫 k8s Docker私有倉庫認證 K8S--docker--搭建私有倉庫 k8s 搭建私有鏡像倉庫Harbor k8s使用私有鏡像倉庫 K8S之集群搭建搭建k8s集群 k8s集群搭建（一） k8s集群的搭建 Blog.094 K8S 集群架構：kubadm、dashboard 與 Harbor 私有倉庫部署