ps -elf | grep containerd 0 S root 19894 19862 0 80 0 - 1418 pipe_w 14:36 pts/3 00:00:00 grep --color=auto containerd 4 S root 39827 1 0 80 0 - 3197 futex_ Oct15 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 1ccf9e9d7f19e6b4159443e4f9eb69520efae7178ad14d8ddef3fb27b2a42a42 -address /run/containerd/containerd.sock 4 S root 39851 1 0 80 0 - 2565 futex_ Oct15 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 07c09e1bfbed1ddd50dbd6fa6fa0c986e82f83bcfaa8dd30c27c895b78173030 -address /run/containerd/containerd.sock 4 S root 39879 1 0 80 0 - 3181 futex_ Oct15 ? 00:00:03 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 98f42915d08acd929bed377113f7a9e72b1026ff87790b366bd56afa6c32e893 -address /run/containerd/containerd.sock 4 S root 39914 1 0 80 0 - 2829 futex_ Oct15 ? 00:00:03 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 04606a708483b150e8b6ad3ccd34fc8d7f1ee8ca54a830211dfe366346556e82 -address /run/containerd/containerd.sock 4 S root 40180 1 0 80 0 - 3181 futex_ Oct15 ? 00:00:12 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id c2469551e412cc8b357b1990ccdfe48e4996a42f2b505f65cfb88928fa932d44 -address /run/containerd/containerd.sock 4 S root 40229 1 0 80 0 - 3181 futex_ Oct15 ? 00:00:13 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 9cc604741074d6f92ca6ad819f2c334cc14e5febbe213d30c880c352a5c4a126 -address /run/containerd/containerd.sock 4 S root 40260 1 0 80 0 - 3181 futex_ Oct15 ? 00:00:12 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 44a16229f3aa90bb3925152715bfb7acbc58ef0cca4e4cb923a34d3c57bdfc66 -address /run/containerd/containerd.sock 4 S root 40287 1 0 80 0 - 3181 futex_ Oct15 ? 00:00:11 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id f09a3b37ef24e98e5966422cc21398cb5b937b56d604a436441fdf4b5c4d8a53 -address /run/containerd/containerd.sock 4 S root 40895 1 5 80 0 - 1567151 futex_ Oct15 ? 00:58:57 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=/run/containerd/containerd.sock --cgroup-driver=systemd 4 S root 41268 1 0 80 0 - 3181 futex_ Oct15 ? 00:00:04 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 4226cdd5d357d36a8d4727ffe2a0fb17b305aa720f6c2e5ca3d6213702f71fb8 -address /run/containerd/containerd.sock 4 S root 41341 1 0 80 0 - 3181 futex_ Oct15 ? 00:00:12 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 625a5a2542d87024d4d0b5c198d09d7dbfd690891154144511043f9de94f8fb8 -address /run/containerd/containerd.sock 4 S root 41567 1 0 80 0 - 3181 futex_ Oct15 ? 00:00:03 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 67a906f68abb8b942e2e27ab4b144c0b42a4bd5cfa69ff5a34d312cd728726fe -address /run/containerd/containerd.sock 4 S root 41651 1 0 80 0 - 3181 futex_ Oct15 ? 00:00:03 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 7f76ee366fdda2d50c266ef416582f07af53b41d757266c40df6ade0339c0474 -address /run/containerd/containerd.sock 4 S root 42604 1 0 80 0 - 2246 futex_ Oct13 ? 00:00:11 containerd-shim -namespace moby -workdir /var/lib/containerd/io.containerd.runtime.v1.linux/moby/5fc9ee9cd73f5f3b737c65d08c02d622e508846f076be133e3620815682d6863 -address /run/containerd/containerd.sock -containerd-binary /usr/bin/containerd -runtime-root /var/run/docker/runtime-runc 0 S root 44377 1 0 80 0 - 328983 futex_ Oct15 ? 00:02:15 /usr/local/bin/containerd-shim-kata-v2 -namespace k8s.io -address /run/containerd/containerd.sock -publish-binary /usr/bin/containerd -id 7d5e7ea32ba10ed89e2d87b9b025e371ba7a839c223d62c6d34ff7512b47a586 -debug 0 S root 53651 1 0 80 0 - 347416 futex_ 08:37 ? 00:00:49 /usr/local/bin/containerd-shim-kata-v2 -namespace k8s.io -address /run/containerd/containerd.sock -publish-binary /usr/bin/containerd -id 41005e66e4c59ae11c6533c279a2868f7912634995212dd5d131e3371d8f93e7 -debug 4 S root 96887 1 1 80 0 - 2069709 futex_ Oct15 ? 00:14:08 /usr/bin/containerd 4 S root 96888 1 0 80 0 - 1172301 futex_ Oct15 ? 00:03:39 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --add-runtime kata-runtime=/usr/local/bin/kata-runtime
root@ubuntu:/usr/share/defaults/kata-containers# ls /usr/local/bin/ containerd-shim-kata-v2 kata-runtime qemu-io ivshmem-client kata-runtime.bak qemu-nbd ivshmem-server qemu-ga qemu-system-aarch64 kata-collect-data.sh qemu-img
root@ubuntu:/etc/containerd# ps -elf | grep containerd 0 S root 33334 1 0 80 0 - 27454 futex_ 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 13faedcf4cf2d8501b8dd240a7c886e55bcdb911178b666c4ead6aa29f7ba4a7 -address /run/containerd/containerd.sock 0 S root 33451 1 0 80 0 - 27806 futex_ 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id b811984e3cda7c014a70fcb2f6e1a61292af2b7deb93f740a3c51c9e7a52b7fe -address /run/containerd/containerd.sock 0 S root 33486 1 0 80 0 - 27806 futex_ 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 16288c7c88cb53922e96af85c73d3bc178d1be522ec3cd9578dfa3de55c60030 -address /run/containerd/containerd.sock 0 S root 33748 1 0 80 0 - 27454 futex_ 12:56 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 8de3978642c12908b0f0cdc6bfa8dbab3156979bb280c40b4c3ebc8e33080937 -address /run/containerd/containerd.sock 0 S root 33824 1 0 80 0 - 27806 futex_ 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id f6cbd0a77bee3a313fd8f8417166315a5c0f19212caba3f4aa95061bb943dfe1 -address /run/containerd/containerd.sock 0 S root 33945 1 0 80 0 - 27806 futex_ 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 31246780d074d791990a0c68b562f84bdd821834fb53def9bdf0183f978aaa99 -address /run/containerd/containerd.sock 0 S root 34005 1 0 80 0 - 27806 futex_ 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id ad48842a9a8d481fbd20a936860ec9243a4892f2e6e9c064ab5093cf083ba0fd -address /run/containerd/containerd.sock 0 S root 34030 1 0 80 0 - 27806 futex_ 12:56 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 7dc3992c40a06c1f60b5d115e1d6bac8c16950e6b0aec6c891a20e85bf906e7f -address /run/containerd/containerd.sock 0 S root 34079 1 0 80 0 - 27806 futex_ 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 6a57995cd991dd4e2c41c1421ab53880cd3d39a84f804194ed4fd1bc80a85cc4 -address /run/containerd/containerd.sock 4 S root 48603 1 5 80 0 - 993600 futex_ 13:14 ? 00:00:01 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=/run/containerd/containerd.sock --resolv-conf=/run/systemd/resolve/resolv.conf --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock
root@ubuntu:/home/ubuntu# ctr run --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh / # ls bin dev etc home proc root run sys tmp usr var / # quit sh: quit: not found / # exit root@ubuntu:/home/ubuntu#
ubuntu@ubuntu:~$ ps -elf | grep containerd 0 S root 33334 1 0 80 0 - 27454 - 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 13faedcf4cf2d8501b8dd240a7c886e55bcdb911178b666c4ead6aa29f7ba4a7 -address /run/containerd/containerd.sock 0 S root 33451 1 0 80 0 - 27806 - 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id b811984e3cda7c014a70fcb2f6e1a61292af2b7deb93f740a3c51c9e7a52b7fe -address /run/containerd/containerd.sock 0 S root 33486 1 0 80 0 - 27806 - 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 16288c7c88cb53922e96af85c73d3bc178d1be522ec3cd9578dfa3de55c60030 -address /run/containerd/containerd.sock 0 S root 33748 1 0 80 0 - 27454 - 12:56 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 8de3978642c12908b0f0cdc6bfa8dbab3156979bb280c40b4c3ebc8e33080937 -address /run/containerd/containerd.sock 0 S root 33824 1 0 80 0 - 27806 - 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id f6cbd0a77bee3a313fd8f8417166315a5c0f19212caba3f4aa95061bb943dfe1 -address /run/containerd/containerd.sock 0 S root 33945 1 0 80 0 - 27806 - 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 31246780d074d791990a0c68b562f84bdd821834fb53def9bdf0183f978aaa99 -address /run/containerd/containerd.sock 0 S root 34005 1 0 80 0 - 27806 - 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id ad48842a9a8d481fbd20a936860ec9243a4892f2e6e9c064ab5093cf083ba0fd -address /run/containerd/containerd.sock 0 S root 34030 1 0 80 0 - 27806 - 12:56 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 7dc3992c40a06c1f60b5d115e1d6bac8c16950e6b0aec6c891a20e85bf906e7f -address /run/containerd/containerd.sock 0 S root 34079 1 0 80 0 - 27806 - 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 6a57995cd991dd4e2c41c1421ab53880cd3d39a84f804194ed4fd1bc80a85cc4 -address /run/containerd/containerd.sock 0 S root 50495 56922 0 80 0 - 271362 - 13:16 pts/3 00:00:00 ctr run --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh 0 S root 50518 1 1 80 0 - 328755 - 13:17 ? 00:00:00 /usr/local/bin/containerd-shim-kata-v2 -namespace default -address /run/containerd/containerd.sock -publish-binary /usr/bin/containerd -id hello -debug 0 S ubuntu 50634 41843 0 80 0 - 1097 pipe_w 13:17 pts/4 00:00:00 grep --color=auto containerd 4 S root 59577 1 1 80 0 - 1496304 - 10:23 ? 00:02:52 /usr/bin/containerd
root@ubuntu:/etc/containerd# ps -elf | grep containerd 0 S root 33334 1 0 80 0 - 27454 futex_ 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 13faedcf4cf2d8501b8dd240a7c886e55bcdb911178b666c4ead6aa29f7ba4a7 -address /run/containerd/containerd.sock 0 S root 33451 1 0 80 0 - 27806 futex_ 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id b811984e3cda7c014a70fcb2f6e1a61292af2b7deb93f740a3c51c9e7a52b7fe -address /run/containerd/containerd.sock 0 S root 33486 1 0 80 0 - 27806 futex_ 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 16288c7c88cb53922e96af85c73d3bc178d1be522ec3cd9578dfa3de55c60030 -address /run/containerd/containerd.sock 0 S root 33748 1 0 80 0 - 27454 futex_ 12:56 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 8de3978642c12908b0f0cdc6bfa8dbab3156979bb280c40b4c3ebc8e33080937 -address /run/containerd/containerd.sock 0 S root 33824 1 0 80 0 - 27806 futex_ 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id f6cbd0a77bee3a313fd8f8417166315a5c0f19212caba3f4aa95061bb943dfe1 -address /run/containerd/containerd.sock 0 S root 33945 1 0 80 0 - 27806 futex_ 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 31246780d074d791990a0c68b562f84bdd821834fb53def9bdf0183f978aaa99 -address /run/containerd/containerd.sock 0 S root 34005 1 0 80 0 - 27806 futex_ 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id ad48842a9a8d481fbd20a936860ec9243a4892f2e6e9c064ab5093cf083ba0fd -address /run/containerd/containerd.sock 0 S root 34030 1 0 80 0 - 27806 futex_ 12:56 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 7dc3992c40a06c1f60b5d115e1d6bac8c16950e6b0aec6c891a20e85bf906e7f -address /run/containerd/containerd.sock 0 S root 34079 1 0 80 0 - 27806 futex_ 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 6a57995cd991dd4e2c41c1421ab53880cd3d39a84f804194ed4fd1bc80a85cc4 -address /run/containerd/containerd.sock 4 S root 52666 1 1 80 0 - 590735 futex_ 13:20 ? 00:00:00 /usr/bin/containerd 0 S root 52951 37885 1 80 0 - 35131 futex_ 13:20 pts/1 00:00:00 kubeadm init --kubernetes-version=v1.18.1 --pod-network-cidr=10.244.0.0/16 --apiserver-advertise-address=10.10.16.82 --ignore-preflight-errors=all --cri-socket /run/containerd/containerd.sock 4 S root 53019 52951 0 80 0 - 30113 ep_pol 13:20 pts/1 00:00:00 crictl -r unix:///run/containerd/containerd.sock pull k8s.gcr.io/kube-apiserver:v1.18.1 0 S root 53036 56922 0 80 0 - 1097 pipe_w 13:20 pts/3 00:00:00 grep --color=auto containerd
root@ubuntu:/etc/containerd# ps -elf | grep containerd 0 S root 33334 1 0 80 0 - 27454 futex_ 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 13faedcf4cf2d8501b8dd240a7c886e55bcdb911178b666c4ead6aa29f7ba4a7 -address /run/containerd/containerd.sock 0 S root 33451 1 0 80 0 - 27806 futex_ 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id b811984e3cda7c014a70fcb2f6e1a61292af2b7deb93f740a3c51c9e7a52b7fe -address /run/containerd/containerd.sock 0 S root 33486 1 0 80 0 - 27806 futex_ 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 16288c7c88cb53922e96af85c73d3bc178d1be522ec3cd9578dfa3de55c60030 -address /run/containerd/containerd.sock 0 S root 33748 1 0 80 0 - 27454 futex_ 12:56 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 8de3978642c12908b0f0cdc6bfa8dbab3156979bb280c40b4c3ebc8e33080937 -address /run/containerd/containerd.sock 0 S root 33824 1 0 80 0 - 27806 futex_ 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id f6cbd0a77bee3a313fd8f8417166315a5c0f19212caba3f4aa95061bb943dfe1 -address /run/containerd/containerd.sock 0 S root 33945 1 0 80 0 - 27806 futex_ 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 31246780d074d791990a0c68b562f84bdd821834fb53def9bdf0183f978aaa99 -address /run/containerd/containerd.sock 0 S root 34005 1 0 80 0 - 27806 futex_ 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id ad48842a9a8d481fbd20a936860ec9243a4892f2e6e9c064ab5093cf083ba0fd -address /run/containerd/containerd.sock 0 S root 34030 1 0 80 0 - 27806 futex_ 12:56 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 7dc3992c40a06c1f60b5d115e1d6bac8c16950e6b0aec6c891a20e85bf906e7f -address /run/containerd/containerd.sock 0 S root 34079 1 0 80 0 - 27806 futex_ 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 6a57995cd991dd4e2c41c1421ab53880cd3d39a84f804194ed4fd1bc80a85cc4 -address /run/containerd/containerd.sock 4 S root 52666 1 0 80 0 - 1015398 futex_ 13:20 ? 00:00:02 /usr/bin/containerd 0 S root 52951 37885 1 80 0 - 35323 ep_pol 13:20 pts/1 00:00:09 kubeadm init --kubernetes-version=v1.18.1 --pod-network-cidr=10.244.0.0/16 --apiserver-advertise-address=10.10.16.82 --ignore-preflight-errors=all --cri-socket /run/containerd/containerd.sock 4 S root 54588 1 11 80 0 - 938109 futex_ 13:35 ? 00:00:01 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=/run/containerd/containerd.sock --resolv-conf=/run/systemd/resolve/resolv.conf --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock 0 S root 54949 56922 0 80 0 - 1097 pipe_w 13:35 pts/3 00:00:00 grep --color=auto containerd
Binary Naming Users specify the runtime they wish to use when creating a container. The runtime can also be changed via a container update. > ctr run --runtime io.containerd.runc.v1 When a user specifies a runtime name, io.containerd.runc.v1, they will specify the name and version of the runtime. This will be translated by containerd into a binary name for the shim. io.containerd.runc.v1 -> containerd-shim-runc-v1 containerd keeps the containerd-shim-* prefix so that users can ps aux | grep containerd-shim to see running shims on their system.
root@ubuntu:/etc/containerd# ctr run --runtime io.containerd.runc.v1 -t --rm docker.io/library/busybox:latest hello sh / # ls bin dev etc home proc root run sys tmp usr var / #
ubuntu@ubuntu:~$ ps -elf | grep containerd 0 S root 33334 1 0 80 0 - 27454 - 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 13faedcf4cf2d8501b8dd240a7c886e55bcdb911178b666c4ead6aa29f7ba4a7 -address /run/containerd/containerd.sock 0 S root 33451 1 0 80 0 - 27806 - 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id b811984e3cda7c014a70fcb2f6e1a61292af2b7deb93f740a3c51c9e7a52b7fe -address /run/containerd/containerd.sock 0 S root 33486 1 0 80 0 - 27806 - 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 16288c7c88cb53922e96af85c73d3bc178d1be522ec3cd9578dfa3de55c60030 -address /run/containerd/containerd.sock 0 S root 33748 1 0 80 0 - 27454 - 12:56 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 8de3978642c12908b0f0cdc6bfa8dbab3156979bb280c40b4c3ebc8e33080937 -address /run/containerd/containerd.sock 0 S root 33824 1 0 80 0 - 27806 - 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id f6cbd0a77bee3a313fd8f8417166315a5c0f19212caba3f4aa95061bb943dfe1 -address /run/containerd/containerd.sock 0 S root 33945 1 0 80 0 - 27806 - 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 31246780d074d791990a0c68b562f84bdd821834fb53def9bdf0183f978aaa99 -address /run/containerd/containerd.sock 0 S root 34005 1 0 80 0 - 27806 - 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id ad48842a9a8d481fbd20a936860ec9243a4892f2e6e9c064ab5093cf083ba0fd -address /run/containerd/containerd.sock 0 S root 34030 1 0 80 0 - 27806 - 12:56 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 7dc3992c40a06c1f60b5d115e1d6bac8c16950e6b0aec6c891a20e85bf906e7f -address /run/containerd/containerd.sock 0 S root 34079 1 0 80 0 - 27806 - 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 6a57995cd991dd4e2c41c1421ab53880cd3d39a84f804194ed4fd1bc80a85cc4 -address /run/containerd/containerd.sock 0 S root 50495 56922 0 80 0 - 271362 - 13:16 pts/3 00:00:00 ctr run --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh 0 S root 50518 1 1 80 0 - 328755 - 13:17 ? 00:00:00 /usr/local/bin/containerd-shim-kata-v2 -namespace default -address /run/containerd/containerd.sock -publish-binary /usr/bin/containerd -id hello -debug 0 S ubuntu 50634 41843 0 80 0 - 1097 pipe_w 13:17 pts/4 00:00:00 grep --color=auto containerd 4 S root 59577 1 1 80 0 - 1496304 - 10:23 ? 00:02:52 /usr/bin/containerd ubuntu@ubuntu:~$ ps -elf | grep containerd 0 S root 33334 1 0 80 0 - 27454 - 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 13faedcf4cf2d8501b8dd240a7c886e55bcdb911178b666c4ead6aa29f7ba4a7 -address /run/containerd/containerd.sock 0 S root 33451 1 0 80 0 - 27806 - 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id b811984e3cda7c014a70fcb2f6e1a61292af2b7deb93f740a3c51c9e7a52b7fe -address /run/containerd/containerd.sock 0 S root 33486 1 0 80 0 - 27806 - 12:55 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 16288c7c88cb53922e96af85c73d3bc178d1be522ec3cd9578dfa3de55c60030 -address /run/containerd/containerd.sock 0 S root 33748 1 0 80 0 - 27454 - 12:56 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 8de3978642c12908b0f0cdc6bfa8dbab3156979bb280c40b4c3ebc8e33080937 -address /run/containerd/containerd.sock 0 S root 33824 1 0 80 0 - 27806 - 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id f6cbd0a77bee3a313fd8f8417166315a5c0f19212caba3f4aa95061bb943dfe1 -address /run/containerd/containerd.sock 0 S root 33945 1 0 80 0 - 27806 - 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 31246780d074d791990a0c68b562f84bdd821834fb53def9bdf0183f978aaa99 -address /run/containerd/containerd.sock 0 S root 34005 1 0 80 0 - 27806 - 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id ad48842a9a8d481fbd20a936860ec9243a4892f2e6e9c064ab5093cf083ba0fd -address /run/containerd/containerd.sock 0 S root 34030 1 0 80 0 - 27806 - 12:56 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 7dc3992c40a06c1f60b5d115e1d6bac8c16950e6b0aec6c891a20e85bf906e7f -address /run/containerd/containerd.sock 0 S root 34079 1 0 80 0 - 27806 - 09:47 ? 00:00:01 /usr/bin/containerd-shim-runc-v1 -namespace k8s.io -id 6a57995cd991dd4e2c41c1421ab53880cd3d39a84f804194ed4fd1bc80a85cc4 -address /run/containerd/containerd.sock 4 S root 52666 1 0 80 0 - 1162990 - 13:20 ? 00:00:04 /usr/bin/containerd 0 S root 57554 56922 0 80 0 - 289859 - 13:39 pts/3 00:00:00 ctr run --runtime io.containerd.runc.v1 -t --rm docker.io/library/busybox:latest hello sh 0 S root 57577 1 0 80 0 - 27774 - 13:39 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace default -id hello -address /run/containerd/containerd.sock 4 S root 57675 1 4 80 0 - 568937 - 13:40 ? 00:00:00 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=/run/containerd/containerd.sock --resolv-conf=/run/systemd/resolve/resolv.conf --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock 0 S ubuntu 57803 41843 0 80 0 - 1097 - 13:40 pts/4 00:00:00 grep --color=auto containerd ubuntu@ubuntu:~$
可以看到/usr/bin/目錄下有如下可執行文件：
kata-runtime：kata-containers的運行時
還有三個containerd開頭的bin containerd：
containerd運行時，docker官方提供的
containerd-shim：shim，docker官方提供的（假如用kata的運行時，再用這個shim，似乎跑起來就是v1的kata？）
containerd-shim-kata-v2：kata-containers連接到containerd的shim，是kata-containers安裝上去的。
運行
使用kata-runtime來運行容器
kata-containers提供了一個命令叫kata-runtime，它本質上是一個類似於runC的low level container runtime。只能進行一個容器的生命周期管理（就是啟動，停止，銷毀等基本操作），沒有鏡像下載等功能，鏡像之類的高級功能一般是由containerd提供的。
所以我們假如直接用kata-runtime來運行容器，我們需要自己手動准備好一個容器的鏡像。
# 准備一個busybox鏡像
# 構造rootfs
mkdir rootfs
docker export $(docker create busybox) | tar -xf - -C rootfs
# 構造config.json
runc spec
# 使用kata-runtime來運行榮齊全
kata-runtime run busybox這樣子啟動container，默認運行的是kata containers v1（就是有kata-proxy進程）。
使用containerd運行kata容器
使用containerd，關鍵就是要設置好底層的runtime，讓containerd知道要用kata的runtime。 有如下兩種方法：
ctr命令行指定runtime
sudo ctr image pull docker.io/library/busybox:latest
sudo ctr run --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh值得注意的是io.containerd.run.kata.v2是v2版本，至於v1版本，我不知道應該怎么從命令行指定。
修改containerd的config文件
默認config文件：/etc/containerd/config.toml。
注意：如果后續要使用CRI接口，要讓kubelet通過CRI接口連接上containerd，則必須去掉一開始的disabled_plugins = ["cri"]。 可以參照如下兩個鏈接來修改containerd的config。
將默認runtime設置成kata v1
[plugins.linux]
# shim binary name/path
shim = "containerd-shim"
# runtime binary name/path
runtime = "/usr/bin/kata-runtime"
# do not use a shim when starting containers, saves on memory but
# live restore is not supported
no_shim = false
# display shim logs in the containerd daemon's log output
shim_debug = true將默認runtime設置成kata v2
注意：下面這個配置會報錯ctr: read unix @->@/containerd-shim/default/hello_5/shim.sock: read: connection reset by peer: unknown，具體原因未知。
所以運行kata v2還是直接用上面說的ctr命令行指定runtime來搞吧。
[plugins.linux]
# shim binary name/path
shim = "containerd-shim-kata-v2"
# runtime binary name/path
runtime = "/usr/bin/kata-runtime"
# do not use a shim when starting containers, saves on memory but
# live restore is not supported
no_shim = false # display shim logs in the containerd daemon's log output shim_debug = true注意：對config文件做了任何修改后，都必須要運行sudo service containerd restart來重新reload config。
用ctr命令運行，注意，這次命令行不需要指定runtime了。
sudo ctr image pull docker.io/library/busybox:latest sudo ctr run -t --rm docker.io/library/busybox:latest hello sh查看啟動kata容器后，在系統中多了多少進程
可以看看啟動一個kata容器后，系統里面到底多了幾個進程？
# 查看系統中的所有進程
# 其中PID是process ID
# PPID：parent process ID，從這個可以看某個進程到底是由哪個進程創建的
ps -AF
# 查看系統中的所有線程
# LWP：線程ID
# NLWP：當前進程的總線程數
ps -AFL H對比啟動前和啟動后的進程以及線程，可以發現多了如下的進程：
1. sudo命令行進程（因為命令是sudo開頭的，所以是先運行sudo，然后sudo在開新進程執行后續的命令行）
2. kata-runtime/ctr/containerd-shim的命令行進程。
3. qemu進程，對於單CPU的處理器，它有四個線程
4. kvm-nx-lpage-re
5. kvm-pit
6. kata-proxy
7. kata-shim
如果是v2的就沒有kata-proxy和kata-shim進程。
暫時還有如下問題沒有搞清楚：
1. 是一個container一個shim，還是一個container process一個shim？
2. 文檔里面說v2里面沒有kata-proxy，但是沒有說沒有shim？還是說在v2里面kata-shim被整合進了containerd-shim-kata-v2？暫時還不確定。
977 37392 container_manager_linux.go:271] Creating Container Manager object based on Node Config: {RuntimeCgroup 445 37392 topology_manager.go:126] [topologymanager] Creating topology manager with none policy 472 37392 container_manager_linux.go:301] [topologymanager] Initializing Topology Manager with none policy 485 37392 container_manager_linux.go:306] Creating device plugin manager: true 867 37392 remote_runtime.go:59] parsed scheme: "" 889 37392 remote_runtime.go:59] scheme "" not registered, fallback to default scheme 977 37392 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/run/containerd/containerd.sock <nil> 998 37392 clientconn.go:933] ClientConn switching balancer to "pick_first" 088 37392 remote_image.go:50] parsed scheme: "" 103 37392 remote_image.go:50] scheme "" not registered, fallback to default scheme 121 37392 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/run/containerd/containerd.sock <nil> 133 37392 clientconn.go:933] ClientConn switching balancer to "pick_first" 214 37392 kubelet.go:292] Adding pod path: /etc/kubernetes/manifests 270 37392 kubelet.go:317] Watching apiserver
k3sのcontainerdに設定を入れる
現在利用しているk8sはk3sで動いているので、「通常のcontainerd」ではなく、「k3sにくっついてくるcontainerd」対して設定を行います。
Rancher Docs: Advanced Options and Configuration によると、/var/lib/rancher/k3s/agent/etc/containerd/config.toml に設定ファイルがあり、`/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl を置いておくとそちらが優先して読み込まれるとのことなので、元の設定ファイルをコピーして追記する形にしました。
config.toml.tmpl
(snip)
[plugins.cri.containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" [plugins.cri.containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml"
RuntimeClassをk8s上に作成する
ブログ記事では觸れられていません4が、k8s上にKata Containerに対応するRuntimeClassを作成する必要があります。
RuntimeClassは下記のように、metadata.name と handler しかないシンプルな設定です。
containerdをruntimeに利用している場合は、handlerは[plugins.cri.containerd.runtimes.${HANDLER_NAME}] の部分、つまり今回はkataが正解です。
kata-RuntimeClass.yaml
apiVersion: node.k8s.io/v1beta1 kind: RuntimeClass metadata: name: kata handler: kata
動かしてみる
ctrで直接コンテナを起動してみるのと、k8sのPodをKata Containerを動かしてみます。
crt
# runtimeにkataを指定して、k3sのcontainerdを使ってコンテナを起動
tk3fftk@nuc1:~$ sudo k3s ctr run --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh / # # 別シェルで動いているコンテナを確認。runtimeがkataになっている tk3fftk@nuc1:~$ sudo k3s ctr c ls | grep hello hello docker.io/library/busybox:latest io.containerd.run.kata.v2
k8s
# 適當にPodのYAMLを吐き出す
$ kubectl run --image nginx nginx --dry-run=client -o yaml > nginx_pod.yaml # spec.runtimeClassName を追加する。下記は追加後のYAML $ cat nginx_pod.yaml apiVersion: v1 kind: Pod metadata: creationTimestamp: null labels: run: nginx name: nginx spec: runtimeClassName: kata containers: - image: nginx name: nginx resources: {} dnsPolicy: ClusterFirst restartPolicy: Always status: {} # deploy $ kubectl apply -f nginx_pod.yaml # worker側で確認。pauseコンテナとnginxコンテナのruntimeがkataになっている tk3fftk@nuc1:~$ sudo k3s ctr containers list | grep kata 910d6bf06948071677690a9cc482455617c7d17e386a0fe3a02c15762a0a7f88 docker.io/library/nginx@sha256:c870bf53de0357813af37b9500cb1c2ff9fb4c00120d5fe1d75c21591293c34d io.containerd.kata.v2 95dbaab3534ba6a87036d2f0057cad600734466293baec222b998817d3b84b40 docker.io/rancher/pause:3.1 io.containerd.kata.v2
OK, in this example we’ll run Kata containers on existing and working Kubernetes on-premise cluster, created with the help of Kubeadm utility. Things that we need: Configured and working Kubernetes cluster v 1.14 or v 1.15 One or more worker nodes in the cluster must be able to run Qemu as the virtualization platform, so these nodes must be bare-metal and have a real CPU with virtualization support. You can check this by running this command on your Kubernetes worker nodes: kubernetes-node# cat /proc/cpuinfo | grep vmx The Kata containers can only run with CRI-O or containerd, but not with dockershim that is the main CRI API for many Kubernetes clusters by default. If you didn’t specify any other CRI API when you created the cluster, you will get a dockershim on all your Kubernetes nodes. Try to run: kubernetes-node# docker ps If you see any output, that means you are using dockershim CRI runtime for this host. First, let’s change the CRI runtime on nodes that will be used for running Kata containers in the future. I’ll use the containerd for this example. Very often the containerd are installed with Docker CE as an additional package, on my Debian hosts it was already installed: kubernetes-node# dpkg -l | grep container ii containerd.io 1.2.5-1 amd64 An open and reliable container runtime ii docker-ce 5:18.09.4~3-0~debian-stretch amd64 Docker: the open-source application container engine ii docker-ce-cli 5:18.09.4~3-0~debian-stretch amd64 Docker CLI: the open-source application container engine If nope, you can install it manually as a package or from source code. OK, first of all, if the node that we’ll migrate to containerd have any running production pods, first drain this node or migrate all pods to other nodes. Then login to this node and remove cri plugin from the list ofdisabled_plugins in the containerd configuration file /etc/containerd/config.toml. before disabled_plugins = ["cri"] after disabled_plugins = [""] Next, we need to prepare this node for the containerd: kubernetes-node# modprobe overlay kubernetes-node# modprobe br_netfilter kubernetes-node# cat > /etc/sysctl.d/99-kubernetes-cri.conf <<EOF net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward = 1 net.bridge.bridge-nf-call-ip6tables = 1 EOF kubernetes-node# sysctl --system Now we need to configure Kubelet on this node for using containerd: kubernetes-node# cat << EOF | sudo tee /etc/systemd/system/kubelet.service.d/0-containerd.conf [Service] Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock" EOF On some systems, you also may need to add default config for kubelet: kubernetes-node# vi /etc/default/kubelet KUBELET_EXTRA_ARGS= --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock After that done, restart containerd and kubelet services on this node: kubernetes-node# systemctl daemon-reload kubernetes-node# systemctl restart containerd kubernetes-node# systemctl restart kubelet Now wait a bit till kubelet will change the runtime engine, and then try to run docker ps command again: kubernetes-node# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES You must see the empty output, that means Kuibernetes start using containerd instead of docker as a container runtime. Also, you can check this by running kubectl get nodes -o wide command: kubectl# kubectl get nodes -o wide NAME STATUS ..... CONTAINER-RUNTIME kubernetes-node Ready . .... containerd://1.2.5 For listing containers on this node, you now must use ctr command: kubernetes-node# ctr --namespace k8s.io containers ls Good, now we have the right configured kubelet and containerd and we can move on and install Kata containers inside our cluster. There’s are prepared deployments and some documentation about how to do it in the kata-deploy section of the repository. Clone it and use the readme file for additional information. Run next commands for deploy Kata containers support into your Kubernetes cluster: kubectl# kubectl apply -f https://raw.githubusercontent.com/kata-containers/packaging/master/kata-deploy/kata-rbac.yaml kubectl# kubectl apply -f https://raw.githubusercontent.com/kata-containers/packaging/master/kata-deploy/kata-deploy.yaml #For the Kubernetes 1.14 and 1.15 run: kubectl# kubectl apply -f https://raw.githubusercontent.com/kata-containers/packaging/master/kata-deploy/k8s-1.14/kata-qemu-runtimeClass.yaml After Kata containers installation the nodes that available for running it must get specific labels (katacontainers.io/kata-runtime=true): kubectl# kubectl get nodes --show-labels | grep kata kubernetes-node Ready <none> 5d23h v1.15.0 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,katacontainers.io/kata-runtime=true,kubernetes.io/arch=amd64,kubernetes.io/hostname=kubernetes-node,kubernetes.io/os=linux If you get these labels on previously configured nodes, that means you can run Kata containers now. For testing it we can use example deployment from the kata-deploy repo: kubectl# kubectl apply -f https://raw.githubusercontent.com/kata-containers/packaging/master/kata-deploy/examples/test-deploy-kata-qemu.yaml Congrats, now you have a first running container on the base of real VM with Qemu virtualization. Good luck.