一 kafka本地目錄結構
[root@hadoop ~]# cd /tmp/kafka-logs1 [root@hadoop kafka-logs1]# find . . ./.lock ./recovery-point-offset-checkpoint ./log-start-offset-checkpoint ./cleaner-offset-checkpoint ./replication-offset-checkpoint ./meta.properties ./mytest-1 ./mytest-1/leader-epoch-checkpoint ./mytest-1/00000000000000000000.log ./mytest-1/00000000000000000000.index ./mytest-1/00000000000000000000.timeindex ./mytest-0 ./mytest-0/leader-epoch-checkpoint ./mytest-0/00000000000000000000.log ./mytest-0/00000000000000000000.index ./mytest-0/00000000000000000000.timeindex |
搭建單節點多broker的kafka后，啟動zk和kafka。
[root@hadoop ~]# cd /usr/local/kafka [root@hadoop kafka]# zookeeper-server-start.sh config/zookeeper.properties ... [root@hadoop kafka]# kafka-server-start.sh config/server0.properties & ... [root@hadoop kafka]# kafka-server-start.sh config/server1.properties & ... [root@hadoop kafka]# kafka-server-start.sh config/server2.properties & ... [root@hadoop ~]# jps 4133 QuorumPeerMain 4791 Kafka 5452 Kafka 5780 Kafka 6164 Jps
創建kafka集群時我已經創建了一個主題test02,現在我們再創建一個主題mytest（2個分區）
[root@hadoop ~]# cd /usr/local/kafka [root@hadoop kafka]# kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic mytest Created topic "mytest".
查看日志目錄可以發現，3個日志目錄幾乎是一致的（__consumer_offsets-0是個什么鬼？）
test02和mytest副本數均是3，所以3個broker的log下面均有；
test02只有一個分區，mytest有2個分區，所以有一個test02目錄，和2個mytest目錄。
[root@hadoop ~]# ll /tmp/kafka-logs0 總用量 16 -rw-r--r-- 1 root root 0 8月 3 11:02 cleaner-offset-checkpoint drwxr-xr-x 2 root root 141 8月 3 11:02 __consumer_offsets-0 -rw-r--r-- 1 root root 4 8月 5 00:44 log-start-offset-checkpoint -rw-r--r-- 1 root root 54 8月 3 11:02 meta.properties drwxr-xr-x 2 root root 141 8月 5 00:31 mytest-0 #命名格式：主題-分區 drwxr-xr-x 2 root root 141 8月 5 00:31 mytest-1 -rw-r--r-- 1 root root 1241 8月 5 00:44 recovery-point-offset-checkpoint -rw-r--r-- 1 root root 1241 8月 5 00:44 replication-offset-checkpoint drwxr-xr-x 2 root root 178 8月 3 11:49 test02-0 [root@hadoop ~]# ll /tmp/kafka-logs1 總用量 16 -rw-r--r-- 1 root root 0 7月 31 16:05 cleaner-offset-checkpoint -rw-r--r-- 1 root root 4 8月 5 00:48 log-start-offset-checkpoint -rw-r--r-- 1 root root 54 7月 31 16:05 meta.properties drwxr-xr-x 2 root root 141 8月 5 00:31 mytest-0 drwxr-xr-x 2 root root 141 8月 5 00:31 mytest-1 -rw-r--r-- 1 root root 85 8月 5 00:48 recovery-point-offset-checkpoint -rw-r--r-- 1 root root 85 8月 5 00:49 replication-offset-checkpoint drwxr-xr-x 2 root root 178 8月 4 23:29 test02-0 [root@hadoop ~]# ll /tmp/kafka-logs2 總用量 16 -rw-r--r-- 1 root root 0 7月 31 16:06 cleaner-offset-checkpoint -rw-r--r-- 1 root root 4 8月 5 00:48 log-start-offset-checkpoint -rw-r--r-- 1 root root 54 7月 31 16:06 meta.properties drwxr-xr-x 2 root root 141 8月 5 00:31 mytest-0 drwxr-xr-x 2 root root 141 8月 5 00:31 mytest-1 -rw-r--r-- 1 root root 85 8月 5 00:48 recovery-point-offset-checkpoint -rw-r--r-- 1 root root 85 8月 5 00:49 replication-offset-checkpoint drwxr-xr-x 2 root root 178 8月 4 23:29 test02-0
查看主題目錄
[root@hadoop ~]# ll /tmp/kafka-logs0/test02-0/ 總用量 16 -rw-r--r-- 1 root root 10485760 8月 5 00:12 00000000000000000000.index -rw-r--r-- 1 root root 79 8月 3 11:06 00000000000000000000.log -rw-r--r-- 1 root root 10485756 8月 5 00:12 00000000000000000000.timeindex -rw-r--r-- 1 root root 10 8月 3 11:49 00000000000000000001.snapshot -rw-r--r-- 1 root root 8 8月 3 11:06 leader-epoch-checkpoint
查看元數據信息
[root@hadoop ~]# cat /tmp/kafka-logs0/meta.properties version=0 broker.id=0 [root@hadoop ~]# cat /tmp/kafka-logs1/meta.properties version=0 broker.id=1 [root@hadoop ~]# cat /tmp/kafka-logs2/meta.properties version=0 broker.id=2
向 mytest主題 生產消息並消費消息，可以看出消息分別保存在了不同分區的log里
#生產消息 [root@hadoop kafka]# kafka-console-producer.sh --broker-list localhost:9092,localhost:9093,localhost:9094 --topic mytest >hello kafka >hello world #消費消息 [root@hadoop kafka]# kafka-console-consumer.sh --bootstrap-server localhost:9092,localhost:9093,localhost:9094 --topic mytest --from-beginning hello kafka hello world #可以看出，消息分別保存在了不同分區的log里 [root@hadoop ~]# cat /tmp/kafka-logs0/mytest-0/00000000000000000000.log C}_Me Ye Yÿÿÿÿÿÿÿÿÿÿÿÿÿÿ"hello kafka[root@hadoop ~]# [root@hadoop ~]# cat /tmp/kafka-logs0/mytest-1/00000000000000000000.log e ꛁe ꞿÿÿÿÿÿÿÿÿÿÿÿÿÿ"hello world[root@hadoop ~]#
二 kafka在zk上的znode
| 1./controller //data = {"version":1,"brokerid":0,"timestamp":"1533396512695"} 2./controller_epoch //data=27#不清楚什么意思，貌似第一次啟動時是1，難道是kafka啟動過的次數？ 3./brokers/ids //實時維護active的broker /brokers/topics /brokers/seqid 4./admin/delete_topics 5./isr_change_notification 6./consumers 7./config 注意：productor不在zk注冊 |
啟動zk客戶端
[root@hadoop ~]# cd /usr/local/kafka [root@hadoop kafka]# zkCli.sh -server hadoop:2181 #啟動zk客戶端 ...
查看根目錄
[zk: hadoop:2181(CONNECTED) 0] ls / #查看znode [cluster, controller_epoch, controller, brokers, zookeeper, admin, isr_change_notification, consumers,
log_dir_event_notification, latest_producer_id_block, config]
查看/controller
[zk: hadoop:2181(CONNECTED) 1] ls /controller [] [zk: hadoop:2181(CONNECTED) 2] get /controller #這里的brokerid為0意思是kafka集群的leader為0 #如果集群中有多個broker，將leader殺死后會發現這里的brokerid變化。 {"version":1,"brokerid":0,"timestamp":"1533396512695"} cZxid = 0x513 ctime = Sat Aug 04 23:28:32 CST 2018 mZxid = 0x513 mtime = Sat Aug 04 23:28:32 CST 2018 pZxid = 0x513 cversion = 0 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x10000711d710001 dataLength = 54 numChildren = 0
查看/controller_epoch
[zk: hadoop:2181(CONNECTED) 3] ls /controller_epoch [] [zk: hadoop:2181(CONNECTED) 4] get /controller_epoch 27 cZxid = 0x1c ctime = Tue Jul 31 14:18:01 CST 2018 mZxid = 0x514 mtime = Sat Aug 04 23:28:32 CST 2018 pZxid = 0x1c cversion = 0 dataVersion = 26 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 2 numChildren = 0
查看/brokers
[zk: hadoop:2181(CONNECTED) 5] get /brokers null cZxid = 0x4 ctime = Tue Jul 31 14:17:50 CST 2018 mZxid = 0x4 mtime = Tue Jul 31 14:17:50 CST 2018 pZxid = 0xd cversion = 3 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 0 numChildren = 3 #3個孩子分別是ids, topics, seqid [zk: hadoop:2181(CONNECTED) 6] ls /brokers [ids, topics, seqid]
查看/brokers/ids
[zk: hadoop:2181(CONNECTED) 7] ls /brokers/ids #顯示kafka集群中的所有active的brokerid #如果殺死broker 0，這里將會顯示[1, 2] [0, 1, 2] [zk: hadoop:2181(CONNECTED) 8] get /brokers/ids/0 {"listener_security_protocol_map":{"PLAINTEXT":"PLAINTEXT"},"endpoints":["PLAINTEXT://hadoop:9092"],
"jmx_port":-1,"host":"hadoop","timestamp":"1533399158574","port":9092,"version":4} cZxid = 0x55f ctime = Sun Aug 05 00:12:38 CST 2018 mZxid = 0x55f mtime = Sun Aug 05 00:12:38 CST 2018 pZxid = 0x55f cversion = 0 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x10000711d710005 dataLength = 182 numChildren = 0
查看/brokers/topics
[zk: hadoop:2181(CONNECTED) 9] ls /brokers/topics [mytest, test02, __consumer_offsets] [zk: hadoop:2181(CONNECTED) 10] ls /brokers/topics/mytest [partitions] [zk: hadoop:2181(CONNECTED) 11] ls /brokers/topics/mytest/partitions #顯示分區個數：mytest主題有2個分區，分別為0和1 [0, 1] [zk: hadoop:2181(CONNECTED) 12] ls /brokers/topics/mytest/partitions/0 [state] [zk: hadoop:2181(CONNECTED) 13] ls /brokers/topics/mytest/partitions/0/state [] #以下可以看出每個分區擁有不同的leader [zk: hadoop:2181(CONNECTED) 14] get /brokers/topics/mytest/partitions/0/state {"controller_epoch":28,"leader":0,"version":1,"leader_epoch":0,"isr":[0,2,1]} cZxid = 0x5a2 ctime = Sun Aug 05 00:31:16 CST 2018 mZxid = 0x5a2 mtime = Sun Aug 05 00:31:16 CST 2018 pZxid = 0x5a2 cversion = 0 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 77 numChildren = 0 [zk: hadoop:2181(CONNECTED) 15] get /brokers/topics/mytest/partitions/1/state {"controller_epoch":28,"leader":1,"version":1,"leader_epoch":0,"isr":[1,0,2]} cZxid = 0x5a1 ctime = Sun Aug 05 00:31:16 CST 2018 mZxid = 0x5a1 mtime = Sun Aug 05 00:31:16 CST 2018 pZxid = 0x5a1 cversion = 0 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 77 numChildren = 0
查看/brokers/seqid
[zk: hadoop:2181(CONNECTED) 16] ls /brokers/seqid [] [zk: hadoop:2181(CONNECTED) 17] get /brokers/seqid null cZxid = 0xd ctime = Tue Jul 31 14:17:50 CST 2018 mZxid = 0xd mtime = Tue Jul 31 14:17:50 CST 2018 pZxid = 0xd cversion = 0 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 0 numChildren = 0
查看/admin/delete_topics
[zk: hadoop:2181(CONNECTED) 18] ls /admin [delete_topics] [zk: hadoop:2181(CONNECTED) 19] ls /admin/delete_topics []
查看/isr_change_notification
[zk: hadoop:2181(CONNECTED) 20] ls /isr_change_notification [] [zk: hadoop:2181(CONNECTED) 21] get /isr_change_notification null cZxid = 0xe ctime = Tue Jul 31 14:17:50 CST 2018 mZxid = 0xe mtime = Tue Jul 31 14:17:50 CST 2018 pZxid = 0x544 cversion = 56 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 0 numChildren = 0
查看/consumers
#不知為何，我這里一直顯示為空，按理說啟動消費者之后這里應該顯示相應信息的 [zk: hadoop:2181(CONNECTED) 22] ls /consumers [] [zk: hadoop:2181(CONNECTED) 23] get /consumers null cZxid = 0x2 ctime = Tue Jul 31 14:17:50 CST 2018 mZxid = 0x2 mtime = Tue Jul 31 14:17:50 CST 2018 pZxid = 0x2 cversion = 0 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 0 numChildren = 0
查看/config
[zk: hadoop:2181(CONNECTED) 24] ls /config [changes, clients, brokers, topics, users] [zk: hadoop:2181(CONNECTED) 25] ls /config/changes [] [zk: hadoop:2181(CONNECTED) 26] ls /config/clients [] [zk: hadoop:2181(CONNECTED) 27] ls /config/brokers [] [zk: hadoop:2181(CONNECTED) 28] ls /config/topics #與ls /brokers/topics結果一致 [mytest, test02, __consumer_offsets] [zk: hadoop:2181(CONNECTED) 29] ls /config/users []
嘗試殺死集群的leader后，查看相對應的znode變化
#1.殺死leader [root@hadoop ~]# ps -ef|grep server0.properties #找到server0的進程號為4791 root 4791 4422 0 8月04 pts/1 00:00:18 ...信息太多，忽略.../server0.properties root 6327 6119 0 00:03 pts/5 00:00:00 grep --color=auto server0.properties [root@hadoop ~]# kill -9 4791 #殺死進程 [root@hadoop ~]# ps -ef|grep server0.properties #再次查看 root 6353 6119 0 00:07 pts/5 00:00:00 grep --color=auto server0.properties #2.查看/controller：brokerid由0變為1 [zk: hadoop:2181(CONNECTED) 30] get /controller {"version":1,"brokerid":1,"timestamp":"1533398833360"} cZxid = 0x54c ctime = Sun Aug 05 00:07:13 CST 2018 mZxid = 0x54c mtime = Sun Aug 05 00:07:13 CST 2018 pZxid = 0x54c cversion = 0 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x10000711d710002 dataLength = 54 numChildren = 0 #3.查看/brokers/ids：active的broker只剩下1和2了 [zk: hadoop:2181(CONNECTED) 31] ls /brokers/ids [1, 2] #4.重新啟動broker 0 [root@hadoop kafka]# kafka-server-start.sh config/server0.properties &
嘗試刪除一個主題test02，查看相對應的znode變化
#1.刪除主題 [root@hadoop ~]# cd /usr/local/kafka [root@hadoop kafka]# kafka-topics.sh --zookeeper localhost:2181 --delete --topic test02 Topic test02 is marked for deletion. Note: This will have no impact if delete.topic.enable is not set to true. #2.查看主題：沒有結果（應該有結果？） [root@hadoop kafka]# kafka-topics.sh --describe --zookeeper localhost:2181 --topic test02 #3.查看log目錄下test02-0目錄是否存在：不存在了 [root@hadoop kafka]# ll /tmp/kafka-logs0 #test02目錄沒有了 總用量 20 -rw-r--r-- 1 root root 4 8月 5 01:34 cleaner-offset-checkpoint drwxr-xr-x 2 root root 141 8月 3 11:02 __consumer_offsets-0 ... drwxr-xr-x 2 root root 141 8月 3 11:02 __consumer_offsets-49 -rw-r--r-- 1 root root 4 8月 5 01:34 log-start-offset-checkpoint -rw-r--r-- 1 root root 54 8月 3 11:02 meta.properties drwxr-xr-x 2 root root 141 8月 5 00:31 mytest-0 drwxr-xr-x 2 root root 141 8月 5 00:31 mytest-1 -rw-r--r-- 1 root root 1230 8月 5 01:34 recovery-point-offset-checkpoint -rw-r--r-- 1 root root 1230 8月 5 01:35 replication-offset-checkpoint #4.查看/admin/delete_topics：為空。按理說這里應該顯示刪除的主題test02 [zk: hadoop:2181(CONNECTED) 32] ls /admin/delete_topics [] #5.查看/brokers/topics：這里顯示test02確實被刪除了 [zk: hadoop:2181(CONNECTED) 33] ls /brokers/topics [mytest, __consumer_offsets] #6.查看/config/topics： [zk: hadoop:2181(CONNECTED) 34] ls /config/topics [mytest, __consumer_offsets] #以上均是我的實際操作結果，與老師演示的有一些出入，暫時無法解釋原因