centos7環境下配置kafka_2.12-2.6.0+zookeeper-3.4.14環境
因為kafka比較穩定，考慮到服務器資源不足，單機版kafka即可滿足業務需求
kafka依賴zookeeper，需要先部署zookeeper
准備工作：
添加啟動kafka和zookeeper的用戶
groupadd -g 1503 zookeeper
useradd -u 1503 -g zookeeper zookeeper
groupadd -g 1502 kafka
useradd -u 1502 -g kafka kafka
1.部署zookeeper
cd /usr/local/elk/
wget https://mirrors.bfsu.edu.cn/apache/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz
tar xf zookeeper-3.4.14.tar.gz
cd zookeeper-3.4.14
# 編輯zookeeper配置
# vim /usr/local/elk/zookeeper-3.4.14/conf/zoo.cfg tickTime=6000 initLimit=10 syncLimit=25 snapshot=5000 preAllocSize=1000 dataDir=/data/zookeeper/data dataLogDir=/data/zookeeper/logs clientPort=2181 autopurge.snapRetainCount=3 autopurge.purgeInterval=1
# 創建相關日志和數據目錄
mkdir -p /data/zookeeper/data
mkdir -p /data/zookeeper/logs
chown -R zookeeper.zookeeper /data/zookeeper
chown -R zookeeper.zookeeper /usr/local/elk/zookeeper-3.4.14
[root@eus-kafka-01:/usr/local/elk/kafka_2.12-2.1.0/config]# egrep -v '^#|^$' server.properties broker.id=249 port=9092 advertised.host.name=eus-image-design-es-kafka01 advertised.listeners=PLAINTEXT://eus-image-design-es-kafka01:9092 num.network.threads=3 num.io.threads=8 socket.send.buffer.bytes=102400 socket.receive.buffer.bytes=102400 socket.request.max.bytes=104857600 message.max.byte=5242880 replica.fetch.max.bytes=5242880 log.dirs=/data/kafka/logs dataDir=/data/kafka/data num.partitions=1 num.recovery.threads.per.data.dir=1 offsets.topic.replication.factor=1 transaction.state.log.replication.factor=1 transaction.state.log.min.isr=1 delete.topic.enable=true log.retention.hours=168 log.segment.bytes=1073741824 log.retention.check.interval.ms=300000 zookeeper.connect=localhost:2181 # 單zook節點 zookeeper.connection.timeout.ms=6000 group.initial.rebalance.delay.ms=0 # 測試啟動 su zookeeper bin/zkServer.sh start bin/zkServer.sh stop
# 使用systemctl管理程序，發現只能用root啟動，否則報錯
# vim /etc/systemd/system/zookeeper.service
[Unit]
Description=zookeeper.service
After=network.target
[Service]
Type=forking
Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/elk/elasticsearch-7.4.2/jdk/bin"
User=root
Group=root
ExecStart=/usr/local/elk/zookeeper-3.4.14/bin/zkServer.sh start
ExecStop=/usr/local/elk/zookeeper-3.4.14/bin/zkServer.sh stop
PIDFile=/data/zookeeper/data/zookeeper_server.pid
[Install]
WantedBy=multi-user.target
# 用root啟動是不安全的
# 修改 /usr/local/elk/zookeeper-3.4.14/bin/zkServer.sh 配置
首行添加 source /etc/profile # vim /etc/systemd/system/zookeeper.service [Unit] Description=zookeeper.service After=network.target [Service] Type=forking # Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/elk/elasticsearch-7.4.2/jdk/bin" Environment=ZOO_LOG_DIR=/data/zookeeper/logs User=zookeeper Group=zookeeper ExecStart=/usr/local/elk/zookeeper-3.4.14/bin/zkServer.sh start ExecStop=/usr/local/elk/zookeeper-3.4.14/bin/zkServer.sh stop PIDFile=/data/zookeeper/data/zookeeper_server.pid [Install] WantedBy=multi-user.target # 問題解決 systemctl daemon-reload systemctl start zookeeper
2.部署kafka
kafka下載地址：
https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.6.0/kafka_2.12-2.6.0.tgz
# 創建kafka相關的日志和存放數據目錄
mkdir -p /data/kafka/{logs,data}
chown -R kafka.kafka /usr/local/elk/kafka_2.12-2.6.0
chown -R kafka.kafka /data/kafka
# 編輯配置文件
# vim /usr/local/elk/kafka_2.12-2.6.0/config/server.properties
broker.id=249 port=9092 advertised.host.name=eus-image-design-es-kafka01 advertised.listeners=PLAINTEXT://eus-image-design-es-kafka01:9092 num.network.threads=3 num.io.threads=8 socket.send.buffer.bytes=102400 socket.receive.buffer.bytes=102400 socket.request.max.bytes=104857600 message.max.byte=5242880 replica.fetch.max.bytes=5242880 log.dirs=/data/kafka/logs dataDir=/data/kafka/data num.partitions=1 num.recovery.threads.per.data.dir=1
auto.create.topics.enable=true offsets.topic.replication.factor=1 transaction.state.log.replication.factor=1 transaction.state.log.min.isr=1 delete.topic.enable=true log.retention.hours=168 log.segment.bytes=1073741824 log.retention.check.interval.ms=300000 zookeeper.connect=localhost:2181 group.initial.rebalance.delay.ms=0 zookeeper.connection.timeout.ms=600000 zookeeper.session.timeout.ms=400000
# 修改hosts(這樣可以做到內網走內網，外網走外網)
# 內網的hosts配置
172.30.0.106 eus-image-design-es-kafka01
外網連接kafka的服務器配置(對應的主機和kafka advertised.listeners 對應)
kafka外網ip eus-image-design-es-kafka01
# vim /usr/local/elk/kafka_2.12-2.6.0/config/zookeeper.properties
dataDir=/data/zookeeper/data
clientPort=2181
maxClientCnxns=0
admin.enableServer=false
#從后台啟動Kafka集群測試
cd /usr/local/elk/kafka_2.12-2.6.0
su kafka
bin/kafka-server-start.sh -daemon config/server.properties
# 啟動kafka報錯：
[2020-08-06 20:06:37,284] INFO Client environment:os.memory.total=1024MB (org.apache.zookeeper.ZooKeeper)
[2020-08-06 20:06:37,288] ERROR Fatal error during KafkaServer startup. Prepare to shutdown (kafka.server.KafkaServer)
java.lang.NumberFormatException: For input string: "2181 # lone"
at java.base/java.lang.NumberFormatException.forInputString(NumberFormatException.java:68)
at java.base/java.lang.Integer.parseInt(Integer.java:658)
at java.base/java.lang.Integer.parseInt(Integer.java:776)
at org.apache.zookeeper.client.ConnectStringParser.<init>(ConnectStringParser.java:75)
at org.apache.zookeeper.ZooKeeper.createDefaultHostProvider(ZooKeeper.java:1311)
at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:1011)
at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:738)
at kafka.zookeeper.ZooKeeperClient.<init>(ZooKeeperClient.scala:111)
at kafka.zk.KafkaZkClient$.apply(KafkaZkClient.scala:1865)
at kafka.server.KafkaServer.createZkClient$1(KafkaServer.scala:419)
at kafka.server.KafkaServer.initZkClient(KafkaServer.scala:444)
at kafka.server.KafkaServer.startup(KafkaServer.scala:222)
at kafka.server.KafkaServerStartable.startup(KafkaServerStartable.scala:44)
at kafka.Kafka$.main(Kafka.scala:82)
at kafka.Kafka.main(Kafka.scala)
解決辦法：
# 去掉后面的注釋即可
zookeeper.connect=localhost:2181 # 單zook節點
# 使用systemctl 管理服務
# vim /etc/systemd/system/kafka.service
[Unit] Description=Apache Kafka server (broker) After=network.target zookeeper.service [Service] Type=simple Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/elk/elasticsearch-7.4.2/jdk/bin" User=kafka Group=kafka ExecStart=/usr/local/elk/kafka_2.12-2.6.0/bin/kafka-server-start.sh /usr/local/elk/kafka_2.12-2.6.0/config/server.properties ExecStop=/usr/local/elk/kafka_2.12-2.6.0/bin/kafka-server-stop.sh [Install] WantedBy=multi-user.target
# 設置開機啟動
# systemctl daemon-reload
# systemctl enable kafka
# systemctl start kafka
kafka可視化工具：https://www.kafkatool.com/download.html
本地hosts解析
1.1.1.1 eus-image-design-es-kafka01
