Python往kafka生產消費數據

安裝 kafka:  pip install kafka-python

生產數據

from kafka import KafkaProducer
import json
 
'''
    生產者demo
    向test_lyl2主題中循環寫入10條json數據
    注意事項：要寫入json數據需加上value_serializer參數，如下代碼
'''
producer = KafkaProducer(
                            value_serializer=lambda v: json.dumps(v).encode('utf-8'),
                            bootstrap_servers=['192.168.12.101:6667','192.168.12.102:6667','192.168.12.103:6667']
                         )
for i in range(10):
    data={
        "name":"李四",
        "age":23,
        "gender":"男",
        "id":i
    }
    producer.send('test_lyl2', data)
producer.close()

消費數據

from kafka import KafkaConsumer
import json
 
'''
    消費者demo
    消費test_lyl2主題中的數據
    注意事項：如需以json格式讀取數據需加上value_deserializer參數
'''
 
 
consumer = KafkaConsumer('test_lyl2',group_id="lyl-gid1",
                         bootstrap_servers=['192.168.12.101:6667','192.168.12.102:6667','192.168.12.103:6667'],
                         auto_offset_reset='earliest',value_deserializer=json.loads
                         )
for message in consumer:
    print(message.value)