今天小編就爲大家分享一篇python消費kafka數據批量插入到es的方法,具有很好的參考價值,希望對大家有所幫助。一起跟隨小編過來看看吧
1、es的批量插入
這是爲了方便後期配置的更改,把配置信息放在logging.conf中
用elasticsearch來實現批量操作,先安裝依賴包,sudo pip install Elasticsearch2
from elasticsearch import Elasticsearch
class ImportEsData:
logging.config.fileConfig("logging.conf")
logger = logging.getLogger("msg")
def __init__(self,hosts,index,type):
self.es = Elasticsearch(hosts=hosts.strip(',').split(','), timeout=5000)
self.index = index
self.type = type
def set_date(self,data):
# 批量處理
# es.index(index="test-index",doc_type="test-type",id=42,body={"any":"data","timestamp":datetime.now()})
self.es.index(index=self.index,doc_type=self.index,body=data)
2、使用pykafka消費kafka
1.因爲kafka是0.8,pykafka不支持zk,只能用get_simple_consumer來實現
2.爲了實現多個應用同時消費而且不重消費,所以一個應用消費一個partition
- 爲是確保消費數據量在不滿足10000這個批量值,能在一個時間範圍內插入到es中,這裏設置consumer_timeout_ms一個超時等待時間,退出等待消費阻塞。
4.退出等待消費阻塞後導致無法再消費數據,因此在獲取self.consumer 的外層加入了while True 一個死循環
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from pykafka import KafkaClient
import logging
import logging.config
from ConfigUtil import ConfigUtil
import datetime
class KafkaPython:
logging.config.fileConfig("logging.conf")
logger = logging.getLogger("msg")
logger_data = logging.getLogger("data")
def __init__(self):
self.server = ConfigUtil().get("kafka","kafka_server")
self.topic = ConfigUtil().get("kafka","topic")
self.group = ConfigUtil().get("kafka","group")
self.partition_id = int(ConfigUtil().get("kafka","partition"))
self.consumer_timeout_ms = int(ConfigUtil().get("kafka","consumer_timeout_ms"))
self.consumer = None
self.hosts = ConfigUtil().get("es","hosts")
self.index_name = ConfigUtil().get("es","index_name")
self.type_name = ConfigUtil().get("es","type_name")
def getConnect(self):
client = KafkaClient(self.server)
topic = client.topics[self.topic]
p = topic.partitions
ps={p.get(self.partition_id)}
self.consumer = topic.get_simple_consumer(
consumer_group=self.group,
auto_commit_enable=True,
consumer_timeout_ms=self.consumer_timeout_ms,
# num_consumer_fetchers=1,
# consumer_id='test1',
partitions=ps
)
self.starttime = datetime.datetime.now()
def beginConsumer(self):
print("beginConsumer kafka-python")
imprtEsData = ImportEsData(self.hosts,self.index_name,self.type_name)
#創建ACTIONS
count = 0
ACTIONS = []
while True:
endtime = datetime.datetime.now()
print (endtime - self.starttime).seconds
for message in self.consumer:
if message is not None:
try:
count = count + 1
# print(str(message.partition.id)+","+str(message.offset)+","+str(count))
# self.logger.info(str(message.partition.id)+","+str(message.offset)+","+str(count))
action = {
"_index": self.index_name,
"_type": self.type_name,
"_source": message.value
}
ACTIONS.append(action)
if len(ACTIONS) >= 10000:
imprtEsData.set_date(ACTIONS)
ACTIONS = []
self.consumer.commit_offsets()
endtime = datetime.datetime.now()
print (endtime - self.starttime).seconds
#break
except (Exception) as e:
# self.consumer.commit_offsets()
print(e)
self.logger.error(e)
self.logger.error(str(message.partition.id)+","+str(message.offset)+","+message.value+"\n")
# self.logger_data.error(message.value+"\n")
# self.consumer.commit_offsets()
if len(ACTIONS) > 0:
self.logger.info("等待時間超過,consumer_timeout_ms,把集合數據插入es")
imprtEsData.set_date(ACTIONS)
ACTIONS = []
self.consumer.commit_offsets()
def disConnect(self):
self.consumer.close()
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
class ImportEsData:
logging.config.fileConfig("logging.conf")
logger = logging.getLogger("msg")
def __init__(self,hosts,index,type):
self.es = Elasticsearch(hosts=hosts.strip(',').split(','), timeout=5000)
self.index = index
self.type = type
def set_date(self,data):
# 批量處理
success = bulk(self.es, data, index=self.index, raise_on_error=True)
self.logger.info(success)
3、運行
if __name__ == '__main__':
kp = KafkaPython()
kp.getConnect()
kp.beginConsumer()
# kp.disConnect()
注:簡單的寫了一個從kafka中讀取數據到一個list裏,當數據達到一個閾值時,在批量插入到 es的插件
現在還在批量的壓測中。。。
推薦我們的python學習基地,看前輩們是如何學習的!從基礎的python腳本、爬蟲、django、數據挖掘等編程技術,還有整理零基礎到項目實戰的資料,送給每一位愛學習python的小夥伴!每天都有老前輩定時講解Python技術,分享一些學習的方法和需要留意的小細節,點擊加入我們的 python學習者聚集地
以上這篇python消費kafka數據批量插入到es的方法就是小編分享給大家的全部內容了