# -*- coding: utf-8 -*-
import datetime
import re
import sys
import time
from datetime import timedelta
from io import StringIO
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfpage import PDFPage
import requests
import pymysql
import redis
import os
#日期校驗
def isVaildDate(date):
try:
if ":" in date:
time.strptime(date, "%Y-%m-%d %H:%M:%S")
else:
time.strptime(date, "%Y-%m-%d")
return True
except:
return False
#根據給定的日期，獲取前n天或後n天的日期，n爲正數則是以後的n天，n爲負數則是以前的n天
def get_day_of_day(str2date,n=0):
if(n<0):
n = abs(n)
return (str2date-timedelta(days=n))
else:
return str2date+timedelta(days=n)
#開始日期與結束日期校驗
def valid_date(startDate,endDate):
#獲取當前時間日期
# nowTime_str = datetime.datetime.now().strftime('%Y-%m-%d')
# print(nowTime_str)
#mktime參數爲struc_time,將日期轉化爲秒，
e_time = time.mktime(time.strptime(endDate,"%Y-%m-%d"))
print(e_time)
try:
s_time = time.mktime(time.strptime(startDate, '%Y-%m-%d'))
#print(s_time)
#日期轉化爲int比較
diff = int(e_time)-int(s_time)
#print(diff)
if diff >= 0:
return True
else:
return False
except Exception as e:
print(e)
return False
#讀取pdf文件到字符串
def getContet(pdfPath,pages=None):
if(pdfPath is None):
return
print("pdfUrl===="+pdfPath)
if not pages:
pagenums = set()
else:
pagenums = set(pages)
output = StringIO()
# 創建一個PDF資源管理器對象來存儲共賞資源
manager = PDFResourceManager()
# 創建一個PDF設備對象
converter = TextConverter(manager, output, laparams=LAParams())
# 創建一個PDF解釋器對象
interpreter = PDFPageInterpreter(manager, converter)
# 打開源pdf文件
with open(pdfPath, 'rb') as infile:
# 對pdf每一頁進行分析
try:
for page in PDFPage.get_pages(infile, pagenums):
interpreter.process_page(page)
except Exception as e:
print(e)
print("PDF內容提取失敗")
converter.close()
# 得到每一頁的txt文檔
text = output.getvalue()
output.close
#print(text)
pattern = r'\s+'
#logging.info("txt轉換成的字符串==="+re.sub(pattern, '',text))
#print(re.sub(pattern, '',text))
content=re.sub(pattern, '',text)
return content
#下載pdf到本地
def downpdf(urlPDF):
res = requests.get(urlPDF)
if(res.status_code==200):
#獲得文件id
pdfId = re.findall("[^/]+(?!.*/)", urlPDF)
res.raise_for_status()
#pdf_path="C:\\Users\\shangshibang\\Desktop\\ho\\0806\\"+str(pdfId[0])
pdf_path="/pdf/"+str(pdfId[0])#linux下文件路徑
with open(pdf_path, 'wb') as file:
for chunk in res.iter_content(100000):
file.write(chunk)
return pdf_path
else:
return
#獲取配置文件信息
import configparser
config = configparser.ConfigParser()
config.read('property.conf',encoding='UTF-8')
#從數據庫中獲取pdf的下載鏈接
ip=config['mysql']['ip']
user=config['mysql']['user']
password=config['mysql']['password']
dabase=config['mysql']['dabase']
# 打開數據庫連接
db = pymysql.connect(ip, user, password,dabase, charset='utf8' )
# 使用cursor()方法獲取操作遊標
cursor = db.cursor()
redisHost=config['redis']['ip']
redisPort=config['redis']['port']
rdp = redis.ConnectionPool(host=redisHost, port=redisPort, password="")
#redis連接池
r = redis.StrictRedis(connection_pool=rdp)
if __name__ == "__main__":
if((len(sys.argv)<=2) or (len(sys.argv)>3)):
raise Exception("請按格式執行腳本，如：python pdf_handle_store.py 2017-08-01 2017-08-30")
#日期字符串校驗
frist = isVaildDate(sys.argv[1])
if(not frist):
raise Exception("請輸入有效的日期")
second = isVaildDate(sys.argv[2])
if (not second):
raise Exception("請輸入有效的日期")
result = valid_date(sys.argv[1], sys.argv[2])
if(not result):
raise Exception("開始時間不能大於結束時間")
startDate=sys.argv[1]
endDate=sys.argv[2]
#獲得腳本需要停止的日期
strStop = endDate.replace("-", "")
str2dateStop = datetime.datetime.strptime(strStop, "%Y%m%d") # 字符串轉化爲date形式
stopDate = get_day_of_day(str2dateStop, 1).date().strftime("%Y-%m-%d")
#根據當前日期獲取明天的日期
while True:
print(startDate)
#數據查詢及文件下載，PDF內容抽取===================================開始
sql = "select url,notice_id from xxxx where public_time like '%" + startDate + "%' and content is null and (url like '%pdf%' or url like '%PDF%')"
db.ping(reconnect=True)
cursor.execute(sql)
results = cursor.fetchall()
for row in results:
url = row[0]
notice_id = row[1]
pdfPath = downpdf(url)
print(pdfPath)
print(notice_id)
# content中的單引號替換爲雙引號，以免插入數據庫時因爲引號的問題導致語法錯誤無法插入
pdfContent=getContet(pdfPath)
if(pdfContent is not None):
content = pdfContent.replace('\'', '\"')
#將抽取內容後的PDF刪除
if os.path.exists(pdfPath): # 如果文件存在
# 刪除文件
os.remove(pdfPath)
else:
print('no such file:%s' % pdfPath) # 則返回文件不存在
sql = "update ssb_insight_notice set xxxx.content='%s',es_store=0 where notice_id=%s" % (
content, notice_id)
with r.pipeline(transaction=False) as p:
try:
cursor.execute(sql)
db.commit()
#將成功解析的notice_id插入redis success_url
p.sadd("success_noticeId", notice_id)
except Exception as e:
print(e)
print("修改失敗")
p.sadd("fail_noticeId", notice_id)
#將成功解析的notice_id插入redis fail_noticeId
finally:
p.execute()
db.close()
#數據查詢及文件下載，PDF內容抽取===================================結束
strContent = startDate.replace("-", "")
str2date = datetime.datetime.strptime(strContent, "%Y%m%d") # 字符串轉化爲date形式
nextDay = get_day_of_day(str2date, 1).date().strftime("%Y-%m-%d")
startDate=nextDay
if(startDate==stopDate):
break