目錄
1.Intro
文件名:pageDownloader.py
模塊名:網頁下載器
引用庫:
selenium | random | sys |
socket | time | urllib2 |
自定義引用文件:configManager、pageResolver、dataDisposer、Console_Color
Console_Color源碼:
#!/usr/bin/env python
# -*- coding:utf-8 -*-
'''
Author: YSW
Time: 2018-7-20
Version: 1.0
Describe: 控制檯顏色打印控制
Update: None
'''
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
'''
開頭部分:
\033[顯示方式;前景色;背景色m
結尾部分:
\033[0m
完整格式:
<\033[顯示方式;前景色;背景色m><要打印的文字><\033[0m>
如果有空格,空格也會打印出來
'''
# 顯示方式參數
DISPLAY_TYPE = {
"默認值": 0,
"高亮": 1,
"非粗體": 22,
"下劃線": 4,
"非下劃線": 24,
"閃爍": 5,
"非閃爍": 25,
"反顯": 7,
"非反顯": 27,
}
# 前景色參數
FOREGROUND_COLOR = {
"黑色": 30,
"紅色": 31,
"綠色": 32,
"黃色": 33,
"藍色": 34,
"洋紅": 35,
"青色": 36,
"白色": 37,
}
# 背景色參數
BACKGROUND_COLOR = {
"黑色": 40,
"紅色": 41,
"綠色": 42,
"黃色": 43,
"藍色": 44,
"洋紅": 45,
"青色": 46,
"白色": 47,
}
def _parameters_3(type, forecolor, backcolor):
'''
返回參數(3)
:param type: 顯示方式
:param forecolor: 前景色
:param backcolor: 背景色
:return: 返回兩個參數值
'''
display_type = DISPLAY_TYPE[type.encode('utf-8')]
foreground_color = FOREGROUND_COLOR[forecolor.encode('utf-8')]
back_color = BACKGROUND_COLOR[backcolor.encode('utf-8')]
return display_type, foreground_color, back_color
def _parameters_2(type, forecolor):
'''
返回參數(2)
:param type: 顯示方式
:param forecolor: 前景色
:return: 返回兩個參數值
'''
display_type = DISPLAY_TYPE[type.encode('utf-8')]
foreground_color = FOREGROUND_COLOR[forecolor.encode('utf-8')]
return display_type, foreground_color
def print_color_back(str, type, forecolor, backcolor):
'''
打印帶背景色的字體
:param str: 字符串
:param type: 顯示方式
:param forecolor: 前景色
:param backcolor: 背景色
:return: 返回打印結果
'''
display_type, foreground_color, back_color = _parameters_3(type, forecolor, backcolor)
head = "\033[{0};{1};{2}m".format(display_type, foreground_color, back_color)
end = "\033[0m"
str_color = head + str + end
print(str_color)
def print_color(str, type="默認值", forecolor="綠色"):
'''
打印字體
:param str: 字符串
:param type: 顯示方式
:param forecolor: 前景色
:return: 返回打印結果
'''
display_type, foreground_color = _parameters_2(type, forecolor)
head = "\033[{0};{1}m".format(display_type, foreground_color)
end = "\033[0m"
str_color = head + str + end
print(str_color)
def print_color_line(str, sign, step, type="默認值", forecolor="綠色"):
'''
打印帶符號行的字體
:param str: 字符串
:param sign: 符號
:param step: 步長
:param type: 顯示方式
:param forecolor: 前景色
:return: 返回打印結果
'''
display_type, foreground_color = _parameters_2(type, forecolor)
head = "\033[{0};{1}m".format(display_type, foreground_color)
end = "\033[0m"
print(head)
print(sign*int(step))
print(str)
print(sign*int(step))
print(end)
功能:大部分網站都是JS動態加載的,所以主要通過selenium獲取網頁源代碼,urllib2獲取靜態頁面源代碼。
2.Source
#!/usr/bin/env Python
# -*- coding: utf-8 -*-
'''
# Author : YSW
# Time : 2018/6/6 14:03
# File : pageDownloader.py
# Version : 2.0
# Describe: 網頁下載器
# Update :
1.重構了網頁下載器,只存儲當天的數據
2.優化了排序算法
'''
from selenium import webdriver
# 引入配置對象DesiredCapabilities
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import configManager
import random
import sys
import socket
import time
import pageResolver
import dataDisposer
import urllib2
from Lib import Console_Color
# 設置默認編碼,防止出現中文字符亂碼
defaultencoding = 'utf-8'
if sys.getdefaultencoding() != defaultencoding:
reload(sys)
sys.setdefaultencoding(defaultencoding)
HEADERS = {
"User-Agent": random.choice(configManager.headers)
}
URL = configManager.urlData
URL_ZB = configManager.urlData_ZB
class DownLoader(object):
def __init__(self, headers):
self.headers = headers
self.dcap = dict(DesiredCapabilities.PHANTOMJS)
# 從USER_AGENTS列表中隨機選一個瀏覽器頭,僞裝瀏覽器
self.dcap["phantomjs.page.settings.userAgent"] = self.headers
# 不載入圖片,爬頁面速度會快很多
self.dcap["phantomjs.page.settings.loadImages"] = False
@staticmethod
def get_url(url, proxy_dict):
'''
獲得靜態頁面
:param url: 靜態頁面 URL 地址
:param proxy_dict: 代理
:return: 返回靜態網頁源代碼
'''
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
proxy_handler = urllib2.ProxyHandler({proxyProtocol: "{0}:{1}".format(proxyIP, proxyPort)})
opener_proxy = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener_proxy)
request = urllib2.Request(url=url, headers=HEADERS)
response = urllib2.urlopen(request)
html = response.read()
return html
def current_time_parse(self, current_date):
'''
獲取當前時間,返回月份和天數
:return: 當前月份和天數
'''
current_month = current_date.month
current_day = current_date.day
return current_month, current_day
def check_exist(self, tender_table, condition1, value1, condition2, value2):
'''
判斷數據是否存在
:param tender_table: 數據表
:param condition1: 條件1
:param value1: 條件值1
:param condition2: 條件2
:param value2: 條件值2
:return: 不爲空返回 False, 爲空返回 True
'''
list_data = list(tender_table.find(
{
condition1: value1,
condition2: value2
}
)
)
if len(list_data) == 0:
return True
return False
#### 招投標數據 ####
def downloader_ynsggzxxt(self, url, proxy_dict):
'''
雲南省公共資源交易中心電子服務系統下載器
'''
website_name = "雲南省公共資源交易中心電子服務系統_工程建設"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('ynsggzxxt')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 201):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
# 解析網頁
resolve_result = resolver.resovler_ynsggzxxt(html, page_num)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_class_name("mmggxlh").find_element_by_link_text('下一頁').click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
projectNumber = resolve["項目編號"]
title = resolve["公告標題"]
start_time = resolve["發佈時間"]
end_time = resolve["截止時間"]
status = resolve["狀態"]
href = resolve["鏈接"]
Console_Color.print_color("[+] 項目編號:{0},公告標題:{1},發佈時間:{2},截止時間:{3},狀態:{4},鏈接:{5}".format(projectNumber, title.encode('utf-8'),
start_time, end_time,
status.encode('utf-8'), href))
for resolve in data_list:
# 打印
projectNumber = resolve["項目編號"]
title = resolve["公告標題"]
start_time = resolve["發佈時間"]
end_time = resolve["截止時間"]
status = resolve["狀態"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="項目編號", value1=projectNumber, condition2="公告標題", value2=title):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 項目編號:{0},公告標題:{1},發佈時間:{2},截止時間:{3},狀態:{4},鏈接:{5}".format(projectNumber, title.encode('utf-8'), start_time, end_time, status.encode('utf-8'), href))
print("[+] 存儲成功")
def downloader_ynsggzxxt_zf(self, url, proxy_dict):
'''
雲南省公共資源交易中心電子服務系統下載器
'''
website_name = "雲南省公共資源交易中心電子服務系統_政府採購"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('ynsggzxxt_zf')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 201):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
# 解析網頁
resolve_result = resolver.resovler_ynsggzxxt_zf(html, page_num)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_class_name("mmggxlh").find_element_by_link_text('下一頁').click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
# 打印
projectNumber = resolve["項目編號"]
title = resolve["公告標題"]
start_time = resolve["發佈時間"]
end_time = resolve["截止時間"]
status = resolve["狀態"]
href = resolve["鏈接"]
Console_Color.print_color(
"[+] 項目編號:{0},公告標題:{1},發佈時間:{2},截止時間:{3},狀態:{4},鏈接:{5}".format(projectNumber, title.encode('utf-8'),
start_time, end_time,
status.encode('utf-8'), href))
for resolve in data_list:
# 打印
projectNumber = resolve["項目編號"]
title = resolve["公告標題"]
start_time = resolve["發佈時間"]
end_time = resolve["截止時間"]
status = resolve["狀態"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="項目編號", value1=projectNumber, condition2="公告標題", value2=title):
# 存儲信息
dataSaver.insert_data(resolve)
print(
"[+] 項目編號:{0},公告標題:{1},發佈時間:{2},截止時間:{3},狀態:{4},鏈接:{5}".format(projectNumber, title.encode('utf-8'),
start_time, end_time,
status.encode('utf-8'), href))
print("[+] 存儲成功")
def downloader_ynsggzzw(self, url, proxy_dict):
'''
雲南省公共資源交易中心網(舊)下載器
'''
website_name = "雲南省公共資源交易中心網_工程建設"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('ynsggzzw')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 101):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
resolve_result = resolver.resovler_ynsggzzw(html, page_num)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_class_name("mmggxlh").find_element_by_link_text('下一頁').click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
# 打印
projectNumber = resolve["項目編號"]
title = resolve["公告標題"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
Console_Color.print_color("[+] 項目編號:{0},公告標題:{1},發佈時間:{2},鏈接:{3}".format(projectNumber, title.encode('utf-8'), start_time,
href))
for resolve in data_list:
# 打印
projectNumber = resolve["項目編號"]
title = resolve["公告標題"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="項目編號", value1=projectNumber, condition2="公告標題", value2=title):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 項目編號:{0},公告標題:{1},發佈時間:{2},鏈接:{3}".format(projectNumber, title.encode('utf-8'), start_time,
href))
print("[+] 存儲成功")
def downloader_kmsgg(self, url, proxy_dict):
'''
昆明市公共資源交易中心網下載器
'''
website_name = "昆明市公共資源交易中心網_政府採購"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('kmsgg')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 51):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
# 解析網頁
resolve_result = resolver.resovler_kmsgg(html, page_num)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_id("btnAjax_NextPage").click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
# 打印
projectNumber = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
end_time = resolve["結束時間"]
status = resolve["狀態"]
href = resolve["鏈接"]
Console_Color.print_color("[+] 編號:{0},工程名稱:{1},發佈時間:{2},結束時間:{3},狀態:{4},鏈接:{5}".format(projectNumber, project_name,
start_time, end_time, status, href))
for resolve in data_list:
# 打印
projectNumber = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
end_time = resolve["結束時間"]
status = resolve["狀態"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="編號", value1=projectNumber, condition2="工程名稱",
value2=project_name):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 編號:{0},工程名稱:{1},發佈時間:{2},結束時間:{3},狀態:{4},鏈接:{5}".format(projectNumber, project_name,
start_time, end_time, status, href))
print("[+] 存儲成功")
def downloader_kmsgg_gc(self, url, proxy_dict):
'''
昆明市公共資源交易中心網下載器
'''
website_name = "昆明市公共資源交易中心網_工程建設"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('kmsgg_gc')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 51):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
# 解析網頁
resolve_result = resolver.resovler_kmsgg_gc(html, page_num)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_id("btnAjax_NextPage").click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
# 打印
projectNumber = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
end_time = resolve["結束時間"]
status = resolve["狀態"]
href = resolve["鏈接"]
Console_Color.print_color("[+] 編號:{0},工程名稱:{1},發佈時間:{2},結束時間:{3},狀態:{4},鏈接:{5}".format(projectNumber, project_name,
start_time, end_time, status, href))
for resolve in data_list:
# 打印
projectNumber = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
end_time = resolve["結束時間"]
status = resolve["狀態"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="編號", value1=projectNumber, condition2="工程名稱",
value2=project_name):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 編號:{0},工程名稱:{1},發佈時間:{2},結束時間:{3},狀態:{4},鏈接:{5}".format(projectNumber, project_name,
start_time, end_time, status, href))
print("[+] 存儲成功")
def downloader_ynszfcgw(self, url, proxy_dict):
'''
雲南省政府採購網下載器
'''
website_name = "雲南省政府採購網"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
# 不知道爲什麼 PhantomJS 打不開這個網站
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('ynszfcgw')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 201):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
# 解析網頁
resolve_result = resolver.resovler_ynszfcgw(html, page_num)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_xpath('//a[@data-page="next"]').click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
# 打印
area = resolve["區劃"]
project_number = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
Console_Color.print_color("[+] 區劃:{0},編號:{1},工程名稱:{2},發佈時間:{3},鏈接:{4}".format(area.encode('utf-8'),
project_number.encode('utf-8'),
project_name.encode('utf-8'), start_time,
href))
for resolve in data_list:
# 打印
area = resolve["區劃"]
project_number = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="編號", value1=project_number, condition2="工程名稱",
value2=project_name):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 區劃:{0},編號:{1},工程名稱:{2},發佈時間:{3},鏈接:{4}".format(area.encode('utf-8'),
project_number.encode('utf-8'),
project_name.encode('utf-8'), start_time,
href))
print("[+] 存儲成功")
#### 中標數據 ####
def downloader_ynsggzxxt_gc_zb(self, url, proxy_dict):
'''
雲南省公共資源交易信息網_工程建設_中標公告下載器
'''
website_name = "雲南省公共資源交易信息網_工程建設_中標公告"
print(url)
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('ynsggzxxt_gc_zb')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 201):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
# 解析網頁
resolve_result = resolver.resovler_ynsggzxxt_gc_zb(html, page_num, proxy_dict)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_class_name("mmggxlh").find_element_by_link_text('下一頁').click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
project_name_parse = resolve["公告名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
people = resolve["中標公司"]
price = resolve["中標價格"]
Console_Color.print_color("[+] 項目名稱:{0},發佈時間:{1},鏈接:{2},中標人:{3},中標價:{4}".format(project_name_parse.encode('utf-8'), start_time, href.encode('utf-8'), people, price))
for resolve in data_list:
# 打印
project_name_parse = resolve["公告名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
people = resolve["中標公司"]
price = resolve["中標價格"]
if self.check_exist(tender_table, condition1="鏈接", value1=href, condition2="公告名稱", value2=project_name_parse):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 項目名稱:{0},發佈時間:{1},鏈接:{2},中標人:{3},中標價:{4}".format(project_name_parse.encode('utf-8'), start_time, href.encode('utf-8'), people, price))
print("[+] 存儲成功")
def downloader_ynsggzxxt_zf_zb(self, url, proxy_dict):
'''
雲南省公共資源交易信息網_政府採購_中標結果下載器
'''
website_name = "雲南省公共資源交易信息網_政府採購_中標結果"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('ynsggzxxt_zf_zb')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 201):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
# 解析網頁
resolve_result = resolver.resovler_ynsggzxxt_zf_zb(html, page_num)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_class_name("mmggxlh").find_element_by_link_text('下一頁').click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
project_name_parse = resolve["公告名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
Console_Color.print_color("[+] 項目名稱:{0},發佈時間:{1},鏈接:{2}".format(project_name_parse.encode('utf-8'), start_time, href.encode('utf-8')))
for resolve in data_list:
# 打印
project_name_parse = resolve["公告名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="鏈接", value1=href, condition2="公告名稱", value2=project_name_parse):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 項目名稱:{0},發佈時間:{1},鏈接:{2}".format(project_name_parse.encode('utf-8'), start_time, href.encode('utf-8')))
print("[+] 存儲成功")
def downloader_ynsggzzw_gc_zb(self, url, proxy_dict):
'''
雲南省公共資源交易中心_工程建設_中標結果下載器
'''
website_name = "雲南省公共資源交易中心_工程建設_中標結果"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('ynsggzzw_gc_zb')
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 101):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
resolve_result = resolver.resovler_ynsggzzw_gc_zb(html, page_num, proxy_dict)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_class_name("mmggxlh").find_element_by_link_text('下一頁').click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
# 打印
title = resolve["公告標題"]
people = resolve["中標公司"]
price = resolve["中標價格"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
Console_Color.print_color("[+] 公告標題:{0},發佈時間:{1},中標人:{2},中標價:{3},鏈接:{4}".format(title.encode('utf-8'), start_time,
people.encode('utf-8'), price.encode('utf-8'),
href))
for resolve in data_list:
# 打印
title = resolve["公告標題"]
people = resolve["中標公司"]
price = resolve["中標價格"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="鏈接", value1=href, condition2="公告標題", value2=title):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 公告標題:{0},發佈時間:{1},中標人:{2},中標價:{3},鏈接:{4}".format(title.encode('utf-8'), start_time,
people.encode('utf-8'), price.encode('utf-8'),
href))
print("[+] 存儲成功")
def downloader_ynsggzzw_zf_zb(self, url, proxy_dict):
'''
雲南省公共資源交易中心_政府採購_結果公示下載器
'''
website_name = "雲南省公共資源交易中心_政府採購_結果公示"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('ynsggzzw_zf_zb')
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 101):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
resolve_result = resolver.resovler_ynsggzzw_zf_zb(html, page_num)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_class_name("mmggxlh").find_element_by_link_text('下一頁').click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
# 打印
title = resolve["公告標題"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
Console_Color.print_color("[+] 公告標題:{0},發佈時間:{1},鏈接:{2}".format(title.encode('utf-8'), start_time, href))
for resolve in data_list:
# 打印
title = resolve["公告標題"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="鏈接", value1=href, condition2="公告標題", value2=title):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 公告標題:{0},發佈時間:{1},鏈接:{2}".format(title.encode('utf-8'), start_time, href))
print("[+] 存儲成功")
def downloader_kmsgg_gc_zb(self, url, proxy_dict):
'''
昆明市公共資源交易平臺公共服務系統_工程建設_中標結果公示下載器
'''
website_name = "昆明市公共資源交易平臺公共服務系統_工程建設_中標結果公示"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('kmsgg_gc_zb')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 51):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
# 解析網頁
resolve_result = resolver.resolver_kmsgg_gc_zb(html, page_num)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_id("btnAjax_NextPage").click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
# 打印
projectNumber = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
Console_Color.print_color("[+] 編號:{0},工程名稱:{1},發佈時間:{2},鏈接:{3}".format(projectNumber, project_name,
start_time, href))
for resolve in data_list:
# 打印
projectNumber = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="編號", value1=projectNumber, condition2="工程名稱",
value2=project_name):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 編號:{0},工程名稱:{1},發佈時間:{2},鏈接:{3}".format(projectNumber, project_name,
start_time, href))
print("[+] 存儲成功")
def downloader_kmsgg_zf_zb(self, url, proxy_dict):
'''
昆明市公共資源交易平臺公共服務系統_政府採購_結果公示下載器
'''
website_name = "昆明市公共資源交易平臺公共服務系統_政府採購_結果公示"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('kmsgg_zf_zb')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 51):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
# 解析網頁
resolve_result = resolver.resolver_kmsgg_zf_zb(html, page_num)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_id("btnAjax_NextPage").click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
# 打印
projectNumber = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
Console_Color.print_color("[+] 編號:{0},工程名稱:{1},發佈時間:{2},鏈接:{3}".format(projectNumber, project_name,
start_time, href))
for resolve in data_list:
# 打印
projectNumber = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="編號", value1=projectNumber, condition2="工程名稱",
value2=project_name):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 編號:{0},工程名稱:{1},發佈時間:{2},鏈接:{3}".format(projectNumber, project_name,
start_time, href))
print("[+] 存儲成功")
def downloader_kmsgg_gc_by(self, url, proxy_dict):
'''
昆明市公共資源交易平臺公共服務系統_工程建設_補遺通知下載器
'''
website_name = "昆明市公共資源交易平臺公共服務系統_工程建設_補遺通知"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('kmsgg_gc_by')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 51):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
# 解析網頁
resolve_result = resolver.resolver_kmsgg_gc_by(html, page_num)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_id("btnAjax_NextPage").click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
# 打印
projectNumber = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
Console_Color.print_color("[+] 編號:{0},工程名稱:{1},發佈時間:{2},鏈接:{3}".format(projectNumber, project_name,
start_time, href))
for resolve in data_list:
# 打印
projectNumber = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="編號", value1=projectNumber, condition2="工程名稱",
value2=project_name):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 編號:{0},工程名稱:{1},發佈時間:{2},鏈接:{3}".format(projectNumber, project_name,
start_time, href))
print("[+] 存儲成功")
def downloader_kmsgg_zf_by(self, url, proxy_dict):
'''
昆明市公共資源交易平臺公共服務系統_政府採購_補遺通知下載器
'''
website_name = "昆明市公共資源交易平臺公共服務系統_政府採購_補遺通知"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('kmsgg_zf_by')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(1, 51):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
# 解析網頁
resolve_result = resolver.resolver_kmsgg_zf_by(html, page_num)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_id("btnAjax_NextPage").click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
# 打印
projectNumber = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
Console_Color.print_color("[+] 編號:{0},工程名稱:{1},發佈時間:{2},鏈接:{3}".format(projectNumber, project_name,
start_time, href))
for resolve in data_list:
# 打印
projectNumber = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
if self.check_exist(tender_table, condition1="編號", value1=projectNumber, condition2="工程名稱",
value2=project_name):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 編號:{0},工程名稱:{1},發佈時間:{2},鏈接:{3}".format(projectNumber, project_name,
start_time, href))
print("[+] 存儲成功")
def downloader_ynszfcgw_cg(self, url, proxy_dict):
'''
雲南省政府採購網_採購結果下載器
'''
website_name = "雲南省政府採購網_採購結果"
Console_Color.print_color("[+] 當前網站:{0}".format(website_name), forecolor="青色")
# 設置代理
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
service_args = ['--proxy={0}:{1}', '--proxy-type={2}'.format(proxyIP, proxyPort, proxyProtocol)]
# 初始化driver對象,傳入瀏覽器報頭參數和代理IP,並獲取網頁URL
# 不知道爲什麼 PhantomJS 打不開這個網站
driver = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver_second = webdriver.PhantomJS(desired_capabilities=self.dcap, service_args=service_args)
driver.get(url)
# 點擊後加載到中標頁面
driver.find_element_by_class_name("panel-heading-search").find_element_by_link_text('中標、成交公告').click()
# 創建網頁解析器對象
resolver = pageResolver.Resolver()
# 創建數據存儲對象
dataSaver = dataDisposer.DataStore('ynszfcgw_cg')
# 創建表對象
tender_table = dataSaver.tender_table()
# 獲取當前時間
current_date = dataDisposer.current_time()
data_list = []
print("[*] 開始存儲數據")
for page_num in range(0, 201):
# 設置時間戳,防止被識別
timeout = 20
socket.setdefaulttimeout(timeout)
sleep_download_time = 3
time.sleep(sleep_download_time)
print("[+] 正在抓取第{0}頁信息".format(page_num))
html = driver.page_source
print("[+] 抓取成功")
# 解析網頁
resolve_result = resolver.resolver_ynszfcgw_cg(html, page_num, driver_second)
# 獲取當前的月數和天數
current_month, current_day = self.current_time_parse(current_date)
# 如果加載到最後一頁就停止點擊
try:
for resolve in sorted(resolve_result, key=lambda x: x['發佈時間'], reverse=True):
# 獲取發佈時間的月數和天數
resolve_month, resolve_day = self.current_time_parse(resolve['發佈時間'])
# 如果是當天時間的數據,則進行存儲
if int(current_month) == int(resolve_month):
if int(current_day) == int(resolve_day):
data_list.append(resolve)
else:
print("[+] 獲取完成")
raise Exception
else:
print("[+] 獲取完成")
raise Exception
# 模擬點擊網頁的下一頁,獲取動態加載的全部網頁
driver.find_element_by_xpath('//a[@data-page="next"]').click()
except Exception:
break
# 關閉當前頁面,如果只有一個頁面
driver.close()
# 關閉瀏覽器
driver.quit()
for resolve in data_list:
# 打印
area = resolve["區劃"]
project_number = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
people = resolve["中標公司"]
price = resolve["中標價格"]
Console_Color.print_color("[+] 區劃:{0},編號:{1},工程名稱:{2},發佈時間:{3},鏈接:{4},中標公司:{5},中標價格:{6}".format(
area.encode('utf-8'),
project_number.encode('utf-8'),
project_name.encode('utf-8'),
start_time,
href,
people.encode('utf-8'),
price.encode('utf-8')
))
for resolve in data_list:
# 打印
area = resolve["區劃"]
project_number = resolve["編號"]
project_name = resolve["工程名稱"]
start_time = resolve["發佈時間"]
href = resolve["鏈接"]
people = resolve["中標公司"]
price = resolve["中標價格"]
if self.check_exist(tender_table, condition1="編號", value1=project_number, condition2="工程名稱",
value2=project_name):
# 存儲信息
dataSaver.insert_data(resolve)
print("[+] 區劃:{0},編號:{1},工程名稱:{2},發佈時間:{3},鏈接:{4},中標公司:{5},中標價格:{6}".format(area.encode('utf-8'),
project_number.encode('utf-8'),
project_name.encode('utf-8'), start_time,
href,
people.encode('utf-8'),
price.encode('utf-8')))
print("[+] 存儲成功")