一、scrapy框架
Spiders文件-
Yi_long.py
import requests
import re
import time
import json
class Yi_long():
def __init__(self):
headers = {
"Accept": "application/json, text/javascript, */*; q=0.01",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Cache-Control": "no-cache",
"Host": "www.elong.com",
"Referer": "http://www.elong.com/",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36",
}
city_name_url = 'http://www.elong.com/ajax/search/stayincity?_=1565849748672'
#http://hotel.qunar.com/render/hoteldiv.jsp?&__jscallback=XQScript_8
session = requests.Session()
response = session.get(city_name_url, headers = headers,verify=False)
# response.encoding = 'utf-8'
self.city_list = json.loads(response.text)
# 熱點城市
self.hotCityList = self.city_list["hotCityList"]
# print(self.hotCityList)
# self.hotelCityList = self.city_list["hotelCityList"]
# print(hotelCityList)
with open('D:/Scrapywork/elong_hotels_master/all_city.txt', 'w',encoding='utf-8') as f:
f.write(json.dumps(self.hotCityList))
# f.write(json.dumps(self.hotelCityList))
def get_url(self):
url = 'http://hotel.elong.com/search/list_cn_%s.html'
# 熱點城市
for cityId in self.hotCityList:
# for cityId in self.hotelCityList:
yield url % (cityId["cityId"])
with open(r'D:/Scrapywork/elong_hotels_master/all_city.txt','r',encoding='utf-8') as f:
cold_city_list = f.read()
# print(type(cold_city_list))
for cityid in (json.loads(cold_city_list)):
# print(cityid["cityId"])
if cityid not in self.hotCityList:
# if cityid not in self.hotelCityList:
yield url % (cityid["cityId"])
def get_cold_city(self):
pass
if __name__ == '__main__':
Yilong = Yi_long()
city_list = Yilong.get_url()
for city in city_list:
print(city)
elong.py
# -*- coding: utf-8 -*-
import scrapy
import requests
import re
from .yi_long import Yi_long
from scrapy import Request
class ElongSpider(scrapy.Spider):
name = 'elong'
allowed_domains = ['www.elong.com']
start_urls = ['http://www.elong.com/']
def start_requests(self):
Yilong = Yi_long()
urls = Yilong.get_url()
for url in urls:
headers = {
"Accept": "application/json, text/javascript, */*; q=0.01",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"Host": "hotel.elong.com",
"Origin": "http://hotel.elong.com",
"Pragma": "no-cache",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36",
"X-Requested-With": "XMLHttpRequest",
}
yield Request(url, callback=self.parse) #headers=headers,data=data,
# {"city": "上海", "hotelname": "上海大飯店", "hoteladdress": "徐家彙路679-1號", "score": "4.5", "comment": "6014", "price": "350"},
def parse(self, response):
# result =response.text
city = response.xpath('//div[@data-wrap="cityWrap"]/label/input/@citynamecn').extract()[0]
hotelname = re.compile('data-hotelname="(.*?)"').findall(response.text)
# print(hotelname)
hoteladdress = re.compile('data-hoteladdress="(.*?)"').findall(response.text)
# print(hoteladdress) <span class="c555 block mt5" data-score="4.1">共<b>365</b>條點評</span>
# score = re.compile('data-score="(\d+?)"').findall(response.text)
score = re.compile('<i class="t20 c37e">(.*?) </i>').findall(response.text)
# print("score", score)
comment = response.xpath('//div[@class="h_info_comt"]/a/span[2]/b/text()').extract()
# print("comment ",comment) # 共954條點評
price = re.compile('<span class="h_pri_num ">(\d+?)</span>').findall(response.text)
# print("price",price)
yield {"city": city,'hotelname': hotelname,'hoteladdress': hoteladdress, 'score': score,'comment':comment,'price':price}
item.py
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class ElongHotelsMasterItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass
middlewares.py
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
class ElongHotelsMasterSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class ElongHotelsMasterDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
import datetime
import os
import pymongo
class ElongHotelsMasterPipeline(object):
def open_spider(self, spider):
today = datetime.datetime.now().strftime('%Y%m%d')
self.path = r'json/%s' % today
if not os.path.exists(self.path):
os.makedirs(self.path)
self.client = pymongo.MongoClient()
self.db = self.client.elong
self.collection = self.db[today]
def close_spider(self, spider):
self.db.close()
self.client.close()
def process_item(self, item, spider):
self.write_in_json(item)
return item
def write_in_mongo(self, hotel_info):
self.collection.insert(hotel_info)
# {"city": "上海", "hotelname": "上海大飯店", "hoteladdress": "徐家彙路679-1號", "score": "4.5", "comment": "共6014條點評", "price": "350"},
def write_in_json(self, item):
if item['city']:
city = item['city']
hotelname = item['hotelname']
hoteladdress = item['hoteladdress']
score = item['score']
comment = item['comment']
price= item['price']
print(city)
with open(os.path.join(self.path, city) + '.json', 'w', encoding='utf-8') as fp:
fp.write('{')
for i, hotel in enumerate(hotelname):
hotel_info = {
"city": city,
"hotelname": hotel,
"hoteladdress": hoteladdress[i],
"score": score[i],
"comment": comment[i],
"price": price[i]
}
fp.write(json.dumps(hotel_info, ensure_ascii=False) + ',\n')
self.write_in_mongo(hotel_info)
fp.write('}')
settings.py
# -*- coding: utf-8 -*-
# Scrapy settings for elong_hotels_master project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'elong_hotels_master'
SPIDER_MODULES = ['elong_hotels_master.spiders']
NEWSPIDER_MODULE = 'elong_hotels_master.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'elong_hotels_master (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 2
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
}
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'elong_hotels_master.middlewares.ElongHotelsMasterSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'elong_hotels_master.middlewares.ElongHotelsMasterDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
# MONGO_URI = '127.0.0.1'
# MONGO_PORT = 27017
# MONGO_DATABASE = 'elong'
# COLLECTION_NAME = 'yi_long_hotel'
ITEM_PIPELINES = {
'elong_hotels_master.pipelines.ElongHotelsMasterPipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
AUTOTHROTTLE_ENABLED = True
# The initial download delay
AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
run_elong.py
from scrapy.cmdline import execute
execute(['scrapy','crawl','elong'])
返回結果: