请求的手动发送
class SecondPipeline(object):
f = None
def open_spider(self,spider):
print('start')
self.f = open('./qiubai.text','w',encoding='utf-8')
def process_item(self, item, spider):
self.f.write(item['author']+':'+ item['content'])
return item
def close_spider(self,spider):
self.f.close()
print('end')
import scrapy
from second.items import SecondItem
class QiubaiSpider(scrapy.Spider):
name = 'qiubai'
# allowed_domains = ['www.qiushibaike.com']
start_urls = ['https://www.qiushibaike.com/text/']
url = 'https://www.qiushibaike.com/text/page/%d/'
pageNum = 1
def parse(self, response):
print('正在爬虫')
div_list = response.xpath("//div[@id='content-left']/div")
for div in div_list:
author = div.xpath('./div/a[2]/h2/text()').extract_first()
content = div.xpath(".//div[@class='content']/span/text()").extract_first()
items = SecondItem()
items['author'] = author
items['content'] = content
yield items
# 执行到此,第一个url已经获取成功,
# 手动添加url,yield scrapy.Request,
# callback 函数解析,可以自定义,也可以再次利用parse,不加(),递归加条件
# url = 'https://www.qiushibaike.com/text/page/2/'
# yield scrapy.Request(url=url, callback=self.parse)
if self.pageNum <= 13:
self.pageNum += 1
new_url = format(self.url % self.pageNum)
yield scrapy.Request(url=new_url, callback=self.parse)
小结:
- 多个url手动添加,callback函数执行
- 页面布局一致时,循环调用解析函数递归注意终止条件
- yield scrapy.Request
- 注意面向对象 数据属性 与实例化对象 结合的特性!
- format函数的使用