爬蟲 scrapy 抓取小說實例

以http://www.biquge.com/2_2970/ 這部小說爲例，用scrapy對這部小說的章節進行抓取

#coding=utf-8
import scrapy,os
curpath = os.getcwd()
noveldir = ''
class novelSpider(scrapy.Spider):
	name = 'xiaoshuo'
	start_urls = ['http://www.biquge.com/2_2970/']	
	def __init__(self):
		self.noveldir = ''
		
	def parse(self,response):
		title = response.css('div#info h1::text').extract_first()    #小說名
		self.noveldir = os.path.join(curpath,title)
		self.log(self.noveldir )
		if not os.path.exists(self.noveldir ):
			os.makedirs(self.noveldir )                          #創建小說目錄
		self.log('開始下載%s' % title.encode('utf8'))  
		for href in  response.css('dd a').css('a::attr(href)'):      #小說章節鏈接
			yield response.follow(href,self.parse_page)          
	def parse_page(self,response):
		filename = response.css('div.bookname h1::text').extract_first().strip() #章節名，同時作爲文件名
		self.log('開始下載 %s' % filename.encode('utf8'))
		with open(os.path.join(self.noveldir ,filename),'w+') as f:
			for item in response.css('div#content::text').extract():         #小說章節內容寫到文件
				f.write(item.encode("utf8")+"\n")

保存後執行scrapy crawl xiaoshuo就可以看到在目錄下的全部小說章節了