scrapy爬知乎返回500

用scrapy爬 知乎日報時，總是返回500

# -*- coding: utf-8 -*-
import scrapy
#import pdfkit
from zhihudaily.items import ZhihudailyItem
class ZhihuSpider(scrapy.Spider):
    name = 'zhihu'
    allowed_domains = ['daily.zhihu.com']
    start_urls = ['http://daily.zhihu.com/']

    def parse(self, response):
        item = ZhihudailyItem()
        sub_url = response.xpath("//div[@class='col-lg-4'][1]//div[@class='wrap'][1]//@href").extract_first()
        fix_url = "http://daily.zhihu.com"
        item["url"] = fix_url+sub_url

        title  = response.xpath("//div[@class='col-lg-4'][1]//div[@class='wrap'][1]//span/text()").extract_first()
        item["filename"] =  "C:/Users/高翔/Desktop/"+title+".pdf"
        print(item["filename"], item['url'])
        yield item
        #pdfkit.from_url(mix_url, filename)

        

在settings文件中添加USER_AGENT也不好使：

DEFAULT_REQUEST_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
  'Accept-Language': 'zh-CN,zh;q=0.9',
  'Cache-Control':'max-age=0',
  'Connection':'keep-alive',
  'Host':'daily.zhihu.com',
  'USER_AGENT':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3269.3 Safari/537.36'
}

原因在於， settings文件有自己的USER_ANGET，並不是在headers中添加的。

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3269.3 Safari/537.36'

問題解決
基礎不紮實...