網絡爬蟲-課時18信息抽取函式

網絡學習筆記

commentURL = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=gn&newsid=comos-{}&group=&compress=0&ie=utf-8&oe=utf-8&page=1&page_size=20'
commentURL.format(newsid)

import re
import json
news = 'http://news.sina.com.cn/c/nd/2016-08-20/doc-ifxvctcc8121090.shtml'

def getCommentCount(newsurl):
    m = re.search('doc-i(.*).shtml',newsurl)
    newsid = m.group(1)
    comments = requests.get(commentURL.format(newsid))
    jd = json.loads(comments.text.strip('var data='))
    return jd['result']['count']['total']

def getNewsDetail(newsurl):
    result = {}
    res = requests.get(newsurl)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text, 'html.parser')
    result['title'] = soup.select('#artibodyTitle')[0].text
    result['newssource'] = soup.select('.time-source span a')[0].text
    timesource = soup.select('.time-source')[0].contents[0].strip()
    result['dt'] = datetime.strptime(timesource, '%Y年%m月%d日%H:%M')
    result['article'] = ' '.join([p.text.strip() for p in soup.select('#artibody p')[:-1]])
    result['editor'] = soup.select('.article-editor')[0].text.lstrip('責任編輯：')
    result['comments'] = getCommentCount(newsurl)
    return result

getNewsDetail('http://news.sina.com.cn/c/nd/2016-08-20/doc-ifxvctcc8121090.shtml')

{'title': '環保部門不再核發機動車環保合格標誌',
 'newssource': '新華網',
 'dt': datetime.datetime(2016, 8, 20, 22, 21),
 'article': '原標題：擋風玻璃“減負”：環保部門不再核發機動車環保合格標誌 新華社北京8月20日電 記者日前從環境保護部獲悉，環保部、公安部、國家認監委聯合發佈《關於進一步規範排放檢驗加強機動車環境監督管理工作的通知》，進一步規範機動車排放檢驗。通知明確，環保部門不再核發機動車環保檢驗合格標誌，以後汽車擋風玻璃上的年檢標誌將從三個變成兩個。',
 'editor': '瞿崑 SN117',
 'comments': 916}