网络爬虫-课时18信息抽取函式

网络学习笔记

commentURL = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=gn&newsid=comos-{}&group=&compress=0&ie=utf-8&oe=utf-8&page=1&page_size=20'
commentURL.format(newsid)

import re
import json
news = 'http://news.sina.com.cn/c/nd/2016-08-20/doc-ifxvctcc8121090.shtml'

def getCommentCount(newsurl):
    m = re.search('doc-i(.*).shtml',newsurl)
    newsid = m.group(1)
    comments = requests.get(commentURL.format(newsid))
    jd = json.loads(comments.text.strip('var data='))
    return jd['result']['count']['total']

def getNewsDetail(newsurl):
    result = {}
    res = requests.get(newsurl)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text, 'html.parser')
    result['title'] = soup.select('#artibodyTitle')[0].text
    result['newssource'] = soup.select('.time-source span a')[0].text
    timesource = soup.select('.time-source')[0].contents[0].strip()
    result['dt'] = datetime.strptime(timesource, '%Y年%m月%d日%H:%M')
    result['article'] = ' '.join([p.text.strip() for p in soup.select('#artibody p')[:-1]])
    result['editor'] = soup.select('.article-editor')[0].text.lstrip('责任编辑：')
    result['comments'] = getCommentCount(newsurl)
    return result

getNewsDetail('http://news.sina.com.cn/c/nd/2016-08-20/doc-ifxvctcc8121090.shtml')

{'title': '环保部门不再核发机动车环保合格标志',
 'newssource': '新华网',
 'dt': datetime.datetime(2016, 8, 20, 22, 21),
 'article': '原标题：挡风玻璃“减负”：环保部门不再核发机动车环保合格标志 新华社北京8月20日电 记者日前从环境保护部获悉，环保部、公安部、国家认监委联合发布《关于进一步规范排放检验加强机动车环境监督管理工作的通知》，进一步规范机动车排放检验。通知明确，环保部门不再核发机动车环保检验合格标志，以后汽车挡风玻璃上的年检标志将从三个变成两个。',
 'editor': '瞿昆 SN117',
 'comments': 916}