数据解析
1.Xpath语法和lxml模块
#使用方式:使用//获取整个页面当中的元素,然后写标签名,然后再写谓词进行提取。
//div[@clas='abc']
需要注意的知识点:
1./和//的区别:/只获取直接子节点,//可以获取子孙节点
2.contains:有时候某个属性包含多个值,可以使用cntains
//div[contains(@class,'job_detail')]
3.谓词的下标是从1开始
使用lxml解析HTML代码:
1.解析html字符串:使用’lxml.etree.HTML’进行
htmlElement = etree.HTML(text)
print(etree.tostring(htmlElement,encoding='utf-8').decode('utf-8'))
2.解析html文件:使用’lxml.etree.parse’进行,如果这个函数默认使用xml解析器,需要自己创建html解析器。
htmlElement = etree.parse('qingyunian.html')
print(etree.tostring(htmlElement,encoding='utf-8').decode('utf-8'))
实例
from lxml import etree
#解析庆余年短评
def parse_qyn_file():
parser = etree.HTMLParser(encoding='utf-8')
htmlElement = etree.parse('qingyunian.html',parser=parser)
print(etree.tostring(htmlElement,encoding='utf-8').decode('utf-8'))
#解析拉钩网页
def parse_lagou_file():
parser = etree.HTMLParser(encoding='utf-8')
htmlElement = etree.parse('lagou.html',parser=parser)
print(etree.tostring(htmlElement,encoding='utf-8').decode('utf-8'))
if __name__ == '__main__':
#parse_lagou_file()
#parse_text()
parse_qyn_file()
实例:解析腾讯招聘的网页信息
from lxml import etree
parser = etree.HTMLParser(encoding='utf-8')
html = etree.parse("tencent.html",parser=parser)
#1.获取所有的a标签
#//a
#xpath返回的是一个列表
alis = html.xpath("//a[@class='recruit-list-link']")
for a in alis:
print(etree.tostring(a,encoding='utf-8').decode("utf-8"))
#2.获取所有岗位名称
alis = html.xpath("//h4")
for a in alis:
print(etree.tostring(a,encoding='utf-8').decode("utf-8"))
#3.获取所有职位信息
alis = html.xpath("//p[@class='recruit-text']")
for a in alis:
print(etree.tostring(a,encoding='utf-8').decode("utf-8"))
#4.获取所有的纯文本信息
alis = html.xpath("//a[@class='recruit-list-link']")
positions = []
for a in alis:
#在某个标签下,再执行xpath函数,获取子孙元素,那么应该在//前加一个点,代表在当前元素下获取
title = a.xpath(".//h4[@class='recruit-title']/text()")
daihao = a.xpath("./p[1]//span[1]/text()")
address = a.xpath("./p[1]//span[2]/text()")
category = a.xpath("./p[1]//span[3]/text()")
time = a.xpath("./p[1]//span[5]/text()")
needs = a.xpath("./p[2]/text()")
position = {
'title':title,
'daihao':daihao,
'address':address,
'category':category,
'time':time,
'needs':needs
}
positions.append(position)
print(positions)
#写入Excel表格
#写入Excel表格
import pandas as pd
datadf = pd.DataFrame(positions)
datadf.to_excel('result.xlsx',sheet_name='pachong_cc')
# 导出excel