python 网络爬虫01

简介

简单的爬取一下百度

"""
爬起百度网页
"""
import requests

# 指定要爬取的路径
url = "https://www.baidu.com/s"

# 设置请求头，添加UA字段，模拟浏览器操作 
# 如果不设置的话就会出现这样的请求，百度那边可以通过  User-Agent 识别的出来你是通过pythoon爬取网页数据的 
# 这里有个坑，一定要自己打开浏览器找到属于自己的那个对应的请求头，否则jj了。可以在浏览器通过f12在浏览器控制台查看到你的请求头里面对应的参数
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3741.400 QQBrowser/10.5.3863.400",
    "Cookie": "BAIDUID=80318213050C70FA4B0C391550F555AE:FG=1; BIDUPSID=80318213050C70FA4B0C391550F555AE; PSTM=1572569970; BD_UPN=1a314753; BDUSS=lUSC1Yd09ldWZ4Sm9OV0Nlc3FiTVVNZVVLclhETFVwSnF-Nn4xMkM1ZHJTfnRkRVFBQUFBJCQAAAAAAAAAAAEAAAD64E5HAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGu-011rvtNdbV; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; ispeed_lsm=0; COOKIE_SESSION=35417_72_9_0_239_142_1_9_0_9_2_9_35536_0_3_232_1575198744_1574348164_1575198741%7C9%23452652_22_1574347932%7C5; BD_HOME=1; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; delPer=0; BD_CK_SAM=1; PSINO=7; H_PS_PSSID=1452_21102_30211_20880_29700; H_PS_645EC=00a7HVWKUJg8wftDg%2FwzzfxmJUP2dfpRwq21%2FMDNtQdM4%2FKNp9fv5hF%2BXkSwdACfD97T",
    "Host": "www.baidu.com"
}

# 设置请求参数
params = {
    "wd": "中国"
}

# 请求地址，获得相响应
response = requests.get(url, params=params, headers=headers)


print(f"响应编码:{response.encoding}")
print(f"响应状态码:{response.status_code}")

# 设置编码格式 否则可能编码
response.encoding="utf-8"
# 打印内容
print(response.text)
print(response.content)
print(response.url)

# 写这玩意好麻烦
# f = open("百度.html", "w", encoding="UTF-8")
# f.write(response.text)
# f.close()


# 打开文件写完就自动关闭，这是文件读写的特有方式
with open("中国.html", "w", encoding="UTF-8") as f:
    f.write(response.text)

爬取豆瓣电影TOP250，分页保存电影数据

"""
爬取豆瓣电影TOP250，分页保存电影数据
"""
import requests
import time

# 设置请求头
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3741.400 QQBrowser/10.5.3863.400",
}

# 循环爬取10页数据
for i in range(10):
    # 这是请求地址
    url = f"https://movie.douban.com/top250?start={i * 25}"
    # 这是请求数据 verify=False 这是忽略掉 https请求中的警告，如果不设置就jj了 
    response = requests.get(url, headers=headers, verify=False)
    print(response.status_code)
    if response.status_code == 200:
        with open(f"第{i+1}页.txt", "w", encoding="UTF-8") as f:
            f.write(response.text)
            print(f"{url} 保存成功")
    # 睡个两秒 防止网页封杀
    time.sleep(2)

voer