動態網頁爬取方法

import requests
from bs4 import BeautifulSoup
import json
import re
import csv

Cookie = "HAList=a-sz-300520-%u79D1%u5927%u56FD%u521B; em_hq_fls=js; st_si=07173812051945; st_asi=delete; st_pvi=43571872917364; st_sp=2019-08-19%2016%3A08%3A52; st_inirUrl=https%3A%2F%2Fcn.bing.com%2F; st_sn=4; st_psi=20190920155504470-111000300841-5541718974; qgqp_b_id=8305e24557b9f3a02c7da68092480aca"
url = "http://push2.eastmoney.com/api/qt/stock/trends2/get?secid=1.000002&fields1=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13&fields2=f51,f52,f53,f54,f55,f56,f57,f58&ut=e1e6871893c6386c5ff6967026016627&iscr=0&cb=cb_1568966339259_56575186&cb_1568966339259_56575186=cb_1568966339259_56575186"
headers = {
    'User-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36',
    'Cookie': Cookie,
    'Connection': 'keep-alive',
    'Accept': '*/*',
    'Accept-Encoding': 'gzip, deflate, sdch',
    'Accept-Language': 'zh-CN,zh;q=0.8',
    'Host': 'query.sse.com.cn',
    'Referer': 'http://quote.eastmoney.com/center/hszs.html'
}                            #可以在瀏覽器F12 network中查到動態網頁數據，複製過來就好了
# gethtml(url,headers):
req = requests.get(url,headers=headers)
req.content.decode('gb18030')
req = req.text

data = re.search(r'"trends":([\s\S]*?)$',req).group(1)[:-1]   #抽取json中的trend
data = data.replace('[','').replace(']','').replace('"','')    #處理成長字符串列表
data = data.split(',')　　　　　　　　　　　　　　　　　　　　　　　　#用逗號分割開
data = [data[i:i+8] for i in range(0,len(data),8)]
print(data)

f = open('e:\\shuju\\3.csv', 'w', encoding='gb18030', newline="")    #csv數據存儲
writer = csv.writer(f)
writer.writerow(('日期', '最新價', '單位淨值', '累計淨值', '日增長額', '日增長率', '申購', '贖回'))             #字段名瞎寫的，懶得一個一個對了
for i in data:
    writer.writerow(i)