70行python代碼爬取新浪財經中股票歷史成交明細

最近在研究股票量化，想從每筆成交的明細着手，但歷史數據的獲取便是一個大問題，一些股票證券軟件又不能批量導出成交數據。所以，我花了兩天時間，成功的從新浪財經爬取了我要的數據

下面開始 

新浪股票明細數據接口爲

http://market.finance.sina.com.cn/transHis.php?symbol=sz000001&date=2018-07-17&page=3

格式不用多說symbol=股票代碼 date=日期 page=頁碼

 

進去就可以看到一個表格，這其實就是新浪官方網頁中展現出來的，但我們在這裏比較好爬取，因爲沒有其他多餘的內容，以前好像是可以直接下載下來的，很不幸現在不行了，但沒關係，我們開始爬取

1.先獲取整個html，這裏我加了個超時重試機制，

def getHtml(url):
    while True:
        try:
            html = urllib.request.urlopen(url, timeout=5).read()
            break
        except:
            print("超時重試")
    html = html.decode('gbk')
    return html

2.獲取列名

def getTitle(tableString):
    s = r'(?<=<thead)>.*?([\s\S]*?)(?=</thead>)'
    pat = re.compile(s)
    code = pat.findall(tableString)
    s2 = r'(?<=<tr).*?>([\s\S]*?)(?=</tr>)'
    pat2 = re.compile(s2)
    code2 = pat2.findall(code[0])
    s3 = r'(?<=<t[h,d]).*?>([\s\S]*?)(?=</t[h,d]>)'
    pat3 = re.compile(s3)
    code3 = pat3.findall(code2[0])
    return code3

3.獲取所有記錄

def getBody(tableString):
    s = r'(?<=<tbody)>.*?([\s\S]*?)(?=</tbody>)'
    pat = re.compile(s)
    code = pat.findall(tableString)
    s2 = r'(?<=<tr).*?>([\s\S]*?)(?=</tr>)'
    pat2 = re.compile(s2)
    code2 = pat2.findall(code[0])
    s3 = r'(?<=<t[h,d]).*?>(?!<)([\s\S]*?)(?=</)[^>]*>'
    pat3 = re.compile(s3)
    code3 = []
    for tr in code2:
        code3.append(pat3.findall(tr))
    return code3

4.然後用個while把每一頁都獲取出來，當然還要判斷一下是不是到最後一頁了，或者其他一些細節

while True:
    Url = 'http://market.finance.sina.com.cn/transHis.php?symbol=' + symbol + '&date=' + date + '&page=' + str(page)
    html = getHtml(Url)
    table = getTable(html)
    if len(table) != 0:
        tbody = getBody(table[0])
        if len(tbody) == 0:
            print("結束")
            break
        if page == 1:
            thead = getTitle(table[0])
            print(thead)
        for tr in tbody:
            print(tr)
    else:
        print("當日無數據")
        break
    page += 1

5.輸出結果

怎麼樣，還不錯吧，然後導入數據庫就可以了

 

本人學生一枚，自學python沒多久，正則表達式也才花了半天時間學的，代碼會比較簡陋，不足的地方還請多指教

源碼：

import urllib.request
import re
import datetime


def getHtml(url):
    while True:
        try:
            html = urllib.request.urlopen(url, timeout=5).read()
            break
        except:
            print("超時重試")
    html = html.decode('gbk')
    return html


def getTable(html):
    s = r'(?<=<table class="datatbl" id="datatbl">)([\s\S]*?)(?=</table>)'
    pat = re.compile(s)
    code = pat.findall(html)
    return code


def getTitle(tableString):
    s = r'(?<=<thead)>.*?([\s\S]*?)(?=</thead>)'
    pat = re.compile(s)
    code = pat.findall(tableString)
    s2 = r'(?<=<tr).*?>([\s\S]*?)(?=</tr>)'
    pat2 = re.compile(s2)
    code2 = pat2.findall(code[0])
    s3 = r'(?<=<t[h,d]).*?>([\s\S]*?)(?=</t[h,d]>)'
    pat3 = re.compile(s3)
    code3 = pat3.findall(code2[0])
    return code3


def getBody(tableString):
    s = r'(?<=<tbody)>.*?([\s\S]*?)(?=</tbody>)'
    pat = re.compile(s)
    code = pat.findall(tableString)
    s2 = r'(?<=<tr).*?>([\s\S]*?)(?=</tr>)'
    pat2 = re.compile(s2)
    code2 = pat2.findall(code[0])
    s3 = r'(?<=<t[h,d]).*?>(?!<)([\s\S]*?)(?=</)[^>]*>'
    pat3 = re.compile(s3)
    code3 = []
    for tr in code2:
        code3.append(pat3.findall(tr))
    return code3


# 股票代碼
symbol = 'sz000001'
# 日期
dateObj = datetime.datetime(2018, 6, 1)
date = dateObj.strftime("%Y-%m-%d")

# 頁碼，因爲不止1頁，從第一頁開始爬取
page = 1

while True:
    Url = 'http://market.finance.sina.com.cn/transHis.php?symbol=' + symbol + '&date=' + date + '&page=' + str(page)
    html = getHtml(Url)
    table = getTable(html)
    if len(table) != 0:
        tbody = getBody(table[0])
        if len(tbody) == 0:
            print("結束")
            break
        if page == 1:
            thead = getTitle(table[0])
            print(thead)
        for tr in tbody:
            print(tr)
    else:
        print("當日無數據")
        break
    page += 1