Python爬蟲學習之下載源文件與數據存儲

下載源文件與數據存儲

下載源文件 urlretrieve

多用於非結構化的數據（圖片等），以下載http://www.pythonscraping.com 上的logo.jpg爲例：

'''urlretrieve'''
from urllib.request import urlretrieve
from urllib.request import urlopen
from bs4 import BeautifulSoup
html=urlopen("http://www.pythonscraping.com")
bsObj=BeautifulSoup(html,"html.parser")
imageLocation=bsObj.find("a",id="logo").find("img").attrs["src"]
urlretrieve(imageLocation,"logo.jpg")

'''顯示logo.jpg'''
import matplotlib.pyplot as plt # plt 用於顯示圖片
import matplotlib.image as mpimg # mpimg 用於讀取圖片
 
logo = mpimg.imread('logo.jpg') # 讀取和代碼處於同一目錄下的logo.jpg
plt.imshow(logo) # 顯示圖片
plt.axis('off') # 不顯示座標軸
plt.show()

下載多個文件到指定路徑下

• 根據圖形文件在服務器上存儲的路徑在本機上建立相同的存儲路徑，引進 os 庫
• 如果不是該服務器上的圖形文件，則不下載

# from urllib.request import urlretrieve
from urllib.request import urlopen
from bs4 import BeautifulSoup
import os
baseUrl="http://pythonscraping.com"
baseDirectory="downloaded"
def getAbsoluteUrl(baseUrl,source):
    if source.startswith("http://www."):
        absoluteUrl="http://"+source[11:]
    elif source.startswith("http://"):
        absoluteUrl=source
    elif source.startswith("wwww."):
        absoluteUrl="http://"+source[4:]
    else:
        absoluteUrl=baseUrl+"/"+source
    if baseUrl in absoluteUrl:
        return absoluteUrl

def getDownloadPath(baseUrl,absoluteUrl,baseDirectory):
    downloadPath=baseDirectory+absoluteUrl.replace(baseUrl,"")
    directory=os.path.dirname(downloadPath)
    if not os.path.exists(directory):
        os.makedirs(directory)
    return downloadPath

html=urlopen("http://www.pythonscraping.com")
bsObj=BeautifulSoup(html,"html.parser")
imageList=bsObj.findAll("img",src=True)
for image in imageList:
    absoluteUrl=getAbsoluteUrl(baseUrl,image.attrs["src"])
    if absoluteUrl is not None:
        print(absoluteUrl)
        downloadPath=getDownloadPath(baseUrl,absoluteUrl,baseDirectory)
        print(downloadPath)
        urlretrieve(absoluteUrl,downloadPath)

http://pythonscraping.com/sites/default/files/lrg_0.jpg
downloaded/sites/default/files/lrg_0.jpg
http://pythonscraping.com/img/lrg%20(1).jpg
downloaded/img/lrg%20(1).jpg

將結果寫入文件（csv）

適用於結構化的數據（表格形式的）。

import csv
csvFile=open("test.csv","w+",newline="",encoding="utf-8-sig")
writer=csv.writer(csvFile)
writer.writerow(('number','number plus 2','number times 2'))
for i in range(10):
    writer.writerow((i,i+2,i*2))
csvFile.close()

csvFile=open("test.csv","r",encoding="utf-8-sig")
reader=csv.reader(csvFile)
for row in reader:
    print(str(row))
csvFile.close()

['number', 'number plus 2', 'number times 2']
['0', '2', '0']
['1', '3', '2']
['2', '4', '4']
['3', '5', '6']
['4', '6', '8']
['5', '7', '10']
['6', '8', '12']
['7', '9', '14']
['8', '10', '16']
['9', '11', '18']