python多線程爬蟲爬取喜馬拉雅網頁所有帶id的音頻

 本學期python課程設計：

設計流程圖：

 

代碼實現：

 

#-*-coding:utf-8-*-
import requests
import re
import os
import multiprocessing
import time
from bs4 import BeautifulSoup

def save_path():
    if os.path.exists(str('D:/') + str('ximalaya')):
        pass
    else:
        os.mkdir(str('D:/') + str('ximalaya'))
    file_name = 'D:/' + str('ximalaya')
    os.chdir(file_name)
    if os.path.exists(file_name + str('/mp3/')):
        pass
    else:
        os.mkdir(file_name + str('/mp3/'))
    mp3file_name = file_name + str('/mp3/')

    if os.path.exists(file_name + str('/geci/')):
        pass
    else:
        os.mkdir(file_name + str('/geci/'))
    gecifile_name = file_name + str('/geci/')
    return gecifile_name,mp3file_name

def job(i,id,url1):
    mp3_title = []
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:65.0) Gecko/20100101 Firefox/65.0'
    }
    gecifile_name,mp3file_name = save_path()

    url3 = "https://www.ximalaya.com/revision/play/album?albumId="+str(id)+"&pageNum="+str(i+1)+"&sort=0&pageSize=30"
    print (url3)
    url_html = requests.get(url3, headers=header).text
    mp3title = re.findall(r'"trackName":"(.*?)"', url_html)
    for title in mp3title:
        mp3_title.append(title)


    # 下載MP3
    num = 0
    mp3download = re.findall(r'"src":"(.*?)"', url_html)
    for l in mp3download:
        print (num)
        print (l)
        mp3filename = mp3file_name + mp3_title[num]
        mp3filename = str(mp3filename + '.m4a')
        print(mp3filename)
        url_mp3 = requests.get(l, headers=header).content
        with open(mp3file_name + mp3_title[num]+'.m4a','wb') as f:
            f.write(url_mp3)
        print("OVER!!!!!!!!!!!!!!!!!!!!{}".format(num))
        num += 1

    #下載聲音簡介
    mp3url = re.findall(r'"trackUrl":"(.*?)"', url_html)
    num1 = 0
    for m in mp3url:
        url = url1 + m
        print(url)
        print(gecifile_name + mp3_title[num1] + '.txt')
        print(num1)
        html = requests.get(url, headers=header)
        soup = BeautifulSoup(html.text, 'lxml')
        all_p = soup.find_all('p', {'data-flag': 'normal'})
        f = open(gecifile_name+mp3_title[num1]+'.txt','a+')
        for p in all_p:
            content = p.string
            print(content)
            try:
                f.writelines(content+'\n')
            except:
                print(EOFError)
        print("OVER###################{}".format(num1))
        num1 +=1



def multi_process(id,url1):
   processes = []
   for i in range(2):
       print(i)
       t = multiprocessing.Process(target=job,args=(i,id,url1))
       t.start()
       processes.append(t)
   for process in processes:
       process.join()


def main():
   print("歡迎來到喜馬拉雅山(https://www.ximalaya.com/)")
   url1 = "https://www.ximalaya.com/"
   url2 = input("輸入包含id的網址(https://www.ximalaya.com/yinyue/3092791中的3092791):")
   id = url2
   start = time.time()
   multi_process(id,url1)
   end = time.time()
   print ("下載所用時間：{}s".format(end-start))

if __name__ == '__main__':
   main()

 

效果展示：