python+selenium爬蟲按照名單循環爬取作者知網下載量等信息

主要爬取下面的表格內的信息如文獻篇數，被引用數等等
用的是selenium爬蟲

from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
import csv
import time
browser = webdriver.Chrome()
browser.minimize_window()  # 最小化窗口
url = 'http://epub.cnki.net/grid2008/brief/result_src.aspx?comptype=scho&stype=1&&auscho_1_sel=%E5%AD%A6%E8%80%85&auscho_1_value1=%E8%B5%B5%E5%AA%9B&auscho_1_special1=%3D&auscho2_1_sel=%E7%A0%94%E7%A9%B6%E9%A2%86%E5%9F%9F%2C%E7%A0%94%E7%A9%B6%E6%96%B9%E5%90%91%2C%E5%AD%A6%E8%80%85%E7%9F%A5%E8%AF%86%E5%90%91%E9%87%8F&auscho2_1_value1=%E5%9B%BE%E4%B9%A6%E6%83%85%E6%8A%A5&auscho2_1_special1=%25&navicode=&showtitle=%u5B66%u8005%u68C0%u7D22&dbCatalog=%u4E2D%u56FD%u5B66%u672F%u6587%u732E%u7F51%u7EDC%u51FA%u7248%u603B%u5E93'


def start_spider():
    # 請求url
    browser.get(url)

    time.sleep(5)
    try:
        browser.find_element_by_id('iframeResult')# 定位到iframe
    except NoSuchElementException:
        print('no')
    browser.switch_to.parent_frame()#切換到父iframe
    x=browser.switch_to.frame('iframeResult')
    # browser.switch_to.frame('iframeResult')
    a = browser.find_element_by_class_name('s_table')
    tr_content = a.find_elements_by_tag_name("tr")  # 進一步定位到表格內容所在的tr節點
    lst = []  # 存儲爲list
    for tr in tr_content:
        tds =tr.find_elements_by_tag_name("td")  # 進一步定位到表格內容所在的td節點
        for td in tds:

            lst.append(td.text)
        with open('D:\Python_DATA\data.csv','a', encoding='utf-8',newline='') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(lst)
            lst.clear()
    # a=browser.find_element_by_class_name('s_tabletd_rb')
    print(lst)  # 輸出表格內容
    browser.switch_to.default_content()
    f = open('D:\Python_DATA\sd.csv', 'r')
    content = f.read()
    final_list = list()
    rows = content.split('\n')
    #名單轉換成list
    for row in rows:
        final_list.append(row.split(','))
    for word in final_list:
        try:#異常處理，有的人查詢不到
            print(word)
            browser.find_element_by_id('auscho_1_value1').clear()
            browser.find_element_by_id('auscho_1_value1').send_keys(word)
            browser.find_element_by_class_name('butt04').click()
            try:
                time.sleep(2)
                browser.find_element_by_id('iframeResult')
            except NoSuchElementException:
                print('no')
            browser.switch_to.parent_frame()#切換到父iframe
            x=browser.switch_to.frame('iframeResult')
            # browser.switch_to.frame('iframeResult')
            a = browser.find_element_by_class_name('s_table')
            tr_content = a.find_elements_by_tag_name("tr")  # 進一步定位到表格內容所在的tr節點
            for tr in tr_content:
                tds =tr.find_elements_by_tag_name("td")  # 進一步定位到表格內容所在的td節點
                for td in tds:

                    lst.append(td.text)
                with open('D:\Python_DATA\data.csv','a', encoding='utf-8',newline='') as csvfile:
                    writer = csv.writer(csvfile)
                    writer.writerow(lst)
                    lst.clear()
            # a=browser.find_element_by_class_name('s_tabletd_rb')
            browser.switch_to.default_content()
        except:
            print('查無此人')
        continue

if __name__ == '__main__':
    start_spider()
    #browser.close()
    print("爬取完成，請到相應文件夾查看！")

效果圖：