爬蟲項目6[爬取網易雲python課程數據]

登錄界面是在iframe表單中,要獲取iframe表單中的數據必須先進入iframe表單

模塊selenium
在定位網頁中的數據時,如果標籤是在iframe表單種,直接定位是找不到的,必須跳轉到iframe表單中才能整個程序中最重要的就是selenium中進入iframe標籤,代碼如下:

iframe_element = browser.find_element_by_xpath()
browser.switch_to_frame(iframe_element) 

from selenium import webdriver
import time
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
browser = webdriver.Chrome(chrome_options=chrome_options)
browser.get("https://study.163.com/")
browser.maximize_window()

#同意協議按鈕
agree_button = browser.find_element_by_xpath(".//span[@class='ux-btn th-bk-main ux-btn- ux-btn- ux-modal-btn um-modal-btn_ok th-bk-main']")
agree_button.click()

#關閉提示表單按鈕
form_close = browser.find_element_by_xpath(".//i[@class='ux-icon ux-icon-close']")
form_close.click()

#登錄按鈕
login_button = browser.find_element_by_xpath(".//a[@class='f-fr j-nav-loginBtn loginBtn']")
login_button.click()
time.sleep(5)

#iframe元素
iframe_element = browser.find_element_by_xpath(".//div[@class='ux-modal mn-login-dialog ux-modal-fadeIn']//div[@id='j-ursContainer-1']/iframe")

browser.switch_to_frame(iframe_element)  #這一步最重要,要進入到iframe元素中,否則下邊的都無法進行

tel = browser.find_element_by_xpath(".//input[@type='tel']")   #賬號輸入標籤
password = browser.find_element_by_xpath(".//input[@class='j-inputtext dlemail'][1]")  #密碼輸入標籤
submit = browser.find_element_by_xpath(".//a[@id='submitBtn'][1]")

tel.send_keys("13839817517")   #輸入賬號
password.send_keys("yhr104653")   #輸入密碼
submit.click()  #提交

#browser.switch_to.default_content()
#print(len(browser.page_source))
#print(browser.page_source)
browser.refresh()

#搜索輸入框
search = browser.find_element_by_xpath(".//div[@class='m-indextopwrap f-pr']//div[@class='box  j-search f-cb']/input")
#提交搜索框
go_search = browser.find_element_by_xpath(".//div[@class='m-indextopwrap f-pr']//div[@class='submit j-submit f-pa']/span[text()='搜索']")
#輸入要搜索的關鍵詞
search.send_keys("python")
go_search.click()
time.sleep(5)


if __name__ == "__main__":
    while 1:
        try:
            next = browser.find_element_by_xpath(".//a[@class='th-bk-disable-gh']")
        except:
            next = None
        lesson_names = browser.find_elements_by_xpath(".//ul[@class='uc-course-list_ul']/li//span[@class='uc-ykt-coursecard-wrap_tit_name']")
        for i in range(len(lesson_names)):
            print(lesson_names[i].text)
        if next:
            next.click()
            time.sleep(5)
        else:
            break