目標網站:news.bitauto.com/
由於推薦頁的加載更多不方便操作
選擇單項頁面爬取 例如新車頁
在頁面右鍵選擇 檢查
找到目標位置
/html/body/div[3]/div/div[1]/div[3]/div/div/h2/a (推薦使用Xpath helper 可以直接複製Xpath)
#coding: utf8
from selenium import webdriver
f=open("url6.txt","w",encoding="utf-8")
fw = open("news.txt", "w", encoding="utf-8")
driver = webdriver.Chrome('C:\Program Files (x86)\Google\Chrome\Application\chromedriver')
def geturl(url,k):
driver.get(url)
urls = driver.find_elements_by_xpath('//div[@class="article-card horizon"]//a') #目標url存在於多個位置 可以選一個方便找到的
url_list=[]
for url in urls:
u = url.get_attribute('href')
if u == 'None':
continue
else:
url_list.append(str(url.get_attribute("href")))
url_list=list(set(url_list))
#print(url_list)
for new_url in url_list:
if(len(new_url)<2):
continue
if(new_url[-1]=='l'):
print(new_url)
f.write(new_url+"\n")
#
if __name__ == '__main__':
#url= 'http://news.bitauto.com/xinche/'
a_list=[("xinche",4786)]
for t,am in a_list:
url = "http://news.bitauto.com/" + t + "/?pageindex="
k=len(t)
for i in range(1, am):
new_url = url + str(i)
print(t," page:", i)
geturl(new_url,k)
f.close()
driver.close()