一、需求
有一批专利号需要在智慧芽上找它对应的相似专利。
抓包看了下智慧芽登录接口，登录可以直接使用用户名、密码登录，没验证码限制少了很多麻烦。又看它使用token请求，开始想着获取到登录后的token再批量请求接口就好了，但尝试了下发现不行。因为请求次数多了就会有限制，再重新登录换新token都不行，所以改成了使用selenium模拟浏览器请求抓取数据。
二、思路
按照人手动操作，先登录，然后打开页面抓取数据，然后保存到excel中，由于数据比较多，代码运行过程中容易出现中断，所以抓取一部分就保存一部分到excel中，一直循环抓取。
三、输入输出格式
输入
输出
四、代码
代码中涉及到的知识点还是比较多的，所以贴了上来，方便后面遇到类似的直接拿来使用。
from selenium import webdriver import time import pandas import pandas as pd from openpyxl import load_workbook from selenium.common import NoSuchElementException from selenium.webdriver.common.by import By from collections import Counter from collections import OrderedDict INPUT_FILE='PNS_1.xlsx' OUT_FILE='PN_Out_1.xlsx' # 设置WebDriver driver = webdriver.Chrome() # 存放需要保存的数据 save_data=[] # 登录 def login(): # 打开Google首页 driver.get('https://account.zhihuiya.com/#/') time.sleep(5) current_url=driver.current_url print('当前页面url:',current_url) if 'account.zhihuiya.com' in current_url: tab_element = driver.find_element(By.ID, 'tab-EMAIL_PASSWORD') if tab_element.is_displayed(): tab_element.click() time.sleep(3) # 找到账号输入框，输入账号 account_element = driver.find_element(By.XPATH, '//*[@id="pane-EMAIL_PASSWORD"]/form/div[1]/div/input') account_element.click() account_element.clear() account_element.send_keys('[email protected]') pwd_element = driver.find_element(By.XPATH, '//*[@id="pane-EMAIL_PASSWORD"]/form/div[2]/div/div/input') pwd_element.send_keys('xxxxx') #登录按钮 btn_element = driver.find_element(By.XPATH, '//*[@id="pane-EMAIL_PASSWORD"]/form/div[4]/div/button') btn_element.click() time.sleep(5) try: # 定位元素 element = driver.find_element(By.XPATH, '/html/body/div[6]/div') # 判断元素是否可见 is_visible = element.is_displayed() print(f"元素是否可见: {is_visible}") if is_visible: agree_element = driver.find_element(By.XPATH, '/html/body/div[6]/div/div[2]/button[2]') agree_element.click() time.sleep(5) # 对元素进行操作 except NoSuchElementException: print("元素不存在") # 获取PN对应的patentId def find_patentId(pn): url='https://analytics.zhihuiya.com/search/result/tablelist/1?sort=pdesc&limit=100&q='+pn+'&_type=query&search_mode=publication' driver.get(url) time.sleep(2) a_element=driver.find_element(By.XPATH,'//*[@data-link-type="PN"][@data-link-data="'+pn+'"]') print(a_element) # 获取元素属性 patent_id = a_element.get_attribute("data-patent-id") print(patent_id) find_similar(pn, patent_id) # 根据pn和patentId找相似 def find_similar(pn,patent_Id): url='https://analytics.zhihuiya.com/patent-view/similar?_type=query&source_type=search_result&rows=100&patentId='+patent_Id+'&sort=pdesc&page=1&q='+pn print(url) driver.get(url) time.sleep(6) span_elements=driver.find_elements(By.XPATH,'//*[@class="PN"]/span[@class="snap-table__pn-row"]') print('相似专利:',span_elements) global save_data for ele in span_elements: print(pn,ele.text) ele_tup=(pn,ele.text) save_data.append(ele_tup) # 超过一百条写入excel一次 if len(save_data)>=100: write_diff_PNs() print('写入数据') # 获取已保存到Excel中的PN def find_out_PNs(sheetname,output_file): # 加载工作簿 workbook = load_workbook(output_file) sheet = workbook[sheetname] print(sheet) rows = [row[0].value for row in sheet.iter_rows(min_col=1, max_col=2, min_row=2)]; # print(rows) workbook.close() return rows # 获取所有需要获取相似专利的PN def find_all_PNs(sheetname,output_file): # 加载工作簿 workbook = load_workbook(output_file) sheet = workbook[sheetname] print(sheet) rows = [row[0].value for row in sheet.iter_rows(min_col=1, max_col=1, min_row=2)]; # print(rows) workbook.close() return rows # 获取所有和已获取PN的差集 def find_diff_PNs(): out_pns=find_out_PNs('PN',OUT_FILE) out_pns=list(set(out_pns)) print('已存pns个数:',len(out_pns)) all_pns=find_all_PNs('PN',INPUT_FILE) all_pns=list(set(all_pns)) print('所有pns个数:',len(all_pns)) diff_pns = [x for x in out_pns if x not in all_pns] print('多余pns:',diff_pns) diff_pns = [x for x in all_pns if x not in out_pns] print('差异pns:',diff_pns) return diff_pns # 找到待获取的PN并将其写入到Excel def write_diff_PNs(): # 合并原来旧数据并写入到excel中 global save_data save_data_df = pandas.DataFrame(save_data, columns=['Query', 'Similar']) df = pd.read_excel(OUT_FILE, sheet_name='PN') print('已存数据行数：',len(df)) df = pd.concat([df, save_data_df], ignore_index=True) # 合并数据 print('删除前数据数：',len(df)) # drop_duplicates 去除重复数据 df=df.drop_duplicates(subset=['Query','Similar'],keep="first",ignore_index=True) print('删除后数据数：',len(df)) with pd.ExcelWriter(OUT_FILE, engine='openpyxl', mode='a', if_sheet_exists="replace") as writer: df.to_excel(writer, sheet_name='PN', index=False) save_data=[] if __name__ == '__main__': # 记录开始时间 start_time = time.time() print(start_time) if len(find_diff_PNs())>0: login() time.sleep(10) while len(find_diff_PNs())>0: try: # 找到待获取的PN diff_pns = find_diff_PNs() print('差异pns数量',diff_pns) for pn in diff_pns: # 找到对应的patentId find_patentId(pn) time.sleep(1) time.sleep(10) print(save_data) write_diff_PNs() except Exception: print("打印异常") time.sleep(10) # 关闭浏览器 driver.quit() else: print('已无需要查找相似的专利') driver.quit() # 记录结束时间 end_time = time.time() # 计算耗时 elapsed_time = end_time - start_time print(f"耗时: {elapsed_time} 秒") # 统计列表元素重复次数 # data_list = find_out_PNs('PN',OUT_FILE) # counts = Counter(data_list) # unique_values = list(OrderedDict.fromkeys(data_list)) # for unique in unique_values: # value = counts.get(unique) # if value>20: # print(str(unique) + '\t' + str(value))