利用百度搜索尋找目標網站

需求：需要尋找一批遊戲或者教育網站地址
思路：1.先百度關鍵字得到一批結果；2.然後訪問百度到的結果，根據網頁內容關鍵字判斷 是否是目標站

# -*- coding: utf-8 -*-
# @Time    : 2018/8/7 13:02
# @Author  : xiangchaoming
# @QQ      : 239036082

# 引入瀏覽器驅動
from selenium import webdriver
# 引入Keys類包 發起鍵盤操作

from lxml import html


def keys():
    f = open("keys.txt", "r", encoding="utf8")
    fr = f.read()
    # str轉dict
    return eval(fr)


def MainFunction():

    temp = []
    fw = open("urls.txt", "a")

    brower = webdriver.Firefox()

    brower.set_page_load_timeout(10)
    brower.set_script_timeout(10)  # 這兩種設置都進行纔有效

    brower.delete_all_cookies()

    for p in range(keys()["pages"]):
        try:
            # 訪問百度
            ku = "https://www.baidu.com/s?wd=" + keys()["key1"] + "&pn=" + str(10 * p)
            brower.get(ku)

            # 先找到百度到的一級連接
            ps = brower.page_source
            urls = html.etree.HTML(ps).xpath("//h3[@class='t']/a/@href")

            for url in urls:
                #print("一級域名："+url)
                try:
                    # 打開黃站，獲取網站所有鏈接
                    brower.get(url)
                    webdata2 = brower.page_source
                    page2 = html.etree.HTML(webdata2)
                    # 將獲得的鏈接作處理
                    a = brower.current_url.split("/")
                    su = a[0] + "//" + a[2]
                    if su not in temp:
                        # print("二級域名：" + su)
                        temp.append(su)
                        try:
                            # 打開黃站中的鏈接，然後根據網頁中的關鍵詞判斷是否是賭博站
                            brower.get(su)
                            webdata3 = brower.page_source
                            for key in keys()["key2"]:
                                if webdata3.__contains__(key):
                                    print("關鍵域名：" + su)
                                    fw.write(su)
                                    fw.write("\n")
                                    fw.flush()
                                    break
                                else:
                                    continue
                        except Exception as e:
                            print("*" * 50)
                            print(e)
                            pass
                    hrefs2 = page2.xpath(r"//a/@href")
                    for href2 in hrefs2:
                        if href2.startswith("http"):
                            # 將獲得的鏈接作處理
                            a = href2.split("/")
                            su = a[0] + "//" + a[2]
                            if su not in temp:
                                #print("二級域名：" + su)
                                temp.append(su)
                                try:
                                    # 打開黃站中的鏈接，然後根據網頁中的關鍵詞判斷是否是賭博站
                                    brower.get(su)
                                    webdata3 = brower.page_source
                                    for key in keys()["key2"]:
                                        if webdata3.__contains__(key):
                                            print("關鍵域名：" + su)
                                            fw.write(su)
                                            fw.write("\n")
                                            fw.flush()
                                            break
                                        else:
                                            continue
                                except Exception as e:
                                    print("*"*50)
                                    print(e)
                                    pass
                except Exception as e:
                    print("#" * 50)
                    print(e)
                    pass
        except Exception as e:
            print("$" * 50)
            print(e)
            pass
        fw.close()
MainFunction()

keys.txt ：key1百度關鍵字，key2爲網頁內容關鍵字，pages表示爬取百度結果的前多少頁

{
"key1": "炸金花",
"key2": ["捕魚遊戲", "新葡京", "棋牌"],
"pages": 100
}