使用Python和selenium的Chromedriver模擬登陸爬取網站信息(beautifulsoup)

爬取的信息很多，所以需要設置斷點，在程序重啓時能繼續爬取。並且能在斷掉之後自動重啓。

1.setting.py

對爬取的常量進行設置

"""
基本信息設置
"""

base_url = "https://com"

login_page = "https://com/users/sign_in"

company_bsae_url = "https://com/companies/{}?branch=&commit=Go&inactive=false&mode=best_fields&nonprofit=&order=score&page={}&q=&search_fields%5B%5D=name&search_fields%5B%5D=previous_names&search_fields%5B%5D=company_number&utf8=%E2%9C%93"

max_page = 30

DATABASE = {
    "host": '127.0.0.1',  # 數據庫地址
    "port": 3306,  # 數據庫端口
    "db": 'opencorporates',  # 數據庫名
    "user": 'root',  # 數據庫用戶名
    "passwd": 'root',  # 數據庫密碼
    "charset": 'utf8',  # 編碼方式
}

# 登陸信息
login_email = "[email protected]"
login_password = "15184378218lk"

# 發送方郵箱
SENDEMAIL = 'com'
# 郵箱地址
EMAILPSD = ''
# 收件人郵箱
TOEMAIL = 'com'

# 縮寫 131個國家
country_list = ['us_la', 'us_sd', 'be', 'us_ok', 'je', 'ca_nl', 'us_nm', 'ca_qc', 'bg', 'us_fl', 'rw', 'dk', 'us_ut', 'us_sc', 'do', 'mt', 'us_mo',
                'us_ks', 'us_ms', 'si', 'is', 'th', 'bm', 'aw', 'us_nd', 'me', 'us_or', 'us_az', 'ro', 'ae_az', 'gi', 'mx', 'bo', 'us_ri', 'fr',
                'us_wv', 'au', 'ch', 'jm', 'mm', 'vu', 'bz', 'us_wy', 'us_nv', 'us_co', 'jp', 'us_tx', 'us_ne', 'ca_ns', 'us_ak', 'bs', 'al', 'gb',
                'sk', 'za', 'us_ga', 'ua', 'gl', 'gg', 'mu', 'us_de', 'us_tn', 'nl', 'ir', 'us_me', 'bh', 'de', 'ca', 'us_nc', 'pa', 'li', 'pr',
                'us_id', 'no', 'us_nh', 'us_ar', 'il', 'us_ca', 'hr', 'se', 'us_al', 'us_mi', 'us_ny', 'us_mt', 'us_oh', 'es', 'us_ct', 'cw', 'us_ia',
                'br', 'ca_pe', 'us_va', 'im', 'bb', 'us_ky', 'bd', 'us_vt', 'us_mn', 'to', 'md', 'tj', 'my', 'nz', 'vn', 'us_pa', 'hk', 'kh', 'sg',
                'lv', 'fi', 'us_md', 'gr', 'us_in', 'us_wi', 'cy', 'tn', 'tz', 'pk', 'in', 'pl', 'ca_nb', 'ae_du', 'us_ma', 'ug', 'us_dc', 'us_hi',
                'ie', 'lu', 'us_wa', 'by', 'us_nj']

# 縮寫與國家參照
country_dic = {'gl': 'Greenland', 'us_in': 'Indiana (US)', 'ie': 'Ireland', 'us_wi': 'Wisconsin (US)', 'jm': 'Jamaica', 'bg': 'Bulgaria',
               'tj': 'Tajikistan',
               'us_ok': 'Oklahoma (US)', 'im': 'Isle of Man', 'ug': 'Uganda', 'ro': 'Romania', 'ca_nb': 'New Brunswick (Canada)',
               'us_wv': 'West Virginia (US)',
               'in': 'India', 'mx': 'Mexico', 'pl': 'Poland', 'us_wy': 'Wyoming (US)', 'us_mo': 'Missouri (US)', 'us_pa': 'Pennsylvania (US)',
               'bb': 'Barbados',
               'us_az': 'Arizona (US)', 'us_dc': 'District of Columbia (US)', 'nl': 'Netherlands', 'gr': 'Greece', 'us_nm': 'New Mexico (US)',
               'aw': 'Aruba',
               'ir': 'Iran', 'fr': 'France', 'us_nc': 'North Carolina (US)', 'us_vt': 'Vermont (US)', 'us_ut': 'Utah (US)',
               'us_nh': 'New Hampshire (US)',
               'by': 'Belarus', 'ch': 'Switzerland', 'us_al': 'Alabama (US)', 'us_ar': 'Arkansas (US)', 'us_va': 'Virginia (US)', 'es': 'Spain',
               'mm': 'Myanmar', 'hk': 'Hong Kong', 'us_md': 'Maryland (US)', 'be': 'Belgium', 'lv': 'Latvia', 'ca_qc': 'Quebec (Canada)',
               'bz': 'Belize',
               'us_ny': 'New York (US)', 'us_sd': 'South Dakota (US)', 'us_mt': 'Montana (US)', 'us_oh': 'Ohio (US)', 'us_ma': 'Massachusetts (US)',
               'nz': 'New Zealand', 'br': 'Brazil', 'kh': 'Cambodia', 'pk': 'Pakistan', 'us_mn': 'Minnesota (US)', 'us_ct': 'Connecticut (US)',
               'us_wa': 'Washington (US)', 'us_me': 'Maine (US)', 'fi': 'Finland', 'gg': 'Guernsey', 'gi': 'Gibraltar', 'bd': 'Bangladesh',
               'us_ga': 'Georgia (US)', 'th': 'Thailand', 'do': 'Dominican Republic', 'li': 'Liechtenstein', 'us_ky': 'Kentucky (US)',
               'my': 'Malaysia',
               'je': 'Jersey', 'us_sc': 'South Carolina (US)', 'us_id': 'Idaho (US)', 'cw': 'Curaçao', 'se': 'Sweden', 'pa': 'Panama',
               'ca_nl': 'Newfoundland and Labrador (Ca...', 'de': 'Germany', 'vn': 'Viet Nam', 'us_nv': 'Nevada (US)', 'us_ne': 'Nebraska (US)',
               'ua': 'Ukraine', 'us_fl': 'Florida (US)', 'ca_pe': 'Prince Edward Island (Canada)', 'mt': 'Malta', 'us_hi': 'Hawaii (US)',
               'vu': 'Vanuatu',
               'si': 'Slovenia', 'rw': 'Rwanda', 'sg': 'Singapore', 'us_ks': 'Kansas (US)', 'me': 'Montenegro', 'is': 'Iceland', 'bh': 'Bahrain',
               'au': 'Australia', 'us_ms': 'Mississippi (US)', 'us_ak': 'Alaska (US)', 'us_co': 'Colorado (US)', 'pr': 'Puerto Rico', 'md': 'Moldova',
               'us_mi': 'Michigan (US)', 'to': 'Tonga', 'bo': 'Bolivia', 'za': 'South Africa', 'ca_ns': 'Nova Scotia (Canada)', 'il': 'Israel',
               'us_or': 'Oregon (US)', 'tz': 'Tanzania', 'us_tn': 'Tennessee (US)', 'ae_az': 'Abu Dhabi (UAE)', 'lu': 'Luxembourg', 'tn': 'Tunisia',
               'us_ca': 'California (US)', 'us_la': 'Louisiana (US)', 'al': 'Albania', 'us_ia': 'Iowa (US)', 'gb': 'United Kingdom',
               'mu': 'Mauritius',
               'hr': 'Croatia', 'dk': 'Denmark', 'sk': 'Slovakia', 'cy': 'Cyprus', 'ae_du': 'Dubai (UAE)', 'us_tx': 'Texas (US)',
               'us_de': 'Delaware (US)',
               'us_nd': 'North Dakota (US)', 'no': 'Norway', 'bm': 'Bermuda', 'jp': 'Japan', 'us_ri': 'Rhode Island (US)', 'bs': 'Bahamas',
               'ca': 'Canada',
               'us_nj': 'New Jersey (US)'}

2.allcountry.py

主要爬蟲文件，利用瀏覽器驅動模擬登錄網站，然後對網站信息進行爬取，如果爬取出錯，繼續下一個頁面的爬取

# -*- coding: utf-8 -*-
import time
import datetime
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import os
from setting import *
from db import db_insert, db_update
from tools import format_str, send_email, dic_change_str, query_company_is_exist, find_breakpoint, save_breakpoint


def main():
    chrome_profile = webdriver.ChromeOptions()
    # chrome_profile.add_argument("--disable-extensions")
    # chrome_profile.add_argument('--headless')

    driver = webdriver.Chrome(executable_path=r"D:\chromedriver\chromedriver.exe", chrome_options=chrome_profile)
    wait = WebDriverWait(driver, 60)
    driver.get(login_page)

    txtemail = wait.until(
        EC.presence_of_element_located((By.ID, 'user_email'))
    )
    txtpassword = wait.until(
        EC.presence_of_element_located((By.ID, 'user_password'))
    )
    btnsearch = wait.until(
        EC.presence_of_element_located((By.NAME, 'submit'))
    )

    time.sleep(3)
    txtemail.clear()
    txtemail.send_keys(login_email)
    txtpassword.clear()
    txtpassword.send_keys(login_password)
    btnsearch.click()

    time.sleep(3)

    country, page_number = find_breakpoint()


    # 異常處理
    try:
        for country in new_country_list:
            # 繼續上一次位置
            for page_number1 in range(page_number, max_page + 1):
                # 頁碼更新，存斷點
                print(page_number1)
                save_breakpoint(country, page_number1)

                data_page = company_bsae_url.format(country, page_number1)
                print(data_page)
                jurisdiction = country_dic[country]

                driver.get(data_page)
                html_page = driver.page_source.encode('utf-8')  # 取得網頁的源代碼
                html_page_soup = BeautifulSoup(html_page, 'lxml')
                company_list = html_page_soup.select('li.search-result a.company_search_result')
                print(company_list)
                if len(company_list) > 0:
                    for i in company_list:

                        company_url = base_url + i.get('href')

                        company_name = i.get_text()

                        driver.get(company_url)
                        company_html = driver.page_source.encode('utf-8')
                        bs_company_html = BeautifulSoup(company_html, 'lxml')
                        company_dt_list = bs_company_html.select('div#attributes dl.attributes')
                        if company_dt_list:
                            company_dt = company_dt_list[0]

                            # 先判斷是否存在
                            company_number = ""
                            bs_company_number = company_dt.select('dd.company_number')
                            if bs_company_number:
                                company_number = bs_company_number[0].get_text()

                                # 數據庫中存在就繼續下一個循環
                                # if company_number in company_number_list:
                                #     continue
                            else:
                                print(company_url + "中公司編號不存在")

                            status = ""
                            bs_status = company_dt.select('dd.status')
                            if bs_status:
                                status = bs_status[0].get_text()

                            incorporation_date = ""
                            bs_incorporation_date = company_dt.select('span[itemprop=foundingDate]')
                            if bs_incorporation_date:
                                incorporation_date = bs_incorporation_date[0].get_text()
                            else:
                                incorporation_date1_dl = bs_company_html.select('div#events div.oc-events-timeline dl')
                                if incorporation_date1_dl:
                                    for incorporation_date1 in incorporation_date1_dl:
                                        incorporation_date_text = incorporation_date1.get_text()
                                        # print(incorporation_date_text)
                                        if "Incorporated" in incorporation_date_text:
                                            incorporation_date = incorporation_date_text.split('Incorporated')[0].strip()
                                            # print(incorporation_date)

                            company_type = ""
                            bs_company_type = company_dt.select('dd.company_type')
                            if bs_company_type:
                                company_type = bs_company_type[0].get_text()

                            business_number = ""
                            bs_business_number = company_dt.select('dd.business_number ')
                            if bs_business_number:
                                business_number = bs_business_number[0].get_text()

                            agent_name = ""
                            bs_agent_name = company_dt.select('dd.agent_name')
                            if bs_agent_name:
                                agent_name = bs_agent_name[0].get_text()

                            agent_address = ""
                            bs_agent_address = company_dt.select('dd.agent_address')
                            if bs_agent_address:
                                agent_address = bs_agent_address[0].get_text()

                            directors_officers = ""
                            bs_directors_officers = company_dt.select('dd.officers')
                            if bs_directors_officers:
                                directors_officers_li = bs_directors_officers[0].select('ul.attribute_list li.attribute_item')
                                directors_officers_dic = {}
                                for directors_officers in directors_officers_li:
                                    directors_officers_name = ""
                                    directors_officers_href = ""
                                    directors_officers_position = ""
                                    directors_officers_date = ""

                                    directors_officers_href_name = directors_officers.select('a.officer')
                                    if directors_officers_href_name:
                                        directors_officers_href = directors_officers_href_name[0].get('href')
                                        directors_officers_href = base_url + directors_officers_href
                                        directors_officers_name = directors_officers_href_name[0].get_text()
                                    else:
                                        continue
                                    directors_officers_text = directors_officers.get_text()
                                    directors_officers_text_list = directors_officers_text.split(',')
                                    if ',' not in directors_officers_name:
                                        if len(directors_officers_text_list) > 2:
                                            directors_officers_position = directors_officers_text_list[1]
                                            directors_officers_date = directors_officers_text_list[2]
                                    # for d directors_officers_text_list:

                                    # directors_officers_position_font = directors_officers.select('font')
                                    # if len(directors_officers_position_font) > 1:
                                    #     directors_officers_position = directors_officers_position_font[0].get_text().rstirp(',').lstirp(',')
                                    #     directors_officers_date = directors_officers.get_text().split(',')[-1]

                                    directors_officers_name_dic = {"directors_officers_position": directors_officers_position,
                                                                   "directors_officers_href": directors_officers_href,
                                                                   "directors_officers_date": directors_officers_date}
                                    directors_officers_dic[directors_officers_name] = directors_officers_name_dic
                                print(directors_officers_dic)
                            else:
                                directors_officers_dic = {}
                            registry_page = ""
                            bs_registry_page = company_dt.select('dd.registry_page a.url')
                            if bs_registry_page:
                                registry_page = bs_registry_page[0].get('href')

                            branch = ""
                            bs_branch = company_dt.select('dd.branch')
                            if bs_branch:
                                branch = bs_branch[0].get_text()

                            industry_codes = ""
                            bs_industry_codes = company_dt.select('dd.industry_codes')
                            if bs_industry_codes:
                                industry_codes = bs_industry_codes[0].get_text()

                            ultimate_beneficial_owners = ""
                            bs_ultimate_beneficial_owners = company_dt.select('dd.ultimate_beneficial_owners')
                            if bs_ultimate_beneficial_owners:
                                ultimate_beneficial_owners = bs_ultimate_beneficial_owners[0].get_text()

                            latest_accounts_date = ""
                            bs_latest_accounts_date = company_dt.select('dd.latest_accounts_date')
                            if bs_latest_accounts_date:
                                latest_accounts_date = bs_latest_accounts_date[0].get_text()

                            registered_address = ""
                            bs_registered_address = company_dt.select('dd.registered_address')
                            if bs_registered_address:
                                # registered_address = bs_registered_address[0].get_text()
                                bs_registered_address_li = bs_registered_address[0].select('ul.address_lines li.address_line')
                                if bs_registered_address_li:
                                    for registered_address_li in bs_registered_address_li:
                                        registered_address_li_str = str(registered_address_li)
                                        registered_address = registered_address + registered_address_li_str.replace('<br/>', ' ').replace(
                                            '<li class="address_line">', ' ').replace('</li>', ' ')

                            alternative_names = ""
                            bs_alternative_names = company_dt.select('dd.alternative_names')
                            if bs_alternative_names:
                                alternative_names = bs_alternative_names[0].get_text()

                            print(company_name, company_number, status, incorporation_date, company_type, jurisdiction)
                        else:
                            print("沒有抓取到該公司數據，url=" + company_url)

                        source_mechanism = ""
                        bs_source_mechanism = bs_company_html.select('div#source span.publisher')
                        if bs_source_mechanism:
                            source_mechanism = bs_source_mechanism[0].get_text()

                        source_page = ""
                        bs_source_page = bs_company_html.select('div#source a.url')
                        if bs_source_page:
                            source_page = bs_source_page[0].get('href')

                        source_time = ""
                        bs_source_time = bs_company_html.select('div#source span.retrieved')
                        if bs_source_time:
                            source_time = bs_source_time[0].get_text()

                        update_time = datetime.datetime.now()
                        # print(directors_officers_dic)
                        company_is_exist = query_company_is_exist(company_number, jurisdiction)

                        # 存在該公司就更新，不存在就插入
                        if company_is_exist:
                            print("數據庫中存在，更新"+company_number)
                            db_update(format_str(company_name), format_str(status), format_str(incorporation_date), format_str(company_type),
                                      format_str(business_number), format_str(agent_name), format_str(agent_address), format_str(alternative_names),
                                      format_str(registered_address), format_str(registry_page), format_str(dic_change_str(directors_officers_dic)),
                                      format_str(branch), format_str(ultimate_beneficial_owners), format_str(industry_codes),
                                      format_str(latest_accounts_date), format_str(source_mechanism), format_str(source_page),
                                      format_str(source_time), format_str(company_url), update_time, page_number1, format_str(company_number),
                                      format_str(jurisdiction))
                        else:
                            print("數據庫中不存在，插入" + company_number)
                            db_insert(format_str(company_name), format_str(company_number), format_str(status), format_str(incorporation_date),
                                      format_str(company_type), format_str(jurisdiction), format_str(business_number), format_str(agent_name),
                                      format_str(agent_address), format_str(alternative_names), format_str(registered_address),
                                      format_str(registry_page),
                                      format_str(dic_change_str(directors_officers_dic)), format_str(branch),
                                      format_str(ultimate_beneficial_owners), format_str(industry_codes), format_str(latest_accounts_date),
                                      format_str(source_mechanism), format_str(source_page), format_str(source_time),
                                      format_str(company_url), update_time, page_number1)

                    page_number = 1
                else:
                    print("訪問公司頁碼出錯，出錯url" + data_page + "   所屬國家：" + country_dic[country])
                time.sleep(5)
    # 捕獲所有異常
    except Exception as e:
        # title = "爬取出現異常,時間:" + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S %A')
        print(e)
        # send_email(title, "異常")
        driver.quit()
        time.sleep(20)
        os.system('python run.py')
        # finally:
        # print('爬取結束')
        # send_email("爬取結束", "爬取結束")

3.tools.py

通用函數文件，工具類

"""
功能函數
"""
from db import select_latest, select_all_by_company_number_jurisdiction
from setting import SENDEMAIL, EMAILPSD, TOEMAIL
import smtplib
from email.mime.text import MIMEText

import json


# 轉換特殊字符
def format_str(str1):
    str2 = str1.replace("'", "\\'")
    str3 = str2.replace('"', '\\"')
    return str3


# 查詢最近爬取數據的頁碼和國家
def query_page_and_country():
    ret = select_latest()
    page_number = ret[20]
    jurisdiction = ret[6]
    print(page_number, jurisdiction)
    return page_number, jurisdiction


# 發郵件
def send_email(title, articletext):
    msg_from = SENDEMAIL  # 發送方郵箱
    passwd = EMAILPSD  # 密碼
    msg_to = TOEMAIL  # 收件人郵箱

    subject = title  # 主題
    content = articletext  # 正文
    msg = MIMEText(content)
    msg['Subject'] = subject
    msg['From'] = msg_from
    msg['To'] = msg_to
    try:
        s = smtplib.SMTP_SSL("smtp.163.com", 465)  # 郵件服務器及端口號
        s.login(msg_from, passwd)
        s.sendmail(msg_from, msg_to, msg.as_string())
        print("郵件發送成功")
    except Exception as e:
        print(e)
    finally:
        s.quit()


# 將字典轉換爲字符串 利用json
def dic_change_str(dic1):
    str1 = json.dumps(dic1)
    return str1


# 查詢數據庫中是否存在該公司
def query_company_is_exist(company_number, jurisdiction):
    ret = select_all_by_company_number_jurisdiction(company_number, jurisdiction)
    if ret:
        return True
    else:
        return False


# if query_company_is_exist('43364123K', 'Louisiana (US)'):
#     print(1)


# 找到斷點，返回國家和頁碼
def find_breakpoint():
    with open('breakpoint.txt', "r") as f:
        str1 = f.read()
    country = str1.split(',')[0]
    page_number = int(str1.split(',')[1])
    return country, page_number


# 存斷點,傳入國家和頁碼
def save_breakpoint(country, page_number):
    str1 = country + ',' + str(page_number)
    with open('breakpoint.txt', 'w') as f:
        f.write(str1)


        # print(find_breakpoint())
        # save_breakpoint('china',11)

4.db.py

數據庫操作文件

"""
數據庫操作函數
"""
import pymysql
from setting import DATABASE


# 插入數據
def db_insert(company_name, company_number, status, incorporation_date, company_type, jurisdiction, business_number, agent_name, agent_address,
              alternative_names, registered_address, registry_page, directors_officers, branch, ultimate_beneficial_owners, industry_codes,
              latest_accounts_date, source_mechanism, source_page, source_time, company_url, update_time, page_number):
    conn = pymysql.connect(host=DATABASE['host'],
                           port=DATABASE['port'],
                           db=DATABASE['db'],
                           user=DATABASE['user'],
                           passwd=DATABASE['passwd'],
                           charset=DATABASE['charset'], )
    cur = conn.cursor()
    sql = "INSERT INTO company(company_name,company_number,status,incorporation_date,company_type,jurisdiction,business_number,agent_name,agent_address,alternative_names,registered_address,registry_page,directors_officers,branch,ultimate_beneficial_owners,industry_codes,latest_accounts_date,source_mechanism,source_page,source_time,company_url,update_time,page_number) VALUES ('{}', '{}','{}', '{}', '{}', '{}','{}','{}', '{}', '{}', '{}','{}','{}', '{}', '{}', '{}','{}','{}', '{}', '{}', '{}','{}','{}')".format(
        company_name, company_number, status, incorporation_date, company_type, jurisdiction, business_number, agent_name, agent_address,
        alternative_names, registered_address, registry_page, directors_officers, branch, ultimate_beneficial_owners, industry_codes,
        latest_accounts_date, source_mechanism, source_page, source_time, company_url, update_time, page_number)
    # print(sql)
    cur.execute(sql.encode('utf-8'))
    conn.commit()
    conn.close()


# 更新數據
def db_update(company_name, company_number, status, incorporation_date, company_type, jurisdiction, business_number, agent_name, agent_address,
              alternative_names, registered_address, registry_page, directors_officers, branch, ultimate_beneficial_owners, industry_codes,
              latest_accounts_date, source_mechanism, source_page, source_time, company_url, update_time, page_number):
    conn = pymysql.connect(host=DATABASE['host'],
                           port=DATABASE['port'],
                           db=DATABASE['db'],
                           user=DATABASE['user'],
                           passwd=DATABASE['passwd'],
                           charset=DATABASE['charset'], )
    cur = conn.cursor()
    sql = "UPDATE company SET company_name='{}',status='{}',incorporation_date='{}',company_type='{}',business_number='{}',agent_name='{}',agent_address='{}',alternative_names='{}',registered_address='{}',registry_page='{}',directors_officers='{}',branch='{}',ultimate_beneficial_owners='{}',industry_codes='{}',latest_accounts_date='{}',source_mechanism='{}',source_page='{}',source_time='{}',company_url='{}',update_time='{}',page_number='{}' WHERE company_number='{}' AND jurisdiction='{}'".format(
        company_name, status, incorporation_date, company_type, business_number, agent_name, agent_address,
        alternative_names, registered_address, registry_page, directors_officers, branch, ultimate_beneficial_owners, industry_codes,
        latest_accounts_date, source_mechanism, source_page, source_time, company_url, update_time, page_number, company_number, jurisdiction)
    cur.execute(sql.encode('utf-8'))
    conn.commit()
    conn.close()


# 查詢最新一條記錄
def select_latest():
    conn = pymysql.connect(host=DATABASE['host'],
                           port=DATABASE['port'],
                           db=DATABASE['db'],
                           user=DATABASE['user'],
                           passwd=DATABASE['passwd'],
                           charset=DATABASE['charset'], )
    cur = conn.cursor()
    sql = "SELECT * FROM company ORDER BY id DESC LIMIT 1"
    cur.execute(sql.encode('utf-8'))
    ret = cur.fetchone()
    cur.close()
    conn.close()
    print(ret)
    return ret


# 根據頁碼查詢該頁碼的公司編號
def select_all_by_page_number(page_number):
    conn = pymysql.connect(host=DATABASE['host'],
                           port=DATABASE['port'],
                           db=DATABASE['db'],
                           user=DATABASE['user'],
                           passwd=DATABASE['passwd'],
                           charset=DATABASE['charset'], )
    cur = conn.cursor()
    sql = "SELECT company_number FROM company WHERE page_number={}".format(page_number)
    print(sql)
    cur.execute(sql.encode('utf-8'))
    results = cur.fetchall()
    company_number_list = []
    for row in results:
        company_number_list.append(row[0])
    return company_number_list


# 通過國家和公司編號查詢公司信息
def select_all_by_company_number_jurisdiction(company_number, jurisdiction):
    conn = pymysql.connect(host=DATABASE['host'],
                           port=DATABASE['port'],
                           db=DATABASE['db'],
                           user=DATABASE['user'],
                           passwd=DATABASE['passwd'],
                           charset=DATABASE['charset'], )
    cur = conn.cursor()
    sql = "SELECT * FROM company WHERE company_number='{}' AND  jurisdiction='{}'".format(company_number, jurisdiction)
    cur.execute(sql.encode('utf-8'))
    ret = cur.fetchone()
    cur.close()
    conn.close()
    return ret

5.breakpoint.txt

斷點記錄文件，程序重啓後繼續斷點運行

ca,21

6.run.py

啓動文件

from allcountry import main

if __name__ == '__main__':
    main()