[python]爬取全國城市歷史天氣數據

python爬取全國城市歷史天氣數據

• 爬取全國城市2011至2020每天天氣數據
• 以requests+BeautifulSoup的方式抓取數據
• 多線程爬取
• 按城市名爬取後按省份存爲xls
• 從全國城市名稱對應拼音構造字典時，存在城市拼音相同問題
• 這個網站對城市的拼音有錯誤導致數據爬不到
• 構造省份城市字典，按照省份創建文件夾歸檔
• 數據來源：天氣後報網站
• 完整代碼及數據：項目地址

分析資源URL

http://www.tianqihoubao.com/lishi/beijing/month/201101.html

易見http://www.tianqihoubao.com/lishi/{城市拼音}/month/{年月}.html

主要代碼

class Crawler(threading.Thread,):

    def run(self):
        print("%s is running" % threading.current_thread())
        while True:
            # 上鎖
            gLock.acquire()
            if len(city_dict) == 0:
                # 釋放鎖
                gLock.release()
                continue
            else:
                item = city_dict.popitem()
                gLock.release()
                data_ = list()
                urls = self.get_urls(item[0])
                for url in urls:
                    try:
                        data_.extend(self.get_data(url))  # 列表合併，將某個城市所有月份的天氣信息寫到data_
                    except Exception as e:
                        print(e)
                        pass
                self.saveTocsv(data_, item[1])  # 保存爲csv
                if len(city_dict) == 0:
                    end = time.time()
                    print("消耗的時間爲：", (end - start))
                    exit()

    # 獲取城市歷史天氣url
    def get_urls(self,city_pinyin):
        urls = []
        for year in target_year_list:
            for month in target_month_list:
                date = year + month
                # url = "http://www.tianqihoubao.com/lishi/beijing/month/201812.html"
                urls.append("http://www.tianqihoubao.com/lishi/{}/month/{}.html".format(city_pinyin, date))
        return urls

    def get_soup(self,url):
        try:
            r = requests.get(url, timeout=30)
            r.raise_for_status()  # 若請求不成功,拋出HTTPError 異常
            soup = BeautifulSoup(r.text, "html.parser")
            return soup
        except Exception as e:
            print(e)
            pass

    # 將天氣數據保存至xls文件
    def saveTocsv(self,data, city):
        fileName = './weather_data/' + city + '天氣.xls'
        result_weather = pd.DataFrame(data, columns=['日期', '天氣狀況', '氣溫', '風力風向'])
        # print(result_weather)
        result_weather.to_excel(fileName, index=False)
        print('Save all weather success!')
        print('remain{}'.format(len(city_dict)))

    def get_data(self,url):
        print(url)
        try:
            soup = self.get_soup(url)
            all_weather = soup.find('div', class_="wdetail").find('table').find_all("tr")
            data = list()
            for tr in all_weather[1:]:
                td_li = tr.find_all("td")
                for td in td_li:
                    s = td.get_text()
                    data.append("".join(s.split()))
            res = np.array(data).reshape(-1, 4)
            return res

        except Exception as e:
            print(e)
            pass