請注意本文采用的可視化工具爲：pyecharts 0.1.9.4
爬取全國367個城市2019年春節期間的空氣質量指數數據（六萬條數據全面解析，城市春節禁放煙花爆竹真的有用嗎？）
首先是數據獲取，貼出爬蟲demo，如果有需要數據的請評論
# -*- coding:utf-8 -*-
import time
import requests
import pandas as pd
from lxml import etree
class AQI(object):
"""
爬取城市AQI實時數據
"""
def __init__(self):
"""
初始化函數
:attr encoding: 編碼
"""
self.encoding = None
def get_encoding(self):
"""
獲取網頁的編碼
:return: None
"""
res = requests.get('http://datacenter.mee.gov.cn/aqiweb2/')
self.encoding = res.apparent_encoding
print('Successfully crawled encoding!')
time.sleep(2)
def crawl_aqi(self, sleep_time=3600):
"""
爬取全國具有監測點的所有城市的AQI實時數據，每小時爬取一次
:param sleep_time: 爬取間隔時間，默認3600秒
:return: None
"""
write_header = True
while 1:
res = requests.get('http://datacenter.mee.gov.cn/aqiweb2/')
parsed_text = etree.HTML(res.text)
timestamp = parsed_text.xpath('/html/body/div[3]/p/i/text()')[0].replace('年', '-'). \
replace('月', '-').replace('日', ' ').replace('時', ':00:00')
print('Successfully crawled timestamp!')
# 直接使用pandas獲取和解析數據
data_res = pd.read_html('http://datacenter.mee.gov.cn/aqiweb2/', encoding=self.encoding)
data = data_res[0]
header = ['city', 'AQI', 'PM2.5', 'PM10', 'SO2', 'NO2', 'CO', 'O3', 'main_pollution']
data.columns = header
data['time'] = timestamp
if write_header is True:
data.to_csv('data2.csv', index=False, mode='a', header=True)
write_header = False
else:
data.to_csv('data2.csv', index=False, mode='a', header=False)
print('Successfully crawled data of {} and saved it to file!'.format(timestamp))
time.sleep(sleep_time)
if __name__ == '__main__':
aqi = AQI()
aqi.get_encoding()
aqi.crawl_aqi()
清洗數據
import pandas as pd
import numpy as np
data = pd.read_csv('AQI_Data.csv')
# 打印基本信息
print(data.info())
print(data.head())
# 數據清洗
data['time'] = pd.to_datetime(data['time'])
data = data[data['time'] <= pd.to_datetime('2019-02-11 23:59:59')] # 選取2月4日——2月12日的數據
data = data.replace('—', np.nan) # 打印全部數據會發現：原始數據中有很多'-'
for col in ['AQI', 'PM2.5', 'PM10', 'SO2', 'NO2', 'CO', 'O3']:
data[col] = data[col].astype(float)
print(data.info()) # 確認屬性信息
print(len(data['city'].unique())) # 查看城市數量
data.to_csv('data_clean.csv', index=False) # 可以保存清洗好的數據
正式開始分析之旅
共有68155條數據，時間爲2月4日到2月11日，每隔1小時收集一次，囊括全國367個城市
包含字段：城市、空氣質量指數、PM2.5指數、PM10指數、SO2指數、NO2指數、CO指數、O3指數、主要污染源、時間
請注意：2019年2月4日是除夕夜。
問題：
1 燃放煙花爆竹真的對空氣質量有影響嗎？
2 煙花爆竹對空氣質量的影響體現在哪些指標上？
3 煙花爆竹對空氣質量的污染程度有多大？
4 哪些區域污染最嚴重(輕)？
5 哪些城市屬於一秒破功型？
6 除夕中午到初一中午的超標城市個數
7 除夕到初七的超標城市個數
1、燃放煙花爆竹真的對空氣質量有影響嗎？
# 參考空氣質量指數，對所有城市取均值
AQI_total_mean = data[data['time'] <= pd.to_datetime('2019-02-06 23:59:59')].groupby(['time'])['AQI'].mean()
# 畫出整體趨勢
line = Line(title="全國春節期間空氣質量指數總體趨勢", subtitle="2019年除夕到初二", width=800)
line.add("", AQI_total_mean.index, np.round(AQI_total_mean.values, 0),mark_point=['max'])
line.render() # 生成本地的html文件
# 畫出具體城市變化
def city_AQI(data, cities):
line = Line("春節期間空氣質量指數", "2019年除夕到初二", width=800)
for city in cities:
city_aqi = data[(data['city'] == city) & (data['time'] <= pd.to_datetime('2019-02-06 23:59:59'))]
line.add(city, city_aqi.index, np.round(city_aqi['AQI'].values, 0), is_smooth=True, mark_point=["max"])
line.render()
return line
city_AQI(data=data, cities=['北京市', '天津市'])
city_AQI(data=data, cities=['長沙市', '南寧市'])
2、煙花爆竹對空氣質量的影響體現在哪些指標上？
data_total_idx = data[data['time'] <= pd.to_datetime('2019-02-06 23:59:59')].groupby(['time']).mean()
print(data_total_idx.columns) # 查看各種指標
line = Line("全國春節期間空氣質量各指標（2019年除夕到初二）", "單位:μg/m3(CO爲mg/m3)", width=800)
for idx in data_total_idx.columns[1:]:
line.add(idx, data_total_idx.index, np.round(data_total_idx[idx].values, 0), is_smooth=True,
legend_text_size=18, xaxis_label_textsize=14, yaxis_label_textsize=18, yaxis_min=8, legend_top=30)
line.render('各種指標.html')
tianjin_idx = data[(data['city']=='天津市')&(data['time']<=pd.to_datetime('2019-02-06 23:59:59'))]
line = Line("天津市春節期間空氣質量各指標（2019年除夕到初二）", "單位:μg/m3(CO爲mg/m3)", width=800)
for idx in tianjin_idx.columns[2:8]:
line.add(idx, tianjin_idx.index, np.round(tianjin_idx[idx].values, 0), is_smooth=True,
legend_text_size=18, xaxis_label_textsize=14, yaxis_label_textsize=18,
yaxis_min=8, legend_top=30)
line.render('天津.html')
我們可以得到結論：煙花爆竹對PM2.5和PM10影響最大。
main_pol = data.loc[data['time']==pd.to_datetime('2019-02-05 02:00:00'), 'main_pollution'].value_counts()
print(main_pol)
from pyecharts import Bar
bar = Bar("全國城市首要污染物", "2019年2月5日凌晨2點", width=600)
bar.add("", main_pol.index, main_pol.values, is_stack=True, is_label_show=True,
bar_category_gap='40%', label_color=['#130f40'], label_text_size=18,
legend_text_size=18, xaxis_label_textsize=18, yaxis_label_textsize=18)
bar.render('main.html')
3、煙花爆竹對空氣質量的污染程度有多大？
data2 = data[data['time'] <= pd.to_datetime('2019-02-05 23:59:59')]
data_AQI_min = data2.groupby('city')['AQI'].min()
data_AQI_max = data2.groupby('city')['AQI'].max()
data_AQI_times = np.round(data_AQI_max/data_AQI_min, 1) # 用最大值/最小值 作爲污染增長倍數
data_AQI_times_top10 = data_AQI_times.nlargest(10)
print(data_AQI_times_top10)
bar = Bar("全國除夕和春節期間空氣質量最高最低比Top10城市", "時間：2019年除夕至初一", width=600)
bar.add("", data_AQI_times_top10.index, data_AQI_times_top10.values, is_stack=True, is_label_show=True)
bar.render()
def city_AQI(data, cities, name):
line = Line("春節期間空氣質量指數", "2019年除夕到初二", width=800)
for city in cities:
city_aqi = data[(data['city'] == city) & (data['time'] <= pd.to_datetime('2019-02-06 23:59:59'))]
line.add(city, city_aqi.index, np.round(city_aqi['AQI'].values, 0), is_smooth=True, mark_point=["max"])
line.render(name)
return line
# 分別看一下top10城市
city_AQI(data=data, cities=['伊春市', '雞西市', '大興安嶺地區'], name='3.html')
city_AQI(data=data, cities=['盤錦市', '葫蘆島市', '瓦房店市', '錦州市'], name='4.html')
data_AQI_times_counts = data_AQI_times.value_counts(bins=[1, 5, 10, 15, 20, 25]) # 注意這個方法
print(data_AQI_times_counts)
bar = Bar("全國除夕和春節期間空氣質量最高最低比的城市數量", "時間：2019年除夕至初一", width=600)
times = ['1-5倍', '5-10倍', '10-15倍', '15-20倍', '20-25倍']
bar.add("", times, data_AQI_times_counts.values, is_stack=True,
is_label_show=True, bar_category_gap='40%', label_color=['#130f40'], label_text_size=18,
legend_text_size=18, xaxis_label_textsize=18, yaxis_label_textsize=18)
bar.render()
4、哪些區域污染最嚴重(輕)？
start_time = pd.to_datetime('2019-02-04 20:00:00')
end_time = pd.to_datetime('2019-02-05 04:00:00')
AQI_by_city = data[(data['time'] >= start_time) & (data['time'] <= end_time)].groupby('city')['AQI'].mean()
# 可視化，但是好像不好使
from pyecharts import Geo
geo = Geo(
"全國各城市空氣質量指數",
"除夕晚上20點至初一凌晨4點平均值",
title_color="#fff",
title_pos="center",
width=800,
height=500,
background_color="#404a59",
)
attr, value = AQI_by_city.index, AQI_by_city.values
geo.add(
"",
attr,
value,
visual_range=[0, 500],
visual_text_color="#fff",
symbol_size=10,
is_visualmap=True,
is_piecewise=True,
visual_split_number=10)
geo.render()
def city_AQI(data, cities, name):
line = Line("春節期間空氣質量指數", "2019年除夕到初二", width=800)
for city in cities:
city_aqi = data[(data['city'] == city) & (data['time'] <= pd.to_datetime('2019-02-06 23:59:59'))]
line.add(city, city_aqi.index, np.round(city_aqi['AQI'].values, 0), is_smooth=True, mark_point=["max"])
line.render(name)
return line
print('污染最重top10：')
print(AQI_by_city.nlargest(10))
city_AQI(data=data, cities=['呼和浩特市', '包頭市', '咸陽市'], name='lagest.html')
print('污染最輕top10：')
print(AQI_by_city.nsmallest(10))
# 查看小於50的所有城市
# print(AQI_by_city[AQI_by_city <= 50])
5、哪些城市屬於一秒破功型？
city1 = data.loc[(data['time'] == pd.to_datetime('2019-02-05 02:00:00')) & (data['AQI']<100), 'city']
data3 = data[(data['time'] >= pd.to_datetime('2019-02-05 18:00:00'))&
(data['time'] <= pd.to_datetime('2019-02-06 23:59:59'))]
city2 = data3.loc[data3['AQI'] > 200, 'city']
# 得到城市
print(list(set(city1).intersection(set(city2))))
提示：其實這些變化究竟是否是因爲煙火？需要進一步考量。
6、除夕中午到初一中午的超標城市個數
start_time = pd.to_datetime('2019-02-04 12:00:00')
end_time = pd.to_datetime('2019-02-05 12:00:00')
city_over_new_year = data.loc[(data['time'] >= start_time) & (data['time'] <= end_time) &
(data['AQI'] > 100)].groupby('time')['AQI'].count().plot(title='The number of city')
plt.show()
7、除夕到初七的超標城市個數
data = data.set_index(data['time']) # 這裏我們將time設置爲索引，並刪除原列
city_over_count = (data.groupby('city')['AQI'].resample('D').mean() > 100).unstack(level=-1).sum()
print(city_over_count)
bar = Bar("全國除夕至初七期間空氣質量超標城市數量", "時間: 2019年 總城市數: 367", width=800)
date = ['02-04(除夕)', '02-05(初一)', '02-06(初二)', '02-07(初三)',
'02-08(初四)', '02-09(初五)', '02-10(初六)', '02-11(初七)']
bar.add("", date, city_over_count.values, is_stack=True,
is_label_show=True, bar_category_gap='40%', label_color=['#130f40'], label_text_size=18,
legend_text_size=18, xaxis_label_textsize=18, yaxis_label_textsize=18)
bar.render('1.html')
line = Line("", width=800)
line.add("", date, city_over_count.values+5, is_smooth=True)
line.render('2.html')
