python代理

1、採集代理ip網站的ip，
2、設置訪問ip爲代理ip，
3、檢測是否訪問成功通過匹配返回的代理ip是否和訪問的ip一致
# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup as bs
import re
def proxy_list(mbUrl):
    headers={'User-Agent':"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0"}
    url='http://www.xicidaili.com/nn/'
    r=requests.get(url=url,headers=headers)
    soup=bs(r.content)
    datas=soup.find_all(name='tr',attrs={'class':re.compile('|[^odd]')})
    for data in datas:
        soup_proxy_content=bs(str(data))
        soup_proxys=soup_proxy_content.find_all(name='td')
        #for i in[1,2,5]:
        #   print soup_proxys[i].string
        ip=str(soup_proxys[1].string)
        port=str(soup_proxys[2].string)
        types=str(soup_proxys[5].string)
        proxy_test(mbUrl,ip,port,types)

def proxy_test(url,ip,port,types):
    proxy={}
    proxy[types.lower()]='%s:%s'%(ip,port)
    #proxy={'http':'106.46.136.24:808'}
    try:
        r=requests.get(url,proxies=proxy,timeout=3) #會把每個代理ip都測試一遍，超時設置爲六秒
        ip_content=re.findall(r'\[(.*?)\]',r.text)[0]#匹配[]中的ip
        #print r.text
        if ip==ip_content:#判斷代理是否測試成功
            print proxy
    except Exception,e:
        #print e
        pass
#測試一下
proxy_list("http://1212.ip138.com/ic.asp")