python 網頁抓取經驗

一定要設置好user-agnet 和accept

#coding:utf-8
'''
Created on 2009-7-15

@author: Administrator
'''
import urllib2
import newhttplib
import lxml.html as x  

def getmusic(num,soc=None):
    s=num.split(':')
    s[0]=s[0].strip()
    s[1]=s[1].strip()
    s[2]=s[2].strip()
    h1 = newhttplib.HTTPConnection('10.0.0.172',80)
    hs={'Accept': 'text/html,application/xhtml+xml,application/xml','User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1) Gecko/20090624 Firefox/3.5'}
    h1.auto_open = 0
    h1.connect(soc)
    url='http://218.200.160.29/s3/i/app/search/musicResult.jsp?qd=1956&CH=12530-wap-lslb&v=1864&tplpath=/s3/i/qrc/&type=all&keyword='+urllib2.quote(s[0].encode('utf-8')+" "+s[1].encode('utf-8')) 
    h1.request("GET",url,headers=hs) 
    r1 = h1.getresponse()
    print r1.getheader('Content-Type')
    content=r1.read()   
    #content=content.decode('utf-8')
    doc=x.document_fromstring(content)  
    alist=doc.xpath("//a")
    print s[0],s[1],len(alist)
    for a in alist:
        c1= a.text_content()
        try:
            if c1.find(s[0])>=0 and c1.find(s[1])>=0:
                print c1
                print c1.find(s[0]),c1.find(s[1])
                nexturl='http://218.200.160.29'+a.get('href') 
                print nexturl
                h1.request("GET",nexturl,headers=hs) 
                r1 = h1.getresponse()
                content=r1.read()
                print r1.getheaders()
                #content=content.decode('utf-8')
                print content
                f=open('./b.html','wb')
                f.write(content)
                f.write(nexturl)
                f.close()
                doc=x.document_fromstring(content)  
                blist=doc.xpath("//a")
                print 'blen=',len(blist)
                for b in blist:
                    bcontent=b.text_content()
                    if bcontent.find(u'高潮版')>=0:
                         print bcontent
                         bhref= b.get('href')
                         nnexturl='http://218.200.160.29'+bhref
                         h1.request("GET",nnexturl,headers=hs) 
                         r1 = h1.getresponse()
                         ccontent=r1.read()
                         #content=content.decode('utf-8')
                         #print ccontent
                         doc=x.document_fromstring(ccontent)  
                         clist=doc.xpath("//a")
                         print 'clen=',len(clist)
                         for c in clist:
                             chref=c.get('href')
                             if chref.find('218.200.160.10')>=0:
                                 print chref
                                 sb2=s[2][11:]
                                 print sb2
                                 if chref.find(sb2)>=0:
                                     print u'找到匹配歌曲開始下載。。。'
                                     h1.request("GET",chref) 
                                     r1 = h1.getresponse()
                                     print r1.status
                                     print r1.getheaders()
        except BaseException,e:
            print e

if __name__ == '__main__':
    #f=open('./test1.htm','wb')
    #con=f.read()  
    #print con
    getmusic('相信:曾建軍:600902000005714466')