查看文章
|
zz from http://www.buaaer.com/bbs/blog.php?tid=39937 HttpClient 4.0出來不久,所以網絡上面相關的實例教程不多,搜httpclient得到的大部分都是基於原 Commons HttpClient 3.1 (legacy) 包的,官網下載頁面:http://hc.apache.org/downloads.cgi,如果大家看了官網說明就明白httpclient4.0是從原包分支出來獨立成包的,以後原來那個包中的httpclient不會再升級,所以以後我們是用httpclient新分支,由於4.0與之前的3.1包結構以及接口等都有較大變化,所以網上搜到的實例大部分都是不適合4.0的,當然,我們可以通過那些實例去琢磨4.0的用法,我也是新手,記錄下學習過程方便以後檢索本實例我們來獲取抓取網頁編碼,內容等信息 默認情況下,服務器端會根據客戶端的請求頭信息來返回服務器支持的編碼,像google.cn他本身支持utf-8,gb2312等編碼,所以如果你在頭部中不指定任何頭部信息的話他默認會返回gb2312編碼,而如果我們在瀏覽器中直接訪問google.cn,通過httplook,或者firefox的firebug插件查看返回頭部信息的話會發現他返回的是UTF-8編碼 下面我們還是看實例來解說吧,註釋等我也放代碼裏面解釋,放完整代碼,方便新手理解 本實例將 使用的httpclient相關包 httpclient-4.0.jar httpcore-4.0.1.jar httpmime-4.0.jar commons-logging-1.0.4.jar等其它相關包 // HttpClientTest.java package com.baihuo.crawler.test; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.util.EntityUtils; class HttpClientTest { public final static void main(String[] args) throws Exception { // 初始化,此處構造函數就與3.1中不同 HttpClient httpclient = new DefaultHttpClient(); HttpHost targetHost = new HttpHost("www.google.cn"); //HttpGet httpget = new HttpGet("http://www.apache.org/"); HttpGet httpget = new HttpGet("/"); // 查看默認request頭部信息 System.out.println("Accept-Charset:" + httpget.getFirstHeader("Accept-Charset")); // 以下這條如果不加會發現無論你設置Accept-Charset爲gbk還是utf-8,他都會默認返回gb2312(本例針對google.cn來說) httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.2)"); // 用逗號分隔顯示可以同時接受多種編碼 httpget.setHeader("Accept-Language", "zh-cn,zh;q=0.5"); httpget.setHeader("Accept-Charset", "GB2312,utf-8;q=0.7,*;q=0.7"); // 驗證頭部信息設置生效 System.out.println("Accept-Charset:" + httpget.getFirstHeader("Accept-Charset").getValue()); // Execute HTTP request System.out.println("executing request " + httpget.getURI()); HttpResponse response = httpclient.execute(targetHost, httpget); //HttpResponse response = httpclient.execute(httpget); System.out.println("----------------------------------------"); System.out.println("Location: " + response.getLastHeader("Location")); System.out.println(response.getStatusLine().getStatusCode()); System.out.println(response.getLastHeader("Content-Type")); System.out.println(response.getLastHeader("Content-Length")); System.out.println("----------------------------------------"); // 判斷頁面返回狀態判斷是否進行轉向抓取新鏈接 int statusCode = response.getStatusLine().getStatusCode(); if ((statusCode == HttpStatus.SC_MOVED_PERMANENTLY) || (statusCode == HttpStatus.SC_MOVED_TEMPORARILY) || (statusCode == HttpStatus.SC_SEE_OTHER) || (statusCode == HttpStatus.SC_TEMPORARY_REDIRECT)) { // 此處重定向處理 此處還未驗證 String newUri = response.getLastHeader("Location").getValue(); httpclient = new DefaultHttpClient(); httpget = new HttpGet(newUri); response = httpclient.execute(httpget); } // Get hold of the response entity HttpEntity entity = response.getEntity(); // 查看所有返回頭部信息 Header headers[] = response.getAllHeaders(); int ii = 0; while (ii < headers.length) { System.out.println(headers[ii].getName() + ": " + headers[ii].getValue()); ++ii; } // If the response does not enclose an entity, there is no need // to bother about connection release if (entity != null) { // 將源碼流保存在一個byte數組當中,因爲可能需要兩次用到該流, byte[] bytes = EntityUtils.toByteArray(entity); String charSet = ""; // 如果頭部Content-Type中包含了編碼信息,那麼我們可以直接在此處獲取 charSet = EntityUtils.getContentCharSet(entity); System.out.println("In header: " + charSet); // 如果頭部中沒有,那麼我們需要 查看頁面源碼,這個方法雖然不能說完全正確,因爲有些粗糙的網頁編碼者沒有在頁面中寫頭部編碼信息 if (charSet == "") { regEx="(?=<meta).*?(?<=charset=[\\'|\\\"]?)([[a-z]|[A-Z]|[0-9]|-]*)"; p=Pattern.compile(regEx, Pattern.CASE_INSENSITIVE); m=p.matcher(new String(bytes)); // 默認編碼轉成字符串,因爲我們的匹配中無中文,所以串中可能的亂碼對我們沒有影響 result=m.find(); if (m.groupCount() == 1) { charSet = m.group(1); } else { charSet = ""; } } System.out.println("Last get: " + charSet); // 至此,我們可以將原byte數組按照正常編碼專成字符串輸出(如果找到了編碼的話) System.out.println("Encoding string is: " + new String(bytes, charSet)); } httpclient.getConnectionManager().shutdown(); } } [ 本帖最後由 darkness 於 2009-9-9 17:24 編輯 ]
|