java爬虫带你爬天爬地爬人生，爬新浪

HttpClient简介

HttpClient是Apache Jakarta Common下的子项目，可以用来提供高效的、最新的、功能丰富的支持HTTP协议的客户端编程工具包，并且它支持 HTTP 协议最新的版本。它的主要功能有：

(1) 实现了所有 HTTP 的方法（GET,POST,PUT,HEAD 等）

(2) 支持自动转向

(3) 支持 HTTPS 协议

(4) 支持代理服务器等

Jsoup简介

jsoup是一款Java的HTML解析器，可直接解析某个URL地址、HTML文本内容。它提供了一套非常省力的API，可通过DOM，CSS以及类似于jQuery的操作方法来取出和操作数据。它的主要功能有：

(1) 从一个URL，文件或字符串中解析HTML；

(2) 使用DOM或CSS选择器来查找、取出数据；

(3) 可操作HTML元素、属性、文本；

使用步骤

代码
import org.apache.http.HttpEntity;

import org.apache.http.client.config.RequestConfig;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.client.protocol.HttpClientContext;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClientBuilder;

import org.apache.http.util.EntityUtils;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.junit.Test;

import java.util.List;

/**

HttpClient & Jsoup libruary test class
Created by xuyh at 2017/11/6 15:28.

public classHttpClientJsoupTest{

@Test

public void test() {

        //通过httpClient获取网页响应,将返回的响应解析为纯文本

    HttpGet httpGet = new HttpGet("http://sports.sina.com.cn/");

    httpGet.setConfig(RequestConfig.custom().setSocketTimeout(30000).setConnectTimeout(30000).build());

    CloseableHttpClient httpClient = null;

    CloseableHttpResponse response = null;

    String responseStr = "";

    try {

        httpClient = HttpClientBuilder.create().build();

        HttpClientContext context = HttpClientContext.create();

        response = httpClient.execute(httpGet, context);

        int state = response.getStatusLine().getStatusCode();

        if (state != 200)

            responseStr = "";

        HttpEntity entity = response.getEntity();

        if (entity != null)

            responseStr = EntityUtils.toString(entity, "utf-8");

    } catch (Exception e) {

        e.printStackTrace();

    } finally {

        try {

            if (response != null)

                response.close();

            if (httpClient != null)

                httpClient.close();

        } catch (Exception ex) {

            ex.printStackTrace();

        }

    }

    if (responseStr == null)

        return;

    //将解析到的纯文本用Jsoup工具转换成Document文档并进行操作

    Document document = Jsoup.parse(responseStr);

    List<Element> elements = document.getElementsByAttributeValue("class", "phdnews_txt fr").first()

            .getElementsByAttributeValue("class", "phdnews_hdline");

    elements.forEach(element -> {

        for (Element e : element.getElementsByTag("a")) {

            System.out.println(e.attr("href"));

            System.out.println(e.text());

        }

    });

}

详解

新建HttpGet对象，对象将从 http://sports.sina.com.cn/ 这个URL地址获取GET响应。并设置socket超时时间和连接超时时间分别为30000ms。

将HttpClient和Jsoup进行封装，形成一个工具类，内容如下：

import org.apache.http.HttpEntity;

import org.apache.http.NameValuePair;

import org.apache.http.client.CookieStore;

import org.apache.http.client.config.RequestConfig;

import org.apache.http.client.entity.UrlEncodedFormEntity;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.client.methods.HttpPost;

import org.apache.http.client.protocol.HttpClientContext;

import org.apache.http.conn.ssl.SSLConnectionSocketFactory;

import org.apache.http.cookie.Cookie;

import org.apache.http.entity.ContentType;

import org.apache.http.entity.StringEntity;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClientBuilder;

import org.apache.http.impl.client.HttpClients;

import org.apache.http.message.BasicNameValuePair;

import org.apache.http.ssl.SSLContextBuilder;

import org.apache.http.util.EntityUtils;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import javax.net.ssl.*;

import java.io.IOException;

import java.security.GeneralSecurityException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

/**

Http工具，包含：
普通http请求工具(使用httpClient进行http,https请求的发送)
Created by xuyh at 2017/7/17 19:08.

public classHttpUtils{

/**

请求超时时间,默认20000ms

private int timeout = 20000;

/**

cookie表

private Map<String, String> cookieMap = new HashMap<>();

/**

请求编码(处理返回结果)，默认UTF-8

private String charset = "UTF-8";

private static HttpUtils httpUtils;

privateHttpUtils(){

}

/**

获取实例

*@return

publicstaticHttpUtilsgetInstance(){

    if (httpUtils == null)

        httpUtils = new HttpUtils();

    return httpUtils;

}

/**

清空cookieMap

publicvoidinvalidCookieMap(){

    cookieMap.clear();

}

publicintgetTimeout(){

    return timeout;

}

/**

设置请求超时时间

*@paramtimeout

publicvoidsetTimeout(inttimeout){

    this.timeout = timeout;

}

publicStringgetCharset(){

    return charset;

}

/**

设置请求字符编码集

*@paramcharset

publicvoidsetCharset(String charset){

    this.charset = charset;

}

/**

将网页返回为解析后的文档格式

*@paramhtml

*@return

*@throwsException

publicstaticDocumentparseHtmlToDoc(String html)throwsException{

    return removeHtmlSpace(html);

}

privatestaticDocumentremoveHtmlSpace(String str){

    Document doc = Jsoup.parse(str);

    String result = doc.html().replace("&nbsp;", "");

    return Jsoup.parse(result);

}

/**

执行get请求,返回doc

*@paramurl

*@return

*@throwsException

publicDocumentexecuteGetAsDocument(String url)throwsException{

    return parseHtmlToDoc(executeGet(url));

}

/**

执行get请求

*@paramurl

*@return

*@throwsException

publicStringexecuteGet(String url)throwsException{

    HttpGet httpGet = new HttpGet(url);

    httpGet.setHeader("Cookie", convertCookieMapToString(cookieMap));

    httpGet.setConfig(RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout).build());

    CloseableHttpClient httpClient = null;

    String str = "";

    try {

        httpClient = HttpClientBuilder.create().build();

        HttpClientContext context = HttpClientContext.create();

        CloseableHttpResponse response = httpClient.execute(httpGet, context);

        getCookiesFromCookieStore(context.getCookieStore(), cookieMap);

        int state = response.getStatusLine().getStatusCode();

        if (state == 404) {

            str = "";

        }

        try {

            HttpEntity entity = response.getEntity();

            if (entity != null) {

                str = EntityUtils.toString(entity, charset);

            }

        } finally {

            response.close();

        }

    } catch (IOException e) {

        throw e;

    } finally {

        try {

            if (httpClient != null)

                httpClient.close();

        } catch (IOException e) {

            throw e;

        }

    }

    return str;

}

/**

用https执行get请求,返回doc

*@paramurl

*@return

*@throwsException

publicDocumentexecuteGetWithSSLAsDocument(String url)throwsException{

    return parseHtmlToDoc(executeGetWithSSL(url));

}

/**

用https执行get请求

*@paramurl

*@return

*@throwsException

publicStringexecuteGetWithSSL(String url)throwsException{

    HttpGet httpGet = new HttpGet(url);

    httpGet.setHeader("Cookie", convertCookieMapToString(cookieMap));

    httpGet.setConfig(RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout).build());

    CloseableHttpClient httpClient = null;

    String str = "";

    try {

        httpClient = createSSLInsecureClient();

        HttpClientContext context = HttpClientContext.create();

        CloseableHttpResponse response = httpClient.execute(httpGet, context);

        getCookiesFromCookieStore(context.getCookieStore(), cookieMap);

        int state = response.getStatusLine().getStatusCode();

        if (state == 404) {

            str = "";

        }

        try {

            HttpEntity entity = response.getEntity();

            if (entity != null) {

                str = EntityUtils.toString(entity, charset);

            }

        } finally {

            response.close();

        }

    } catch (IOException e) {

        throw e;

    } catch (GeneralSecurityException ex) {

        throw ex;

    } finally {

        try {

            if (httpClient != null)

                httpClient.close();

        } catch (IOException e) {

            throw e;

        }

    }

    return str;

}

/**

执行post请求,返回doc

*@paramurl

*@paramparams

*@return

*@throwsException

publicDocumentexecutePostAsDocument(String url, Map<String, String> params)throwsException{

    return parseHtmlToDoc(executePost(url, params));

}

/**

执行post请求

*@paramurl

*@paramparams

*@return

*@throwsException

publicStringexecutePost(String url, Map<String, String> params)throwsException{

    String reStr = "";

    HttpPost httpPost = new HttpPost(url);

    httpPost.setConfig(RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout).build());

    httpPost.setHeader("Cookie", convertCookieMapToString(cookieMap));

    List<NameValuePair> paramsRe = new ArrayList<>();

    for (String key : params.keySet()) {

        paramsRe.add(new BasicNameValuePair(key, params.get(key)));

    }

    CloseableHttpClient httpclient = HttpClientBuilder.create().build();

    CloseableHttpResponse response;

    try {

        httpPost.setEntity(new UrlEncodedFormEntity(paramsRe));

        HttpClientContext context = HttpClientContext.create();

        response = httpclient.execute(httpPost, context);

        getCookiesFromCookieStore(context.getCookieStore(), cookieMap);

        HttpEntity entity = response.getEntity();

        reStr = EntityUtils.toString(entity, charset);

    } catch (IOException e) {

        throw e;

    } finally {

        httpPost.releaseConnection();

    }

    return reStr;

}

/**

用https执行post请求,返回doc

*@paramurl

*@paramparams

*@return

*@throwsException

publicDocumentexecutePostWithSSLAsDocument(String url, Map<String, String> params)throwsException{

    return parseHtmlToDoc(executePostWithSSL(url, params));

}

/**

用https执行post请求

*@paramurl

*@paramparams

*@return

*@throwsException

publicStringexecutePostWithSSL(String url, Map<String, String> params)throwsException{

    String re = "";

    HttpPost post = new HttpPost(url);

    List<NameValuePair> paramsRe = new ArrayList<>();

    for (String key : params.keySet()) {

        paramsRe.add(new BasicNameValuePair(key, params.get(key)));

    }

    post.setHeader("Cookie", convertCookieMapToString(cookieMap));

    post.setConfig(RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout).build());

    CloseableHttpResponse response;

    try {

        CloseableHttpClient httpClientRe = createSSLInsecureClient();

        HttpClientContext contextRe = HttpClientContext.create();

        post.setEntity(new UrlEncodedFormEntity(paramsRe));

        response = httpClientRe.execute(post, contextRe);

        HttpEntity entity = response.getEntity();

        if (entity != null) {

            re = EntityUtils.toString(entity, charset);

        }

        getCookiesFromCookieStore(contextRe.getCookieStore(), cookieMap);

    } catch (Exception e) {

        throw e;

    }

    return re;

}

/**

发送JSON格式body的POST请求

*@paramurl 地址

*@paramjsonBody json body

*@return

*@throwsException

publicStringexecutePostWithJson(String url, String jsonBody)throwsException{

    String reStr = "";

    HttpPost httpPost = new HttpPost(url);

    httpPost.setConfig(RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout).build());

    httpPost.setHeader("Cookie", convertCookieMapToString(cookieMap));

    CloseableHttpClient httpclient = HttpClientBuilder.create().build();

    CloseableHttpResponse response;

    try {

        httpPost.setEntity(new StringEntity(jsonBody, ContentType.APPLICATION_JSON));

        HttpClientContext context = HttpClientContext.create();

        response = httpclient.execute(httpPost, context);

        getCookiesFromCookieStore(context.getCookieStore(), cookieMap);

        HttpEntity entity = response.getEntity();

        reStr = EntityUtils.toString(entity, charset);

    } catch (IOException e) {

        throw e;

    } finally {

        httpPost.releaseConnection();

    }

    return reStr;

}

/**

发送JSON格式body的SSL POST请求

*@paramurl 地址

*@paramjsonBody json body

*@return

*@throwsException

publicStringexecutePostWithJsonAndSSL(String url, String jsonBody)throwsException{

    String re = "";

    HttpPost post = new HttpPost(url);

    post.setHeader("Cookie", convertCookieMapToString(cookieMap));

    post.setConfig(RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout).build());

    CloseableHttpResponse response;

    try {

        CloseableHttpClient httpClientRe = createSSLInsecureClient();

        HttpClientContext contextRe = HttpClientContext.create();

        post.setEntity(new StringEntity(jsonBody, ContentType.APPLICATION_JSON));

        response = httpClientRe.execute(post, contextRe);

        HttpEntity entity = response.getEntity();

        if (entity != null) {

            re = EntityUtils.toString(entity, charset);

        }

        getCookiesFromCookieStore(contextRe.getCookieStore(), cookieMap);

    } catch (Exception e) {

        throw e;

    }

    return re;

}

privatevoidgetCookiesFromCookieStore(CookieStore cookieStore, Map<String, String> cookieMap){

    List<Cookie> cookies = cookieStore.getCookies();

    for (Cookie cookie : cookies) {

        cookieMap.put(cookie.getName(), cookie.getValue());

    }

}

privateStringconvertCookieMapToString(Map<String, String> map){

    String cookie = "";

    for (String key : map.keySet()) {

        cookie += (key + "=" + map.get(key) + "; ");

    }

    if (map.size() > 0) {

        cookie = cookie.substring(0, cookie.length() - 2);

    }

    return cookie;

}

/**

创建 SSL连接

*@return

*@throwsGeneralSecurityException

privatestaticCloseableHttpClientcreateSSLInsecureClient()throwsGeneralSecurityException{

    try {

        SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, (chain, authType) -> true).build();

        SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory(sslContext,

                (s, sslContextL) -> true);

        return HttpClients.custom().setSSLSocketFactory(sslConnectionSocketFactory).build();

    } catch (GeneralSecurityException e) {

        throw e;

    }

}

}

给大家推荐一个程序员学习交流群：863621962。群里有分享的视频，还有思维导图

群公告有视频，都是干货的，你可以下载来看。主要分享分布式架构、高可扩展、高性能、高并发、性能优化、Spring boot、Redis、ActiveMQ、Nginx、Mycat、Netty、Jvm大型分布式项目实战学习架构师视频。

java爬虫带你爬天爬地爬人生，爬新浪

MySQL 核心模块揭秘 | 18 期 | 锁在内存里长什么样*

使用perf工具生成火焰图

大龄程序员思考

响应式界面控件DevExtreme * 更强的数据分析和可视化功能

HttpSecurity 是如何组装过滤器链的

数说海南——近6年海南各市县人口简单看

长序列中Transformers的高级注意力机制总结

WebStorm 创建 Vue 项目

頂級實用乾貨——談談Java中的volatile

爲什麼Spring框架如此流行？

Springboot 打jar包分離lib，配置文件正確方式

阿里P8架構師深度概述分佈式架構

java的poi技術讀取Excel數據到MySQL

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結