OkHttpClient和Jsoup進行網頁爬取

通過http請求，返回一個json格式的數據，然後將json數據轉化爲java對象返回給調用方。Http採用OkHttp庫，json轉化採用fastjson庫。

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"&gt;
<modelVersion>4.0.0</modelVersion>

<groupId>com.ok.http.client</groupId>
<artifactId>okhttp</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>

<name>okhttp</name>
<url>http://maven.apache.org</url&gt;

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>

<dependencies>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.11.3</version>
</dependency>

<dependency>
    <groupId>org.mongodb</groupId>
    <artifactId>bson</artifactId>
    <version>3.6.4</version>
</dependency>

<dependency>
    <groupId>com.google.cloud.trace.instrumentation.jdbc</groupId>
    <artifactId>driver</artifactId>
    <version>0.1.1</version>
    <type>pom</type>
</dependency>
<dependency>
    <groupId>ch.qos.logback.contrib</groupId>
    <artifactId>logback-mongodb-access</artifactId>
    <version>0.1.5</version>
</dependency>
<!-- MongoDB數據庫連接驅動 -->
<dependency>
    <groupId>org.mongodb</groupId>
    <artifactId>mongo-java-driver</artifactId>
    <version>3.0.0</version>
</dependency>
<dependency>
    <groupId>com.squareup.okio</groupId>
    <artifactId>okio</artifactId>
    <version>1.11.0</version>

</dependency>
<dependency>
    <groupId>com.squareup.okhttp3</groupId>
    <artifactId>okhttp</artifactId>
    <version>3.6.0</version>
</dependency>

<dependency>
    <groupId>com.alibaba</groupId>
    <artifactId>fastjson</artifactId>
    <version>1.2.47</version>
</dependency>

<dependency>
  <groupId>junit</groupId>
  <artifactId>junit</artifactId>
  <version>3.8.1</version>
  <scope>test</scope>
</dependency>

</dependencies>
</project>

package com.ok.http.client.okhttp;

import java.util.List;
import java.util.Map;

public class ExecuteTask {
public static void main(String[] args) throws Exception {
// 調用downloadHtml下載網頁
CrawlData crawlData = new CrawlData();
String url = null;
url = "http://top.chinaz.com/all/index.html";
System.out.println("開始爬取,請等待.");
String htmlBody = crawlData.downloadHtml(url);
System.out.println("爬取成功");
// 將下載的數據進行分析
List<Map<String, Object>> dataList = Analysis.analysisData(htmlBody);
System.out.println("數據解析成功");
for (Map<String, Object> data : dataList) {
StoreData.adds(data);
System.out.println("存儲成功");
}
}
}

package com.ok.http.client.okhttp;

import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;

/**

• Hello world!

• */
  public class CrawlData
  {
  public static String downloadHtml(String url) {
  String body = null;
  OkHttpClient client = new OkHttpClient();
  //請求
  Request request = new Request.Builder().url(url).build();
  //發起請求
  try {

      Response response = client.newCall(request).execute();
      body = new String(response.body().bytes());
  
  } catch (Exception e) {
      e.printStackTrace();
  }
  return body;//取得目標

  }
  }

  package com.ok.http.client.okhttp;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.util.*;

public class Analysis {
/**

• 解析數據
• @param htmlBody
• @return

• @throws IOException
  */
  public static List<Map<String, Object>> analysisData(String htmlBody) throws Exception {
  // 獲取目標HTML代碼
  List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
  Document doc = Jsoup.parse(htmlBody);
  Elements elements = doc.select("ul.listCentent").select("li");
  System.out.println(" 數據集合大小=====" + elements.size());
  for (Element elmemt : elements) {
  Map<String, Object> map1 = new HashMap<String, Object>();
  // 獲取公司名
  String siteName = elmemt.select("div.CentTxt > h3.rightTxtHead > a").text();
  System.out.println("siteName=====" + siteName);
  // 獲取域名
  String domainName = elmemt.select("div.CentTxt > h3.rightTxtHead > span").text();
  System.out.println("domainName=====" + domainName);
  // 獲取AlexaRank排名
  String AlexaRank = elmemt.select("li.clearfix >div.CentTxt > div.RtCPart >p").text();
  System.out.println("AlexaRank=====" + AlexaRank);
  // 獲取公司簡介
  String Synopsis = elmemt.select("div.CentTxt> p").text();
  System.out.println("公司簡介====" + Synopsis);
  // 獲取得分
  String score = elmemt.select("div.RtCRateCent>span").text();
  System.out.println(score);
  // 獲取排名
  String siteRank = elmemt.select("div.RtCRateCent> strong").text();
  System.out.println("排名:" + siteRank);
  // 獲取網址
  String webSite = "http://top.chinaz.com" + elmemt.select("a").first().attr("href");
  System.out.println("網址:" + webSite);
  // 獲取備案信息
  String stringecordInformation = getGecordInformation(webSite);
  System.out.println("備案信息" + stringecordInformation);
  System.out.println("\t");
  // StoreData.add(siteName,domainName, AlexaRank , Synopsis, score, siteRank, webSite ,RecordInformation);
  map1.put("siteName", siteName);
  map1.put("domainName", domainName);
  map1.put("AlexaRank", AlexaRank);
  map1.put("公司簡介", Synopsis);
  map1.put("排名", siteRank);
  map1.put("網址", webSite);
  map1.put("備案信息", stringecordInformation);
  list.add(map1);
  }
  return list;
  }

  /**

• 獲取備案信息
• @param url
• @return
• @throws Exception
  */
  private static String getGecordInformation(String url) throws Exception {
  String htmlBody = CrawlData.downloadHtml(url);
  if (htmlBody != null) {
  Document doc = Jsoup.parse(htmlBody);
  String stringecordInformation = doc.select("li.TMain06List-Left>p").text();
  return stringecordInformation;
  }
  return null;
  }
  }

  package com.ok.http.client.okhttp;
  import com.alibaba.fastjson.JSONObject;
  import com.mongodb.MongoClient;
  import com.mongodb.client.MongoCollection;
  import com.mongodb.client.MongoDatabase;
  import org.bson.Document;
  import java.util.Map;

public class StoreData{

public static void adds(Map<String,Object> dataMap){
    try{
        // 連接到 mongodb 服務String siteRank
        MongoClient mongoClient = new MongoClient( "localhost" , 27017 );
        // 連接到數據庫
        MongoDatabase mongoDatabase = mongoClient.getDatabase("sit_rank");
        System.out.println(mongoDatabase);
        System.out.println("成功連接數據庫");

        MongoCollection<Document> collection = mongoDatabase.getCollection("information");
        System.out.println(collection);
        System.out.println("集合 information 選擇成功");
        //插入文檔
        /**
         * 1. 創建文檔 org.bson.Document 參數爲key-value的格式
         * 2. 創建文檔集合List<Document>
         * 3. 將文檔集合插入數據庫集合中 mongoCollection.insertMany(List<Document>) 插入單個文檔可以用 mongoCollection.insertOne(Document)
         * */
        String siteName=null;String domainName=null;String AlexaRank=null;String Synopsis=null;
                String score=null;String siteRank=null;String webSite=null;String RecordInformation=null;
        JSONObject josn = JSONObject.parseObject(dataMap.toString());
                Document document = new Document(josn);
        document.put("_id",siteName);
        document.append("domainName", domainName);
        document.append("AlexaRank",AlexaRank);
        document.append("Synopsis",Synopsis);
        document.append("score",score);
        document.append("siteRank",siteRank);
        document.append("webSite",webSite);
        document.append("RecordInformation",RecordInformation);
        collection.insertOne(document);
        System.out.println("文檔插入成功");
        //關閉mongodb連接
        mongoClient.close();
        System.out.println("MongoDB連接已關閉");
    }catch(Exception e){
        System.err.println( e.getClass().getName() + ": " + e.getMessage() );
    }
}

}