從高德地圖大批量扒取數據，並寫入excel

首先，源代碼下載地址：http://download.csdn.net/detail/u010134178/9714703
github: https://github.com/whsgzcy/Data_From_GaoDe
然後，如果有問題，或者是積分不夠可以發送郵件到我的郵箱[email protected]，我可以把源程序分享給大家，很高興能和各位同行分享，如果是打電話要我上廁所，我可是要罵人的呦~。
最後，直接上代碼。

演示視頻：鏈接: https://pan.baidu.com/s/1h3FYUPJIEoSCNTux3R2t0w 密碼: xjf7

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;

import javax.naming.InitialContext;

import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;

import com.iwant.download2geodata.data.ShopInfo;
import com.iwant.download2geodata.data.ShopList;
import com.iwant.download2geodata.data.TemplateData;

import jxl.Workbook;
import jxl.write.Label;
import jxl.write.WritableSheet;
import jxl.write.WritableWorkbook;
import net.sf.json.JSONArray;
import net.sf.json.JSONException;
import net.sf.json.JSONObject; / ** * @Description：獲取高德所有數據* @author：whsgzcy * @date：2016-12-17下午1:04:31首先是從高德複製一個鏈接，只要修改pagernum參數即可拼接成新的鏈接*抓取方式較原始，但可維護性高* / public class GeoGetDataThread extends Thread {   public int pagernum = 1; //跳轉到下一頁參數 public String murl =“http：//ditu.amap。 ？COM /服務/ poiInfo QUERY_TYPE = TQuery的頁大小＆= 20＆頁次=“;     public String nurl =“＆          ”           +“city = 321200＆geoobj = 119.848676％7C32.40107％7C119.93279％7C32.527222＆keywords =％E7％94％ B5％E5％8A％A8％E8％BD％A6" ;
















    private ShopList shopList = new ShopList();
    private List<ShopInfo> shopInfoList = new ArrayList<ShopInfo>();
    private List<TemplateData> templateDataList = new ArrayList<TemplateData>();

    /**
     * @Description: 根據pagernum跳轉下一頁
     * @author: whsgzcy
     * @date: 2016-12-17 下午1:17:08
     * @param pagernum
     */

    @Override
    public void run() {
        super.run();

//      // /先用本次文件進行測試
//      String encoding = "UTF-8";
//      StringBuilder builder = null;
//      try {
//          File file = new File("C:\\Users\\suzhe\\Desktop\\t.txt");
//          InputStreamReader read = new InputStreamReader(new FileInputStream(
//                  file), encoding);// 考慮到編碼格式
//          BufferedReader bufferedReader = new BufferedReader(read);
//          builder = new StringBuilder();
//          for (String s = bufferedReader.readLine(); s != null; s = bufferedReader
//                  .readLine()) {
//              builder.append(s);
//          }
//      } catch (Exception e) {
//          e.printStackTrace();
//      }
//      if (null == builder) return;
//      jsonObject = JSONObject.fromObject(builder.toString());
        // 請求數據
        // 根據現有的數據量沒有到list極限
        // 使用list存儲數據，並在每次請求數據時休眠5秒，放緩線程以及防止扒取數據IP被封

        try {
            for(;;pagernum++){
                // 線程休眠
//              try {Thread.sleep(5000);} catch (InterruptedException e1) {e1.printStackTrace();}
                String url = murl+pagernum+nurl;
                System.out.println(url);
                JSONObject jsonObject = new JSONObject();
                jsonObject = HttpRequestUtil.getJsonObject(url);
                // 先打印count 判斷是否有數據
                int count = jsonObject.getInt("count");
                if(0 == count) break;
                System.out.println("count = " + count);
                if (0 != count) {
                    JSONArray dataArray = jsonObject.getJSONArray("data");
                    JSONArray listArray = dataArray.getJSONObject(0).getJSONArray("list");
                    JSONObject line = null;
                    for (int i = 0; i < listArray.size(); i++) {
                        ShopInfo shopInfo = new ShopInfo();
                        TemplateData template = new TemplateData();
                        line = listArray.getJSONObject(i);
                        String name = line.getString("name");
                        shopInfo.setName(name);
                        String tel = line.getString("tel");
                        shopInfo.setTel(tel);
                        String address = line.getString("address");
                        shopInfo.setAddress(address);
                        String longitude = line.getString("longitude");
                        shopInfo.setLongitude(longitude);
                        String latitude = line.getString("latitude");
                        shopInfo.setLatitude(latitude);
                        JSONObject templateData = line.getJSONObject("templateData");
                        if (templateData.has("pic_info")) {
                            String pic_info = templateData.getString("pic_info");
                            template.setPic_info(pic_info);
                            templateDataList.add(template);
                            shopInfo.setTemplateData(templateDataList);
                        }else{
                            template.setPic_info("");
                            templateDataList.add(template);
                            shopInfo.setTemplateData(templateDataList);
                        }
                        shopInfoList.add(shopInfo);
                        System.out.println("數據讀寫中");
                    }
                    // 至此 第一頁請求數據完成
                    shopList.setmShopInfoList(shopInfoList);
            }

                // 將數據寫進Excel
                String title[] = {"店鋪名稱","店主","店鋪簡介","店鋪電話","地址","經度","緯度","101充電樁","10有充電樁","照片","是否提供上門服務","圖片鏈接"}; 
                createExcel("C:/Users/suzhe/Desktop/test2.xls","sheet1",title);
                    try {  
                        workbook = new HSSFWorkbook(new FileInputStream("C:/Users/suzhe/Desktop/test2.xls"));
                        //流  
                        FileOutputStream out = null;  
                        HSSFSheet sheet = workbook.getSheet("sheet1");
                        // 獲取表格的總行數  
                        int rowCount = sheet.getLastRowNum() + 1; // 需要加一  
                        // 獲取表頭的列數  
                        int columnCount = sheet.getRow(0).getLastCellNum();  
                        Row row = sheet.createRow(rowCount);     //最新要添加的一行  
                        HSSFRow titleRow = sheet.getRow(0);  

                        if(titleRow!=null){  
                                for(int i = 0; i < shopList.getmShopInfoList().size(); i++){

                                    row = sheet.createRow(i+1);     //最新要添加的一行  

                                    for(int k = 0; k < 12; k++){

                                        String name = shopInfoList.get(i).getName();
                                        Cell cellName = row.createCell(0);
                                        cellName.setCellValue(name);

                                        Cell cellMaster = row.createCell(1);                                
                                        cellMaster.setCellValue("無");
                                        Cell cellSay = row.createCell(2);
                                        cellSay.setCellValue("無");

                                        String tel = shopInfoList.get(i).getTel();
                                        Cell cellTel = row.createCell(3);
                                        cellTel.setCellValue(tel);

                                        String address = shopInfoList.get(i).getAddress();
                                        Cell cellAddress = row.createCell(4);
                                        cellAddress.setCellValue(address);

                                        String longitude = shopInfoList.get(i).getLongitude();
                                        Cell cellLongitude = row.createCell(5);
                                        cellLongitude.setCellValue(longitude);

                                        String latitude = shopInfoList.get(i).getLatitude();
                                        Cell cellLatitude = row.createCell(6);
                                        cellLatitude.setCellValue(latitude);

                                        Cell cellIsCharge = row.createCell(7);
                                        cellIsCharge.setCellValue("0");                                 
                                        Cell cellHaveCharge = row.createCell(8);
                                        cellHaveCharge.setCellValue("0");

                                        String pic_info = shopInfoList.get(i).getTemplateData().get(i).getPic_info();
                                        if(pic_info.equals("")){
                                            Cell cellPicUrl= row.createCell(11);
                                            cellPicUrl.setCellValue("");
                                            Cell cellPic = row.createCell(9);
                                            cellPic.setCellValue("");
                                        }else{
                                            Cell cellPicUrl= row.createCell(11);
                                            cellPicUrl.setCellValue(pic_info);
                                            Cell cellPic = row.createCell(9);
                                            cellPic.setCellValue("taizhou/"+(i+1)+".jpg");
                                        }

                                        Cell cellHome = row.createCell(10);
                                        cellHome.setCellValue("暫不提供上門服務");  
                                    }
                                }
                        }  
                        out = new FileOutputStream("C:/Users/suzhe/Desktop/test2.xls");  
                        workbook.write(out);  
                    }  catch (Exception e) {  
                        e.printStackTrace();  
                    }  
            }
        } catch (JSONException e) {
            e.printStackTrace();
        }
    }

    public void writeToExcel(String fileDir,String sheetName){  
        //創建workbook  
        File file = new File(fileDir);  
        try {  
            workbook = new HSSFWorkbook(new FileInputStream(file));  
        } catch (FileNotFoundException e) {  
            e.printStackTrace();  
        } catch (IOException e) {  
            e.printStackTrace();  
        }  
        //流  
        FileOutputStream out = null;  
        HSSFSheet sheet = workbook.getSheet(sheetName);  
        // 獲取表格的總行數  
        int rowCount = sheet.getLastRowNum() + 1; // 需要加一  
        // 獲取表頭的列數  
        int columnCount = sheet.getRow(0).getLastCellNum();  
        try {  
            Row row = sheet.createRow(rowCount);     //最新要添加的一行  
            //通過反射獲得object的字段,對應表頭插入  
            // 獲取該對象的class對象  
//            Class class_ = object.getClass();  
            // 獲得表頭行對象  
            HSSFRow titleRow = sheet.getRow(0);  
            if(titleRow!=null){  
                for (int columnIndex = 0; columnIndex < columnCount; columnIndex++) {  //遍歷表頭  
                    String title = titleRow.getCell(columnIndex).toString().trim().toString().trim();  
                    Cell cell1 = row.createCell(0);
                    cell1.setCellValue("111");
                    Cell cell2 = row.createCell(1);
                    cell2.setCellValue("222");
                    Cell cell3 = row.createCell(2);
                    cell3.setCellValue("333");
                }  
            }  
            out = new FileOutputStream(fileDir);  
            workbook.write(out);  
        } catch (Exception e) {  
            e.printStackTrace();  
        } finally {    
            try {    
                out.close();    
            } catch (IOException e) {    
                e.printStackTrace();  
            }    
        }    
    } 
      /** 
     * 創建新excel. 
     * @param fileDir  excel的路徑 
     * @param sheetName 要創建的表格索引 
     * @param titleRow excel的第一行即表格頭 
     */  
    public void createExcel(String fileDir,String sheetName,String titleRow[]){  
        //創建workbook  
        workbook = new HSSFWorkbook();  
        //添加Worksheet（不添加sheet時生成的xls文件打開時會報錯)  
        Sheet sheet1 = workbook.createSheet(sheetName);    
        //新建文件  
        FileOutputStream out = null;  
        try {  
            //添加表頭  
            Row row = workbook.getSheet(sheetName).createRow(0);    //創建第一行    
            for(int i = 0;i < titleRow.length;i++){  
                Cell cell = row.createCell(i);  
                cell.setCellValue(titleRow[i]);  
            }  
            out = new FileOutputStream(fileDir);  
            workbook.write(out);  
        } catch (Exception e) {  
            e.printStackTrace();  
        } finally {    
            try {    
                out.close();    
            } catch (IOException e) {    
                e.printStackTrace();  
            }    
        }    
    }  

    /**
     * 
     * @Description:讀取本地文件---/Users/whsgzcy/Desktop/t.txt
     * @author: whsgzcy
     * @date: 2016-12-17 下午5:35:21
     * @param filePath
     *            void
     * @throws
     */
    public static void readTxtFile(String filePath) {
        try {
            String encoding = "UTF-8";
            File file = new File(filePath);
            if (file.isFile() && file.exists()) { // 判斷文件是否存在
                InputStreamReader read = new InputStreamReader(
                        new FileInputStream(file), encoding);// 考慮到編碼格式
                BufferedReader bufferedReader = new BufferedReader(read);
                String lineTxt = null;
                while ((lineTxt = bufferedReader.readLine()) != null) {
                    System.out.println(lineTxt);
                }
                read.close();
            } else {
                System.out.println("找不到指定的文件");
            }
        } catch (Exception e) {
            System.out.println("讀取文件內容出錯");
            e.printStackTrace();
        }

    }

    public static void main(String[] args) {
        GeoGetDataThread dg = new GeoGetDataThread();
        dg.start();
    }

}

還有一段代碼：

這裏寫package com.iwant.download2geodata;

import java.io.BufferedReader;
import java.io.InputStreamReader;

import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;

import net.sf.json.JSONObject;

/**
* @Description:
* @author: whsgzcy
* @date: 2016-12-17 下午1:30:05
*
*/
public class HttpRequestUtil {

/**
* @Description:純get請求
* @author: whsgzcy
* @date: 2016-12-17 下午1:51:14  
* @param url
* @return
* JSONObject
* @throws
 */
public static JSONObject getJsonObject(String url) {
    JSONObject jsonObject = null;
    try {
        HttpClient httpClient = new DefaultHttpClient();
        HttpGet httpGet = new HttpGet(url);
        HttpParams httpParams = httpClient.getParams();
        HttpConnectionParams.setConnectionTimeout(httpParams, 5000);
        HttpResponse response = httpClient.execute(httpGet);
        StringBuilder builder = new StringBuilder();
        BufferedReader bufferedReader = new BufferedReader(
                new InputStreamReader(response.getEntity().getContent(),
                        "utf-8"));
        for (String s = bufferedReader.readLine(); s != null; s = bufferedReader
                .readLine()) {
            builder.append(s);
        }
        jsonObject = JSONObject.fromObject(builder.toString());
    } catch (Exception e) {
        e.printStackTrace();
        jsonObject = null;
    }
    return jsonObject;
}

}

序列化數據：

這裏package com.iwant.download2geodata.data;

import java.io.Serializable;
import java.util.List;

/**
* @Description:
* @author: whsgzcy
* @date: 2016-12-17 下午2:07:25
*
*/
public class ShopInfo implements Serializable{

private String name;
private String tel;
private String address;
private String longitude;//經度
private String latitude;//緯度
private List<TemplateData> templateData;// 圖片

public String getName() {
    return name;
}
public void setName(String name) {
    this.name = name;
}
public String getTel() {
    return tel;
}
public void setTel(String tel) {
    this.tel = tel;
}
public String getAddress() {
    return address;
}
public void setAddress(String address) {
    this.address = address;
}
public String getLongitude() {
    return longitude;
}
public void setLongitude(String longitude) {
    this.longitude = longitude;
}
public String getLatitude() {
    return latitude;
}
public void setLatitude(String latitude) {
    this.latitude = latitude;
}
public List<TemplateData> getTemplateData() {
    return templateData;
}
public void setTemplateData(List<TemplateData> templateData) {
    this.templateData = templateData;
}

} package com.iwant.download2geodata.data; import java.io.Serializable; import java.util.List; / * @Description：* @author：whsgzcy * @date：2016-12-17下午2:40:12 * * / public class ShopList implements Serializable { private List mShopInfoList; public List getmShopInfoList（）{ return mShopInfoList; } public void setmShopInfoList（List mShopInfoList）{ this.mShopInfoList = mShopInfoList; } } package com.iwant.download2geodata.data; import java.io.Serializable; / * @說明：

• @author: whsgzcy
• @date: 2016-12-17 下午2:20:13

• */
  public class TemplateData implements Serializable{

  private String pic_info;

  public String getPic_info() {
  return pic_info;
  }

  public void setPic_info(String pic_info) {
  this.pic_info = pic_info;
  }
  }

很簡單，懂java的應該都能看懂，在此提出一個設計思路，我代碼中屏蔽了一個睡眠線程，是防止我的IP被屏蔽，如果加入sleep，那麼，線程就會在這段時間執行主線程上的邏輯，我是一次性通過，就懶得加了，有興趣的同學可以將這塊加上，thanks a lot。