從網易獲取股票每日交易數據的爬蟲程序

注意由於網易限制,此爬蟲對於連續批量取值不合適.

爬蟲類:

package com.ufo.hy.agumaster.crawler.daytransact;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;

import com.ufo.hy.agumaster.entity.DayTransact;

public class NeteaseDTCrawler {
    private List<DayTransact> dtList;
    
    public List<DayTransact> getDtList(){
        return dtList;
    }
    
    public void download(String originalCode,String name,String fromDate,String toDate) {
        try {
            String code="";
            if(originalCode.startsWith("6")) {
                code="0"+originalCode;
            }else {
                code="1"+originalCode;
            }
            
            String raw="http://quotes.money.163.com/service/chddata.html?code={0}&start={1}&end={2}&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP";
            Object[] arr={code,fromDate,toDate};
            String urlPath = MessageFormat.format(raw, arr);

            URL url = new URL(urlPath);
            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
            connection.setRequestMethod("GET");
            connection.connect();
            int responseCode = connection.getResponseCode();
            if (responseCode == HttpURLConnection.HTTP_OK) {
                //InputStream is = connection.getInputStream();
                
                //BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
                
                dtList=new ArrayList<>();
                try(InputStream is = connection.getInputStream();
                    BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"));){
                    String line = null;  
                    int idx=0;
                    while( ( line = br.readLine() ) != null ) {
                        if(idx>0) {
                            String[] arr4Dt=line.split(",");
                            DayTransact dt=new DayTransact(arr4Dt);
                            dt.setCode(originalCode);
                            dt.setName(name);
                            
                            dtList.add(dt);
                        }
                        idx++;
                    }
                }
            }

        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    
    public static void main(String[] args) {
        NeteaseDTCrawler n=new NeteaseDTCrawler();
        n.download("002101","廣東鴻圖", "20200401", "20200410");
        
        for(DayTransact dt:n.getDtList()) {
            System.out.println(dt);
        }
    }
}

實體類:

package com.ufo.hy.agumaster.entity;

/**
 * 每日交易數據實體類
 * @author ufo
 *
 */
public class DayTransact {
    private long    id;            // ID
    private String  day;        // 日期
    private String  code;        // 代號
    private String  name;        // 名稱
    private double  tclose;        // 收盤價
    private double  high;        // 最高價
    private double  low;        // 最低價
    private double  topen;        // 開盤價
    private double  lclose;        // 前日收盤價
    private double  chg;        // 漲跌額
    private double  pchg;        // 漲跌幅
    private double  turnover;    // 換手率
    private long    voturnover;    // 成交量
    private double  vaturnover;    // 成交金額
    private double  tcap;        // 總市值
    private double  mcap;        // 流通市值
    
    public DayTransact() {
        
    }
    
    public DayTransact(String[] arr) {
        if(arr.length!=15) {
            throw new ArrayIndexOutOfBoundsException("Array size should be 15 but now it is "+arr.length);
        }
        
        day=arr[0];

        try {
            tclose=Double.parseDouble(arr[3]);
        }catch(NumberFormatException ex) {
            throw new NumberFormatException("Can not get tclose from string:"+arr[3]);
        }
        
        try {
            high=Double.parseDouble(arr[4]);
        }catch(NumberFormatException ex) {
            throw new NumberFormatException("Can not get high from string:"+arr[4]);
        }
        
        try {
            low=Double.parseDouble(arr[5]);
        }catch(NumberFormatException ex) {
            throw new NumberFormatException("Can not get low from string:"+arr[5]);
        }
        
        try {
            topen=Double.parseDouble(arr[6]);
        }catch(NumberFormatException ex) {
            throw new NumberFormatException("Can not get topen from string:"+arr[6]);
        }
        
        try {
            lclose=Double.parseDouble(arr[7]);
        }catch(NumberFormatException ex) {
            throw new NumberFormatException("Can not get lclose from string:"+arr[7]);
        }
    
        try {
            chg=Double.parseDouble(arr[8]);
        }catch(NumberFormatException ex) {
            throw new NumberFormatException("Can not get chg from string:"+arr[8]);
        }
        
        try {
            pchg=Double.parseDouble(arr[9]);
        }catch(NumberFormatException ex) {
            throw new NumberFormatException("Can not get pchg from string:"+arr[9]);
        }
        
        try {
            turnover=Double.parseDouble(arr[10]);
        }catch(NumberFormatException ex) {
            throw new NumberFormatException("Can not get turnover from string:"+arr[10]);
        }
        
        try {
            voturnover=Long.parseLong(arr[11]);
        }catch(NumberFormatException ex) {
            throw new NumberFormatException("Can not get voturnover from string:"+arr[11]);
        }
        
        try {
            vaturnover=Double.parseDouble(arr[12]);
        }catch(NumberFormatException ex) {
            throw new NumberFormatException("Can not get vaturnover from string:"+arr[12]);
        }
        
        try {
            tcap=Double.parseDouble(arr[13]);
        }catch(NumberFormatException ex) {
            throw new NumberFormatException("Can not get tcap from string:"+arr[13]);
        }
        
        try {
            mcap=Double.parseDouble(arr[14]);
        }catch(NumberFormatException ex) {
            throw new NumberFormatException("Can not get mcap from string:"+arr[14]);
        }
    
    }
    
    public String toString() {
        StringBuilder sb=new StringBuilder();
        sb.append("id:"+id);
        sb.append(" 日期day:"+day);
        sb.append(" 代號code:"+code);
        sb.append(" 名稱name:"+name);
        sb.append(" 收盤價tclose:"+tclose);
        sb.append(" 最高價high:"+high);
        sb.append(" 最低價low:"+low);
        sb.append(" 開盤價topen:"+topen);
        sb.append(" 前日收盤價lclose:"+lclose);
        sb.append(" 漲跌額chg:"+chg);
        sb.append(" 漲跌幅pchg:"+pchg);
        sb.append(" 換手率turnover:"+turnover);
        sb.append(" 成交量voturnover:"+voturnover);
        sb.append(" 成交金額vaturnover:"+vaturnover);
        sb.append(" 總市值tcap:"+tcap);
        sb.append(" 流通市值mcap:"+mcap);
        
        return sb.toString();//"code:"+code+" name:"+name+" date:"+day+" tclose:"+tclose;
    }
    
    public long getId() {
        return id;
    }
    public void setId(long id) {
        this.id = id;
    }
    public String getDay() {
        return day;
    }
    public void setDay(String day) {
        this.day = day;
    }
    public String getCode() {
        return code;
    }
    public void setCode(String code) {
        this.code = code;
    }
    public String getName() {
        return name;
    }
    public void setName(String name) {
        this.name = name;
    }
    public double getTclose() {
        return tclose;
    }
    public void setTclose(double tclose) {
        this.tclose = tclose;
    }
    public double getHigh() {
        return high;
    }
    public void setHigh(double high) {
        this.high = high;
    }
    public double getLow() {
        return low;
    }
    public void setLow(double low) {
        this.low = low;
    }
    public double getTopen() {
        return topen;
    }
    public void setTopen(double topen) {
        this.topen = topen;
    }
    public double getLclose() {
        return lclose;
    }
    public void setLclose(double lclose) {
        this.lclose = lclose;
    }
    public double getChg() {
        return chg;
    }
    public void setChg(double chg) {
        this.chg = chg;
    }
    public double getPchg() {
        return pchg;
    }
    public void setPchg(double pchg) {
        this.pchg = pchg;
    }
    public double getTurnover() {
        return turnover;
    }
    public void setTurnover(double turnover) {
        this.turnover = turnover;
    }
    public long getVoturnover() {
        return voturnover;
    }
    public void setVoturnover(long voturnover) {
        this.voturnover = voturnover;
    }
    public double getVaturnover() {
        return vaturnover;
    }
    public void setVaturnover(double vaturnover) {
        this.vaturnover = vaturnover;
    }
    public double getTcap() {
        return tcap;
    }
    public void setTcap(double tcap) {
        this.tcap = tcap;
    }
    public double getMcap() {
        return mcap;
    }
    public void setMcap(double mcap) {
        this.mcap = mcap;
    }
}

運行結果:

id:0 日期day:2020-04-10 代號code:002101 名稱name:廣東鴻圖 收盤價tclose:7.95 最高價high:8.23 最低價low:7.81 開盤價topen:8.11 前日收盤價lclose:8.1 漲跌額chg:-0.15 漲跌幅pchg:-1.8519 換手率turnover:2.201 成交量voturnover:9367912 成交金額vaturnover:7.500989349E7 總市值tcap:4.2322887597E9 流通市值mcap:3.38368519305E9
id:0 日期day:2020-04-09 代號code:002101 名稱name:廣東鴻圖 收盤價tclose:8.1 最高價high:8.18 最低價low:7.94 開盤價topen:8.13 前日收盤價lclose:8.1 漲跌額chg:0.0 漲跌幅pchg:0.0 換手率turnover:2.7466 成交量voturnover:11690195 成交金額vaturnover:9.441648164E7 總市值tcap:4.3121432646E9 流通市值mcap:3.4475283099E9
id:0 日期day:2020-04-08 代號code:002101 名稱name:廣東鴻圖 收盤價tclose:8.1 最高價high:8.27 最低價low:8.03 開盤價topen:8.06 前日收盤價lclose:8.23 漲跌額chg:-0.13 漲跌幅pchg:-1.5796 換手率turnover:2.7482 成交量voturnover:11697086 成交金額vaturnover:9.499666166E7 總市值tcap:4.3121432646E9 流通市值mcap:3.4475283099E9
id:0 日期day:2020-04-07 代號code:002101 名稱name:廣東鴻圖 收盤價tclose:8.23 最高價high:8.28 最低價low:7.9 開盤價topen:8.04 前日收盤價lclose:7.9 漲跌額chg:0.33 漲跌幅pchg:4.1772 換手率turnover:3.7546 成交量voturnover:15980408 成交金額vaturnover:1.293773521E8 總市值tcap:4.38135050218E9 流通市值mcap:3.50285901117E9
id:0 日期day:2020-04-03 代號code:002101 名稱name:廣東鴻圖 收盤價tclose:7.9 最高價high:8.11 最低價low:7.82 開盤價topen:8.11 前日收盤價lclose:8.2 漲跌額chg:-0.3 漲跌幅pchg:-3.6585 換手率turnover:3.2445 成交量voturnover:13809146 成交金額vaturnover:1.0978952972E8 總市值tcap:4.2056705914E9 流通市值mcap:3.3624041541E9
id:0 日期day:2020-04-02 代號code:002101 名稱name:廣東鴻圖 收盤價tclose:8.2 最高價high:8.2 最低價low:7.58 開盤價topen:7.7 前日收盤價lclose:7.75 漲跌額chg:0.45 漲跌幅pchg:5.8065 換手率turnover:4.5431 成交量voturnover:19336424 成交金額vaturnover:1.5326836152E8 總市值tcap:4.3653796012E9 流通市值mcap:3.4900903878E9
id:0 日期day:2020-04-01 代號code:002101 名稱name:廣東鴻圖 收盤價tclose:7.75 最高價high:8.07 最低價low:7.62 開盤價topen:7.62 前日收盤價lclose:7.44 漲跌額chg:0.31 漲跌幅pchg:4.1667 換手率turnover:3.703 成交量voturnover:15760812 成交金額vaturnover:1.2279217002E8 總市值tcap:4.1258160865E9 流通市值mcap:3.29856103725E9

就到這裏,有緣人儘管拿去用,如果覺得不錯就念聲好就行.

--2020年5月7日--