Java實現mongoDB的數據導入TIDB

Java實現mongoDB的數據導入TIDB

目錄

用 [TOC]來生成目錄：

一、mongoDB的優缺點：

優點：
- 面向文檔存儲(類json數據模式簡單而強大)
- 動態查詢
- 全索引支持，擴展到內部對象和內嵌數據
- MapResuce支持複雜聚合
- 高效存儲二進制大對象（比如照片和視頻）
- 查詢記錄分析
- 內置GridFS，支持大容量的存儲
- 複製和故障切換支持
- Auto-Sharding自動分片支持雲級擴展性
缺點：
- MongoDB沒有如Mysql那樣成熟的維護工具
- 不支持事務
- MongoDB佔用空間過大
- 無法進行關聯表查詢，不適用於關係多的數據
- 複雜聚合操作通過MapReduce創建，速度慢
- 模式自由，自由靈活的文件存儲格式帶來的數據錯誤

---

二、TiDB的核心特性：

• SQL支持（TiDB是mysql兼容的）
• 水平線性彈性擴展
• 分佈式事務
• 跨數據中心數據強一致性保證
• 故障自恢復的高可用

---

程序實現兩個數據庫的數據的轉移

該程序是將mongoDB的數據庫中一個表的數據轉移到TIDB

代碼塊

DBConnet類-用來實現mongodb的連接，例如：

import com.mongodb.MongoClient;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import org.bson.Document;
import java.sql.SQLException;
import java.text.ParseException;
import static com.mongodb.client.model.Filters.*;


public class DBConnet {

    //獲取今天開始的毫秒數
    public long starttime = 1496246400;
    //獲取今天結束的毫秒數
    public  long endtime= starttime + 86400;
    //查詢結束的時間
    public long end =1498838400;

    public void Connet() throws SQLException, ParseException {
        //mongoDB的相關連接操作
        MongoClient mongoClient = new MongoClient("localhost", 27017);
        MongoDatabase database = mongoClient.getDatabase("test");
        MongoCollection<Document> track = database.getCollection("test");
        //mysql的連接
        DBHelper db = new DBHelper();
        //獲取系統當前的毫秒數

        while(endtime <= end) {
            MongoCursor<Document> mongoCursor = find(track);

                while (mongoCursor.hasNext()) {
                    Document t = mongoCursor.next();
                    try {
                        db.mosaic(t);
                    } catch (SQLException e) {
                        e.printStackTrace();
                    }
                }


        }
        db.close();
    }


    public  MongoCursor<Document> find(MongoCollection<Document> table) {

        //FindIterable<Document> t = table.find().skip(skip).limit(limit).batchSize(30);
        Document sortKey = new Document("rtime", 1);
        FindIterable<Document> t = table.find(and(gte("rtime", starttime), lte("rtime", endtime))).sort(sortKey);
        MongoCursor<Document> mongoCursor = t.iterator();
        starttime = starttime + 86400;
        endtime = endtime + 86400;
        return mongoCursor;
    }
}

DBHelper類 –實現mysql的連接，例如：

import org.bson.Document;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;


public class DBHelper {
    public static final String url = "jdbc:mysql://127.0.0.1:3306/test";
    public static final String name = "com.mysql.jdbc.Driver";
    public static final String user = "root";
    public static final String password = "";

    public Connection conn = null;
    public PreparedStatement pst = null;
    public int COUNT =1;
    public String prefix ="insert into test (Id,field1,field2,field3,field4,field5,field6,field7,field8,field9,field10,field11,field12) values ";
    public StringBuilder suffix = new StringBuilder();
    public String deletesql ="delete from wifinew where rtime < 1498838400 limit 10000";


    public DBHelper() {
        try {

            Class.forName(name);//指定連接類型
            conn = DriverManager.getConnection(url, user, password);//獲取連接
            conn.setAutoCommit(false);
            pst = conn.prepareStatement("");//準備執行語句
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    //批量添加的功能
    public void mosaic(Document p) throws SQLException {
        if ( COUNT <= 100000 ) {
            suffix.append("('").append(p.getString("Id")).append("',"
                    ).append(String.valueOf(p.getDouble("field1")) ).append(","
                    ).append(String.valueOf(p.getDouble("field2")) ).append(","
                    ).append("'"+p.getString("field3")+"',"
                    ).append(p.getInteger("field4") ).append(","
                    ).append(p.getInteger("field5") ).append(","
                    ).append(p.getInteger("field6") ).append(","
                    ).append("'" ).append(p.getString("field7")    ).append("',"
                    ).append("'" ).append(p.getString("field8")).append("',"
                    ).append(p.getLong("field9") ).append(","
                    ).append(p.getLong("field10") ).append(","
                    ).append(p.getLong("field11") ).append(","
                    ).append(p.getInteger("field12")
                    ).append("),");


            if (COUNT % 10000 == 0) {
                String sql = prefix  + suffix.substring(0, suffix.length() - 1);
                pst.addBatch(sql);
                pst.executeBatch();
                conn.commit();
                suffix = new StringBuilder();
            }
            COUNT++;
        }else {
            COUNT = 0;
        }
    }
    //批量刪除的功能
    public void delete() throws SQLException {
        pst.addBatch(deletesql);
        int[] result =pst.executeBatch();
        conn.commit();
        if (result.length >= 0){
            delete();
        }else{
            close();
            return;
        }

    }

    public void close() {
        try {
            this.conn.close();
            this.pst.close();
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }
}

test類 – 程序的主入口，例如：

import java.sql.SQLException;
import java.text.ParseException;

public class test {
    public static void main(String[] args) throws SQLException {
        //批量刪除數據庫
        delete();
        //add();


    }

    public static void add(){
        //連接到Db
        DBConnet DBConnet = new DBConnet();
        try {
            DBConnet.Connet();
        } catch (SQLException e) {
            e.printStackTrace();
        } catch (ParseException e) {
            e.printStackTrace();
        }
    }

    public static void delete() throws SQLException {
        DBHelper db = new DBHelper();
        try {
            db.delete();
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }

}

pom.xml的配置文檔，例如：

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>carl</groupId>
    <artifactId>xlx2_2biaoqian</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <mysql.version>5.1.24</mysql.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>1.2.17</version>
        </dependency>

        <dependency>
            <groupId>org.mongodb</groupId>
            <artifactId>mongo-java-driver</artifactId>
            <version>3.2.0</version>
        </dependency>

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>${mysql.version}</version>
        </dependency>

    </dependencies>

</project>