上億級數據量-JDBC插入數據庫

轉載：http://blog.csdn.net/jzshmyt/article/details/7255761

 
package com.database;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
/**
 * 將億級數據量,通過JDBC方式,插入到數據庫中.關鍵技術說明
 *   一、爲了避免內存溢出問題,每100萬條重新建立一次jdbc連接.
 *   二、將連接屬性設置爲不自動提交,每5萬條批量執行一次提交.
 * @author Administrator
 */
public class Import {
	private static final String JDBC_URL = "jdbc:oracle:thin:@localhost:1521:orcl";
	private static final String JDBC_USER = "ttt";
	private static final String JDBC_PASSWORD = "pw123456";
	private static final String JDBC_SQL = "insert into tb_test (col1,col2) values (?,?)";
	
	public static void main(String[] args) throws Exception {
		Class.forName("oracle.jdbc.driver.OracleDriver");
		exe(100000005);//1億數據量測試通過
	}
	
	public static void exe(int sl) throws SQLException{
		int SIZE = 1000000;//每批的記錄數(建議值100萬)
		int page = sl/SIZE;//計算批數
		
		for (int i = 0; i < page; i++) {//批量執行
			execute(i*SIZE,(i+1)*SIZE);
		}
		if(sl%SIZE != 0){
			execute(page*SIZE,sl);
		}
	}
	
	private static void execute(int begin, int end) throws SQLException {
		Connection conn = null;
		PreparedStatement pst= null;
		try {
			conn = DriverManager.getConnection(JDBC_URL, JDBC_USER,JDBC_PASSWORD);
			System.out.println("重新打開連接 :" + begin + " - " + end);
			
			conn.setAutoCommit(false);// 設置不自動提交
			pst = conn.prepareStatement(JDBC_SQL);
			
			int recordNum = 0; // 計數器
			int commit_size = 50000;// 每次提交記錄數5萬
			for (int i = begin; i < end; i++) {
				recordNum++; // 計數
				pst.setString(1, "n" + i);
				pst.setString(2, "v" + i);
				pst.addBatch();
				if (recordNum % commit_size == 0) {
					pst.executeBatch();
					conn.commit();
					System.out.println("提交:" + i);
					conn.setAutoCommit(false);
					pst = conn.prepareStatement(JDBC_SQL);
				}
			}
			if (recordNum % commit_size != 0) {
				pst.executeBatch();
				conn.commit();
				System.out.println("提交:" + recordNum);
			}
		} catch (SQLException e) {
			e.printStackTrace();
		} finally {
			try {
				if (pst != null) pst.close();
			} catch (SQLException e) {
				e.printStackTrace();
			}finally{
				try{
					if (conn != null) conn.close();
					System.out.println("成功關閉連接！");
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}

}