關係型數據庫批量取數，fetchSize相關設置調研，mysql、pg、sqlserver、oracle

爲了支持對關係型數據庫的批量取數，防止內存溢出；調研了mysql、postgresql、sqlserver、oracle數據庫的fetchSize設置；

數據庫連接和處理的代碼大同小異，首先貼一份可運行代碼；

@Slf4j
public class TestFetchMysql {

    private Connection connection;

    private String driver;
    private String url;
    private String userName;
    private String password;

    public TestFetchMysql(String driver, String url, String userName, String password) {
        this.driver = driver;
        this.url = url;
        this.userName = userName;
        this.password = password;
    }

    private static void close(ResultSet rs, Statement st, PreparedStatement ps, Connection conn) {
        try {
            if (rs != null) rs.close();
            if (st != null) st.close();
            if (ps != null) ps.close();
            if (conn != null) conn.close();
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }

    private synchronized Connection getConnection(){
        try {
            if(connection == null || connection.isClosed()){
                Class.forName(driver);
                connection = DriverManager.getConnection(url, userName, password);
            }
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return connection;
    }

    public ArrayNode query(String query){
        Connection connection = getConnection();
        PreparedStatement preparedStatement = null;
        ResultSet resultSet = null;
        ArrayNode result = null;
        try {
            //preparedStatement = connection.prepareStatement(query, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
            preparedStatement = connection.prepareStatement(query);
            preparedStatement.setFetchSize(Integer.MIN_VALUE);
            //preparedStatement.setFetchDirection(ResultSet.FETCH_REVERSE);
            resultSet = preparedStatement.executeQuery();
            printMem();
            result = toJsonStr(resultSet);
            printMem();
        } catch (SQLException e) {
            e.printStackTrace();
        }finally {
            close(resultSet, null, preparedStatement, null);
        }
        log.info("查找sql [{}] {} 查詢結果 {}", query, System.getProperty("line.separator"), result);
        return result;
    }

    public static ArrayNode toJsonStr(ResultSet resultSet) throws SQLException {
        ObjectMapper objectMapper = new ObjectMapper();
        ArrayNode arrayNode = objectMapper.createArrayNode();

        ResultSetMetaData metaData = resultSet.getMetaData();
        int columnCount = metaData.getColumnCount();

        int count = 0;
        int sum = 0;
        while(resultSet.next()){
            count++;
            ObjectNode objectNode = objectMapper.createObjectNode();
            for(int i = 1; i <= columnCount; i++){
                String columnName  = metaData.getColumnLabel(i);
                String value = resultSet.getString(columnName);
                objectNode.put(columnName, value);
            }
            arrayNode.add(objectNode);
            if(count % 100 == 0){
                sum++;
                log.info("100個結果集 {}", sum);
                arrayNode = objectMapper.createArrayNode();
            }
        }
        return arrayNode;
    }

    public static void printMem(){
        log.info("totalMemory:{}, maxMemory:{}, freeMemory:{}", Runtime.getRuntime().maxMemory(), Runtime.getRuntime().freeMemory(),  Runtime.getRuntime().totalMemory());
    }

    public static void main(String[] args) {
       
        TestFetchMysql mysqlClient = new TestFetchMysql(DbType.getDriver("MYSQL") , "jdbc:mysql://127.0.0.1:3306/test_fetch", "test", "123456");

        ArrayNode query = mysqlClient.query("select * from test");
        log.info("打印下行數：" + query.size());
    }
}

DbType配置類

public enum DbType {

    SQLSERVER("SQLSERVER", "com.microsoft.sqlserver.jdbc.SQLServerDriver"),
    POSTGRESQL("POSTGRESQL", "org.postgresql.Driver"),
    MYSQL("MYSQL", "com.mysql.jdbc.Driver"),
    ORACLE("ORACLE", "oracle.jdbc.driver.OracleDriver");

    private String name;
    private String driver;

    DbType(String name, String driver) {
        this.name = name;
        this.driver = driver;
    }

    public static String getDriver(String name){
        for(DbType dbType : DbType.values()){
            if(dbType.getName().equals(name)){
                return dbType.getDriver();
            }
        }
        return name;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public String getDriver() {
        return driver;
    }

    public void setDriver(String driver) {
        this.driver = driver;
    }

    public static void main(String[] args) {
      //System.out.println( getDriver("POSTGRESQL") );
    }

測試是否生效手段

①設置ide運行參數-Xms8m -Xmx8m -Dfile.encoding=UTF-8

②printMem方法打印運行內存

③註釋toJsonStr方法的以下代碼，不生效內存溢出的會在執行sql得到resultSet的時刻，註釋掉會在拼接Json的時刻

        if(count % 100 == 0){
                sum++;
                log.info("100個結果集 {}", sum);
                arrayNode = objectMapper.createArrayNode();
            }

下面是結論，測試過程可以自己復現

1、Mysql fetchSize設置

mysql版本5.7，配置參數如下

    preparedStatement.setFetchSize(Integer.MIN_VALUE);

網上教程需要設置一下兩個參數，實際發現不需要

connection.prepareStatement(query, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
preparedStatement.setFetchDirection(ResultSet.FETCH_REVERSE);

必須設置 preparedStatement.setFetchSize(Integer.MIN_VALUE);  設置setFetchSize =20 不起作用會內存溢出

2、postgresql fetchSize設置

pg版本9.2.24，配置參數如下，設置自動提交爲false,不設置FetchSize的設置會無效；

fetchSize按照自己的需要設置大小，一般來說越大越快

connection.setAutoCommit(false);  
preparedStatement.setFetchSize(1000);

*     類似mysql的設置會報錯 報錯fetch size必須大於0 或者 操作要求可捲動的 ResultSet，但此 ResultSet 是 FORWARD_ONLY。
*     connection.prepareStatement(query, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
*     preparedStatement.setFetchDirection(ResultSet.FETCH_REVERSE);
*     preparedStatement.setFetchSize(Integer.MIN_VALUE)

3、SqlServer fetchSize設置

sqlserver版本2014 12.0，sqlserver可以設置fetchSize，但是fetchSize不生效，不過也不會內存溢出;

一次查詢大量數據不會內存溢出，好像一次只會查詢128條？底層待了解

4、Oracle fetchSize設置

oracle版本 11，只需要設置fetchSize即可，而且fetchSize生效，按需設置；

默認好像是一次拿10條數據，效率很慢；

preparedStatement.setFetchSize(1000);

ps：新建的表，查詢報找不到異常；所以使用程序創建了一張表，然後插入1w條數據做的測試。