大數據IMF傳奇行動絕密課程第96課：通過SparkStreaming的foreachRDD把處理後的數據寫入外部存儲系統中

通過SparkStreaming的foreachRDD把處理後的數據寫入外部存儲系統中

1、技術實現解析
2、實現實戰

關鍵部分代碼：

        resultRowRDD.foreachPartition( partitionOfRecords => {
          // ConnectionPool is a static, lazily initialized pool of connections

            val connection = ConnectionPool.getConnection()
            partitionOfRecords.foreach(record => {
              val sql = "insert into categorytop3(category, item, click_count) values ('" + record.getAs("category") + "','" +
                record.getAs("item") + "'," + record.getAs("click_count") + ")"
              val stmt = connection.createStatement()
              stmt.executeUpdate(sql)
            })
            ConnectionPool.returnConnection(connection)

        })

ConnectionPool.java

package com.tom.spark.sparkstreaming;

import java.sql.Connection;
import java.sql.DriverManager;
import java.util.LinkedList;

public class ConnectionPool {
    private static LinkedList<Connection> connectionQueue;

    static {
        try {
            Class.forName("com.mysql.jdbc.Driver");
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }
    }
    public synchronized static Connection getConnection() {
        try {
            if(connectionQueue == null) {
                connectionQueue = new LinkedList<Connection>();
                for(int i = 0; i < 5; i++) {
                    Connection conn = DriverManager.getConnection(
                            "jdbc:mysql://Master:3306/sparkstreaming",
                            "root",
                            "778899");
                    connectionQueue.push(conn);
                }
            }
        } catch(Exception e) {
            e.printStackTrace();
        }
        return connectionQueue.poll();
    }
    public static void returnConnection(Connection conn) {connectionQueue.push(conn);}
}