Flume 1.8.0 + Spark Streaming 2.2.0 + Mysql 環境搭建

用到的軟件

---

    1、Hercules ：用來模擬發送UDP TCP數據給指定端口，官網下載即可

    2、Flume 1.8.0 下載，然後添加flume.conf配置文件即可，具體配置看上一篇https://blog.csdn.net/qq_22310551/article/details/89873681

    3、spark-streaming ，maven項目pom依賴如下

<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming -->
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-streaming_2.11</artifactId>
    <version>2.2.0</version>
    <scope>provided</scope>
</dependency>

<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-core_2.11</artifactId>
    <version>2.2.0</version>
</dependency>

<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-flume -->
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-streaming-flume_2.11</artifactId>
    <version>2.1.0</version>
</dependency>

<!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
<dependency>
    <groupId>mysql</groupId>
    <artifactId>mysql-connector-java</artifactId>
    <version>5.1.44</version>
</dependency>

<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-sql_2.11</artifactId>
    <version>2.2.0</version>
</dependency>

 

代碼實現 

---

package fri.fang.flume_streaming_demo;

import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.apache.log4j.Level;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.streaming.Duration;
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.streaming.flume.FlumeUtils;
import org.apache.spark.streaming.flume.SparkFlumeEvent;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.mysql.jdbc.Connection;

public final class JavaFlumeEventBatch {
	private static final String URL = "jdbc:mysql://localhost:3306/cityguard";
	private static final String USER = "root";
	private static final String PASS = "";

	static {
		// 註冊驅動
		try {
			Class.forName("com.mysql.jdbc.Driver");
		} catch (ClassNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}

	private static Connection getConnnection() throws Exception {
		// 獲取連接
		return (Connection) DriverManager.getConnection(URL, USER, PASS);
	}

	private static void closeAll(Connection conn, PreparedStatement carOnline, PreparedStatement carOffline,
			PreparedStatement idCard) {
		try {
			carOnline.close();
			carOffline.close();
			idCard.close();
		} catch (SQLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		try {
			conn.close();
		} catch (SQLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	private static void setPreparedStatement(Row row, PreparedStatement prep) {

		System.out.println("insert start ：" + row.mkString());
		// 在線 離線車
		// check_car(car_number,time_length,search_time,device_number,region_cn)
		if ("".equals(row.getString(5))) {
			try {
				prep.setString(1, row.getString(0));
				prep.setInt(2, row.getInt(1));
				prep.setTimestamp(3, row.getTimestamp(2));
				prep.setString(4, row.getString(3));
				prep.setString(5, row.getString(4));
			} catch (SQLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}

		} else {
			// 身份信息
			// car_number,time_length,search_time,device_number,region_cn,papers_number
			try {
				prep.setString(1, row.getString(0));
				prep.setInt(2, row.getInt(1));
				prep.setTimestamp(3, row.getTimestamp(2));
				prep.setString(4, row.getString(3));
				prep.setString(5, row.getString(4));
				prep.setString(6, row.getString(5));
			} catch (SQLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}

		}
		System.out.println("insert end ... ");
	}

	private JavaFlumeEventBatch() {
	}

	public static void main(String[] args) {
        //不顯示error以下的日誌信息
		org.apache.log4j.Logger.getLogger("org.apache.spark").setLevel(Level.ERROR);

		String host = "127.0.0.1";
		int port = 44444;
		// Duration batchInterval = new Duration(1000);
		SparkConf sparkConf = new SparkConf().setAppName("JavaFlumeEventCount");
		JavaStreamingContext ssc = new JavaStreamingContext(sparkConf.setMaster("local[2]"), Duration.apply(10000));
		JavaReceiverInputDStream<SparkFlumeEvent> flumeStream = FlumeUtils.createStream(ssc, host, port);

		flumeStream.foreachRDD(new VoidFunction<JavaRDD<SparkFlumeEvent>>() {

			@Override
			public void call(JavaRDD<SparkFlumeEvent> t) throws Exception {
				t.foreachPartition(new VoidFunction<Iterator<SparkFlumeEvent>>() {
					@Override
					public void call(Iterator<SparkFlumeEvent> t) throws Exception {
						Connection conn = getConnnection();
						PreparedStatement carCheckOffline = conn.prepareStatement(
								"insert into check_car(car_number,time_length,search_time,device_number,region_cn) values(?,?,?,?,?)");
						PreparedStatement carCheckOnline = conn.prepareStatement(
								"insert into check_car_online(car_number,time_length,search_time,device_number,region_cn) values(?,?,?,?,?)");
						PreparedStatement idCard = conn.prepareStatement(
								"insert into check_idcard(car_number,time_length,search_time,device_number,region_cn,papers_number) values(?,?,?,?,?,?)");
						boolean carCheckOfflineExist = false;
						boolean carCheckOnlineExist = false;
						boolean idCardExist = false;

						int count = 0;
						while (t.hasNext()) {
							count++;
							SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
							String temp = new String(t.next().event().getBody().array());
							JSONObject json = JSON.parseObject(temp);
							List<Object> values = new ArrayList<>();
							values.add(json.getString("car_number"));
							values.add(json.getIntValue("time_length"));
							values.add(Timestamp.valueOf(json.getString("search_time")));
							values.add(json.getString("device_number"));
							values.add(json.getString("region_cn"));
							if (json.getString("papers_number") != null) {
								values.add(json.getString("papers_number"));
							} else {
								values.add("");
							}
							if ("off_line".equals(json.getString("check_state"))) {
								carCheckOfflineExist = true;
								setPreparedStatement(RowFactory.create(values.toArray()), carCheckOffline);
								carCheckOffline.addBatch();
							} else if ("on_line".equals(json.getString("check_state"))) {
								carCheckOnlineExist = true;
								setPreparedStatement(RowFactory.create(values.toArray()), carCheckOnline);
								carCheckOnline.addBatch();
							} else {
								idCardExist = true;
								setPreparedStatement(RowFactory.create(values.toArray()), idCard);
								idCard.addBatch();
							}
						}
						System.out.println("本分片總共數據條數：" + count);
						if (carCheckOfflineExist) {
							System.out.println("本次離線查詢車輛批插入條數：" + carCheckOffline.executeBatch().length);
						}
						if (carCheckOnlineExist) {
							System.out.println("本次在線查詢車輛批插入條數：" + carCheckOnline.executeBatch().length);
						}
						if (idCardExist) {
							System.out.println("本次人員查詢批插入條數：" + idCard.executeBatch().length);
						}
						closeAll(conn, carCheckOnline, carCheckOffline, idCard);
					}
				});
			}
		});

		ssc.start();
		try {
			ssc.awaitTermination();
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
}

運行測試 

---

      運行Flume，運行java程序，Hercules發送udp數據，那麼就可以實現接收數據和處理數據。