输出操作指定了对流数据经转化操作得到的数据所要执行的操作(例如把结果推入外部数据库或输出到屏幕上)。与 RDD 中的惰性求值类似,如果一个 DStream 及其派生出的 DStream 都没有被执行输出操作,那么这些 DStream 就都不会被求值。如果StreamingContext 中没有设定输出操作,整个 context 就都不会启动。
package com.ljpbd.bigdata.spark.streaming import com.ljpbd.bigdata.spark.Util.JdbcUtil import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord} import org.apache.spark.SparkConf import org.apache.spark.rdd.RDD import org.apache.spark.streaming.dstream.{DStream, InputDStream} import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies} import org.apache.spark.streaming.{Seconds, StreamingContext} import java.sql.{Connection, PreparedStatement, ResultSet} import java.text.SimpleDateFormat import java.util.Date import scala.collection.mutable.ListBuffer object SparkStreaming11_Req1BlockList { def main(args: Array[String]): Unit = { //1.创建 SparkConf val sparkConf: SparkConf = new SparkConf().setAppName("ReceiverWordCount").setMaster("local[*]") //2.创建 StreamingContext val ssc = new StreamingContext(sparkConf, Seconds(3)) //3.定义 Kafka 参数 val kafkaPara: Map[String, Object] = Map[String, Object]( ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "linux1:9092,linux2:9092,linux3:9092", ConsumerConfig.GROUP_ID_CONFIG -> "atguigu", "key.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer", "value.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer" ) //4.读取 Kafka 数据创建 DStream val kafkaDStream: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream[String, String](ssc, LocationStrategies.PreferConsistent, ConsumerStrategies.Subscribe[String, String](Set("atguigu"), kafkaPara)) //5.将每条消息的 KV 取出 val adClickData: DStream[AdClickData] = kafkaDStream.map( kafkaData => { val data: String = kafkaData.value() val datas: Array[String] = data.split(" ") AdClickData(datas(0), datas(1), datas(2), datas(3), datas(4)) } ) //周期性获取黑名单数据 //判断点击用户是否在黑名单中 //如果用户不在黑名单中,那么进行统计数量(每个采集周期) val ds: DStream[((String, String, String), Int)] = adClickData.transform( rdd => { //通过jdbc周期性获取黑名单数据 val blackList = ListBuffer[String]() val connection: Connection = JdbcUtil.getConnection val pstat: PreparedStatement = connection.prepareStatement("select userid from black_list") val rs: ResultSet = pstat.executeQuery() while (rs.next()) { blackList.append(rs.getString(1)) } rs.close() pstat.close() connection.close() val filterRdd: RDD[AdClickData] = rdd.filter( data => { //判断点击用户是不是在黑名单中 !blackList.contains(data.user) } ) //如果用户不在黑名单中,那么进行统计数量(每个采集周期) filterRdd.map( data => { val sdf = new SimpleDateFormat("yyyy-MM-dd") val day = sdf.format(new Date(data.ts)) val user = data.user val ad = data.ad ((day, user, ad), 1) } ).reduceByKey(_ + _) } ) ds.foreachRDD( rdd => { //rdd.foreach会每一条数据创建连接 /* foreach是rdd的算子,算子之外的代码是在driver端执行,算子之内的代码是在executor执行 ,可以将对象从driver传输到executor,这样就会涉及到闭包操作,需要将数据序列化 但是数据库连接对象是不能序列化的 val conn: Connection = JdbcUtil.getConnection rdd提供了一个算子可以有效提供效率,foreachPartition 可以一个分区创建一个连接对象,这样就可以大幅度减少连接对象的创建 */ /* rdd.foreachPartition( iter=>{ val conn: Connection = JdbcUtil.getConnection conn.close() } )*/ rdd.foreach { case ((day, user, ad), count) => { println(s"${day} ${user} ${ad} ${count}") if (count >= 30) { //如果统计数量超过点击域值,那么将用户拉入到黑名单中 val conn: Connection = JdbcUtil.getConnection var sql = """ |insert into black_list(userid) values(?) |on DUPLICATE KEY |UPDATE userid = ? |""".stripMargin JdbcUtil.executeUpdate(conn, sql, Array(user, user)) conn.close() } else { //如果没有超过域值,那么需要将当天的广告点击数量进行更新, val conn: Connection = JdbcUtil.getConnection val sql = """ |select * from user_ad_count where dt=? and userid=? and adid=? |""".stripMargin //查询统计表数据 var flag = JdbcUtil.isExist(conn, sql, Array(day, user, ad)) //如果存在数据,则更新 if (flag) { val sql1 = """ |update user_ad_count |set count=count+? |where dt=? and userid=? and adid=? |""".stripMargin JdbcUtil.executeUpdate(conn, sql1, Array(count, day, user, ad)) //判断更新后的点击数量是否超过域值,如果超过,那么将用户拉入到黑名单中 val sql2 = """ |select * from user_ad_count |where dt=? and userid=? and adid=? and count>=30 |""".stripMargin var flag1 = JdbcUtil.isExist(conn, sql2, Array(day, user, ad)) if (flag1) { val sql3 = """ |insert into black_list(userid) values(?) |on DUPLICATE KEY |UPDATE userid = ? |""".stripMargin JdbcUtil.executeUpdate(conn, sql3, Array(user, user)) } } else { //如果不存在数据,那么新增 val sql4 = """ |insert into user_ad_count(dt,userid,adid,count) values(?,?,?,?) |""".stripMargin JdbcUtil.executeUpdate(conn, sql4, Array(day, user, ad, count)) } conn.close() } } } } ) //7.开启任务 ssc.start() ssc.awaitTermination() } //广告点击数据 case class AdClickData(ts: String, area: String, city: String, user: String, ad: String) }