使用KafkaUtil以direct方式獲取kafka的數據時遇到的問題

使用KafkaUtil以direct方式獲取kafka的數據時遇到的問題

報的是nio.channels.closeException類似的錯誤

原因是我的server.properties配置文件裏面

zookeeper的端口配錯了

正確的代碼是

package kafkaTest


import kafka.serializer.StringDecoder
import org.apache.spark.streaming.dstream.{DStream, InputDStream}
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by Administrator on 2018/5/13.
  */
object kafkaTest {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setAppName("kafkaTest").setMaster("local[2]")
    val sc = new SparkContext(conf)
    val scc = new StreamingContext(sc,Seconds(2))
    //checkpoint目錄
    scc.checkpoint("hdfs://hadoop02:9000/sparkStreaming")
    val TOPIC=Set("sparkstream")
    //kafkaUtils創建direct dstream
    var kafkaParams=Map("metadata.broker.list"->"hadoop02:9092")

    val dstream: DStream[String] = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](scc, kafkaParams, TOPIC)
      .map(_._2)
    dstream.flatMap(line=>line.split(",")).map(x=>(x,1)).reduceByKey(_+_).print()
    /**
      * 偏移量 數據 解碼器 解碼器
      * K: ClassTag,
    V: ClassTag,
    KD <: Decoder[K]: ClassTag,
    VD <: Decoder[V]: ClassTag
      *  ssc: StreamingContext,
      kafkaParams: Map[String, String],
      topics: Set[String]
      * */
    scc.start()
    scc.awaitTermination()
    scc.stop()
  }
}