sparkstreaming實現hdfs目錄的監控，並實時寫入hbase

部署環境參考https://blog.csdn.net/luoye4321/article/details/99745877

代碼如下：

import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapred.TableOutputFormat
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.mapred.JobConf
import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.apache.spark.streaming.{Seconds, StreamingContext}

object HbaseOpe {
  def writeStreamToHbase(txtPath:String)={

    val sparkConf = new SparkConf().setAppName("streamToHbase")
    val ssc = new StreamingContext(sparkConf, Seconds(10))
    val lines = ssc.textFileStream(txtPath)

    lines.foreachRDD(txtRdd=>{
      val tablename= "tb:table3"
      val hbaseConf = HBaseConfiguration.create()
      hbaseConf.set(TableOutputFormat.OUTPUT_TABLE, tablename)
      val jobConf = new JobConf(hbaseConf)
      jobConf.setOutputFormat(classOf[TableOutputFormat])
      txtRdd.map(_.split(",")).map(arr=>{
        val put = new Put(Bytes.toBytes(arr(0)))
                
      put.addColumn(Bytes.toBytes("static"),Bytes.toBytes("col1"),Bytes.toBytes(arr(1)))
            
      put.addColumn(Bytes.toBytes("static"),Bytes.toBytes("col2"),Bytes.toBytes(arr(2)))
        
      put.addColumn(Bytes.toBytes("static"),Bytes.toBytes("col3"),Bytes.toBytes(arr(3)))
        
      put.addColumn(Bytes.toBytes("static"),Bytes.toBytes("col4"),Bytes.toBytes(arr(4)))
        
      put.addColumn(Bytes.toBytes("static"),Bytes.toBytes("col5"),Bytes.toBytes(arr(5)))
        
      put.addColumn(Bytes.toBytes("static"),Bytes.toBytes("col6"),Bytes.toBytes(arr(6)))
        
      put.addColumn(Bytes.toBytes("static"),Bytes.toBytes("col7"),Bytes.toBytes(arr(7)))
        (new ImmutableBytesWritable, put)
      }).saveAsHadoopDataset(jobConf)
    })

    ssc.start()
    ssc.awaitTermination()

  }
}