Flink入門示例——wordCount(二)

說明（批處理）

• 讀取HDFS數據
• 將處理結果寫入HDFS

代碼示例

package com.test

import org.apache.flink.api.scala._
import org.apache.flink.core.fs.FileSystem.WriteMode

/**
  * 讀取hdfs數據，把處理結果結果再寫入HDFS
  */
object WordCountBatch{

  /**
    * main函數傳遞的參數
    * hdfs://artemis-02:9000/tmp/lvxw/tmp/logs  hdfs://artemis-02:9000//tmp/lvxw/tmp/out
    * @param args
    */
  def main(args: Array[String]): Unit = {
    if(args.length!=2){
      println(s"${this.getClass.getSimpleName} must be two param:inputDir outputDir" )
      System.exit(1)
    }

    // 在window環境下，以hadoop身份遠程放完HDFS
    System.setProperty("HADOOP_USER_NAME","hadoop")
    val Array(inputDir,outputDir) = args

    val env = ExecutionEnvironment.getExecutionEnvironment

    val text = env.readTextFile(inputDir)

    val result = text.flatMap ( _.split("\\s"))
      .map ((_, 1))
      .groupBy(0)
      .sum(1)

    result.setParallelism(2).writeAsCsv(outputDir,"\n",",",WriteMode.OVERWRITE)
    env.execute(this.getClass.getSimpleName)

  }
}