spark 2.2.0 WordCount scala版

安裝scala Ide for eclipse

導入spark/jar 下面的所有jar包

package cn.spark.study.core
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
object WordCount {
  def main(args : Array[String]){
    val conf =new SparkConf().setAppName("WordCountScala");
    val sc = new SparkContext(conf);
    val lines=sc.textFile("hdfs://172.16.2.235:9000/user/root/README.md")  
    val words = lines.flatMap(line => line.split(" "))  
    val pairs = words.map(word => (word,1))  
    val wordCounts = pairs.reduceByKey(_+_)  
    wordCounts.foreach(wordCount => println (wordCount._1 +"apperared" +wordCount._2 + " times."))
  }
}

export 生成jar 包導入集羣

寫一個腳本word.sh

/opt/spark/bin/spark-submit \

--master spark://172.16.2.235:7077 \

--class cn.spark.study.core.WordCount \

--num-executors 3 \

--driver-memory 1G \

--executor-memory 1G \

--executor-cores 1 \

/root/SparkScala/wordcount.jar\

運行腳本

結果在SparkUI stdout 中查看