spark （RDD 的持久化、排序、二次排序）

RDD 的持久化

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;

/**
 * RDD 的持久化
 */

public class Persion_9 {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf()
                .setAppName("Persist_9")
                .setMaster("local");

        JavaSparkContext sc =  new JavaSparkContext(conf);

        // cache()或者persist()的使用，是有規則的
        // 必須在transformation或者textFile等創建了一個RDD之後，直接連續調用cache()或persist()纔可以
        // 如果你先創建一個RDD，然後單獨另起一行執行cache()或persist()方法，是沒有用的
        // 而且，會報錯，大量的文件會丟失
        JavaRDD<String> lines = sc.textFile("D:\\eclipse\\wc\\scalaworid\\spark.txt")
                .cache();


        long beginTime = System.currentTimeMillis();
        long count = lines.count();
        System.out.println(count);
        long endTime = System.currentTimeMillis();
        System.out.println("消耗的時間" + (endTime - beginTime));


        beginTime = System.currentTimeMillis();
        count = lines.count();
        System.out.println(count);
        endTime = System.currentTimeMillis();
        System.out.println("消耗的時間" + (endTime - beginTime));
    }
}

1、對文本文件內的每個單詞都統計出其出現的次數。
2、按照每個單詞出現次數的數量，降序排序。

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.*;
import org.apache.spark.sql.sources.In;
import scala.Tuple2;

import java.util.Arrays;

/**
 * 1、對文本文件內的每個單詞都統計出其出現的次數。
 * 2、按照每個單詞出現次數的數量，降序排序.
 */
public class SortWorldCount {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf()
                .setAppName("SortWorldCount")
                .setMaster("local");

        JavaSparkContext sc = new JavaSparkContext(conf);

        JavaRDD<String> stringJavaRDD = sc.textFile("D:\\eclipse\\wc\\scalaworid\\spark.txt");

        JavaRDD<String> objectJavaRDD = stringJavaRDD.flatMap(new FlatMapFunction<String, String>() {
            @Override
            public Iterable<String> call(String s) throws Exception {
                return Arrays.asList(s.split(" "));
            }
        });
        JavaPairRDD<String, Integer> worldAndOne = objectJavaRDD.mapToPair(new PairFunction<String, String, Integer>() {
            @Override
            public Tuple2<String, Integer> call(String s) throws Exception {
                return new Tuple2<String, Integer>(s, 1);
            }
        });
        JavaPairRDD<String, Integer> stringIntegerJavaPairRDD = worldAndOne.reduceByKey(new Function2<Integer, Integer, Integer>() {
            @Override
            public Integer call(Integer integer, Integer integer2) throws Exception {
                return integer + integer2;
            }
        });
        //到這裏爲止就得到了每個單詞出現的次數，但是我們的新需求是，按照每個單詞出現的次數降序排列
        //wAndC DRR內的元素是什麼？應該是這樣的格式：(hello，2)(word，3)
        //我們需要將RDD中轉換爲(3,word)的格式，才能根據單詞出現的次數進行排序
        // 進行key-value的反轉

        JavaPairRDD<Integer, String> pairRDD = stringIntegerJavaPairRDD.mapToPair(new PairFunction<Tuple2<String, Integer>, Integer, String>() {
            @Override
            public Tuple2<Integer, String> call(Tuple2<String, Integer> tuple2) throws Exception {
                return new Tuple2<Integer, String>(tuple2._2,tuple2._1);
            }
        });

        //按照key排序
        JavaPairRDD<Integer, String> sortByKey = pairRDD.sortByKey(false);
        sortByKey.foreach(new VoidFunction<Tuple2<Integer, String>>() {
            @Override
            public void call(Tuple2<Integer, String> tuple2) throws Exception {
                System.out.println(tuple2._1 + " : "+ tuple2._2);
            }
        });
    }
}

1、按照文件中的第一列排序。
2、如果第一列相同，則按照第二列排序。

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import scala.Int;
import scala.Tuple2;

/**
 * 二次排序
 * 1.實現自定義的key，要實現Ordered接口和Serializable接口，在key自己對多個列的排序算法
 * 2.將包含文本的RDD，映射稱key爲自定義key，value爲文本的JavaPairRDD
 * 3.使用sortedByKey算子，按照自定義的key進行排列
 * 4.再次映射，剔除自定義的key，只保留value元文本行
 *
 */
public class SecondarySort {

    public static void main(String[] args) {
        SparkConf conf = new SparkConf()
                .setMaster("local")
                .setAppName("SecondarySort_12");

        JavaSparkContext sc = new JavaSparkContext(conf);

        JavaRDD<String> lines = sc.textFile("D:\\eclipse\\wc\\input\\sort.txt");

        /**
         * 1 5
         * 2 4
         * 3 6
         * 1 3
         * 2 1
         */
        JavaPairRDD<SecondarySortKey, String> pair = lines.mapToPair(new PairFunction<String, SecondarySortKey, String>() {
            @Override
            public Tuple2<SecondarySortKey, String> call(String s) throws Exception {
                String[] split = s.split(" ");
                SecondarySortKey key = new SecondarySortKey(Integer.parseInt(split[0]), Integer.parseInt(split[1]));
                //(1,5:1 5)
                //(2,4:2 4)
                //(3,6:3 6)
                //(1,3:1 3)
                //(2,1:2 1)
                return new Tuple2<SecondarySortKey, String>(key, s);
            }
        });


        //(1,3:1 3)
        //(1,5:1 5)
        //(2,1:2 1)
        //(2,4:2 4)
        //(3,6:3 6)
        JavaPairRDD<SecondarySortKey, String> sorted = pair.sortByKey();

        final JavaRDD<String> sorteded = sorted.map(new Function<Tuple2<SecondarySortKey, String>, String>() {
            @Override
            public String call(Tuple2<SecondarySortKey, String> tuple2) throws Exception {
                return tuple2._2;
            }
        });

        sorteded.foreach(new VoidFunction<String>() {
            @Override
            public void call(String s) throws Exception {
                System.out.println(s);
            }
        });
        sc.close();
    }
}

SecondarySortKey類

import scala.math.Ordered;

import java.io.Serializable;

public class SecondarySortKey implements Ordered<SecondarySortKey>, Serializable {

    //首先在自定義的key裏面，定義需要進行排序的列
    private int first;
    private int second;

    // 爲要進行排序的多個列提供getter和setter方法，以及hashcode和equals和構造方法方法
    public SecondarySortKey(int first, int second) {
        this.first = first;
        this.second = second;
    }

    public int getFirst() {
        return first;
    }

    public int getSecond() {
        return second;
    }

    public void setFirst(int first) {
        this.first = first;
    }

    public void setSecond(int second) {
        this.second = second;
    }


    @Override
    public int compare(SecondarySortKey that) {

        if(this.getFirst() - that.getFirst() !=0){
            return this.getFirst() - that.getFirst();
        }else
            return this.getSecond() - that.getSecond();
    }

    //小於
    @Override
    public boolean $less(SecondarySortKey that) {
        if(this.getFirst() < that.getFirst()){
            return true;
        }else if (this.getFirst() == that.getFirst() && that.getSecond() < that.getSecond()){
            return true;
        }
        return false;
    }

    //大於
    @Override
    public boolean $greater(SecondarySortKey that) {
        if(this.getFirst() > that.getFirst()){
            return true;
        }else if (this.getFirst() == that.getFirst() && this.getSecond() > that.getSecond()){
            return true;
        }
        return false;
    }

    //小於等於
    @Override
    public boolean $less$eq(SecondarySortKey that) {
        if(this.getFirst() < that.getFirst()){
            return true;
        }else if (this.getFirst() == that.getFirst() && that.getSecond() == that.getSecond()){
            return true;
        }
        return false;
    }

    //大於等於
    @Override
    public boolean $greater$eq(SecondarySortKey that) {
        if(this.getFirst() > (that.getFirst())){
            return true;
        }else if (this.first == that.getFirst() && this.second == this.getSecond()){
                return true;
        }
        return false;
    }

    @Override
    public int compareTo(SecondarySortKey that) {
       if (this.getFirst() - that.getFirst() !=0){
           return this.getFirst() - that.getFirst();
       }else
           return this.getSecond() - that.getSecond();
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) return true;
        if (o == null || getClass() != o.getClass()) return false;

        SecondarySortKey that = (SecondarySortKey) o;

        if (first != that.first) return false;
        return second == that.second;
    }

    @Override
    public int hashCode() {
        int result = first;
        result = 31 * result + second;
        return result;
    }
}