使用反射的方式將RDD轉換爲DataFrame

Java：

package cn.spark.sql;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;

import java.util.List;

public class RDD2DataFrameReflection {
    public static void main(String[] args){
        SparkConf conf = new SparkConf().setAppName("RDD2DataFrameReflection")
        .setMaster("local");
        JavaSparkContext sc = new JavaSparkContext(conf);
        SQLContext sqlContext = new SQLContext(sc);

        JavaRDD<String> lines = sc.textFile("C://Users//Desktop//students.txt");

        JavaRDD<Student> students = lines.map(
                new Function<String, Student>() {
                    private static final long serialVersionUID = 1L;
            @Override
            public Student call(String s) throws Exception {
                String[] lineSplit = s.split(",");
                Student stu = new Student();
                stu.setID(Integer.valueOf(lineSplit[0].trim()));
                stu.setName(lineSplit[1]);
                stu.setAge(Integer.valueOf(lineSplit[2].trim()));
                return stu;


            }
        });

        /*
        * 使用反射方式 將RDD轉換爲DataFrame
        * 將Student.class 傳入進去 其實就是用反射的方式來創建DataFrame
        * 因爲底層通過Student class 進行反射 來獲取其中的field
        * 要求 javabean中需要實現serializable 接口
        * */
        DataFrame studentDF = sqlContext.createDataFrame(
                students,Student.class);
        //拿到一個DataFrame 後 就可以註冊一個臨時表
        studentDF.registerTempTable("students");
        DataFrame teengerDF = sqlContext.sql("select * from students where age <= 18");

        //將查詢出來的DataFrame 再次轉換成RDD
        JavaRDD<Row> teengerRDD = teengerDF.javaRDD();


        //將EDD中的數據 進行映射 映射爲student
        JavaRDD<Student> teengerStudentRDD = teengerRDD.map(new Function<Row, Student>() {
            @Override
            public Student call(Row row) throws Exception {
                Student stu = new Student();
                stu.setAge(row.getInt(0));
                stu.setID(row.getInt(1));
                stu.setName(row.getString(2));
                return stu;
            }
        });
        List<Student> studentList = teengerStudentRDD.collect();
        for (Student stu : studentList){
            System.out.println(stu);
        }
    }
}

測試：