MapReduce--14--學生成績（增強版）--需求3

題目描述

關於對於學生成績相關的練習題，之前是一個入門級別的需求，現在對這些需求進行增強，首先看數據的改變：

computer,huangxiaoming,85,86,41,75,93,42,85
computer,xuzheng,54,52,86,91,42
computer,huangbo,85,42,96,38
english,zhaobenshan,54,52,86,91,42,85,75
english,liuyifei,85,41,75,21,85,96,14
algorithm,liuyifei,75,85,62,48,54,96,15
computer,huangjiaju,85,75,86,85,85
english,liuyifei,76,95,86,74,68,74,48
english,huangdatou,48,58,67,86,15,33,85
algorithm,huanglei,76,95,86,74,68,74,48
algorithm,huangjiaju,85,75,86,85,85,74,86
computer,huangdatou,48,58,67,86,15,33,85
english,zhouqi,85,86,41,75,93,42,85,75,55,47,22
english,huangbo,85,42,96,38,55,47,22
algorithm,liutao,85,75,85,99,66
computer,huangzitao,85,86,41,75,93,42,85
math,wangbaoqiang,85,86,41,75,93,42,85
computer,liujialing,85,41,75,21,85,96,14,74,86
computer,liuyifei,75,85,62,48,54,96,15
computer,liutao,85,75,85,99,66,88,75,91
computer,huanglei,76,95,86,74,68,74,48
english,liujialing,75,85,62,48,54,96,15
math,huanglei,76,95,86,74,68,74,48
math,huangjiaju,85,75,86,85,85,74,86
math,liutao,48,58,67,86,15,33,85
english,huanglei,85,75,85,99,66,88,75,91
math,xuzheng,54,52,86,91,42,85,75
math,huangxiaoming,85,75,85,99,66,88,75,91
math,liujialing,85,86,41,75,93,42,85,75
english,huangxiaoming,85,86,41,75,93,42,85
algorithm,huangdatou,48,58,67,86,15,33,85
algorithm,huangzitao,85,86,41,75,93,42,85,75

一、數據解釋

數據字段個數不固定：
第一個是課程名稱，總共四個課程，computer，math，english，algorithm，
第二個是學生姓名，後面是每次考試的分數

 

二、統計需求：

1、統計每門課程的參考人數和課程平均分

2、統計每門課程參考學生的平均分，並且按課程存入不同的結果文件，要求一門課程一個結果文件，並且按平均分從高到低排序，分數保留一位小數

3、求出每門課程參考學生成績最高的2個學生的信息：課程，姓名和平均分

 

三、解題思路

mapper階段的輸出：

key： CourseScore

value： NullWritable

reducer階段的輸出：

key： CourseScore

value：NullWritable

實現難點：

                分組條件（課程） 和 排序規則（課程，成績）不一致，所以需要自定義分組

自定義分組的代碼 CourseScoreGroupComparator.java 在 MR 程序裏頭

 

四、代碼實現

package com.ghgj.mazh.mapreduce.exercise.coursescore3;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class CourseScoreMR_Pro_03 {

    public static void main(String[] args) throws Exception {
        /**
         * 一些參數的初始化
         */
        String inputPath = "D:\\bigdata\\coursescore2\\input";
        String outputPath = "D:\\bigdata\\coursescore2\\output3";

        /**
         * 初始化一個Job對象
         */
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        /**
         * 設置jar包所在路徑
         */
        job.setJarByClass(CourseScoreMR_Pro_03.class);

        /**
         * 指定mapper類和reducer類 等各種其他業務邏輯組件
         */
        job.setMapperClass(Mapper_CS.class);
        job.setReducerClass(Reducer_CS.class);
        // 指定maptask的輸出類型
        job.setMapOutputKeyClass(CourseScore.class);
        job.setMapOutputValueClass(NullWritable.class);
        // 指定reducetask的輸出類型
        job.setOutputKeyClass(CourseScore.class);
        job.setOutputValueClass(NullWritable.class);

        job.setGroupingComparatorClass(CourseScoreGroupComparator.class);

        /**
         * 指定該mapreduce程序數據的輸入和輸出路徑
         */
        Path input = new Path(inputPath);
        Path output = new Path(outputPath);
        FileSystem fs = FileSystem.get(conf);
        if (fs.exists(output)) {
            fs.delete(output, true);
        }
        FileInputFormat.setInputPaths(job, input);
        FileOutputFormat.setOutputPath(job, output);

        /**
         * 最後提交任務
         */
        boolean waitForCompletion = job.waitForCompletion(true);
        System.exit(waitForCompletion ? 0 : 1);
    }

    /**
     * Mapper組件：
     * <p>
     * 輸入的key:
     * 輸入的value: computer,xuzheng,54,52,86,91,42
     * <p>
     * 輸出的key: CourseScore
     * 輸入的value: NullWritable
     */
    private static class Mapper_CS extends Mapper<LongWritable, Text, CourseScore, NullWritable> {

        CourseScore keyOut = new CourseScore();

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

            String[] splits = value.toString().split(",");
            String course = splits[0];
            String name = splits[1];

            int sum = 0;
            int num = 0;
            for(int i=2; i<splits.length; i++){
                sum += Integer.valueOf(splits[i]);
                num ++;
            }
            double avgScore = Math.round(sum * 1D / num * 10) / 10D;

            keyOut.setCourse(course);
            keyOut.setName(name);
            keyOut.setScore(avgScore);

            context.write(keyOut, NullWritable.get());
        }
    }

    /**
     * Reducer組件：
     * <p>
     * 輸入的key: CourseScore
     * 輸入的values: NullWritable
     * <p>
     * 輸出的key: CourseScore
     * 輸入的value: NullWritable
     */
    private static class Reducer_CS extends Reducer<CourseScore, NullWritable, CourseScore, NullWritable> {

        // 成績最高的兩個人的信息
        int topN = 2;

        @Override
        protected void reduce(CourseScore key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {

            int number = 0;
            for(NullWritable nvl: values){
                context.write(key, nvl);
                number ++;
                if(number == topN){
                    break;
                }
            }
        }
    }

    /**
     * 自定義分組組件
     */
    public static class CourseScoreGroupComparator extends WritableComparator{

        CourseScoreGroupComparator(){
            super(CourseScore.class, true);
        }

        @Override
        public int compare(WritableComparable a, WritableComparable b) {

            CourseScore cs1 = (CourseScore)a;
            CourseScore cs2 = (CourseScore)b;

            int result = cs1.getCourse().compareTo(cs2.getCourse());
            return result;
        }
    }
}

其中 CourseScore類的實現：

package com.ghgj.mazh.mapreduce.exercise.coursescore3;

import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

public class CourseScore implements WritableComparable<CourseScore> {

    private String course;
    private String name;
    private double score;

    public CourseScore(String course, String name, double score) {
        super();
        this.course = course;
        this.name = name;
        this.score = score;
    }

    public CourseScore() {
    }

    public String getCourse() {
        return course;
    }

    public void setCourse(String course) {
        this.course = course;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public double getScore() {
        return score;
    }

    public void setScore(double score) {
        this.score = score;
    }

    @Override
    public void write(DataOutput out) throws IOException {
        // TODO Auto-generated method stub
        out.writeUTF(course);
        out.writeUTF(name);
        out.writeDouble(score);
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        // TODO Auto-generated method stub
        this.course = in.readUTF();
        this.name = in.readUTF();
        this.score = in.readDouble();
    }

    /**
     * 排序規則
     * compareTo方法既充當排序用，用充當分組規則
     */
    @Override
    public int compareTo(CourseScore cs) {

        int courseDiff = this.course.compareTo(cs.getCourse());

        if (courseDiff == 0) {
            double diff = cs.getScore() - this.score;
            if (diff == 0) {
                return 0;
            } else {
                return diff > 0 ? 1 : -1;
            }
        } else {
            return courseDiff > 0 ? 1 : -1;
        }
    }


    @Override
    public String toString() {
        return course + "\t" + name + "\t" + score;
    }
}

 

五、執行結果

最後的結果：每個課程的最高成績的前2名

 

至此，大功告成