代碼:MapReduce程序模板

MapReduce:

public class MRDriver extends Configured implements Tool {

    @Override
    public int run(String[] args) throws Exception {
        //1.創建job
        Job job = Job.getInstance(this.getConf(), "xxx");
        job.setJarByClass( MRDriver.class );

        //2.input
        Path inputPath = new Path(args[0]);
        FileInputFormat.setInputPaths(job,inputPath);

        //3.mapper
        job.setMapperClass(null);
        job.setMapOutputKeyClass(null);
        job.setMapOutputValueClass(null);

        //4.shuffle
        job.setPartitionerClass(null);//分區
        job.setGroupingComparatorClass(null);//分組
        job.setSortComparatorClass(null);//排序

        //5.reducer
        job.setReducerClass(null);
        job.setOutputKeyClass(null);
        job.setOutputValueClass(null);

        //6.output
        Path outputPath = new Path(args[1]);
        FileSystem hdfs = FileSystem.get(this.getConf());
        //查看輸出路徑是否存在,存在則先刪除
        if(hdfs.exists(outputPath)){
            //boolean delete(Path f, boolean recursive)
            //如果是目錄則爲true
            hdfs.delete(outputPath,true);
        }
        FileOutputFormat.setOutputPath(job,outputPath);

        boolean isSuccess = job.waitForCompletion(true);
        return isSuccess?0:1;
    }

    public static void main(String[] args) {
        Configuration configuration = new Configuration();
        try {
            int status = ToolRunner.run(configuration, new MRDriver(), args);
            System.exit(status);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

Mapper:

public class MRMapper extends Mapper<LongWritable,Text,Text,Text>{
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //根據業務處理
    }
}

Reducer:

public class MRReducer extends Reducer<Text,Text,Text,Text> {
    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        //根據業務處理
    }
}
發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章