hadoop07--mapreduce工作流程,Combiner, 二次排序

MapReduce工作流程

Combiner

Combiner對於使用, 嚴格來說, 最適合的場景就是合併數量

Combiner 輸出的類型爲K,V對. reduce的K,V對類型一致

實例: 實現Combiner

分析: 要想實現 Combiner 則需要繼承一個reducer類, 在dirver 類中設置Combiner類

1. 繼承Reducer類, 實現重載reduce方法

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class WordCountCombiner extends Reducer<Text, IntWritable, Text, IntWritable> {

	@Override
	protected void reduce(Text key, Iterable<IntWritable> values, Context context)
			throws IOException, InterruptedException {
		int sum = 0;

		for (IntWritable value : values) {
			sum += value.get();
		}
		context.write(key, new IntWritable(sum));

	}

}

2. 在主函數Driver中設置Combiner類

job.setCombinerClass(WordCountCombiner.class);

在運行結果中可以看到Combiner的變化

Combine input records=429
Combine output records=100

二次排序

自定義序列化

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class Driver {
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		Configuration conf = new Configuration();

		Job job = Job.getInstance(conf);

		job.setJarByClass(Driver.class);
		job.setMapperClass(SecondSortMap.class);
		job.setReducerClass(SecondSortReduce.class);
		job.setGroupingComparatorClass(SecondSortGroup.class);

		job.setMapOutputKeyClass(CustomKey.class);
		job.setMapOutputValueClass(Text.class);

		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);

		FileInputFormat.setInputPaths(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));

		boolean res = job.waitForCompletion(true);
		System.exit(res ? 0 : 1);

	}
}

實現map

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class SecondSortMap extends Mapper<LongWritable, Text, CustomKey, Text> {

	private CustomKey outputkey = new CustomKey();
	private Text outputValue = new Text();

	@Override
	protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

		String line = value.toString();

		String[] words = line.split(",");

		outputkey.setWord1(words[0]);
		outputkey.setWord2(words[1]);
		outputkey.setWord3(words[2]);

		outputValue.set(words[2]);
		//System.out.println(outputkey+outputValue.toString());
		context.write(outputkey, outputValue);
	}
}

實現reduce

import java.io.IOException;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class SecondSortReduce extends Reducer<CustomKey, Text, Text, Text> {

	Text outputkey = new Text();
	Text outputValue = new Text();

	@Override
	protected void reduce(CustomKey key, Iterable<Text> values, Context context)
			throws IOException, InterruptedException {
		
		
		String tmp = "";
		for (Text value : values) {
			tmp += value.toString() + ",";
		}
		
		
		outputkey.set(key.getWord1());
		outputValue.set(tmp.substring(0, tmp.length() - 1));

		context.write(outputkey, outputValue);
	}
}

實現分組

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

public class SecondSortGroup extends WritableComparator {

	@Override
	public int compare(WritableComparable a, WritableComparable b) {
		// TODO Auto-generated method stub
		CustomKey key1 = (CustomKey) a;
		CustomKey key2 = (CustomKey) b;
		System.out.print(key1.getWord1());
		System.out.print(key2.getWord1());
		System.out.println(key1.getWord1().compareTo(key2.getWord1()));
		return key1.getWord1().compareTo(key2.getWord1());
	}

	public SecondSortGroup() {
		// TODO Auto-generated constructor stub
		super(CustomKey.class, true);
	}

}

實現主函數Driver

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class Driver {
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		Configuration conf = new Configuration();

		Job job = Job.getInstance(conf);

		job.setJarByClass(Driver.class);
		job.setMapperClass(SecondSortMap.class);
		job.setReducerClass(SecondSortReduce.class);
		job.setGroupingComparatorClass(SecondSortGroup.class);

		job.setMapOutputKeyClass(CustomKey.class);
		job.setMapOutputValueClass(Text.class);

		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);

		FileInputFormat.setInputPaths(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));

		boolean res = job.waitForCompletion(true);
		System.exit(res ? 0 : 1);

	}
}