Hadoop的MapReduce中多文件輸出

這兩天在網上看了個MapReduce的多文件輸出的帖子： http://blog.csdn.net/inkfish。寫的不錯。

我試着完成了一下。也是分爲三個文件：我這三個文件，跟原作者的稍有不同。其中有些類是我原來寫的，我直接拷貝過來的，所以有點不同。

My_LineRead.java

public class My_LineRead<K, V> extends RecordWriter<K, V>{ private static final String utf8 = "UTF-8"; private static final String colon = "----"; //劃分符號 private static final byte[] newline; static { try { newline = "/n".getBytes(utf8); } catch (UnsupportedEncodingException uee) { throw new IllegalArgumentException("can't find " + utf8 + " encoding"); } } protected DataOutputStream out; private final byte[] keyValueSeparator; public My_LineRead(DataOutputStream out) { this(out, colon); //調用下面的構造函數 } public My_LineRead(DataOutputStream out, String keyValueSeparator) { // TODO Auto-generated constructor stub this.out = out; try { this.keyValueSeparator = keyValueSeparator.getBytes(utf8); } catch (UnsupportedEncodingException e) { // TODO Auto-generated catch block throw new IllegalArgumentException("can't find " + utf8 + " encoding"); } } @Override public void close(TaskAttemptContext arg0) throws IOException, InterruptedException { // TODO Auto-generated method stub out.close(); } @Override public void write(K key, V value) throws IOException, InterruptedException { if (!(key == null && key instanceof NullWritable)){ //如果key不爲空者輸出key if ((Object)key instanceof Text){ Text to = (Text) key; out.write(to.getBytes(), 0, to.getLength()); } else { out.write(key.toString().getBytes(utf8)); } out.write(keyValueSeparator); } if (!(value == null && value instanceof NullWritable)){ //如果value不爲空則輸出value if ((Object)value instanceof Text){ Text to = (Text) value; out.write(to.getBytes(), 0, to.getLength()); } else { out.write(value.toString().getBytes(utf8)); } out.write(newline); } } }

MyMultipleOutputFormat.java //這個類，我添加了些註釋便於理解

public abstract class MyMultipleOutputFormat <K extends WritableComparable<?>, V extends Writable> extends FileOutputFormat<K, V> { //接口類，需要在主程序中實現generateFileNameForKeyValue來獲取文件名 private MultiRecordWriter writer = null; @Override public RecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException { // TODO Auto-generated method stub //如果第一次調用那麼writer=null if (writer == null) { //getTaskOutputPath獲取output路徑 writer = new MultiRecordWriter(job, getTaskOutputPath(job)); } return writer; } private Path getTaskOutputPath(TaskAttemptContext conf) throws IOException { Path workPath = null; OutputCommitter committer = super.getOutputCommitter(conf); if (committer instanceof FileOutputCommitter) { workPath = ((FileOutputCommitter) committer).getWorkPath(); } else { Path outputPath = super.getOutputPath(conf); if (outputPath == null) { throw new IOException("Undefined job output-path"); } workPath = outputPath; } return workPath; } /**通過key, value, conf來確定輸出文件名（含擴展名）*/ //返回值就是文件名。可以根據key,value來判斷 protected abstract String generateFileNameForKeyValue(K key, V value, Configuration conf); //MultiRecordWriter類 public class MultiRecordWriter extends RecordWriter<K, V> { /**RecordWriter的緩存*/ private HashMap<String, RecordWriter<K, V>> recordWriters = null; private TaskAttemptContext job = null; /**輸出目錄*/ private Path workPath = null; //構造函數 public MultiRecordWriter(TaskAttemptContext job, Path workPath) { super(); this.job = job; this.workPath = workPath; recordWriters = new HashMap<String, RecordWriter<K, V>>(); } //關閉，應該可能是多個文件進行關閉，所有采用循環 //recordWriters.values() 就是指的getBaseRecordWriter返回的值。 @Override public void close(TaskAttemptContext context) throws IOException, InterruptedException { Iterator<RecordWriter<K, V>> values = this.recordWriters.values().iterator(); while (values.hasNext()) { values.next().close(context); } this.recordWriters.clear(); } @Override public void write(K key, V value) throws IOException, InterruptedException { //得到輸出文件名 String baseName = generateFileNameForKeyValue(key, value, job.getConfiguration()); //如果recordWriters裏沒有文件名，那麼就建立。否則就直接寫值。 RecordWriter<K, V> rw = this.recordWriters.get(baseName); if (rw == null) { rw = getBaseRecordWriter(job, baseName); //放入HashMap this.recordWriters.put(baseName, rw); } rw.write(key, value); } // ${mapred.out.dir}/_temporary/_${taskid}/${nameWithExtension} private RecordWriter<K, V> getBaseRecordWriter(TaskAttemptContext job, String baseName) throws IOException, InterruptedException { //獲取配置文件 Configuration conf = job.getConfiguration(); //查看是否使用解碼器 boolean isCompressed = getCompressOutput(job); String keyValueSeparator = ","; RecordWriter<K, V> recordWriter = null; if (isCompressed) { Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, GzipCodec.class); CompressionCodec codec = ReflectionUtils.newInstance(codecClass, conf); Path file = new Path(workPath, baseName + codec.getDefaultExtension()); FSDataOutputStream fileOut = file.getFileSystem(conf).create(file, false); recordWriter = new My_LineRead<K, V>(new DataOutputStream(codec .createOutputStream(fileOut)), keyValueSeparator); } //如果不使用解碼器 else { Path file = new Path(workPath, baseName); FSDataOutputStream fileOut = file.getFileSystem(conf).create(file, false); //recordWriter = new My_LineRead<K, V>(fileOut, keyValueSeparator); //這裏我使用的我自己的OutputFormat recordWriter = new My_LineRead<K, V>(fileOut); } return recordWriter; } } }

最後就是測試類，WordCount_MulFileOut.java

public class WordCount_MulFileOut { public static class wordcountMapper extends Mapper<LongWritable, Text, Text, IntWritable>{ private final static IntWritable one = new IntWritable(1); private Text word = new Text(); public void map(LongWritable key, Text value, Context context)throws IOException, InterruptedException{ String line = value.toString(); StringTokenizer itr = new StringTokenizer(line); while(itr.hasMoreElements()){ word.set(itr.nextToken()); context.write(word, one); } } } public static class wordcountReduce extends Reducer<Text, IntWritable, Text, IntWritable>{ public void reduce(Text key, Iterable<IntWritable>values, Context context)throws IOException, InterruptedException{ int sum = 0; for (IntWritable str : values){ sum += str.get(); } context.write(key, new IntWritable(sum)); } } public static class MyMultiple extends MyMultipleOutputFormat{ @Override protected String generateFileNameForKeyValue(WritableComparable key, Writable value, Configuration conf) { // TODO Auto-generated method stub return "other.txt"; } } public static void main(String args[])throws Exception{ Configuration conf = new Configuration(); Job job = new Job(conf, "wordcount"); job.setJarByClass(WordCount_MulFileOut.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(MyMultiple.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setMapperClass(wordcountMapper.class); job.setReducerClass(wordcountReduce.class); job.setCombinerClass(wordcountReduce.class); FileInputFormat.setInputPaths(job, new Path(args[1])); FileOutputFormat.setOutputPath(job, new Path(args[2])); job.waitForCompletion(true); } }