hadoop實現topk

package max;
 
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import max.TopKApp2.MyReducer.LongArrayWrible;
 
/**
 * 
 * @author
 *
 */
public class TopKApp2 {
	//要統計的文件位置
		static final String INPUT_PATH = "hdfs://test1:9000/input/num-*.txt";
		//統計結果輸出的位置
		static final String OUT_PATH = "hdfs://test1:9000/out";
		public static void main(String[] args) throws Exception {
			
				Configuration conf = new Configuration();
				final FileSystem fileSystem = FileSystem.get(new URI(INPUT_PATH), conf);
				final Path outPath = new Path(OUT_PATH);
				if(fileSystem.exists(outPath)){
					fileSystem.delete(outPath, true);
				}
			
				final Job job = new Job(conf , TopKApp2.class.getSimpleName());
				
				FileInputFormat.setInputPaths(job, INPUT_PATH);
				job.setMapperClass(MyMapper.class);
				job.setReducerClass(MyReducer.class);
				job.setOutputKeyClass(Text.class);
				job.setOutputValueClass(LongArrayWrible.class);
				FileOutputFormat.setOutputPath(job, outPath);
				job.waitForCompletion(true);
			
		}
		static class MyMapper extends Mapper<LongWritable, Text, Text, LongArrayWrible>{
			
			LongArrayWrible aw=new LongArrayWrible(3);
			Text max=new Text("max");
			LongWritable lw=new LongWritable();
			
			@Override
			protected void cleanup(Mapper<LongWritable, Text, Text, LongArrayWrible>.Context context)
					throws IOException, InterruptedException {
				// TODO Auto-generated method stub
				context.write(max, aw);
				
			}

			@Override
			protected void map(LongWritable key, Text value,
					Mapper<LongWritable, Text, Text, LongArrayWrible>.Context context)
					throws IOException, InterruptedException {
				// TODO Auto-generated method stub
				String[] strs=value.toString().split(" ");
				
				
				
				aw.offer(new Person(Long.parseLong(strs[0]), strs[1]));
				
			}

			
			
			
		
		}
		
		public static class MyReducer extends Reducer<Text,LongArrayWrible, Text, LongArrayWrible>{
			Text max=new Text("max");
			
			@Override
			protected void reduce(Text arg0, Iterable<LongArrayWrible> arg1,
					Reducer<Text, LongArrayWrible, Text, LongArrayWrible>.Context context)
					throws IOException, InterruptedException {
				// TODO Auto-generated method stub
				
				LongArrayWrible arrayWritable=new LongArrayWrible(3);
				for(LongArrayWrible law:arg1){
					arrayWritable.addAll(law);
				}
				
				System.out.println(arrayWritable.toString());
				
				context.write(max, arrayWritable);
				System.out.println("reduceing");
			}
			static class LongArrayWrible implements Writable {
				PriorityQueue<Person> queue=new PriorityQueue<>(new Comparator<Person>() {

					@Override
					public int compare(Person o1, Person o2) {
						// TODO Auto-generated method stub
						return o1.getAge().get()>o2.getAge().get()? 1:-1;
					}
				});
				int maxSize=5;
				public LongArrayWrible(int maxSize) {
				
				//	this.maxSize=maxSize;
					// TODO Auto-generated constructor stub
				}
				public LongArrayWrible() {
					
					
					// TODO Auto-generated constructor stub
				}
			

				
				public Person[] toPersons() {
					// TODO Auto-generated method stub
					Person[] strs=new Person[queue.size()];
					int i=0;
					while(!queue.isEmpty()){
						Person num=queue.poll();
						strs[i++]=num;
					}
					for(int j=0;j<strs.length;++j)
						this.offer(strs[j]);
					return strs;
				}

				@Override
				public String toString() {
					// TODO Auto-generated method stub
					Person[] persons= this.toPersons();
					String str="";
					for(Person s:persons){
						str=str+s.toString()+", ";
					}
					return str;
				}
				
				public void offer(Person e){
					
					queue.offer(e);
					if(queue.size()>maxSize) queue.poll();
				}
				
				public void addAll(LongArrayWrible law){
					PriorityQueue<Person> q1=law.queue;
					while(!q1.isEmpty()){
						Person l= q1.poll();
						this.offer(l);
					}
					System.out.println("after addAll:"+toString());
				}



				@Override
				public void readFields(DataInput arg0) throws IOException {
					// TODO Auto-generated method stub
					
					int i=0;
					try{
						
						while(true){
							Person lw=new Person();
							lw.readFields(arg0);
							System.out.println("lw="+lw.toString());
							offer(lw);
							++i;
						}
					}catch(Exception e){
						e.printStackTrace();
						System.out.println("readcount'"+i);
					}
					System.out.println("after ReadFile");
					
				}



				@Override
				public void write(DataOutput arg0) throws IOException {
					// TODO Auto-generated method stub
					Person[] s=this.toPersons();
					for(int i=0;i<s.length;++i){
						s[i].write(arg0);
					}
					
				}
				
			}
			
			
			
		}	
		static class Person implements Writable{
			LongWritable age;
			Text name;
			public LongWritable getAge() {
				return age;
			}
			public void setAge(LongWritable age) {
				this.age = age;
			}
			public Text getName() {
				return name;
			}
			public void setName(Text name) {
				this.name = name;
			}
			@Override
			public void readFields(DataInput arg0) throws IOException {
				// TODO Auto-generated method stub
				age=new LongWritable();
				name=new Text();
				age.readFields(arg0);
				name.readFields(arg0);
				
			}
			@Override
			public void write(DataOutput arg0) throws IOException {
				// TODO Auto-generated method stub
				age.write(arg0);
				name.write(arg0);
				
			}
			@Override
			public String toString() {
				return "Person [age=" + age.get() + ", name=" + name.toString() + "]";
			}
			
			public Person(long age1,String name1){
				this.setAge(new LongWritable(age1));
				this.setName(new Text(name1));
			}
			public Person(){
				
			}
		}
}