package max;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import max.TopKApp2.MyReducer.LongArrayWrible;
/**
*
* @author
*
*/
public class TopKApp2 {
//要统计的文件位置
static final String INPUT_PATH = "hdfs://test1:9000/input/num-*.txt";
//统计结果输出的位置
static final String OUT_PATH = "hdfs://test1:9000/out";
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
final FileSystem fileSystem = FileSystem.get(new URI(INPUT_PATH), conf);
final Path outPath = new Path(OUT_PATH);
if(fileSystem.exists(outPath)){
fileSystem.delete(outPath, true);
}
final Job job = new Job(conf , TopKApp2.class.getSimpleName());
FileInputFormat.setInputPaths(job, INPUT_PATH);
job.setMapperClass(MyMapper.class);
job.setReducerClass(MyReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongArrayWrible.class);
FileOutputFormat.setOutputPath(job, outPath);
job.waitForCompletion(true);
}
static class MyMapper extends Mapper<LongWritable, Text, Text, LongArrayWrible>{
LongArrayWrible aw=new LongArrayWrible(3);
Text max=new Text("max");
LongWritable lw=new LongWritable();
@Override
protected void cleanup(Mapper<LongWritable, Text, Text, LongArrayWrible>.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
context.write(max, aw);
}
@Override
protected void map(LongWritable key, Text value,
Mapper<LongWritable, Text, Text, LongArrayWrible>.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
String[] strs=value.toString().split(" ");
aw.offer(new Person(Long.parseLong(strs[0]), strs[1]));
}
}
public static class MyReducer extends Reducer<Text,LongArrayWrible, Text, LongArrayWrible>{
Text max=new Text("max");
@Override
protected void reduce(Text arg0, Iterable<LongArrayWrible> arg1,
Reducer<Text, LongArrayWrible, Text, LongArrayWrible>.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
LongArrayWrible arrayWritable=new LongArrayWrible(3);
for(LongArrayWrible law:arg1){
arrayWritable.addAll(law);
}
System.out.println(arrayWritable.toString());
context.write(max, arrayWritable);
System.out.println("reduceing");
}
static class LongArrayWrible implements Writable {
PriorityQueue<Person> queue=new PriorityQueue<>(new Comparator<Person>() {
@Override
public int compare(Person o1, Person o2) {
// TODO Auto-generated method stub
return o1.getAge().get()>o2.getAge().get()? 1:-1;
}
});
int maxSize=5;
public LongArrayWrible(int maxSize) {
// this.maxSize=maxSize;
// TODO Auto-generated constructor stub
}
public LongArrayWrible() {
// TODO Auto-generated constructor stub
}
public Person[] toPersons() {
// TODO Auto-generated method stub
Person[] strs=new Person[queue.size()];
int i=0;
while(!queue.isEmpty()){
Person num=queue.poll();
strs[i++]=num;
}
for(int j=0;j<strs.length;++j)
this.offer(strs[j]);
return strs;
}
@Override
public String toString() {
// TODO Auto-generated method stub
Person[] persons= this.toPersons();
String str="";
for(Person s:persons){
str=str+s.toString()+", ";
}
return str;
}
public void offer(Person e){
queue.offer(e);
if(queue.size()>maxSize) queue.poll();
}
public void addAll(LongArrayWrible law){
PriorityQueue<Person> q1=law.queue;
while(!q1.isEmpty()){
Person l= q1.poll();
this.offer(l);
}
System.out.println("after addAll:"+toString());
}
@Override
public void readFields(DataInput arg0) throws IOException {
// TODO Auto-generated method stub
int i=0;
try{
while(true){
Person lw=new Person();
lw.readFields(arg0);
System.out.println("lw="+lw.toString());
offer(lw);
++i;
}
}catch(Exception e){
e.printStackTrace();
System.out.println("readcount'"+i);
}
System.out.println("after ReadFile");
}
@Override
public void write(DataOutput arg0) throws IOException {
// TODO Auto-generated method stub
Person[] s=this.toPersons();
for(int i=0;i<s.length;++i){
s[i].write(arg0);
}
}
}
}
static class Person implements Writable{
LongWritable age;
Text name;
public LongWritable getAge() {
return age;
}
public void setAge(LongWritable age) {
this.age = age;
}
public Text getName() {
return name;
}
public void setName(Text name) {
this.name = name;
}
@Override
public void readFields(DataInput arg0) throws IOException {
// TODO Auto-generated method stub
age=new LongWritable();
name=new Text();
age.readFields(arg0);
name.readFields(arg0);
}
@Override
public void write(DataOutput arg0) throws IOException {
// TODO Auto-generated method stub
age.write(arg0);
name.write(arg0);
}
@Override
public String toString() {
return "Person [age=" + age.get() + ", name=" + name.toString() + "]";
}
public Person(long age1,String name1){
this.setAge(new LongWritable(age1));
this.setName(new Text(name1));
}
public Person(){
}
}
}