Hadoop_day06_MapReduce 的 經典案例（求共同好友）

1. 需求分析

以下是qq的好友列表數據，冒號前是一個用戶，冒號後是該用戶的所有好友（數據中的好友關係是單向的）

A:B,C,D,F,E,O
B:A,C,E,K
C:A,B,D,E,I 
D:A,E,F,L
E:B,C,D,M,L
F:A,B,C,D,E,O,M
G:A,C,D,E,F
H:A,C,D,E,O
I:A,O
J:B,O
K:A,C,D
L:D,E,F
M:E,F,G
O:A,H,I,J

求出哪些人兩兩之間有共同好友，及他倆的共同好友都有誰？

2. 實現步驟

第一步：代碼實現

Mapper類

public class Step1Mapper extends Mapper<LongWritable,Text,Text,Text> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //1:以冒號拆分行文本數據: 冒號左邊就是V2
        String[] split = value.toString().split(":");

        //2:將冒號右邊的字符串以逗號拆分,每個成員就是K2
        String[] split1 = split[1].split(",");
        for (String s : split1) {
            context.write(new Text(s),new Text(split[0]));
        }
    }
}

Reducer類

public class Step1Reducer extends Reducer<Text,Text,Text,Text> {
    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        //1:遍歷集合,並將每一個元素拼接,得到K3
        StringBuffer buffer = new StringBuffer();
        for (Text value : values) {
            buffer.append(value.toString()).append("-");
        }
        context.write(new Text(buffer.toString()),key);
    }
}

JobMain

public class JobMain extends Configured implements Tool{
    @Override
    public int run(String[] strings) throws Exception {

        Job job = Job.getInstance(super.getConf(), "common_friends_step1");

        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job,new Path("d:\\mapreduce\\common_friends_step1_in"));

        job.setMapperClass(Step1Mapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);

        job.setReducerClass(Step1Reducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job,new Path("d:\\mapreduce\\common_friends_step1_out"));

        boolean bl = job.waitForCompletion(true);
        return bl ? 0 : 1;
    }

    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();
        int run = ToolRunner.run(configuration, new JobMain(), args);
        System.exit(run);
    }
}

第二步：代碼實現

Mapper類

public class Step2Mapper extends Mapper<LongWritable,Text,Text,Text> {
     /*
         K1           V1

         0            A-F-C-J-E-	B
        ----------------------------------
    
         K2             V2
         A-C            B
         A-E            B
         A-F            B
         C-E            B
    
     */
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //1:拆分行文本數據,結果的第二部分可以得到V2
        String[] split = value.toString().split("\t");

        //2:繼續以'-'爲分隔符拆分行文本數據第一部分,得到數組
        String[] split1 = split[0].split("-");

        //3:對數組做一個排序
        Arrays.sort(split1);

        //4:對數組中的元素進行兩兩組合,得到K2
        /*
          A-E-C ----->  A  C  E

          A  C  E
            A  C  E

         */
        for (int i = 0; i < split1.length-1; i++){
            for (int j = i + 1; j < split1.length; j++){
                String str = split1[i] + "-" + split1[j];
                context.write(new Text(str),new Text(split[1]));
            }
        }
    }
}

Reducer類

public class Step2Reducer extends Reducer<Text,Text,Text,Text> {
    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        //1:原來的K2就是K3
        //2:將集合進行遍歷,將集合中的元素拼接,得到V3
        StringBuffer buffer = new StringBuffer();
        for (Text value : values) {
            buffer.append(value.toString()).append("-");
        }
        context.write(key,new Text(buffer.toString()));
    }
}

JobMain

public class JobMain extends Configured implements Tool {
    @Override
    public int run(String[] strings) throws Exception {

        Job job = Job.getInstance(super.getConf(), "common_friends_step2");

        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job,new Path("d:\\mapreduce\\common_friends_step1_out"));

        job.setMapperClass(Step2Mapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);

        job.setReducerClass(Step2Reducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job,new Path("d:\\mapreduce\\common_friends_step2_out"));

        boolean bl = job.waitForCompletion(true);
        return bl ? 0 : 1;
    }

    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();
        int run = ToolRunner.run(configuration,new JobMain(),args);
        System.exit(run);
    }
}