需求:
有一個文件保存瞭如下信息:
A:B,C,D,F,E,O(代表A的好友有BCDFEO)
B:A,C,E,K
C:F,A,D,I
D:A,E,F,L
E:B,C,D,M,L
F:A,B,C,D,E,O,M
G:A,C,D,E,F
H:A,C,D,E,O
I:A,O
J:B,O
K:A,C,D
L:D,E,F
M:E,F,G
O:A,H,I,J
求出兩兩之間有共同好友的"用戶對",及他倆的共同好友
比如:
A--B C ,E
思路:
- 先將文件內容輸出爲:<好友 擁有該好友的人1,擁有該好友的人2, ....>的格式
- 然後將擁有該好友的人兩兩配對,作爲map的key,好友作爲value,reduce且調整格式後輸出的則爲< A--B 共同好友1,共同好友2,....>的格式
第一步代碼:
package com.bpf.mr.friend;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class FriendStepOne {
static class FriendStepOneMapper extends Mapper<LongWritable, Text, Text, Text>{
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
//A:B,C,D,F,E,O
String line = value.toString();
String[] p_friend = line.split(":");
//System.out.println(p_friend[1]);
String p = p_friend[0];
String[] friends = p_friend[1].split(",");
//輸出<好友,人>
for (String friend : friends) {
context.write(new Text(friend), new Text(p));
}
}
}
static class FriendStepOneRuducer extends Reducer<Text, Text, Text, Text>{
@Override
protected void reduce(Text key, Iterable<Text> persons, Context context) throws IOException, InterruptedException {
StringBuffer b = new StringBuffer();
for (Text person : persons) {
b.append(person.toString() + ",");
}
context.write(key, new Text(b.toString()));
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(FriendStepOne.class);
job.setMapperClass(FriendStepOneMapper.class);
job.setReducerClass(FriendStepOneRuducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.setInputPaths(job, new Path("D:\\測試數據\\輸入"));
FileOutputFormat.setOutputPath(job, new Path("D:\\測試數據\\輸出"));
boolean res = job.waitForCompletion(true);
System.exit(res?0:1);
}
}
結果:第二步代碼:
package com.bpf.mr.friend;
import java.io.IOException;
import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class FriendStepTwo {
static class FriendStepOneMapper extends Mapper<LongWritable, Text, Text, Text>{
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
//A I,K,C,B,G,F,H,O,D
String line = value.toString();
String[] f_p = line.split("\t");
String f = f_p[0];
String[] p = f_p[1].split(",");
Arrays.sort(p);
for(int i = 0; i < p.length - 1; i ++) {
for(int j = i + 1; j < p.length; j ++) {
context.write(new Text(p[i] + "--" + p[j]), new Text(f));
}
}
}
}
static class FriendStepOneRuducer extends Reducer<Text, Text, Text, Text>{
@Override
protected void reduce(Text key, Iterable<Text> friends, Context context) throws IOException, InterruptedException {
StringBuffer b = new StringBuffer();
for (Text friend : friends) {
b.append(friend.toString() + ",");
}
context.write(key, new Text(b.toString()));
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(FriendStepTwo.class);
job.setMapperClass(FriendStepOneMapper.class);
job.setReducerClass(FriendStepOneRuducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.setInputPaths(job, new Path("D:\\測試數據\\輸出"));
FileOutputFormat.setOutputPath(job, new Path("D:\\測試數據\\再輸出"));
boolean res = job.waitForCompletion(true);
System.exit(res?0:1);
}
}
結果: