wordcount
測試:文件1 4.3m,文件100 430m;不設置combiner,設置combiner
單機環境
1.java
eclipse,maven一直沒有成功,工程手動添加依賴
hadoop-common-2.7.3.jar
hadoop-mapreduce-client-core-2.7.3.jar
commons-cli-1.2.jar
hadoop-client-2.7.3.jar
hadoop-hdfs-2.7.3.jar
eclipse僅作爲編輯器,自動補全之類的
linux下編譯class,打包jar
保存hadoop classpath,後續引用方便
tmp=`bin/hadoop classpath`
javac -classpath $tmp XMWordCount.java
MANIFEST.MF
Main-Class: com.XMWordCount
jar cfm XMWordCount1.jar com/MANIFEST.MF com/*.class
增加combiner,生成XMWordCount2.jar
jar cfm XMWordCount2.jar com/MANIFEST.MF com/*.class
bin/hadoop jar /home/xiumu/XMWordCount1.jar input1 output9
//XMWordCount.java
package com;
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
public class XMWordCount {
public static class TokenizerMapper
extends Mapper<Object, Text, Text, IntWritable>{
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(Object key, Text value, Context context
) throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
String s1 = itr.nextToken();
StringBuilder s2 = new StringBuilder("");
int len = s1.length();
for (int i = 0; i < len; ++i) {
char c = s1.charAt(i);
if (c >= 'a' && c <= 'z') s2.append(c);
if (c >= 'A' && c <= 'Z') s2.append(c);
if (c == '\'' || c == '-') s2.append(c);
}
if(!s2.toString().equals("")) {
word.set(s2.toString().toLowerCase());
context.write(word, one);
}
}
}
}
public static class IntSumReducer
extends Reducer<Text,IntWritable,Text,IntWritable> {
private IntWritable result = new IntWritable();
public void reduce(Text key, Iterable<IntWritable> values,
Context context
) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length < 2) {
System.err.println("Usage: XMwordcount <in> [<in>...] <out>");
System.exit(2);
}
Job job = Job.getInstance(conf, "xiumu word count");
// Job job = new Job();
job.setJarByClass(XMWordCount.class);
job.setMapperClass(TokenizerMapper.class);
// job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
for (int i = 0; i < otherArgs.length - 1; ++i) {
FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
}
FileOutputFormat.setOutputPath(job,
new Path(otherArgs[otherArgs.length - 1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
1 | 100 | |
無 | 0.5min | 7min |
有 | 0.5min | 3min |
2.streaming,python3
bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.3.jar -input input1 -output output1 -mapper "/home/xiumu/mapper.py" -reducer "/home/xiumu/reduce.py"
bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.3.jar -input input1 -output output1 -mapper "/home/xiumu/mapper.py" -reducer "/home/xiumu/reduce.py" -combiner "/home/xiumu/reduce.py"
mapper.py
#!/usr/bin/python3
import sys
for line in sys.stdin :
words = line.strip().split()
for word in words :
newword = str()
for c in word :
if c >= 'a' and c <= 'z' :
newword += c
if c >= 'A' and c <= 'Z' :
newword += c
if c == '\'' or c == '-' :
newword += c
newword = newword.strip().lower()
if newword == '' :
continue
print("%s %d" % (newword, 1))
reducer.py
#!/usr/bin/python3
import sys
(last_key, count) = (None, 0)
for line in sys.stdin :
(key, value) = line.strip().split()
if last_key == key :
count += int(value)
else :
if last_key != None :
print("%s %d" % (last_key, count))
(last_key, count) = (key, int(value))
if last_key != None :
print("%s %d" % (last_key, count))
1 | 100 | |
無 | 4min | 14min |
有 | 0.5min | 10min |
結論:
java比streaming快好多,streaming可以選自己喜歡的腳本語言,簡單,但是效率低,而且對整個過程的控制不如java
combiner,比較大的影響效率,尤其是在reduce階段會快很多,原因很顯然,不表
而且java運行時,機器負載較低