</pre><pre name="code" class="java">package hbase.test;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.Tool;
public class HBaseMRTest extends Configured implements Tool {
@Override
public int run(String[] strings) throws Exception {
Configuration configuration = HBaseConfiguration.create();
Job job = new Job(configuration);
job.setJarByClass(HBaseMRTest.class);
Scan scan = new Scan();
// scan.addColumn(Bytes.toBytes("common"), Bytes.toBytes("common"));
/**
* 1. input tableName 2. scan 3.MapperClass
* 4.mapOutKeyClass 5.MapperOutValueClass 6.job
*/
TableMapReduceUtil.initTableMapperJob("AccessLog", scan, HBaseMRTestMapper.class, Text.class, IntWritable.class, job);
/**
* 1.output tableName 2. ReducerClass 3. job
*/
TableMapReduceUtil.initTableReducerJob("AccessLog", HBaseMRTestReducer.class, job);
job.setNumReduceTasks(1);
boolean flag = job.waitForCompletion(true);
if (flag) {
return 1;
} else {
return 0;
}
//job.setJobName("hbase-test-job");
//job.setOutputKeyClass(null);
//job.setOutputValueClass(null);
//job.setMapperClass(HBaseMRTestMapper.class);
//job.setReducerClass(HBaseMRTestReducer.class);
}
public static class HBaseMRTestMapper extends TableMapper<Text, IntWritable> {
private final IntWritable ONE = new IntWritable(1);
private Text text = new Text();
@Override
protected void map(ImmutableBytesWritable row, Result value, Context context) throws IOException, InterruptedException {
String ip = Bytes.toString(row.get());
// String url = new String(value.getValue(Bytes.toBytes("ip"), Bytes.toBytes("url")));
text.set(ip);
System.out.println("=======" + ip);
context.write(text, ONE);
}
}
public static class HBaseMRTestReducer extends TableReducer<Text, IntWritable, ImmutableBytesWritable> {
@Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
System.out.println("--------" + sum);
Put put = new Put(key.getBytes());
put.add(Bytes.toBytes("common"), Bytes.toBytes("count"), Bytes.toBytes(String.valueOf(sum)));
context.write(null, put);
}
}
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException, Exception {
HBaseMRTest test = new HBaseMRTest();
int res = test.run(args);
System.exit(res);
}
}
hbase-MR
發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.