Hadoop之MapReduce 根据用户流量日志文件数据统计每个用户流量总和

1.实现需求

1.根据手机号统计流量日志文件中的上行流量和下行流量，以及总流量
2.13开头的手机号写到文件一中，15开头的手机号写到文件二中，其它的手机号写到文件三中
3.手机号是第二列，上行流量是倒数第三列，下行流量是倒数第二列

1363157985066 	13726230503	00-FD-07-A4-72-B8:CMCC	120.196.100.82	i02.c.aliimg.com		24	27	2481	24681	200
1363157995052 	13826544101	5C-0E-8B-C7-F1-E0:CMCC	120.197.40.4			4	0	264	0	200
1363157991076 	13926435656	20-10-7A-28-CC-0A:CMCC	120.196.100.99			2	4	132	1512	200
1363154400022 	13926251106	5C-0E-8B-8B-B1-50:CMCC	120.197.40.4			4	0	240	0	200
1363157993044 	18211575961	94-71-AC-CD-E6-18:CMCC-EASY	120.196.100.99	iface.qiyi.com	视频网站	15	12	1527	2106	200
1363157995074 	84138413	5C-0E-8B-8C-E8-20:7DaysInn	120.197.40.4	122.72.52.12		20	16	4116	1432	200
1363157993055 	13560439658	C4-17-FE-BA-DE-D9:CMCC	120.196.100.99			18	15	1116	954	200
1363157995033 	15920133257	5C-0E-8B-C7-BA-20:CMCC	120.197.40.4	sug.so.360.cn	信息安全	20	20	3156	2936	200
1363157983019 	13719199419	68-A1-B7-03-07-B1:CMCC-EASY	120.196.100.82			4	0	240	0	200
1363157984041 	13660577991	5C-0E-8B-92-5C-20:CMCC-EASY	120.197.40.4	s19.cnzz.com	站点统计	24	9	6960	690	200
1363157973098 	15013685858	5C-0E-8B-C7-F7-90:CMCC	120.197.40.4	rank.ie.sogou.com	搜索引擎	28	27	3659	3538	200
1363157986029 	15989002119	E8-99-C4-4E-93-E0:CMCC-EASY	120.196.100.99	www.umeng.com	站点统计	3	3	1938	180	200
1363157992093 	13560439658	C4-17-FE-BA-DE-D9:CMCC	120.196.100.99			15	9	918	4938	200
1363157986041 	13480253104	5C-0E-8B-C7-FC-80:CMCC-EASY	120.197.40.4			3	3	180	180	200
1363157984040 	13602846565	5C-0E-8B-8B-B6-00:CMCC	120.197.40.4	2052.flash2-http.qq.com	综合门户	15	12	1938	2910	200
1363157995093 	13922314466	00-FD-07-A2-EC-BA:CMCC	120.196.100.82	img.qfc.cn		12	12	3008	3720	200
1363157982040 	13502468823	5C-0A-5B-6A-0B-D4:CMCC-EASY	120.196.100.99	y0.ifengimg.com	综合门户	57	102	7335	110349	200
1363157986072 	18320173382	84-25-DB-4F-10-1A:CMCC-EASY	120.196.100.99	input.shouji.sogou.com	搜索引擎	21	18	9531	2412	200
1363157990043 	13925057413	00-1F-64-E1-E6-9A:CMCC	120.196.100.55	t3.baidu.com	搜索引擎	69	63	11058	48243	200
1363157988072 	13760778710	00-FD-07-A4-7B-08:CMCC	120.196.100.82			2	2	120	120	200
1363157985066 	13726238888	00-FD-07-A4-72-B8:CMCC	120.196.100.82	i02.c.aliimg.com		24	27	2481	24681	200
1363157993055 	13560436666	C4-17-FE-BA-DE-D9:CMCC	120.196.100.99			18	15	1116	954	200
1363157985066 	13726238888	00-FD-07-A4-72-B8:CMCC	120.196.100.82	i02.c.aliimg.com		24	27	10000	20000	200

2.MapReduce编程模型

表层图解：

实现过程图解

• input
  读取文件
• splitting
  分割文件，框架自动完成
• mapping
  处理文件，以key，value的方式分类 ，需要自己实现
• combiner
  mapper端的聚合操作，优点：能减少IO，提升作业性能。局限性：求平均数这块就有问题了。可选
• shuffing
  把相同的key归类到一起，框架自动完成
• partitioner 输出分区，定义分区规则 可选
• Reducing
  处理相同的key的数据，需要自己实现
• Final result
  处理最后结果

3.编程实现

3.1 依赖

		<dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>

3.2 自定义复杂数据类型 Access

关键点有三个，为了网络传输的序列化和反序列化

• 1.实现Writable接口
• 2.实现write和readFields方法，并且里面的顺序要一致
• 3.定义一个默认的无参构造方法

package com.zc.bigdata.mapreduce;

import lombok.*;
import org.apache.hadoop.io.Writable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

/**
 * 自定义复杂类型
 * 1.实现Writable接口
 * 2.实现write和readFields方法，并且里面的顺序要一致
 * 3.定义一个默认的无参构造方法
 */
@Data
@NoArgsConstructor
public class Access implements Writable {
    private String phone;
    private long up;
    private long down;
    private long sum;

    public Access(String phone, long up, long down) {
        this.phone = phone;
        this.up = up;
        this.down = down;
        this.sum = down + up;

    }

    @Override
    public void write(DataOutput out) throws IOException {
        out.writeUTF(this.phone);
        out.writeLong(this.up);
        out.writeLong(this.down);
        out.writeLong(this.sum);
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        this.phone = in.readUTF();
        this.up = in.readLong();
        this.down = in.readLong();
        this.sum = in.readLong();
    }
}

3.3 重写Mapper

package com.zc.bigdata.mapreduce;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

/**
 * 自定义Mapper处理类
 */
public class AccessMapper extends Mapper<LongWritable,Text, Text, Access>{

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

        String[] lines = value.toString().split("\t");

        String phone = lines[1]; // 取出手机号
        long up = Long.parseLong(lines[lines.length-3]); //取出上行流量
        long down = Long.parseLong(lines[lines.length-2]); //取出下行流量

        context.write(new Text(phone), new Access(phone, up, down));
    }
}

3.4 重写Reducer

如果不想在文件中输出key，可以使用NullWritable，继承时的声明和启动类的Reducer输出都要记得改一下哦

context.write(NullWritable.get(), new Access(key.toString(), ups, downs));

package com.zc.bigdata.mapreduce;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class AccessReducer extends Reducer<Text, Access, Text, Access> {

    @Override
    protected void reduce(Text key, Iterable<Access> values, Context context) throws IOException, InterruptedException {
        long up = 0, down = 0;
        for (Access value : values) {
            up += value.getUp();
            down += value.getDown();
        }
        context.write(key, new Access(key.toString(), up, down));
    }
}

3.5 重写Partitioner

package com.zc.bigdata.mapreduce;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Partitioner;

public class AccessPartitioner extends Partitioner<Text, Access> {
    @Override
    public int getPartition(Text phone, Access access, int numPartitions) {
        if(phone.toString().startsWith("13")){
            return 0;
        }else if(phone.toString().startsWith("15")){
            return 1;
        }else {
            return 2;
        }
    }
}

3.6 实现Job启动类

package com.zc.bigdata.mapreduce;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.File;
import java.io.IOException;

public class AccessApp {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        // windows系统适配，还要下载hadoop-2.6.0，配置环境变量，替换windows/system32里面的2歌文件
        // 步骤还挺多
        System.setProperty("hadoop.home.dir","D://gitee//hadoop-2.6.0");

        // 创建job
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        // 设置驱动类
        job.setJarByClass(AccessApp.class);

        // 设置自定义的mapper和reducer
        job.setMapperClass(AccessMapper.class);
        job.setReducerClass(AccessReducer.class);

        // 设置mapper端的聚合规则
        job.setCombinerClass(AccessReducer.class);

        // 设置mapper的输出key,value类型和reducer的输出key,value类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Access.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Access.class);

        // 设置自定义分区规则
        job.setPartitionerClass(AccessPartitioner.class);
        // 设置Reducer个数
        job.setNumReduceTasks(3);

        // 提前删除输入目录，以免运行报错
        FileUtils.deleteDirectory(new File("output//access//"));

        // 设置输入文件夹和输出文件夹
        FileInputFormat.setInputPaths(job,new Path("input//access//"));
        FileOutputFormat.setOutputPath(job,new Path("output//access//"));

        // 执行job
        boolean result = job.waitForCompletion(true);
        System.out.println(result);

    }
}

3.7 运行AccessApp.main()

本地运行，执行成功，windows执行会报错，要做windows的hadoop适配，网上有很多适配的文章，推荐一个：https://blog.csdn.net/sunshine920103/article/details/52431138，如果没用，就试试其它文章，还挺麻烦的

part-r-00000：全是13开头的电话号码

part-r-00001：全是15开头的电话号码

part-r-00002：其它