SpringBoot集成Hadoop系列二 ---- MapReduce明星微博統計

代碼:

package com.hadoop.reduce.model;

import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

/**
 * 統計明星微博數據實體
 * @author linhaiy
 * @date 2019.05.18
 */
public class Weibo implements WritableComparable<Weibo> {
	// 微博粉絲數
	private int friends;
	// 微博關注
	private int followers;
	// 發微博數目
	private int num;

	public Weibo() {
	}

	public Weibo(int friends, int followers, int num) {
		this.friends = friends;
		this.followers = followers;
		this.num = num;
	}

	public void set(int friends, int followers, int num) {
		this.friends = friends;
		this.followers = followers;
		this.num = num;
	}

	@Override
	public int compareTo(Weibo weibo) {
		return weibo.getFriends() - this.friends;
	}

	@Override
	public void write(DataOutput output) throws IOException {
		output.writeInt(friends);
		output.writeInt(followers);
		output.writeInt(num);
	}

	@Override
	public void readFields(DataInput input) throws IOException {
		friends = input.readInt();
		followers = input.readInt();
		num = input.readInt();
	}

	public int getFriends() {
		return friends;
	}

	public void setFriends(int friends) {
		this.friends = friends;
	}

	public int getFollowers() {
		return followers;
	}

	public void setFollowers(int followers) {
		this.followers = followers;
	}

	public int getNum() {
		return num;
	}

	public void setNum(int num) {
		this.num = num;
	}
}

package com.hadoop.reduce.mapper;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import com.hadoop.reduce.model.Weibo;

import java.io.IOException;

/**
 * 統計明星微博的粉絲，關注，微博數
 * @author linhaiy
 * @date 2019.05.18
 */
public class WeiboMapper extends Mapper<Text, Weibo, Text, Text> {
	/**
	 * 讀取 /java/weibo/weibo.txt 文件，內容格式如下 唐嫣 唐嫣 24301532 200 2391 明星名稱 名稱微博名 粉絲數
	 * 關注 微博數
	 * @param key
	 * @param value
	 * @param context
	 * @throws IOException
	 * @throws InterruptedException
	 */
	@Override
	protected void map(Text key, Weibo value, Context context) throws IOException, InterruptedException {
		// 輸出格式 key = friends values =
		// [{"friends":22898071,"followers":11,"num":268}...]
		context.write(new Text("friends"), new Text(key.toString() + "\t" + value.getFriends()));
		context.write(new Text("followers"), new Text(key.toString() + "\t" + value.getFollowers()));
		context.write(new Text("num"), new Text(key.toString() + "\t" + value.getNum()));
	}
}

package com.hadoop.reduce.reducer;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;

import com.hadoop.util.SortUtil;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

/**
 * 明星微博統計
 * @author linhaiy
 * @date 2019.05.18
 */
public class WeiboReduce extends Reducer<Text, Text, Text, IntWritable> {
	private MultipleOutputs<Text, IntWritable> outputs;

	@Override
	protected void setup(Context context) throws IOException, InterruptedException {
		outputs = new MultipleOutputs<>(context);
	}

	private Text text = new Text();

	/**
	 * 讀取 WeiboMapper的輸出，內容格式 key=friends, value= 姚晨 627 ...
	 * @param key
	 * @param values
	 * @param context
	 * @throws IOException
	 * @throws InterruptedException
	 */
	@Override
	protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
		// 將輸出內容放到map中
		Map<String, Integer> map = new HashMap<>();
		for (Text value : values) {
			String[] spilt = value.toString().split("\t");
			// 將數據放到map中
			map.put(spilt[0], Integer.parseInt(spilt[1].toString()));
		}

		// 對map內容排序
		Map.Entry<String, Integer>[] entries = SortUtil.sortHashMapByValue(map);
		// map排序後格式 [{"陳坤":73343207},{"姚晨":71382446}...]
		for (Map.Entry<String, Integer> entry : entries) {
			// 份文件輸出，格式: friends 陳坤 73343207
			outputs.write(key.toString(), entry.getKey(), entry.getValue());
		}
	}

	@Override
	protected void cleanup(Context context) throws IOException, InterruptedException {
		outputs.close();
	}
}

package com.hadoop.reduce.service;

import java.io.IOException;

import javax.annotation.PostConstruct;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import com.hadoop.reduce.bean.StaffProvincePartitioner;
import com.hadoop.reduce.bean.WeiboInputFormat;
import com.hadoop.reduce.mapper.CounterMapper;
import com.hadoop.reduce.mapper.FriendsMapper;
import com.hadoop.reduce.mapper.JoinMapper;
import com.hadoop.reduce.mapper.StaffMap;
import com.hadoop.reduce.mapper.WeatherMap;
import com.hadoop.reduce.mapper.WeiboMapper;
import com.hadoop.reduce.mapper.WordCount;
import com.hadoop.reduce.mapper.WordCountMap;
import com.hadoop.reduce.model.GroupSortModel;
import com.hadoop.reduce.model.OrderInfo;
import com.hadoop.reduce.model.StaffModel;
import com.hadoop.reduce.model.Weibo;
import com.hadoop.reduce.reducer.FriendsReduce;
import com.hadoop.reduce.reducer.JoinReduce;
import com.hadoop.reduce.reducer.StaffReduce;
import com.hadoop.reduce.reducer.WeatherReduce;
import com.hadoop.reduce.reducer.WeiboReduce;
import com.hadoop.reduce.reducer.WordCountReduce;
import com.hadoop.util.GroupSort;

/**
 * Map/Reduce工具類
 * @author linhaiy
 * @date 2019.05.18
 */
@Component
public class ReduceJobsUtils {

	@Value("${hdfs.path}")
	private String path;

	private static String hdfsPath;

	/**
	 * 獲取HDFS配置信息
	 * @return
	 */
	public static Configuration getConfiguration() {
		Configuration configuration = new Configuration();
		configuration.set("fs.defaultFS", hdfsPath);
		configuration.set("mapred.job.tracker", hdfsPath);
		// 運行在yarn的集羣模式
		// configuration.set("mapreduce.framework.name", "yarn");
		// 這個配置是讓main方法尋找該機器的mr環境
		// configuration.set("yarn.resourcemanmager.hostname", "node1");
		return configuration;
	}

	/**
	 * 明星微博統計
	 * 
	 * @param jobName
	 * @param inputPath
	 * @param outputPath
	 * @throws IOException
	 * @throws ClassNotFoundException
	 * @throws InterruptedException
	 */
	public static void weibo(String jobName, String inputPath, String outputPath)
			throws IOException, ClassNotFoundException, InterruptedException {
		Configuration conf = getConfiguration();
		Job job = Job.getInstance(conf, jobName);
		job.setJarByClass(Weibo.class);

		// 指定Mapper的類
		job.setMapperClass(WeiboMapper.class);
		// 指定reduce的類
		job.setReducerClass(WeiboReduce.class);

		// 設置Mapper的輸出
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(Text.class);

		// 設置Mapper輸出的類型
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);

		// 指定輸入文件的位置
		FileInputFormat.addInputPath(job, new Path(inputPath));
		// 指定輸入文件的位置
		FileOutputFormat.setOutputPath(job, new Path(outputPath));

		/**
		 * 自定義輸入輸出格式
		 */
		job.setInputFormatClass(WeiboInputFormat.class);
		MultipleOutputs.addNamedOutput(job, "friends", org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.class,
				Text.class, IntWritable.class);
		MultipleOutputs.addNamedOutput(job, "followers", org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.class,
				Text.class, IntWritable.class);
		MultipleOutputs.addNamedOutput(job, "num", org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.class,
				Text.class, IntWritable.class);

		// 將job中的參數，提交到yarn中運行
		job.waitForCompletion(true);
	}

	@PostConstruct
	public void getPath() {
		hdfsPath = this.path;
	}

	public static String getHdfsPath() {
		return hdfsPath;
	}
}

package com.hadoop.reduce.service;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.springframework.stereotype.Service;
import com.hadoop.hdfs.service.HdfsService;

/**
 * 單詞統計
 * @author linhaiy
 * @date 2019.05.18
 */
@Service
public class MapReduceService {

	// 默認reduce輸出目錄
	private static final String OUTPUT_PATH = "/output";

	/**
	 * 明星微博統計
	 * @param jobName
	 * @param inputPath
	 * @throws Exception
	 */
	public void weibo(String jobName, String inputPath) throws Exception {
		if (StringUtils.isEmpty(jobName) || StringUtils.isEmpty(inputPath)) {
			return;
		}
		// 輸出目錄 = output/當前Job
		String outputPath = OUTPUT_PATH + "/" + jobName;
		if (HdfsService.existFile(outputPath)) {
			HdfsService.deleteFile(outputPath);
		}
		ReduceJobsUtils.weibo(jobName, inputPath, outputPath);
	}
}

package com.hadoop.reduce.controller;

import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import com.hadoop.reduce.service.MapReduceService;
import com.hadoop.util.Result;

/**
 * MapReduce處理控制層
 * @author linhaiy
 * @date 2019.05.18
 */
@RestController
@RequestMapping("/hadoop/reduce")
public class MapReduceAction {

	@Autowired
	MapReduceService mapReduceService;

	/**
	 * 明星微博統計
	 * @param jobName
	 * @param inputPath
	 * @return
	 * @throws Exception
	 */
	@RequestMapping(value = "weibo", method = RequestMethod.POST)
	@ResponseBody
	public Result weibo(@RequestParam("jobName") String jobName, @RequestParam("inputPath") String inputPath)
			throws Exception {
		if (StringUtils.isEmpty(jobName) || StringUtils.isEmpty(inputPath)) {
			return new Result(Result.FAILURE, "請求參數爲空");
		}
		mapReduceService.weibo(jobName, inputPath);
		return new Result(Result.SUCCESS, "明星微博統計成功");
	}

}