大數據----使用MapReduce【1】

偏移量:
指的是每行行首字母移動到文本的最前面需要一定的字符

MapReduce的數據類型
LongWritable 長整型
IntWritable 整型
DoubleWritable 雙字節數值
FloatWritable 浮點型
Text 文本
BooleanWritable 布爾型數值

POM文件【配置文件】

<?xml version="1.0" encoding="UTF-8"?>

4.0.0

cn.itcast
mapreduce
1.0-SNAPSHOT


cloudera
https://repository.cloudera.com/artifactory/cloudera-repos/




org.apache.Hadoop
Hadoop-client
2.6.0-mr1-cdh5.14.0


org.apache.Hadoop
Hadoop-common
2.6.0-cdh5.14.0


org.apache.Hadoop
Hadoop-hdfs
2.6.0-cdh5.14.0

<dependency>
    <groupId>org.apache.Hadoop</groupId>
    <artifactId>Hadoop-mapreduce-client-core</artifactId>
    <version>2.6.0-cdh5.14.0</version>
</dependency>
<dependency>
    <groupId>junit</groupId>
    <artifactId>junit</artifactId>
    <version>4.11</version>
    <scope>test</scope>
</dependency>
<dependency>
    <groupId>org.testng</groupId>
    <artifactId>testng</artifactId>
    <version>RELEASE</version>
</dependency>

org.apache.maven.plugins maven-compiler-plugin 3.0 1.8 1.8 UTF-8

    <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <version>2.4.3</version>
        <executions>
            <execution>
                <phase>package</phase>
                <goals>
                    <goal>shade</goal>
                </goals>
                <configuration>
                    <minimizeJar>true</minimizeJar>
                </configuration>
            </execution>
        </executions>
    </plugin>

</plugins>

***編寫Map代碼***
*

public class WordCountMap extends Mapper<LongWritable, Text, Text, LongWritable> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String data = value.toString();
String[] split = data.split(" “);
int sum = 0;
for (String datas : split) {
if (”".equals(datas.trim())){
continue;
}
sum+=Integer.parseInt(datas.trim());
}
context.write(new Text(“數據總和”),new IntWritable(sum));
}
}*
編寫public class WordCountReduce extends Reducer<Text, LongWritable, Text, LongWritable> {
@Override
protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {
long sum =0;
for (IntWritable value : values) {
sum+=value.get();
}
context.write(new Text(key+“總和”),new LongWritable(sum));
}

編寫Reduce代碼

public class WordCountReduce extends Reducer<Text, LongWritable, Text, LongWritable> {
@Override
protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {
long sum =0;
for (IntWritable value : values) {
sum+=value.get();
}
context.write(new Text(key+“總和”),new LongWritable(sum));
}

編寫Driver代碼
public class WordCountDrive {

public static void main(String[] args) throws Exception {
Configuration configuration = new Configuration();
Job instance = Job.getInstance(configuration);
instance.setJarByClass(WordCountDrive.class);

instance.setMapperClass(WordCountMap.class);
instance.setMapOutputKeyClass(Text.class);
instance.setMapOutputValueClass(LongWritable.class);

instance.setReducerClass(WordCountReduce.class);
instance.setOutputKeyClass(Text.class);
instance.setOutputValueClass(LongWritable.class);

FileInputFormat.setInputPaths(instance, new Path("D:\\練習\\SumData.txt"));
FileSystem fs = FileSystem.get(configuration);
if (fs.exists(new Path("D:\\練習\\BBB"))) {
    fs.delete(new Path("D:\\練習\\BBB"), true);
}
FileOutputFormat.setOutputPath(instance, new Path("D:\BBB"));
boolean b = instance.waitForCompletion(true);
System.exit(b ? 0 : 1);
}

}