MapReduce: WordCount的Eclipse實現

WordCountMapper.java

package cds.hadoop.wordcount;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.util.StringUtils;

/**
 * Mapper Task
 */
public class WordCountMapper extends Mapper<LongWritable, Text, Text, LongWritable> {

    @Override
    /**
     * key: line number;
     * value: a line of input
     */
    protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, LongWritable>.Context context)
            throws IOException, InterruptedException {
        // get the line
        String line = value.toString();

        // split a line onto words
        String[] words = StringUtils.split(line, ' ');

        // output <word, 1>
        for (String word : words) {
            context.write(new Text(word), new LongWritable(1L));

        }

    }

}

---

WordCountReducer.java

package cds.hadoop.wordcount;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

/**
 * Reducer Task
 * */
public class WordCountReducer extends Reducer<Text, LongWritable, Text, LongWritable> {

    @Override
    // input is <key, <v1,v2,...,vn>>
    protected void reduce(Text key, Iterable<LongWritable> values,
            Reducer<Text, LongWritable, Text, LongWritable>.Context context) throws IOException, InterruptedException {
        // count the number that every word appears
        long count = 0;
        for (LongWritable value : values) {
            count += value.get();
        }

        // output is <word, count>, it's what we want
        context.write(new Text(key), new LongWritable(count));

    }

}

---

WordCountRunner.java

package cds.hadoop.wordcount;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
 * describe a job:
 * which class is mapper?
 * which class is reducer?
 * where is the input file?
 * where to output?
 * 
 * Then submit this job to Hadoop
 * */
public class WordCountRunner {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration conf = new Configuration();
        Job wcjob = Job.getInstance(conf);

        wcjob.setJarByClass(WordCountRunner.class);

        // set Mapper
        wcjob.setMapperClass(WordCountMapper.class);
        wcjob.setMapOutputKeyClass(Text.class);
        wcjob.setMapOutputValueClass(LongWritable.class);

        // set Reducer
        wcjob.setReducerClass(WordCountReducer.class);
        wcjob.setOutputKeyClass(Text.class);
        wcjob.setOutputValueClass(LongWritable.class);

        // where is the input file?
        // FileInputFormat is a component of Job, but it has a default realization,
        // so we don't write a class of FileInputFormat in this demo
        FileInputFormat.setInputPaths(wcjob, "hdfs://master:9000/wc/srcData");

        // where to output?
        FileOutputFormat.setOutputPath(wcjob, new Path("hdfs://master:9000/wc/output"));

        boolean ret = wcjob.waitForCompletion(true);
    }

}

---

輸出與結果：

2016-04-04 08:28:39,173 WARN  [main] util.NativeCodeLoader (NativeCodeLoader.java:<clinit>(62)) - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2016-04-04 08:28:40,150 INFO  [main] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1173)) - session.id is deprecated. Instead, use dfs.metrics.session-id
2016-04-04 08:28:40,151 INFO  [main] jvm.JvmMetrics (JvmMetrics.java:init(76)) - Initializing JVM Metrics with processName=JobTracker, sessionId=
2016-04-04 08:28:40,707 WARN  [main] mapreduce.JobResourceUploader (JobResourceUploader.java:uploadFiles(64)) - Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
2016-04-04 08:28:40,757 WARN  [main] mapreduce.JobResourceUploader (JobResourceUploader.java:uploadFiles(171)) - No job jar file set.  User classes may not be found. See Job or Job#setJar(String).
2016-04-04 08:28:41,003 INFO  [main] input.FileInputFormat (FileInputFormat.java:listStatus(283)) - Total input paths to process : 1
2016-04-04 08:28:41,314 INFO  [main] mapreduce.JobSubmitter (JobSubmitter.java:submitJobInternal(198)) - number of splits:1
2016-04-04 08:28:41,604 INFO  [main] mapreduce.JobSubmitter (JobSubmitter.java:printTokens(287)) - Submitting tokens for job: job_local1963603364_0001
2016-04-04 08:28:42,074 INFO  [main] mapreduce.Job (Job.java:submit(1294)) - The url to track the job: http://localhost:8080/
2016-04-04 08:28:42,075 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1339)) - Running job: job_local1963603364_0001
2016-04-04 08:28:42,121 INFO  [Thread-15] mapred.LocalJobRunner (LocalJobRunner.java:createOutputCommitter(471)) - OutputCommitter set in config null
2016-04-04 08:28:42,124 INFO  [Thread-15] output.FileOutputCommitter (FileOutputCommitter.java:<init>(100)) - File Output Committer Algorithm version is 1
2016-04-04 08:28:42,126 INFO  [Thread-15] mapred.LocalJobRunner (LocalJobRunner.java:createOutputCommitter(489)) - OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
2016-04-04 08:28:42,302 INFO  [Thread-15] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(448)) - Waiting for map tasks
2016-04-04 08:28:42,303 INFO  [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:run(224)) - Starting task: attempt_local1963603364_0001_m_000000_0
2016-04-04 08:28:42,543 INFO  [LocalJobRunner Map Task Executor #0] output.FileOutputCommitter (FileOutputCommitter.java:<init>(100)) - File Output Committer Algorithm version is 1
2016-04-04 08:28:42,612 INFO  [LocalJobRunner Map Task Executor #0] mapred.Task (Task.java:initialize(612)) -  Using ResourceCalculatorProcessTree : [ ]
2016-04-04 08:28:42,614 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:runNewMapper(756)) - Processing split: hdfs://master:9000/wc/srcData/testText.txt:0+1678
2016-04-04 08:28:43,130 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1360)) - Job job_local1963603364_0001 running in uber mode : false
2016-04-04 08:28:43,132 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1367)) -  map 0% reduce 0%
2016-04-04 08:28:44,978 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:setEquator(1205)) - (EQUATOR) 0 kvi 26214396(104857584)
2016-04-04 08:28:44,979 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:init(998)) - mapreduce.task.io.sort.mb: 100
2016-04-04 08:28:44,979 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:init(999)) - soft limit at 83886080
2016-04-04 08:28:44,979 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:init(1000)) - bufstart = 0; bufvoid = 104857600
2016-04-04 08:28:44,979 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:init(1001)) - kvstart = 26214396; length = 6553600
2016-04-04 08:28:44,983 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:createSortingCollector(403)) - Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
2016-04-04 08:28:45,404 INFO  [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) - 
2016-04-04 08:28:45,410 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1460)) - Starting flush of map output
2016-04-04 08:28:45,410 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1482)) - Spilling map output
2016-04-04 08:28:45,410 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1483)) - bufstart = 0; bufend = 3718; bufvoid = 104857600
2016-04-04 08:28:45,410 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1485)) - kvstart = 26214396(104857584); kvend = 26213380(104853520); length = 1017/6553600
2016-04-04 08:28:45,468 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:sortAndSpill(1667)) - Finished spill 0
2016-04-04 08:28:45,536 INFO  [LocalJobRunner Map Task Executor #0] mapred.Task (Task.java:done(1038)) - Task:attempt_local1963603364_0001_m_000000_0 is done. And is in the process of committing
2016-04-04 08:28:45,548 INFO  [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) - map
2016-04-04 08:28:45,548 INFO  [LocalJobRunner Map Task Executor #0] mapred.Task (Task.java:sendDone(1158)) - Task 'attempt_local1963603364_0001_m_000000_0' done.
2016-04-04 08:28:45,548 INFO  [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:run(249)) - Finishing task: attempt_local1963603364_0001_m_000000_0
2016-04-04 08:28:45,550 INFO  [Thread-15] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(456)) - map task executor complete.
2016-04-04 08:28:45,597 INFO  [Thread-15] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(448)) - Waiting for reduce tasks
2016-04-04 08:28:45,598 INFO  [pool-6-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:run(302)) - Starting task: attempt_local1963603364_0001_r_000000_0
2016-04-04 08:28:45,651 INFO  [pool-6-thread-1] output.FileOutputCommitter (FileOutputCommitter.java:<init>(100)) - File Output Committer Algorithm version is 1
2016-04-04 08:28:45,651 INFO  [pool-6-thread-1] mapred.Task (Task.java:initialize(612)) -  Using ResourceCalculatorProcessTree : [ ]
2016-04-04 08:28:45,665 INFO  [pool-6-thread-1] mapred.ReduceTask (ReduceTask.java:run(362)) - Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@4d739343
2016-04-04 08:28:45,735 INFO  [pool-6-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:<init>(197)) - MergerManager: memoryLimit=671245888, maxSingleShuffleLimit=167811472, mergeThreshold=443022304, ioSortFactor=10, memToMemMergeOutputsThreshold=10
2016-04-04 08:28:45,848 INFO  [EventFetcher for fetching Map Completion Events] reduce.EventFetcher (EventFetcher.java:run(61)) - attempt_local1963603364_0001_r_000000_0 Thread started: EventFetcher for fetching Map Completion Events
2016-04-04 08:28:45,987 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1367)) -  map 100% reduce 0%
2016-04-04 08:28:46,004 INFO  [localfetcher#1] reduce.LocalFetcher (LocalFetcher.java:copyMapOutput(144)) - localfetcher#1 about to shuffle output of map attempt_local1963603364_0001_m_000000_0 decomp: 4230 len: 4234 to MEMORY
2016-04-04 08:28:46,006 INFO  [localfetcher#1] reduce.InMemoryMapOutput (InMemoryMapOutput.java:shuffle(100)) - Read 4230 bytes from map-output for attempt_local1963603364_0001_m_000000_0
2016-04-04 08:28:46,007 INFO  [localfetcher#1] reduce.MergeManagerImpl (MergeManagerImpl.java:closeInMemoryFile(315)) - closeInMemoryFile -> map-output of size: 4230, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->4230
2016-04-04 08:28:46,040 INFO  [EventFetcher for fetching Map Completion Events] reduce.EventFetcher (EventFetcher.java:run(76)) - EventFetcher is interrupted.. Returning
2016-04-04 08:28:46,041 INFO  [pool-6-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) - 1 / 1 copied.
2016-04-04 08:28:46,041 INFO  [pool-6-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(687)) - finalMerge called with 1 in-memory map-outputs and 0 on-disk map-outputs
2016-04-04 08:28:46,050 INFO  [pool-6-thread-1] mapred.Merger (Merger.java:merge(606)) - Merging 1 sorted segments
2016-04-04 08:28:46,050 INFO  [pool-6-thread-1] mapred.Merger (Merger.java:merge(705)) - Down to the last merge-pass, with 1 segments left of total size: 4227 bytes
2016-04-04 08:28:46,108 INFO  [pool-6-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(754)) - Merged 1 segments, 4230 bytes to disk to satisfy reduce memory limit
2016-04-04 08:28:46,109 INFO  [pool-6-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(784)) - Merging 1 files, 4234 bytes from disk
2016-04-04 08:28:46,109 INFO  [pool-6-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(799)) - Merging 0 segments, 0 bytes from memory into reduce
2016-04-04 08:28:46,109 INFO  [pool-6-thread-1] mapred.Merger (Merger.java:merge(606)) - Merging 1 sorted segments
2016-04-04 08:28:46,109 INFO  [pool-6-thread-1] mapred.Merger (Merger.java:merge(705)) - Down to the last merge-pass, with 1 segments left of total size: 4227 bytes
2016-04-04 08:28:46,110 INFO  [pool-6-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) - 1 / 1 copied.
2016-04-04 08:28:46,164 INFO  [pool-6-thread-1] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1173)) - mapred.skip.on is deprecated. Instead, use mapreduce.job.skiprecords
2016-04-04 08:28:46,701 INFO  [pool-6-thread-1] mapred.Task (Task.java:done(1038)) - Task:attempt_local1963603364_0001_r_000000_0 is done. And is in the process of committing
2016-04-04 08:28:46,707 INFO  [pool-6-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) - 1 / 1 copied.
2016-04-04 08:28:46,707 INFO  [pool-6-thread-1] mapred.Task (Task.java:commit(1199)) - Task attempt_local1963603364_0001_r_000000_0 is allowed to commit now
2016-04-04 08:28:46,727 INFO  [pool-6-thread-1] output.FileOutputCommitter (FileOutputCommitter.java:commitTask(482)) - Saved output of task 'attempt_local1963603364_0001_r_000000_0' to hdfs://master:9000/wc/output/_temporary/0/task_local1963603364_0001_r_000000
2016-04-04 08:28:46,754 INFO  [pool-6-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) - reduce > reduce
2016-04-04 08:28:46,754 INFO  [pool-6-thread-1] mapred.Task (Task.java:sendDone(1158)) - Task 'attempt_local1963603364_0001_r_000000_0' done.
2016-04-04 08:28:46,754 INFO  [pool-6-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:run(325)) - Finishing task: attempt_local1963603364_0001_r_000000_0
2016-04-04 08:28:46,754 INFO  [Thread-15] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(456)) - reduce task executor complete.
2016-04-04 08:28:46,994 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1367)) -  map 100% reduce 100%
2016-04-04 08:28:46,995 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1378)) - Job job_local1963603364_0001 completed successfully
2016-04-04 08:28:47,102 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1385)) - Counters: 35
    File System Counters
        FILE: Number of bytes read=8820
        FILE: Number of bytes written=574570
        FILE: Number of read operations=0
        FILE: Number of large read operations=0
        FILE: Number of write operations=0
        HDFS: Number of bytes read=3356
        HDFS: Number of bytes written=1637
        HDFS: Number of read operations=13
        HDFS: Number of large read operations=0
        HDFS: Number of write operations=4
    Map-Reduce Framework
        Map input records=23
        Map output records=255
        Map output bytes=3718
        Map output materialized bytes=4234
        Input split bytes=107
        Combine input records=0
        Combine output records=0
        Reduce input groups=172
        Reduce shuffle bytes=4234
        Reduce input records=255
        Reduce output records=172
        Spilled Records=510
        Shuffled Maps =1
        Failed Shuffles=0
        Merged Map outputs=1
        GC time elapsed (ms)=57
        Total committed heap usage (bytes)=331489280
    Shuffle Errors
        BAD_ID=0
        CONNECTION=0
        IO_ERROR=0
        WRONG_LENGTH=0
        WRONG_MAP=0
        WRONG_REDUCE=0
    File Input Format Counters 
        Bytes Read=1678
    File Output Format Counters 
        Bytes Written=1637