windows10上Eclipse运行MapReduce wordcount程序遇到的坑

我的wordcount代码：

package com.walloce.wordcount;

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import com.walloce.wordcount.MyMapReduce.myMap.myReduce;

public class MyMapReduce extends Configured implements Tool {
    
    /**
     * KEYIN LongWritable 传入的key类型(偏移量)
     * VALUEIN Text 传入的value类型(文本)
     * KEYOUT 传出的key类型
     * VALUEOUT 传出的value类型
     * @author Walloce
     * 2018
     */
    static class myMap extends Mapper<LongWritable, Text, Text, IntWritable> {
        //输出结果的key
        Text text = new Text();
        //输出结果的value
        IntWritable mr_value = new IntWritable(1);
        
        int line_count = 1;
        
        @Override
        protected void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {
            
            System.out.println("map阶段开始...");
            //将获取的文本类型转为字符串类型
            String line = value.toString();
            
            System.out.println("第 "+ line_count +" 行的字符串的偏移量为：" + key.get());
            
            //将的到的一行字符串拆解为多个单词的字符串数组
            String[] words = line.split(" ");
            
            //遍历得到的所有word
            for (String word : words) {
                text.set(word);
                context.write(text, mr_value);
            }
            line_count++;
        }
        
        /**
         * Text, IntWritable, Text, IntWritable
         * reduce时输入的key-value和输出的key-value.
         * eg:(hello,2)
         * @author Walloce
         * 2018
         */
        static class myReduce extends Reducer<Text, IntWritable, Text, IntWritable> {
            
            private IntWritable result = new IntWritable();
            
            int reduce_time = 0;
            
            @Override
            protected void reduce(Text key, Iterable<IntWritable> values, Context context)
                    throws IOException, InterruptedException {
                System.out.println("这是第"+ reduce_time +"次reduce");
                System.out.println("Reduce阶段开始....");
                
                int sum = 0;
                for (IntWritable value : values) {
                    sum += value.get();
                }
                result.set(sum);
                context.write(key, result);
                reduce_time++;
            }
            
        }

    }
    
    public int run(String[] args) throws Exception {
        
        //hadoop的八股文
        Configuration conf = this.getConf();
        Job job = Job.getInstance(conf, this.getClass().getSimpleName());
        //对job进行具体的配置
        
        //当你本地运行，这个设置可以不写，不会报错
        //当提价到集群上面运行的时候，这个设置不写，会报类找不到的异常 
        job.setJarByClass(MyMapReduce.class);
        
        //写一个输入路径
        Path input = new Path(args[0]);
        FileInputFormat.addInputPath(job, input);
        //写一个输出路径
        Path output = new Path(args[1]);
        FileOutputFormat.setOutputPath(job, output);
        
        //执行前先判断输出路径是否存在，存在就删除
        FileSystem fs = output.getFileSystem(conf);
        if(fs.exists(output)){
            fs.delete(output,true);
        }
        
        //设置运行的map类的相关参数
        job.setMapperClass(myMap.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        //==============shuffle=======================
//        job.setCombinerClass(MyCombiner.class);
        
        
        //==============shuffle=======================
        //设置运行的Reduce的相关参数
        job.setReducerClass(myReduce.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        
        boolean isSuccess =  job.waitForCompletion(true);
        
        
        return isSuccess?0:1;
    }
    
    public static void main(String[] args) {
        // TODO Auto-generated method stub
        Configuration conf = new Configuration();
        args = new String[]{
                "hdfs://bigdata-study-104:8020/testdata/word.txt",
                "hdfs://bigdata-study-104:8020/testresult/output/"
        };

        try {
            ToolRunner.run(conf, new MyMapReduce(), args);
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

}

 

异常信息：

18/03/22 20:41:24 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
18/03/22 20:41:25 INFO Configuration.deprecation: session.id is deprecated. Instead, use dfs.metrics.session-id
18/03/22 20:41:25 INFO jvm.JvmMetrics: Initializing JVM Metrics with processName=JobTracker, sessionId=
18/03/22 20:41:25 WARN mapreduce.JobSubmitter: No job jar file set.  User classes may not be found. See Job or Job#setJar(String).
18/03/22 20:41:25 INFO input.FileInputFormat: Total input paths to process : 1
18/03/22 20:41:26 INFO mapreduce.JobSubmitter: number of splits:1
18/03/22 20:41:26 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_local1064879045_0001
18/03/22 20:41:26 WARN conf.Configuration: file:/opt/modules/cdh/hadoop-2.5.0-cdh5.3.6/data/temp/mapred/staging/Walloce1064879045/.staging/job_local1064879045_0001/job.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.retry.interval;  Ignoring.
18/03/22 20:41:26 WARN conf.Configuration: file:/opt/modules/cdh/hadoop-2.5.0-cdh5.3.6/data/temp/mapred/staging/Walloce1064879045/.staging/job_local1064879045_0001/job.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.attempts;  Ignoring.
18/03/22 20:41:26 INFO mapreduce.JobSubmitter: Cleaning up the staging area file:/opt/modules/cdh/hadoop-2.5.0-cdh5.3.6/data/temp/mapred/staging/Walloce1064879045/.staging/job_local1064879045_0001
Exception in thread "main" java.lang.UnsatisfiedLinkError: org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Ljava/lang/String;I)Z
    at org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Native Method)
    at org.apache.hadoop.io.nativeio.NativeIO$Windows.access(NativeIO.java:571)
    at org.apache.hadoop.fs.FileUtil.canRead(FileUtil.java:977)
    at org.apache.hadoop.util.DiskChecker.checkAccessByFileMethods(DiskChecker.java:173)
    at org.apache.hadoop.util.DiskChecker.checkDirAccess(DiskChecker.java:160)
    at org.apache.hadoop.util.DiskChecker.checkDir(DiskChecker.java:94)
    at org.apache.hadoop.fs.LocalDirAllocator$AllocatorPerContext.confChanged(LocalDirAllocator.java:285)
    at org.apache.hadoop.fs.LocalDirAllocator$AllocatorPerContext.getLocalPathForWrite(LocalDirAllocator.java:344)
    at org.apache.hadoop.fs.LocalDirAllocator.getLocalPathForWrite(LocalDirAllocator.java:150)
    at org.apache.hadoop.fs.LocalDirAllocator.getLocalPathForWrite(LocalDirAllocator.java:131)
    at org.apache.hadoop.fs.LocalDirAllocator.getLocalPathForWrite(LocalDirAllocator.java:115)
    at org.apache.hadoop.mapred.LocalDistributedCacheManager.setup(LocalDistributedCacheManager.java:131)
    at org.apache.hadoop.mapred.LocalJobRunner$Job.<init>(LocalJobRunner.java:163)
    at org.apache.hadoop.mapred.LocalJobRunner.submitJob(LocalJobRunner.java:731)
    at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:432)
    at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1285)
    at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1282)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Unknown Source)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1614)
    at org.apache.hadoop.mapreduce.Job.submit(Job.java:1282)
    at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1303)
    at com.walloce.wordcount.MyMapReduce.run(MyMapReduce.java:130)
    at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
    at com.walloce.wordcount.MyMapReduce.main(MyMapReduce.java:145)

 

修改代码：将570行注释掉，直接return true;

 

再运行还是报错！！！

18/03/22 20:53:42 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
18/03/22 20:53:42 INFO Configuration.deprecation: session.id is deprecated. Instead, use dfs.metrics.session-id
18/03/22 20:53:42 INFO jvm.JvmMetrics: Initializing JVM Metrics with processName=JobTracker, sessionId=
18/03/22 20:53:43 WARN mapreduce.JobSubmitter: No job jar file set.  User classes may not be found. See Job or Job#setJar(String).
18/03/22 20:53:43 INFO input.FileInputFormat: Total input paths to process : 1
18/03/22 20:53:43 INFO mapreduce.JobSubmitter: number of splits:1
18/03/22 20:53:43 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_local105563724_0001
18/03/22 20:53:43 INFO mapreduce.Job: The url to track the job: http://localhost:8080/
18/03/22 20:53:43 INFO mapreduce.Job: Running job: job_local105563724_0001
18/03/22 20:53:43 INFO mapred.LocalJobRunner: OutputCommitter set in config null
18/03/22 20:53:43 INFO mapred.LocalJobRunner: OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
18/03/22 20:53:43 INFO mapred.LocalJobRunner: Waiting for map tasks
18/03/22 20:53:43 INFO mapred.LocalJobRunner: Starting task: attempt_local105563724_0001_m_000000_0
18/03/22 20:53:43 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
18/03/22 20:53:43 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@4a86a3cd
18/03/22 20:53:43 INFO mapred.MapTask: Processing split: hdfs://bigdata-study-104:8020/testdata/word.txt:0+210
18/03/22 20:53:43 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
18/03/22 20:53:43 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
18/03/22 20:53:43 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
18/03/22 20:53:43 INFO mapred.MapTask: soft limit at 83886080
18/03/22 20:53:43 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
18/03/22 20:53:43 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
map阶段开始...
第 1 行的字符串的偏移量为：0
map阶段开始...
第 2 行的字符串的偏移量为：46
map阶段开始...
第 3 行的字符串的偏移量为：86
map阶段开始...
第 4 行的字符串的偏移量为：120
map阶段开始...
第 5 行的字符串的偏移量为：149
map阶段开始...
第 6 行的字符串的偏移量为：172
map阶段开始...
第 7 行的字符串的偏移量为：190
map阶段开始...
第 8 行的字符串的偏移量为：204
18/03/22 20:53:44 INFO mapred.LocalJobRunner: 
18/03/22 20:53:44 INFO mapred.MapTask: Starting flush of map output
18/03/22 20:53:44 INFO mapred.MapTask: Spilling map output
18/03/22 20:53:44 INFO mapred.MapTask: bufstart = 0; bufend = 348; bufvoid = 104857600
18/03/22 20:53:44 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 26214256(104857024); length = 141/6553600
18/03/22 20:53:44 INFO mapred.MapTask: Finished spill 0
18/03/22 20:53:44 INFO mapred.Task: Task:attempt_local105563724_0001_m_000000_0 is done. And is in the process of committing
18/03/22 20:53:44 INFO mapred.LocalJobRunner: map
18/03/22 20:53:44 INFO mapred.Task: Task 'attempt_local105563724_0001_m_000000_0' done.
18/03/22 20:53:44 INFO mapred.LocalJobRunner: Finishing task: attempt_local105563724_0001_m_000000_0
18/03/22 20:53:44 INFO mapred.LocalJobRunner: map task executor complete.
18/03/22 20:53:44 INFO mapred.LocalJobRunner: Waiting for reduce tasks
18/03/22 20:53:44 INFO mapred.LocalJobRunner: Starting task: attempt_local105563724_0001_r_000000_0
18/03/22 20:53:44 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
18/03/22 20:53:44 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@66a2f2c4
18/03/22 20:53:44 INFO mapred.ReduceTask: Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@7c3f147f
18/03/22 20:53:44 INFO reduce.MergeManagerImpl: MergerManager: memoryLimit=1323407744, maxSingleShuffleLimit=330851936, mergeThreshold=873449152, ioSortFactor=10, memToMemMergeOutputsThreshold=10
18/03/22 20:53:44 INFO reduce.EventFetcher: attempt_local105563724_0001_r_000000_0 Thread started: EventFetcher for fetching Map Completion Events
18/03/22 20:53:44 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local105563724_0001_m_000000_0 decomp: 422 len: 426 to MEMORY
18/03/22 20:53:44 INFO reduce.InMemoryMapOutput: Read 422 bytes from map-output for attempt_local105563724_0001_m_000000_0
18/03/22 20:53:44 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 422, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->422
18/03/22 20:53:44 INFO reduce.EventFetcher: EventFetcher is interrupted.. Returning
18/03/22 20:53:44 INFO mapred.LocalJobRunner: 1 / 1 copied.
18/03/22 20:53:44 INFO reduce.MergeManagerImpl: finalMerge called with 1 in-memory map-outputs and 0 on-disk map-outputs
18/03/22 20:53:44 INFO mapred.Merger: Merging 1 sorted segments
18/03/22 20:53:44 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 413 bytes
18/03/22 20:53:44 INFO reduce.MergeManagerImpl: Merged 1 segments, 422 bytes to disk to satisfy reduce memory limit
18/03/22 20:53:44 INFO reduce.MergeManagerImpl: Merging 1 files, 426 bytes from disk
18/03/22 20:53:44 INFO reduce.MergeManagerImpl: Merging 0 segments, 0 bytes from memory into reduce
18/03/22 20:53:44 INFO mapred.Merger: Merging 1 sorted segments
18/03/22 20:53:44 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 413 bytes
18/03/22 20:53:44 INFO mapred.LocalJobRunner: 1 / 1 copied.
18/03/22 20:53:44 INFO mapred.LocalJobRunner: reduce task executor complete.
18/03/22 20:53:44 WARN mapred.LocalJobRunner: job_local105563724_0001
java.lang.Exception: java.lang.NoSuchMethodError: org.apache.hadoop.fs.FSOutputSummer.<init>(Ljava/util/zip/Checksum;II)V
    at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
    at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:529)
Caused by: java.lang.NoSuchMethodError: org.apache.hadoop.fs.FSOutputSummer.<init>(Ljava/util/zip/Checksum;II)V
    at org.apache.hadoop.hdfs.DFSOutputStream.<init>(DFSOutputStream.java:1563)
    at org.apache.hadoop.hdfs.DFSOutputStream.<init>(DFSOutputStream.java:1594)
    at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1626)
    at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1488)
    at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1413)
    at org.apache.hadoop.hdfs.DistributedFileSystem$6.doCall(DistributedFileSystem.java:387)
    at org.apache.hadoop.hdfs.DistributedFileSystem$6.doCall(DistributedFileSystem.java:383)
    at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
    at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:383)
    at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:327)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:908)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:889)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:786)
    at org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.getRecordWriter(TextOutputFormat.java:132)
    at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.<init>(ReduceTask.java:540)
    at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:614)
    at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389)
    at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:319)
    at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source)
    at java.util.concurrent.FutureTask.run(Unknown Source)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
    at java.lang.Thread.run(Unknown Source)
18/03/22 20:53:44 INFO mapreduce.Job: Job job_local105563724_0001 running in uber mode : false
18/03/22 20:53:44 INFO mapreduce.Job:  map 100% reduce 0%
18/03/22 20:53:44 INFO mapreduce.Job: Job job_local105563724_0001 failed with state FAILED due to: NA
18/03/22 20:53:44 INFO mapreduce.Job: Counters: 38
    File System Counters
        FILE: Number of bytes read=175
        FILE: Number of bytes written=262184
        FILE: Number of read operations=0
        FILE: Number of large read operations=0
        FILE: Number of write operations=0
        HDFS: Number of bytes read=210
        HDFS: Number of bytes written=0
        HDFS: Number of read operations=6
        HDFS: Number of large read operations=0
        HDFS: Number of write operations=2
    Map-Reduce Framework
        Map input records=8
        Map output records=36
        Map output bytes=348
        Map output materialized bytes=426
        Input split bytes=112
        Combine input records=0
        Combine output records=0
        Reduce input groups=0
        Reduce shuffle bytes=426
        Reduce input records=0
        Reduce output records=0
        Spilled Records=36
        Shuffled Maps =1
        Failed Shuffles=0
        Merged Map outputs=1
        GC time elapsed (ms)=0
        CPU time spent (ms)=0
        Physical memory (bytes) snapshot=0
        Virtual memory (bytes) snapshot=0
        Total committed heap usage (bytes)=232783872
    Shuffle Errors
        BAD_ID=0
        CONNECTION=0
        IO_ERROR=0
        WRONG_LENGTH=0
        WRONG_MAP=0
        WRONG_REDUCE=0
    File Input Format Counters 
        Bytes Read=210
    File Output Format Counters 
        Bytes Written=0

 

主要异常信息：

 java.lang.Exception: java.lang.NoSuchMethodError: org.apache.hadoop.fs.FSOutputSummer.<init>(Ljava/util/zip/Checksum;II)V 

 

好不容易解决一个，又来一个....要崩溃了，继续百度！

问题大致的原因是因为jar包冲突了，所以我buildpath看下导入的jar包有哪些，结果发现问题了。。。。。

 

同时存在cdh版本的和apache版本的hadoop-common-2.5.0jar包，删掉cdh版本的后运行结果：

18/03/22 20:57:26 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
18/03/22 20:57:26 INFO Configuration.deprecation: session.id is deprecated. Instead, use dfs.metrics.session-id
18/03/22 20:57:26 INFO jvm.JvmMetrics: Initializing JVM Metrics with processName=JobTracker, sessionId=
18/03/22 20:57:27 WARN mapreduce.JobSubmitter: No job jar file set.  User classes may not be found. See Job or Job#setJar(String).
18/03/22 20:57:27 INFO input.FileInputFormat: Total input paths to process : 1
18/03/22 20:57:27 INFO mapreduce.JobSubmitter: number of splits:1
18/03/22 20:57:27 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_local1889225814_0001
18/03/22 20:57:27 WARN conf.Configuration: file:/opt/modules/cdh/hadoop-2.5.0-cdh5.3.6/data/temp/mapred/staging/Walloce1889225814/.staging/job_local1889225814_0001/job.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.retry.interval;  Ignoring.
18/03/22 20:57:27 WARN conf.Configuration: file:/opt/modules/cdh/hadoop-2.5.0-cdh5.3.6/data/temp/mapred/staging/Walloce1889225814/.staging/job_local1889225814_0001/job.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.attempts;  Ignoring.
18/03/22 20:57:27 WARN conf.Configuration: file:/opt/modules/cdh/hadoop-2.5.0-cdh5.3.6/data/temp/mapred/local/localRunner/Walloce/job_local1889225814_0001/job_local1889225814_0001.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.retry.interval;  Ignoring.
18/03/22 20:57:27 WARN conf.Configuration: file:/opt/modules/cdh/hadoop-2.5.0-cdh5.3.6/data/temp/mapred/local/localRunner/Walloce/job_local1889225814_0001/job_local1889225814_0001.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.attempts;  Ignoring.
18/03/22 20:57:27 INFO mapreduce.Job: The url to track the job: http://localhost:8080/
18/03/22 20:57:27 INFO mapreduce.Job: Running job: job_local1889225814_0001
18/03/22 20:57:27 INFO mapred.LocalJobRunner: OutputCommitter set in config null
18/03/22 20:57:27 INFO mapred.LocalJobRunner: OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
18/03/22 20:57:28 INFO mapred.LocalJobRunner: Waiting for map tasks
18/03/22 20:57:28 INFO mapred.LocalJobRunner: Starting task: attempt_local1889225814_0001_m_000000_0
18/03/22 20:57:28 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
18/03/22 20:57:28 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@73a61b48
18/03/22 20:57:28 INFO mapred.MapTask: Processing split: hdfs://bigdata-study-104:8020/testdata/word.txt:0+210
18/03/22 20:57:28 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
18/03/22 20:57:28 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
18/03/22 20:57:28 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
18/03/22 20:57:28 INFO mapred.MapTask: soft limit at 83886080
18/03/22 20:57:28 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
18/03/22 20:57:28 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
map阶段开始...
第 1 行的字符串的偏移量为：0
map阶段开始...
第 2 行的字符串的偏移量为：46
map阶段开始...
第 3 行的字符串的偏移量为：86
map阶段开始...
第 4 行的字符串的偏移量为：120
map阶段开始...
第 5 行的字符串的偏移量为：149
map阶段开始...
第 6 行的字符串的偏移量为：172
map阶段开始...
第 7 行的字符串的偏移量为：190
map阶段开始...
第 8 行的字符串的偏移量为：204
18/03/22 20:57:28 INFO mapred.LocalJobRunner: 
18/03/22 20:57:28 INFO mapred.MapTask: Starting flush of map output
18/03/22 20:57:28 INFO mapred.MapTask: Spilling map output
18/03/22 20:57:28 INFO mapred.MapTask: bufstart = 0; bufend = 348; bufvoid = 104857600
18/03/22 20:57:28 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 26214256(104857024); length = 141/6553600
18/03/22 20:57:28 INFO mapred.MapTask: Finished spill 0
18/03/22 20:57:28 INFO mapred.Task: Task:attempt_local1889225814_0001_m_000000_0 is done. And is in the process of committing
18/03/22 20:57:28 INFO mapred.LocalJobRunner: map
18/03/22 20:57:28 INFO mapred.Task: Task 'attempt_local1889225814_0001_m_000000_0' done.
18/03/22 20:57:28 INFO mapred.LocalJobRunner: Finishing task: attempt_local1889225814_0001_m_000000_0
18/03/22 20:57:28 INFO mapred.LocalJobRunner: map task executor complete.
18/03/22 20:57:28 INFO mapred.LocalJobRunner: Waiting for reduce tasks
18/03/22 20:57:28 INFO mapred.LocalJobRunner: Starting task: attempt_local1889225814_0001_r_000000_0
18/03/22 20:57:28 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
18/03/22 20:57:28 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@504037fc
18/03/22 20:57:28 INFO mapred.ReduceTask: Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@1de1d428
18/03/22 20:57:28 INFO reduce.MergeManagerImpl: MergerManager: memoryLimit=1323407744, maxSingleShuffleLimit=330851936, mergeThreshold=873449152, ioSortFactor=10, memToMemMergeOutputsThreshold=10
18/03/22 20:57:28 INFO reduce.EventFetcher: attempt_local1889225814_0001_r_000000_0 Thread started: EventFetcher for fetching Map Completion Events
18/03/22 20:57:28 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local1889225814_0001_m_000000_0 decomp: 422 len: 426 to MEMORY
18/03/22 20:57:28 INFO reduce.InMemoryMapOutput: Read 422 bytes from map-output for attempt_local1889225814_0001_m_000000_0
18/03/22 20:57:28 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 422, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->422
18/03/22 20:57:28 INFO reduce.EventFetcher: EventFetcher is interrupted.. Returning
18/03/22 20:57:28 INFO mapred.LocalJobRunner: 1 / 1 copied.
18/03/22 20:57:28 INFO reduce.MergeManagerImpl: finalMerge called with 1 in-memory map-outputs and 0 on-disk map-outputs
18/03/22 20:57:28 INFO mapred.Merger: Merging 1 sorted segments
18/03/22 20:57:28 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 413 bytes
18/03/22 20:57:28 INFO reduce.MergeManagerImpl: Merged 1 segments, 422 bytes to disk to satisfy reduce memory limit
18/03/22 20:57:28 INFO reduce.MergeManagerImpl: Merging 1 files, 426 bytes from disk
18/03/22 20:57:28 INFO reduce.MergeManagerImpl: Merging 0 segments, 0 bytes from memory into reduce
18/03/22 20:57:28 INFO mapred.Merger: Merging 1 sorted segments
18/03/22 20:57:28 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 413 bytes
18/03/22 20:57:28 INFO mapred.LocalJobRunner: 1 / 1 copied.
18/03/22 20:57:28 INFO Configuration.deprecation: mapred.skip.on is deprecated. Instead, use mapreduce.job.skiprecords
这是第0次reduce
Reduce阶段开始....
这是第1次reduce
Reduce阶段开始....
这是第2次reduce
Reduce阶段开始....
这是第3次reduce
Reduce阶段开始....
这是第4次reduce
Reduce阶段开始....
这是第5次reduce
Reduce阶段开始....
这是第6次reduce
Reduce阶段开始....
这是第7次reduce
Reduce阶段开始....
18/03/22 20:57:28 INFO mapred.Task: Task:attempt_local1889225814_0001_r_000000_0 is done. And is in the process of committing
18/03/22 20:57:28 INFO mapred.LocalJobRunner: 1 / 1 copied.
18/03/22 20:57:28 INFO mapred.Task: Task attempt_local1889225814_0001_r_000000_0 is allowed to commit now
18/03/22 20:57:28 INFO output.FileOutputCommitter: Saved output of task 'attempt_local1889225814_0001_r_000000_0' to hdfs://bigdata-study-104:8020/testresult/output/_temporary/0/task_local1889225814_0001_r_000000
18/03/22 20:57:28 INFO mapred.LocalJobRunner: reduce > reduce
18/03/22 20:57:28 INFO mapred.Task: Task 'attempt_local1889225814_0001_r_000000_0' done.
18/03/22 20:57:28 INFO mapred.LocalJobRunner: Finishing task: attempt_local1889225814_0001_r_000000_0
18/03/22 20:57:28 INFO mapred.LocalJobRunner: reduce task executor complete.
18/03/22 20:57:28 INFO mapreduce.Job: Job job_local1889225814_0001 running in uber mode : false
18/03/22 20:57:28 INFO mapreduce.Job:  map 100% reduce 100%
18/03/22 20:57:28 INFO mapreduce.Job: Job job_local1889225814_0001 completed successfully
18/03/22 20:57:28 INFO mapreduce.Job: Counters: 38
    File System Counters
        FILE: Number of bytes read=1234
        FILE: Number of bytes written=508548
        FILE: Number of read operations=0
        FILE: Number of large read operations=0
        FILE: Number of write operations=0
        HDFS: Number of bytes read=420
        HDFS: Number of bytes written=61
        HDFS: Number of read operations=15
        HDFS: Number of large read operations=0
        HDFS: Number of write operations=6
    Map-Reduce Framework
        Map input records=8
        Map output records=36
        Map output bytes=348
        Map output materialized bytes=426
        Input split bytes=112
        Combine input records=0
        Combine output records=0
        Reduce input groups=8
        Reduce shuffle bytes=426
        Reduce input records=36
        Reduce output records=8
        Spilled Records=72
        Shuffled Maps =1
        Failed Shuffles=0
        Merged Map outputs=1
        GC time elapsed (ms)=0
        CPU time spent (ms)=0
        Physical memory (bytes) snapshot=0
        Virtual memory (bytes) snapshot=0
        Total committed heap usage (bytes)=471859200
    Shuffle Errors
        BAD_ID=0
        CONNECTION=0
        IO_ERROR=0
        WRONG_LENGTH=0
        WRONG_MAP=0
        WRONG_REDUCE=0
    File Input Format Counters 
        Bytes Read=210
    File Output Format Counters 
        Bytes Written=61

运行成功，在dfs上看到运行的结果文件！！

 

 

参考：https://getpocket.com/a/read/2123376367 解决问题。