运行mapreduce - java.lang.InterruptedException

错误日志:

 

2018-11-19 05:23:51,686 WARN  [main] util.NativeCodeLoader (NativeCodeLoader.java:<clinit>(62)) - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2018-11-19 05:23:52,595 INFO  [main] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1181)) - session.id is deprecated. Instead, use dfs.metrics.session-id
2018-11-19 05:23:52,596 INFO  [main] jvm.JvmMetrics (JvmMetrics.java:init(79)) - Initializing JVM Metrics with processName=JobTracker, sessionId=
2018-11-19 05:23:53,215 WARN  [main] mapreduce.JobResourceUploader (JobResourceUploader.java:uploadFiles(64)) - Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
2018-11-19 05:23:53,289 INFO  [main] input.FileInputFormat (FileInputFormat.java:listStatus(289)) - Total input files to process : 1
2018-11-19 05:23:53,375 INFO  [main] mapreduce.JobSubmitter (JobSubmitter.java:submitJobInternal(200)) - number of splits:1
2018-11-19 05:23:53,684 INFO  [main] mapreduce.JobSubmitter (JobSubmitter.java:printTokens(289)) - Submitting tokens for job: job_local2033293629_0001
2018-11-19 05:23:54,051 INFO  [main] mapreduce.Job (Job.java:submit(1345)) - The url to track the job: http://localhost:8080/
2018-11-19 05:23:54,052 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1390)) - Running job: job_local2033293629_0001
2018-11-19 05:23:54,053 INFO  [Thread-23] mapred.LocalJobRunner (LocalJobRunner.java:createOutputCommitter(498)) - OutputCommitter set in config null
2018-11-19 05:23:54,059 INFO  [Thread-23] output.FileOutputCommitter (FileOutputCommitter.java:<init>(123)) - File Output Committer Algorithm version is 1
2018-11-19 05:23:54,061 INFO  [Thread-23] output.FileOutputCommitter (FileOutputCommitter.java:<init>(138)) - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
2018-11-19 05:23:54,070 INFO  [Thread-23] mapred.LocalJobRunner (LocalJobRunner.java:createOutputCommitter(516)) - OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
2018-11-19 05:23:54,156 INFO  [Thread-23] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(475)) - Waiting for map tasks
2018-11-19 05:23:54,157 INFO  [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:run(251)) - Starting task: attempt_local2033293629_0001_m_000000_0
2018-11-19 05:23:54,185 INFO  [LocalJobRunner Map Task Executor #0] output.FileOutputCommitter (FileOutputCommitter.java:<init>(123)) - File Output Committer Algorithm version is 1
2018-11-19 05:23:54,187 INFO  [LocalJobRunner Map Task Executor #0] output.FileOutputCommitter (FileOutputCommitter.java:<init>(138)) - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
2018-11-19 05:23:54,219 INFO  [LocalJobRunner Map Task Executor #0] mapred.Task (Task.java:initialize(619)) -  Using ResourceCalculatorProcessTree : [ ]
2018-11-19 05:23:54,224 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:runNewMapper(756)) - Processing split: hdfs://hadoop1:9000/Input/test.txt:0+1149413
2018-11-19 05:23:54,303 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:setEquator(1205)) - (EQUATOR) 0 kvi 26214396(104857584)
2018-11-19 05:23:54,303 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:init(998)) - mapreduce.task.io.sort.mb: 100
2018-11-19 05:23:54,304 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:init(999)) - soft limit at 83886080
2018-11-19 05:23:54,304 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:init(1000)) - bufstart = 0; bufvoid = 104857600
2018-11-19 05:23:54,304 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:init(1001)) - kvstart = 26214396; length = 6553600
2018-11-19 05:23:54,305 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:createSortingCollector(403)) - Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
2018-11-19 05:23:54,983 INFO  [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(618)) - 
2018-11-19 05:23:54,987 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1462)) - Starting flush of map output
2018-11-19 05:23:54,987 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1484)) - Spilling map output
2018-11-19 05:23:54,987 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1485)) - bufstart = 0; bufend = 1574408; bufvoid = 104857600
2018-11-19 05:23:54,988 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1487)) - kvstart = 26214396(104857584); kvend = 25789404(103157616); length = 424993/6553600
2018-11-19 05:23:55,054 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1411)) - Job job_local2033293629_0001 running in uber mode : false
2018-11-19 05:23:55,055 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1418)) -  map 0% reduce 0%
2018-11-19 05:23:55,483 INFO  [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:sortAndSpill(1669)) - Finished spill 0
2018-11-19 05:23:55,496 INFO  [LocalJobRunner Map Task Executor #0] mapred.Task (Task.java:done(1099)) - Task:attempt_local2033293629_0001_m_000000_0 is done. And is in the process of committing
2018-11-19 05:23:55,516 INFO  [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(618)) - map
2018-11-19 05:23:55,516 INFO  [LocalJobRunner Map Task Executor #0] mapred.Task (Task.java:sendDone(1219)) - Task 'attempt_local2033293629_0001_m_000000_0' done.
2018-11-19 05:23:55,516 INFO  [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:run(276)) - Finishing task: attempt_local2033293629_0001_m_000000_0
2018-11-19 05:23:55,517 INFO  [Thread-23] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(483)) - map task executor complete.
2018-11-19 05:23:55,518 INFO  [Thread-23] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(475)) - Waiting for reduce tasks
2018-11-19 05:23:55,518 INFO  [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:run(329)) - Starting task: attempt_local2033293629_0001_r_000000_0
2018-11-19 05:23:55,530 INFO  [pool-7-thread-1] output.FileOutputCommitter (FileOutputCommitter.java:<init>(123)) - File Output Committer Algorithm version is 1
2018-11-19 05:23:55,530 INFO  [pool-7-thread-1] output.FileOutputCommitter (FileOutputCommitter.java:<init>(138)) - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
2018-11-19 05:23:55,531 INFO  [pool-7-thread-1] mapred.Task (Task.java:initialize(619)) -  Using ResourceCalculatorProcessTree : [ ]
2018-11-19 05:23:55,542 INFO  [pool-7-thread-1] mapred.ReduceTask (ReduceTask.java:run(362)) - Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@2de8a214
2018-11-19 05:23:55,564 INFO  [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:<init>(206)) - MergerManager: memoryLimit=678356544, maxSingleShuffleLimit=169589136, mergeThreshold=447715328, ioSortFactor=10, memToMemMergeOutputsThreshold=10
2018-11-19 05:23:55,574 INFO  [EventFetcher for fetching Map Completion Events] reduce.EventFetcher (EventFetcher.java:run(61)) - attempt_local2033293629_0001_r_000000_0 Thread started: EventFetcher for fetching Map Completion Events
2018-11-19 05:23:55,690 INFO  [localfetcher#1] reduce.LocalFetcher (LocalFetcher.java:copyMapOutput(145)) - localfetcher#1 about to shuffle output of map attempt_local2033293629_0001_m_000000_0 decomp: 569824 len: 569828 to MEMORY
2018-11-19 05:23:55,702 INFO  [localfetcher#1] reduce.InMemoryMapOutput (InMemoryMapOutput.java:doShuffle(93)) - Read 569824 bytes from map-output for attempt_local2033293629_0001_m_000000_0
2018-11-19 05:23:55,706 INFO  [localfetcher#1] reduce.MergeManagerImpl (MergeManagerImpl.java:closeInMemoryFile(321)) - closeInMemoryFile -> map-output of size: 569824, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->569824
2018-11-19 05:23:55,707 INFO  [EventFetcher for fetching Map Completion Events] reduce.EventFetcher (EventFetcher.java:run(76)) - EventFetcher is interrupted.. Returning
2018-11-19 05:23:55,708 INFO  [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(618)) - 1 / 1 copied.
2018-11-19 05:23:55,708 INFO  [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(693)) - finalMerge called with 1 in-memory map-outputs and 0 on-disk map-outputs
2018-11-19 05:23:55,722 INFO  [pool-7-thread-1] mapred.Merger (Merger.java:merge(606)) - Merging 1 sorted segments
2018-11-19 05:23:55,729 INFO  [pool-7-thread-1] mapred.Merger (Merger.java:merge(705)) - Down to the last merge-pass, with 1 segments left of total size: 569809 bytes
2018-11-19 05:23:55,838 INFO  [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(760)) - Merged 1 segments, 569824 bytes to disk to satisfy reduce memory limit
2018-11-19 05:23:55,844 INFO  [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(790)) - Merging 1 files, 569828 bytes from disk
2018-11-19 05:23:55,848 INFO  [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(805)) - Merging 0 segments, 0 bytes from memory into reduce
2018-11-19 05:23:55,850 INFO  [pool-7-thread-1] mapred.Merger (Merger.java:merge(606)) - Merging 1 sorted segments
2018-11-19 05:23:55,852 INFO  [pool-7-thread-1] mapred.Merger (Merger.java:merge(705)) - Down to the last merge-pass, with 1 segments left of total size: 569809 bytes
2018-11-19 05:23:55,857 INFO  [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(618)) - 1 / 1 copied.
2018-11-19 05:23:55,901 INFO  [pool-7-thread-1] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1181)) - mapred.skip.on is deprecated. Instead, use mapreduce.job.skiprecords
2018-11-19 05:23:56,056 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1418)) -  map 100% reduce 0%
2018-11-19 05:23:56,084 INFO  [pool-7-thread-1] mapred.Task (Task.java:done(1099)) - Task:attempt_local2033293629_0001_r_000000_0 is done. And is in the process of committing
2018-11-19 05:23:56,089 INFO  [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(618)) - 1 / 1 copied.
2018-11-19 05:23:56,091 INFO  [pool-7-thread-1] mapred.Task (Task.java:commit(1260)) - Task attempt_local2033293629_0001_r_000000_0 is allowed to commit now
2018-11-19 05:23:56,114 INFO  [pool-7-thread-1] output.FileOutputCommitter (FileOutputCommitter.java:commitTask(582)) - Saved output of task 'attempt_local2033293629_0001_r_000000_0' to hdfs://hadoop1:9000/Output/_temporary/0/task_local2033293629_0001_r_000000
2018-11-19 05:23:56,120 INFO  [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(618)) - reduce > reduce
2018-11-19 05:23:56,120 INFO  [pool-7-thread-1] mapred.Task (Task.java:sendDone(1219)) - Task 'attempt_local2033293629_0001_r_000000_0' done.
2018-11-19 05:23:56,120 INFO  [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:run(352)) - Finishing task: attempt_local2033293629_0001_r_000000_0
2018-11-19 05:23:56,120 INFO  [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:run(329)) - Starting task: attempt_local2033293629_0001_r_000001_0
2018-11-19 05:23:56,127 INFO  [pool-7-thread-1] output.FileOutputCommitter (FileOutputCommitter.java:<init>(123)) - File Output Committer Algorithm version is 1
2018-11-19 05:23:56,127 INFO  [pool-7-thread-1] output.FileOutputCommitter (FileOutputCommitter.java:<init>(138)) - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
2018-11-19 05:23:56,128 INFO  [pool-7-thread-1] mapred.Task (Task.java:initialize(619)) -  Using ResourceCalculatorProcessTree : [ ]
2018-11-19 05:23:56,128 INFO  [pool-7-thread-1] mapred.ReduceTask (ReduceTask.java:run(362)) - Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@2bfb9fd5
2018-11-19 05:23:56,133 INFO  [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:<init>(206)) - MergerManager: memoryLimit=678356544, maxSingleShuffleLimit=169589136, mergeThreshold=447715328, ioSortFactor=10, memToMemMergeOutputsThreshold=10
2018-11-19 05:23:56,138 INFO  [EventFetcher for fetching Map Completion Events] reduce.EventFetcher (EventFetcher.java:run(61)) - attempt_local2033293629_0001_r_000001_0 Thread started: EventFetcher for fetching Map Completion Events
2018-11-19 05:23:56,151 INFO  [localfetcher#2] reduce.LocalFetcher (LocalFetcher.java:copyMapOutput(145)) - localfetcher#2 about to shuffle output of map attempt_local2033293629_0001_m_000000_0 decomp: 1217086 len: 1217090 to MEMORY
2018-11-19 05:23:56,153 INFO  [localfetcher#2] reduce.InMemoryMapOutput (InMemoryMapOutput.java:doShuffle(93)) - Read 1217086 bytes from map-output for attempt_local2033293629_0001_m_000000_0
2018-11-19 05:23:56,157 INFO  [localfetcher#2] reduce.MergeManagerImpl (MergeManagerImpl.java:closeInMemoryFile(321)) - closeInMemoryFile -> map-output of size: 1217086, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->1217086
2018-11-19 05:23:56,157 INFO  [EventFetcher for fetching Map Completion Events] reduce.EventFetcher (EventFetcher.java:run(76)) - EventFetcher is interrupted.. Returning
2018-11-19 05:23:56,158 INFO  [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(618)) - 1 / 1 copied.
2018-11-19 05:23:56,158 INFO  [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(693)) - finalMerge called with 1 in-memory map-outputs and 0 on-disk map-outputs
2018-11-19 05:23:56,160 INFO  [pool-7-thread-1] mapred.Merger (Merger.java:merge(606)) - Merging 1 sorted segments
2018-11-19 05:23:56,161 INFO  [pool-7-thread-1] mapred.Merger (Merger.java:merge(705)) - Down to the last merge-pass, with 1 segments left of total size: 1217075 bytes
2018-11-19 05:23:56,303 INFO  [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(760)) - Merged 1 segments, 1217086 bytes to disk to satisfy reduce memory limit
2018-11-19 05:23:56,305 INFO  [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(790)) - Merging 1 files, 1217090 bytes from disk
2018-11-19 05:23:56,307 INFO  [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(805)) - Merging 0 segments, 0 bytes from memory into reduce
2018-11-19 05:23:56,307 INFO  [pool-7-thread-1] mapred.Merger (Merger.java:merge(606)) - Merging 1 sorted segments
2018-11-19 05:23:56,308 INFO  [pool-7-thread-1] mapred.Merger (Merger.java:merge(705)) - Down to the last merge-pass, with 1 segments left of total size: 1217075 bytes
2018-11-19 05:23:56,313 INFO  [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(618)) - 1 / 1 copied.
2018-11-19 05:23:56,529 WARN  [DataStreamer for file /Output/_temporary/0/_temporary/attempt_local2033293629_0001_r_000001_0/part-r-00001] hdfs.DataStreamer (DataStreamer.java:closeResponder(929)) - Caught exception
java.lang.InterruptedException
    at java.lang.Object.wait(Native Method)
    at java.lang.Thread.join(Thread.java:1252)
    at java.lang.Thread.join(Thread.java:1326)
    at org.apache.hadoop.hdfs.DataStreamer.closeResponder(DataStreamer.java:927)
    at org.apache.hadoop.hdfs.DataStreamer.endBlock(DataStreamer.java:578)
    at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:755)
2018-11-19 05:23:56,531 INFO  [pool-7-thread-1] mapred.Task (Task.java:done(1099)) - Task:attempt_local2033293629_0001_r_000001_0 is done. And is in the process of committing
2018-11-19 05:23:56,534 INFO  [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(618)) - 1 / 1 copied.
2018-11-19 05:23:56,534 INFO  [pool-7-thread-1] mapred.Task (Task.java:commit(1260)) - Task attempt_local2033293629_0001_r_000001_0 is allowed to commit now
2018-11-19 05:23:56,543 INFO  [pool-7-thread-1] output.FileOutputCommitter (FileOutputCommitter.java:commitTask(582)) - Saved output of task 'attempt_local2033293629_0001_r_000001_0' to hdfs://hadoop1:9000/Output/_temporary/0/task_local2033293629_0001_r_000001
2018-11-19 05:23:56,548 INFO  [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(618)) - reduce > reduce
2018-11-19 05:23:56,549 INFO  [pool-7-thread-1] mapred.Task (Task.java:sendDone(1219)) - Task 'attempt_local2033293629_0001_r_000001_0' done.
2018-11-19 05:23:56,549 INFO  [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:run(352)) - Finishing task: attempt_local2033293629_0001_r_000001_0
2018-11-19 05:23:56,549 INFO  [Thread-23] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(483)) - reduce task executor complete.
2018-11-19 05:23:57,057 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1418)) -  map 100% reduce 100%
2018-11-19 05:23:57,058 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1429)) - Job job_local2033293629_0001 completed successfully
2018-11-19 05:23:57,089 INFO  [main] mapreduce.Job (Job.java:monitorAndPrintJob(1436)) - Counters: 35
    File System Counters
        FILE: Number of bytes read=4750713
        FILE: Number of bytes written=8708886
        FILE: Number of read operations=0
        FILE: Number of large read operations=0
        FILE: Number of write operations=0
        HDFS: Number of bytes read=3448239
        HDFS: Number of bytes written=472
        HDFS: Number of read operations=27
        HDFS: Number of large read operations=0
        HDFS: Number of write operations=12
    Map-Reduce Framework
        Map input records=101421
        Map output records=106249
        Map output bytes=1574408
        Map output materialized bytes=1786918
        Input split bytes=99
        Combine input records=0
        Combine output records=0
        Reduce input groups=23
        Reduce shuffle bytes=1786918
        Reduce input records=106249
        Reduce output records=23
        Spilled Records=212498
        Shuffled Maps =2
        Failed Shuffles=0
        Merged Map outputs=2
        GC time elapsed (ms)=37
        Total committed heap usage (bytes)=497233920
    Shuffle Errors
        BAD_ID=0
        CONNECTION=0
        IO_ERROR=0
        WRONG_LENGTH=0
        WRONG_MAP=0
        WRONG_REDUCE=0
    File Input Format Counters 
        Bytes Read=1149413
    File Output Format Counters 
        Bytes Written=361
OK

 

错误代码:

package WordCount;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCountMain {

    public static final String HADOOP_ROOT_PATH = "hdfs://hadoop1:9000";
    public static final String HADOOP_INPUT_PATH = "hdfs://hadoop1:9000/Input";
    public static final String HADOOP_OUTPUT_PATH = "hdfs://hadoop1:9000/Output";

    public static void main(String[] args) throws IOException,
            URISyntaxException, ClassNotFoundException, InterruptedException {

        Configuration conf = new Configuration();
        // 1、设置job运行时要访问的默认文件系统
        //conf.set("fs.defaultFS", HADOOP_ROOT_PATH);
        // 2、设置job提交到哪去运行
        //conf.set("mapreduce.framework.name", "yarn");
        //conf.set("yarn.resourcemanager.hostname", "hadoop1");
        // 3、如果要从windows系统上运行这个job提交客户端程序，则需要加这个跨平台提交的参数
        //conf.set("mapreduce.app-submission.cross-platform", "true");

        Job job = Job.getInstance(conf);

        // 1、封装参数：jar包所在的位置
        job.setJar("/home/hadoop/wordcount.jar");
        //job.setJarByClass(WordCountMain.class);

        // 2、封装参数： 本次job所要调用的Mapper实现类、Reducer实现类
        job.setMapperClass(WordCountMapper.class);
        job.setReducerClass(WordcountReducer.class);

        // 3、封装参数：本次job的Mapper实现类、Reducer实现类产生的结果数据的key、value类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        // 4、封装参数：本次job要处理的输入数据集所在路径、最终结果的输出路径
        Path output = new Path(HADOOP_OUTPUT_PATH);
        FileSystem fs = FileSystem.get(new URI(HADOOP_ROOT_PATH), conf);
        if (fs.exists(output)) {
            fs.delete(output, true);
        }
        FileInputFormat.setInputPaths(job, new Path(HADOOP_INPUT_PATH));
        FileOutputFormat.setOutputPath(job, output); // 注意：输出路径必须不存在

        // 5、封装参数：想要启动的reduce task的数量
        job.setNumReduceTasks(2);

        // 6、提交job给yarn
        boolean res = job.waitForCompletion(true);
        System.out.println("OK");
        System.exit(res ? 0 : -1);

    }

}

 

 

修改方式添加 job 提交的地址

开放以下代码

// 2、设置job提交到哪去运行
        conf.set("mapreduce.framework.name", "yarn");