hadoop 源码解析其二(reduce)
reduce 过程的实现(不包含数据拉取部分)
首先调用 ReduceTask run 方法开始:
1, 实现 shuffer 拉取数据 返回是迭代器 ,然后构造reducerContext(Reducer 上下文),返还给 reducer.run(reducerContext)
2, 在 reducer.run(reducerContext)之中 调用 ReduceContextImpl nextKey()方法
3, 在 nextKey() 之中 调用 nextKeyValue()
4, nextKeyValue() 之中:
1),map 输出 将 key-val 序列化字符数组,首先反序列化得到 k,v
2),nextKeyIsSame 分组比较器 (前一个key 与 后一个 key 进行比较),对下条key 进行预判,比较的结果赋值给 nextKeyIsSame
5, 执行 reduce(context.getCurrentKey(), context.getValues(), context)
1),context.getValues() 返回迭代器 (firstValue || nextKeyIsSame) 决定的
6, reduce 的过程:reduce 是由两次迭代器迭代过程:
1),首先执行 nextKeyValue() 之中的 input.next(), 此时取出的数据是真实的数据
2),context.getValues() 也是一个迭代器, 迭代是根据 nextKeyIsSame来决定的, nextKeyIsSame 由 input.next()来进行配置配置
3), nextKeyIsSame 作为两个迭代器的 中间转换角色, 来自于分组比较器
7,分组比较器:
1),分组的比较器(决定什么数据算一组) (是与不是两种结果)
2),排序比较器可以当做分组比较器来使用
3),reduce 端首先自定义分组比较器, 然后排序比较器(来自map 端,大于小于等于 三种返回结果) ,最后默认分组比较器
4),没有配置 直接取key比较器(mapreduce.job.output.key.comparator.class)
8,综述:map 端 进行k,v 排序, 通过input.next() 迭代器获取所有的数据 , 使用 nextKeyIsSame(reduce比较器判断是否与下一个key值相同)最为中间转换来确定是否为一组数据
以上来自(hadoop-mapreduce-client 之中 Reducer ReduceContextImpl ReduceTask 类)
ReduceTask 类
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem.Statistics; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableFactories; import org.apache.hadoop.io.WritableFactory; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskCounter; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter; import org.apache.hadoop.mapreduce.task.reduce.Shuffle; import org.apache.hadoop.util.Progress; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; /** A Reduce task. */ @InterfaceAudience.Private @InterfaceStability.Unstable public class ReduceTask extends Task { static { // register a ctor WritableFactories.setFactory (ReduceTask.class, new WritableFactory() { public Writable newInstance() { return new ReduceTask(); } }); } private static final Log LOG = LogFactory.getLog(ReduceTask.class.getName()); private int numMaps; private CompressionCodec codec; // If this is a LocalJobRunner-based job, this will // be a mapping from map task attempts to their output files. // This will be null in other cases. private Map<TaskAttemptID, MapOutputFile> localMapFiles; { getProgress().setStatus("reduce"); setPhase(TaskStatus.Phase.SHUFFLE); // phase to start with } private Progress copyPhase; private Progress sortPhase; private Progress reducePhase; private Counters.Counter shuffledMapsCounter = getCounters().findCounter(TaskCounter.SHUFFLED_MAPS); private Counters.Counter reduceShuffleBytes = getCounters().findCounter(TaskCounter.REDUCE_SHUFFLE_BYTES); private Counters.Counter reduceInputKeyCounter = getCounters().findCounter(TaskCounter.REDUCE_INPUT_GROUPS); private Counters.Counter reduceInputValueCounter = getCounters().findCounter(TaskCounter.REDUCE_INPUT_RECORDS); private Counters.Counter reduceOutputCounter = getCounters().findCounter(TaskCounter.REDUCE_OUTPUT_RECORDS); private Counters.Counter reduceCombineInputCounter = getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS); private Counters.Counter reduceCombineOutputCounter = getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS); private Counters.Counter fileOutputByteCounter = getCounters().findCounter(FileOutputFormatCounter.BYTES_WRITTEN); // A custom comparator for map output files. Here the ordering is determined // by the file's size and path. In case of files with same size and different // file paths, the first parameter is considered smaller than the second one. // In case of files with same size and path are considered equal. private Comparator<FileStatus> mapOutputFileComparator = new Comparator<FileStatus>() { public int compare(FileStatus a, FileStatus b) { if (a.getLen() < b.getLen()) return -1; else if (a.getLen() == b.getLen()) if (a.getPath().toString().equals(b.getPath().toString())) return 0; else return -1; else return 1; } }; // A sorted set for keeping a set of map output files on disk private final SortedSet<FileStatus> mapOutputFilesOnDisk = new TreeSet<FileStatus>(mapOutputFileComparator); public ReduceTask() { super(); } public ReduceTask(String jobFile, TaskAttemptID taskId, int partition, int numMaps, int numSlotsRequired) { super(jobFile, taskId, partition, numSlotsRequired); this.numMaps = numMaps; } /** * Register the set of mapper outputs created by a LocalJobRunner-based * job with this ReduceTask so it knows where to fetch from. * * This should not be called in normal (networked) execution. */ public void setLocalMapFiles(Map<TaskAttemptID, MapOutputFile> mapFiles) { this.localMapFiles = mapFiles; } private CompressionCodec initCodec() { // check if map-outputs are to be compressed if (conf.getCompressMapOutput()) { Class<? extends CompressionCodec> codecClass = conf.getMapOutputCompressorClass(DefaultCodec.class); return ReflectionUtils.newInstance(codecClass, conf); } return null; } @Override public boolean isMapTask() { return false; } public int getNumMaps() { return numMaps; } /** * Localize the given JobConf to be specific for this task. */ @Override public void localizeConfiguration(JobConf conf) throws IOException { super.localizeConfiguration(conf); conf.setNumMapTasks(numMaps); } @Override public void write(DataOutput out) throws IOException { super.write(out); out.writeInt(numMaps); // write the number of maps } @Override public void readFields(DataInput in) throws IOException { super.readFields(in); numMaps = in.readInt(); } // Get the input files for the reducer (for local jobs). private Path[] getMapFiles(FileSystem fs) throws IOException { List<Path> fileList = new ArrayList<Path>(); for(int i = 0; i < numMaps; ++i) { fileList.add(mapOutputFile.getInputFile(i)); } return fileList.toArray(new Path[0]); } private class ReduceValuesIterator<KEY,VALUE> extends ValuesIterator<KEY,VALUE> { public ReduceValuesIterator (RawKeyValueIterator in, RawComparator<KEY> comparator, Class<KEY> keyClass, Class<VALUE> valClass, Configuration conf, Progressable reporter) throws IOException { super(in, comparator, keyClass, valClass, conf, reporter); } @Override public VALUE next() { reduceInputValueCounter.increment(1); return moveToNext(); } protected VALUE moveToNext() { return super.next(); } public void informReduceProgress() { reducePhase.set(super.in.getProgress().getProgress()); // update progress reporter.progress(); } } private class SkippingReduceValuesIterator<KEY,VALUE> extends ReduceValuesIterator<KEY,VALUE> { private SkipRangeIterator skipIt; private TaskUmbilicalProtocol umbilical; private Counters.Counter skipGroupCounter; private Counters.Counter skipRecCounter; private long grpIndex = -1; private Class<KEY> keyClass; private Class<VALUE> valClass; private SequenceFile.Writer skipWriter; private boolean toWriteSkipRecs; private boolean hasNext; private TaskReporter reporter; public SkippingReduceValuesIterator(RawKeyValueIterator in, RawComparator<KEY> comparator, Class<KEY> keyClass, Class<VALUE> valClass, Configuration conf, TaskReporter reporter, TaskUmbilicalProtocol umbilical) throws IOException { super(in, comparator, keyClass, valClass, conf, reporter); this.umbilical = umbilical; this.skipGroupCounter = reporter.getCounter(TaskCounter.REDUCE_SKIPPED_GROUPS); this.skipRecCounter = reporter.getCounter(TaskCounter.REDUCE_SKIPPED_RECORDS); this.toWriteSkipRecs = toWriteSkipRecs() && SkipBadRecords.getSkipOutputPath(conf)!=null; this.keyClass = keyClass; this.valClass = valClass; this.reporter = reporter; skipIt = getSkipRanges().skipRangeIterator(); mayBeSkip(); } public void nextKey() throws IOException { super.nextKey(); mayBeSkip(); } public boolean more() { return super.more() && hasNext; } private void mayBeSkip() throws IOException { hasNext = skipIt.hasNext(); if(!hasNext) { LOG.warn("Further groups got skipped."); return; } grpIndex++; long nextGrpIndex = skipIt.next(); long skip = 0; long skipRec = 0; while(grpIndex<nextGrpIndex && super.more()) { while (hasNext()) { VALUE value = moveToNext(); if(toWriteSkipRecs) { writeSkippedRec(getKey(), value); } skipRec++; } super.nextKey(); grpIndex++; skip++; } //close the skip writer once all the ranges are skipped if(skip>0 && skipIt.skippedAllRanges() && skipWriter!=null) { skipWriter.close(); } skipGroupCounter.increment(skip); skipRecCounter.increment(skipRec); reportNextRecordRange(umbilical, grpIndex); } @SuppressWarnings("unchecked") private void writeSkippedRec(KEY key, VALUE value) throws IOException{ if(skipWriter==null) { Path skipDir = SkipBadRecords.getSkipOutputPath(conf); Path skipFile = new Path(skipDir, getTaskID().toString()); skipWriter = SequenceFile.createWriter( skipFile.getFileSystem(conf), conf, skipFile, keyClass, valClass, CompressionType.BLOCK, reporter); } skipWriter.append(key, value); } } @Override @SuppressWarnings("unchecked") public void run(JobConf job, final TaskUmbilicalProtocol umbilical) throws IOException, InterruptedException, ClassNotFoundException { job.setBoolean(JobContext.SKIP_RECORDS, isSkipping()); if (isMapOrReduce()) { // copy 阶段 copyPhase = getProgress().addPhase("copy"); // 排序阶段 sortPhase = getProgress().addPhase("sort"); // reduce 阶段 reducePhase = getProgress().addPhase("reduce"); } // start thread that will handle communication with parent TaskReporter reporter = startReporter(umbilical); boolean useNewApi = job.getUseNewReducer(); initialize(job, getJobID(), reporter, useNewApi); // check if it is a cleanupJobTask if (jobCleanup) { runJobCleanupTask(umbilical, reporter); return; } if (jobSetup) { runJobSetupTask(umbilical, reporter); return; } if (taskCleanup) { runTaskCleanupTask(umbilical, reporter); return; } // Initialize the codec codec = initCodec(); RawKeyValueIterator rIter = null; ShuffleConsumerPlugin shuffleConsumerPlugin = null; Class combinerClass = conf.getCombinerClass(); CombineOutputCollector combineCollector = (null != combinerClass) ? new CombineOutputCollector(reduceCombineOutputCounter, reporter, conf) : null; Class<? extends ShuffleConsumerPlugin> clazz = job.getClass(MRConfig.SHUFFLE_CONSUMER_PLUGIN, Shuffle.class, ShuffleConsumerPlugin.class); shuffleConsumerPlugin = ReflectionUtils.newInstance(clazz, job); LOG.info("Using ShuffleConsumerPlugin: " + shuffleConsumerPlugin); ShuffleConsumerPlugin.Context shuffleContext = new ShuffleConsumerPlugin.Context(getTaskID(), job, FileSystem.getLocal(job), umbilical, super.lDirAlloc, reporter, codec, combinerClass, combineCollector, spilledRecordsCounter, reduceCombineInputCounter, shuffledMapsCounter, reduceShuffleBytes, failedShuffleCounter, mergedMapOutputsCounter, taskStatus, copyPhase, sortPhase, this, mapOutputFile, localMapFiles); shuffleConsumerPlugin.init(shuffleContext); // 实现 shuffer 拉取数据 返回是 迭代器 // reduce 的输入源 rIter = shuffleConsumerPlugin.run(); // free up the data structures mapOutputFilesOnDisk.clear(); sortPhase.complete(); // sort is complete setPhase(TaskStatus.Phase.REDUCE); statusUpdate(umbilical); Class keyClass = job.getMapOutputKeyClass(); Class valueClass = job.getMapOutputValueClass(); // 分组的比较器(决定什么数据算一组) (是与不是两种结果) // 排序比较器可以当做分组比较器来使用 // 首先取 分组比较器, 然后排序比较器(来自map 端,大于小于等于 三种返回结果) ,最后默认分组比较器 // 没有配置 直接取key比较器(mapreduce.job.output.key.comparator.class) RawComparator comparator = job.getOutputValueGroupingComparator(); if (useNewApi) { runNewReducer(job, umbilical, reporter, rIter, comparator, keyClass, valueClass); } else { runOldReducer(job, umbilical, reporter, rIter, comparator, keyClass, valueClass); } shuffleConsumerPlugin.close(); done(umbilical, reporter); } @SuppressWarnings("unchecked") private <INKEY,INVALUE,OUTKEY,OUTVALUE> void runOldReducer(JobConf job, TaskUmbilicalProtocol umbilical, final TaskReporter reporter, RawKeyValueIterator rIter, RawComparator<INKEY> comparator, Class<INKEY> keyClass, Class<INVALUE> valueClass) throws IOException { Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE> reducer = ReflectionUtils.newInstance(job.getReducerClass(), job); // make output collector String finalName = getOutputName(getPartition()); RecordWriter<OUTKEY, OUTVALUE> out = new OldTrackingRecordWriter<OUTKEY, OUTVALUE>( this, job, reporter, finalName); final RecordWriter<OUTKEY, OUTVALUE> finalOut = out; OutputCollector<OUTKEY,OUTVALUE> collector = new OutputCollector<OUTKEY,OUTVALUE>() { public void collect(OUTKEY key, OUTVALUE value) throws IOException { finalOut.write(key, value); // indicate that progress update needs to be sent reporter.progress(); } }; // apply reduce function try { //increment processed counter only if skipping feature is enabled boolean incrProcCount = SkipBadRecords.getReducerMaxSkipGroups(job)>0 && SkipBadRecords.getAutoIncrReducerProcCount(job); ReduceValuesIterator<INKEY,INVALUE> values = isSkipping() ? new SkippingReduceValuesIterator<INKEY,INVALUE>(rIter, comparator, keyClass, valueClass, job, reporter, umbilical) : new ReduceValuesIterator<INKEY,INVALUE>(rIter, job.getOutputValueGroupingComparator(), keyClass, valueClass, job, reporter); values.informReduceProgress(); while (values.more()) { reduceInputKeyCounter.increment(1); reducer.reduce(values.getKey(), values, collector, reporter); if(incrProcCount) { reporter.incrCounter(SkipBadRecords.COUNTER_GROUP, SkipBadRecords.COUNTER_REDUCE_PROCESSED_GROUPS, 1); } values.nextKey(); values.informReduceProgress(); } reducer.close(); reducer = null; out.close(reporter); out = null; } finally { IOUtils.cleanup(LOG, reducer); closeQuietly(out, reporter); } } static class OldTrackingRecordWriter<K, V> implements RecordWriter<K, V> { private final RecordWriter<K, V> real; private final org.apache.hadoop.mapred.Counters.Counter reduceOutputCounter; private final org.apache.hadoop.mapred.Counters.Counter fileOutputByteCounter; private final List<Statistics> fsStats; @SuppressWarnings({ "deprecation", "unchecked" }) public OldTrackingRecordWriter(ReduceTask reduce, JobConf job, TaskReporter reporter, String finalName) throws IOException { this.reduceOutputCounter = reduce.reduceOutputCounter; this.fileOutputByteCounter = reduce.fileOutputByteCounter; List<Statistics> matchedStats = null; if (job.getOutputFormat() instanceof FileOutputFormat) { matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job), job); } fsStats = matchedStats; FileSystem fs = FileSystem.get(job); long bytesOutPrev = getOutputBytes(fsStats); this.real = job.getOutputFormat().getRecordWriter(fs, job, finalName, reporter); long bytesOutCurr = getOutputBytes(fsStats); fileOutputByteCounter.increment(bytesOutCurr - bytesOutPrev); } @Override public void write(K key, V value) throws IOException { long bytesOutPrev = getOutputBytes(fsStats); real.write(key, value); long bytesOutCurr = getOutputBytes(fsStats); fileOutputByteCounter.increment(bytesOutCurr - bytesOutPrev); reduceOutputCounter.increment(1); } @Override public void close(Reporter reporter) throws IOException { long bytesOutPrev = getOutputBytes(fsStats); real.close(reporter); long bytesOutCurr = getOutputBytes(fsStats); fileOutputByteCounter.increment(bytesOutCurr - bytesOutPrev); } private long getOutputBytes(List<Statistics> stats) { if (stats == null) return 0; long bytesWritten = 0; for (Statistics stat: stats) { bytesWritten = bytesWritten + stat.getBytesWritten(); } return bytesWritten; } } static class NewTrackingRecordWriter<K,V> extends org.apache.hadoop.mapreduce.RecordWriter<K,V> { private final org.apache.hadoop.mapreduce.RecordWriter<K,V> real; private final org.apache.hadoop.mapreduce.Counter outputRecordCounter; private final org.apache.hadoop.mapreduce.Counter fileOutputByteCounter; private final List<Statistics> fsStats; @SuppressWarnings("unchecked") NewTrackingRecordWriter(ReduceTask reduce, org.apache.hadoop.mapreduce.TaskAttemptContext taskContext) throws InterruptedException, IOException { this.outputRecordCounter = reduce.reduceOutputCounter; this.fileOutputByteCounter = reduce.fileOutputByteCounter; List<Statistics> matchedStats = null; if (reduce.outputFormat instanceof org.apache.hadoop.mapreduce.lib.output.FileOutputFormat) { matchedStats = getFsStatistics(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat .getOutputPath(taskContext), taskContext.getConfiguration()); } fsStats = matchedStats; long bytesOutPrev = getOutputBytes(fsStats); this.real = (org.apache.hadoop.mapreduce.RecordWriter<K, V>) reduce.outputFormat .getRecordWriter(taskContext); long bytesOutCurr = getOutputBytes(fsStats); fileOutputByteCounter.increment(bytesOutCurr - bytesOutPrev); } @Override public void close(TaskAttemptContext context) throws IOException, InterruptedException { long bytesOutPrev = getOutputBytes(fsStats); real.close(context); long bytesOutCurr = getOutputBytes(fsStats); fileOutputByteCounter.increment(bytesOutCurr - bytesOutPrev); } @Override public void write(K key, V value) throws IOException, InterruptedException { long bytesOutPrev = getOutputBytes(fsStats); real.write(key,value); long bytesOutCurr = getOutputBytes(fsStats); fileOutputByteCounter.increment(bytesOutCurr - bytesOutPrev); outputRecordCounter.increment(1); } private long getOutputBytes(List<Statistics> stats) { if (stats == null) return 0; long bytesWritten = 0; for (Statistics stat: stats) { bytesWritten = bytesWritten + stat.getBytesWritten(); } return bytesWritten; } } @SuppressWarnings("unchecked") private <INKEY,INVALUE,OUTKEY,OUTVALUE> void runNewReducer(JobConf job, final TaskUmbilicalProtocol umbilical, final TaskReporter reporter, RawKeyValueIterator rIter, RawComparator<INKEY> comparator, Class<INKEY> keyClass, Class<INVALUE> valueClass ) throws IOException,InterruptedException, ClassNotFoundException { // wrap value iterator to report progress. final RawKeyValueIterator rawIter = rIter; rIter = new RawKeyValueIterator() { public void close() throws IOException { rawIter.close(); } public DataInputBuffer getKey() throws IOException { return rawIter.getKey(); } public Progress getProgress() { return rawIter.getProgress(); } public DataInputBuffer getValue() throws IOException { return rawIter.getValue(); } public boolean next() throws IOException { boolean ret = rawIter.next(); reporter.setProgress(rawIter.getProgress().getProgress()); return ret; } }; // make a task context so we can get the classes // 任务上下文 org.apache.hadoop.mapreduce.TaskAttemptContext taskContext = new org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl(job, getTaskID(), reporter); // make a reducer // 反射 reducer 对象 org.apache.hadoop.mapreduce.Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE> reducer = (org.apache.hadoop.mapreduce.Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE>) ReflectionUtils.newInstance(taskContext.getReducerClass(), job); org.apache.hadoop.mapreduce.RecordWriter<OUTKEY,OUTVALUE> trackedRW = new NewTrackingRecordWriter<OUTKEY, OUTVALUE>(this, taskContext); job.setBoolean("mapred.skip.on", isSkipping()); job.setBoolean(JobContext.SKIP_RECORDS, isSkipping()); // Reducer 上下文 org.apache.hadoop.mapreduce.Reducer.Context reducerContext = createReduceContext(reducer, job, getTaskID(), rIter, reduceInputKeyCounter, reduceInputValueCounter, trackedRW, committer, reporter, comparator, keyClass, valueClass); try { reducer.run(reducerContext); } finally { trackedRW.close(reducerContext); } } private <OUTKEY, OUTVALUE> void closeQuietly(RecordWriter<OUTKEY, OUTVALUE> c, Reporter r) { if (c != null) { try { c.close(r); } catch (Exception e) { LOG.info("Exception in closing " + c, e); } } } }
Reducer 类
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapreduce; import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.task.annotation.Checkpointable; import java.util.Iterator; /** * Reduces a set of intermediate values which share a key to a smaller set of * values. * * <p><code>Reducer</code> implementations * can access the {@link Configuration} for the job via the * {@link JobContext#getConfiguration()} method.</p> * <p><code>Reducer</code> has 3 primary phases:</p> * <ol> * <li> * * <b id="Shuffle">Shuffle</b> * * <p>The <code>Reducer</code> copies the sorted output from each * {@link Mapper} using HTTP across the network.</p> * </li> * * <li> * <b id="Sort">Sort</b> * * <p>The framework merge sorts <code>Reducer</code> inputs by * <code>key</code>s * (since different <code>Mapper</code>s may have output the same key).</p> * * <p>The shuffle and sort phases occur simultaneously i.e. while outputs are * being fetched they are merged.</p> * * <b id="SecondarySort">SecondarySort</b> * * <p>To achieve a secondary sort on the values returned by the value * iterator, the application should extend the key with the secondary * key and define a grouping comparator. The keys will be sorted using the * entire key, but will be grouped using the grouping comparator to decide * which keys and values are sent in the same call to reduce.The grouping * comparator is specified via * {@link Job#setGroupingComparatorClass(Class)}. The sort order is * controlled by * {@link Job#setSortComparatorClass(Class)}.</p> * * * For example, say that you want to find duplicate web pages and tag them * all with the url of the "best" known example. You would set up the job * like: * <ul> * <li>Map Input Key: url</li> * <li>Map Input Value: document</li> * <li>Map Output Key: document checksum, url pagerank</li> * <li>Map Output Value: url</li> * <li>Partitioner: by checksum</li> * <li>OutputKeyComparator: by checksum and then decreasing pagerank</li> * <li>OutputValueGroupingComparator: by checksum</li> * </ul> * </li> * * <li> * <b id="Reduce">Reduce</b> * * <p>In this phase the * {@link #reduce(Object, Iterable, org.apache.hadoop.mapreduce.Reducer.Context)} * method is called for each <code><key, (collection of values)></code> in * the sorted inputs.</p> * <p>The output of the reduce task is typically written to a * {@link RecordWriter} via * {@link Context#write(Object, Object)}.</p> * </li> * </ol> * * <p>The output of the <code>Reducer</code> is <b>not re-sorted</b>.</p> * * <p>Example:</p> * <p><blockquote><pre> * public class IntSumReducer<Key> extends Reducer<Key,IntWritable, * Key,IntWritable> { * private IntWritable result = new IntWritable(); * * public void reduce(Key key, Iterable<IntWritable> values, * Context context) throws IOException, InterruptedException { * int sum = 0; * for (IntWritable val : values) { * sum += val.get(); * } * result.set(sum); * context.write(key, result); * } * } * </pre></blockquote> * * @see Mapper * @see Partitioner */ @Checkpointable @InterfaceAudience.Public @InterfaceStability.Stable public class Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT> { /** * The <code>Context</code> passed on to the {@link Reducer} implementations. */ public abstract class Context implements ReduceContext<KEYIN,VALUEIN,KEYOUT,VALUEOUT> { } /** * Called once at the start of the task. */ protected void setup(Context context ) throws IOException, InterruptedException { // NOTHING } /** * This method is called once for each key. Most applications will define * their reduce class by overriding this method. The default implementation * is an identity function. */ @SuppressWarnings("unchecked") protected void reduce(KEYIN key, Iterable<VALUEIN> values, Context context ) throws IOException, InterruptedException { for(VALUEIN value: values) { context.write((KEYOUT) key, (VALUEOUT) value); } } /** * Called once at the end of the task. */ protected void cleanup(Context context ) throws IOException, InterruptedException { // NOTHING } /** * Advanced application writers can use the * {@link #run(org.apache.hadoop.mapreduce.Reducer.Context)} method to * control how the reduce task works. */ public void run(Context context) throws IOException, InterruptedException { setup(context); try { // 根据 key 值进行 分组 // ReduceContextImpl 下的 nextKey() // 1),对 k,v 的赋值 // 2),对 k 进行预判断 while (context.nextKey()) { // reduce 的结果依赖于 map 的排序结果 // 一次 reduce 的过程: // context.getValues() 返回迭代器 (firstValue || nextKeyIsSame) 决定的 // nextKeyIsSame 调用 nextKeyValue // nextKeyIsSame 作为两个迭代器的 中间转换角色 , // nextKeyValue来自于 input // context.getValues() 只迭代一组数据 reduce(context.getCurrentKey(), context.getValues(), context); // If a back up store is used, reset it Iterator<VALUEIN> iter = context.getValues().iterator(); if(iter instanceof ReduceContext.ValueIterator) { ((ReduceContext.ValueIterator<VALUEIN>)iter).resetBackupStore(); } } } finally { cleanup(context); } } }
ReduceContextImpl类
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapreduce.task; import java.io.DataOutputStream; import java.io.IOException; import java.util.Iterator; import java.util.NoSuchElementException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.io.serializer.Deserializer; import org.apache.hadoop.io.serializer.SerializationFactory; import org.apache.hadoop.io.serializer.Serializer; import org.apache.hadoop.mapred.BackupStore; import org.apache.hadoop.mapred.RawKeyValueIterator; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.ReduceContext; import org.apache.hadoop.mapreduce.StatusReporter; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.util.Progressable; /** * The context passed to the {@link Reducer}. * @param <KEYIN> the class of the input keys * @param <VALUEIN> the class of the input values * @param <KEYOUT> the class of the output keys * @param <VALUEOUT> the class of the output values */ @InterfaceAudience.Private @InterfaceStability.Unstable public class ReduceContextImpl<KEYIN,VALUEIN,KEYOUT,VALUEOUT> extends TaskInputOutputContextImpl<KEYIN,VALUEIN,KEYOUT,VALUEOUT> implements ReduceContext<KEYIN, VALUEIN, KEYOUT, VALUEOUT> { private RawKeyValueIterator input; private Counter inputValueCounter; private Counter inputKeyCounter; private RawComparator<KEYIN> comparator; private KEYIN key; // current key private VALUEIN value; // current value private boolean firstValue = false; // first value in key private boolean nextKeyIsSame = false; // more w/ this key private boolean hasMore; // more in file protected Progressable reporter; private Deserializer<KEYIN> keyDeserializer; private Deserializer<VALUEIN> valueDeserializer; private DataInputBuffer buffer = new DataInputBuffer(); private BytesWritable currentRawKey = new BytesWritable(); private ValueIterable iterable = new ValueIterable(); private boolean isMarked = false; private BackupStore<KEYIN,VALUEIN> backupStore; private final SerializationFactory serializationFactory; private final Class<KEYIN> keyClass; private final Class<VALUEIN> valueClass; private final Configuration conf; private final TaskAttemptID taskid; private int currentKeyLength = -1; private int currentValueLength = -1; public ReduceContextImpl(Configuration conf, TaskAttemptID taskid, RawKeyValueIterator input, Counter inputKeyCounter, Counter inputValueCounter, RecordWriter<KEYOUT,VALUEOUT> output, OutputCommitter committer, StatusReporter reporter, RawComparator<KEYIN> comparator, Class<KEYIN> keyClass, Class<VALUEIN> valueClass ) throws InterruptedException, IOException{ super(conf, taskid, output, committer, reporter); this.input = input; this.inputKeyCounter = inputKeyCounter; this.inputValueCounter = inputValueCounter; this.comparator = comparator; this.serializationFactory = new SerializationFactory(conf); this.keyDeserializer = serializationFactory.getDeserializer(keyClass); this.keyDeserializer.open(buffer); // 反序列化 this.valueDeserializer = serializationFactory.getDeserializer(valueClass); this.valueDeserializer.open(buffer); hasMore = input.next(); this.keyClass = keyClass; this.valueClass = valueClass; this.conf = conf; this.taskid = taskid; } /** Start processing next unique key. */ public boolean nextKey() throws IOException,InterruptedException { while (hasMore && nextKeyIsSame) { nextKeyValue(); } // 累加器 if (hasMore) { if (inputKeyCounter != null) { inputKeyCounter.increment(1); } return nextKeyValue(); } else { return false; } } /** * Advance to the next key/value pair. */ @Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!hasMore) { key = null; value = null; return false; } firstValue = !nextKeyIsSame; DataInputBuffer nextKey = input.getKey(); // map 输出 将 key-val 序列化字符数组 currentRawKey.set(nextKey.getData(), nextKey.getPosition(), nextKey.getLength() - nextKey.getPosition()); // buffer.reset(currentRawKey.getBytes(), 0, currentRawKey.getLength()); // 反序列化 key = keyDeserializer.deserialize(key); // 对 key 赋值 DataInputBuffer nextVal = input.getValue(); buffer.reset(nextVal.getData(), nextVal.getPosition(), nextVal.getLength() - nextVal.getPosition()); // 对 value 进行赋值 value = valueDeserializer.deserialize(value); currentKeyLength = nextKey.getLength() - nextKey.getPosition(); currentValueLength = nextVal.getLength() - nextVal.getPosition(); if (isMarked) { backupStore.write(nextKey, nextVal); } hasMore = input.next(); // 如果有第二条数据 if (hasMore) { nextKey = input.getKey(); // 分组比较器 (前一个key 与 后一个 key 进行比较) // 对下条key 进行预判 比较的 结果赋值给 nextKeyIsSame nextKeyIsSame = comparator.compare(currentRawKey.getBytes(), 0, currentRawKey.getLength(), nextKey.getData(), nextKey.getPosition(), nextKey.getLength() - nextKey.getPosition() ) == 0; } else { nextKeyIsSame = false; } inputValueCounter.increment(1); return true; } public KEYIN getCurrentKey() { return key; } @Override public VALUEIN getCurrentValue() { return value; } BackupStore<KEYIN,VALUEIN> getBackupStore() { return backupStore; } protected class ValueIterator implements ReduceContext.ValueIterator<VALUEIN> { private boolean inReset = false; private boolean clearMarkFlag = false; @Override public boolean hasNext() { try { if (inReset && backupStore.hasNext()) { return true; } } catch (Exception e) { e.printStackTrace(); throw new RuntimeException("hasNext failed", e); } // nextKeyIsSame 是由 return firstValue || nextKeyIsSame; } @Override public VALUEIN next() { if (inReset) { try { if (backupStore.hasNext()) { backupStore.next(); DataInputBuffer next = backupStore.nextValue(); buffer.reset(next.getData(), next.getPosition(), next.getLength() - next.getPosition()); value = valueDeserializer.deserialize(value); return value; } else { inReset = false; backupStore.exitResetMode(); if (clearMarkFlag) { clearMarkFlag = false; isMarked = false; } } } catch (IOException e) { e.printStackTrace(); throw new RuntimeException("next value iterator failed", e); } } // if this is the first record, we don't need to advance if (firstValue) { firstValue = false; return value; } // if this isn't the first record and the next key is different, they // can't advance it here. if (!nextKeyIsSame) { throw new NoSuchElementException("iterate past last value"); } // otherwise, go to the next key/value pair try { nextKeyValue(); return value; } catch (IOException ie) { throw new RuntimeException("next value iterator failed", ie); } catch (InterruptedException ie) { // this is bad, but we can't modify the exception list of java.util throw new RuntimeException("next value iterator interrupted", ie); } } @Override public void remove() { throw new UnsupportedOperationException("remove not implemented"); } @Override public void mark() throws IOException { if (getBackupStore() == null) { backupStore = new BackupStore<KEYIN,VALUEIN>(conf, taskid); } isMarked = true; if (!inReset) { backupStore.reinitialize(); if (currentKeyLength == -1) { // The user has not called next() for this iterator yet, so // there is no current record to mark and copy to backup store. return; } assert (currentValueLength != -1); int requestedSize = currentKeyLength + currentValueLength + WritableUtils.getVIntSize(currentKeyLength) + WritableUtils.getVIntSize(currentValueLength); DataOutputStream out = backupStore.getOutputStream(requestedSize); writeFirstKeyValueBytes(out); backupStore.updateCounters(requestedSize); } else { backupStore.mark(); } } @Override public void reset() throws IOException { // We reached the end of an iteration and user calls a // reset, but a clearMark was called before, just throw // an exception if (clearMarkFlag) { clearMarkFlag = false; backupStore.clearMark(); throw new IOException("Reset called without a previous mark"); } if (!isMarked) { throw new IOException("Reset called without a previous mark"); } inReset = true; backupStore.reset(); } @Override public void clearMark() throws IOException { if (getBackupStore() == null) { return; } if (inReset) { clearMarkFlag = true; backupStore.clearMark(); } else { inReset = isMarked = false; backupStore.reinitialize(); } } /** * This method is called when the reducer moves from one key to * another. * @throws IOException */ public void resetBackupStore() throws IOException { if (getBackupStore() == null) { return; } inReset = isMarked = false; backupStore.reinitialize(); currentKeyLength = -1; } /** * This method is called to write the record that was most recently * served (before a call to the mark). Since the framework reads one * record in advance, to get this record, we serialize the current key * and value * @param out * @throws IOException */ private void writeFirstKeyValueBytes(DataOutputStream out) throws IOException { assert (getCurrentKey() != null && getCurrentValue() != null); WritableUtils.writeVInt(out, currentKeyLength); WritableUtils.writeVInt(out, currentValueLength); Serializer<KEYIN> keySerializer = serializationFactory.getSerializer(keyClass); keySerializer.open(out); keySerializer.serialize(getCurrentKey()); Serializer<VALUEIN> valueSerializer = serializationFactory.getSerializer(valueClass); valueSerializer.open(out); valueSerializer.serialize(getCurrentValue()); } } protected class ValueIterable implements Iterable<VALUEIN> { private ValueIterator iterator = new ValueIterator(); @Override public Iterator<VALUEIN> iterator() { return iterator; } } /** * Iterate through the values for the current key, reusing the same value * object, which is stored in the context. * @return the series of values associated with the current key. All of the * objects returned directly and indirectly from this method are reused. */ public Iterable<VALUEIN> getValues() throws IOException, InterruptedException { return iterable; } }