org.apache.tez.runtime.library.common.sort.impl

Class ExternalSorter

java.lang.Object

org.apache.tez.runtime.library.common.sort.impl.ExternalSorter

Direct Known Subclasses:

DefaultSorter, PipelinedSorter

public abstract class ExternalSorter
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	ExternalSorter.MapBufferTooSmallException Exception indicating that the allocated sort buffer is insufficient to hold the current record.

Field Summary

Fields

Modifier and Type	Field and Description
protected org.apache.tez.common.counters.TezCounter	additionalSpillBytesRead
protected org.apache.tez.common.counters.TezCounter	additionalSpillBytesWritten
protected long	availableMemoryMb
protected org.apache.hadoop.io.compress.CompressionCodec	codec
protected Combiner	combiner
protected org.apache.hadoop.io.RawComparator	comparator
protected org.apache.hadoop.conf.Configuration	conf
protected org.apache.tez.common.counters.TezCounter	fileOutputByteCounter
protected org.apache.hadoop.fs.Path	finalIndexFile
protected org.apache.hadoop.fs.Path	finalOutputFile
protected int	ifileBufferSize
protected boolean	ifileReadAhead
protected int	ifileReadAheadLength
protected Class	keyClass
protected org.apache.hadoop.io.serializer.Serializer	keySerializer
protected org.apache.tez.common.counters.TezCounter	mapOutputByteCounter
protected TezTaskOutput	mapOutputFile
protected org.apache.tez.common.counters.TezCounter	mapOutputRecordCounter
protected org.apache.hadoop.util.Progressable	nullProgressable
protected org.apache.tez.common.counters.TezCounter	numAdditionalSpills
protected int	numSpills
protected org.apache.tez.common.counters.TezCounter	outputBytesWithOverheadCounter
protected org.apache.tez.runtime.api.OutputContext	outputContext
protected Partitioner	partitioner
protected int	partitions
protected org.apache.hadoop.fs.FileSystem	rfs
protected org.apache.hadoop.io.serializer.SerializationFactory	serializationFactory
protected org.apache.hadoop.util.IndexedSorter	sorter
protected org.apache.tez.common.counters.TezCounter	spilledRecordsCounter
protected Map<Integer,org.apache.hadoop.fs.Path>	spillFileIndexPaths
protected Map<Integer,org.apache.hadoop.fs.Path>	spillFilePaths
protected Class	valClass
protected org.apache.hadoop.io.serializer.Serializer	valSerializer

Constructor Summary

Constructors

Constructor and Description
ExternalSorter(org.apache.tez.runtime.api.OutputContext outputContext, org.apache.hadoop.conf.Configuration conf, int numOutputs, long initialMemoryAvailable)

Method Summary

Methods

Modifier and Type	Method and Description
void	close()
abstract void	flush()
org.apache.hadoop.fs.Path	getFinalIndexFile()
org.apache.hadoop.fs.Path	getFinalOutputFile()
static long	getInitialMemoryRequirement(org.apache.hadoop.conf.Configuration conf, long maxAvailableTaskMemory)
TezTaskOutput	getMapOutput()
int	getNumSpills()
ShuffleHeader	getShuffleHeader(int reduce)
InputStream	getSortedStream(int partition)
protected void	runCombineProcessor(TezRawKeyValueIterator kvIter, IFile.Writer writer)
protected void	sameVolRename(org.apache.hadoop.fs.Path srcPath, org.apache.hadoop.fs.Path dstPath) Rename srcPath to dstPath on the same volume.
void	write(Object key, Iterable<Object> values)
abstract void	write(Object key, Object value)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

nullProgressable

protected final org.apache.hadoop.util.Progressable nullProgressable

outputContext

protected final org.apache.tez.runtime.api.OutputContext outputContext

combiner

protected final Combiner combiner

partitioner

protected final Partitioner partitioner

conf

protected final org.apache.hadoop.conf.Configuration conf

rfs

protected final org.apache.hadoop.fs.FileSystem rfs

mapOutputFile

protected final TezTaskOutput mapOutputFile

partitions

protected final int partitions

keyClass

protected final Class keyClass

valClass

protected final Class valClass

comparator

protected final org.apache.hadoop.io.RawComparator comparator

serializationFactory

protected final org.apache.hadoop.io.serializer.SerializationFactory serializationFactory

keySerializer

protected final org.apache.hadoop.io.serializer.Serializer keySerializer

valSerializer

protected final org.apache.hadoop.io.serializer.Serializer valSerializer

ifileReadAhead

protected final boolean ifileReadAhead

ifileReadAheadLength

protected final int ifileReadAheadLength

ifileBufferSize

protected final int ifileBufferSize

availableMemoryMb

protected final long availableMemoryMb

sorter

protected final org.apache.hadoop.util.IndexedSorter sorter

codec

protected final org.apache.hadoop.io.compress.CompressionCodec codec

spillFilePaths

protected final Map<Integer,org.apache.hadoop.fs.Path> spillFilePaths

spillFileIndexPaths

protected final Map<Integer,org.apache.hadoop.fs.Path> spillFileIndexPaths

finalOutputFile

protected org.apache.hadoop.fs.Path finalOutputFile

finalIndexFile

protected org.apache.hadoop.fs.Path finalIndexFile

numSpills

protected int numSpills

mapOutputByteCounter

protected final org.apache.tez.common.counters.TezCounter mapOutputByteCounter

mapOutputRecordCounter

protected final org.apache.tez.common.counters.TezCounter mapOutputRecordCounter

outputBytesWithOverheadCounter

protected final org.apache.tez.common.counters.TezCounter outputBytesWithOverheadCounter

fileOutputByteCounter

protected final org.apache.tez.common.counters.TezCounter fileOutputByteCounter

spilledRecordsCounter

protected final org.apache.tez.common.counters.TezCounter spilledRecordsCounter

additionalSpillBytesWritten

protected final org.apache.tez.common.counters.TezCounter additionalSpillBytesWritten

additionalSpillBytesRead

protected final org.apache.tez.common.counters.TezCounter additionalSpillBytesRead

numAdditionalSpills

protected final org.apache.tez.common.counters.TezCounter numAdditionalSpills

Constructor Detail

ExternalSorter

public ExternalSorter(org.apache.tez.runtime.api.OutputContext outputContext,
              org.apache.hadoop.conf.Configuration conf,
              int numOutputs,
              long initialMemoryAvailable)
               throws IOException

Throws:

IOException

Method Detail

close

public void close()
           throws IOException

Throws:

IOException

flush

public abstract void flush()
                    throws IOException

Throws:

IOException

write

public abstract void write(Object key,
         Object value)
                    throws IOException

Throws:

IOException

write

public void write(Object key,
         Iterable<Object> values)
           throws IOException

Throws:

IOException

getMapOutput

@InterfaceAudience.Private
public TezTaskOutput getMapOutput()

getFinalIndexFile

@InterfaceAudience.Private
public org.apache.hadoop.fs.Path getFinalIndexFile()

getFinalOutputFile

public org.apache.hadoop.fs.Path getFinalOutputFile()

runCombineProcessor

protected void runCombineProcessor(TezRawKeyValueIterator kvIter,
                       IFile.Writer writer)
                            throws IOException

Throws:

IOException

sameVolRename

protected void sameVolRename(org.apache.hadoop.fs.Path srcPath,
                 org.apache.hadoop.fs.Path dstPath)
                      throws IOException

Rename srcPath to dstPath on the same volume. This is the same as RawLocalFileSystem's rename method, except that it will not fall back to a copy, and it will create the target directory if it doesn't exist.

Throws:

IOException

getSortedStream

public InputStream getSortedStream(int partition)

getShuffleHeader

public ShuffleHeader getShuffleHeader(int reduce)

getInitialMemoryRequirement

public static long getInitialMemoryRequirement(org.apache.hadoop.conf.Configuration conf,
                               long maxAvailableTaskMemory)

getNumSpills

public int getNumSpills()