public class ShuffleUtils extends Object
| Modifier and Type | Field and Description |
|---|---|
static String |
SHUFFLE_HANDLER_SERVICE_ID |
| Constructor and Description |
|---|
ShuffleUtils() |
| Modifier and Type | Method and Description |
|---|---|
static StringBuilder |
constructBaseURIForShuffleHandler(String host,
int port,
int partition,
String appId,
boolean sslShuffle) |
static StringBuilder |
constructBaseURIForShuffleHandler(String hostIdentifier,
int partition,
String appId,
boolean sslShuffle) |
static HttpConnection.HttpConnectionParams |
constructHttpShuffleConnectionParams(org.apache.hadoop.conf.Configuration conf) |
static URL |
constructInputURL(String baseURI,
List<InputAttemptIdentifier> inputs,
boolean keepAlive) |
static ByteBuffer |
convertJobTokenToBytes(org.apache.hadoop.security.token.Token<org.apache.tez.common.security.JobTokenIdentifier> jobToken) |
static int |
deserializeShuffleProviderMetaData(ByteBuffer meta) |
static SecretKey |
getJobTokenSecretFromTokenBytes(ByteBuffer meta) |
static void |
shuffleToDisk(OutputStream output,
String hostIdentifier,
InputStream input,
long compressedLength,
org.apache.commons.logging.Log LOG,
String identifier) |
static void |
shuffleToMemory(byte[] shuffleData,
InputStream input,
int decompressedLength,
int compressedLength,
org.apache.hadoop.io.compress.CompressionCodec codec,
boolean ifileReadAhead,
int ifileReadAheadLength,
org.apache.commons.logging.Log LOG,
String identifier) |
static String |
stringify(ShuffleUserPayloads.DataMovementEventPayloadProto dmProto) |
public static String SHUFFLE_HANDLER_SERVICE_ID
public static SecretKey getJobTokenSecretFromTokenBytes(ByteBuffer meta) throws IOException
IOExceptionpublic static ByteBuffer convertJobTokenToBytes(org.apache.hadoop.security.token.Token<org.apache.tez.common.security.JobTokenIdentifier> jobToken) throws IOException
IOExceptionpublic static int deserializeShuffleProviderMetaData(ByteBuffer meta) throws IOException
IOExceptionpublic static void shuffleToMemory(byte[] shuffleData,
InputStream input,
int decompressedLength,
int compressedLength,
org.apache.hadoop.io.compress.CompressionCodec codec,
boolean ifileReadAhead,
int ifileReadAheadLength,
org.apache.commons.logging.Log LOG,
String identifier)
throws IOException
IOExceptionpublic static void shuffleToDisk(OutputStream output, String hostIdentifier, InputStream input, long compressedLength, org.apache.commons.logging.Log LOG, String identifier) throws IOException
IOExceptionpublic static StringBuilder constructBaseURIForShuffleHandler(String host, int port, int partition, String appId, boolean sslShuffle)
public static StringBuilder constructBaseURIForShuffleHandler(String hostIdentifier, int partition, String appId, boolean sslShuffle)
public static URL constructInputURL(String baseURI, List<InputAttemptIdentifier> inputs, boolean keepAlive) throws MalformedURLException
MalformedURLExceptionpublic static HttpConnection.HttpConnectionParams constructHttpShuffleConnectionParams(org.apache.hadoop.conf.Configuration conf)
public static String stringify(ShuffleUserPayloads.DataMovementEventPayloadProto dmProto)
Copyright © 2014 Apache Software Foundation. All rights reserved.