co.cask.cdap.etl.spark.batch

Class BasicSparkExecutionPluginContext

java.lang.Object

co.cask.cdap.etl.common.AbstractStageContext

co.cask.cdap.etl.common.AbstractTransformContext

co.cask.cdap.etl.spark.batch.BasicSparkExecutionPluginContext

All Implemented Interfaces:

DatasetContext, ServiceDiscoverer, SparkExecutionPluginContext, LookupProvider, StageContext, TransformContext

public class BasicSparkExecutionPluginContext
extends AbstractTransformContext
implements SparkExecutionPluginContext

Implementation of SparkExecutionPluginContext by delegating to JavaSparkExecutionContext.

Field Summary

Fields inherited from class co.cask.cdap.etl.common.AbstractStageContext

arguments

Constructor Summary

Constructors

Constructor and Description
BasicSparkExecutionPluginContext(JavaSparkExecutionContext sec, org.apache.spark.api.java.JavaSparkContext jsc, DatasetContext datasetContext, PipelineRuntime pipelineRuntime, StageSpec stageSpec)

Method Summary

Modifier and Type	Method and Description
SparkInterpreter	createSparkInterpreter()
void	discardDataset(Dataset dataset)
<K,V> org.apache.spark.api.java.JavaPairRDD<K,V>	fromDataset(String datasetName)
<K,V> org.apache.spark.api.java.JavaPairRDD<K,V>	fromDataset(String datasetName, Map<String,String> arguments)
<K,V> org.apache.spark.api.java.JavaPairRDD<K,V>	fromDataset(String datasetName, Map<String,String> arguments, Iterable<? extends Split> splits)
org.apache.spark.api.java.JavaRDD<StreamEvent>	fromStream(String streamName)
<V> org.apache.spark.api.java.JavaPairRDD<Long,V>	fromStream(String streamName, Class<V> valueType)
org.apache.spark.api.java.JavaRDD<StreamEvent>	fromStream(String streamName, long startTime, long endTime)
<K,V> org.apache.spark.api.java.JavaPairRDD<K,V>	fromStream(String streamName, long startTime, long endTime, Class<? extends co.cask.cdap.api.stream.StreamEventDecoder<K,V>> decoderClass, Class<K> keyType, Class<V> valueType)
<V> org.apache.spark.api.java.JavaPairRDD<Long,V>	fromStream(String streamName, long startTime, long endTime, Class<V> valueType)
<T extends Dataset> T	getDataset(String name)
<T extends Dataset> T	getDataset(String name, Map<String,String> arguments)
<T extends Dataset> T	getDataset(String namespace, String name)
<T extends Dataset> T	getDataset(String namespace, String name, Map<String,String> arguments)
long	getLogicalStartTime()
PluginContext	getPluginContext()
Map<String,String>	getRuntimeArguments()
org.apache.spark.api.java.JavaSparkContext	getSparkContext()
<T> Lookup<T>	provide(String table, Map<String,String> arguments)
void	releaseDataset(Dataset dataset)
<K,V> void	saveAsDataset(org.apache.spark.api.java.JavaPairRDD<K,V> rdd, String datasetName)
<K,V> void	saveAsDataset(org.apache.spark.api.java.JavaPairRDD<K,V> rdd, String datasetName, Map<String,String> arguments)

Methods inherited from class co.cask.cdap.etl.common.AbstractStageContext

getArguments, getInputSchema, getInputSchemas, getMetrics, getNamespace, getOutputPortSchemas, getOutputSchema, getPipelineName, getPluginProperties, getPluginProperties, getServiceURL, getServiceURL, getStageName, loadPluginClass, newPluginInstance

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface co.cask.cdap.etl.api.StageContext

getArguments, getInputSchema, getInputSchemas, getMetrics, getNamespace, getOutputPortSchemas, getOutputSchema, getPipelineName, getPluginProperties, getPluginProperties, getStageName, loadPluginClass, newPluginInstance

Methods inherited from interface co.cask.cdap.api.ServiceDiscoverer

getServiceURL, getServiceURL

Constructor Detail

BasicSparkExecutionPluginContext

public BasicSparkExecutionPluginContext(JavaSparkExecutionContext sec,
                                        org.apache.spark.api.java.JavaSparkContext jsc,
                                        DatasetContext datasetContext,
                                        PipelineRuntime pipelineRuntime,
                                        StageSpec stageSpec)

Method Detail

getLogicalStartTime

public long getLogicalStartTime()

Specified by:

getLogicalStartTime in interface SparkExecutionPluginContext

Specified by:

getLogicalStartTime in interface StageContext

Overrides:

getLogicalStartTime in class AbstractStageContext

getRuntimeArguments

public Map<String,String> getRuntimeArguments()

Specified by:

getRuntimeArguments in interface SparkExecutionPluginContext

fromDataset

public <K,V> org.apache.spark.api.java.JavaPairRDD<K,V> fromDataset(String datasetName)

Specified by:

fromDataset in interface SparkExecutionPluginContext

fromDataset

public <K,V> org.apache.spark.api.java.JavaPairRDD<K,V> fromDataset(String datasetName,
                                                                    Map<String,String> arguments)

Specified by:

fromDataset in interface SparkExecutionPluginContext

fromDataset

public <K,V> org.apache.spark.api.java.JavaPairRDD<K,V> fromDataset(String datasetName,
                                                                    Map<String,String> arguments,
                                                                    @Nullable
                                                                    Iterable<? extends Split> splits)

Specified by:

fromDataset in interface SparkExecutionPluginContext

fromStream

public org.apache.spark.api.java.JavaRDD<StreamEvent> fromStream(String streamName)

Specified by:

fromStream in interface SparkExecutionPluginContext

fromStream

public org.apache.spark.api.java.JavaRDD<StreamEvent> fromStream(String streamName,
                                                                 long startTime,
                                                                 long endTime)

Specified by:

fromStream in interface SparkExecutionPluginContext

fromStream

public <V> org.apache.spark.api.java.JavaPairRDD<Long,V> fromStream(String streamName,
                                                                    Class<V> valueType)

Specified by:

fromStream in interface SparkExecutionPluginContext

fromStream

public <V> org.apache.spark.api.java.JavaPairRDD<Long,V> fromStream(String streamName,
                                                                    long startTime,
                                                                    long endTime,
                                                                    Class<V> valueType)

Specified by:

fromStream in interface SparkExecutionPluginContext

fromStream

public <K,V> org.apache.spark.api.java.JavaPairRDD<K,V> fromStream(String streamName,
                                                                   long startTime,
                                                                   long endTime,
                                                                   Class<? extends co.cask.cdap.api.stream.StreamEventDecoder<K,V>> decoderClass,
                                                                   Class<K> keyType,
                                                                   Class<V> valueType)

Specified by:

fromStream in interface SparkExecutionPluginContext

saveAsDataset

public <K,V> void saveAsDataset(org.apache.spark.api.java.JavaPairRDD<K,V> rdd,
                                String datasetName)

Specified by:

saveAsDataset in interface SparkExecutionPluginContext

saveAsDataset

public <K,V> void saveAsDataset(org.apache.spark.api.java.JavaPairRDD<K,V> rdd,
                                String datasetName,
                                Map<String,String> arguments)

Specified by:

saveAsDataset in interface SparkExecutionPluginContext

getSparkContext

public org.apache.spark.api.java.JavaSparkContext getSparkContext()

Specified by:

getSparkContext in interface SparkExecutionPluginContext

getPluginContext

public PluginContext getPluginContext()

Specified by:

getPluginContext in interface SparkExecutionPluginContext

createSparkInterpreter

public SparkInterpreter createSparkInterpreter()
                                        throws IOException

Specified by:

createSparkInterpreter in interface SparkExecutionPluginContext

Throws:

IOException

getDataset

public <T extends Dataset> T getDataset(String name)
                                 throws DatasetInstantiationException

Specified by:

getDataset in interface DatasetContext

Throws:

DatasetInstantiationException

getDataset

public <T extends Dataset> T getDataset(String namespace,
                                        String name)
                                 throws DatasetInstantiationException

Specified by:

getDataset in interface DatasetContext

Throws:

DatasetInstantiationException

getDataset

public <T extends Dataset> T getDataset(String name,
                                        Map<String,String> arguments)
                                 throws DatasetInstantiationException

Specified by:

getDataset in interface DatasetContext

Throws:

DatasetInstantiationException

getDataset

public <T extends Dataset> T getDataset(String namespace,
                                        String name,
                                        Map<String,String> arguments)
                                 throws DatasetInstantiationException

Specified by:

getDataset in interface DatasetContext

Throws:

DatasetInstantiationException

releaseDataset

public void releaseDataset(Dataset dataset)

Specified by:

releaseDataset in interface DatasetContext

discardDataset

public void discardDataset(Dataset dataset)

Specified by:

discardDataset in interface DatasetContext

provide

public <T> Lookup<T> provide(String table,
                             Map<String,String> arguments)

Specified by:

provide in interface LookupProvider

Overrides:

provide in class AbstractTransformContext