co.cask.cdap.etl.spark

Class SparkPipelineRunner

java.lang.Object

co.cask.cdap.etl.spark.SparkPipelineRunner

Direct Known Subclasses:

BatchSparkPipelineDriver

public abstract class SparkPipelineRunner
extends Object

Base Spark program to run a Hydrator pipeline.

Constructor Summary

Constructors

Constructor and Description
SparkPipelineRunner()

Method Summary

Modifier and Type	Method and Description
protected abstract SparkPairCollection<Object,Object>	addJoinKey(StageSpec stageSpec, String inputStageName, SparkCollection<Object> inputCollection, StageStatisticsCollector collector)
protected abstract SparkCollection<RecordInfo<Object>>	getSource(StageSpec stageSpec, StageStatisticsCollector collector)
protected abstract SparkCollection<Object>	mergeJoinResults(StageSpec stageSpec, SparkPairCollection<Object,List<JoinElement<Object>>> joinedInputs, StageStatisticsCollector collector)
void	runPipeline(PipelinePhase pipelinePhase, String sourcePluginType, JavaSparkExecutionContext sec, Map<String,Integer> stagePartitions, PluginContext pluginContext, Map<String,StageStatisticsCollector> collectors)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

SparkPipelineRunner

public SparkPipelineRunner()

Method Detail

getSource

protected abstract SparkCollection<RecordInfo<Object>> getSource(StageSpec stageSpec,
                                                                 StageStatisticsCollector collector)
                                                          throws Exception

Throws:

Exception

addJoinKey

protected abstract SparkPairCollection<Object,Object> addJoinKey(StageSpec stageSpec,
                                                                 String inputStageName,
                                                                 SparkCollection<Object> inputCollection,
                                                                 StageStatisticsCollector collector)
                                                          throws Exception

Throws:

Exception

mergeJoinResults

protected abstract SparkCollection<Object> mergeJoinResults(StageSpec stageSpec,
                                                            SparkPairCollection<Object,List<JoinElement<Object>>> joinedInputs,
                                                            StageStatisticsCollector collector)
                                                     throws Exception

Throws:

Exception

runPipeline

public void runPipeline(PipelinePhase pipelinePhase,
                        String sourcePluginType,
                        JavaSparkExecutionContext sec,
                        Map<String,Integer> stagePartitions,
                        PluginContext pluginContext,
                        Map<String,StageStatisticsCollector> collectors)
                 throws Exception

Throws:

Exception