co.cask.cdap.etl.spark.streaming

Class DStreamCollection<T>

java.lang.Object

co.cask.cdap.etl.spark.streaming.DStreamCollection<T>

Type Parameters:

T - type of objects in the collection

All Implemented Interfaces:

SparkCollection<T>

public class DStreamCollection<T>
extends Object
implements SparkCollection<T>

JavaDStream backed SparkCollection

Constructor Summary

Constructors

Constructor and Description
DStreamCollection(JavaSparkExecutionContext sec, org.apache.spark.streaming.api.java.JavaDStream<T> stream)

Method Summary

Modifier and Type	Method and Description
SparkCollection<RecordInfo<Object>>	aggregate(StageSpec stageSpec, Integer partitions, StageStatisticsCollector collector)
SparkCollection<T>	cache()
<U> SparkCollection<U>	compute(StageSpec stageSpec, SparkCompute<T,U> compute)
<U> SparkCollection<U>	flatMap(StageSpec stageSpec, org.apache.spark.api.java.function.FlatMapFunction<T,U> function)
<K,V> SparkPairCollection<K,V>	flatMapToPair(org.apache.spark.api.java.function.PairFlatMapFunction<T,K,V> function)
org.apache.spark.streaming.api.java.JavaDStream<T>	getUnderlying()
SparkCollection<RecordInfo<Object>>	multiOutputTransform(StageSpec stageSpec, StageStatisticsCollector collector)
void	publishAlerts(StageSpec stageSpec, StageStatisticsCollector collector)
void	store(StageSpec stageSpec, org.apache.spark.api.java.function.PairFlatMapFunction<T,Object,Object> sinkFunction)
void	store(StageSpec stageSpec, SparkSink<T> sink)
SparkCollection<RecordInfo<Object>>	transform(StageSpec stageSpec, StageStatisticsCollector collector)
SparkCollection<T>	union(SparkCollection<T> other)
SparkCollection<T>	window(StageSpec stageSpec, Windower windower)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

DStreamCollection

public DStreamCollection(JavaSparkExecutionContext sec,
                         org.apache.spark.streaming.api.java.JavaDStream<T> stream)

Method Detail

getUnderlying

public org.apache.spark.streaming.api.java.JavaDStream<T> getUnderlying()

Specified by:

getUnderlying in interface SparkCollection<T>

cache

public SparkCollection<T> cache()

Specified by:

cache in interface SparkCollection<T>

union

public SparkCollection<T> union(SparkCollection<T> other)

Specified by:

union in interface SparkCollection<T>

transform

public SparkCollection<RecordInfo<Object>> transform(StageSpec stageSpec,
                                                     StageStatisticsCollector collector)

Specified by:

transform in interface SparkCollection<T>

multiOutputTransform

public SparkCollection<RecordInfo<Object>> multiOutputTransform(StageSpec stageSpec,
                                                                StageStatisticsCollector collector)

Specified by:

multiOutputTransform in interface SparkCollection<T>

flatMap

public <U> SparkCollection<U> flatMap(StageSpec stageSpec,
                                      org.apache.spark.api.java.function.FlatMapFunction<T,U> function)

Specified by:

flatMap in interface SparkCollection<T>

flatMapToPair

public <K,V> SparkPairCollection<K,V> flatMapToPair(org.apache.spark.api.java.function.PairFlatMapFunction<T,K,V> function)

Specified by:

flatMapToPair in interface SparkCollection<T>

aggregate

public SparkCollection<RecordInfo<Object>> aggregate(StageSpec stageSpec,
                                                     @Nullable
                                                     Integer partitions,
                                                     StageStatisticsCollector collector)

Specified by:

aggregate in interface SparkCollection<T>

compute

public <U> SparkCollection<U> compute(StageSpec stageSpec,
                                      SparkCompute<T,U> compute)
                               throws Exception

Specified by:

compute in interface SparkCollection<T>

Throws:

Exception

store

public void store(StageSpec stageSpec,
                  org.apache.spark.api.java.function.PairFlatMapFunction<T,Object,Object> sinkFunction)

Specified by:

store in interface SparkCollection<T>

store

public void store(StageSpec stageSpec,
                  SparkSink<T> sink)
           throws Exception

Specified by:

store in interface SparkCollection<T>

Throws:

Exception

publishAlerts

public void publishAlerts(StageSpec stageSpec,
                          StageStatisticsCollector collector)
                   throws Exception

Specified by:

publishAlerts in interface SparkCollection<T>

Throws:

Exception

window

public SparkCollection<T> window(StageSpec stageSpec,
                                 Windower windower)

Specified by:

window in interface SparkCollection<T>