co.cask.cdap.etl.spark

Class StreamingCompat

java.lang.Object

co.cask.cdap.etl.spark.StreamingCompat

public final class StreamingCompat
extends Object

Utility class to handle incompatibilities between Spark1 and Spark2 streaming. All hydrator-spark-core modules must have this class with the exact same method signatures. Incompatibilities are in a few places. DStream.foreachRDD() no longer takes a Function2 in Spark2 in favor of VoidFunction2. In Spark1.2, a Function2 must be used because VoidFunction2 was not yet introduced. Outer join methods in Spark1 use guava's Optional whereas Spark2 uses its own Optional. JavaStreamingContext.getOrCreate() does not use a JavaStreamingContextFactory in Spark2, but requires it in Spark1.2.

Method Summary

Modifier and Type	Method and Description
static <T> void	foreachRDD(org.apache.spark.streaming.api.java.JavaDStream<T> stream, org.apache.spark.api.java.function.Function2<org.apache.spark.api.java.JavaRDD<T>,org.apache.spark.streaming.Time,Void> func)
static <K,V1,V2> org.apache.spark.streaming.api.java.JavaPairDStream<K,scala.Tuple2<com.google.common.base.Optional<V1>,com.google.common.base.Optional<V2>>>	fullOuterJoin(org.apache.spark.streaming.api.java.JavaPairDStream<K,V1> left, org.apache.spark.streaming.api.java.JavaPairDStream<K,V2> right)
static <K,V1,V2> org.apache.spark.streaming.api.java.JavaPairDStream<K,scala.Tuple2<com.google.common.base.Optional<V1>,com.google.common.base.Optional<V2>>>	fullOuterJoin(org.apache.spark.streaming.api.java.JavaPairDStream<K,V1> left, org.apache.spark.streaming.api.java.JavaPairDStream<K,V2> right, int numPartitions)
static org.apache.spark.streaming.api.java.JavaStreamingContext	getOrCreate(String checkpointDir, org.apache.spark.api.java.function.Function0<org.apache.spark.streaming.api.java.JavaStreamingContext> contextFunc)
static <K,V1,V2> org.apache.spark.streaming.api.java.JavaPairDStream<K,scala.Tuple2<V1,com.google.common.base.Optional<V2>>>	leftOuterJoin(org.apache.spark.streaming.api.java.JavaPairDStream<K,V1> left, org.apache.spark.streaming.api.java.JavaPairDStream<K,V2> right)
static <K,V1,V2> org.apache.spark.streaming.api.java.JavaPairDStream<K,scala.Tuple2<V1,com.google.common.base.Optional<V2>>>	leftOuterJoin(org.apache.spark.streaming.api.java.JavaPairDStream<K,V1> left, org.apache.spark.streaming.api.java.JavaPairDStream<K,V2> right, int numPartitions)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Method Detail

getOrCreate

public static org.apache.spark.streaming.api.java.JavaStreamingContext getOrCreate(String checkpointDir,
                                                                                   org.apache.spark.api.java.function.Function0<org.apache.spark.streaming.api.java.JavaStreamingContext> contextFunc)

foreachRDD

public static <T> void foreachRDD(org.apache.spark.streaming.api.java.JavaDStream<T> stream,
                                  org.apache.spark.api.java.function.Function2<org.apache.spark.api.java.JavaRDD<T>,org.apache.spark.streaming.Time,Void> func)

leftOuterJoin

public static <K,V1,V2> org.apache.spark.streaming.api.java.JavaPairDStream<K,scala.Tuple2<V1,com.google.common.base.Optional<V2>>> leftOuterJoin(org.apache.spark.streaming.api.java.JavaPairDStream<K,V1> left,
                                                                                                                                                  org.apache.spark.streaming.api.java.JavaPairDStream<K,V2> right)

leftOuterJoin

public static <K,V1,V2> org.apache.spark.streaming.api.java.JavaPairDStream<K,scala.Tuple2<V1,com.google.common.base.Optional<V2>>> leftOuterJoin(org.apache.spark.streaming.api.java.JavaPairDStream<K,V1> left,
                                                                                                                                                  org.apache.spark.streaming.api.java.JavaPairDStream<K,V2> right,
                                                                                                                                                  int numPartitions)

fullOuterJoin

public static <K,V1,V2> org.apache.spark.streaming.api.java.JavaPairDStream<K,scala.Tuple2<com.google.common.base.Optional<V1>,com.google.common.base.Optional<V2>>> fullOuterJoin(org.apache.spark.streaming.api.java.JavaPairDStream<K,V1> left,
                                                                                                                                                                                   org.apache.spark.streaming.api.java.JavaPairDStream<K,V2> right)

fullOuterJoin

public static <K,V1,V2> org.apache.spark.streaming.api.java.JavaPairDStream<K,scala.Tuple2<com.google.common.base.Optional<V1>,com.google.common.base.Optional<V2>>> fullOuterJoin(org.apache.spark.streaming.api.java.JavaPairDStream<K,V1> left,
                                                                                                                                                                                   org.apache.spark.streaming.api.java.JavaPairDStream<K,V2> right,
                                                                                                                                                                                   int numPartitions)