org.apache.beam.runners.spark.structuredstreaming.translation

Class AbstractTranslationContext

java.lang.Object

org.apache.beam.runners.spark.structuredstreaming.translation.AbstractTranslationContext

Direct Known Subclasses:

TranslationContext

public abstract class AbstractTranslationContext
extends java.lang.Object

Base class that gives a context for PTransform translation: keeping track of the datasets, the SparkSession, the current transform being translated.

Constructor Summary

Constructors

Constructor and Description
AbstractTranslationContext(SparkStructuredStreamingPipelineOptions options)

Method Summary

Modifier and Type	Method and Description
<T> org.apache.spark.sql.Dataset<T>	emptyDataset()
org.apache.beam.sdk.runners.AppliedPTransform<?,?,?>	getCurrentTransform()
<T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>>	getDataset(PValue value)
PValue	getInput()
java.util.Map<TupleTag<?>,PCollection<?>>	getInputs()
PValue	getOutput()
java.util.Map<TupleTag<?>,Coder<?>>	getOutputCoders()
java.util.Map<TupleTag<?>,PCollection<?>>	getOutputs()
org.apache.beam.runners.core.construction.SerializablePipelineOptions	getSerializableOptions()
<T> org.apache.spark.sql.Dataset<T>	getSideInputDataSet(PCollectionView<?> value)
org.apache.spark.sql.SparkSession	getSparkSession()
abstract void	launchStreaming(org.apache.spark.sql.streaming.DataStreamWriter<?> dataStreamWriter)
static void	printDatasetContent(org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue> dataset)
<T> void	putDataset(PValue value, org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset)
void	putDatasetWildcard(PValue value, org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<?>> dataset) TODO: All these 3 methods (putDataset*) are temporary and they are used only for generics type checking.
void	setCurrentTransform(org.apache.beam.sdk.runners.AppliedPTransform<?,?,?> currentTransform)
<ViewT,ElemT> void	setSideInputDataset(PCollectionView<ViewT> value, org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<ElemT>> set)
void	startPipeline() Starts the pipeline.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

AbstractTranslationContext

public AbstractTranslationContext(SparkStructuredStreamingPipelineOptions options)

Method Detail

getSparkSession

public org.apache.spark.sql.SparkSession getSparkSession()

getSerializableOptions

public org.apache.beam.runners.core.construction.SerializablePipelineOptions getSerializableOptions()

setCurrentTransform

public void setCurrentTransform(org.apache.beam.sdk.runners.AppliedPTransform<?,?,?> currentTransform)

getCurrentTransform

public org.apache.beam.sdk.runners.AppliedPTransform<?,?,?> getCurrentTransform()

emptyDataset

public <T> org.apache.spark.sql.Dataset<T> emptyDataset()

getDataset

public <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> getDataset(PValue value)

putDatasetWildcard

public void putDatasetWildcard(PValue value,
                               org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<?>> dataset)

TODO: All these 3 methods (putDataset*) are temporary and they are used only for generics type checking. We should unify them in the future.

putDataset

public <T> void putDataset(PValue value,
                           org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset)

setSideInputDataset

public <ViewT,ElemT> void setSideInputDataset(PCollectionView<ViewT> value,
                                              org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<ElemT>> set)

getSideInputDataSet

public <T> org.apache.spark.sql.Dataset<T> getSideInputDataSet(PCollectionView<?> value)

getInput

public PValue getInput()

getInputs

public java.util.Map<TupleTag<?>,PCollection<?>> getInputs()

getOutput

public PValue getOutput()

getOutputs

public java.util.Map<TupleTag<?>,PCollection<?>> getOutputs()

getOutputCoders

public java.util.Map<TupleTag<?>,Coder<?>> getOutputCoders()

startPipeline

public void startPipeline()

Starts the pipeline.

launchStreaming

public abstract void launchStreaming(org.apache.spark.sql.streaming.DataStreamWriter<?> dataStreamWriter)

printDatasetContent

public static void printDatasetContent(org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue> dataset)