org.apache.beam.runners.spark.structuredstreaming.translation

Class TransformTranslator.Context

java.lang.Object

org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator.Context

Enclosing class:

TransformTranslator<InT extends PInput,OutT extends POutput,TransformT extends PTransform<? extends InT,OutT>>

protected class TransformTranslator.Context
extends java.lang.Object

Available mutable context to translate a PTransform. The context is backed by the shared PipelineTranslator.TranslationState of the PipelineTranslator.

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderProvider

EncoderProvider.Factory<T>

Method Summary

Modifier and Type	Method and Description
<T> org.apache.spark.broadcast.Broadcast<T>	broadcast(T value)
<T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>>	createDataset(java.util.List<org.apache.beam.sdk.util.WindowedValue<T>> data, org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> enc)
<T> org.apache.spark.sql.Encoder<T>	encoderOf(Coder<T> coder, EncoderProvider.Factory<T> factory)
org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,PTransform<InT,OutT>>	getCurrentTransform()
<T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>>	getDataset(PCollection<T> pCollection)
InT	getInput()
java.util.Map<TupleTag<?>,PCollection<?>>	getInputs()
PipelineOptions	getOptions()
OutT	getOutput()
<T> PCollection<T>	getOutput(TupleTag<T> tag)
java.util.Map<TupleTag<?>,PCollection<?>>	getOutputs()
org.apache.beam.runners.core.construction.SerializablePipelineOptions	getSerializableOptions()
org.apache.spark.sql.SparkSession	getSparkSession()
default <T> void	putDataset(PCollection<T> pCollection, org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset)
<T> void	putDataset(PCollection<T> pCollection, org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset, boolean cache)
<T1,T2> org.apache.spark.sql.Encoder<scala.Tuple2<T1,T2>>	tupleEncoder(org.apache.spark.sql.Encoder<T1> e1, org.apache.spark.sql.Encoder<T2> e2)
<T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>>	windowedEncoder(Coder<T> coder)
<T,W extends BoundedWindow> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>>	windowedEncoder(Coder<T> coder, Coder<W> windowCoder)
<T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>>	windowedEncoder(org.apache.spark.sql.Encoder<T> enc)
org.apache.spark.sql.Encoder<BoundedWindow>	windowEncoder()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderProvider

encoderFactory, encoderOf, keyEncoderOf, kvEncoderOf, valueEncoderOf

Method Detail

getInput

public InT getInput()

getInputs

public java.util.Map<TupleTag<?>,PCollection<?>> getInputs()

getOutputs

public java.util.Map<TupleTag<?>,PCollection<?>> getOutputs()

getOutput

public OutT getOutput()

getOutput

public <T> PCollection<T> getOutput(TupleTag<T> tag)

getCurrentTransform

public org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,PTransform<InT,OutT>> getCurrentTransform()

getDataset

public <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> getDataset(PCollection<T> pCollection)

putDataset

public <T> void putDataset(PCollection<T> pCollection,
                           org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset,
                           boolean cache)

getSerializableOptions

public org.apache.beam.runners.core.construction.SerializablePipelineOptions getSerializableOptions()

getOptions

public PipelineOptions getOptions()

createDataset

public <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> createDataset(java.util.List<org.apache.beam.sdk.util.WindowedValue<T>> data,
                                                                                                 org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> enc)

broadcast

public <T> org.apache.spark.broadcast.Broadcast<T> broadcast(T value)

getSparkSession

public org.apache.spark.sql.SparkSession getSparkSession()

encoderOf

public <T> org.apache.spark.sql.Encoder<T> encoderOf(Coder<T> coder,
                                                     EncoderProvider.Factory<T> factory)

tupleEncoder

public <T1,T2> org.apache.spark.sql.Encoder<scala.Tuple2<T1,T2>> tupleEncoder(org.apache.spark.sql.Encoder<T1> e1,
                                                                              org.apache.spark.sql.Encoder<T2> e2)

windowedEncoder

public <T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> windowedEncoder(Coder<T> coder)

windowedEncoder

public <T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> windowedEncoder(org.apache.spark.sql.Encoder<T> enc)

windowedEncoder

public <T,W extends BoundedWindow> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> windowedEncoder(Coder<T> coder,
                                                                                                                           Coder<W> windowCoder)

windowEncoder

public org.apache.spark.sql.Encoder<BoundedWindow> windowEncoder()

putDataset

public <T> void putDataset(PCollection<T> pCollection,
                           org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset)