org.apache.beam.runners.spark.structuredstreaming.translation

Class TransformTranslator.Context

java.lang.Object

org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator.Context

All Implemented Interfaces:

EncoderProvider, PipelineTranslator.TranslationState

Enclosing class:

TransformTranslator<InT extends PInput,OutT extends POutput,TransformT extends PTransform<InT,OutT>>

protected class TransformTranslator.Context
extends java.lang.Object
implements PipelineTranslator.TranslationState

Available mutable context to translate a PTransform. The context is backed by the shared PipelineTranslator.TranslationState of the PipelineTranslator.

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderProvider

EncoderProvider.Factory<T>

Method Summary

Modifier and Type	Method and Description
<T> org.apache.spark.broadcast.Broadcast<T>	broadcast(T value)
<T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>>	createDataset(java.util.List<org.apache.beam.sdk.util.WindowedValue<T>> data, org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> enc)
<T> org.apache.spark.sql.Encoder<T>	encoderOf(Coder<T> coder, EncoderProvider.Factory<T> factory)
org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,TransformT>	getCurrentTransform()
<T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>>	getDataset(PCollection<T> pCollection)
InT	getInput()
java.util.Map<TupleTag<?>,PCollection<?>>	getInputs()
PipelineOptions	getOptions()
java.util.function.Supplier<PipelineOptions>	getOptionsSupplier()
OutT	getOutput()
<T> PCollection<T>	getOutput(TupleTag<T> tag)
java.util.Map<TupleTag<?>,PCollection<?>>	getOutputs()
<T> org.apache.spark.broadcast.Broadcast<SideInputValues<T>>	getSideInputBroadcast(PCollection<T> pCollection, SideInputValues.Loader<T> loader)
org.apache.spark.sql.SparkSession	getSparkSession()
boolean	isLeaf(PCollection<?> pCollection)
<T> void	putDataset(PCollection<T> pCollection, org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset, boolean cache)
<InputT,T> void	putUnresolved(PCollection<T> out, PipelineTranslator.UnresolvedTranslation<InputT,T> unresolved)
<T1,T2> org.apache.spark.sql.Encoder<scala.Tuple2<T1,T2>>	tupleEncoder(org.apache.spark.sql.Encoder<T1> e1, org.apache.spark.sql.Encoder<T2> e2)
<T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>>	windowedEncoder(Coder<T> coder)
<T,W extends BoundedWindow> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>>	windowedEncoder(Coder<T> coder, Coder<W> windowCoder)
<T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>>	windowedEncoder(org.apache.spark.sql.Encoder<T> enc)
org.apache.spark.sql.Encoder<BoundedWindow>	windowEncoder()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator.TranslationState

putDataset

Methods inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderProvider

encoderFactory, encoderOf, keyEncoderOf, kvEncoderOf, valueEncoderOf

Method Detail

getInput

public InT getInput()

getInputs

public java.util.Map<TupleTag<?>,PCollection<?>> getInputs()

getOutputs

public java.util.Map<TupleTag<?>,PCollection<?>> getOutputs()

getOutput

public OutT getOutput()

getOutput

public <T> PCollection<T> getOutput(TupleTag<T> tag)

getCurrentTransform

public org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,TransformT> getCurrentTransform()

getDataset

public <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> getDataset(PCollection<T> pCollection)

Specified by:

getDataset in interface PipelineTranslator.TranslationState

getSideInputBroadcast

public <T> org.apache.spark.broadcast.Broadcast<SideInputValues<T>> getSideInputBroadcast(PCollection<T> pCollection,
                                                                                          SideInputValues.Loader<T> loader)

Specified by:

getSideInputBroadcast in interface PipelineTranslator.TranslationState

putDataset

public <T> void putDataset(PCollection<T> pCollection,
                           org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset,
                           boolean cache)

Specified by:

putDataset in interface PipelineTranslator.TranslationState

putUnresolved

public <InputT,T> void putUnresolved(PCollection<T> out,
                                     PipelineTranslator.UnresolvedTranslation<InputT,T> unresolved)

Specified by:

putUnresolved in interface PipelineTranslator.TranslationState

isLeaf

public boolean isLeaf(PCollection<?> pCollection)

Specified by:

isLeaf in interface PipelineTranslator.TranslationState

getOptionsSupplier

public java.util.function.Supplier<PipelineOptions> getOptionsSupplier()

Specified by:

getOptionsSupplier in interface PipelineTranslator.TranslationState

getOptions

public PipelineOptions getOptions()

Specified by:

getOptions in interface PipelineTranslator.TranslationState

createDataset

public <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> createDataset(java.util.List<org.apache.beam.sdk.util.WindowedValue<T>> data,
                                                                                                 org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> enc)

broadcast

public <T> org.apache.spark.broadcast.Broadcast<T> broadcast(T value)

getSparkSession

public org.apache.spark.sql.SparkSession getSparkSession()

Specified by:

getSparkSession in interface PipelineTranslator.TranslationState

encoderOf

public <T> org.apache.spark.sql.Encoder<T> encoderOf(Coder<T> coder,
                                                     EncoderProvider.Factory<T> factory)

Specified by:

encoderOf in interface EncoderProvider

tupleEncoder

public <T1,T2> org.apache.spark.sql.Encoder<scala.Tuple2<T1,T2>> tupleEncoder(org.apache.spark.sql.Encoder<T1> e1,
                                                                              org.apache.spark.sql.Encoder<T2> e2)

windowedEncoder

public <T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> windowedEncoder(Coder<T> coder)

windowedEncoder

public <T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> windowedEncoder(org.apache.spark.sql.Encoder<T> enc)

windowedEncoder

public <T,W extends BoundedWindow> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> windowedEncoder(Coder<T> coder,
                                                                                                                           Coder<W> windowCoder)

windowEncoder

public org.apache.spark.sql.Encoder<BoundedWindow> windowEncoder()