Package org.apache.beam.runners.spark.structuredstreaming.translation

Class TransformTranslator.Context

java.lang.Object

org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator.Context

All Implemented Interfaces:

EncoderProvider, PipelineTranslator.TranslationState

Enclosing class:

TransformTranslator<InT extends PInput,OutT extends POutput,TransformT extends PTransform<InT,OutT>>

protected class TransformTranslator.Context
extends Object
implements PipelineTranslator.TranslationState

Available mutable context to translate a PTransform. The context is backed by the shared PipelineTranslator.TranslationState of the PipelineTranslator.

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderProvider

EncoderProvider.Factory<T>

Method Summary

Modifier and Type	Method	Description
<T> org.apache.spark.broadcast.Broadcast<T>	broadcast(T value)
<T> org.apache.spark.sql.Dataset<WindowedValue<T>>	createDataset(List<WindowedValue<T>> data, org.apache.spark.sql.Encoder<WindowedValue<T>> enc)
<T> org.apache.spark.sql.Encoder<T>	encoderOf(Coder<T> coder, EncoderProvider.Factory<T> factory)
org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,TransformT>	getCurrentTransform()
<T> org.apache.spark.sql.Dataset<WindowedValue<T>>	getDataset(PCollection<T> pCollection)
InT	getInput()
Map<TupleTag<?>,PCollection<?>>	getInputs()
PipelineOptions	getOptions()
Supplier<PipelineOptions>	getOptionsSupplier()
OutT	getOutput()
<T> PCollection<T>	getOutput(TupleTag<T> tag)
Map<TupleTag<?>,PCollection<?>>	getOutputs()
<T> org.apache.spark.broadcast.Broadcast<SideInputValues<T>>	getSideInputBroadcast(PCollection<T> pCollection, SideInputValues.Loader<T> loader)
org.apache.spark.sql.SparkSession	getSparkSession()
boolean	isLeaf(PCollection<?> pCollection)
<T> void	putDataset(PCollection<T> pCollection, org.apache.spark.sql.Dataset<WindowedValue<T>> dataset, boolean cache)
<InputT, T> void	putUnresolved(PCollection<T> out, PipelineTranslator.UnresolvedTranslation<InputT,T> unresolved)
<T1, T2> org.apache.spark.sql.Encoder<scala.Tuple2<T1,T2>>	tupleEncoder(org.apache.spark.sql.Encoder<T1> e1, org.apache.spark.sql.Encoder<T2> e2)
<T> org.apache.spark.sql.Encoder<WindowedValue<T>>	windowedEncoder(Coder<T> coder)
<T, W extends BoundedWindow> org.apache.spark.sql.Encoder<WindowedValue<T>>	windowedEncoder(Coder<T> coder, Coder<W> windowCoder)
<T> org.apache.spark.sql.Encoder<WindowedValue<T>>	windowedEncoder(org.apache.spark.sql.Encoder<T> enc)
org.apache.spark.sql.Encoder<BoundedWindow>	windowEncoder()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderProvider

encoderFactory, encoderOf, keyEncoderOf, kvEncoderOf, valueEncoderOf

Methods inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator.TranslationState

putDataset

Method Details

getInput

public InT getInput()

getInputs

public Map<TupleTag<?>,PCollection<?>> getInputs()

getOutputs

public Map<TupleTag<?>,PCollection<?>> getOutputs()

getOutput

public OutT getOutput()

getOutput

public <T> PCollection<T> getOutput(TupleTag<T> tag)

getCurrentTransform

public org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,TransformT> getCurrentTransform()

getDataset

public <T>
org.apache.spark.sql.Dataset<WindowedValue<T>> getDataset(PCollection<T> pCollection)

Specified by:

getDataset in interface PipelineTranslator.TranslationState

getSideInputBroadcast

public <T>
org.apache.spark.broadcast.Broadcast<SideInputValues<T>> getSideInputBroadcast(PCollection<T> pCollection,
 SideInputValues.Loader<T> loader)

Specified by:

getSideInputBroadcast in interface PipelineTranslator.TranslationState

putDataset

public <T> void putDataset(PCollection<T> pCollection,
 org.apache.spark.sql.Dataset<WindowedValue<T>> dataset,
 boolean cache)

Specified by:

putDataset in interface PipelineTranslator.TranslationState

putUnresolved

public <InputT,
T> void putUnresolved(PCollection<T> out,
 PipelineTranslator.UnresolvedTranslation<InputT,T> unresolved)

Specified by:

putUnresolved in interface PipelineTranslator.TranslationState

isLeaf

public boolean isLeaf(PCollection<?> pCollection)

Specified by:

isLeaf in interface PipelineTranslator.TranslationState

getOptionsSupplier

public Supplier<PipelineOptions> getOptionsSupplier()

Specified by:

getOptionsSupplier in interface PipelineTranslator.TranslationState

getOptions

public PipelineOptions getOptions()

Specified by:

getOptions in interface PipelineTranslator.TranslationState

createDataset

public <T>
org.apache.spark.sql.Dataset<WindowedValue<T>> createDataset(List<WindowedValue<T>> data,
 org.apache.spark.sql.Encoder<WindowedValue<T>> enc)

broadcast

public <T> org.apache.spark.broadcast.Broadcast<T> broadcast(T value)

getSparkSession

public org.apache.spark.sql.SparkSession getSparkSession()

Specified by:

getSparkSession in interface PipelineTranslator.TranslationState

encoderOf

public <T> org.apache.spark.sql.Encoder<T> encoderOf(Coder<T> coder,
 EncoderProvider.Factory<T> factory)

Specified by:

encoderOf in interface EncoderProvider

tupleEncoder

public <T1,
T2>
org.apache.spark.sql.Encoder<scala.Tuple2<T1,T2>> tupleEncoder(org.apache.spark.sql.Encoder<T1> e1,
 org.apache.spark.sql.Encoder<T2> e2)

windowedEncoder

public <T>
org.apache.spark.sql.Encoder<WindowedValue<T>> windowedEncoder(Coder<T> coder)

windowedEncoder

public <T>
org.apache.spark.sql.Encoder<WindowedValue<T>> windowedEncoder(org.apache.spark.sql.Encoder<T> enc)

windowedEncoder

public <T,
W extends BoundedWindow>
org.apache.spark.sql.Encoder<WindowedValue<T>> windowedEncoder(Coder<T> coder,
 Coder<W> windowCoder)

windowEncoder

public org.apache.spark.sql.Encoder<BoundedWindow> windowEncoder()