org.apache.beam.runners.spark.structuredstreaming.translation

Class TransformTranslator.Context

java.lang.Object

org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator.Context

Enclosing class:

TransformTranslator<InT extends PInput,OutT extends POutput,TransformT extends PTransform<? extends InT,OutT>>

protected class TransformTranslator.Context
extends java.lang.Object

Constructor Summary

Constructors

Modifier	Constructor and Description
protected	Context(org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,PTransform<InT,OutT>> transform, TranslationContext cxt)

Method Summary

Modifier and Type	Method and Description
<T> org.apache.spark.broadcast.Broadcast<T>	broadcast(T value)
<T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>>	createDataset(java.util.List<org.apache.beam.sdk.util.WindowedValue<T>> data, org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> enc)
<T> org.apache.spark.sql.Encoder<T>	encoderOf(Coder<T> coder)
org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,PTransform<InT,OutT>>	getCurrentTransform()
<T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>>	getDataset(PCollection<T> pCollection)
InT	getInput()
java.util.Map<TupleTag<?>,PCollection<?>>	getInputs()
PipelineOptions	getOptions()
OutT	getOutput()
<T> PCollection<T>	getOutput(TupleTag<T> tag)
java.util.Map<TupleTag<?>,PCollection<?>>	getOutputs()
org.apache.beam.runners.core.construction.SerializablePipelineOptions	getSerializableOptions()
<T> org.apache.spark.sql.Dataset<T>	getSideInputDataset(PCollectionView<?> sideInput)
org.apache.spark.sql.SparkSession	getSparkSession()
<K,V> org.apache.spark.sql.Encoder<K>	keyEncoderOf(KvCoder<K,V> coder)
<K,V> org.apache.spark.sql.Encoder<KV<K,V>>	kvEncoderOf(KvCoder<K,V> coder)
<T> void	putDataset(PCollection<T> pCollection, org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset)
<ViewT,ElemT> void	setSideInputDataset(PCollectionView<ViewT> value, org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<ElemT>> set)
<T1,T2> org.apache.spark.sql.Encoder<scala.Tuple2<T1,T2>>	tupleEncoder(org.apache.spark.sql.Encoder<T1> e1, org.apache.spark.sql.Encoder<T2> e2)
<K,V> org.apache.spark.sql.Encoder<V>	valueEncoderOf(KvCoder<K,V> coder)
<T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>>	windowedEncoder(Coder<T> coder)
<T,W extends BoundedWindow> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>>	windowedEncoder(Coder<T> coder, Coder<W> windowCoder)
<T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>>	windowedEncoder(org.apache.spark.sql.Encoder<T> enc)
org.apache.spark.sql.Encoder<BoundedWindow>	windowEncoder()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

Context

protected Context(org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,PTransform<InT,OutT>> transform,
                  TranslationContext cxt)

Method Detail

getInput

public InT getInput()

getInputs

public java.util.Map<TupleTag<?>,PCollection<?>> getInputs()

getOutput

public OutT getOutput()

getOutput

public <T> PCollection<T> getOutput(TupleTag<T> tag)

getOutputs

public java.util.Map<TupleTag<?>,PCollection<?>> getOutputs()

getCurrentTransform

public org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,PTransform<InT,OutT>> getCurrentTransform()

getDataset

public <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> getDataset(PCollection<T> pCollection)

putDataset

public <T> void putDataset(PCollection<T> pCollection,
                           org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset)

getSerializableOptions

public org.apache.beam.runners.core.construction.SerializablePipelineOptions getSerializableOptions()

getOptions

public PipelineOptions getOptions()

setSideInputDataset

public <ViewT,ElemT> void setSideInputDataset(PCollectionView<ViewT> value,
                                              org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<ElemT>> set)

getSideInputDataset

public <T> org.apache.spark.sql.Dataset<T> getSideInputDataset(PCollectionView<?> sideInput)

createDataset

public <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> createDataset(java.util.List<org.apache.beam.sdk.util.WindowedValue<T>> data,
                                                                                                 org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> enc)

broadcast

public <T> org.apache.spark.broadcast.Broadcast<T> broadcast(T value)

getSparkSession

public org.apache.spark.sql.SparkSession getSparkSession()

encoderOf

public <T> org.apache.spark.sql.Encoder<T> encoderOf(Coder<T> coder)

kvEncoderOf

public <K,V> org.apache.spark.sql.Encoder<KV<K,V>> kvEncoderOf(KvCoder<K,V> coder)

keyEncoderOf

public <K,V> org.apache.spark.sql.Encoder<K> keyEncoderOf(KvCoder<K,V> coder)

valueEncoderOf

public <K,V> org.apache.spark.sql.Encoder<V> valueEncoderOf(KvCoder<K,V> coder)

windowedEncoder

public <T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> windowedEncoder(Coder<T> coder)

windowedEncoder

public <T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> windowedEncoder(org.apache.spark.sql.Encoder<T> enc)

tupleEncoder

public <T1,T2> org.apache.spark.sql.Encoder<scala.Tuple2<T1,T2>> tupleEncoder(org.apache.spark.sql.Encoder<T1> e1,
                                                                              org.apache.spark.sql.Encoder<T2> e2)

windowedEncoder

public <T,W extends BoundedWindow> org.apache.spark.sql.Encoder<org.apache.beam.sdk.util.WindowedValue<T>> windowedEncoder(Coder<T> coder,
                                                                                                                           Coder<W> windowCoder)

windowEncoder

public org.apache.spark.sql.Encoder<BoundedWindow> windowEncoder()