public class TranslationContext
extends java.lang.Object
PTransform
translation: keeping track of the
datasets, the SparkSession
, the current transform being translated.Constructor and Description |
---|
TranslationContext(SparkStructuredStreamingPipelineOptions options) |
Modifier and Type | Method and Description |
---|---|
<T> org.apache.spark.sql.Encoder<T> |
encoderOf(Coder<T> coder,
java.util.function.Function<Coder<T>,org.apache.spark.sql.Encoder<T>> loadFn) |
<T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> |
getDataset(PCollection<T> pCollection) |
org.apache.beam.runners.core.construction.SerializablePipelineOptions |
getSerializableOptions() |
<T> org.apache.spark.sql.Dataset<T> |
getSideInputDataSet(PCollectionView<?> value) |
org.apache.spark.sql.SparkSession |
getSparkSession() |
static <T> void |
printDatasetContent(org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset) |
<T> void |
putDataset(PCollection<T> pCollection,
org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset) |
<ViewT,ElemT> |
setSideInputDataset(PCollectionView<ViewT> value,
org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<ElemT>> set) |
void |
startPipeline()
Starts the batch pipeline, streaming is not supported.
|
public TranslationContext(SparkStructuredStreamingPipelineOptions options)
public org.apache.spark.sql.SparkSession getSparkSession()
public org.apache.beam.runners.core.construction.SerializablePipelineOptions getSerializableOptions()
public <T> org.apache.spark.sql.Encoder<T> encoderOf(Coder<T> coder, java.util.function.Function<Coder<T>,org.apache.spark.sql.Encoder<T>> loadFn)
public <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> getDataset(PCollection<T> pCollection)
public <T> void putDataset(PCollection<T> pCollection, org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset)
public <ViewT,ElemT> void setSideInputDataset(PCollectionView<ViewT> value, org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<ElemT>> set)
public <T> org.apache.spark.sql.Dataset<T> getSideInputDataSet(PCollectionView<?> value)
public void startPipeline()
SparkStructuredStreamingRunner
public static <T> void printDatasetContent(org.apache.spark.sql.Dataset<org.apache.beam.sdk.util.WindowedValue<T>> dataset)