org.apache.beam.runners.flink

Class FlinkStreamingAggregationsTranslators

java.lang.Object

org.apache.beam.runners.flink.FlinkStreamingAggregationsTranslators

public class FlinkStreamingAggregationsTranslators
extends java.lang.Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	FlinkStreamingAggregationsTranslators.ConcatenateAsIterable<T>

Constructor Summary

Constructors

Constructor and Description
FlinkStreamingAggregationsTranslators()

Method Summary

Modifier and Type	Method and Description
static <K,InputT,AccumT,OutputT> org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputT>>>	batchCombinePerKey(org.apache.beam.runners.flink.FlinkStreamingTranslationContext context, PTransform<PCollection<KV<K,InputT>>,PCollection<KV<K,OutputT>>> transform, CombineFnBase.GlobalCombineFn<InputT,AccumT,OutputT> combineFn, java.util.Map<java.lang.Integer,PCollectionView<?>> sideInputTagMapping, java.util.List<PCollectionView<?>> sideInputs)
static <K,InputT,AccumT,OutputT> org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputT>>>	batchCombinePerKeyNoSideInputs(org.apache.beam.runners.flink.FlinkStreamingTranslationContext context, PTransform<PCollection<KV<K,InputT>>,PCollection<KV<K,OutputT>>> transform, CombineFnBase.GlobalCombineFn<InputT,AccumT,OutputT> combineFn)
static <K,InputT> org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator<org.apache.beam.sdk.util.WindowedValue<KV<K,java.lang.Iterable<InputT>>>>	batchGroupByKey(org.apache.beam.runners.flink.FlinkStreamingTranslationContext context, PTransform<PCollection<KV<K,InputT>>,PCollection<KV<K,java.lang.Iterable<InputT>>>> transform) Creates a two-steps GBK operation.
static <K,InputT,OutputT> org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputT>>>	buildTwoInputStream(org.apache.flink.streaming.api.datastream.KeyedStream<org.apache.beam.sdk.util.WindowedValue<KV<K,InputT>>,FlinkKey> keyedStream, org.apache.flink.streaming.api.datastream.DataStream<RawUnionValue> sideInputStream, java.lang.String name, <any> operator, org.apache.flink.api.common.typeinfo.TypeInformation<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputT>>> outputTypeInfo)
static <K,InputT,AccumT,OutputT> org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputT>>>	getBatchCombinePerKeyOperator(org.apache.beam.runners.flink.FlinkStreamingTranslationContext context, PCollection<KV<K,InputT>> input, java.util.Map<java.lang.Integer,PCollectionView<?>> sideInputTagMapping, java.util.List<PCollectionView<?>> sideInputs, Coder<org.apache.beam.sdk.util.WindowedValue<KV<K,AccumT>>> windowedAccumCoder, CombineFnBase.GlobalCombineFn<InputT,AccumT,?> combineFn, <any> finalDoFnOperator, org.apache.flink.api.common.typeinfo.TypeInformation<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputT>>> outputTypeInfo)
static <K,InputAccumT,OutputAccumT,InputT,OutputT> <any>	getWindowedAggregateDoFnOperator(org.apache.beam.runners.flink.FlinkStreamingTranslationContext context, PTransform<PCollection<KV<K,InputT>>,PCollection<KV<K,OutputT>>> transform, KvCoder<K,InputAccumT> inputKvCoder, Coder<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputAccumT>>> outputCoder, CombineFnBase.GlobalCombineFn<? super InputAccumT,?,OutputAccumT> combineFn, java.util.Map<java.lang.Integer,PCollectionView<?>> sideInputTagMapping, java.util.List<PCollectionView<?>> sideInputs)
static <K,InputAccumT,OutputAccumT,InputT,OutputT> <any>	getWindowedAggregateDoFnOperator(org.apache.beam.runners.flink.FlinkStreamingTranslationContext context, PTransform<PCollection<KV<K,InputT>>,PCollection<KV<K,OutputT>>> transform, KvCoder<K,InputAccumT> inputKvCoder, Coder<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputAccumT>>> outputCoder, org.apache.beam.runners.core.SystemReduceFn<K,InputAccumT,?,OutputAccumT,BoundedWindow> reduceFn, java.util.Map<java.lang.Integer,PCollectionView<?>> sideInputTagMapping, java.util.List<PCollectionView<?>> sideInputs) Create a DoFnOperator instance that group elements per window and apply a combine function on them.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

FlinkStreamingAggregationsTranslators

public FlinkStreamingAggregationsTranslators()

Method Detail

getWindowedAggregateDoFnOperator

public static <K,InputAccumT,OutputAccumT,InputT,OutputT> <any> getWindowedAggregateDoFnOperator(org.apache.beam.runners.flink.FlinkStreamingTranslationContext context,
                                                                                                 PTransform<PCollection<KV<K,InputT>>,PCollection<KV<K,OutputT>>> transform,
                                                                                                 KvCoder<K,InputAccumT> inputKvCoder,
                                                                                                 Coder<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputAccumT>>> outputCoder,
                                                                                                 org.apache.beam.runners.core.SystemReduceFn<K,InputAccumT,?,OutputAccumT,BoundedWindow> reduceFn,
                                                                                                 java.util.Map<java.lang.Integer,PCollectionView<?>> sideInputTagMapping,
                                                                                                 java.util.List<PCollectionView<?>> sideInputs)

Create a DoFnOperator instance that group elements per window and apply a combine function on them.

getWindowedAggregateDoFnOperator

public static <K,InputAccumT,OutputAccumT,InputT,OutputT> <any> getWindowedAggregateDoFnOperator(org.apache.beam.runners.flink.FlinkStreamingTranslationContext context,
                                                                                                 PTransform<PCollection<KV<K,InputT>>,PCollection<KV<K,OutputT>>> transform,
                                                                                                 KvCoder<K,InputAccumT> inputKvCoder,
                                                                                                 Coder<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputAccumT>>> outputCoder,
                                                                                                 CombineFnBase.GlobalCombineFn<? super InputAccumT,?,OutputAccumT> combineFn,
                                                                                                 java.util.Map<java.lang.Integer,PCollectionView<?>> sideInputTagMapping,
                                                                                                 java.util.List<PCollectionView<?>> sideInputs)

getBatchCombinePerKeyOperator

public static <K,InputT,AccumT,OutputT> org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputT>>> getBatchCombinePerKeyOperator(org.apache.beam.runners.flink.FlinkStreamingTranslationContext context,
                                                                                                                                                                                                  PCollection<KV<K,InputT>> input,
                                                                                                                                                                                                  java.util.Map<java.lang.Integer,PCollectionView<?>> sideInputTagMapping,
                                                                                                                                                                                                  java.util.List<PCollectionView<?>> sideInputs,
                                                                                                                                                                                                  Coder<org.apache.beam.sdk.util.WindowedValue<KV<K,AccumT>>> windowedAccumCoder,
                                                                                                                                                                                                  CombineFnBase.GlobalCombineFn<InputT,AccumT,?> combineFn,
                                                                                                                                                                                                  <any> finalDoFnOperator,
                                                                                                                                                                                                  org.apache.flink.api.common.typeinfo.TypeInformation<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputT>>> outputTypeInfo)

batchGroupByKey

public static <K,InputT> org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator<org.apache.beam.sdk.util.WindowedValue<KV<K,java.lang.Iterable<InputT>>>> batchGroupByKey(org.apache.beam.runners.flink.FlinkStreamingTranslationContext context,
                                                                                                                                                                                        PTransform<PCollection<KV<K,InputT>>,PCollection<KV<K,java.lang.Iterable<InputT>>>> transform)

Creates a two-steps GBK operation. Elements are first aggregated locally to save on serialized size since in batch it's very likely that all the elements will be within the same window and pane. The only difference with batchCombinePerKey is the nature of the SystemReduceFn used. It uses SystemReduceFn.buffering() instead of SystemReduceFn.combining() so that new element can simply be appended without accessing the existing state.

batchCombinePerKey

public static <K,InputT,AccumT,OutputT> org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputT>>> batchCombinePerKey(org.apache.beam.runners.flink.FlinkStreamingTranslationContext context,
                                                                                                                                                                                       PTransform<PCollection<KV<K,InputT>>,PCollection<KV<K,OutputT>>> transform,
                                                                                                                                                                                       CombineFnBase.GlobalCombineFn<InputT,AccumT,OutputT> combineFn,
                                                                                                                                                                                       java.util.Map<java.lang.Integer,PCollectionView<?>> sideInputTagMapping,
                                                                                                                                                                                       java.util.List<PCollectionView<?>> sideInputs)

buildTwoInputStream

public static <K,InputT,OutputT> org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputT>>> buildTwoInputStream(org.apache.flink.streaming.api.datastream.KeyedStream<org.apache.beam.sdk.util.WindowedValue<KV<K,InputT>>,FlinkKey> keyedStream,
                                                                                                                                                                                 org.apache.flink.streaming.api.datastream.DataStream<RawUnionValue> sideInputStream,
                                                                                                                                                                                 java.lang.String name,
                                                                                                                                                                                 <any> operator,
                                                                                                                                                                                 org.apache.flink.api.common.typeinfo.TypeInformation<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputT>>> outputTypeInfo)

batchCombinePerKeyNoSideInputs

public static <K,InputT,AccumT,OutputT> org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator<org.apache.beam.sdk.util.WindowedValue<KV<K,OutputT>>> batchCombinePerKeyNoSideInputs(org.apache.beam.runners.flink.FlinkStreamingTranslationContext context,
                                                                                                                                                                                                   PTransform<PCollection<KV<K,InputT>>,PCollection<KV<K,OutputT>>> transform,
                                                                                                                                                                                                   CombineFnBase.GlobalCombineFn<InputT,AccumT,OutputT> combineFn)