org.apache.beam.runners.flink

Interface FlinkPipelineOptions

All Superinterfaces:

ApplicationNameOptions, FileStagingOptions, HasDisplayData, PipelineOptions, StreamingOptions, VersionDependentFlinkPipelineOptions

public interface FlinkPipelineOptions
extends PipelineOptions, ApplicationNameOptions, StreamingOptions, FileStagingOptions, VersionDependentFlinkPipelineOptions

Options which can be used to configure the Flink Runner.

Avoid using `org.apache.flink.*` members below. This allows including the flink runner without requiring flink on the classpath (e.g. to use with the direct runner).

Nested Class Summary

Nested Classes

Modifier and Type	Interface and Description
static class	FlinkPipelineOptions.MaxBundleSizeFactory Maximum bundle size factory.
static class	FlinkPipelineOptions.MaxBundleTimeFactory Maximum bundle time factory.

Nested classes/interfaces inherited from interface org.apache.beam.sdk.options.PipelineOptions

PipelineOptions.AtomicLongFactory, PipelineOptions.CheckEnabled, PipelineOptions.DirectRunner, PipelineOptions.JobNameFactory, PipelineOptions.UserAgentFactory

Field Summary

Fields

Modifier and Type	Field and Description
static java.lang.String	AUTO
static java.lang.String	EXACTLY_ONCE
static java.lang.String	PIPELINED

Method Summary

Modifier and Type	Method and Description
static FlinkPipelineOptions	defaults()
java.lang.Boolean	getAllowNonRestoredState()
boolean	getAttachedMode()
java.lang.Long	getAutoWatermarkInterval()
java.lang.Long	getCheckpointingInterval()
java.lang.String	getCheckpointingMode()
java.lang.Long	getCheckpointTimeoutMillis()
java.lang.Boolean	getDisableMetrics()
java.lang.Boolean	getEnableStableInputDrain()
java.lang.String	getExecutionModeForBatch()
java.lang.Long	getExecutionRetryDelay()
java.lang.Boolean	getFailOnCheckpointingErrors()
java.lang.Boolean	getFasterCopy()
java.lang.Long	getFileInputSplitMaxSizeMB()
boolean	getFinishBundleBeforeCheckpointing()
java.lang.String	getFlinkConfDir()
java.lang.String	getFlinkMaster() The url of the Flink JobManager on which to execute pipelines.
int	getJobCheckIntervalInSecs()
java.lang.Long	getLatencyTrackingInterval()
java.lang.Long	getMaxBundleSize()
java.lang.Long	getMaxBundleTimeMills()
java.lang.Integer	getMaxParallelism()
java.lang.Long	getMinPauseBetweenCheckpoints()
java.lang.Integer	getNumberOfExecutionRetries()
int	getNumConcurrentCheckpoints()
java.lang.Boolean	getObjectReuse()
java.lang.Boolean	getOperatorChaining()
java.lang.Integer	getParallelism()
java.lang.Boolean	getReIterableGroupByKeyResult()
java.lang.String	getReportCheckpointDuration()
java.lang.Boolean	getRetainExternalizedCheckpointsOnCancellation()
java.lang.String	getSavepointPath()
java.lang.Long	getShutdownSourcesAfterIdleMs()
java.lang.String	getStateBackend()
java.lang.Class<? extends FlinkStateBackendFactory>	getStateBackendFactory() Deprecated. Please use setStateBackend below.
java.lang.String	getStateBackendStoragePath()
java.lang.Boolean	isAutoBalanceWriteFilesShardingEnabled()
java.lang.Boolean	isExternalizedCheckpointsEnabled() Enables or disables externalized checkpoints.
void	setAllowNonRestoredState(java.lang.Boolean allowNonRestoredState)
void	setAttachedMode(boolean attachedMode)
void	setAutoBalanceWriteFilesShardingEnabled(java.lang.Boolean autoBalanceWriteFilesShardingEnabled)
void	setAutoWatermarkInterval(java.lang.Long interval)
void	setCheckpointingInterval(java.lang.Long interval)
void	setCheckpointingMode(java.lang.String mode)
void	setCheckpointTimeoutMillis(java.lang.Long checkpointTimeoutMillis)
void	setDisableMetrics(java.lang.Boolean disableMetrics)
void	setEnableStableInputDrain(java.lang.Boolean enableStableInputDrain)
void	setExecutionModeForBatch(java.lang.String executionMode)
void	setExecutionRetryDelay(java.lang.Long delay)
void	setExternalizedCheckpointsEnabled(java.lang.Boolean externalCheckpoints)
void	setFailOnCheckpointingErrors(java.lang.Boolean failOnCheckpointingErrors)
void	setFasterCopy(java.lang.Boolean fasterCopy)
void	setFileInputSplitMaxSizeMB(java.lang.Long fileInputSplitMaxSizeMB)
void	setFinishBundleBeforeCheckpointing(boolean finishBundleBeforeCheckpointing)
void	setFlinkConfDir(java.lang.String confDir)
void	setFlinkMaster(java.lang.String value)
void	setJobCheckIntervalInSecs(int seconds)
void	setLatencyTrackingInterval(java.lang.Long interval)
void	setMaxBundleSize(java.lang.Long size)
void	setMaxBundleTimeMills(java.lang.Long time)
void	setMaxParallelism(java.lang.Integer value)
void	setMinPauseBetweenCheckpoints(java.lang.Long minPauseInterval)
void	setNumberOfExecutionRetries(java.lang.Integer retries)
void	setNumConcurrentCheckpoints(int maxConcurrentCheckpoints)
void	setObjectReuse(java.lang.Boolean reuse)
void	setOperatorChaining(java.lang.Boolean chaining)
void	setParallelism(java.lang.Integer value)
void	setReIterableGroupByKeyResult(java.lang.Boolean reIterableGroupByKeyResult)
void	setReportCheckpointDuration(java.lang.String metricNamespace)
void	setRetainExternalizedCheckpointsOnCancellation(java.lang.Boolean retainOnCancellation)
void	setSavepointPath(java.lang.String path)
void	setShutdownSourcesAfterIdleMs(java.lang.Long timeoutMs)
void	setStateBackend(java.lang.String stateBackend)
void	setStateBackendFactory(java.lang.Class<? extends FlinkStateBackendFactory> stateBackendFactory) Deprecated. Please use setStateBackend below.
void	setStateBackendStoragePath(java.lang.String path)

Methods inherited from interface org.apache.beam.sdk.options.StreamingOptions

getUpdateCompatibilityVersion, isStreaming, setStreaming, setUpdateCompatibilityVersion

Methods inherited from interface org.apache.beam.sdk.options.ApplicationNameOptions

getAppName, setAppName

Methods inherited from interface org.apache.beam.sdk.options.PipelineOptions

as, getJobName, getOptionsId, getRunner, getStableUniqueNames, getTempLocation, getUserAgent, outputRuntimeOptions, revision, setJobName, setOptionsId, setRunner, setStableUniqueNames, setTempLocation, setUserAgent

Methods inherited from interface org.apache.beam.sdk.transforms.display.HasDisplayData

populateDisplayData

Methods inherited from interface org.apache.beam.sdk.options.FileStagingOptions

getFilesToStage, setFilesToStage

Methods inherited from interface org.apache.beam.runners.flink.VersionDependentFlinkPipelineOptions

getUseDataStreamForBatch, setUseDataStreamForBatch

Field Detail

AUTO

static final java.lang.String AUTO

See Also:

Constant Field Values

PIPELINED

static final java.lang.String PIPELINED

See Also:

Constant Field Values

EXACTLY_ONCE

static final java.lang.String EXACTLY_ONCE

See Also:

Constant Field Values

Method Detail

getFlinkMaster

@Default.String(value="[auto]")
java.lang.String getFlinkMaster()

The url of the Flink JobManager on which to execute pipelines. This can either be the address of a cluster JobManager, in the form "host:port" or one of the special Strings "[local]", "[collection]" or "[auto]". "[local]" will start a local Flink Cluster in the JVM, "[collection]" will execute the pipeline on Java Collections while "[auto]" will let the system decide where to execute the pipeline based on the environment.

setFlinkMaster

void setFlinkMaster(java.lang.String value)

getParallelism

@Default.Integer(value=-1)
java.lang.Integer getParallelism()

setParallelism

void setParallelism(java.lang.Integer value)

getMaxParallelism

@Default.Integer(value=-1)
java.lang.Integer getMaxParallelism()

setMaxParallelism

void setMaxParallelism(java.lang.Integer value)

getCheckpointingInterval

@Default.Long(value=-1L)
java.lang.Long getCheckpointingInterval()

setCheckpointingInterval

void setCheckpointingInterval(java.lang.Long interval)

getCheckpointingMode

@Default.String(value="EXACTLY_ONCE")
java.lang.String getCheckpointingMode()

setCheckpointingMode

void setCheckpointingMode(java.lang.String mode)

getCheckpointTimeoutMillis

@Default.Long(value=-1L)
java.lang.Long getCheckpointTimeoutMillis()

setCheckpointTimeoutMillis

void setCheckpointTimeoutMillis(java.lang.Long checkpointTimeoutMillis)

getMinPauseBetweenCheckpoints

@Default.Long(value=-1L)
java.lang.Long getMinPauseBetweenCheckpoints()

setMinPauseBetweenCheckpoints

void setMinPauseBetweenCheckpoints(java.lang.Long minPauseInterval)

getNumConcurrentCheckpoints

@Default.Integer(value=1)
int getNumConcurrentCheckpoints()

setNumConcurrentCheckpoints

void setNumConcurrentCheckpoints(int maxConcurrentCheckpoints)

getFailOnCheckpointingErrors

@Default.Boolean(value=true)
java.lang.Boolean getFailOnCheckpointingErrors()

setFailOnCheckpointingErrors

void setFailOnCheckpointingErrors(java.lang.Boolean failOnCheckpointingErrors)

getFinishBundleBeforeCheckpointing

@Default.Boolean(value=false)
boolean getFinishBundleBeforeCheckpointing()

setFinishBundleBeforeCheckpointing

void setFinishBundleBeforeCheckpointing(boolean finishBundleBeforeCheckpointing)

getShutdownSourcesAfterIdleMs

@Default.Long(value=-1L)
java.lang.Long getShutdownSourcesAfterIdleMs()

setShutdownSourcesAfterIdleMs

void setShutdownSourcesAfterIdleMs(java.lang.Long timeoutMs)

getNumberOfExecutionRetries

@Default.Integer(value=-1)
java.lang.Integer getNumberOfExecutionRetries()

setNumberOfExecutionRetries

void setNumberOfExecutionRetries(java.lang.Integer retries)

getJobCheckIntervalInSecs

@Default.Integer(value=5)
int getJobCheckIntervalInSecs()

setJobCheckIntervalInSecs

void setJobCheckIntervalInSecs(int seconds)

getAttachedMode

@Default.Boolean(value=true)
boolean getAttachedMode()

setAttachedMode

void setAttachedMode(boolean attachedMode)

getExecutionRetryDelay

@Default.Long(value=-1L)
java.lang.Long getExecutionRetryDelay()

setExecutionRetryDelay

void setExecutionRetryDelay(java.lang.Long delay)

getObjectReuse

@Default.Boolean(value=false)
java.lang.Boolean getObjectReuse()

setObjectReuse

void setObjectReuse(java.lang.Boolean reuse)

getOperatorChaining

@Default.Boolean(value=true)
java.lang.Boolean getOperatorChaining()

setOperatorChaining

void setOperatorChaining(java.lang.Boolean chaining)

getStateBackendFactory

@Deprecated
java.lang.Class<? extends FlinkStateBackendFactory> getStateBackendFactory()

Deprecated. Please use setStateBackend below.

State backend to store Beam's state during computation. Note: Only applicable when executing in streaming mode.

setStateBackendFactory

@Deprecated
void setStateBackendFactory(java.lang.Class<? extends FlinkStateBackendFactory> stateBackendFactory)

Deprecated. Please use setStateBackend below.

setStateBackend

void setStateBackend(java.lang.String stateBackend)

getStateBackend

java.lang.String getStateBackend()

setStateBackendStoragePath

void setStateBackendStoragePath(java.lang.String path)

getStateBackendStoragePath

java.lang.String getStateBackendStoragePath()

getDisableMetrics

@Default.Boolean(value=false)
java.lang.Boolean getDisableMetrics()

setDisableMetrics

void setDisableMetrics(java.lang.Boolean disableMetrics)

isExternalizedCheckpointsEnabled

@Default.Boolean(value=false)
java.lang.Boolean isExternalizedCheckpointsEnabled()

Enables or disables externalized checkpoints.

setExternalizedCheckpointsEnabled

void setExternalizedCheckpointsEnabled(java.lang.Boolean externalCheckpoints)

getRetainExternalizedCheckpointsOnCancellation

@Default.Boolean(value=false)
java.lang.Boolean getRetainExternalizedCheckpointsOnCancellation()

setRetainExternalizedCheckpointsOnCancellation

void setRetainExternalizedCheckpointsOnCancellation(java.lang.Boolean retainOnCancellation)

getMaxBundleSize

@Default.InstanceFactory(value=FlinkPipelineOptions.MaxBundleSizeFactory.class)
java.lang.Long getMaxBundleSize()

setMaxBundleSize

void setMaxBundleSize(java.lang.Long size)

getMaxBundleTimeMills

@Default.InstanceFactory(value=FlinkPipelineOptions.MaxBundleTimeFactory.class)
java.lang.Long getMaxBundleTimeMills()

setMaxBundleTimeMills

void setMaxBundleTimeMills(java.lang.Long time)

getLatencyTrackingInterval

@Default.Long(value=0L)
java.lang.Long getLatencyTrackingInterval()

setLatencyTrackingInterval

void setLatencyTrackingInterval(java.lang.Long interval)

getAutoWatermarkInterval

java.lang.Long getAutoWatermarkInterval()

setAutoWatermarkInterval

void setAutoWatermarkInterval(java.lang.Long interval)

getExecutionModeForBatch

@Default.String(value="PIPELINED")
java.lang.String getExecutionModeForBatch()

setExecutionModeForBatch

void setExecutionModeForBatch(java.lang.String executionMode)

getSavepointPath

java.lang.String getSavepointPath()

setSavepointPath

void setSavepointPath(java.lang.String path)

getAllowNonRestoredState

@Default.Boolean(value=false)
java.lang.Boolean getAllowNonRestoredState()

setAllowNonRestoredState

void setAllowNonRestoredState(java.lang.Boolean allowNonRestoredState)

isAutoBalanceWriteFilesShardingEnabled

@Default.Boolean(value=false)
java.lang.Boolean isAutoBalanceWriteFilesShardingEnabled()

setAutoBalanceWriteFilesShardingEnabled

void setAutoBalanceWriteFilesShardingEnabled(java.lang.Boolean autoBalanceWriteFilesShardingEnabled)

getReportCheckpointDuration

java.lang.String getReportCheckpointDuration()

setReportCheckpointDuration

void setReportCheckpointDuration(java.lang.String metricNamespace)

getReIterableGroupByKeyResult

@Default.Boolean(value=false)
java.lang.Boolean getReIterableGroupByKeyResult()

setReIterableGroupByKeyResult

void setReIterableGroupByKeyResult(java.lang.Boolean reIterableGroupByKeyResult)

getFasterCopy

@Default.Boolean(value=false)
java.lang.Boolean getFasterCopy()

setFasterCopy

void setFasterCopy(java.lang.Boolean fasterCopy)

getFlinkConfDir

java.lang.String getFlinkConfDir()

setFlinkConfDir

void setFlinkConfDir(java.lang.String confDir)

getFileInputSplitMaxSizeMB

@Default.Long(value=0L)
java.lang.Long getFileInputSplitMaxSizeMB()

setFileInputSplitMaxSizeMB

void setFileInputSplitMaxSizeMB(java.lang.Long fileInputSplitMaxSizeMB)

getEnableStableInputDrain

@Default.Boolean(value=false)
java.lang.Boolean getEnableStableInputDrain()

setEnableStableInputDrain

void setEnableStableInputDrain(java.lang.Boolean enableStableInputDrain)

defaults

static FlinkPipelineOptions defaults()