org.apache.beam.runners.dataflow.options

Interface DataflowStreamingPipelineOptions

All Superinterfaces:

HasDisplayData, PipelineOptions

All Known Subinterfaces:

DataflowPipelineOptions, DataflowWorkerHarnessOptions, TestDataflowPipelineOptions

@Hidden
public interface DataflowStreamingPipelineOptions
extends PipelineOptions

[Internal] Options for configuring StreamingDataflowWorker.

Nested Class Summary

Nested Classes

Modifier and Type	Interface and Description
static class	DataflowStreamingPipelineOptions.GlobalConfigRefreshPeriodFactory Read global get config request period from system property 'windmill.global_config_refresh_period'.
static class	DataflowStreamingPipelineOptions.HarnessUpdateReportingPeriodFactory Read counter reporting period from system property 'windmill.harness_update_reporting_period'.
static class	DataflowStreamingPipelineOptions.LocalWindmillHostportFactory Factory for creating local Windmill address.
static class	DataflowStreamingPipelineOptions.MaxStackTraceDepthToReportFactory Read 'MaxStackTraceToReport' from system property 'windmill.max_stack_trace_to_report' or Integer.MAX_VALUE if unspecified.
static class	DataflowStreamingPipelineOptions.PeriodicStatusPageDirectoryFactory Read 'PeriodicStatusPageOutputDirector' from system property 'windmill.periodic_status_page_directory' or null if unspecified.
static class	DataflowStreamingPipelineOptions.WindmillServiceStreamingRpcBatchLimitFactory Factory for setting value of WindmillServiceStreamingRpcBatchLimit based on environment.

Nested classes/interfaces inherited from interface org.apache.beam.sdk.options.PipelineOptions

PipelineOptions.AtomicLongFactory, PipelineOptions.CheckEnabled, PipelineOptions.DirectRunner, PipelineOptions.JobNameFactory, PipelineOptions.UserAgentFactory

Method Summary

Modifier and Type	Method and Description
int	getActiveWorkRefreshPeriodMillis()
boolean	getChannelzShowOnlyWindmillServiceChannels()
Duration	getGlobalConfigRefreshPeriod()
boolean	getIsWindmillServiceDirectPathEnabled()
java.lang.String	getLocalWindmillHostport()
int	getMaxBundlesFromWindmillOutstanding() Maximum number of bundles outstanding from windmill before the worker stops requesting.
long	getMaxBytesFromWindmillOutstanding() Maximum number of bytes outstanding from windmill before the worker stops requesting.
int	getMaxStackTraceDepthToReport()
java.lang.String	getOverrideWindmillBinary() Custom windmill_main binary to use with the streaming runner.
java.lang.String	getPeriodicStatusPageOutputDirectory()
int	getPerWorkerMetricsUpdateReportingPeriodMillis()
java.lang.Integer	getStreamingSideInputCacheExpirationMillis()
java.lang.Integer	getStreamingSideInputCacheMb()
int	getStuckCommitDurationMillis()
boolean	getUseSeparateWindmillHeartbeatStreams()
boolean	getUseWindmillIsolatedChannels()
int	getWindmillGetDataStreamCount()
Duration	getWindmillHarnessUpdateReportingPeriod()
int	getWindmillMessagesBetweenIsReadyChecks()
java.lang.Integer	getWindmillServiceCommitThreads()
java.lang.String	getWindmillServiceEndpoint() Custom windmill service endpoint.
int	getWindmillServicePort()
int	getWindmillServiceRpcChannelAliveTimeoutSec()
int	getWindmillServiceStreamingLogEveryNStreamFailures()
int	getWindmillServiceStreamingRpcBatchLimit()
int	getWindmillServiceStreamingRpcHealthCheckPeriodMs()
int	getWindmillServiceStreamMaxBackoffMillis()
void	setActiveWorkRefreshPeriodMillis(int value)
void	setChannelzShowOnlyWindmillServiceChannels(boolean value)
void	setGlobalConfigRefreshPeriod(Duration value)
void	setIsWindmillServiceDirectPathEnabled(boolean isWindmillServiceDirectPathEnabled)
void	setLocalWindmillHostport(java.lang.String value)
void	setMaxBundlesFromWindmillOutstanding(int value)
void	setMaxBytesFromWindmillOutstanding(long value)
void	setMaxStackTraceDepthToReport(int value)
void	setOverrideWindmillBinary(java.lang.String value)
void	setPeriodicStatusPageOutputDirectory(java.lang.String directory)
void	setPerWorkerMetricsUpdateReportingPeriodMillis(int value)
void	setStreamingSideInputCacheExpirationMillis(java.lang.Integer value)
void	setStreamingSideInputCacheMb(java.lang.Integer value)
void	setStuckCommitDurationMillis(int value)
void	setUseSeparateWindmillHeartbeatStreams(boolean value)
void	setUseWindmillIsolatedChannels(boolean value)
void	setWindmillGetDataStreamCount(int value)
void	setWindmillHarnessUpdateReportingPeriod(Duration value)
void	setWindmillMessagesBetweenIsReadyChecks(int value)
void	setWindmillServiceCommitThreads(java.lang.Integer value)
void	setWindmillServiceEndpoint(java.lang.String value)
void	setWindmillServicePort(int value)
void	setWindmillServiceRpcChannelAliveTimeoutSec(int value)
void	setWindmillServiceStreamingLogEveryNStreamFailures(int value)
void	setWindmillServiceStreamingRpcBatchLimit(int value)
void	setWindmillServiceStreamingRpcHealthCheckPeriodMs(int value)
void	setWindmillServiceStreamMaxBackoffMillis(int value)

Methods inherited from interface org.apache.beam.sdk.options.PipelineOptions

as, getJobName, getOptionsId, getRunner, getStableUniqueNames, getTempLocation, getUserAgent, outputRuntimeOptions, revision, setJobName, setOptionsId, setRunner, setStableUniqueNames, setTempLocation, setUserAgent

Methods inherited from interface org.apache.beam.sdk.transforms.display.HasDisplayData

populateDisplayData

Method Detail

getOverrideWindmillBinary

java.lang.String getOverrideWindmillBinary()

Custom windmill_main binary to use with the streaming runner.

setOverrideWindmillBinary

void setOverrideWindmillBinary(java.lang.String value)

getWindmillServiceEndpoint

java.lang.String getWindmillServiceEndpoint()

Custom windmill service endpoint.

setWindmillServiceEndpoint

void setWindmillServiceEndpoint(java.lang.String value)

getWindmillServicePort

@Default.Integer(value=443)
int getWindmillServicePort()

setWindmillServicePort

void setWindmillServicePort(int value)

getLocalWindmillHostport

@Default.InstanceFactory(value=DataflowStreamingPipelineOptions.LocalWindmillHostportFactory.class)
java.lang.String getLocalWindmillHostport()

setLocalWindmillHostport

void setLocalWindmillHostport(java.lang.String value)

getMaxBundlesFromWindmillOutstanding

@Default.Integer(value=0)
int getMaxBundlesFromWindmillOutstanding()

Maximum number of bundles outstanding from windmill before the worker stops requesting.

If <= 0, use the default value of 100 + getNumberOfWorkerHarnessThreads()

setMaxBundlesFromWindmillOutstanding

void setMaxBundlesFromWindmillOutstanding(int value)

getMaxBytesFromWindmillOutstanding

@Default.Long(value=0L)
long getMaxBytesFromWindmillOutstanding()

Maximum number of bytes outstanding from windmill before the worker stops requesting.

If <= 0, use the default value of 50% of jvm memory.

setMaxBytesFromWindmillOutstanding

void setMaxBytesFromWindmillOutstanding(long value)

getStreamingSideInputCacheMb

@Default.Integer(value=100)
java.lang.Integer getStreamingSideInputCacheMb()

setStreamingSideInputCacheMb

void setStreamingSideInputCacheMb(java.lang.Integer value)

getStreamingSideInputCacheExpirationMillis

@Default.Integer(value=60000)
java.lang.Integer getStreamingSideInputCacheExpirationMillis()

setStreamingSideInputCacheExpirationMillis

void setStreamingSideInputCacheExpirationMillis(java.lang.Integer value)

getWindmillServiceCommitThreads

@Default.Integer(value=1)
java.lang.Integer getWindmillServiceCommitThreads()

setWindmillServiceCommitThreads

void setWindmillServiceCommitThreads(java.lang.Integer value)

getActiveWorkRefreshPeriodMillis

@Default.Integer(value=10000)
int getActiveWorkRefreshPeriodMillis()

setActiveWorkRefreshPeriodMillis

void setActiveWorkRefreshPeriodMillis(int value)

getWindmillServiceStreamingRpcHealthCheckPeriodMs

@Default.Integer(value=10000)
int getWindmillServiceStreamingRpcHealthCheckPeriodMs()

setWindmillServiceStreamingRpcHealthCheckPeriodMs

void setWindmillServiceStreamingRpcHealthCheckPeriodMs(int value)

getWindmillMessagesBetweenIsReadyChecks

@Default.Integer(value=10)
int getWindmillMessagesBetweenIsReadyChecks()

setWindmillMessagesBetweenIsReadyChecks

void setWindmillMessagesBetweenIsReadyChecks(int value)

getUseWindmillIsolatedChannels

@Default.Boolean(value=false)
boolean getUseWindmillIsolatedChannels()

setUseWindmillIsolatedChannels

void setUseWindmillIsolatedChannels(boolean value)

getUseSeparateWindmillHeartbeatStreams

@Default.Boolean(value=false)
boolean getUseSeparateWindmillHeartbeatStreams()

setUseSeparateWindmillHeartbeatStreams

void setUseSeparateWindmillHeartbeatStreams(boolean value)

getWindmillGetDataStreamCount

@Default.Integer(value=1)
int getWindmillGetDataStreamCount()

setWindmillGetDataStreamCount

void setWindmillGetDataStreamCount(int value)

getChannelzShowOnlyWindmillServiceChannels

@Default.Boolean(value=true)
boolean getChannelzShowOnlyWindmillServiceChannels()

setChannelzShowOnlyWindmillServiceChannels

void setChannelzShowOnlyWindmillServiceChannels(boolean value)

getWindmillHarnessUpdateReportingPeriod

@Default.InstanceFactory(value=DataflowStreamingPipelineOptions.HarnessUpdateReportingPeriodFactory.class)
Duration getWindmillHarnessUpdateReportingPeriod()

setWindmillHarnessUpdateReportingPeriod

void setWindmillHarnessUpdateReportingPeriod(Duration value)

getPerWorkerMetricsUpdateReportingPeriodMillis

@Default.Integer(value=30000)
int getPerWorkerMetricsUpdateReportingPeriodMillis()

setPerWorkerMetricsUpdateReportingPeriodMillis

void setPerWorkerMetricsUpdateReportingPeriodMillis(int value)

getMaxStackTraceDepthToReport

@Default.InstanceFactory(value=DataflowStreamingPipelineOptions.MaxStackTraceDepthToReportFactory.class)
int getMaxStackTraceDepthToReport()

setMaxStackTraceDepthToReport

void setMaxStackTraceDepthToReport(int value)

getStuckCommitDurationMillis

@Default.Integer(value=600000)
int getStuckCommitDurationMillis()

setStuckCommitDurationMillis

void setStuckCommitDurationMillis(int value)

getGlobalConfigRefreshPeriod

@Default.InstanceFactory(value=DataflowStreamingPipelineOptions.GlobalConfigRefreshPeriodFactory.class)
Duration getGlobalConfigRefreshPeriod()

setGlobalConfigRefreshPeriod

void setGlobalConfigRefreshPeriod(Duration value)

getPeriodicStatusPageOutputDirectory

@Default.InstanceFactory(value=DataflowStreamingPipelineOptions.PeriodicStatusPageDirectoryFactory.class)
java.lang.String getPeriodicStatusPageOutputDirectory()

setPeriodicStatusPageOutputDirectory

void setPeriodicStatusPageOutputDirectory(java.lang.String directory)

getWindmillServiceStreamingRpcBatchLimit

@Default.InstanceFactory(value=DataflowStreamingPipelineOptions.WindmillServiceStreamingRpcBatchLimitFactory.class)
int getWindmillServiceStreamingRpcBatchLimit()

setWindmillServiceStreamingRpcBatchLimit

void setWindmillServiceStreamingRpcBatchLimit(int value)

getWindmillServiceStreamingLogEveryNStreamFailures

@Default.Integer(value=20)
int getWindmillServiceStreamingLogEveryNStreamFailures()

setWindmillServiceStreamingLogEveryNStreamFailures

void setWindmillServiceStreamingLogEveryNStreamFailures(int value)

getWindmillServiceRpcChannelAliveTimeoutSec

@Default.Integer(value=40)
int getWindmillServiceRpcChannelAliveTimeoutSec()

setWindmillServiceRpcChannelAliveTimeoutSec

void setWindmillServiceRpcChannelAliveTimeoutSec(int value)

getWindmillServiceStreamMaxBackoffMillis

@Default.Integer(value=30000)
int getWindmillServiceStreamMaxBackoffMillis()

setWindmillServiceStreamMaxBackoffMillis

void setWindmillServiceStreamMaxBackoffMillis(int value)

getIsWindmillServiceDirectPathEnabled

@Default.Boolean(value=false)
boolean getIsWindmillServiceDirectPathEnabled()

setIsWindmillServiceDirectPathEnabled

void setIsWindmillServiceDirectPathEnabled(boolean isWindmillServiceDirectPathEnabled)