Package org.apache.beam.runners.spark.structuredstreaming

Interface SparkStructuredStreamingPipelineOptions

All Superinterfaces:

ApplicationNameOptions, FileStagingOptions, HasDisplayData, PipelineOptions, SparkCommonPipelineOptions, StreamingOptions

public interface SparkStructuredStreamingPipelineOptions
extends SparkCommonPipelineOptions

Spark runner PipelineOptions handles Spark execution-related configurations, such as the master address, and other user-related knobs.

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.beam.sdk.options.PipelineOptions

PipelineOptions.AtomicLongFactory, PipelineOptions.CheckEnabled, PipelineOptions.DirectRunner, PipelineOptions.JobNameFactory, PipelineOptions.UserAgentFactory

Nested classes/interfaces inherited from interface org.apache.beam.runners.spark.SparkCommonPipelineOptions

SparkCommonPipelineOptions.StorageLevelFactory, SparkCommonPipelineOptions.TmpCheckpointDirFactory

Field Summary

Fields inherited from interface org.apache.beam.runners.spark.SparkCommonPipelineOptions

DEFAULT_MASTER_URL

Method Summary

Modifier and Type	Method	Description
long	getMaxBatchDurationMillis()
Long	getMaxRecordsPerBatch()
long	getReaderIdleTimeoutMillis()
int	getStreamingStopAfterIdleBatches()
boolean	getTestMode()	Set to true to run the job in test mode.
boolean	getUseActiveSparkSession()
long	getWatermarkDelayMillis()
void	setMaxBatchDurationMillis(long value)
void	setMaxRecordsPerBatch(Long maxRecordsPerBatch)
void	setReaderIdleTimeoutMillis(long value)
void	setStreamingStopAfterIdleBatches(int value)
void	setTestMode(boolean testMode)
void	setUseActiveSparkSession(boolean value)
void	setWatermarkDelayMillis(long value)

Methods inherited from interface org.apache.beam.sdk.options.ApplicationNameOptions

getAppName, setAppName

Methods inherited from interface org.apache.beam.sdk.options.FileStagingOptions

getFilesToStage, setFilesToStage

Methods inherited from interface org.apache.beam.sdk.transforms.display.HasDisplayData

populateDisplayData

Methods inherited from interface org.apache.beam.sdk.options.PipelineOptions

as, getGbek, getJobName, getOptionsId, getRunner, getStableUniqueNames, getTempLocation, getUserAgent, outputRuntimeOptions, revision, setGbek, setJobName, setOptionsId, setRunner, setStableUniqueNames, setTempLocation, setUserAgent

Methods inherited from interface org.apache.beam.runners.spark.SparkCommonPipelineOptions

getCheckpointDir, getEnableSparkMetricSinks, getPreferGroupByKeyToHandleHugeValues, getSparkMaster, getStorageLevel, setCheckpointDir, setEnableSparkMetricSinks, setPreferGroupByKeyToHandleHugeValues, setSparkMaster, setStorageLevel

Methods inherited from interface org.apache.beam.sdk.options.StreamingOptions

getUpdateCompatibilityVersion, isStreaming, setStreaming, setUpdateCompatibilityVersion

Method Details

getTestMode

@Boolean(false)
boolean getTestMode()

Set to true to run the job in test mode.

setTestMode

void setTestMode(boolean testMode)

getUseActiveSparkSession

@Boolean(false)
boolean getUseActiveSparkSession()

setUseActiveSparkSession

void setUseActiveSparkSession(boolean value)

getWatermarkDelayMillis

@Long(0L)
long getWatermarkDelayMillis()

setWatermarkDelayMillis

void setWatermarkDelayMillis(long value)

getMaxRecordsPerBatch

@Long(-1L)
Long getMaxRecordsPerBatch()

setMaxRecordsPerBatch

void setMaxRecordsPerBatch(Long maxRecordsPerBatch)

getMaxBatchDurationMillis

@Long(500L)
long getMaxBatchDurationMillis()

setMaxBatchDurationMillis

void setMaxBatchDurationMillis(long value)

getReaderIdleTimeoutMillis

@Long(600000L)
long getReaderIdleTimeoutMillis()

setReaderIdleTimeoutMillis

void setReaderIdleTimeoutMillis(long value)

getStreamingStopAfterIdleBatches

@Integer(-1)
int getStreamingStopAfterIdleBatches()

setStreamingStopAfterIdleBatches

void setStreamingStopAfterIdleBatches(int value)