org.apache.beam.runners.spark

Interface SparkPipelineOptions

All Superinterfaces:

ApplicationNameOptions, HasDisplayData, PipelineOptions, StreamingOptions

All Known Subinterfaces:

SparkContextOptions, TestSparkPipelineOptions

public interface SparkPipelineOptions
extends PipelineOptions, StreamingOptions, ApplicationNameOptions

Spark runner PipelineOptions handles Spark execution-related configurations, such as the master address, batch-interval, and other user-related knobs.

Nested Class Summary

Nested Classes

Modifier and Type	Interface and Description
static class	SparkPipelineOptions.TmpCheckpointDirFactory Returns the default checkpoint directory of /tmp/${job.name}.

Nested classes/interfaces inherited from interface org.apache.beam.sdk.options.PipelineOptions

PipelineOptions.AtomicLongFactory, PipelineOptions.CheckEnabled, PipelineOptions.DirectRunner, PipelineOptions.JobNameFactory, PipelineOptions.NoOpMetricsSink, PipelineOptions.UserAgentFactory

Method Summary

Modifier and Type	Method and Description
java.lang.Long	getBatchIntervalMillis()
java.lang.Long	getBundleSize()
java.lang.String	getCheckpointDir()
java.lang.Long	getCheckpointDurationMillis()
java.lang.Boolean	getEnableSparkMetricSinks()
java.util.List<java.lang.String>	getFilesToStage() List of local files to make available to workers.
java.lang.Long	getMaxRecordsPerBatch()
java.lang.Long	getMinReadTimeMillis()
java.lang.Double	getReadTimePercentage()
java.lang.String	getSparkMaster()
java.lang.String	getStorageLevel()
boolean	getUsesProvidedSparkContext()
void	setBatchIntervalMillis(java.lang.Long batchInterval)
void	setBundleSize(java.lang.Long value)
void	setCheckpointDir(java.lang.String checkpointDir)
void	setCheckpointDurationMillis(java.lang.Long durationMillis)
void	setEnableSparkMetricSinks(java.lang.Boolean enableSparkMetricSinks)
void	setFilesToStage(java.util.List<java.lang.String> value)
void	setMaxRecordsPerBatch(java.lang.Long maxRecordsPerBatch)
void	setMinReadTimeMillis(java.lang.Long minReadTimeMillis)
void	setReadTimePercentage(java.lang.Double readTimePercentage)
void	setSparkMaster(java.lang.String master)
void	setStorageLevel(java.lang.String storageLevel)
void	setUsesProvidedSparkContext(boolean value)

Methods inherited from interface org.apache.beam.sdk.options.StreamingOptions

isStreaming, setStreaming

Methods inherited from interface org.apache.beam.sdk.options.ApplicationNameOptions

getAppName, setAppName

Methods inherited from interface org.apache.beam.sdk.options.PipelineOptions

as, getJobName, getMetricsHttpSinkUrl, getMetricsPushPeriod, getMetricsSink, getOptionsId, getRunner, getStableUniqueNames, getTempLocation, getUserAgent, outputRuntimeOptions, setJobName, setMetricsHttpSinkUrl, setMetricsPushPeriod, setMetricsSink, setOptionsId, setRunner, setStableUniqueNames, setTempLocation, setUserAgent

Methods inherited from interface org.apache.beam.sdk.transforms.display.HasDisplayData

populateDisplayData

Method Detail

getSparkMaster

@Default.String(value="local[4]")
java.lang.String getSparkMaster()

setSparkMaster

void setSparkMaster(java.lang.String master)

getBatchIntervalMillis

@Default.Long(value=500L)
java.lang.Long getBatchIntervalMillis()

setBatchIntervalMillis

void setBatchIntervalMillis(java.lang.Long batchInterval)

getStorageLevel

@Default.String(value="MEMORY_ONLY")
java.lang.String getStorageLevel()

setStorageLevel

void setStorageLevel(java.lang.String storageLevel)

getMinReadTimeMillis

@Default.Long(value=200L)
java.lang.Long getMinReadTimeMillis()

setMinReadTimeMillis

void setMinReadTimeMillis(java.lang.Long minReadTimeMillis)

getMaxRecordsPerBatch

@Default.Long(value=-1L)
java.lang.Long getMaxRecordsPerBatch()

setMaxRecordsPerBatch

void setMaxRecordsPerBatch(java.lang.Long maxRecordsPerBatch)

getReadTimePercentage

@Default.Double(value=0.1)
java.lang.Double getReadTimePercentage()

setReadTimePercentage

void setReadTimePercentage(java.lang.Double readTimePercentage)

getCheckpointDir

@Default.InstanceFactory(value=SparkPipelineOptions.TmpCheckpointDirFactory.class)
java.lang.String getCheckpointDir()

setCheckpointDir

void setCheckpointDir(java.lang.String checkpointDir)

getCheckpointDurationMillis

@Default.Long(value=-1L)
java.lang.Long getCheckpointDurationMillis()

setCheckpointDurationMillis

void setCheckpointDurationMillis(java.lang.Long durationMillis)

getBundleSize

@Default.Long(value=0L)
java.lang.Long getBundleSize()

setBundleSize

@Experimental
void setBundleSize(java.lang.Long value)

getEnableSparkMetricSinks

@Default.Boolean(value=true)
java.lang.Boolean getEnableSparkMetricSinks()

setEnableSparkMetricSinks

void setEnableSparkMetricSinks(java.lang.Boolean enableSparkMetricSinks)

getUsesProvidedSparkContext

@Default.Boolean(value=false)
boolean getUsesProvidedSparkContext()

setUsesProvidedSparkContext

void setUsesProvidedSparkContext(boolean value)

getFilesToStage

java.util.List<java.lang.String> getFilesToStage()

List of local files to make available to workers.

Jars are placed on the worker's classpath.

The default value is the list of jars from the main program's classpath.

setFilesToStage

void setFilesToStage(java.util.List<java.lang.String> value)