org.apache.beam.sdk.io.gcp.bigquery

Interface BigQueryOptions

All Superinterfaces:

ApplicationNameOptions, GcpOptions, GoogleApiDebugOptions, HasDisplayData, PipelineOptions, StreamingOptions

All Known Subinterfaces:

DataflowPipelineOptions, DataflowWorkerHarnessOptions, TestBigQueryOptions, TestDataflowPipelineOptions

public interface BigQueryOptions
extends ApplicationNameOptions, GcpOptions, PipelineOptions, StreamingOptions

Properties needed when using Google BigQuery with the Apache Beam SDK.

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.beam.sdk.extensions.gcp.options.GcpOptions

GcpOptions.DefaultProjectFactory, GcpOptions.EnableStreamingEngineFactory, GcpOptions.GcpOAuthScopesFactory, GcpOptions.GcpTempLocationFactory, GcpOptions.GcpUserCredentialsFactory

Nested classes/interfaces inherited from interface org.apache.beam.sdk.extensions.gcp.options.GoogleApiDebugOptions

GoogleApiDebugOptions.GoogleApiTracer

Field Summary

Fields inherited from interface org.apache.beam.sdk.extensions.gcp.options.GcpOptions

STREAMING_ENGINE_EXPERIMENT, WINDMILL_SERVICE_EXPERIMENT

Method Summary

Modifier and Type	Method and Description
java.lang.String	getBigQueryEndpoint() BQ endpoint to use.
java.lang.String	getBigQueryProject()
java.lang.Integer	getBqStreamingApiLoggingFrequencySec()
java.lang.Boolean	getEnableStorageReadApiV2()
java.lang.Boolean	getGroupFilesFileLoad() Choose to use a GBK when gathering a list of files in batch FILE_LOAD.
java.lang.Integer	getHTTPReadTimeout()
java.lang.Integer	getHTTPWriteTimeout()
java.lang.Integer	getInsertBundleParallelism()
java.util.Map<java.lang.String,java.lang.String>	getJobLabelsMap()
java.lang.Integer	getMaxBufferingDurationMilliSec()
java.lang.Integer	getMaxConnectionPoolConnections()
java.lang.Long	getMaxStreamingBatchSize()
java.lang.Long	getMaxStreamingRowsToBatch()
java.lang.Integer	getMinConnectionPoolConnections()
java.lang.Integer	getNumStorageWriteApiStreamAppendClients()
java.lang.Integer	getNumStorageWriteApiStreams()
java.lang.Integer	getNumStreamingKeys()
java.lang.Integer	getStorageApiAppendThresholdBytes()
java.lang.Integer	getStorageApiAppendThresholdRecordCount()
java.lang.Long	getStorageWriteApiMaxRequestSize()
java.lang.Integer	getStorageWriteApiMaxRetries()
java.lang.Integer	getStorageWriteApiTriggeringFrequencySec()
java.lang.Long	getStorageWriteMaxInflightBytes()
java.lang.Long	getStorageWriteMaxInflightRequests()
java.lang.String	getTempDatasetId()
java.lang.Boolean	getUseStorageApiConnectionPool()
java.lang.Boolean	getUseStorageWriteApi()
java.lang.Boolean	getUseStorageWriteApiAtLeastOnce()
void	setBigQueryEndpoint(java.lang.String value)
void	setBigQueryProject(java.lang.String value)
void	setBqStreamingApiLoggingFrequencySec(java.lang.Integer value)
void	setEnableStorageReadApiV2(java.lang.Boolean value)
void	setGroupFilesFileLoad(java.lang.Boolean value)
void	setHTTPReadTimeout(java.lang.Integer timeout)
void	setHTTPWriteTimeout(java.lang.Integer timeout)
void	setInsertBundleParallelism(java.lang.Integer parallelism)
void	setJobLabelsMap(java.util.Map<java.lang.String,java.lang.String> value)
void	setMaxBufferingDurationMilliSec(java.lang.Integer value)
void	setMaxConnectionPoolConnections(java.lang.Integer value)
void	setMaxStreamingBatchSize(java.lang.Long value)
void	setMaxStreamingRowsToBatch(java.lang.Long value)
void	setMinConnectionPoolConnections(java.lang.Integer value)
void	setNumStorageWriteApiStreamAppendClients(java.lang.Integer value)
void	setNumStorageWriteApiStreams(java.lang.Integer value)
void	setNumStreamingKeys(java.lang.Integer value)
void	setStorageApiAppendThresholdBytes(java.lang.Integer value)
void	setStorageApiAppendThresholdRecordCount(java.lang.Integer value)
void	setStorageWriteApiMaxRequestSize(java.lang.Long value)
void	setStorageWriteApiMaxRetries(java.lang.Integer value)
void	setStorageWriteApiTriggeringFrequencySec(java.lang.Integer value)
void	setStorageWriteMaxInflightBytes(java.lang.Long value)
void	setStorageWriteMaxInflightRequests(java.lang.Long value)
void	setTempDatasetId(java.lang.String value)
void	setUseStorageApiConnectionPool(java.lang.Boolean value)
void	setUseStorageWriteApi(java.lang.Boolean value)
void	setUseStorageWriteApiAtLeastOnce(java.lang.Boolean value)

Methods inherited from interface org.apache.beam.sdk.extensions.gcp.options.GcpOptions

getCredentialFactoryClass, getDataflowKmsKey, getGcpCredential, getGcpOauthScopes, getGcpTempLocation, getImpersonateServiceAccount, getProject, getWorkerRegion, getWorkerZone, getZone, isEnableStreamingEngine, setCredentialFactoryClass, setDataflowKmsKey, setEnableStreamingEngine, setGcpCredential, setGcpOauthScopes, setGcpTempLocation, setImpersonateServiceAccount, setProject, setWorkerRegion, setWorkerZone, setZone

Methods inherited from interface org.apache.beam.sdk.extensions.gcp.options.GoogleApiDebugOptions

getGoogleApiTrace, setGoogleApiTrace

Methods inherited from interface org.apache.beam.sdk.options.StreamingOptions

getUpdateCompatibilityVersion, isStreaming, setStreaming, setUpdateCompatibilityVersion, updateCompatibilityVersionLessThan

Methods inherited from interface org.apache.beam.sdk.options.ApplicationNameOptions

getAppName, setAppName

Method Detail

getTempDatasetId

@Default.String(value="bigquery.googleapis.com/cloud_dataflow")
java.lang.String getTempDatasetId()

setTempDatasetId

void setTempDatasetId(java.lang.String value)

getHTTPReadTimeout

@Default.Integer(value=80000)
java.lang.Integer getHTTPReadTimeout()

setHTTPReadTimeout

void setHTTPReadTimeout(java.lang.Integer timeout)

getHTTPWriteTimeout

@Default.Integer(value=900000)
java.lang.Integer getHTTPWriteTimeout()

setHTTPWriteTimeout

void setHTTPWriteTimeout(java.lang.Integer timeout)

getInsertBundleParallelism

@Default.Integer(value=3)
java.lang.Integer getInsertBundleParallelism()

setInsertBundleParallelism

void setInsertBundleParallelism(java.lang.Integer parallelism)

getNumStreamingKeys

@Default.Integer(value=50)
java.lang.Integer getNumStreamingKeys()

setNumStreamingKeys

void setNumStreamingKeys(java.lang.Integer value)

getMaxStreamingRowsToBatch

@Default.Long(value=500L)
java.lang.Long getMaxStreamingRowsToBatch()

setMaxStreamingRowsToBatch

void setMaxStreamingRowsToBatch(java.lang.Long value)

getMaxStreamingBatchSize

@Default.Long(value=65536L)
java.lang.Long getMaxStreamingBatchSize()

setMaxStreamingBatchSize

void setMaxStreamingBatchSize(java.lang.Long value)

getBqStreamingApiLoggingFrequencySec

@Default.Integer(value=180)
java.lang.Integer getBqStreamingApiLoggingFrequencySec()

setBqStreamingApiLoggingFrequencySec

void setBqStreamingApiLoggingFrequencySec(java.lang.Integer value)

getUseStorageWriteApi

@Default.Boolean(value=false)
java.lang.Boolean getUseStorageWriteApi()

setUseStorageWriteApi

void setUseStorageWriteApi(java.lang.Boolean value)

getUseStorageWriteApiAtLeastOnce

@Default.Boolean(value=false)
java.lang.Boolean getUseStorageWriteApiAtLeastOnce()

setUseStorageWriteApiAtLeastOnce

void setUseStorageWriteApiAtLeastOnce(java.lang.Boolean value)

getNumStorageWriteApiStreams

@Default.Integer(value=0)
java.lang.Integer getNumStorageWriteApiStreams()

setNumStorageWriteApiStreams

void setNumStorageWriteApiStreams(java.lang.Integer value)

getNumStorageWriteApiStreamAppendClients

@Default.Integer(value=1)
java.lang.Integer getNumStorageWriteApiStreamAppendClients()

setNumStorageWriteApiStreamAppendClients

void setNumStorageWriteApiStreamAppendClients(java.lang.Integer value)

getMinConnectionPoolConnections

@Default.Integer(value=2)
java.lang.Integer getMinConnectionPoolConnections()

setMinConnectionPoolConnections

void setMinConnectionPoolConnections(java.lang.Integer value)

getMaxConnectionPoolConnections

@Default.Integer(value=20)
java.lang.Integer getMaxConnectionPoolConnections()

setMaxConnectionPoolConnections

void setMaxConnectionPoolConnections(java.lang.Integer value)

getStorageWriteMaxInflightRequests

@Default.Long(value=1000L)
java.lang.Long getStorageWriteMaxInflightRequests()

setStorageWriteMaxInflightRequests

void setStorageWriteMaxInflightRequests(java.lang.Long value)

getStorageWriteMaxInflightBytes

@Default.Long(value=104857600L)
java.lang.Long getStorageWriteMaxInflightBytes()

setStorageWriteMaxInflightBytes

void setStorageWriteMaxInflightBytes(java.lang.Long value)

getUseStorageApiConnectionPool

@Default.Boolean(value=false)
java.lang.Boolean getUseStorageApiConnectionPool()

setUseStorageApiConnectionPool

void setUseStorageApiConnectionPool(java.lang.Boolean value)

getStorageWriteApiTriggeringFrequencySec

java.lang.Integer getStorageWriteApiTriggeringFrequencySec()

setStorageWriteApiTriggeringFrequencySec

void setStorageWriteApiTriggeringFrequencySec(java.lang.Integer value)

getStorageWriteApiMaxRetries

@Default.Integer(value=500)
java.lang.Integer getStorageWriteApiMaxRetries()

setStorageWriteApiMaxRetries

void setStorageWriteApiMaxRetries(java.lang.Integer value)

getMaxBufferingDurationMilliSec

@Default.Integer(value=0)
java.lang.Integer getMaxBufferingDurationMilliSec()

setMaxBufferingDurationMilliSec

void setMaxBufferingDurationMilliSec(java.lang.Integer value)

getBigQueryProject

java.lang.String getBigQueryProject()

setBigQueryProject

void setBigQueryProject(java.lang.String value)

getStorageApiAppendThresholdBytes

@Default.Integer(value=2097152)
java.lang.Integer getStorageApiAppendThresholdBytes()

setStorageApiAppendThresholdBytes

void setStorageApiAppendThresholdBytes(java.lang.Integer value)

getStorageApiAppendThresholdRecordCount

@Default.Integer(value=150000)
java.lang.Integer getStorageApiAppendThresholdRecordCount()

setStorageApiAppendThresholdRecordCount

void setStorageApiAppendThresholdRecordCount(java.lang.Integer value)

getStorageWriteApiMaxRequestSize

@Default.Long(value=10000000L)
java.lang.Long getStorageWriteApiMaxRequestSize()

setStorageWriteApiMaxRequestSize

void setStorageWriteApiMaxRequestSize(java.lang.Long value)

getEnableStorageReadApiV2

@Default.Boolean(value=false)
java.lang.Boolean getEnableStorageReadApiV2()

setEnableStorageReadApiV2

void setEnableStorageReadApiV2(java.lang.Boolean value)

getJobLabelsMap

java.util.Map<java.lang.String,java.lang.String> getJobLabelsMap()

setJobLabelsMap

void setJobLabelsMap(java.util.Map<java.lang.String,java.lang.String> value)

getBigQueryEndpoint

@Hidden
java.lang.String getBigQueryEndpoint()

BQ endpoint to use. If unspecified, uses the default endpoint.

setBigQueryEndpoint

void setBigQueryEndpoint(java.lang.String value)

getGroupFilesFileLoad

@Hidden
 @Default.Boolean(value=false)
java.lang.Boolean getGroupFilesFileLoad()

Choose to use a GBK when gathering a list of files in batch FILE_LOAD.

The purpose of this option is to accommodate the runner compatibility, for example, some runners having known issues on large side input, turning on this option avoids trigger side input related issues.

This is an experimental pipeline option, no backward compatibility guaranteed.

setGroupFilesFileLoad

void setGroupFilesFileLoad(java.lang.Boolean value)