Package org.apache.beam.sdk.io.gcp.bigquery

Interface BigQueryOptions

All Superinterfaces:
ApplicationNameOptions, GcpOptions, GoogleApiDebugOptions, HasDisplayData, PipelineOptions, StreamingOptions
All Known Subinterfaces:
DataflowPipelineOptions, DataflowWorkerHarnessOptions, TestBigQueryOptions, TestDataflowPipelineOptions
public interface BigQueryOptions extends ApplicationNameOptions, GcpOptions, PipelineOptions, StreamingOptions
Properties needed when using Google BigQuery with the Apache Beam SDK.

  • Method Details

    • getTempDatasetId

      @String("bigquery.googleapis.com/cloud_dataflow") String getTempDatasetId()

    • setTempDatasetId

      void setTempDatasetId(String value)

    • getHTTPReadTimeout

      @Integer(80000) Integer getHTTPReadTimeout()

    • setHTTPReadTimeout

      void setHTTPReadTimeout(Integer timeout)

    • getHTTPWriteTimeout

      @Integer(900000) Integer getHTTPWriteTimeout()

    • setHTTPWriteTimeout

      void setHTTPWriteTimeout(Integer timeout)

    • getInsertBundleParallelism

      @Integer(3) Integer getInsertBundleParallelism()

    • setInsertBundleParallelism

      void setInsertBundleParallelism(Integer parallelism)

    • getNumStreamingKeys

      @Integer(50) Integer getNumStreamingKeys()

    • setNumStreamingKeys

      void setNumStreamingKeys(Integer value)

    • getMaxStreamingRowsToBatch

      @Long(500L) Long getMaxStreamingRowsToBatch()

    • setMaxStreamingRowsToBatch

      void setMaxStreamingRowsToBatch(Long value)

    • getMaxStreamingBatchSize

      @Long(65536L) Long getMaxStreamingBatchSize()

    • setMaxStreamingBatchSize

      void setMaxStreamingBatchSize(Long value)

    • getBqStreamingApiLoggingFrequencySec

      @Integer(180) Integer getBqStreamingApiLoggingFrequencySec()

    • setBqStreamingApiLoggingFrequencySec

      void setBqStreamingApiLoggingFrequencySec(Integer value)

    • getUseStorageWriteApi

      @Boolean(false) Boolean getUseStorageWriteApi()

    • setUseStorageWriteApi

      void setUseStorageWriteApi(Boolean value)

    • getUseStorageWriteApiAtLeastOnce

      @Boolean(false) Boolean getUseStorageWriteApiAtLeastOnce()

    • setUseStorageWriteApiAtLeastOnce

      void setUseStorageWriteApiAtLeastOnce(Boolean value)

    • getNumStorageWriteApiStreams

      @Integer(0) Integer getNumStorageWriteApiStreams()

    • setNumStorageWriteApiStreams

      void setNumStorageWriteApiStreams(Integer value)

    • getNumStorageWriteApiStreamAppendClients

      @Integer(1) Integer getNumStorageWriteApiStreamAppendClients()

    • setNumStorageWriteApiStreamAppendClients

      void setNumStorageWriteApiStreamAppendClients(Integer value)

    • getMinConnectionPoolConnections

      @Integer(2) Integer getMinConnectionPoolConnections()

    • setMinConnectionPoolConnections

      void setMinConnectionPoolConnections(Integer value)

    • getMaxConnectionPoolConnections

      @Integer(20) Integer getMaxConnectionPoolConnections()

    • setMaxConnectionPoolConnections

      void setMaxConnectionPoolConnections(Integer value)

    • getStorageWriteMaxInflightRequests

      @Long(1000L) Long getStorageWriteMaxInflightRequests()

    • setStorageWriteMaxInflightRequests

      void setStorageWriteMaxInflightRequests(Long value)

    • getStorageWriteMaxInflightBytes

      @Long(104857600L) Long getStorageWriteMaxInflightBytes()

    • setStorageWriteMaxInflightBytes

      void setStorageWriteMaxInflightBytes(Long value)

    • getUseStorageApiConnectionPool

      @Boolean(false) Boolean getUseStorageApiConnectionPool()

    • setUseStorageApiConnectionPool

      void setUseStorageApiConnectionPool(Boolean value)

    • getStorageWriteApiTriggeringFrequencySec

      Integer getStorageWriteApiTriggeringFrequencySec()

    • setStorageWriteApiTriggeringFrequencySec

      void setStorageWriteApiTriggeringFrequencySec(Integer value)

    • getStorageWriteApiMaxRetries

      @Integer(500) Integer getStorageWriteApiMaxRetries()

    • setStorageWriteApiMaxRetries

      void setStorageWriteApiMaxRetries(Integer value)

    • getMaxBufferingDurationMilliSec

      @Integer(0) Integer getMaxBufferingDurationMilliSec()

    • setMaxBufferingDurationMilliSec

      void setMaxBufferingDurationMilliSec(Integer value)

    • getBigQueryProject

      String getBigQueryProject()

    • setBigQueryProject

      void setBigQueryProject(String value)

    • getStorageApiAppendThresholdBytes

      @Integer(2097152) Integer getStorageApiAppendThresholdBytes()

    • setStorageApiAppendThresholdBytes

      void setStorageApiAppendThresholdBytes(Integer value)

    • getStorageApiAppendThresholdRecordCount

      @Integer(150000) Integer getStorageApiAppendThresholdRecordCount()

    • setStorageApiAppendThresholdRecordCount

      void setStorageApiAppendThresholdRecordCount(Integer value)

    • getStorageWriteApiMaxRequestSize

      @Long(10000000L) Long getStorageWriteApiMaxRequestSize()

    • setStorageWriteApiMaxRequestSize

      void setStorageWriteApiMaxRequestSize(Long value)

    • getEnableStorageReadApiV2

      @Boolean(false) Boolean getEnableStorageReadApiV2()

    • setEnableStorageReadApiV2

      void setEnableStorageReadApiV2(Boolean value)

    • getJobLabelsMap

      Map<String,String> getJobLabelsMap()

    • setJobLabelsMap

      void setJobLabelsMap(Map<String,String> value)

    • getBigQueryEndpoint

      @Hidden String getBigQueryEndpoint()
      BQ endpoint to use. If unspecified, uses the default endpoint.

    • setBigQueryEndpoint

      void setBigQueryEndpoint(String value)

    • getGroupFilesFileLoad

      @Hidden @Boolean(false) Boolean getGroupFilesFileLoad()
      Choose to use a GBK when gathering a list of files in batch FILE_LOAD.

      The purpose of this option is to accommodate the runner compatibility, for example, some runners having known issues on large side input, turning on this option avoids trigger side input related issues.

      This is an experimental pipeline option, no backward compatibility guaranteed.

    • setGroupFilesFileLoad

      void setGroupFilesFileLoad(Boolean value)