Package org.apache.beam.sdk.io.iceberg

Class AddFilesSchemaTransformProvider.Configuration

java.lang.Object
org.apache.beam.sdk.io.iceberg.AddFilesSchemaTransformProvider.Configuration
Enclosing class:
AddFilesSchemaTransformProvider
@DefaultSchema(AutoValueSchema.class) public abstract static class AddFilesSchemaTransformProvider.Configuration extends Object

  • Constructor Details

    • Configuration

      public Configuration()

  • Method Details

    • builder

      public static AddFilesSchemaTransformProvider.Configuration.Builder builder()

    • getTable

      @SchemaFieldDescription("A fully-qualified table identifier.") public abstract String getTable()

    • getCatalogProperties

      @SchemaFieldDescription("Properties used to set up the Iceberg catalog.") public abstract @Nullable Map<String,String> getCatalogProperties()

    • getConfigProperties

      @SchemaFieldDescription("Properties passed to the Hadoop ") public abstract @Nullable Map<String,String> getConfigProperties()

    • getTriggeringFrequencySeconds

      @SchemaFieldDescription("For a streaming pipeline, sets the frequency at which incoming files are appended (default 600, or 10min).") public abstract @Nullable Integer getTriggeringFrequencySeconds()

    • getManifestFileSize

      @SchemaFieldDescription("The number of data files per manifest (default 10,000 files).") public abstract @Nullable Integer getManifestFileSize()

    • getLocationPrefix

      @SchemaFieldDescription("The prefix shared among all partitions. For example, a data file may have the following location:%n\'gs://bucket/namespace/table/data/id=13/name=beam/data_file.parquet\'%n%nThe provided prefix should go up until the partition information:%n\'gs://bucket/namespace/table/data/\'.%nIf not provided, will try determining each DataFile\'s partition from its metrics metadata.") public abstract @Nullable String getLocationPrefix()

    • getPartitionFields

      @SchemaFieldDescription("Fields used to create a partition spec that is applied when tables are created. For a field \'foo\', the available partition transforms are:\n\n- `foo`\n- `truncate(foo, N)`\n- `bucket(foo, N)`\n- `hour(foo)`\n- `day(foo)`\n- `month(foo)`\n- `year(foo)`\n- `void(foo)`\n\nFor more information on partition transforms, please visit https://iceberg.apache.org/spec/#partition-transforms.") public abstract @Nullable List<String> getPartitionFields()

    • getTableProperties

      @SchemaFieldDescription("Iceberg table properties to be set on the table when it is created.\nFor more information on table properties, please visit https://iceberg.apache.org/docs/latest/configuration/#table-properties.") public abstract @Nullable Map<String,String> getTableProperties()

    • getErrorHandling

      @SchemaFieldDescription("This option specifies whether and where to output unwritable rows.") public abstract @Nullable ErrorHandling getErrorHandling()

    • getIcebergCatalog

      public IcebergCatalogConfig getIcebergCatalog()