apache_beam.transforms.environments module

Environments concepts.

For internal use only. No backwards compatibility guarantees.

class apache_beam.transforms.environments.Environment(capabilities: Iterable[str] = (), artifacts: Iterable[ArtifactInformation] = (), resource_hints: Mapping[str, bytes] | None = None)

Bases: object

Abstract base class for environments.

Represents a type and configuration of environment. Each type of Environment should have a unique urn.

For internal use only. No backwards compatibility guarantees.

artifacts() → Iterable[ArtifactInformation]

to_runner_api_parameter(context: PipelineContext) → Tuple[str, message.Message | bytes | str | None]

capabilities() → Iterable[str]

resource_hints() → Mapping[str, bytes]

classmethod register_urn(urn, parameter_type)

classmethod register_urn(urn, parameter_type)

classmethod register_urn(urn, parameter_type, constructor)

classmethod register_urn(urn, parameter_type, constructor)

classmethod get_env_cls_from_urn(urn: str) → Type[Environment]

to_runner_api(context: PipelineContext) → beam_runner_api_pb2.Environment

classmethod from_runner_api(proto: beam_runner_api_pb2.Environment | None, context: PipelineContext) → Environment | None

classmethod from_options(options: Type[EnvironmentT]) → EnvironmentT

Creates an Environment object from PortableOptions.

Parameters:

options – The PortableOptions object.

class apache_beam.transforms.environments.AnyOfEnvironment(environments)

Bases: Environment

to_runner_api_parameter(context: PipelineContext) → Tuple[str, beam_runner_api_pb2.AnyOfEnvironmentPayload]

static from_runner_api_parameter(payload: beam_runner_api_pb2.AnyOfEnvironmentPayload, capabilities: Iterable[str], artifacts: Iterable[beam_runner_api_pb2.ArtifactInformation], resource_hints: Mapping[str, bytes], context: PipelineContext) → AnyOfEnvironment

static create_proto(environments: Iterable[Environment]) → Environment

class apache_beam.transforms.environments.DefaultEnvironment(capabilities: Iterable[str] = (), artifacts: Iterable[ArtifactInformation] = (), resource_hints: Mapping[str, bytes] | None = None)

Bases: Environment

Used as a stub when context is missing a default environment.

to_runner_api_parameter(context)

static from_runner_api_parameter(payload: beam_runner_api_pb2.DockerPayload, capabilities: Iterable[str], artifacts: Iterable[beam_runner_api_pb2.ArtifactInformation], resource_hints: Mapping[str, bytes], context: PipelineContext) → DefaultEnvironment

class apache_beam.transforms.environments.DockerEnvironment(container_image=None, capabilities=(), artifacts=(), resource_hints=None)

Bases: Environment

to_runner_api_parameter(context: PipelineContext) → Tuple[str, beam_runner_api_pb2.DockerPayload]

static from_runner_api_parameter(payload: beam_runner_api_pb2.DockerPayload, capabilities: Iterable[str], artifacts: Iterable[beam_runner_api_pb2.ArtifactInformation], resource_hints: Mapping[str, bytes], context: PipelineContext) → DockerEnvironment

classmethod from_options(options: PortableOptions) → DockerEnvironment

classmethod from_container_image(container_image: str, artifacts: Iterable[ArtifactInformation] = (), resource_hints: Mapping[str, bytes] | None = None) → DockerEnvironment

static default_docker_image() → str

class apache_beam.transforms.environments.ProcessEnvironment(command: str, os: str = '', arch: str = '', env: Mapping[str, str] | None = None, capabilities: Iterable[str] = (), artifacts: Iterable[ArtifactInformation] = (), resource_hints: Mapping[str, bytes] | None = None)

Bases: Environment

to_runner_api_parameter(context: PipelineContext) → Tuple[str, beam_runner_api_pb2.ProcessPayload]

static from_runner_api_parameter(payload, capabilities: Iterable[str], artifacts: Iterable[beam_runner_api_pb2.ArtifactInformation], resource_hints: Mapping[str, bytes], context: PipelineContext) → ProcessEnvironment

static parse_environment_variables(variables)

classmethod from_options(options: PortableOptions) → ProcessEnvironment

class apache_beam.transforms.environments.ExternalEnvironment(url, params=None, capabilities=(), artifacts=(), resource_hints=None)

Bases: Environment

to_runner_api_parameter(context: PipelineContext) → Tuple[str, beam_runner_api_pb2.ExternalPayload]

static from_runner_api_parameter(payload: beam_runner_api_pb2.ExternalPayload, capabilities: Iterable[str], artifacts: Iterable[beam_runner_api_pb2.ArtifactInformation], resource_hints: Mapping[str, bytes], context: PipelineContext) → ExternalEnvironment

classmethod from_options(options: PortableOptions) → ExternalEnvironment

class apache_beam.transforms.environments.EmbeddedPythonEnvironment(capabilities: Iterable[str] = (), artifacts: Iterable[ArtifactInformation] = (), resource_hints: Mapping[str, bytes] | None = None)

Bases: Environment

to_runner_api_parameter(context: PipelineContext) → Tuple[str, None]

static from_runner_api_parameter(unused_payload: None, capabilities: Iterable[str], artifacts: Iterable[beam_runner_api_pb2.ArtifactInformation], resource_hints: Mapping[str, bytes], context: PipelineContext) → EmbeddedPythonEnvironment

classmethod from_options(options: PortableOptions) → EmbeddedPythonEnvironment

classmethod default() → EmbeddedPythonEnvironment

class apache_beam.transforms.environments.EmbeddedPythonGrpcEnvironment(state_cache_size=None, data_buffer_time_limit_ms=None, capabilities=(), artifacts=(), resource_hints=None)

Bases: Environment

to_runner_api_parameter(context: PipelineContext) → Tuple[str, bytes]

static from_runner_api_parameter(payload: bytes, capabilities: Iterable[str], artifacts: Iterable[beam_runner_api_pb2.ArtifactInformation], resource_hints: Mapping[str, bytes], context: PipelineContext) → EmbeddedPythonGrpcEnvironment

classmethod from_options(options: PortableOptions) → EmbeddedPythonGrpcEnvironment

static parse_config(s: str) → Dict[str, Any]

classmethod default() → EmbeddedPythonGrpcEnvironment

class apache_beam.transforms.environments.SubprocessSDKEnvironment(command_string, capabilities=(), artifacts=(), resource_hints=None)

Bases: Environment

to_runner_api_parameter(context: PipelineContext) → Tuple[str, bytes]

static from_runner_api_parameter(payload: bytes, capabilities: Iterable[str], artifacts: Iterable[beam_runner_api_pb2.ArtifactInformation], resource_hints: Mapping[str, bytes], context: PipelineContext) → SubprocessSDKEnvironment

classmethod from_options(options: PortableOptions) → SubprocessSDKEnvironment

classmethod from_command_string(command_string: str) → SubprocessSDKEnvironment

class apache_beam.transforms.environments.PyPIArtifactRegistry

Bases: object

classmethod register_artifact(name, version)

classmethod get_artifacts()