Source code for apache_beam.io.aws.clients.client_options

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Builds boto3 client arguments from :class:`AwsOptions`.

Shared across AWS service clients (e.g. the S3 filesystem client) so that every
boto3 client created by Beam uses the same credentials and connection settings.
"""

# pytype: skip-file

import functools
import types
from collections.abc import Mapping
from typing import Any
from typing import Optional
from typing import Union

from apache_beam.options.pipeline_options import AwsOptions
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.utils.secret import _SECRET_CLASSES
from apache_beam.utils.secret import Secret

# Secret manager names that resolve to AwsSecret. AWS credentials cannot be
# stored there, because the credentials are needed to access it.
_AWS_SECRET_MANAGERS = frozenset(
    name for name, secret_cls in _SECRET_CLASSES.items()
    if secret_cls == 'AwsSecret')

# Client settings (see AwsOptions.get_client_settings) that hold credentials.
CREDENTIAL_SETTINGS = frozenset(
    {'access_key_id', 'secret_access_key', 'session_token'})

_TRUE_STRINGS = frozenset({'true', '1', 'yes', 'y', 'on'})
_FALSE_STRINGS = frozenset({'false', '0', 'no', 'n', 'off', ''})


[docs] @functools.lru_cache(maxsize=None) def get_connection_setting_defaults() -> Mapping[str, Any]: """Returns the non-credential client settings and their default values. The names (e.g. ``region_name``, ``endpoint_url``) come from AwsOptions.get_client_settings, so new settings added to AwsOptions are picked up automatically. The result is computed once and is read-only. """ return types.MappingProxyType({ name: value for name, value in AwsOptions(flags=[]).get_client_settings().items() if name not in CREDENTIAL_SETTINGS })
[docs] def parse_connection_settings(settings: Mapping[str, Any]) -> dict[str, Any]: """Validates connection settings, e.g. from a secret spec. Values for boolean settings (e.g. ``disable_ssl``) may be strings such as 'true' or 'false'. Args: settings: A dict of client setting names (without the ``aws_`` prefix) to values. Returns: A dict of the settings with values converted to the setting types. Raises: ValueError: If a setting is a credential or not a known setting, or a boolean value is invalid. """ credentials = sorted(CREDENTIAL_SETTINGS.intersection(settings)) if credentials: raise ValueError( f"Invalid secret parameter {', '.join(credentials)}: AWS credentials " "cannot be set here. Use the " f"{', '.join('--aws_' + name for name in credentials)} pipeline " "option instead.") defaults = get_connection_setting_defaults() unknown = sorted(set(settings) - set(defaults)) if unknown: raise ValueError(f"Invalid secret parameter {', '.join(unknown)}") parsed = {} for name, value in settings.items(): if isinstance(defaults[name], bool) and isinstance(value, str): lowered = value.strip().lower() if lowered in _TRUE_STRINGS: value = True elif lowered in _FALSE_STRINGS: value = False else: raise ValueError(f'Invalid boolean value for {name}: {value}') parsed[name] = value return parsed
[docs] def view_options( options: Union[PipelineOptions, dict], options_cls: type[PipelineOptions]) -> PipelineOptions: """Returns an ``options_cls`` view of PipelineOptions or a dict of values.""" if isinstance(options, PipelineOptions): return options.view_as(options_cls) return options_cls(flags=[], **options)
# S3IO (and therefore the S3 client) is created for almost every filesystem # operation, so cache parsed secrets per process to avoid calling the secret # manager each time. Note that a rotated secret is only picked up by new # worker processes. @functools.lru_cache(maxsize=32) def _fetch_secret(spec: str, secret_manager: Optional[str]) -> str: return Secret.from_json(spec, secret_manager).get_str() def _parse_credential(value: Optional[str], secret_manager: Optional[str]) -> Optional[str]: """Parses a credential option with Secret.from_json. With no secret_manager, Secret.from_json returns the value itself (as a RawSecret), so plain credentials keep working. """ if not value: return value if (secret_manager and secret_manager.strip().lower() in _AWS_SECRET_MANAGERS): raise ValueError( '--aws_credentials_secret_manager cannot be AWS Secrets Manager, ' 'because the AWS credentials are needed to access it. Store the ' 'credentials in another secret manager, e.g. GoogleCloudSecretManager.') return _fetch_secret(value, secret_manager)
[docs] def get_client_kwargs( options: Union[PipelineOptions, dict], service_options_cls: Optional[type[PipelineOptions]] = None, overrides: Optional[dict[str, Any]] = None) -> dict[str, Any]: """Returns common boto3 client keyword arguments. Settings come from the --aws_* defaults in AwsOptions, with the overrides from ``service_options_cls`` applied (e.g. --s3_* in S3Options; see AwsOptions.get_client_settings), and then ``overrides``. Credentials are then resolved through --aws_credentials_secret_manager if it is set. Service-specific settings that are not shared (e.g. --s3_api_version) are not included. Args: options: PipelineOptions, or a dict of option names to values. service_options_cls: Optional options class whose overrides apply, e.g. S3Options. If None, only the --aws_* defaults are used. overrides: Optional dict of connection settings (setting names without the ``aws_`` prefix, e.g. ``region_name``) that take precedence over the options. Values that are None are ignored. Credentials cannot be overridden; see parse_connection_settings. """ aws_options = view_options(options, AwsOptions) service_options = ( view_options(options, service_options_cls) if service_options_cls else None) settings = aws_options.get_client_settings(service_options) if overrides: settings.update( (name, value) for name, value in parse_connection_settings(overrides).items() if value is not None) secret_manager = aws_options.aws_credentials_secret_manager return { 'aws_access_key_id': _parse_credential( settings['access_key_id'], secret_manager), 'aws_secret_access_key': _parse_credential( settings['secret_access_key'], secret_manager), 'aws_session_token': _parse_credential( settings['session_token'], secret_manager), 'region_name': settings['region_name'], 'endpoint_url': settings['endpoint_url'], 'use_ssl': not settings['disable_ssl'], 'verify': settings['verify'], }