Source code for apache_beam.io.flink.flink_streaming_impulse_source

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
A PTransform that provides an unbounded, streaming source of empty byte arrays.

This can only be used with the flink runner.
"""
# pytype: skip-file

import json
from typing import Any
from typing import Dict

from apache_beam import PTransform
from apache_beam import Windowing
from apache_beam import pvalue
from apache_beam.transforms.window import GlobalWindows


[docs]class FlinkStreamingImpulseSource(PTransform): URN = "flink:transform:streaming_impulse:v1" config = {} # type: Dict[str, Any]
[docs] def expand(self, pbegin): assert isinstance(pbegin, pvalue.PBegin), ( 'Input to transform must be a PBegin but found %s' % pbegin) return pvalue.PCollection(pbegin.pipeline, is_bounded=False)
[docs] def get_windowing(self, unused_inputs): return Windowing(GlobalWindows())
[docs] def infer_output_type(self, unused_input_type): return bytes
[docs] def to_runner_api_parameter(self, context): assert isinstance(self, FlinkStreamingImpulseSource), \ "expected instance of StreamingImpulseSource, but got %s" % self.__class__ return (self.URN, json.dumps(self.config))
[docs] def set_interval_ms(self, interval_ms): """Sets the interval (in milliseconds) between messages in the stream. """ self.config["interval_ms"] = interval_ms return self
[docs] def set_message_count(self, message_count): """If non-zero, the stream will produce only this many total messages. Otherwise produces an unbounded number of messages. """ self.config["message_count"] = message_count return self
[docs] @staticmethod @PTransform.register_urn(URN, None) def from_runner_api_parameter(_ptransform, spec_parameter, _context): if isinstance(spec_parameter, bytes): spec_parameter = spec_parameter.decode('utf-8') config = json.loads(spec_parameter) instance = FlinkStreamingImpulseSource() if "interval_ms" in config: instance.set_interval_ms(config["interval_ms"]) if "message_count" in config: instance.set_message_count(config["message_count"]) return instance