Source code for apache_beam.tools.utils

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Utility functions for all microbenchmarks."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import gc
import os
import time

import numpy


[docs]def check_compiled(module): """Check whether given module has been compiled. Args: module: string, module name """ check_module = __import__(module, globals(), locals(), -1) ext = os.path.splitext(check_module.__file__)[-1] if ext in ('.py', '.pyc'): raise RuntimeError( "Profiling uncompiled code.\n" "To compile beam, run " "'pip install Cython; python setup.py build_ext --inplace'")
[docs]class BenchmarkConfig( collections.namedtuple( "BenchmarkConfig", ["benchmark", "size", "num_runs"])): """ Attributes: benchmark: a callable that takes an int argument - benchmark size, and returns a callable. A returned callable must run the code being benchmarked on an input of specified size. For example, one can implement a benchmark as: class MyBenchmark(object): def __init__(self, size): [do necessary initialization] def __call__(self): [run the code in question] size: int, a size of the input. Aggregated per-element metrics are counted based on the size of the input. num_runs: int, number of times to run each benchmark. """ def __str__(self): return "%s, %s element(s)" % ( getattr(self.benchmark, '__name__', str(self.benchmark)), str(self.size))
[docs]def run_benchmarks(benchmark_suite, verbose=True): """Runs benchmarks, and collects execution times. A simple instrumentation to run a callable several times, collect and print its execution times. Args: benchmark_suite: A list of BenchmarkConfig. verbose: bool, whether to print benchmark results to stdout. Returns: A dictionary of the form string -> list of floats. Keys of the dictionary are benchmark names, values are execution times in seconds for each run. """ def run(benchmark_fn, size): # Contain each run of a benchmark inside a function so that any temporary # objects can be garbage-collected after the run. benchmark_instance_callable = benchmark_fn(size) start = time.time() _ = benchmark_instance_callable() return time.time() - start cost_series = collections.defaultdict(list) for benchmark_config in benchmark_suite: name = str(benchmark_config) num_runs = benchmark_config.num_runs size = benchmark_config.size for run_id in range(num_runs): # Do a proactive GC before each run to minimize side-effects of different # runs. gc.collect() time_cost = run(benchmark_config.benchmark, size) cost_series[name].append(time_cost) if verbose: per_element_cost = time_cost / size print("%s: run %d of %d, per element time cost: %g sec" % ( name, run_id + 1, num_runs, per_element_cost)) if verbose: print("") if verbose: pad_length = max([len(str(bc)) for bc in benchmark_suite]) for benchmark_config in benchmark_suite: name = str(benchmark_config) per_element_median_cost = ( numpy.median(cost_series[name]) / benchmark_config.size) std = numpy.std(cost_series[name]) / benchmark_config.size print("%s: per element median time cost: %g sec, relative std: %.2f%%" % ( name.ljust(pad_length, " "), per_element_median_cost, std * 100 / per_element_median_cost)) return cost_series