diff options
| -rw-r--r-- | .gitignore | 5 | ||||
| -rw-r--r-- | LICENSE.txt | 29 | ||||
| -rw-r--r-- | README.md | 0 | ||||
| -rw-r--r-- | benchstrace/__init__.py | 3 | ||||
| -rw-r--r-- | benchstrace/benchmark.py | 264 | ||||
| -rw-r--r-- | benchstrace/cli.py | 54 | ||||
| -rw-r--r-- | benchstrace/strace.py | 113 | ||||
| -rw-r--r-- | pyproject.toml | 26 | ||||
| -rw-r--r-- | setup.py | 3 |
9 files changed, 497 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b5be78d --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.idea +__pycache__ +venv +*.egg-info +benchstrace/_version.py diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..2086dd0 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2023, Association of Universities for Research in Astronomy (AURA) +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/README.md diff --git a/benchstrace/__init__.py b/benchstrace/__init__.py new file mode 100644 index 0000000..a85946d --- /dev/null +++ b/benchstrace/__init__.py @@ -0,0 +1,3 @@ +from . import benchmark +from . import strace +from ._version import __version__, __version_tuple__ diff --git a/benchstrace/benchmark.py b/benchstrace/benchmark.py new file mode 100644 index 0000000..8094bd5 --- /dev/null +++ b/benchstrace/benchmark.py @@ -0,0 +1,264 @@ +import os +import sys +from .strace import STrace, STraceRecord +from .strace import parse_record + + +SHOW_COLLECTION = 1 << 1 +SHOW_AVERAGE = 2 << 1 +SHOW_TOTAL = 3 << 1 +SHOW_ALL = SHOW_COLLECTION | SHOW_AVERAGE | SHOW_TOTAL + + +class Benchmark: + # Input file data begins after + MAX_HEADER_LINES = 3 + + def __init__(self, command="", passes=2, setup="", teardown=""): + """ + A benchmark record + :param command: to execute + :param passes: times to execute command + :param setup: to execute before command + :param teardown: to execute after command + """ + self.result = [] + self.command = command + self.passes = passes + self.count = 0 + + if self.passes < 1: + self.passes = 1 + + if self.command: + for i in range(self.passes): + if self.passes > 1: + print(f"Pass [{i+1}/{passes}]: ", end="") + print(f"Running `{self.command}`") + data = STrace(command, setup=setup, teardown=teardown).records + self.count += len(data) + self.result.append(data) + + def save(self, filename, clobber=False): + """ + Write benchmark data to a file + :param filename: output file path + :param clobber: toggle overwriting the output file + """ + if clobber and os.path.exists(filename): + raise FileExistsError(filename) + + filename = os.path.abspath(filename) + print(f"Writing {filename}") + with open(filename, "w+") as fp: + print(f"# {len(self.result)}", file=fp) + print(f"# {self.command}", file=fp) + print("", file=fp) + for t, data in enumerate(self.result): + print(f"# {t}", file=fp) + for rec in data: + rec_fmt = f"{rec.name} {rec.calls} {rec.seconds:.6f} {rec.min:.6f} {rec.max:.6f} {rec.usecs_call}" + print(rec_fmt, file=fp) + + def load(self, filename): + """ + Read benchmark data from a file + :param filename: input file path + """ + result = [] + filename_s = os.path.basename(filename) + + with open(filename, "r") as fp: + collection_max = int(fp.readline().split("#")[1]) or 0 + command = fp.readline().split("#")[1] or "UNKNOWN" + + if not collection_max: + raise ValueError(f"{filename_s} has no collections!") + + if command == "UNKNOWN": + print(f"{filename_s}: no command stored", file=sys.stderr) + + print(f"{filename_s}: {collection_max} collection(s)") + print(f"{filename_s}: command: {command}") + data_count = 0 + start_collection = 0 + data = [] + + fp = open(filename, "r") + for i, line in enumerate(fp.readlines()): + # Skip header information + if i < self.MAX_HEADER_LINES: + continue + + # Begin collecting records + if not start_collection and line.startswith("#") and i == self.MAX_HEADER_LINES: + start_collection = 1 + continue + + # Append collected records to result list + if line.startswith("#") and start_collection: + result.append(data) + data = [] + continue + + # Store record + rec = parse_record(line) + data.append(rec) + data_count += 1 + + result.append(data) + fp.close() + self.count = data_count + print(f"{filename}: {data_count} records") + self.result = result + + def diff_record(self, a, b): + """ + Calculate the difference between records a and b + :param a: baseline StraceRecord + :param b: comparison StraceRecord + :return: StraceRecord containing the difference between a and b + """ + r_calls = b.calls - a.calls + r_seconds = b.seconds - a.seconds + r_min = b.min - a.min + r_max = b.max - a.max + r_usecs_call = b.usecs_call - a.usecs_call + + return STraceRecord(a.name, r_calls, r_seconds, r_min, r_max, r_usecs_call) + + @property + def total(self): + """Sum of all records""" + result = dict(name="Total", calls=0, usecs_call=0, min=0, max=0, seconds=0) + for trace in self.result: + for rec in trace: + result["calls"] += rec.calls + result["usecs_call"] += rec.usecs_call + result["max"] += rec.max + result["min"] += rec.min + result["seconds"] += rec.seconds + + return STraceRecord(**result) + + @property + def average(self): + """Average of all records""" + result = dict(name="Average", calls=0, usecs_call=0, min=0, max=0, seconds=0) + total = self.total + result["calls"] = total.calls + result["usecs_call"] = int(total.usecs_call / self.count) + result["max"] = int(total.max / self.count) + result["min"] = int(total.min / self.count) + result["seconds"] = int(total.seconds / self.count) + + return STraceRecord(**result) + + @staticmethod + def get_winner(data): + result = "" + if data.usecs_call == 0: + result = "same" + elif data.usecs_call < 0: + result = "faster" + else: + result = "slower" + return result + + @staticmethod + def get_percent(a, b): + if (b.usecs_call - a.usecs_call) < 0: + percent = (a.usecs_call - b.usecs_call) / a.usecs_call * 100 + else: + percent = (b.usecs_call - a.usecs_call) / b.usecs_call * 100 + + return percent + + def diff_show_record(self, title, a, b): + abdiff = self.diff_record(a, b) + fastest = self.get_winner(abdiff) + percent = self.get_percent(a, b) + if not a.calls or not b.calls: + percent = 0 + fastest = "" + + print(f"{title}:") + print(f"\tcalls: {a.calls:10d} {b.calls:10d} {abdiff.calls:+10d}") + print(f"\t\u00B5s/call: {a.usecs_call:8d} {b.usecs_call:10d} {abdiff.usecs_call:+10d} {percent:10.2f}% {fastest}") + + def diff_total(self, b): + """ + Display the total difference between total and b.total + :param b: + """ + total_a = self.total + total_b = b.total + self.diff_show_record("Total", total_a, total_b) + + def diff_average(self, b): + """ + Display the average difference between result and b.average + :param b: + """ + average_a = self.average + average_b = b.average + self.diff_show_record("Average", average_a, average_b) + + @staticmethod + def normalize_results(a, b): + def extract(objs, name): + for x in objs: + if x.name == name: + return x + + x1_result = [] + x2_result = [] + nop = dict(name="", calls=0, usecs_call=0, min=0, max=0, seconds=0) + + for left, right in zip(a.result, b.result): + empty = nop.copy() + keys_a = set(x.name for x in left) + keys_b = set(x.name for x in right) + keys_missing = keys_b ^ keys_a + x1_missing = [] + x2_missing = [] + x1_data = [] + x2_data = [] + + for x in sorted(keys_a): + if x in keys_missing: + x2_missing.append(x) + value = extract(a.result[0], x) + if not value: + continue + x1_data.append(value) + + for x in sorted(keys_b): + if x in keys_missing: + x1_missing.append(x) + value = extract(b.result[0], x) + x2_data.append(value) + + for x in sorted(x1_missing): + empty["name"] = x + x1_data.append(STraceRecord(**empty)) + + for x in sorted(x2_missing): + empty["name"] = x + x2_data.append(STraceRecord(**empty)) + + x1_result.append(sorted(x1_data)) + x2_result.append(sorted(x2_data)) + return zip(x1_result, x2_result) + + def diff(self, b, mode=SHOW_ALL): + """ + Display the difference between stored result and b + :param b: list of StraceRecords + :param mode: flag to handle various output modes (not implemented) + """ + a = self + for i, (left, right) in enumerate(self.normalize_results(a, b)): + print(f"\nCOLLECTION {i+1}\n") + for x1, x2 in zip(left, right): + self.diff_show_record(x1.name, x1, x2)
\ No newline at end of file diff --git a/benchstrace/cli.py b/benchstrace/cli.py new file mode 100644 index 0000000..caa4c19 --- /dev/null +++ b/benchstrace/cli.py @@ -0,0 +1,54 @@ +from argparse import ArgumentParser +from .benchmark import Benchmark +import sys + + +def prof_mode(args): + result = Benchmark(args.COMMAND, passes=args.passes, setup=args.setup, teardown=args.teardown) + + print(f"Records: {result.count}") + if args.output_file: + result.save(args.output_file, clobber=args.clobber) + return 0 + + +def diff_mode(args): + left = Benchmark() + left.load(args.left) + right = Benchmark() + right.load(args.right) + left.diff(right) + + print("\nSUMMARY\n") + left.diff_average(right) + left.diff_total(right) + return 0 + + +def main(): + parser = ArgumentParser() + subparsers = parser.add_subparsers() + + parser_prof = subparsers.add_parser("prof") + parser_prof.add_argument("-o", "--output_file", type=str) + parser_prof.add_argument("-c", "--clobber", action='store_true') + parser_prof.add_argument("-p", "--passes", type=int, default=1) + parser_prof.add_argument("-s", "--setup", type=str) + parser_prof.add_argument("-t", "--teardown", type=str) + parser_prof.add_argument("COMMAND") + parser_prof.set_defaults(func=prof_mode) + + parser_diff = subparsers.add_parser("diff") + parser_diff.add_argument("left") + parser_diff.add_argument("right") + parser_diff.set_defaults(func=diff_mode) + + args = parser.parse_args() + if len(sys.argv) < 2: + parser.print_help() + exit(0) + + args.func(args) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchstrace/strace.py b/benchstrace/strace.py new file mode 100644 index 0000000..efb35d4 --- /dev/null +++ b/benchstrace/strace.py @@ -0,0 +1,113 @@ +import os +import re +import subprocess +import sys +import tempfile +from collections import namedtuple + +KEYS = ["name", "calls", "seconds", "min", "max", "usecs_call"] +STraceRecord = namedtuple("STraceRecord", KEYS) + + +def find_program(name): + syspath = os.environ.get("PATH", "") + for x in syspath.split(":"): + target = os.path.abspath(os.path.join(x, name)) + if os.path.exists(target): + return target + return "" + + +def parse_record(line): + """ + Parse a single line from: strace -w -c + :return: an StraceRecord + """ + rec = line.split() + data = dict(zip(*[KEYS, rec])) + result = dict() + for k, v in data.items(): + result[k] = v + if k == "name": + continue + elif k == "calls" or k == "usecs_call": + if "." in v: + v = v[:v.find(".")] + result[k] = int(v) + else: + result[k] = float(v) + return STraceRecord(**result) + + +def parse_output(lines): + """ + Parse all lines from: strace -w -c + :return: a list of STraceRecords + """ + result = [] + for i, line in enumerate(lines): + if i < 2 or i > len(lines) - 3: + continue + result.append(parse_record(line)) + return result + + +class STrace: + """Run strace command and parse statistical output + """ + VERSION_RE = re.compile(r"strace.*version\s(?P<major>\d+)\.(?P<minor>\d+)\.?(?P<patch>\d+)?") + NEED_VERSION = (6, 0, 0) + + def __init__(self, command="", setup="", teardown="", output_file=""): + self.program = find_program("strace") + self.command = command + major, minor, _ = self.version + if major < 6: + raise RuntimeError(f"strace {major}.{minor} is too old. Install {self.NEED_VERSION}, or greater.") + self.records = self.run(setup, teardown) or [] + + def run(self, setup="", teardown=""): + """ + Execute strace + :param setup: command to execute before `self.command` + :param teardown: command to execute after `self.command` + :return: a list of STraceRecords + """ + handle, tmpfile = tempfile.mkstemp() + os.close(handle) + command = ["strace", "-o", tmpfile, "-w", "-c", "-S", "name", "-U", "name,calls,time-total,time-min,time-max,time-avg"] + self.command.split(" ") + + if setup: + proc_setup = subprocess.run(setup.split()) + if proc_setup.returncode: + print(f"Warning: setup command failed ({proc_setup.returncode})", file=sys.stderr) + + proc = subprocess.run(command, stderr=open("/dev/null", "w")) + if proc.returncode: + print("Warning: non-zero exit ({proc.returncode})", file=sys.stderr) + + if teardown: + proc_teardown = subprocess.run(teardown.split()) + if proc_teardown.returncode: + print(f"Warning: teardown command failed ({proc_teardown.returncode})", file=sys.stderr) + + data = open(tmpfile, "r").read().splitlines() + os.remove(tmpfile) + return parse_output(data) + + @property + def version(self): + """Retrieve version number from strace + :return: tuple containing major, minor, and patch version + """ + command = ["strace", "--version"] + proc = subprocess.run(command, capture_output=True) + data = proc.stdout.decode().splitlines() + match = re.match(self.VERSION_RE, data[0]) + if not match: + return 0, 0, 0 + + result = match.groupdict() + if not result.get("patch"): + result["patch"] = 0 + return int(result["major"]), int(result["minor"]), int(result["patch"]) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f53ca6b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["setuptools>=45", "setuptools-scm[toml]>=6.2"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "benchstrace/_version.py" + +[project] +name = "benchstrace" +authors = [ + {name = "Joseph Hunkeler", email = "jhunkeler@gmail.com"}, +] +description = "Benchmark and compare strace data" +readme = "README.md" +requires-python = ">=3.7" +keywords = ["benchmark", "profile"] +license = {text = "BSD-3-Clause"} +classifiers = [ + "Programming Language :: Python :: 3", +] +dynamic = ["version"] + +[project.optional-dependencies] + +[project.scripts] +benchstrace = "benchstrace.cli:main" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6068493 --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() |