diff options
Diffstat (limited to 'benchstrace/benchmark.py')
| -rw-r--r-- | benchstrace/benchmark.py | 264 |
1 files changed, 264 insertions, 0 deletions
diff --git a/benchstrace/benchmark.py b/benchstrace/benchmark.py new file mode 100644 index 0000000..8094bd5 --- /dev/null +++ b/benchstrace/benchmark.py @@ -0,0 +1,264 @@ +import os +import sys +from .strace import STrace, STraceRecord +from .strace import parse_record + + +SHOW_COLLECTION = 1 << 1 +SHOW_AVERAGE = 2 << 1 +SHOW_TOTAL = 3 << 1 +SHOW_ALL = SHOW_COLLECTION | SHOW_AVERAGE | SHOW_TOTAL + + +class Benchmark: + # Input file data begins after + MAX_HEADER_LINES = 3 + + def __init__(self, command="", passes=2, setup="", teardown=""): + """ + A benchmark record + :param command: to execute + :param passes: times to execute command + :param setup: to execute before command + :param teardown: to execute after command + """ + self.result = [] + self.command = command + self.passes = passes + self.count = 0 + + if self.passes < 1: + self.passes = 1 + + if self.command: + for i in range(self.passes): + if self.passes > 1: + print(f"Pass [{i+1}/{passes}]: ", end="") + print(f"Running `{self.command}`") + data = STrace(command, setup=setup, teardown=teardown).records + self.count += len(data) + self.result.append(data) + + def save(self, filename, clobber=False): + """ + Write benchmark data to a file + :param filename: output file path + :param clobber: toggle overwriting the output file + """ + if clobber and os.path.exists(filename): + raise FileExistsError(filename) + + filename = os.path.abspath(filename) + print(f"Writing {filename}") + with open(filename, "w+") as fp: + print(f"# {len(self.result)}", file=fp) + print(f"# {self.command}", file=fp) + print("", file=fp) + for t, data in enumerate(self.result): + print(f"# {t}", file=fp) + for rec in data: + rec_fmt = f"{rec.name} {rec.calls} {rec.seconds:.6f} {rec.min:.6f} {rec.max:.6f} {rec.usecs_call}" + print(rec_fmt, file=fp) + + def load(self, filename): + """ + Read benchmark data from a file + :param filename: input file path + """ + result = [] + filename_s = os.path.basename(filename) + + with open(filename, "r") as fp: + collection_max = int(fp.readline().split("#")[1]) or 0 + command = fp.readline().split("#")[1] or "UNKNOWN" + + if not collection_max: + raise ValueError(f"{filename_s} has no collections!") + + if command == "UNKNOWN": + print(f"{filename_s}: no command stored", file=sys.stderr) + + print(f"{filename_s}: {collection_max} collection(s)") + print(f"{filename_s}: command: {command}") + data_count = 0 + start_collection = 0 + data = [] + + fp = open(filename, "r") + for i, line in enumerate(fp.readlines()): + # Skip header information + if i < self.MAX_HEADER_LINES: + continue + + # Begin collecting records + if not start_collection and line.startswith("#") and i == self.MAX_HEADER_LINES: + start_collection = 1 + continue + + # Append collected records to result list + if line.startswith("#") and start_collection: + result.append(data) + data = [] + continue + + # Store record + rec = parse_record(line) + data.append(rec) + data_count += 1 + + result.append(data) + fp.close() + self.count = data_count + print(f"{filename}: {data_count} records") + self.result = result + + def diff_record(self, a, b): + """ + Calculate the difference between records a and b + :param a: baseline StraceRecord + :param b: comparison StraceRecord + :return: StraceRecord containing the difference between a and b + """ + r_calls = b.calls - a.calls + r_seconds = b.seconds - a.seconds + r_min = b.min - a.min + r_max = b.max - a.max + r_usecs_call = b.usecs_call - a.usecs_call + + return STraceRecord(a.name, r_calls, r_seconds, r_min, r_max, r_usecs_call) + + @property + def total(self): + """Sum of all records""" + result = dict(name="Total", calls=0, usecs_call=0, min=0, max=0, seconds=0) + for trace in self.result: + for rec in trace: + result["calls"] += rec.calls + result["usecs_call"] += rec.usecs_call + result["max"] += rec.max + result["min"] += rec.min + result["seconds"] += rec.seconds + + return STraceRecord(**result) + + @property + def average(self): + """Average of all records""" + result = dict(name="Average", calls=0, usecs_call=0, min=0, max=0, seconds=0) + total = self.total + result["calls"] = total.calls + result["usecs_call"] = int(total.usecs_call / self.count) + result["max"] = int(total.max / self.count) + result["min"] = int(total.min / self.count) + result["seconds"] = int(total.seconds / self.count) + + return STraceRecord(**result) + + @staticmethod + def get_winner(data): + result = "" + if data.usecs_call == 0: + result = "same" + elif data.usecs_call < 0: + result = "faster" + else: + result = "slower" + return result + + @staticmethod + def get_percent(a, b): + if (b.usecs_call - a.usecs_call) < 0: + percent = (a.usecs_call - b.usecs_call) / a.usecs_call * 100 + else: + percent = (b.usecs_call - a.usecs_call) / b.usecs_call * 100 + + return percent + + def diff_show_record(self, title, a, b): + abdiff = self.diff_record(a, b) + fastest = self.get_winner(abdiff) + percent = self.get_percent(a, b) + if not a.calls or not b.calls: + percent = 0 + fastest = "" + + print(f"{title}:") + print(f"\tcalls: {a.calls:10d} {b.calls:10d} {abdiff.calls:+10d}") + print(f"\t\u00B5s/call: {a.usecs_call:8d} {b.usecs_call:10d} {abdiff.usecs_call:+10d} {percent:10.2f}% {fastest}") + + def diff_total(self, b): + """ + Display the total difference between total and b.total + :param b: + """ + total_a = self.total + total_b = b.total + self.diff_show_record("Total", total_a, total_b) + + def diff_average(self, b): + """ + Display the average difference between result and b.average + :param b: + """ + average_a = self.average + average_b = b.average + self.diff_show_record("Average", average_a, average_b) + + @staticmethod + def normalize_results(a, b): + def extract(objs, name): + for x in objs: + if x.name == name: + return x + + x1_result = [] + x2_result = [] + nop = dict(name="", calls=0, usecs_call=0, min=0, max=0, seconds=0) + + for left, right in zip(a.result, b.result): + empty = nop.copy() + keys_a = set(x.name for x in left) + keys_b = set(x.name for x in right) + keys_missing = keys_b ^ keys_a + x1_missing = [] + x2_missing = [] + x1_data = [] + x2_data = [] + + for x in sorted(keys_a): + if x in keys_missing: + x2_missing.append(x) + value = extract(a.result[0], x) + if not value: + continue + x1_data.append(value) + + for x in sorted(keys_b): + if x in keys_missing: + x1_missing.append(x) + value = extract(b.result[0], x) + x2_data.append(value) + + for x in sorted(x1_missing): + empty["name"] = x + x1_data.append(STraceRecord(**empty)) + + for x in sorted(x2_missing): + empty["name"] = x + x2_data.append(STraceRecord(**empty)) + + x1_result.append(sorted(x1_data)) + x2_result.append(sorted(x2_data)) + return zip(x1_result, x2_result) + + def diff(self, b, mode=SHOW_ALL): + """ + Display the difference between stored result and b + :param b: list of StraceRecords + :param mode: flag to handle various output modes (not implemented) + """ + a = self + for i, (left, right) in enumerate(self.normalize_results(a, b)): + print(f"\nCOLLECTION {i+1}\n") + for x1, x2 in zip(left, right): + self.diff_show_record(x1.name, x1, x2)
\ No newline at end of file |