diff options
Diffstat (limited to 'src/benchmark.py')
| -rw-r--r-- | src/benchmark.py | 409 |
1 files changed, 0 insertions, 409 deletions
diff --git a/src/benchmark.py b/src/benchmark.py index 92e7b52..6ea687d 100644 --- a/src/benchmark.py +++ b/src/benchmark.py @@ -28,7 +28,6 @@ import subprocess from time import sleep import traceback -import matplotlib.pyplot as plt import numpy as np import src.globalvars @@ -36,9 +35,6 @@ import src.util from src.util import print_status, print_error, print_warn from src.util import print_info0, print_info, print_debug -# This is useful when evaluating strings in the plot functions. str(np.NaN) == "nan" -nan = np.NaN - class Benchmark: """Default implementation of most methods allocbench expects from a benchmark""" @@ -602,408 +598,3 @@ class Benchmark: stats["outliers"][dp] = outliers self.results["stats"][alloc][perm] = stats - - ###### Summary helpers ###### - def _eval_with_stat(self, evaluation, alloc, perm, stat): - try: - s = evaluation.format(**self.results["stats"][alloc][perm][stat]) - except KeyError as e: - print_warn(traceback.format_exc()) - print_warn(f"For {alloc} in {perm}") - return nan - return eval(s) - - def plot_single_arg(self, yval, ylabel="'y-label'", xlabel="'x-label'", - autoticks=True, title="'default title'", filepostfix="", - sumdir="", arg="", scale=None, file_ext=src.globalvars.summary_file_ext): - - args = self.results["args"] - allocators = self.results["allocators"] - - arg = arg or list(args.keys())[0] - - if not autoticks: - x_vals = list(range(1, len(args[arg]) + 1)) - else: - x_vals = args[arg] - - for allocator in allocators: - y_vals = [] - for perm in self.iterate_args(args=args): - if scale: - if scale == allocator: - y_vals = [1] * len(x_vals) - else: - mean = self._eval_with_stat(yval, allocator, perm, "mean") - norm_mean = self._eval_with_stat(yval, scale, perm, "mean") - y_vals.append(mean / norm_mean) - else: - y_vals.append(self._eval_with_stat(yval, allocator, perm, "mean")) - - plt.plot(x_vals, y_vals, marker='.', linestyle='-', - label=allocator, color=allocators[allocator]["color"]) - - plt.legend(loc="best") - if not autoticks: - plt.xticks(x_vals, args[arg]) - plt.xlabel(eval(xlabel)) - plt.ylabel(eval(ylabel)) - plt.title(eval(title)) - figname = os.path.join(sumdir, f"{self.name}.{filepostfix}.{file_ext}") - if figname.endswith(".tex"): - import tikzplotlib - tikzplotlib.save(figname) - else: - plt.savefig(figname) - plt.clf() - - def barplot_single_arg(self, yval, ylabel="'y-label'", xlabel="'x-label'", - title="'default title'", filepostfix="", sumdir="", - arg="", scale=None, file_ext=src.globalvars.summary_file_ext, yerr=True): - - args = self.results["args"] - allocators = self.results["allocators"] - nallocators = len(allocators) - - if arg: - arg = args[arg] - elif args.keys(): - arg = args[list(args.keys())[0]] - else: - arg = [""] - - narg = len(arg) - - for i, allocator in enumerate(allocators): - x_vals = list(range(i, narg * (nallocators+1), nallocators+1)) - y_vals = [] - y_errs = None - if yerr: - y_errs = [] - - for perm in self.iterate_args(args=args): - if scale: - if scale == allocator: - y_vals = [1] * len(x_vals) - else: - mean = self._eval_with_stat(yval, allocator, perm, "mean") - norm_mean = self._eval_with_stat(yval, scale, perm, "mean") - y_vals.append(mean / norm_mean) - else: - y_vals.append(self._eval_with_stat(yval, allocator, perm, "mean")) - - if yerr: - y_errs.append(self._eval_with_stat(yval, allocator, perm, "std")) - - plt.bar(x_vals, y_vals, width=1, label=allocator, yerr=y_errs, - color=allocators[allocator]["color"]) - - plt.legend(loc="best") - plt.xticks(list(range(int(np.floor(nallocators/2)), narg*(nallocators+1), nallocators+1)), arg) - plt.xlabel(eval(xlabel)) - plt.ylabel(eval(ylabel)) - plt.title(eval(title)) - figname = os.path.join(sumdir, f"{self.name}.{filepostfix}.{file_ext}") - if figname.endswith(".tex"): - import tikzplotlib - tikzplotlib.save(figname) - else: - plt.savefig(figname) - plt.clf() - - def plot_fixed_arg(self, yval, ylabel="'y-label'", xlabel="loose_arg", - autoticks=True, title="'default title'", filepostfix="", - sumdir="", fixed=[], file_ext=src.globalvars.summary_file_ext, scale=None): - - args = self.results["args"] - allocators = self.results["allocators"] - - for arg in fixed or args: - loose_arg = [a for a in args if a != arg][0] - - if not autoticks: - x_vals = list(range(1, len(args[loose_arg]) + 1)) - else: - x_vals = args[loose_arg] - - for arg_value in args[arg]: - for allocator in allocators: - y_vals = [] - for perm in self.iterate_args_fixed({arg: arg_value}, args=args): - if scale: - if scale == allocator: - y_vals = [1] * len(x_vals) - else: - mean = self._eval_with_stat(yval, allocator, perm, "mean") - norm_mean = self._eval_with_stat(yval, scale, perm, "mean") - y_vals.append(mean / norm_mean) - else: - y_vals.append(self._eval_with_stat(yval, allocator, perm, "mean")) - - plt.plot(x_vals, y_vals, marker='.', linestyle='-', - label=allocator, color=allocators[allocator]["color"]) - - plt.legend(loc="best") - if not autoticks: - plt.xticks(x_vals, args[loose_arg]) - plt.xlabel(eval(xlabel)) - plt.ylabel(eval(ylabel)) - plt.title(eval(title)) - figname = os.path.join(sumdir, - f"{self.name}.{arg}.{arg_value}.{filepostfix}.{file_ext}") - if figname.endswith(".tex"): - import tikzplotlib - tikzplotlib.save(figname) - else: - plt.savefig(figname) - plt.clf() - - def export_facts_to_file(self, comment_symbol, f): - """Write collected facts about used system and benchmark to file""" - print(comment_symbol, self.name, file=f) - print(file=f) - print(comment_symbol, "Common facts:", file=f) - for k, v in src.facter.FACTS.items(): - print(comment_symbol, k + ":", v, file=f) - print(file=f) - print(comment_symbol, "Benchmark facts:", file=f) - for k, v in self.results["facts"].items(): - print(comment_symbol, k + ":", v, file=f) - print(file=f) - - def export_stats_to_csv(self, datapoint, path=None): - """Write descriptive statistics about datapoint to csv file""" - allocators = self.results["allocators"] - args = self.results["args"] - stats = self.results["stats"] - - if path is None: - path = datapoint - - path = path + ".csv" - - stats_fields = list(stats[list(allocators)[0]][list(self.iterate_args(args=args))[0]]) - fieldnames = ["allocator", *args, *stats_fields] - widths = [] - for fieldname in fieldnames: - widths.append(len(fieldname) + 2) - - # collect rows - rows = {} - for alloc in allocators: - rows[alloc] = {} - for perm in self.iterate_args(args=args): - d = [] - d.append(alloc) - d += list(perm._asdict().values()) - d += [stats[alloc][perm][s][datapoint] for s in stats[alloc][perm]] - d[-1] = (",".join([str(x) for x in d[-1]])) - rows[alloc][perm] = d - - # calc widths - for i in range(0, len(fieldnames)): - for alloc in allocators: - for perm in self.iterate_args(args=args): - field_len = len(str(rows[alloc][perm][i])) + 2 - if field_len > widths[i]: - widths[i] = field_len - - with open(path, "w") as f: - headerline = "" - for i, h in enumerate(fieldnames): - headerline += h.capitalize().ljust(widths[i]).replace("_", "-") - print(headerline, file=f) - - for alloc in allocators: - for perm in self.iterate_args(args=args): - line = "" - for i, x in enumerate(rows[alloc][perm]): - line += str(x).ljust(widths[i]) - print(line.replace("_", "-"), file=f) - - def export_stats_to_dataref(self, datapoint, path=None): - """Write descriptive statistics about datapoint to dataref file""" - stats = self.results["stats"] - - if path is None: - path = datapoint - - path = path + ".dataref" - - # Example: \drefset{/mysql/glibc/40/Lower-whisker}{71552.0} - line = "\\drefset{{/{}/{}/{}/{}}}{{{}}}" - - with open(path, "w") as f: - # Write facts to file - self.export_facts_to_file("%", f) - - for alloc in self.results["allocators"]: - for perm in self.iterate_args(args=self.results["args"]): - for statistic, values in stats[alloc][perm].items(): - cur_line = line.format(self.name, alloc, - "/".join([str(p) for p in list(perm)]), - statistic, values[datapoint]) - # Replace empty outliers - cur_line.replace("[]", "") - # Replace underscores - cur_line.replace("_", "-") - print(cur_line, file=f) - - def write_best_doublearg_tex_table(self, evaluation, sort=">", - filepostfix="", sumdir="", std=False): - args = self.results["args"] - keys = list(args.keys()) - allocators = self.results["allocators"] - - header_arg = keys[0] if len(args[keys[0]]) < len(args[keys[1]]) else keys[1] - row_arg = [arg for arg in args if arg != header_arg][0] - - headers = args[header_arg] - rows = args[row_arg] - - cell_text = [] - for av in rows: - row = [] - for perm in self.iterate_args_fixed({row_arg: av}, args=args): - best = [] - best_val = None - for allocator in allocators: - d = [] - for m in self.results[allocator][perm]: - d.append(eval(evaluation.format(**m))) - mean = np.mean(d) - if not best_val: - best = [allocator] - best_val = mean - elif ((sort == ">" and mean > best_val) - or (sort == "<" and mean < best_val)): - best = [allocator] - best_val = mean - elif mean == best_val: - best.append(allocator) - - row.append("{}: {:.3f}".format(best[0], best_val)) - cell_text.append(row) - - fname = os.path.join(sumdir, ".".join([self.name, filepostfix, "tex"])) - with open(fname, "w") as f: - print("\\documentclass{standalone}", file=f) - print("\\begin{document}", file=f) - print("\\begin{tabular}{|", end="", file=f) - print(" l |" * len(headers), "}", file=f) - - print(header_arg+"/"+row_arg, end=" & ", file=f) - for header in headers[:-1]: - print(header, end="& ", file=f) - print(headers[-1], "\\\\", file=f) - - for i, row in enumerate(cell_text): - print(rows[i], end=" & ", file=f) - for e in row[:-1]: - print(e, end=" & ", file=f) - print(row[-1], "\\\\", file=f) - print("\\end{tabular}", file=f) - print("\\end{document}", file=f) - - def write_tex_table(self, entries, sort=">", - filepostfix="", sumdir="", std=False): - """generate a latex standalone table from an list of entries dictionaries - - Entries must have at least the two keys: "label" and "expression". - The optional "sort" key specifies the direction of the order: - ">" : bigger is better. - "<" : smaller is better. - - Table layout: - - | alloc1 | alloc2 | .... - --------------------------------------- - | name1 name2 | ... - --------------------------------------- - perm1 | eavl1 eval2 | ... - perm2 | eval1 eval2 | ... - """ - args = self.results["args"] - allocators = self.results["allocators"] - nallocators = len(allocators) - nentries = len(entries) - perm_fields = self.Perm._fields - nperm_fields = len(perm_fields) - - alloc_header_line = f"\\multicolumn{{{nperm_fields}}}{{c|}}{{}} &" - for alloc in allocators: - alloc_header_line += f"\\multicolumn{{{nentries}}}{{c|}}{{{alloc}}} &" - alloc_header_line = alloc_header_line[:-1] + "\\\\" - - perm_fields_header = "" - for field in self.Perm._fields: - perm_fields_header += f'{field} &' - entry_header_line = "" - for entry in entries: - entry_header_line += f'{entry["label"]} &' - entry_header_line = perm_fields_header + entry_header_line * nallocators - entry_header_line = entry_header_line[:-1] + "\\\\" - - fname = os.path.join(sumdir, ".".join([self.name, filepostfix, "tex"])) - with open(fname, "w") as f: - print("\\documentclass{standalone}", file=f) - print("\\usepackage{booktabs}", file=f) - print("\\usepackage{xcolor}", file=f) - print("\\begin{document}", file=f) - print("\\begin{tabular}{|", f"{'c|'*nperm_fields}", f"{'c'*nentries}|"*nallocators, "}", file=f) - print("\\toprule", file=f) - - print(alloc_header_line, file=f) - print("\\hline", file=f) - print(entry_header_line, file=f) - print("\\hline", file=f) - - for perm in self.iterate_args(args=args): - values = [[] for _ in entries] - maxs = [None for _ in entries] - mins = [None for _ in entries] - for allocator in allocators: - for i, entry in enumerate(entries): - expr = entry["expression"] - values[i].append(eval(expr.format(**self.results["stats"][allocator][perm]["mean"]))) - - # get max and min for each entry - for i, entry in enumerate(entries): - if not "sort" in entry: - continue - # bigger is better - elif entry["sort"] == ">": - maxs[i] = max(values[i]) - mins[i] = min(values[i]) - # smaller is better - elif entry["sort"] == "<": - mins[i] = max(values[i]) - maxs[i] = min(values[i]) - - # build row - row = "" - perm_dict = perm._asdict() - for field in perm_fields: - row += str(perm_dict[field]) + "&" - - for i, _ in enumerate(allocators): - for y, entry_vals in enumerate(values): - val = entry_vals[i] - - # format - val_str = str(val) - if type(val) == float: - val_str = f"{val:.2f}" - - # colorize - if val == maxs[y]: - val_str = f"\\textcolor{{green}}{{{val_str}}}" - elif val == mins[y]: - val_str = f"\\textcolor{{red}}{{{val_str}}}" - row += f"{val_str} &" - #escape _ for latex - row = row.replace("_", "\\_") - print(row[:-1], "\\\\", file=f) - - print("\\end{tabular}", file=f) - print("\\end{document}", file=f) |