aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xbench.py1
-rw-r--r--src/benchmark.py308
-rw-r--r--src/benchmarks/loop.py3
-rw-r--r--src/benchmarks/mysql.py67
-rw-r--r--src/benchmarks/realloc.py2
5 files changed, 218 insertions, 163 deletions
diff --git a/bench.py b/bench.py
index 0b581f8..eef861a 100755
--- a/bench.py
+++ b/bench.py
@@ -168,7 +168,6 @@ def main():
else:
print_error("malt not found. Skipping analyse.")
- print_status("Running", bench.name, "...")
bench.run(runs=args.runs)
if need_resultdir:
diff --git a/src/benchmark.py b/src/benchmark.py
index c3573b7..5169a7a 100644
--- a/src/benchmark.py
+++ b/src/benchmark.py
@@ -89,20 +89,18 @@ class Benchmark (object):
# into lists of dicts.
save_data = {}
save_data.update(self.results)
- save_data["mean"] = {}
- save_data["std"] = {}
+ save_data["stats"] = {}
for allocator in self.results["allocators"]:
measures = []
- means = []
- stds = []
+ stats = []
for ntuple in self.iterate_args(args=self.results["args"]):
measures.append((ntuple._asdict(), self.results[allocator][ntuple]))
- means.append((ntuple._asdict(), self.results["mean"][allocator][ntuple]))
- stds.append((ntuple._asdict(), self.results["std"][allocator][ntuple]))
+ if "stats" in self.results:
+ stats.append((ntuple._asdict(), self.results["stats"][allocator][ntuple]))
save_data[allocator] = measures
- save_data["mean"][allocator] = means
- save_data["std"][allocator] = stds
+ if "stats" in self.results:
+ save_data["stats"][allocator] = stats
with open(f, "wb") as f:
pickle.dump(save_data, f)
@@ -126,11 +124,15 @@ class Benchmark (object):
d[self.Perm(**dic)] = measures
self.results[allocator] = d
- for s in ["std", "mean"]:
- d = {}
- for dic, value in self.results[s][allocator]:
+ d = {}
+ if "stats" in self.results:
+ for dic, value in self.results["stats"][allocator]:
d[self.Perm(**dic)] = value
- self.results[s][allocator] = d
+ self.results["stats"][allocator] = d
+
+ # add missing statistics
+ if not "stats" in self.results:
+ self.calc_desc_statistics()
def prepare(self):
os.environ["PATH"] += os.pathsep + os.path.join("build", "benchmarks", self.name)
@@ -166,6 +168,7 @@ class Benchmark (object):
if runs < 1:
return
+ print_status("Running", self.name, "...")
# check if perf is allowed on this system
if self.measure_cmd == self.defaults["measure_cmd"]:
if Benchmark.perf_allowed == None:
@@ -185,20 +188,25 @@ class Benchmark (object):
if not Benchmark.perf_allowed:
raise Exception("You don't have the needed permissions to use perf")
+ # save one valid result to expand expand invalid results
+ valid_result = {}
+
n = len(list(self.iterate_args())) * len(self.allocators)
for run in range(1, runs + 1):
print_status(run, ". run", sep='')
i = 0
- for tname, t in self.allocators.items():
- if tname not in self.results:
- self.results[tname] = {}
+ for alloc_name, t in self.allocators.items():
+ if alloc_name not in self.results:
+ self.results[alloc_name] = {}
env = dict(os.environ)
- env["LD_PRELOAD"] = env.get("LD_PRELOAD", "") + "build/print_status_on_exit.so " + t["LD_PRELOAD"]
+ env["LD_PRELOAD"] = env.get("LD_PRELOAD", "")
+ env["LD_PRELOAD"] += " " + "build/print_status_on_exit.so"
+ env["LD_PRELOAD"] += " " + t["LD_PRELOAD"]
if hasattr(self, "preallocator_hook"):
- self.preallocator_hook((tname, t), run, env,
+ self.preallocator_hook((alloc_name, t), run, env,
verbose=src.globalvars.verbosity)
for perm in self.iterate_args():
@@ -234,80 +242,115 @@ class Benchmark (object):
actual_env = env
- print_debug("Cmd:", actual_cmd)
+ print_debug("\nCmd:", actual_cmd)
res = subprocess.run(actual_cmd.split(),
env=actual_env,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
universal_newlines=True)
- if res.returncode != 0:
- print()
- print_debug("Stdout:\n" + res.stdout)
- print_debug("Stderr:\n" + res.stderr)
- print_error("{} failed with exit code {} for {}".format(actual_cmd, res.returncode, tname))
- break
+ result = {}
- if "ERROR: ld.so" in res.stderr:
+ if res.returncode != 0 or "ERROR: ld.so" in res.stderr:
print()
+ print_debug("Stdout:\n" + res.stdout)
print_debug("Stderr:\n" + res.stderr)
- print_error("Preloading of {} failed for {}".format(t["LD_PRELOAD"], tname))
- break
-
- result = {}
-
- if not self.server_benchmark:
- # Read VmHWM from status file. If our benchmark didn't fork
- # the first occurance of VmHWM is from our benchmark
- with open("status", "r") as f:
- for l in f.readlines():
- if l.startswith("VmHWM:"):
- result["VmHWM"] = l.split()[1]
- break
- os.remove("status")
-
- if hasattr(self, "process_output"):
- self.process_output(result, res.stdout, res.stderr,
- tname, perm,
- verbose=src.globalvars.verbosity)
+ if res.returncode != 0:
+ print_error("{} failed with exit code {} for {}".format(actual_cmd, res.returncode, alloc_name))
+ else:
+ print_error("Preloading of {} failed for {}".format(t["LD_PRELOAD"], alloc_name))
- # Parse perf output if available
- if self.measure_cmd == self.defaults["measure_cmd"]:
- csvreader = csv.reader(res.stderr.splitlines(),
- delimiter=',')
- for row in csvreader:
- # Split of the user/kernel space info to be better portable
- try:
- result[row[2].split(":")[0]] = row[0]
- except Exception as e:
- print_warn("Exception", e, "occured on", row, "for",
- tname, "and", perm)
+ # parse and store results
+ else:
+ if not self.server_benchmark:
+ # Read VmHWM from status file. If our benchmark didn't fork
+ # the first occurance of VmHWM is from our benchmark
+ with open("status", "r") as f:
+ for l in f.readlines():
+ if l.startswith("VmHWM:"):
+ result["VmHWM"] = l.split()[1]
+ break
+ os.remove("status")
+
+ # Parse perf output if available
+ if self.measure_cmd == self.defaults["measure_cmd"]:
+ csvreader = csv.reader(res.stderr.splitlines(),
+ delimiter=',')
+ for row in csvreader:
+ # Split of the user/kernel space info to be better portable
+ try:
+ result[row[2].split(":")[0]] = row[0]
+ except Exception as e:
+ print_warn("Exception", e, "occured on", row, "for",
+ alloc_name, "and", perm)
+
+ if hasattr(self, "process_output"):
+ self.process_output(result, res.stdout, res.stderr,
+ alloc_name, perm,
+ verbose=src.globalvars.verbosity)
+
+ # save a valid result so we can expand invalid ones
+ if valid_result != None:
+ valid_result = result
if not dry_run:
- if not perm in self.results[tname]:
- self.results[tname][perm] = []
- self.results[tname][perm].append(result)
-
- if run == runs:
- self.results["mean"][tname][perm] = {}
- self.results["std"][tname][perm] = {}
-
- for datapoint in self.results[tname][perm][0]:
- try:
- d = [np.float(m[datapoint]) for m in self.results[tname][perm]]
- except ValueError:
- d = np.NaN
- self.results["mean"][tname][perm][datapoint] = np.mean(d)
- self.results["std"][tname][perm][datapoint] = np.std(d)
+ if not perm in self.results[alloc_name]:
+ self.results[alloc_name][perm] = []
+ self.results[alloc_name][perm].append(result)
if hasattr(self, "postallocator_hook"):
- self.postallocator_hook((tname, t), run,
+ self.postallocator_hook((alloc_name, t), run,
verbose=src.globalvars.verbosity)
print()
# Reset PATH
os.environ["PATH"] = os.environ["PATH"].replace(":build/" + self.name, "")
+ #expand invalid results
+ if valid_result != {}:
+ for allocator in self.allocators:
+ for perm in self.iterate_args():
+ for i, m in enumerate(self.results[allocator][perm]):
+ if m == {}:
+ self.results[allocator][perm][i] = {k: np.NaN for k in valid_result}
+
+ self.calc_desc_statistics()
+
+ def calc_desc_statistics(self):
+ if "stats" in self.results:
+ return
+ allocs = self.results["allocators"]
+ self.results["stats"] = {}
+ for alloc in allocs:
+ self.results["stats"][alloc] = {}
+ for perm in self.iterate_args(self.results["args"]):
+ stats = {s: {} for s in ["min", "max", "mean", "median", "std",
+ "lower_quartile", "upper_quartile",
+ "lower_whiskers", "upper_whiskers",
+ "outliers"]}
+ for dp in self.results[alloc][perm][0]:
+ try:
+ data = [float(m[dp]) for m in self.results[alloc][perm]]
+ except ValueError as e:
+ print_debug(e)
+ continue
+ stats["min"][dp] = np.min(data)
+ stats["max"][dp] = np.max(data)
+ stats["mean"][dp] = np.mean(data)
+ stats["median"][dp] = np.median(data)
+ stats["std"][dp] = np.std(data, ddof=1)
+ stats["lower_quartile"][dp], stats["upper_quartile"][dp] = np.percentile(data, [25, 75])
+ trimmed_range = stats["upper_quartile"][dp] - stats["lower_quartile"][dp]
+ stats["lower_whiskers"][dp] = stats["lower_quartile"][dp] - trimmed_range
+ stats["upper_whiskers"][dp] = stats["upper_quartile"][dp] - trimmed_range
+ outliers = []
+ for d in data:
+ if d > stats["upper_whiskers"][dp] or d < stats["lower_whiskers"][dp]:
+ outliers.append(d)
+ stats["outliers"][dp] = outliers
+
+ self.results["stats"][alloc][perm] = stats
+
def plot_single_arg(self, yval, ylabel="'y-label'", xlabel="'x-label'",
autoticks=True, title="'default title'", filepostfix="",
sumdir="", arg="", scale=None, file_ext="png"):
@@ -329,17 +372,17 @@ class Benchmark (object):
if scale == allocator:
y_vals = [1] * len(x_vals)
else:
- mean = eval(yval.format(**self.results["mean"][allocator][perm]))
- norm_mean = eval(yval.format(**self.results["mean"][scale][perm]))
+ mean = eval(yval.format(**self.results["stats"][allocator][perm]["mean"]))
+ norm_mean = eval(yval.format(**self.results["stats"][scale][perm]["mean"]))
y_vals.append(mean / norm_mean)
else:
- y_vals.append(eval(yval.format(**self.results["mean"][allocator][perm])))
+ y_vals.append(eval(yval.format(**self.results["stats"][allocator][perm]["mean"])))
plt.plot(x_vals, y_vals, marker='.', linestyle='-',
label=allocator, color=allocators[allocator]["color"])
- plt.legend()
+ plt.legend(loc="best")
if not autoticks:
plt.xticks(x_vals, args[arg])
plt.xlabel(eval(xlabel))
@@ -348,6 +391,44 @@ class Benchmark (object):
plt.savefig(os.path.join(sumdir, ".".join([self.name, filepostfix, file_ext])))
plt.clf()
+ def barplot_single_arg(self, yval, ylabel="'y-label'", xlabel="'x-label'",
+ title="'default title'", filepostfix="", sumdir="",
+ arg="", scale=None, file_ext="png"):
+
+ args = self.results["args"]
+ allocators = self.results["allocators"]
+ nallocators = len(allocators)
+
+ arg = arg or list(args.keys())[0]
+ narg = len(args[arg])
+
+
+ for i, allocator in enumerate(allocators):
+ x_vals = list(range(i, narg * (nallocators+1), nallocators+1))
+ y_vals = []
+ for perm in self.iterate_args(args=args):
+ if scale:
+ if scale == allocator:
+ y_vals = [1] * len(x_vals)
+ else:
+ mean = eval(yval.format(**self.results["stats"][allocator][perm]["mean"]))
+ norm_mean = eval(yval.format(**self.results["stats"][scale][perm]["mean"]))
+ y_vals.append(mean / norm_mean)
+ else:
+ y_vals.append(eval(yval.format(**self.results["stats"][allocator][perm]["mean"])))
+
+
+ plt.bar(x_vals, y_vals, width=1, label=allocator,
+ color=allocators[allocator]["color"])
+
+ plt.legend(loc="best")
+ plt.xticks(list(range(int(np.floor(nallocators/2)), narg*(nallocators+1), nallocators+1)), args[arg])
+ plt.xlabel(eval(xlabel))
+ plt.ylabel(eval(ylabel))
+ plt.title(eval(title))
+ plt.savefig(os.path.join(sumdir, ".".join([self.name, filepostfix, file_ext])))
+ plt.clf()
+
def plot_fixed_arg(self, yval, ylabel="'y-label'", xlabel="loose_arg",
autoticks=True, title="'default title'", filepostfix="",
sumdir="", fixed=[], file_ext="png", scale=None):
@@ -371,17 +452,17 @@ class Benchmark (object):
if scale == allocator:
y_vals = [1] * len(x_vals)
else:
- mean = eval(yval.format(**self.results["mean"][allocator][perm]))
- norm_mean = eval(yval.format(**self.results["mean"][scale][perm]))
+ mean = eval(yval.format(**self.results["stats"][allocator][perm]["mean"]))
+ norm_mean = eval(yval.format(**self.results["stats"][scale][perm]["mean"]))
y_vals.append(mean / norm_mean)
else:
- y_vals.append(eval(yval.format(**self.results["mean"][allocator][perm])))
+ y_vals.append(eval(yval.format(**self.results["stats"][allocator][perm]["mean"])))
plt.plot(x_vals, y_vals, marker='.', linestyle='-',
label=allocator, color=allocators[allocator]["color"])
- plt.legend()
+ plt.legend(loc="best")
if not autoticks:
plt.xticks(x_vals, args[loose_arg])
plt.xlabel(eval(xlabel))
@@ -391,47 +472,54 @@ class Benchmark (object):
str(arg_value), filepostfix, file_ext])))
plt.clf()
- def export_to_csv(self, datapoints=None, path=None, std=True):
- args = self.results["args"]
+ def export_to_csv(self, datapoint, path=None):
allocators = self.results["allocators"]
+ args = self.results["args"]
+ stats = self.results["stats"]
if path is None:
- if datapoints is not None:
- path = ".".join(datapoints)
- else:
- path = "full"
+ path = datapoint
path = path + ".csv"
- if datapoints is None:
- first_alloc = list(allocators)[0]
- first_perm = list(self.results[first_alloc])[0]
- datapoints = list(self.results[first_alloc][first_perm])
-
- for allocator in self.results["allocators"]:
- path_alloc = allocator + '_' + path
- with open(path_alloc, "w") as f:
- fieldnames = [*args]
- for d in datapoints:
- fieldnames.append(d)
- if std:
- fieldnames.append(d + "(std)")
-
- writer = csv.DictWriter(f, fieldnames, delimiter="\t",
- lineterminator='\n')
- writer.writeheader()
+ stats_fields = list(stats[list(allocators)[0]][list(self.iterate_args(args=args))[0]])
+ fieldnames = ["allocator", *args, *stats_fields]
+ widths = []
+ for fieldname in fieldnames:
+ widths.append(len(fieldname) + 2)
+ # collect rows
+ rows = {}
+ for alloc in allocators:
+ rows[alloc] = {}
+ for perm in self.iterate_args(args=args):
+ d = []
+ d.append(alloc)
+ d += list(perm._asdict().values())
+ d += [stats[alloc][perm][s][datapoint] for s in stats[alloc][perm]]
+ d[-1] = (",".join([str(x) for x in d[-1]]))
+ rows[alloc][perm] = d
+
+ # calc widths
+ for i in range(0, len(fieldnames)):
+ for alloc in allocators:
for perm in self.iterate_args(args=args):
- d = {}
- d.update(perm._asdict())
+ field_len = len(str(rows[alloc][perm][i])) + 2
+ if field_len > widths[i]:
+ widths[i] = field_len
- for dp in datapoints:
- d[dp] = self.results["mean"][allocator][perm][dp]
- if std:
- fieldname = dp + "(std)"
- d[fieldname] = self.results["std"][allocator][perm][dp]
+ with open(path, "w") as f:
+ headerline = ""
+ for i, h in enumerate(fieldnames):
+ headerline += h.ljust(widths[i])
+ print(headerline, file=f)
- writer.writerow(d)
+ for alloc in allocators:
+ for perm in self.iterate_args(args=args):
+ line = ""
+ for i, x in enumerate(rows[alloc][perm]):
+ line += str(x).ljust(widths[i])
+ print(line, file=f)
def write_best_doublearg_tex_table(self, evaluation, sort=">",
filepostfix="", sumdir="", std=False):
diff --git a/src/benchmarks/loop.py b/src/benchmarks/loop.py
index 099a578..ef291c4 100644
--- a/src/benchmarks/loop.py
+++ b/src/benchmarks/loop.py
@@ -42,8 +42,7 @@ class Benchmark_Loop(Benchmark):
self.write_best_doublearg_tex_table("perm.nthreads / ({task-clock}/1000)",
filepostfix="memusage.matrix")
- self.export_to_csv(datapoints=["task-clock", "L1-dcache-load-misses",
- "L1-dcache-loads"])
+ self.export_to_csv("task-clock")
loop = Benchmark_Loop()
diff --git a/src/benchmarks/mysql.py b/src/benchmarks/mysql.py
index 0ac3f75..1420b4d 100644
--- a/src/benchmarks/mysql.py
+++ b/src/benchmarks/mysql.py
@@ -166,72 +166,39 @@ class Benchmark_MYSQL(Benchmark):
xlabel='"threads"',
ylabel='"transactions"',
title='"sysbench oltp read only"',
- filepostfix="l.ro")
+ filepostfix="l")
- # linear plot
+ # normalized linear plot
ref_alloc = list(allocators)[0]
self.plot_single_arg("{transactions}",
xlabel='"threads"',
ylabel='"transactions scaled at " + scale',
title='"sysbench oltp read only"',
- filepostfix="norm.l.ro",
+ filepostfix="norm.l",
scale=ref_alloc)
# bar plot
- for i, allocator in enumerate(allocators):
- y_vals = []
- for perm in self.iterate_args(args=self.results["args"]):
- d = [int(m["transactions"]) for m in self.results[allocator][perm]]
- y_vals.append(np.mean(d))
- x_vals = [x-i/8 for x in range(1, len(y_vals) + 1)]
- plt.bar(x_vals, y_vals, width=0.2, label=allocator, align="center",
- color=allocators[allocator]["color"])
-
- plt.legend()
- plt.xlabel("threads")
- plt.xticks(range(1, len(y_vals) + 1), self.results["args"]["nthreads"])
- plt.ylabel("transactions")
- plt.title("sysbench oltp read only")
- plt.savefig(self.name + ".b.ro.png")
- plt.clf()
+ self.barplot_single_arg("{transactions}",
+ xlabel='"threads"',
+ ylabel='"transactions"',
+ title='"sysbench oltp read only"',
+ filepostfix="b")
# normalized bar plot
- norm_y_vals = []
- for perm in self.iterate_args(args=self.results["args"]):
- d = [int(m["transactions"]) for m in self.results[ref_alloc][perm]]
- norm_y_vals.append(np.mean(d))
-
- nallocs = len(allocators)
- x_vals = [x for x in range(1, len(norm_y_vals)*nallocs + 1, nallocs)]
- plt.bar(x_vals, [1]*len(norm_y_vals), width=0.7, label=ref_alloc,
- align="center", color=allocators[ref_alloc]["color"])
-
- for i, allocator in enumerate(list(allocators)[1:]):
- y_vals = []
- for y, perm in enumerate(self.iterate_args(args=self.results["args"])):
- d = [int(m["transactions"]) for m in self.results[allocator][perm]]
- y_vals.append(np.mean(d) / norm_y_vals[y])
-
- x_vals = [x+i+1 for x in range(1, len(norm_y_vals)*nallocs + 1, nallocs)]
-
- plt.bar(x_vals, y_vals, width=0.7, label=allocator, align="center",
- color=allocators[allocator]["color"])
-
- plt.legend()
- plt.xlabel("threads")
- plt.xticks(range(1, len(norm_y_vals)*nallocs + 1, nallocs), self.results["args"]["nthreads"])
- plt.ylabel("transactions normalized")
- plt.title("sysbench oltp read only")
- plt.savefig(self.name + ".norm.b.ro.png")
- plt.clf()
+ self.barplot_single_arg("{transactions}",
+ xlabel='"threads"',
+ ylabel='"transactions scaled at " + scale',
+ title='"sysbench oltp read only"',
+ filepostfix="norm.b",
+ scale=ref_alloc)
# Memusage
- self.plot_single_arg("{rssmax}",
+ self.barplot_single_arg("{rssmax}",
xlabel='"threads"',
ylabel='"VmHWM in kB"',
title='"Memusage sysbench oltp read only"',
- filepostfix="ro.mem")
+ filepostfix="mem")
# Colored latex table showing transactions count
d = {allocator: {} for allocator in allocators}
@@ -281,5 +248,7 @@ class Benchmark_MYSQL(Benchmark):
print("\end{tabular}", file=f)
+ self.export_to_csv("transactions")
+
mysql = Benchmark_MYSQL()
diff --git a/src/benchmarks/realloc.py b/src/benchmarks/realloc.py
index d39c7a0..957d5fd 100644
--- a/src/benchmarks/realloc.py
+++ b/src/benchmarks/realloc.py
@@ -33,7 +33,7 @@ class Benchmark_Realloc(Benchmark):
plt.savefig(self.name + ".png")
plt.clf()
- self.export_to_csv(datapoints=["task-clock"])
+ self.export_to_csv("task-clock")
realloc = Benchmark_Realloc()
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-23 18:09:32 +0000