From 25c4d81069f576354d0279bf38417c236e924540 Mon Sep 17 00:00:00 2001
From: Florian Fischer <florian.fl.fischer@fau.de>
Date: Mon, 25 Mar 2019 17:49:39 +0100
Subject: move benchmark definitions into src/benchmarks

bench now loads all *.py files from src/benchmarks as benchmarks
---
 bench.py                       |   6 +-
 src/benchmarks/__init__.py     |   0
 src/benchmarks/dj_trace.py     | 357 +++++++++++++++++++++++++++++++++++++++++
 src/benchmarks/falsesharing.py |  74 +++++++++
 src/benchmarks/larson.py       |  46 ++++++
 src/benchmarks/loop.py         |  36 +++++
 src/benchmarks/mysql.py        | 285 ++++++++++++++++++++++++++++++++
 src/dj_trace.py                | 357 -----------------------------------------
 src/falsesharing.py            |  74 ---------
 src/larson.py                  |  46 ------
 src/loop.py                    |  36 -----
 src/mysql.py                   | 285 --------------------------------
 12 files changed, 802 insertions(+), 800 deletions(-)
 create mode 100644 src/benchmarks/__init__.py
 create mode 100644 src/benchmarks/dj_trace.py
 create mode 100644 src/benchmarks/falsesharing.py
 create mode 100644 src/benchmarks/larson.py
 create mode 100644 src/benchmarks/loop.py
 create mode 100644 src/benchmarks/mysql.py
 delete mode 100644 src/dj_trace.py
 delete mode 100644 src/falsesharing.py
 delete mode 100644 src/larson.py
 delete mode 100644 src/loop.py
 delete mode 100644 src/mysql.py

diff --git a/bench.py b/bench.py
index 412597f..90bd0bf 100755
--- a/bench.py
+++ b/bench.py
@@ -14,7 +14,9 @@ import src.globalvars
 from src.util import *
 
 
-benchmarks = ["loop", "mysql", "falsesharing", "dj_trace", "larson"]
+bench_dir = "src/benchmarks"
+benchmarks = [e[:-3] for e in os.listdir(bench_dir)
+                     if e[-3:] == ".py" and e != "__init__.py"]
 
 parser = argparse.ArgumentParser(description="benchmark memory allocators")
 parser.add_argument("-ds, --dont-save", action='store_true', dest="dont_save",
@@ -143,7 +145,7 @@ def main():
             continue
 
         try:
-            bench = eval("importlib.import_module('src.{0}').{0}".format(bench))
+            bench = eval("importlib.import_module('src.benchmarks.{0}').{0}".format(bench))
 
             if args.load:
                 bench.load(path=args.load)
diff --git a/src/benchmarks/__init__.py b/src/benchmarks/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/benchmarks/dj_trace.py b/src/benchmarks/dj_trace.py
new file mode 100644
index 0000000..062f34c
--- /dev/null
+++ b/src/benchmarks/dj_trace.py
@@ -0,0 +1,357 @@
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+from urllib.request import urlretrieve
+import sys
+import re
+
+from src.benchmark import Benchmark
+from src.util import print_status
+
+comma_sep_number_re = "(?:\d*(?:,\d*)?)*"
+rss_re = "(?P<rss>" + comma_sep_number_re + ")"
+time_re = "(?P<time>" + comma_sep_number_re + ")"
+
+cycles_re = re.compile("^{} cycles$".format(time_re))
+cpu_time_re = re.compile("^{} usec across.*threads$".format(time_re))
+
+max_rss_re = re.compile("^{} Kb Max RSS".format(rss_re))
+ideal_rss_re = re.compile("^{} Kb Max Ideal RSS".format(rss_re))
+
+malloc_re = re.compile("^Avg malloc time:\s*{} in.*calls$".format(time_re))
+calloc_re = re.compile("^Avg calloc time:\s*{} in.*calls$".format(time_re))
+realloc_re = re.compile("^Avg realloc time:\s*{} in.*calls$".format(time_re))
+free_re = re.compile("^Avg free time:\s*{} in.*calls$".format(time_re))
+
+
+class Benchmark_DJ_Trace(Benchmark):
+    def __init__(self):
+        self.name = "dj_trace"
+        self.descrition = """This benchmark uses the workload simulator written
+                             by DJ Delorie to simulate workloads provided by
+                             him under https://delorie.com/malloc. Those
+                             workloads are generated from traces of real
+                             aplications and are also used by delorie to
+                             measure improvements in the glibc allocator."""
+
+        self.cmd = "trace_run{binary_suffix} dj_workloads/{workload}.wl"
+        self.measure_cmd = ""
+
+        self.args = {
+                        "workload": [
+                                        "389-ds-2",
+                                        "dj",
+                                        "dj2",
+                                        "mt_test_one_alloc",
+                                        "oocalc",
+                                        "qemu-virtio",
+                                        "qemu-win7",
+                                        "proprietary-1",
+                                        "proprietary-2",
+                                      ]
+                    }
+        self.results = {
+                        "389-ds-2": {
+                            "malloc": 170500018, "calloc": 161787184,
+                            "realloc": 404134, "free": 314856324,
+                            "threads": 41},
+                        "dj": {
+                            "malloc": 2000000, "calloc": 200, "realloc": 0,
+                            "free": 2003140, "threads": 201},
+                        "dj2": {
+                            "malloc": 29263321, "calloc": 3798404,
+                            "realloc": 122956, "free": 32709054,
+                            "threads": 36},
+                        "mt_test_one_alloc": {
+                            "malloc": 524290, "calloc": 1, "realloc": 0,
+                            "free": 594788, "threads": 2},
+                        "oocalc": {
+                            "malloc": 6731734, "calloc": 38421,
+                            "realloc": 14108, "free": 6826686, "threads": 88},
+                        "qemu-virtio": {
+                            "malloc": 1772163, "calloc": 146634,
+                            "realloc": 59813, "free": 1954732, "threads": 3},
+                        "qemu-win7": {
+                            "malloc": 980904, "calloc": 225420,
+                            "realloc": 89880, "free": 1347825, "threads": 6},
+                        "proprietary-1": {
+                            "malloc": 316032131, "calloc": 5642, "realloc": 84,
+                            "free": 319919727, "threads": 20},
+                        "proprietary-2": {
+                            "malloc": 9753948, "calloc": 4693,
+                            "realloc": 117, "free": 10099261, "threads": 19},
+                        }
+
+        self.requirements = ["trace_run"]
+        super().__init__()
+
+    def prepare(self):
+        super().prepare()
+
+        def reporthook(blocknum, blocksize, totalsize):
+            readsofar = blocknum * blocksize
+            if totalsize > 0:
+                percent = readsofar * 1e2 / totalsize
+                s = "\r%5.1f%% %*d / %d" % (
+                    percent, len(str(totalsize)), readsofar, totalsize)
+                sys.stderr.write(s)
+            else:  # total size is unknown
+                sys.stderr.write("\rdownloaded %d" % (readsofar,))
+
+        if not os.path.isdir("dj_workloads"):
+            os.mkdir("dj_workloads")
+
+        download_all = None
+        wl_sizes = {"dj": "14M", "oocalc": "65M", "mt_test_one_alloc": "5.7M",
+                    "proprietary-1": "2.8G", "qemu-virtio": "34M",
+                    "proprietary-2": "92M", "qemu-win7": "23M",
+                    "389-ds-2": "3.4G", "dj2": "294M"}
+
+        for wl in self.args["workload"]:
+            file_name = wl + ".wl"
+            file_path = os.path.join("dj_workloads", file_name)
+            if not os.path.isfile(file_path):
+                if download_all == None:
+                    choice = input(("Download all missing workloads (upto 6.7GB)"
+                                          " [Y/n/x] "))
+                    if choice == "x":
+                        break
+                    else:
+                        download_all = choice in ['', 'Y', 'y']
+
+                if (not download_all and
+                    input("want to download {} ({}) [Y/n] ".format(wl, wl_sizes[wl])) not in ['', 'Y', 'y']):
+                    continue
+
+                if download_all:
+                    print_status("downloading {} ({}) ...".format(wl, wl_sizes[wl]))
+
+                url = "http://www.delorie.com/malloc/" + file_name
+                urlretrieve(url, file_path, reporthook)
+                sys.stderr.write("\n")
+
+        available_workloads = []
+        for wl in self.args["workload"]:
+            file_name = wl + ".wl"
+            file_path = os.path.join("dj_workloads", file_name)
+            if os.path.isfile(file_path):
+                available_workloads.append(wl)
+
+        if len(available_workloads) > 0:
+            self.args["workload"] = available_workloads
+            return True
+        
+        return False
+
+    def process_output(self, result, stdout, stderr, allocator, perm, verbose):
+        def to_int(s):
+            return int(s.replace(',', ""))
+
+        regexs = {7: malloc_re, 8: calloc_re, 9: realloc_re, 10: free_re}
+        functions = {7: "malloc", 8: "calloc", 9: "realloc", 10: "free"}
+        for i, l in enumerate(stdout.splitlines()):
+            if i == 0:
+                result["cycles"] = to_int(cycles_re.match(l).group("time"))
+            elif i == 2:
+                result["cputime"] = to_int(cpu_time_re.match(l).group("time"))
+            elif i == 3:
+                result["Max_RSS"] = to_int(max_rss_re.match(l).group("rss"))
+            elif i == 4:
+                result["Ideal_RSS"] = to_int(ideal_rss_re.match(l).group("rss"))
+            elif i in [7, 8, 9, 10]:
+                res = regexs[i].match(l)
+                fname = functions[i]
+                result["avg_" + fname] = to_int(res.group("time"))
+
+    def summary(self):
+        args = self.results["args"]
+        allocators = self.results["allocators"]
+
+        cpu_time_means = {allocator: {} for allocator in allocators}
+        cycles_means = {allocator: {} for allocator in allocators}
+        for perm in self.iterate_args(args=args):
+            for i, allocator in enumerate(allocators):
+                d = [x["cputime"] for x in self.results[allocator][perm]]
+                # data is in milliseconds
+                cpu_time_means[allocator][perm] = int(np.mean(d)/1000)
+
+                d = [x["cycles"] for x in self.results[allocator][perm]]
+                cycles_means[allocator][perm] = int(np.mean(d))
+
+                plt.bar([i], cpu_time_means[allocator][perm], label=allocator,
+                        color=allocators[allocator]["color"])
+
+            plt.legend(loc="best")
+            plt.ylabel("Zeit in ms")
+            plt.title("Gesamte Laufzeit")
+            plt.savefig(".".join([self.name, perm.workload, "runtime", "png"]))
+            plt.clf()
+
+        # Function Times
+        func_times_means = {allocator: {} for allocator in allocators}
+        xa = np.arange(0, 6, 1.5)
+        for perm in self.iterate_args(args=args):
+            for i, allocator in enumerate(allocators):
+                x_vals = [x+i/len(allocators) for x in xa]
+
+                func_times_means[allocator][perm] = [0,0,0,0]
+
+                func_times_means[allocator][perm][0] = np.mean([x["avg_malloc"] for x in self.results[allocator][perm]])
+                func_times_means[allocator][perm][1] = np.mean([x["avg_calloc"] for x in self.results[allocator][perm]])
+                func_times_means[allocator][perm][2] = np.mean([x["avg_realloc"] for x in self.results[allocator][perm]])
+                func_times_means[allocator][perm][3] = np.mean([x["avg_free"] for x in self.results[allocator][perm]])
+
+                plt.bar(x_vals, func_times_means[allocator][perm], width=0.25,
+                        align="center", label=allocator,
+                        color=allocators[allocator]["color"])
+
+            plt.legend(loc="best")
+            plt.xticks(xa + 1/len(allocators)*2,
+                       ["malloc\n" + str(self.results[perm.workload]["malloc"]) + "\ncalls",
+                        "calloc\n" + str(self.results[perm.workload]["calloc"]) + "\ncalls",
+                        "realloc\n" + str(self.results[perm.workload]["realloc"]) + "\ncalls",
+                        "free\n" + str(self.results[perm.workload]["free"]) + "\ncalls"])
+            plt.ylabel("Durchschnittliche Zeit in cycles")
+            plt.title("Durchscnittliche Laufzeiten der API Funktionen")
+            plt.savefig(".".join([self.name, perm.workload, "apitimes", "png"]))
+            plt.clf()
+
+        # Memusage
+        rss_means = {allocator: {} for allocator in allocators}
+        for perm in self.iterate_args(args=args):
+            for i, allocator in enumerate(allocators):
+                d = [x["Max_RSS"] for x in self.results[allocator][perm]]
+                # data is in kB
+                rss_means[allocator][perm] = np.mean(d)/1000
+
+                plt.bar([i], rss_means[allocator][perm], label=allocator,
+                        color=allocators[allocator]["color"])
+
+            # add ideal rss
+            y_val = self.results[list(allocators.keys())[0]][perm][0]["Ideal_RSS"]/1000
+            plt.bar([len(allocators)], y_val, label="Ideal RSS")
+
+            plt.legend(loc="best")
+            plt.ylabel("Max RSS in MB")
+            plt.title("Maximal benötigter Speicher (VmHWM)")
+            plt.savefig(".".join([self.name, perm.workload, "rss", "png"]))
+            plt.clf()
+
+        # Tables
+        for perm in self.iterate_args(args=args):
+            # collect data
+            d = {allocator: {} for allocator in allocators}
+            for i, allocator in enumerate(allocators):
+                d[allocator]["time"] = [x["cputime"] for x in self.results[allocator][perm]]
+                d[allocator]["rss"] = [x["Max_RSS"] for x in self.results[allocator][perm]]
+
+            times = {allocator: np.mean(d[allocator]["time"]) for allocator in allocators}
+            tmin = min(times)
+            tmax = max(times)
+
+            rss = {allocator: np.mean(d[allocator]["rss"]) for allocator in allocators}
+            rssmin = min(rss)
+            rssmax = max(rss)
+
+            fname = ".".join([self.name, perm.workload, "table.tex"])
+            with open(fname, "w") as f:
+                print("\\begin{tabular}{| l | l | l |}", file=f)
+                print("& Zeit (ms) / $\\sigma$ (\\%) & VmHWM (KB) / $\\sigma$ (\\%) \\\\", file=f)
+                print("\\hline", file=f)
+
+                for allocator in allocators:
+                    print(allocator, end=" & ", file=f)
+
+                    s = "\\textcolor{{{}}}{{{}}} / {}"
+
+                    t = d[allocator]["time"]
+                    m = times[allocator]
+                    if m == tmin:
+                        color = "green"
+                    elif m == tmax:
+                        color = "red"
+                    else:
+                        color = "black"
+                    print(s.format(color, m, np.std(t)/m), end=" & ", file=f)
+
+                    t = d[allocator]["rss"]
+                    m = rss[allocator]
+                    if m == rssmin:
+                        color = "green"
+                    elif m == rssmax:
+                        color = "red"
+                    else:
+                        color = "black"
+                    print(s.format(color, m, np.std(t)/m if m else 0), "\\\\", file=f)
+
+                print("\end{tabular}", file=f)
+
+        # Create summary similar to DJ's at
+        # https://sourceware.org/ml/libc-alpha/2017-01/msg00452.html
+        with open(self.name + "_plain.txt", "w") as f:
+            # Absolutes
+            fmt = "{:<20} {:>15} {:>7} {:>7} {:>7} {:>7} {:>7}"
+            for i, allocator in enumerate(allocators):
+                print("{0} {1} {0}".format("-" * 10, allocator), file=f)
+                print(fmt.format("Workload", "Total", "malloc", "calloc",
+                                 "realloc", "free", "RSS"), file=f)
+
+                for perm in self.iterate_args(args=args):
+                    cycles = cycles_means[allocator][perm]
+                    times = [int(t) for t in func_times_means[allocator][perm]]
+                    rss = int(rss_means[allocator][perm])
+                    print(fmt.format(perm.workload, cycles, times[0], times[1],
+                                     times[2], times[3], rss), file=f)
+
+                print(file=f)
+
+            # Changes. First allocator in allocators is the reference
+            fmt_changes = "{:<20} {:>14.0f}% {:>6.0f}% {:>6.0f}% {:>6.0f}% {:>6.0f}% {:>6.0f}%"
+            for i, allocator in enumerate(list(allocators)[1:]):
+                print("{0} Changes {1} {0}".format("-" * 10, allocator), file=f)
+                print(fmt.format("Workload", "Total", "malloc", "calloc",
+                                 "realloc", "free", "RSS"), file=f)
+
+                ref_alloc = list(allocators)[0]
+                cycles_change_means = []
+                times_change_means = []
+                rss_change_means = []
+                for perm in self.iterate_args(args=args):
+
+                    normal_cycles = cycles_means[ref_alloc][perm]
+                    if normal_cycles:
+                        cycles = np.round(cycles_means[allocator][perm] / normal_cycles * 100)
+                    else:
+                        cycles = 0
+                    cycles_change_means.append(cycles)
+
+                    normal_times = func_times_means[ref_alloc][perm]
+                    times = [0, 0, 0, 0]
+                    for i in range(0, len(times)):
+                        t = func_times_means[allocator][perm][i]
+                        nt = normal_times[i]
+                        if nt != 0:
+                            times[i] = np.round(t/nt * 100)
+                    times_change_means.append(times)
+
+                    normal_rss = rss_means[ref_alloc][perm]
+                    if normal_rss:
+                        rss = np.round(rss_means[allocator][perm] / normal_rss * 100)
+                    else:
+                        rss = 0
+                    rss_change_means.append(rss)
+
+                    print(fmt_changes.format(perm.workload, cycles, times[0],
+                                             times[1], times[2], times[3], rss),
+                          file=f)
+                print(file=f)
+                tmeans = [0,0,0,0]
+                for i in range(0, len(times)):
+                    tmeans[i] = np.mean([times[i] for times in times_change_means])
+                print(fmt_changes.format("Mean:", np.mean(cycles_change_means),
+                                         tmeans[0], tmeans[1], tmeans[2],
+                                         tmeans[3], np.mean(rss_change_means)),
+                      '\n', file=f)
+
+
+dj_trace = Benchmark_DJ_Trace()
diff --git a/src/benchmarks/falsesharing.py b/src/benchmarks/falsesharing.py
new file mode 100644
index 0000000..f6375a6
--- /dev/null
+++ b/src/benchmarks/falsesharing.py
@@ -0,0 +1,74 @@
+import matplotlib.pyplot as plt
+import numpy as np
+import re
+
+from src.benchmark import Benchmark
+
+time_re = re.compile("^Time elapsed = (?P<time>\d*\.\d*) seconds.$")
+
+
+class Benchmark_Falsesharing(Benchmark):
+    def __init__(self):
+        self.name = "falsesharing"
+        self.descrition = """This benchmarks makes small allocations and writes
+                             to them multiple times. If the allocated objects
+                             are on the same cache line the writes will be
+                             expensive because of cache thrashing."""
+
+        self.cmd = "cache-{bench}{binary_suffix} {threads} 100 8 1000000"
+
+        self.args = {
+                        "bench": ["thrash", "scratch"],
+                        "threads": Benchmark.scale_threads_for_cpus(2)
+                    }
+
+        self.requirements = ["cache-thrash", "cache-scratch"]
+        super().__init__()
+
+    def process_output(self, result, stdout, stderr, allocator, perm, verbose):
+        result["time"] = time_re.match(stdout).group("time")
+
+    def summary(self):
+        # Speedup thrash
+        args = self.results["args"]
+        nthreads = args["threads"]
+        allocators = self.results["allocators"]
+
+        for bench in self.results["args"]["bench"]:
+            for allocator in allocators:
+                y_vals = []
+
+                single_threaded_perm = self.Perm(bench=bench, threads=1)
+                single_threaded = np.mean([float(m["time"])
+                                          for m in self.results[allocator][single_threaded_perm]])
+
+                for perm in self.iterate_args_fixed({"bench": bench}, args=args):
+
+                    d = [float(m["time"]) for m in self.results[allocator][perm]]
+
+                    y_vals.append(single_threaded / np.mean(d))
+
+                plt.plot(nthreads, y_vals, marker='.', linestyle='-',
+                         label=allocator, color=allocators[allocator]["color"])
+
+            plt.legend()
+            plt.xlabel("threads")
+            plt.ylabel("speedup")
+            plt.title(bench + " speedup")
+            plt.savefig(self.name + "." + bench + ".png")
+            plt.clf()
+
+        self.plot_fixed_arg("({L1-dcache-load-misses}/{L1-dcache-loads})*100",
+                            ylabel="'l1 cache misses in %'",
+                            title="'cache misses: ' + arg + ' ' + str(arg_value)",
+                            filepostfix="l1-misses",
+                            fixed=["bench"])
+
+        self.plot_fixed_arg("({LLC-load-misses}/{LLC-loads})*100",
+                            ylabel="'l1 cache misses in %'",
+                            title="'LLC misses: ' + arg + ' ' + str(arg_value)",
+                            filepostfix="llc-misses",
+                            fixed=["bench"])
+
+
+falsesharing = Benchmark_Falsesharing()
diff --git a/src/benchmarks/larson.py b/src/benchmarks/larson.py
new file mode 100644
index 0000000..e070b7b
--- /dev/null
+++ b/src/benchmarks/larson.py
@@ -0,0 +1,46 @@
+import re
+
+from src.benchmark import Benchmark
+
+throughput_re = re.compile("^Throughput =\s*(?P<throughput>\d+) operations per second.$")
+
+
+class Benchmark_Larson(Benchmark):
+    def __init__(self):
+        self.name = "larson"
+        self.descrition = """This benchmark is courtesy of Paul Larson at
+                             Microsoft Research. It simulates a server: each
+                             thread allocates and deallocates objects, and then
+                             transfers some objects (randomly selected) to
+                             other threads to be freed."""
+
+        self.cmd = "larson{binary_suffix} 1 8 {maxsize} 1000 50000 1 {threads}"
+
+        self.args = {
+                        "maxsize": [8, 32, 64, 128, 256, 512, 1024],
+                        "threads": Benchmark.scale_threads_for_cpus(2)
+                    }
+
+        self.requirements = ["larson"]
+        super().__init__()
+
+    def process_output(self, result, stdout, stderr, target, perm, verbose):
+        for l in stdout.splitlines():
+            res = throughput_re.match(l)
+            if res:
+                result["throughput"] = int(res.group("throughput"))
+                return
+
+    def summary(self):
+        # Plot threads->throughput and maxsize->throughput
+        self.plot_fixed_arg("{throughput}/1000000",
+                            ylabel="'MOPS/s'",
+                            title="'Larson: ' + arg + ' ' + str(arg_value)",
+                            filepostfix="throughput")
+
+        self.plot_fixed_arg("({L1-dcache-load-misses}/{L1-dcache-loads})*100",
+                            ylabel="'l1 cache misses in %'",
+                            title="'Larson cache misses: ' + arg + ' ' + str(arg_value)",
+                            filepostfix="cachemisses")
+
+larson = Benchmark_Larson()
diff --git a/src/benchmarks/loop.py b/src/benchmarks/loop.py
new file mode 100644
index 0000000..4683549
--- /dev/null
+++ b/src/benchmarks/loop.py
@@ -0,0 +1,36 @@
+from src.benchmark import Benchmark
+
+
+class Benchmark_Loop(Benchmark):
+    def __init__(self):
+        self.name = "loop"
+        self.descrition = """This benchmark allocates and frees n blocks in t concurrent
+                             threads."""
+
+        self.cmd = "loop{binary_suffix} {nthreads} 1000000 {maxsize}"
+
+        self.args = {"maxsize":  [2 ** x for x in range(6, 16)],
+                     "nthreads": Benchmark.scale_threads_for_cpus(2)}
+
+        self.requirements = ["loop"]
+        super().__init__()
+
+    def summary(self):
+        # Speed
+        self.plot_fixed_arg("perm.nthreads / ({task-clock}/1000)",
+                            ylabel='"MOPS/cpu-second"',
+                            title='"Loop: " + arg + " " + str(arg_value)',
+                            filepostfix="time")
+
+        # L1 cache misses
+        self.plot_fixed_arg("({L1-dcache-load-misses}/{L1-dcache-loads})*100",
+                            ylabel='"L1 misses in %"',
+                            title='"Loop l1 cache misses: " + arg + " " + str(arg_value)',
+                            filepostfix="l1misses")
+
+        # Speed Matrix
+        self.write_best_doublearg_tex_table("perm.nthreads / ({task-clock}/1000)",
+                                            filepostfix="memusage.matrix")
+
+
+loop = Benchmark_Loop()
diff --git a/src/benchmarks/mysql.py b/src/benchmarks/mysql.py
new file mode 100644
index 0000000..e3f1f0f
--- /dev/null
+++ b/src/benchmarks/mysql.py
@@ -0,0 +1,285 @@
+import atexit
+import copy
+import matplotlib.pyplot as plt
+import multiprocessing
+import numpy as np
+import os
+import re
+import shutil
+import subprocess
+from subprocess import PIPE
+import sys
+from time import sleep
+
+from src.globalvars import allocators
+from src.benchmark import Benchmark
+from src.util import *
+
+cwd = os.getcwd()
+
+prepare_cmd = ("sysbench oltp_read_only --db-driver=mysql --mysql-user=root "
+               "--mysql-socket=" + cwd + "/mysql_test/socket --tables=5 "
+               "--table-size=1000000 prepare").split()
+
+cmd = ("sysbench oltp_read_only --threads={nthreads} --time=60 --tables=5 "
+       "--db-driver=mysql --mysql-user=root --mysql-socket="
+       + cwd + "/mysql_test/socket run")
+
+server_cmd = ("{0} -h {2}/mysql_test --socket={2}/mysql_test/socket "
+              "--max-connections={1} "
+              "--secure-file-priv=").format(shutil.which("mysqld"),
+                                            multiprocessing.cpu_count(), cwd).split()
+
+
+class Benchmark_MYSQL(Benchmark):
+    def __init__(self):
+        self.name = "mysql"
+        self.descrition = """See sysbench documentation."""
+
+        # mysqld fails with hoard somehow
+        self.allocators = copy.copy(allocators)
+        if "hoard" in self.allocators:
+            del(self.allocators["hoard"])
+
+        self.args = {"nthreads": Benchmark.scale_threads_for_cpus(1)}
+        self.cmd = cmd
+        self.measure_cmd = ""
+
+        self.requirements = ["mysqld", "sysbench"]
+
+        atexit.register(self.terminate_server)
+
+        super().__init__()
+
+    def start_and_wait_for_server(self, cmd_prefix=""):
+        actual_cmd = cmd_prefix.split() + server_cmd
+        print_info("Starting server with:", actual_cmd)
+
+        self.server = subprocess.Popen(actual_cmd, stdout=PIPE, stderr=PIPE,
+                                       universal_newlines=True)
+        # TODO make sure server comes up !
+        sleep(10)
+        if self.server.poll() is not None:
+            print_debug("cmd_prefix:", cmd_prefix, file=sys.stderr)
+            print_debug("Stderr:", self.server.stderr, file=sys.stderr)
+            return False
+
+        return True
+
+    def terminate_server(self):
+        if hasattr(self, "server"):
+            if self.server.poll() is None:
+                print_info("Terminating mysql server")
+                self.server.terminate()
+
+            for i in range(0,10):
+                sleep(1)
+                if self.server.poll() is not None:
+                    return
+
+            print_info("Killing still running mysql server")
+            self.server.kill()
+            self.server.wait()
+
+    def prepare(self):
+        super().prepare()
+
+        # Setup Test Environment
+        if not os.path.exists("mysql_test"):
+            print_status("Prepare mysqld directory and database")
+            os.makedirs("mysql_test")
+
+            # Init database
+            if b"MariaDB" in subprocess.run(["mysqld", "--version"],
+                                            stdout=PIPE).stdout:
+                init_db_cmd = ["mysql_install_db", "--basedir=/usr",
+                               "--datadir="+cwd+"/mysql_test"]
+                print_info2("MariaDB detected")
+            else:
+                init_db_cmd = ["mysqld", "-h", cwd+"/mysql_test",
+                               "--initialize-insecure"]
+                print_info2("Oracle MySQL detected")
+
+            p = subprocess.run(init_db_cmd, stdout=PIPE, stderr=PIPE)
+
+            if not p.returncode == 0:
+                print_debug(p.stderr, file=sys.stderr)
+                raise Exception("Creating test DB failed with:", p.returncode)
+
+            if not self.start_and_wait_for_server():
+                raise Exception("Starting mysql server failed with {}".format(self.server.returncode))
+
+            # Create sbtest TABLE
+            p = subprocess.run(("mysql -u root -S "+cwd+"/mysql_test/socket").split(" "),
+                               input=b"CREATE DATABASE sbtest;\n",
+                               stdout=PIPE, stderr=PIPE)
+
+            if not p.returncode == 0:
+                print_debug("Stderr:", p.stderr, file=sys.stderr)
+                self.terminate_server()
+                raise Exception("Creating test tables failed with:", p.returncode)
+
+            print_status("Prepare test tables ...")
+            ret = True
+            p = subprocess.run(prepare_cmd, stdout=PIPE, stderr=PIPE)
+            if p.returncode != 0:
+                print_debug("Stdout:", p.stdout, file=sys.stderr)
+                print_debug("Stderr:", p.stderr, file=sys.stderr)
+                self.terminate_server()
+                raise Exception("Preparing test tables failed with:", p.returncode)
+
+            self.terminate_server()
+
+    def cleanup(self):
+        if os.path.exists("mysql_test"):
+            print_status("Delete mysqld directory")
+            shutil.rmtree("mysql_test", ignore_errors=True)
+
+    def preallocator_hook(self, allocator, run, verbose):
+        if not self.start_and_wait_for_server(cmd_prefix=allocator[1]["cmd_prefix"]):
+            print_debug(allocator[1]["cmd_prefix"], file=sys.stderr)
+            raise Exception("Starting mysql server for {} failed with".format(allocator[0], self.server.returncode))
+
+    def postallocator_hook(self, allocator, run, verbose):
+        self.terminate_server()
+
+    def process_output(self, result, stdout, stderr, allocator, perm, verbose):
+        result["transactions"] = re.search("transactions:\s*(\d*)", stdout).group(1)
+        result["queries"] = re.search("queries:\s*(\d*)", stdout).group(1)
+        # Latency
+        result["min"] = re.search("min:\s*(\d*.\d*)", stdout).group(1)
+        result["avg"] = re.search("avg:\s*(\d*.\d*)", stdout).group(1)
+        result["max"] = re.search("max:\s*(\d*.\d*)", stdout).group(1)
+
+        with open("/proc/"+str(self.server.pid)+"/status", "r") as f:
+            for l in f.readlines():
+                if l.startswith("VmHWM:"):
+                    result["rssmax"] = int(l.replace("VmHWM:", "").strip().split()[0])
+                    break
+
+    def summary(self):
+        allocators = self.results["allocators"]
+        args = self.results["args"]
+
+        # linear plot
+        self.plot_single_arg("{transactions}",
+                             xlabel='"threads"',
+                             ylabel='"transactions"',
+                             title='"sysbench oltp read only"',
+                             filepostfix="l.ro")
+
+
+        # linear plot
+        ref_alloc = list(allocators)[0]
+        self.plot_single_arg("{transactions}",
+                             xlabel='"threads"',
+                             ylabel='"transactions scaled at " + scale',
+                             title='"sysbench oltp read only"',
+                             filepostfix="norm.l.ro",
+                             scale=ref_alloc)
+
+        # bar plot
+        for i, allocator in enumerate(allocators):
+            y_vals = []
+            for perm in self.iterate_args(args=self.results["args"]):
+                d = [int(m["transactions"]) for m in self.results[allocator][perm]]
+                y_vals.append(np.mean(d))
+            x_vals = [x-i/8 for x in range(1, len(y_vals) + 1)]
+            plt.bar(x_vals, y_vals, width=0.2, label=allocator, align="center",
+                    color=allocators[allocator]["color"])
+
+        plt.legend()
+        plt.xlabel("threads")
+        plt.xticks(range(1, len(y_vals) + 1), self.results["args"]["nthreads"])
+        plt.ylabel("transactions")
+        plt.title("sysbench oltp read only")
+        plt.savefig(self.name + ".b.ro.png")
+        plt.clf()
+
+        # normalized bar plot
+        norm_y_vals = []
+        for perm in self.iterate_args(args=self.results["args"]):
+            d = [int(m["transactions"]) for m in self.results[ref_alloc][perm]]
+            norm_y_vals.append(np.mean(d))
+
+        nallocs = len(allocators)
+        x_vals = [x for x in range(1, len(norm_y_vals)*nallocs + 1, nallocs)]
+        plt.bar(x_vals, [1]*len(norm_y_vals), width=0.7, label=ref_alloc,
+                align="center", color=allocators[ref_alloc]["color"])
+
+        for i, allocator in enumerate(list(allocators)[1:]):
+            y_vals = []
+            for y, perm in enumerate(self.iterate_args(args=self.results["args"])):
+                d = [int(m["transactions"]) for m in self.results[allocator][perm]]
+                y_vals.append(np.mean(d) / norm_y_vals[y])
+
+            x_vals = [x+i+1 for x in range(1, len(norm_y_vals)*nallocs + 1, nallocs)]
+
+            plt.bar(x_vals, y_vals, width=0.7, label=allocator, align="center",
+                    color=allocators[allocator]["color"])
+
+        plt.legend()
+        plt.xlabel("threads")
+        plt.xticks(range(1, len(norm_y_vals)*nallocs + 1, nallocs), self.results["args"]["nthreads"])
+        plt.ylabel("transactions normalized")
+        plt.title("sysbench oltp read only")
+        plt.savefig(self.name + ".norm.b.ro.png")
+        plt.clf()
+
+        # Memusage
+        self.plot_single_arg("{rssmax}",
+                             xlabel='"threads"',
+                             ylabel='"VmHWM in kB"',
+                             title='"Memusage sysbench oltp read only"',
+                             filepostfix="ro.mem")
+
+        # Colored latex table showing transactions count
+        d = {allocator: {} for allocator in allocators}
+        for perm in self.iterate_args(args=args):
+            for i, allocator in enumerate(allocators):
+                t = [float(x["transactions"]) for x in self.results[allocator][perm]]
+                m = np.mean(t)
+                s = np.std(t)/m
+                d[allocator][perm] = {"mean": m, "std": s}
+
+        mins = {}
+        maxs = {}
+        for perm in self.iterate_args(args=args):
+            cmax = None
+            cmin = None
+            for i, allocator in enumerate(allocators):
+                m = d[allocator][perm]["mean"]
+                if not cmax or m > cmax:
+                    cmax = m
+                if not cmin or m < cmin:
+                    cmin = m
+            maxs[perm] = cmax
+            mins[perm] = cmin
+
+        fname = ".".join([self.name, "transactions.tex"])
+        headers = [perm.nthreads for perm in self.iterate_args(args=args)]
+        with open(fname, "w") as f:
+            print("\\begin{tabular}{| l" + " l"*len(headers) + " |}", file=f)
+            print("Fäden / Allokator ", end=" ", file=f)
+            for head in headers:
+                print("& {}".format(head), end=" ", file=f)
+            print("\\\\\n\\hline", file=f)
+
+            for allocator in allocators:
+                print(allocator, end=" ", file=f)
+                for perm in self.iterate_args(args=args):
+                    m = d[allocator][perm]["mean"]
+                    s = "& \\textcolor{{{}}}{{{:.3f}}}"
+                    if m == maxs[perm]:
+                        color = "green"
+                    elif m == mins[perm]:
+                        color = "red"
+                    else:
+                        color = "black"
+                    print(s.format(color, m), end=" ", file=f)
+                print("\\\\", file=f)
+
+            print("\end{tabular}", file=f)
+
+
+mysql = Benchmark_MYSQL()
diff --git a/src/dj_trace.py b/src/dj_trace.py
deleted file mode 100644
index 062f34c..0000000
--- a/src/dj_trace.py
+++ /dev/null
@@ -1,357 +0,0 @@
-import matplotlib.pyplot as plt
-import numpy as np
-import os
-from urllib.request import urlretrieve
-import sys
-import re
-
-from src.benchmark import Benchmark
-from src.util import print_status
-
-comma_sep_number_re = "(?:\d*(?:,\d*)?)*"
-rss_re = "(?P<rss>" + comma_sep_number_re + ")"
-time_re = "(?P<time>" + comma_sep_number_re + ")"
-
-cycles_re = re.compile("^{} cycles$".format(time_re))
-cpu_time_re = re.compile("^{} usec across.*threads$".format(time_re))
-
-max_rss_re = re.compile("^{} Kb Max RSS".format(rss_re))
-ideal_rss_re = re.compile("^{} Kb Max Ideal RSS".format(rss_re))
-
-malloc_re = re.compile("^Avg malloc time:\s*{} in.*calls$".format(time_re))
-calloc_re = re.compile("^Avg calloc time:\s*{} in.*calls$".format(time_re))
-realloc_re = re.compile("^Avg realloc time:\s*{} in.*calls$".format(time_re))
-free_re = re.compile("^Avg free time:\s*{} in.*calls$".format(time_re))
-
-
-class Benchmark_DJ_Trace(Benchmark):
-    def __init__(self):
-        self.name = "dj_trace"
-        self.descrition = """This benchmark uses the workload simulator written
-                             by DJ Delorie to simulate workloads provided by
-                             him under https://delorie.com/malloc. Those
-                             workloads are generated from traces of real
-                             aplications and are also used by delorie to
-                             measure improvements in the glibc allocator."""
-
-        self.cmd = "trace_run{binary_suffix} dj_workloads/{workload}.wl"
-        self.measure_cmd = ""
-
-        self.args = {
-                        "workload": [
-                                        "389-ds-2",
-                                        "dj",
-                                        "dj2",
-                                        "mt_test_one_alloc",
-                                        "oocalc",
-                                        "qemu-virtio",
-                                        "qemu-win7",
-                                        "proprietary-1",
-                                        "proprietary-2",
-                                      ]
-                    }
-        self.results = {
-                        "389-ds-2": {
-                            "malloc": 170500018, "calloc": 161787184,
-                            "realloc": 404134, "free": 314856324,
-                            "threads": 41},
-                        "dj": {
-                            "malloc": 2000000, "calloc": 200, "realloc": 0,
-                            "free": 2003140, "threads": 201},
-                        "dj2": {
-                            "malloc": 29263321, "calloc": 3798404,
-                            "realloc": 122956, "free": 32709054,
-                            "threads": 36},
-                        "mt_test_one_alloc": {
-                            "malloc": 524290, "calloc": 1, "realloc": 0,
-                            "free": 594788, "threads": 2},
-                        "oocalc": {
-                            "malloc": 6731734, "calloc": 38421,
-                            "realloc": 14108, "free": 6826686, "threads": 88},
-                        "qemu-virtio": {
-                            "malloc": 1772163, "calloc": 146634,
-                            "realloc": 59813, "free": 1954732, "threads": 3},
-                        "qemu-win7": {
-                            "malloc": 980904, "calloc": 225420,
-                            "realloc": 89880, "free": 1347825, "threads": 6},
-                        "proprietary-1": {
-                            "malloc": 316032131, "calloc": 5642, "realloc": 84,
-                            "free": 319919727, "threads": 20},
-                        "proprietary-2": {
-                            "malloc": 9753948, "calloc": 4693,
-                            "realloc": 117, "free": 10099261, "threads": 19},
-                        }
-
-        self.requirements = ["trace_run"]
-        super().__init__()
-
-    def prepare(self):
-        super().prepare()
-
-        def reporthook(blocknum, blocksize, totalsize):
-            readsofar = blocknum * blocksize
-            if totalsize > 0:
-                percent = readsofar * 1e2 / totalsize
-                s = "\r%5.1f%% %*d / %d" % (
-                    percent, len(str(totalsize)), readsofar, totalsize)
-                sys.stderr.write(s)
-            else:  # total size is unknown
-                sys.stderr.write("\rdownloaded %d" % (readsofar,))
-
-        if not os.path.isdir("dj_workloads"):
-            os.mkdir("dj_workloads")
-
-        download_all = None
-        wl_sizes = {"dj": "14M", "oocalc": "65M", "mt_test_one_alloc": "5.7M",
-                    "proprietary-1": "2.8G", "qemu-virtio": "34M",
-                    "proprietary-2": "92M", "qemu-win7": "23M",
-                    "389-ds-2": "3.4G", "dj2": "294M"}
-
-        for wl in self.args["workload"]:
-            file_name = wl + ".wl"
-            file_path = os.path.join("dj_workloads", file_name)
-            if not os.path.isfile(file_path):
-                if download_all == None:
-                    choice = input(("Download all missing workloads (upto 6.7GB)"
-                                          " [Y/n/x] "))
-                    if choice == "x":
-                        break
-                    else:
-                        download_all = choice in ['', 'Y', 'y']
-
-                if (not download_all and
-                    input("want to download {} ({}) [Y/n] ".format(wl, wl_sizes[wl])) not in ['', 'Y', 'y']):
-                    continue
-
-                if download_all:
-                    print_status("downloading {} ({}) ...".format(wl, wl_sizes[wl]))
-
-                url = "http://www.delorie.com/malloc/" + file_name
-                urlretrieve(url, file_path, reporthook)
-                sys.stderr.write("\n")
-
-        available_workloads = []
-        for wl in self.args["workload"]:
-            file_name = wl + ".wl"
-            file_path = os.path.join("dj_workloads", file_name)
-            if os.path.isfile(file_path):
-                available_workloads.append(wl)
-
-        if len(available_workloads) > 0:
-            self.args["workload"] = available_workloads
-            return True
-        
-        return False
-
-    def process_output(self, result, stdout, stderr, allocator, perm, verbose):
-        def to_int(s):
-            return int(s.replace(',', ""))
-
-        regexs = {7: malloc_re, 8: calloc_re, 9: realloc_re, 10: free_re}
-        functions = {7: "malloc", 8: "calloc", 9: "realloc", 10: "free"}
-        for i, l in enumerate(stdout.splitlines()):
-            if i == 0:
-                result["cycles"] = to_int(cycles_re.match(l).group("time"))
-            elif i == 2:
-                result["cputime"] = to_int(cpu_time_re.match(l).group("time"))
-            elif i == 3:
-                result["Max_RSS"] = to_int(max_rss_re.match(l).group("rss"))
-            elif i == 4:
-                result["Ideal_RSS"] = to_int(ideal_rss_re.match(l).group("rss"))
-            elif i in [7, 8, 9, 10]:
-                res = regexs[i].match(l)
-                fname = functions[i]
-                result["avg_" + fname] = to_int(res.group("time"))
-
-    def summary(self):
-        args = self.results["args"]
-        allocators = self.results["allocators"]
-
-        cpu_time_means = {allocator: {} for allocator in allocators}
-        cycles_means = {allocator: {} for allocator in allocators}
-        for perm in self.iterate_args(args=args):
-            for i, allocator in enumerate(allocators):
-                d = [x["cputime"] for x in self.results[allocator][perm]]
-                # data is in milliseconds
-                cpu_time_means[allocator][perm] = int(np.mean(d)/1000)
-
-                d = [x["cycles"] for x in self.results[allocator][perm]]
-                cycles_means[allocator][perm] = int(np.mean(d))
-
-                plt.bar([i], cpu_time_means[allocator][perm], label=allocator,
-                        color=allocators[allocator]["color"])
-
-            plt.legend(loc="best")
-            plt.ylabel("Zeit in ms")
-            plt.title("Gesamte Laufzeit")
-            plt.savefig(".".join([self.name, perm.workload, "runtime", "png"]))
-            plt.clf()
-
-        # Function Times
-        func_times_means = {allocator: {} for allocator in allocators}
-        xa = np.arange(0, 6, 1.5)
-        for perm in self.iterate_args(args=args):
-            for i, allocator in enumerate(allocators):
-                x_vals = [x+i/len(allocators) for x in xa]
-
-                func_times_means[allocator][perm] = [0,0,0,0]
-
-                func_times_means[allocator][perm][0] = np.mean([x["avg_malloc"] for x in self.results[allocator][perm]])
-                func_times_means[allocator][perm][1] = np.mean([x["avg_calloc"] for x in self.results[allocator][perm]])
-                func_times_means[allocator][perm][2] = np.mean([x["avg_realloc"] for x in self.results[allocator][perm]])
-                func_times_means[allocator][perm][3] = np.mean([x["avg_free"] for x in self.results[allocator][perm]])
-
-                plt.bar(x_vals, func_times_means[allocator][perm], width=0.25,
-                        align="center", label=allocator,
-                        color=allocators[allocator]["color"])
-
-            plt.legend(loc="best")
-            plt.xticks(xa + 1/len(allocators)*2,
-                       ["malloc\n" + str(self.results[perm.workload]["malloc"]) + "\ncalls",
-                        "calloc\n" + str(self.results[perm.workload]["calloc"]) + "\ncalls",
-                        "realloc\n" + str(self.results[perm.workload]["realloc"]) + "\ncalls",
-                        "free\n" + str(self.results[perm.workload]["free"]) + "\ncalls"])
-            plt.ylabel("Durchschnittliche Zeit in cycles")
-            plt.title("Durchscnittliche Laufzeiten der API Funktionen")
-            plt.savefig(".".join([self.name, perm.workload, "apitimes", "png"]))
-            plt.clf()
-
-        # Memusage
-        rss_means = {allocator: {} for allocator in allocators}
-        for perm in self.iterate_args(args=args):
-            for i, allocator in enumerate(allocators):
-                d = [x["Max_RSS"] for x in self.results[allocator][perm]]
-                # data is in kB
-                rss_means[allocator][perm] = np.mean(d)/1000
-
-                plt.bar([i], rss_means[allocator][perm], label=allocator,
-                        color=allocators[allocator]["color"])
-
-            # add ideal rss
-            y_val = self.results[list(allocators.keys())[0]][perm][0]["Ideal_RSS"]/1000
-            plt.bar([len(allocators)], y_val, label="Ideal RSS")
-
-            plt.legend(loc="best")
-            plt.ylabel("Max RSS in MB")
-            plt.title("Maximal benötigter Speicher (VmHWM)")
-            plt.savefig(".".join([self.name, perm.workload, "rss", "png"]))
-            plt.clf()
-
-        # Tables
-        for perm in self.iterate_args(args=args):
-            # collect data
-            d = {allocator: {} for allocator in allocators}
-            for i, allocator in enumerate(allocators):
-                d[allocator]["time"] = [x["cputime"] for x in self.results[allocator][perm]]
-                d[allocator]["rss"] = [x["Max_RSS"] for x in self.results[allocator][perm]]
-
-            times = {allocator: np.mean(d[allocator]["time"]) for allocator in allocators}
-            tmin = min(times)
-            tmax = max(times)
-
-            rss = {allocator: np.mean(d[allocator]["rss"]) for allocator in allocators}
-            rssmin = min(rss)
-            rssmax = max(rss)
-
-            fname = ".".join([self.name, perm.workload, "table.tex"])
-            with open(fname, "w") as f:
-                print("\\begin{tabular}{| l | l | l |}", file=f)
-                print("& Zeit (ms) / $\\sigma$ (\\%) & VmHWM (KB) / $\\sigma$ (\\%) \\\\", file=f)
-                print("\\hline", file=f)
-
-                for allocator in allocators:
-                    print(allocator, end=" & ", file=f)
-
-                    s = "\\textcolor{{{}}}{{{}}} / {}"
-
-                    t = d[allocator]["time"]
-                    m = times[allocator]
-                    if m == tmin:
-                        color = "green"
-                    elif m == tmax:
-                        color = "red"
-                    else:
-                        color = "black"
-                    print(s.format(color, m, np.std(t)/m), end=" & ", file=f)
-
-                    t = d[allocator]["rss"]
-                    m = rss[allocator]
-                    if m == rssmin:
-                        color = "green"
-                    elif m == rssmax:
-                        color = "red"
-                    else:
-                        color = "black"
-                    print(s.format(color, m, np.std(t)/m if m else 0), "\\\\", file=f)
-
-                print("\end{tabular}", file=f)
-
-        # Create summary similar to DJ's at
-        # https://sourceware.org/ml/libc-alpha/2017-01/msg00452.html
-        with open(self.name + "_plain.txt", "w") as f:
-            # Absolutes
-            fmt = "{:<20} {:>15} {:>7} {:>7} {:>7} {:>7} {:>7}"
-            for i, allocator in enumerate(allocators):
-                print("{0} {1} {0}".format("-" * 10, allocator), file=f)
-                print(fmt.format("Workload", "Total", "malloc", "calloc",
-                                 "realloc", "free", "RSS"), file=f)
-
-                for perm in self.iterate_args(args=args):
-                    cycles = cycles_means[allocator][perm]
-                    times = [int(t) for t in func_times_means[allocator][perm]]
-                    rss = int(rss_means[allocator][perm])
-                    print(fmt.format(perm.workload, cycles, times[0], times[1],
-                                     times[2], times[3], rss), file=f)
-
-                print(file=f)
-
-            # Changes. First allocator in allocators is the reference
-            fmt_changes = "{:<20} {:>14.0f}% {:>6.0f}% {:>6.0f}% {:>6.0f}% {:>6.0f}% {:>6.0f}%"
-            for i, allocator in enumerate(list(allocators)[1:]):
-                print("{0} Changes {1} {0}".format("-" * 10, allocator), file=f)
-                print(fmt.format("Workload", "Total", "malloc", "calloc",
-                                 "realloc", "free", "RSS"), file=f)
-
-                ref_alloc = list(allocators)[0]
-                cycles_change_means = []
-                times_change_means = []
-                rss_change_means = []
-                for perm in self.iterate_args(args=args):
-
-                    normal_cycles = cycles_means[ref_alloc][perm]
-                    if normal_cycles:
-                        cycles = np.round(cycles_means[allocator][perm] / normal_cycles * 100)
-                    else:
-                        cycles = 0
-                    cycles_change_means.append(cycles)
-
-                    normal_times = func_times_means[ref_alloc][perm]
-                    times = [0, 0, 0, 0]
-                    for i in range(0, len(times)):
-                        t = func_times_means[allocator][perm][i]
-                        nt = normal_times[i]
-                        if nt != 0:
-                            times[i] = np.round(t/nt * 100)
-                    times_change_means.append(times)
-
-                    normal_rss = rss_means[ref_alloc][perm]
-                    if normal_rss:
-                        rss = np.round(rss_means[allocator][perm] / normal_rss * 100)
-                    else:
-                        rss = 0
-                    rss_change_means.append(rss)
-
-                    print(fmt_changes.format(perm.workload, cycles, times[0],
-                                             times[1], times[2], times[3], rss),
-                          file=f)
-                print(file=f)
-                tmeans = [0,0,0,0]
-                for i in range(0, len(times)):
-                    tmeans[i] = np.mean([times[i] for times in times_change_means])
-                print(fmt_changes.format("Mean:", np.mean(cycles_change_means),
-                                         tmeans[0], tmeans[1], tmeans[2],
-                                         tmeans[3], np.mean(rss_change_means)),
-                      '\n', file=f)
-
-
-dj_trace = Benchmark_DJ_Trace()
diff --git a/src/falsesharing.py b/src/falsesharing.py
deleted file mode 100644
index f6375a6..0000000
--- a/src/falsesharing.py
+++ /dev/null
@@ -1,74 +0,0 @@
-import matplotlib.pyplot as plt
-import numpy as np
-import re
-
-from src.benchmark import Benchmark
-
-time_re = re.compile("^Time elapsed = (?P<time>\d*\.\d*) seconds.$")
-
-
-class Benchmark_Falsesharing(Benchmark):
-    def __init__(self):
-        self.name = "falsesharing"
-        self.descrition = """This benchmarks makes small allocations and writes
-                             to them multiple times. If the allocated objects
-                             are on the same cache line the writes will be
-                             expensive because of cache thrashing."""
-
-        self.cmd = "cache-{bench}{binary_suffix} {threads} 100 8 1000000"
-
-        self.args = {
-                        "bench": ["thrash", "scratch"],
-                        "threads": Benchmark.scale_threads_for_cpus(2)
-                    }
-
-        self.requirements = ["cache-thrash", "cache-scratch"]
-        super().__init__()
-
-    def process_output(self, result, stdout, stderr, allocator, perm, verbose):
-        result["time"] = time_re.match(stdout).group("time")
-
-    def summary(self):
-        # Speedup thrash
-        args = self.results["args"]
-        nthreads = args["threads"]
-        allocators = self.results["allocators"]
-
-        for bench in self.results["args"]["bench"]:
-            for allocator in allocators:
-                y_vals = []
-
-                single_threaded_perm = self.Perm(bench=bench, threads=1)
-                single_threaded = np.mean([float(m["time"])
-                                          for m in self.results[allocator][single_threaded_perm]])
-
-                for perm in self.iterate_args_fixed({"bench": bench}, args=args):
-
-                    d = [float(m["time"]) for m in self.results[allocator][perm]]
-
-                    y_vals.append(single_threaded / np.mean(d))
-
-                plt.plot(nthreads, y_vals, marker='.', linestyle='-',
-                         label=allocator, color=allocators[allocator]["color"])
-
-            plt.legend()
-            plt.xlabel("threads")
-            plt.ylabel("speedup")
-            plt.title(bench + " speedup")
-            plt.savefig(self.name + "." + bench + ".png")
-            plt.clf()
-
-        self.plot_fixed_arg("({L1-dcache-load-misses}/{L1-dcache-loads})*100",
-                            ylabel="'l1 cache misses in %'",
-                            title="'cache misses: ' + arg + ' ' + str(arg_value)",
-                            filepostfix="l1-misses",
-                            fixed=["bench"])
-
-        self.plot_fixed_arg("({LLC-load-misses}/{LLC-loads})*100",
-                            ylabel="'l1 cache misses in %'",
-                            title="'LLC misses: ' + arg + ' ' + str(arg_value)",
-                            filepostfix="llc-misses",
-                            fixed=["bench"])
-
-
-falsesharing = Benchmark_Falsesharing()
diff --git a/src/larson.py b/src/larson.py
deleted file mode 100644
index e070b7b..0000000
--- a/src/larson.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import re
-
-from src.benchmark import Benchmark
-
-throughput_re = re.compile("^Throughput =\s*(?P<throughput>\d+) operations per second.$")
-
-
-class Benchmark_Larson(Benchmark):
-    def __init__(self):
-        self.name = "larson"
-        self.descrition = """This benchmark is courtesy of Paul Larson at
-                             Microsoft Research. It simulates a server: each
-                             thread allocates and deallocates objects, and then
-                             transfers some objects (randomly selected) to
-                             other threads to be freed."""
-
-        self.cmd = "larson{binary_suffix} 1 8 {maxsize} 1000 50000 1 {threads}"
-
-        self.args = {
-                        "maxsize": [8, 32, 64, 128, 256, 512, 1024],
-                        "threads": Benchmark.scale_threads_for_cpus(2)
-                    }
-
-        self.requirements = ["larson"]
-        super().__init__()
-
-    def process_output(self, result, stdout, stderr, target, perm, verbose):
-        for l in stdout.splitlines():
-            res = throughput_re.match(l)
-            if res:
-                result["throughput"] = int(res.group("throughput"))
-                return
-
-    def summary(self):
-        # Plot threads->throughput and maxsize->throughput
-        self.plot_fixed_arg("{throughput}/1000000",
-                            ylabel="'MOPS/s'",
-                            title="'Larson: ' + arg + ' ' + str(arg_value)",
-                            filepostfix="throughput")
-
-        self.plot_fixed_arg("({L1-dcache-load-misses}/{L1-dcache-loads})*100",
-                            ylabel="'l1 cache misses in %'",
-                            title="'Larson cache misses: ' + arg + ' ' + str(arg_value)",
-                            filepostfix="cachemisses")
-
-larson = Benchmark_Larson()
diff --git a/src/loop.py b/src/loop.py
deleted file mode 100644
index a8459da..0000000
--- a/src/loop.py
+++ /dev/null
@@ -1,36 +0,0 @@
-from src.benchmark import Benchmark
-
-
-class Benchmark_Loop(Benchmark):
-    def __init__(self):
-        self.name = "loop"
-        self.descrition = """This benchmark allocates and frees n blocks in t concurrent
-                             threads."""
-
-        self.cmd = "loop{binary_suffix} {nthreads} 1000000 {maxsize}"
-
-        self.args = {"maxsize":  [2 ** x for x in range(6, 16)],
-                     "nthreads": Benchmark.scale_threads_for_cpus(2)}
-
-        self.requirements = ["loop"]
-        super().__init__()
-
-    def summary(self):
-        # Speed
-        self.plot_fixed_arg("perm.nthreads / (float({task-clock})/1000)",
-                            ylabel='"MOPS/cpu-second"',
-                            title='"Loop: " + arg + " " + str(arg_value)',
-                            filepostfix="time")
-
-        # L1 cache misses
-        self.plot_fixed_arg("({L1-dcache-load-misses}/{L1-dcache-loads})*100",
-                            ylabel='"L1 misses in %"',
-                            title='"Loop l1 cache misses: " + arg + " " + str(arg_value)',
-                            filepostfix="l1misses")
-
-        # Speed Matrix
-        self.write_best_doublearg_tex_table("perm.nthreads / (float({task-clock})/1000)",
-                                            filepostfix="memusage.matrix")
-
-
-loop = Benchmark_Loop()
diff --git a/src/mysql.py b/src/mysql.py
deleted file mode 100644
index e3f1f0f..0000000
--- a/src/mysql.py
+++ /dev/null
@@ -1,285 +0,0 @@
-import atexit
-import copy
-import matplotlib.pyplot as plt
-import multiprocessing
-import numpy as np
-import os
-import re
-import shutil
-import subprocess
-from subprocess import PIPE
-import sys
-from time import sleep
-
-from src.globalvars import allocators
-from src.benchmark import Benchmark
-from src.util import *
-
-cwd = os.getcwd()
-
-prepare_cmd = ("sysbench oltp_read_only --db-driver=mysql --mysql-user=root "
-               "--mysql-socket=" + cwd + "/mysql_test/socket --tables=5 "
-               "--table-size=1000000 prepare").split()
-
-cmd = ("sysbench oltp_read_only --threads={nthreads} --time=60 --tables=5 "
-       "--db-driver=mysql --mysql-user=root --mysql-socket="
-       + cwd + "/mysql_test/socket run")
-
-server_cmd = ("{0} -h {2}/mysql_test --socket={2}/mysql_test/socket "
-              "--max-connections={1} "
-              "--secure-file-priv=").format(shutil.which("mysqld"),
-                                            multiprocessing.cpu_count(), cwd).split()
-
-
-class Benchmark_MYSQL(Benchmark):
-    def __init__(self):
-        self.name = "mysql"
-        self.descrition = """See sysbench documentation."""
-
-        # mysqld fails with hoard somehow
-        self.allocators = copy.copy(allocators)
-        if "hoard" in self.allocators:
-            del(self.allocators["hoard"])
-
-        self.args = {"nthreads": Benchmark.scale_threads_for_cpus(1)}
-        self.cmd = cmd
-        self.measure_cmd = ""
-
-        self.requirements = ["mysqld", "sysbench"]
-
-        atexit.register(self.terminate_server)
-
-        super().__init__()
-
-    def start_and_wait_for_server(self, cmd_prefix=""):
-        actual_cmd = cmd_prefix.split() + server_cmd
-        print_info("Starting server with:", actual_cmd)
-
-        self.server = subprocess.Popen(actual_cmd, stdout=PIPE, stderr=PIPE,
-                                       universal_newlines=True)
-        # TODO make sure server comes up !
-        sleep(10)
-        if self.server.poll() is not None:
-            print_debug("cmd_prefix:", cmd_prefix, file=sys.stderr)
-            print_debug("Stderr:", self.server.stderr, file=sys.stderr)
-            return False
-
-        return True
-
-    def terminate_server(self):
-        if hasattr(self, "server"):
-            if self.server.poll() is None:
-                print_info("Terminating mysql server")
-                self.server.terminate()
-
-            for i in range(0,10):
-                sleep(1)
-                if self.server.poll() is not None:
-                    return
-
-            print_info("Killing still running mysql server")
-            self.server.kill()
-            self.server.wait()
-
-    def prepare(self):
-        super().prepare()
-
-        # Setup Test Environment
-        if not os.path.exists("mysql_test"):
-            print_status("Prepare mysqld directory and database")
-            os.makedirs("mysql_test")
-
-            # Init database
-            if b"MariaDB" in subprocess.run(["mysqld", "--version"],
-                                            stdout=PIPE).stdout:
-                init_db_cmd = ["mysql_install_db", "--basedir=/usr",
-                               "--datadir="+cwd+"/mysql_test"]
-                print_info2("MariaDB detected")
-            else:
-                init_db_cmd = ["mysqld", "-h", cwd+"/mysql_test",
-                               "--initialize-insecure"]
-                print_info2("Oracle MySQL detected")
-
-            p = subprocess.run(init_db_cmd, stdout=PIPE, stderr=PIPE)
-
-            if not p.returncode == 0:
-                print_debug(p.stderr, file=sys.stderr)
-                raise Exception("Creating test DB failed with:", p.returncode)
-
-            if not self.start_and_wait_for_server():
-                raise Exception("Starting mysql server failed with {}".format(self.server.returncode))
-
-            # Create sbtest TABLE
-            p = subprocess.run(("mysql -u root -S "+cwd+"/mysql_test/socket").split(" "),
-                               input=b"CREATE DATABASE sbtest;\n",
-                               stdout=PIPE, stderr=PIPE)
-
-            if not p.returncode == 0:
-                print_debug("Stderr:", p.stderr, file=sys.stderr)
-                self.terminate_server()
-                raise Exception("Creating test tables failed with:", p.returncode)
-
-            print_status("Prepare test tables ...")
-            ret = True
-            p = subprocess.run(prepare_cmd, stdout=PIPE, stderr=PIPE)
-            if p.returncode != 0:
-                print_debug("Stdout:", p.stdout, file=sys.stderr)
-                print_debug("Stderr:", p.stderr, file=sys.stderr)
-                self.terminate_server()
-                raise Exception("Preparing test tables failed with:", p.returncode)
-
-            self.terminate_server()
-
-    def cleanup(self):
-        if os.path.exists("mysql_test"):
-            print_status("Delete mysqld directory")
-            shutil.rmtree("mysql_test", ignore_errors=True)
-
-    def preallocator_hook(self, allocator, run, verbose):
-        if not self.start_and_wait_for_server(cmd_prefix=allocator[1]["cmd_prefix"]):
-            print_debug(allocator[1]["cmd_prefix"], file=sys.stderr)
-            raise Exception("Starting mysql server for {} failed with".format(allocator[0], self.server.returncode))
-
-    def postallocator_hook(self, allocator, run, verbose):
-        self.terminate_server()
-
-    def process_output(self, result, stdout, stderr, allocator, perm, verbose):
-        result["transactions"] = re.search("transactions:\s*(\d*)", stdout).group(1)
-        result["queries"] = re.search("queries:\s*(\d*)", stdout).group(1)
-        # Latency
-        result["min"] = re.search("min:\s*(\d*.\d*)", stdout).group(1)
-        result["avg"] = re.search("avg:\s*(\d*.\d*)", stdout).group(1)
-        result["max"] = re.search("max:\s*(\d*.\d*)", stdout).group(1)
-
-        with open("/proc/"+str(self.server.pid)+"/status", "r") as f:
-            for l in f.readlines():
-                if l.startswith("VmHWM:"):
-                    result["rssmax"] = int(l.replace("VmHWM:", "").strip().split()[0])
-                    break
-
-    def summary(self):
-        allocators = self.results["allocators"]
-        args = self.results["args"]
-
-        # linear plot
-        self.plot_single_arg("{transactions}",
-                             xlabel='"threads"',
-                             ylabel='"transactions"',
-                             title='"sysbench oltp read only"',
-                             filepostfix="l.ro")
-
-
-        # linear plot
-        ref_alloc = list(allocators)[0]
-        self.plot_single_arg("{transactions}",
-                             xlabel='"threads"',
-                             ylabel='"transactions scaled at " + scale',
-                             title='"sysbench oltp read only"',
-                             filepostfix="norm.l.ro",
-                             scale=ref_alloc)
-
-        # bar plot
-        for i, allocator in enumerate(allocators):
-            y_vals = []
-            for perm in self.iterate_args(args=self.results["args"]):
-                d = [int(m["transactions"]) for m in self.results[allocator][perm]]
-                y_vals.append(np.mean(d))
-            x_vals = [x-i/8 for x in range(1, len(y_vals) + 1)]
-            plt.bar(x_vals, y_vals, width=0.2, label=allocator, align="center",
-                    color=allocators[allocator]["color"])
-
-        plt.legend()
-        plt.xlabel("threads")
-        plt.xticks(range(1, len(y_vals) + 1), self.results["args"]["nthreads"])
-        plt.ylabel("transactions")
-        plt.title("sysbench oltp read only")
-        plt.savefig(self.name + ".b.ro.png")
-        plt.clf()
-
-        # normalized bar plot
-        norm_y_vals = []
-        for perm in self.iterate_args(args=self.results["args"]):
-            d = [int(m["transactions"]) for m in self.results[ref_alloc][perm]]
-            norm_y_vals.append(np.mean(d))
-
-        nallocs = len(allocators)
-        x_vals = [x for x in range(1, len(norm_y_vals)*nallocs + 1, nallocs)]
-        plt.bar(x_vals, [1]*len(norm_y_vals), width=0.7, label=ref_alloc,
-                align="center", color=allocators[ref_alloc]["color"])
-
-        for i, allocator in enumerate(list(allocators)[1:]):
-            y_vals = []
-            for y, perm in enumerate(self.iterate_args(args=self.results["args"])):
-                d = [int(m["transactions"]) for m in self.results[allocator][perm]]
-                y_vals.append(np.mean(d) / norm_y_vals[y])
-
-            x_vals = [x+i+1 for x in range(1, len(norm_y_vals)*nallocs + 1, nallocs)]
-
-            plt.bar(x_vals, y_vals, width=0.7, label=allocator, align="center",
-                    color=allocators[allocator]["color"])
-
-        plt.legend()
-        plt.xlabel("threads")
-        plt.xticks(range(1, len(norm_y_vals)*nallocs + 1, nallocs), self.results["args"]["nthreads"])
-        plt.ylabel("transactions normalized")
-        plt.title("sysbench oltp read only")
-        plt.savefig(self.name + ".norm.b.ro.png")
-        plt.clf()
-
-        # Memusage
-        self.plot_single_arg("{rssmax}",
-                             xlabel='"threads"',
-                             ylabel='"VmHWM in kB"',
-                             title='"Memusage sysbench oltp read only"',
-                             filepostfix="ro.mem")
-
-        # Colored latex table showing transactions count
-        d = {allocator: {} for allocator in allocators}
-        for perm in self.iterate_args(args=args):
-            for i, allocator in enumerate(allocators):
-                t = [float(x["transactions"]) for x in self.results[allocator][perm]]
-                m = np.mean(t)
-                s = np.std(t)/m
-                d[allocator][perm] = {"mean": m, "std": s}
-
-        mins = {}
-        maxs = {}
-        for perm in self.iterate_args(args=args):
-            cmax = None
-            cmin = None
-            for i, allocator in enumerate(allocators):
-                m = d[allocator][perm]["mean"]
-                if not cmax or m > cmax:
-                    cmax = m
-                if not cmin or m < cmin:
-                    cmin = m
-            maxs[perm] = cmax
-            mins[perm] = cmin
-
-        fname = ".".join([self.name, "transactions.tex"])
-        headers = [perm.nthreads for perm in self.iterate_args(args=args)]
-        with open(fname, "w") as f:
-            print("\\begin{tabular}{| l" + " l"*len(headers) + " |}", file=f)
-            print("Fäden / Allokator ", end=" ", file=f)
-            for head in headers:
-                print("& {}".format(head), end=" ", file=f)
-            print("\\\\\n\\hline", file=f)
-
-            for allocator in allocators:
-                print(allocator, end=" ", file=f)
-                for perm in self.iterate_args(args=args):
-                    m = d[allocator][perm]["mean"]
-                    s = "& \\textcolor{{{}}}{{{:.3f}}}"
-                    if m == maxs[perm]:
-                        color = "green"
-                    elif m == mins[perm]:
-                        color = "red"
-                    else:
-                        color = "black"
-                    print(s.format(color, m), end=" ", file=f)
-                print("\\\\", file=f)
-
-            print("\end{tabular}", file=f)
-
-
-mysql = Benchmark_MYSQL()
-- 
cgit v1.2.3