put common run logic into benchmark.py

1 files changed, 61 insertions, 177 deletions

diff --git a/loop.py b/loop.py
index 2709e68..c6cef00 100644
--- a/loop.py
+++ b/loop.py
@@ -1,18 +1,9 @@
-import csv
-import pickle
 import matplotlib.pyplot as plt
 import multiprocessing
 import numpy as np
 import os
-import subprocess
-from subprocess import PIPE
 
 from benchmark import Benchmark
-from common_targets import common_targets
-
-perf_cmd = ("perf stat -x\; -d -e cpu-clock,cache-references,cache-misses,cycles,"
-       "instructions,branches,faults,migrations ")
-cmd = "build/bench_loop{} 1.2 {} 1000000 {} 10"
 
 class Benchmark_Loop( Benchmark ):
     def __init__(self):
@@ -20,182 +11,75 @@ class Benchmark_Loop( Benchmark ):
         self.descrition = """This benchmark makes n allocations in t concurrent threads.
                             How allocations are freed can be changed with the benchmark
                             version""",
-        self.targets = common_targets
-        self.maxsize = [2 ** x for x in range(6, 16)]
-        self.nthreads = range(1, multiprocessing.cpu_count() * 2 + 1)
-
-        self.results = {"args" : {"nthreads" : self.nthreads, "maxsize": self.maxsize},
-                        "targets" : self.targets}
-
-    def prepare(self, verbose=False):
-        req = ["build/bench_loop"]
-        for r in req:
-            if not os.path.isfile(r):
-                print(r, "not found")
-                return False
-            if not os.access(r, os.X_OK):
-                print(r, "not found")
-                return False
-            if verbose:
-                print(r, "found and executable.")
-        return True
-
-
-    def run(self, verbose=False, runs=3):
-        args_permutations = [(x,y) for x in self.nthreads for y in self.maxsize]
-        n = len(args_permutations)
-
-        for run in range(1, runs + 1):
-            print(str(run) + ". run")
-
-            for i, args in enumerate(args_permutations):
-                print(i + 1, "of", n, "\r", end='')
-
-                # run cmd for each target
-                for tname, t in self.targets.items():
-                    if not tname in self.results:
-                        self.results[tname] = {}
-
-                    result = {}
-
-                    os.environ["LD_PRELOAD"] = t["LD_PRELOAD"]
-
-                    cur_cmd = cmd.format(t["binary_suffix"], *args)
-
-                    # Collect memory consumtion on first run
-                    if run == 1:
-                        os.environ["LD_PRELOAD"] = "build/print_status_on_exit.so " + os.environ["LD_PRELOAD"]
-                        subprocess.run(cur_cmd.split(), env=os.environ)
-                        with open("status", "r") as f:
-                            for l in f.readlines():
-                                if l.startswith("VmHWM:"):
-                                    result["rssmax"] = l.split()[1]
-
-                        os.environ["LD_PRELOAD"] = t["LD_PRELOAD"]
-
-                    target_cmd = perf_cmd + cur_cmd
-                    if verbose:
-                        print("\n" + tname, t, "\n", target_cmd, "\n")
-
-                    p = subprocess.run(target_cmd.split(),
-                                         env=os.environ,
-                                         stderr=PIPE,
-                                         stdout=PIPE,
-                                         universal_newlines=True)
-
-
-                    output = p.stderr
-
-                    if p.returncode != 0:
-                        print("\n" + target_cmd, "exited with", p.returncode, ".\n Aborting Benchmark.")
-                        print(tname, t)
-                        print(output)
-                        print(p.stdout)
-                        return False
-
-                    if "ERROR: ld.so" in output:
-                        print("\nPreloading of", t["LD_PRELOAD"], "failed for", tname, ".\n Aborting Benchmark.")
-                        print(output)
-                        return False
 
-                    # Handle perf output
-                    csvreader = csv.reader(output.splitlines(), delimiter=';')
-                    for row in csvreader:
-                        result[row[2].replace("\\", "")] = row[0].replace("\\", "")
+        self.cmd = "build/bench_loop{binary_suffix} 1.2 {nthreads} 1000000 {maxsize} 10"
 
-                    if not args in self.results[tname]:
-                        self.results[tname][args] = [result]
-                    else:
-                        self.results[tname][args].append(result)
+        self.args = {
+                        "maxsize" : [2 ** x for x in range(6, 16)],
+                        "nthreads" : range(1, multiprocessing.cpu_count() * 2 + 1)
+                    }
 
-            print()
-        return True
+        self.requirements = ["build/bench_loop"]
+        print(type(self))
+        super().__init__()
 
     def summary(self, sd=None):
-        nthreads = self.results["args"]["nthreads"]
-        maxsize = self.results["args"]["maxsize"]
+        args = self.results["args"]
         targets = self.results["targets"]
 
         sd = sd or ""
 
-        # MAXSIZE fixed
-        y_mapping = {v : i for i, v in enumerate(nthreads)}
-        for size in maxsize:
-            for target in targets:
-                y_vals = [0] * len(nthreads)
-                for margs, measures in [(a, m) for a, m in self.results[target].items() if a[1] == size]:
-                    d = []
-                    for m in measures:
-                        # nthreads/time = MOPS/s
-                        for e in m:
-                            if "cpu-clock" in e:
-                                d.append(margs[0]/float(m[e]))
-                    y_vals[y_mapping[margs[0]]] = np.mean(d)
-                plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"])
-
-            plt.legend()
-            plt.xlabel("threads")
-            plt.ylabel("MOPS/s")
-            plt.title("Loop: " + str(size) + "B")
-            plt.savefig(os.path.join(sd, self.name + "." + str(size) + "B.png"))
-            plt.clf()
-
-        # NTHREADS fixed
-        y_mapping = {v : i for i, v in enumerate(maxsize)}
-        x_vals = [i + 1 for i in range(0, len(maxsize))]
-        for n in nthreads:
-            for target in targets:
-                y_vals = [0] * len(maxsize)
-                for margs, measures in [(a, m) for a, m in self.results[target].items() if a[0] == n]:
-                    d = []
-                    for m in measures:
-                        # nthreads/time = MOPS/S
-                        for e in m:
-                            if "cpu-clock" in e:
-                                d.append(margs[0]/float(m[e]))
-                    y_vals[y_mapping[margs[1]]] = np.mean(d)
-                plt.plot(x_vals, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"])
-
-            plt.legend()
-            plt.xticks(x_vals, maxsize)
-            plt.xlabel("size in B")
-            plt.ylabel("MOPS/s")
-            plt.title("Loop: " + str(n) + "thread(s)")
-            plt.savefig(os.path.join(sd, self.name + "." + str(n) + "threads.png"))
-            plt.clf()
-
-        #Memusage
-        y_mapping = {v : i for i, v in enumerate(nthreads)}
-        for size in maxsize:
-            for target in targets:
-                y_vals = [0] * len(nthreads)
-                for margs, measures in [(a, m) for a, m in self.results[target].items() if a[1] == size]:
-                    y_vals[y_mapping[margs[0]]] = int(measures[0]["rssmax"])
-                plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"])
-
-            plt.legend()
-            plt.xlabel("threads")
-            plt.ylabel("kb")
-            plt.title("Memusage Loop: " + str(size) + "B")
-            plt.savefig(os.path.join(sd, self.name + "." + str(size) + "B.mem.png"))
-            plt.clf()
-
-        # NTHREADS fixed
-        y_mapping = {v : i for i, v in enumerate(maxsize)}
-        x_vals = [i + 1 for i in range(0, len(maxsize))]
-        for n in nthreads:
-            for target in targets:
-                y_vals = [0] * len(maxsize)
-                for margs, measures in [(a, m) for a, m in self.results[target].items() if a[0] == n]:
-                    y_vals[y_mapping[margs[1]]] = int(measures[0]["rssmax"])
-                plt.plot(x_vals, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"])
-
-            plt.legend()
-            plt.xticks(x_vals, maxsize)
-            plt.xlabel("size in B")
-            plt.ylabel("kb")
-            plt.title("Memusage Loop: " + str(n) + "thread(s)")
-            plt.savefig(os.path.join(sd, self.name + "." + str(n) + "threads.mem.png"))
-            plt.clf()
+        # Speed
+        for arg in args:
+            loose_arg = [a for a in args if a != arg][0]
+            for arg_value in args[arg]:
+                for target in targets:
+                    y_vals = []
+                    for perm in self.iterate_args_fixed({arg : arg_value}):
+                        d = []
+                        for measure in self.results[target][perm]:
+                            # nthreads/time = MOPS/s
+                            for e in measure:
+                                if "task-clock" in e:
+                                    d.append(perm.nthreads/float(measure[e]))
+                        y_vals.append(np.mean(d))
+
+                    x_vals = list(range(1, len(y_vals) + 1))
+
+                    plt.plot(x_vals, y_vals, marker='.', linestyle='-',
+                        label=target, color=targets[target]["color"])
+
+                plt.legend()
+                plt.xticks(x_vals, args[loose_arg])
+                plt.xlabel(loose_arg)
+                plt.ylabel("MOPS/s")
+                plt.title("Loop: " + arg + " " + str(arg_value))
+                plt.savefig(os.path.join(sd, ".".join([self.name, arg, str(arg_value), "png"])))
+                plt.clf()
+
+        # Memusage
+        for arg in args:
+            loose_arg = [a for a in args if a != arg][0]
+            for arg_value in args[arg]:
+                for target in targets:
+                    y_vals = []
+                    for perm in self.iterate_args_fixed({arg : arg_value}):
+                        d = []
+                        for measure in self.results[target][perm]:
+                            d.append(int(measure["VmHWM"]))
+                        y_vals.append(np.mean(d))
+
+                    x_vals = list(range(1, len(y_vals) + 1))
+
+                    plt.plot(x_vals, y_vals, marker='.', linestyle='-',
+                        label=target, color=targets[target]["color"])
+
+                plt.legend()
+                plt.xticks(x_vals, args[loose_arg])
+                plt.xlabel(loose_arg)
+                plt.ylabel("VmHWM")
+                plt.title("Loop Memusage: " + arg + " " + str(arg_value))
+                plt.savefig(os.path.join(sd, ".".join([self.name, arg, str(arg_value), "mem", "png"])))
+                plt.clf()
 
 loop = Benchmark_Loop()