put common run logic into benchmark.py

3 files changed, 278 insertions, 279 deletions

diff --git a/benchmark.py b/benchmark.py
index c68dce7..92a670d 100644
--- a/benchmark.py
+++ b/benchmark.py
@@ -1,17 +1,198 @@
+from collections import namedtuple
+import copy
+import csv
+import itertools
+import os
 import pickle
+import subprocess
+
+from common_targets import common_targets
 
 class Benchmark (object):
+
+    defaults = {
+        "name" : "default_benchmark",
+        "description" : "This is the default benchmark description please add your own useful one.",
+        
+        "perf_cmd" : "perf stat -x, -dd ",
+        "analyse_cmd" : "memusage -p {} -t ",
+        "cmd" : "true",
+        "targets" : common_targets,
+    }
+
+    def __init__(self):
+        # Set default values
+        for k in Benchmark.defaults:
+            if not hasattr(self, k):
+                setattr(self, k, Benchmark.defaults[k])
+
+        # non copy types
+        if not hasattr(self, "args"):
+            self.args = {}
+
+        self.Perm = namedtuple("Perm", self.args.keys())
+        
+        if not hasattr(self, "results"):
+            self.results = {}
+        self.results["args"] = self.args
+        self.results["targets"] = self.targets
+        self.results.update({t : {} for t in self.targets})
+
+        if not hasattr(self, "requirements"):
+            self.requirements = []
+    
     def save(self, path=None, verbose=False):
         f = path if path else self.name + ".save"
         if verbose:
             print("Saving results to:", self.name + ".save")
+        # Pickle can't handle namedtuples so convert the dicts of namedtuples
+        # into lists of dicts.
+        save_data = {"args" : self.results["args"], "targets" : self.results["targets"]}
+        for target in self.results["targets"]:
+            l = []
+            for ntuple, measures in self.results[target].items():
+                l.append((ntuple._asdict(), measures))
+            save_data[target] = l
+
         with open(f, "wb") as f:
-            pickle.dump(self.results, f)
+            pickle.dump(save_data, f)
 
     def load(self, path=None, verbose=False):
         f = path if path else self.name + ".save"
         if verbose:
             print("Loading results from:", self.name + ".save")
         with open(f, "rb") as f:
-            self.results = pickle.load(f)
+            save_data = pickle.load(f)
+        # Build new named tuples
+        self.results["args"] = save_data["args"]
+        self.results["targets"] = save_data["targets"]
+        for target in save_data["targets"]:
+            d = {}
+            for dic, measures in save_data[target]:
+                d[self.Perm(**dic)] = measures
+            self.results[target] = d
+
+    def prepare(self, verbose=False):
+        def is_exe(fpath):
+            return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+        for r in self.requirements:
+            fpath, fname = os.path.split(r)
+            
+            if fpath:
+                if not is_exe(r):
+                    return False
+            else:
+                found = False
+                for path in os.environ["PATH"].split(os.pathsep):
+                    exe_file = os.path.join(path, r)
+                    if is_exe(exe_file):
+                        found = True
+
+                if not found:
+                    return False
+
+        return True
+
+    def iterate_args(self):
+        """Return a dict for each possible combination of args"""
+        arg_names = sorted(self.args.keys())
+        for p in itertools.product(*[self.args[k] for k in arg_names]):
+            Perm = namedtuple("Perm", arg_names)
+            yield Perm(*p)
+
+    def iterate_args_fixed(self, fixed):
+        for p in self.iterate_args():
+            p_dict = p._asdict()
+            is_fixed = True
+            for k in fixed:
+                if p_dict[k] != fixed[k]:
+                    is_fixed = False
+                    break
+            if is_fixed:
+                yield p
+            
+
+    def analyse(self, verbose=False):
+        for perm in self.iterate_args():
+            file_name = ".".join(list(self.name, *p.items()))
+
+            actual_cmd = analyse_cmd.format(file_name + ".png")
+            if "binary_suffix" in cmd:
+                p["binary_suffix"] = ""
+            actual_cmd += cmd.format(**perm._asdict())
+            
+            with open(file_name + ".hist", "w") as f:
+                res = subprocess.run(actual_cmd.split(),
+                                    stderr=f,
+                                    universal_newlines=True)
+
+                if res.returncode != 0:
+                    print(actual_cmd, "failed.")
+                    print("Aborting analysing.")
+                    print("You may look at", file_name + ".hist", "to fix this.")
+                    return
+
+
+    def run(self, verbose=False, runs=5):
+        n = len(list(self.iterate_args()))
+        for run in range(1, runs + 1):
+            print(str(run) + ". run")
+
+            for i, perm in enumerate(self.iterate_args()):
+                print(i + 1, "of", n, "\r", end='')
+
+                for tname, t in self.targets.items():
+                    if not tname in self.results:
+                        self.results[tname] = {}
+                    
+                    actual_cmd = self.perf_cmd
+
+                    perm_dict = perm._asdict()
+                    perm_dict.update(t)
+                    actual_cmd += self.cmd.format(**perm_dict)
+
+                    os.environ["LD_PRELOAD"] = "build/print_status_on_exit.so "
+                    os.environ["LD_PRELOAD"] += t["LD_PRELOAD"]
+
+                    res = subprocess.run(actual_cmd.split(),
+                                        stderr=subprocess.PIPE,
+                                        stdout=subprocess.PIPE,
+                                        universal_newlines=True)
+
+                    if res.returncode != 0:
+                        print("\n" + actual_cmd, "exited with", res.returncode)
+                        print("Aborting Benchmark.")
+                        print("Stdout:\n" + res.stdout)
+                        print("Stderr:\n" + res.stderr)
+                        return False
+
+                    if "ERROR: ld.so" in res.stderr:
+                        print("\nPreloading of", t["LD_PRELOAD"], "failed for", tname)
+                        print("Stderr:\n" + res.stderr)
+                        print("Aborting Benchmark.")
+                        return False
+
+                    result = {}
+                    
+                    # Read VmHWM from status file # If our benchmark didn't fork
+                    # the first occurance of VmHWM is from our benchmark
+                    with open("status", "r") as f:
+                        for l in f.readlines():
+                            if l.startswith("VmHWM:"):
+                                result["VmHWM"] = l.split()[1]
+                                break
+                    os.remove("status")
+                        
+                    # Parse perf output
+                    csvreader = csv.reader(res.stderr.splitlines(), delimiter=',')
+                    for row in csvreader:
+                        # Split of the user/kernel space info to be better portable
+                        result[row[2].split(":")[0]] = row[0]
+
+                    if run == 1:
+                        self.results[tname][perm] = []
+                    self.results[tname][perm].append(result)
+            print()
+        return True
 
diff --git a/falsesharing.py b/falsesharing.py
index 008b477..7159e04 100644
--- a/falsesharing.py
+++ b/falsesharing.py
@@ -8,11 +8,6 @@ import re
 import subprocess
 
 from benchmark import Benchmark
-from common_targets import common_targets
-
-cmd = ("perf stat -d -x\; -e cpu-clock,cache-references,cache-misses,cycles,"
-       "instructions,branches,faults,migrations "
-       "build/cache-{}{} {} 100 8 1000000")
 
 class Benchmark_Falsesharing( Benchmark ):
     def __init__(self):
@@ -20,104 +15,39 @@ class Benchmark_Falsesharing( Benchmark ):
         self.descrition = """This benchmarks makes small allocations and writes
                             to them multiple times. If the allocated objects are
                             on the same cache line the writes will be expensive because
-                            of cache thrashing.""",
-        self.targets = common_targets
-        self.nthreads = range(1, multiprocessing.cpu_count() * 2 + 1)
-
-        self.results = {"args" : {"nthreads" : self.nthreads},
-                        "targets" : self.targets,
-                        "thrash": {x : {} for x in self.targets},
-                        "scratch": {x: {} for x in self.targets}
-                       }
-
-    def prepare(self, verbose=False):
-        req = ["build/cache-thrash", "build/cache-scratch"]
-        for r in req:
-            if not os.path.isfile(r):
-                print(r, "not found")
-                return False
-            if not os.access(r, os.X_OK):
-                print(r, "not executable")
-                return False
-            if verbose:
-                print(r, "found and executable.")
-        return True
-
-
-    def run(self, verbose=False, runs=3):
-        for run in range(1, runs + 1):
-            print(str(run) + ". run")
-
-            n = len(self.nthreads)
-            for i, threads in enumerate(list(range(1, n + 1)) * 2):
-                print(i + 1, "of", n*2, "\r", end='')
-
-                # run cmd for each target
-                for tname, t in self.targets.items():
-                    result = {}
-
-                    os.environ["LD_PRELOAD"] = t["LD_PRELOAD"]
-
-                    for bench in ["thrash", "scratch"]:
-
-                        target_cmd = cmd.format(bench, t["binary_suffix"], threads).split(" ")
-                        if verbose:
-                            print("\n" + tname, t, "\n", " ".join(target_cmd), "\n")
-
-                        p = subprocess.run(target_cmd,
-                                             env=os.environ,
-                                             stderr=subprocess.PIPE,
-                                             stdout=subprocess.PIPE,
-                                             universal_newlines=True)
-
-                        output = str(p.stdout)
-                        err = str(p.stderr)
-
-                        if p.returncode != 0:
-                            print("\n" + " ".join(target_cmd), "exited with",
-                                    p.returncode, ".\n Aborting Benchmark.")
-                            print(tname, t)
-                            print(output)
-                            print(p.stdout)
-                            return False
-
-                        if "ERROR: ld.so" in output:
-                            print("\nPreloading of", t["LD_PRELOAD"], "failed for", tname,
-                                    ".\n Aborting Benchmark.")
-                            print(output)
-                            return False
-
-                        time = float(re.search("(\d*\.\d*)", output)[1])
-                        result["time"] = time
-                        # Handle perf output
-                        csvreader = csv.reader(err.splitlines()[1:], delimiter=';')
-                        for row in csvreader:
-                            result[row[2].replace("\\", "")] = row[0].replace("\\", "")
-
-                        if not threads in self.results[bench][tname]:
-                            self.results[bench][tname][threads] = [result]
-                        else:
-                            self.results[bench][tname][threads].append(result)
-
-            print()
-        return True
+                            of cache thrashing."""
+
+        self.cmd = "build/cache-{bench}{binary_suffix} {threads} 100 8 1000000"
+
+        self.args = {
+                        "bench" : ["thrash", "scratch"],
+                        "threads" : range(1, multiprocessing.cpu_count() * 2 + 1)
+                    }
+
+        self.requirements = ["build/cache-thrash", "build/cache-scratch"]
+        super().__init__()
 
     def summary(self, sd=None):
         # Speedup thrash
-        nthreads = self.results["args"]["nthreads"]
+        nthreads = self.results["args"]["threads"]
         targets = self.results["targets"]
 
         sd = sd or ""
 
-        y_mapping = {v : i for i, v in enumerate(nthreads)}
-        for bench in ["thrash", "scratch"]:
+        for bench in self.results["args"]["bench"]:
             for target in targets:
-                y_vals = [0] * len(nthreads)
-                single_threaded = np.mean([m["time"] for m in self.results[bench][target][1]])
-                for threads, measures in self.results[bench][target].items():
-                    l1_load_misses = []
-                    d = [m["time"] for m in measures]
-                    y_vals[y_mapping[threads]] = single_threaded / np.mean(d)
+                y_vals = []
+
+                single_threaded_perm = self.Perm(bench=bench, threads=1)
+                single_threaded = np.mean([float(m["task-clock"])
+                                                    for m in self.results[target][single_threaded_perm]])
+
+                for perm in self.iterate_args_fixed({"bench" : bench}):
+
+                    d = [float(m["task-clock"]) for m in self.results[target][perm]]
+
+                    y_vals.append(single_threaded / np.mean(d))
+
                 plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target,
                             color=targets[target]["color"])
 
@@ -129,10 +59,12 @@ class Benchmark_Falsesharing( Benchmark ):
             plt.clf()
 
             for target in targets:
-                y_vals = [0] * len(nthreads)
-                for threads, measures in self.results[bench][target].items():
+                y_vals = []
+
+                for perm in self.iterate_args_fixed({"bench" : bench}):
                     l1_load_misses = []
-                    for m in measures:
+
+                    for m in self.results[target][perm]:
                         misses = 0
                         loads = 0
                         for e in m:
@@ -141,7 +73,9 @@ class Benchmark_Falsesharing( Benchmark ):
                             elif "L1-dcache-loads" in e:
                                 loads = float(m[e])
                         l1_load_misses.append(misses/loads)
-                    y_vals[y_mapping[threads]] = np.mean(l1_load_misses) * 100
+
+                    y_vals.append(np.mean(l1_load_misses) * 100)
+
                 plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"])
             plt.legend()
             plt.xlabel("threads")
@@ -150,4 +84,4 @@ class Benchmark_Falsesharing( Benchmark ):
             plt.savefig(os.path.join(sd, self.name + "." + bench + ".l1misses.png"))
             plt.clf()
 
-falsesharing= Benchmark_Falsesharing()
+falsesharing = Benchmark_Falsesharing()
diff --git a/loop.py b/loop.py
index 2709e68..c6cef00 100644
--- a/loop.py
+++ b/loop.py
@@ -1,18 +1,9 @@
-import csv
-import pickle
 import matplotlib.pyplot as plt
 import multiprocessing
 import numpy as np
 import os
-import subprocess
-from subprocess import PIPE
 
 from benchmark import Benchmark
-from common_targets import common_targets
-
-perf_cmd = ("perf stat -x\; -d -e cpu-clock,cache-references,cache-misses,cycles,"
-       "instructions,branches,faults,migrations ")
-cmd = "build/bench_loop{} 1.2 {} 1000000 {} 10"
 
 class Benchmark_Loop( Benchmark ):
     def __init__(self):
@@ -20,182 +11,75 @@ class Benchmark_Loop( Benchmark ):
         self.descrition = """This benchmark makes n allocations in t concurrent threads.
                             How allocations are freed can be changed with the benchmark
                             version""",
-        self.targets = common_targets
-        self.maxsize = [2 ** x for x in range(6, 16)]
-        self.nthreads = range(1, multiprocessing.cpu_count() * 2 + 1)
-
-        self.results = {"args" : {"nthreads" : self.nthreads, "maxsize": self.maxsize},
-                        "targets" : self.targets}
-
-    def prepare(self, verbose=False):
-        req = ["build/bench_loop"]
-        for r in req:
-            if not os.path.isfile(r):
-                print(r, "not found")
-                return False
-            if not os.access(r, os.X_OK):
-                print(r, "not found")
-                return False
-            if verbose:
-                print(r, "found and executable.")
-        return True
-
-
-    def run(self, verbose=False, runs=3):
-        args_permutations = [(x,y) for x in self.nthreads for y in self.maxsize]
-        n = len(args_permutations)
-
-        for run in range(1, runs + 1):
-            print(str(run) + ". run")
-
-            for i, args in enumerate(args_permutations):
-                print(i + 1, "of", n, "\r", end='')
-
-                # run cmd for each target
-                for tname, t in self.targets.items():
-                    if not tname in self.results:
-                        self.results[tname] = {}
-
-                    result = {}
-
-                    os.environ["LD_PRELOAD"] = t["LD_PRELOAD"]
-
-                    cur_cmd = cmd.format(t["binary_suffix"], *args)
-
-                    # Collect memory consumtion on first run
-                    if run == 1:
-                        os.environ["LD_PRELOAD"] = "build/print_status_on_exit.so " + os.environ["LD_PRELOAD"]
-                        subprocess.run(cur_cmd.split(), env=os.environ)
-                        with open("status", "r") as f:
-                            for l in f.readlines():
-                                if l.startswith("VmHWM:"):
-                                    result["rssmax"] = l.split()[1]
-
-                        os.environ["LD_PRELOAD"] = t["LD_PRELOAD"]
-
-                    target_cmd = perf_cmd + cur_cmd
-                    if verbose:
-                        print("\n" + tname, t, "\n", target_cmd, "\n")
-
-                    p = subprocess.run(target_cmd.split(),
-                                         env=os.environ,
-                                         stderr=PIPE,
-                                         stdout=PIPE,
-                                         universal_newlines=True)
-
-
-                    output = p.stderr
-
-                    if p.returncode != 0:
-                        print("\n" + target_cmd, "exited with", p.returncode, ".\n Aborting Benchmark.")
-                        print(tname, t)
-                        print(output)
-                        print(p.stdout)
-                        return False
-
-                    if "ERROR: ld.so" in output:
-                        print("\nPreloading of", t["LD_PRELOAD"], "failed for", tname, ".\n Aborting Benchmark.")
-                        print(output)
-                        return False
 
-                    # Handle perf output
-                    csvreader = csv.reader(output.splitlines(), delimiter=';')
-                    for row in csvreader:
-                        result[row[2].replace("\\", "")] = row[0].replace("\\", "")
+        self.cmd = "build/bench_loop{binary_suffix} 1.2 {nthreads} 1000000 {maxsize} 10"
 
-                    if not args in self.results[tname]:
-                        self.results[tname][args] = [result]
-                    else:
-                        self.results[tname][args].append(result)
+        self.args = {
+                        "maxsize" : [2 ** x for x in range(6, 16)],
+                        "nthreads" : range(1, multiprocessing.cpu_count() * 2 + 1)
+                    }
 
-            print()
-        return True
+        self.requirements = ["build/bench_loop"]
+        print(type(self))
+        super().__init__()
 
     def summary(self, sd=None):
-        nthreads = self.results["args"]["nthreads"]
-        maxsize = self.results["args"]["maxsize"]
+        args = self.results["args"]
         targets = self.results["targets"]
 
         sd = sd or ""
 
-        # MAXSIZE fixed
-        y_mapping = {v : i for i, v in enumerate(nthreads)}
-        for size in maxsize:
-            for target in targets:
-                y_vals = [0] * len(nthreads)
-                for margs, measures in [(a, m) for a, m in self.results[target].items() if a[1] == size]:
-                    d = []
-                    for m in measures:
-                        # nthreads/time = MOPS/s
-                        for e in m:
-                            if "cpu-clock" in e:
-                                d.append(margs[0]/float(m[e]))
-                    y_vals[y_mapping[margs[0]]] = np.mean(d)
-                plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"])
-
-            plt.legend()
-            plt.xlabel("threads")
-            plt.ylabel("MOPS/s")
-            plt.title("Loop: " + str(size) + "B")
-            plt.savefig(os.path.join(sd, self.name + "." + str(size) + "B.png"))
-            plt.clf()
-
-        # NTHREADS fixed
-        y_mapping = {v : i for i, v in enumerate(maxsize)}
-        x_vals = [i + 1 for i in range(0, len(maxsize))]
-        for n in nthreads:
-            for target in targets:
-                y_vals = [0] * len(maxsize)
-                for margs, measures in [(a, m) for a, m in self.results[target].items() if a[0] == n]:
-                    d = []
-                    for m in measures:
-                        # nthreads/time = MOPS/S
-                        for e in m:
-                            if "cpu-clock" in e:
-                                d.append(margs[0]/float(m[e]))
-                    y_vals[y_mapping[margs[1]]] = np.mean(d)
-                plt.plot(x_vals, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"])
-
-            plt.legend()
-            plt.xticks(x_vals, maxsize)
-            plt.xlabel("size in B")
-            plt.ylabel("MOPS/s")
-            plt.title("Loop: " + str(n) + "thread(s)")
-            plt.savefig(os.path.join(sd, self.name + "." + str(n) + "threads.png"))
-            plt.clf()
-
-        #Memusage
-        y_mapping = {v : i for i, v in enumerate(nthreads)}
-        for size in maxsize:
-            for target in targets:
-                y_vals = [0] * len(nthreads)
-                for margs, measures in [(a, m) for a, m in self.results[target].items() if a[1] == size]:
-                    y_vals[y_mapping[margs[0]]] = int(measures[0]["rssmax"])
-                plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"])
-
-            plt.legend()
-            plt.xlabel("threads")
-            plt.ylabel("kb")
-            plt.title("Memusage Loop: " + str(size) + "B")
-            plt.savefig(os.path.join(sd, self.name + "." + str(size) + "B.mem.png"))
-            plt.clf()
-
-        # NTHREADS fixed
-        y_mapping = {v : i for i, v in enumerate(maxsize)}
-        x_vals = [i + 1 for i in range(0, len(maxsize))]
-        for n in nthreads:
-            for target in targets:
-                y_vals = [0] * len(maxsize)
-                for margs, measures in [(a, m) for a, m in self.results[target].items() if a[0] == n]:
-                    y_vals[y_mapping[margs[1]]] = int(measures[0]["rssmax"])
-                plt.plot(x_vals, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"])
-
-            plt.legend()
-            plt.xticks(x_vals, maxsize)
-            plt.xlabel("size in B")
-            plt.ylabel("kb")
-            plt.title("Memusage Loop: " + str(n) + "thread(s)")
-            plt.savefig(os.path.join(sd, self.name + "." + str(n) + "threads.mem.png"))
-            plt.clf()
+        # Speed
+        for arg in args:
+            loose_arg = [a for a in args if a != arg][0]
+            for arg_value in args[arg]:
+                for target in targets:
+                    y_vals = []
+                    for perm in self.iterate_args_fixed({arg : arg_value}):
+                        d = []
+                        for measure in self.results[target][perm]:
+                            # nthreads/time = MOPS/s
+                            for e in measure:
+                                if "task-clock" in e:
+                                    d.append(perm.nthreads/float(measure[e]))
+                        y_vals.append(np.mean(d))
+
+                    x_vals = list(range(1, len(y_vals) + 1))
+
+                    plt.plot(x_vals, y_vals, marker='.', linestyle='-',
+                        label=target, color=targets[target]["color"])
+
+                plt.legend()
+                plt.xticks(x_vals, args[loose_arg])
+                plt.xlabel(loose_arg)
+                plt.ylabel("MOPS/s")
+                plt.title("Loop: " + arg + " " + str(arg_value))
+                plt.savefig(os.path.join(sd, ".".join([self.name, arg, str(arg_value), "png"])))
+                plt.clf()
+
+        # Memusage
+        for arg in args:
+            loose_arg = [a for a in args if a != arg][0]
+            for arg_value in args[arg]:
+                for target in targets:
+                    y_vals = []
+                    for perm in self.iterate_args_fixed({arg : arg_value}):
+                        d = []
+                        for measure in self.results[target][perm]:
+                            d.append(int(measure["VmHWM"]))
+                        y_vals.append(np.mean(d))
+
+                    x_vals = list(range(1, len(y_vals) + 1))
+
+                    plt.plot(x_vals, y_vals, marker='.', linestyle='-',
+                        label=target, color=targets[target]["color"])
+
+                plt.legend()
+                plt.xticks(x_vals, args[loose_arg])
+                plt.xlabel(loose_arg)
+                plt.ylabel("VmHWM")
+                plt.title("Loop Memusage: " + arg + " " + str(arg_value))
+                plt.savefig(os.path.join(sd, ".".join([self.name, arg, str(arg_value), "mem", "png"])))
+                plt.clf()
 
 loop = Benchmark_Loop()