From 5e78c1ccda63bcbfd065f5dbde70f7e3ee1b881c Mon Sep 17 00:00:00 2001
From: Florian Fischer <florian.fl.fischer@fau.de>
Date: Sat, 8 Sep 2018 15:51:20 +0200
Subject: measure wall time in falsesharing and fix summary for results with
 different args

---
 benchmark.py    | 12 +++++++-----
 falsesharing.py | 16 +++++++++++-----
 loop.py         |  4 ++--
 mysql.py        |  7 ++++---
 4 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/benchmark.py b/benchmark.py
index 920772f..caa25fd 100644
--- a/benchmark.py
+++ b/benchmark.py
@@ -93,15 +93,17 @@ class Benchmark (object):
 
         return True
 
-    def iterate_args(self):
+    def iterate_args(self, args=None):
         """Return a dict for each possible combination of args"""
-        arg_names = sorted(self.args.keys())
-        for p in itertools.product(*[self.args[k] for k in arg_names]):
+        if not args:
+            args = self.args
+        arg_names = sorted(args.keys())
+        for p in itertools.product(*[args[k] for k in arg_names]):
             Perm = namedtuple("Perm", arg_names)
             yield Perm(*p)
 
-    def iterate_args_fixed(self, fixed):
-        for p in self.iterate_args():
+    def iterate_args_fixed(self, fixed, args=None):
+        for p in self.iterate_args(args=args):
             p_dict = p._asdict()
             is_fixed = True
             for k in fixed:
diff --git a/falsesharing.py b/falsesharing.py
index 7159e04..05a193e 100644
--- a/falsesharing.py
+++ b/falsesharing.py
@@ -9,6 +9,8 @@ import subprocess
 
 from benchmark import Benchmark
 
+time_re = re.compile("^Time elapsed = (?P<time>\d*\.\d*) seconds.$")
+
 class Benchmark_Falsesharing( Benchmark ):
     def __init__(self):
         self.name = "falsesharing"
@@ -27,9 +29,13 @@ class Benchmark_Falsesharing( Benchmark ):
         self.requirements = ["build/cache-thrash", "build/cache-scratch"]
         super().__init__()
 
+    def process_stdout(self, result, stdout, verbose):
+        result["time"] = time_re.match(stdout).group("time")
+
     def summary(self, sd=None):
         # Speedup thrash
-        nthreads = self.results["args"]["threads"]
+        args = self.results["args"]
+        nthreads = args["threads"]
         targets = self.results["targets"]
 
         sd = sd or ""
@@ -39,12 +45,12 @@ class Benchmark_Falsesharing( Benchmark ):
                 y_vals = []
 
                 single_threaded_perm = self.Perm(bench=bench, threads=1)
-                single_threaded = np.mean([float(m["task-clock"])
+                single_threaded = np.mean([float(m["time"])
                                                     for m in self.results[target][single_threaded_perm]])
 
-                for perm in self.iterate_args_fixed({"bench" : bench}):
+                for perm in self.iterate_args_fixed({"bench" : bench}, args=args):
 
-                    d = [float(m["task-clock"]) for m in self.results[target][perm]]
+                    d = [float(m["time"]) for m in self.results[target][perm]]
 
                     y_vals.append(single_threaded / np.mean(d))
 
@@ -61,7 +67,7 @@ class Benchmark_Falsesharing( Benchmark ):
             for target in targets:
                 y_vals = []
 
-                for perm in self.iterate_args_fixed({"bench" : bench}):
+                for perm in self.iterate_args_fixed({"bench" : bench}, args=args):
                     l1_load_misses = []
 
                     for m in self.results[target][perm]:
diff --git a/loop.py b/loop.py
index 3139f46..e22933b 100644
--- a/loop.py
+++ b/loop.py
@@ -34,7 +34,7 @@ class Benchmark_Loop( Benchmark ):
             for arg_value in args[arg]:
                 for target in targets:
                     y_vals = []
-                    for perm in self.iterate_args_fixed({arg : arg_value}):
+                    for perm in self.iterate_args_fixed({arg : arg_value}, args=args):
                         d = []
                         for measure in self.results[target][perm]:
                             # nthreads/time = MOPS/s
@@ -62,7 +62,7 @@ class Benchmark_Loop( Benchmark ):
             for arg_value in args[arg]:
                 for target in targets:
                     y_vals = []
-                    for perm in self.iterate_args_fixed({arg : arg_value}):
+                    for perm in self.iterate_args_fixed({arg : arg_value}, args=args):
                         d = []
                         for measure in self.results[target][perm]:
                             d.append(int(measure["VmHWM"]))
diff --git a/mysql.py b/mysql.py
index e74c7c5..2f501da 100644
--- a/mysql.py
+++ b/mysql.py
@@ -185,12 +185,13 @@ class Benchmark_MYSQL( Benchmark ):
     def summary(self, sd=None):
         sd = sd or ""
         targets = self.results["targets"]
+        args = self.results["args"]
         nthreads = list(self.results["args"]["nthreads"])
 
         # linear plot
         for target in targets:
             y_vals = []
-            for perm in self.iterate_args():
+            for perm in self.iterate_args(args=args):
                 d = [int(m["transactions"]) for m in self.results[target][perm]]
                 y_vals.append(np.mean(d))
             plt.plot(nthreads, y_vals, label=target, linestyle='-',
@@ -206,7 +207,7 @@ class Benchmark_MYSQL( Benchmark ):
         # bar plot
         for i, target in enumerate(targets):
             y_vals = []
-            for perm in self.iterate_args():
+            for perm in self.iterate_args(args=args):
                 d = [int(m["transactions"]) for m in self.results[target][perm]]
                 y_vals.append(np.mean(d))
             x_vals = [x-i/8 for x in range(1, len(nthreads) + 1)]
@@ -255,7 +256,7 @@ class Benchmark_MYSQL( Benchmark ):
         # Memusage
         for target in targets:
             y_vals = []
-            for perm in self.iterate_args():
+            for perm in self.iterate_args(args=args):
                 d = [int(m["rssmax"]) for m in self.results[target][perm]]
                 y_vals.append(np.mean(d))
             plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target,
-- 
cgit v1.2.3