add analyze and server_benchmark feature

--analyze uses malt to trace the benchmarks behavior. It uses the run
loop but the obtained results are not stored

Benchmark.server_benchmark is used if only a server is started for
each allocator and clients are used to measure its performance in the run loop.
If server_benchmark is set to True the cmds are run with the system default
allocator.

Misc changes:

* The global environment is no longer changed. Instead a custom env dict
is passed to suprocesses containing LD_PRELOAD.

* Failing cmds no longer skip the whole benchmark instead they now skip the
malfunctioning allocator.

* Fix default title in plot_single_arg

an analyse run are not stored

4 files changed, 89 insertions, 70 deletions

diff --git a/bench.py b/bench.py
index 90bd0bf..0b581f8 100755
--- a/bench.py
+++ b/bench.py
@@ -23,6 +23,7 @@ parser.add_argument("-ds, --dont-save", action='store_true', dest="dont_save",
                     help="don't save benchmark results in RESULTDIR")
 parser.add_argument("-l", "--load", help="load benchmark results from directory", type=str)
 parser.add_argument("-a", "--allocators", help="load allocator definitions from file", type=str)
+parser.add_argument("--analyse", help="analyse benchmark behaviour using malt", action="store_true")
 parser.add_argument("-r", "--runs", help="how often the benchmarks run", default=3, type=int)
 parser.add_argument("-v", "--verbose", help="more output", action='count')
 parser.add_argument("-vdebug", "--verbose-debug", help="debug output",
@@ -121,8 +122,8 @@ def main():
         starttime = starttime[:starttime.rfind(':')]
         src.globalvars.facts["starttime"] = starttime
 
-    # Create result directory if we save or summarize results
-    need_resultdir = not (args.nosum and args.dont_save)
+    # Create result directory if we analyse, save or summarize
+    need_resultdir = not (args.nosum and args.dont_save and not args.analyse)
     if need_resultdir:
         if args.resultdir:
             resdir = os.path.join(args.resultdir)
@@ -131,11 +132,9 @@ def main():
                                     src.globalvars.facts["starttime"])
         # make resdir globally available
         src.globalvars.resdir = resdir
-        try:
-            print_info2("Creating result dir:", resdir)
-            os.makedirs(resdir)
-        except FileExistsError:
-            pass
+
+        print_info2("Creating result dir:", resdir)
+        os.makedirs(resdir, exist_ok=True)
 
     # TODO load all results at once
 
@@ -144,16 +143,32 @@ def main():
         if args.benchmarks and not bench in args.benchmarks:
             continue
 
+        if args.analyse or not args.nosum:
+            bench_res_dir = os.path.join(resdir, bench)
+            print_info2("Creating benchmark result dir:", bench_res_dir)
+            os.makedirs(bench_res_dir, exist_ok=True)
+
         try:
             bench = eval("importlib.import_module('src.benchmarks.{0}').{0}".format(bench))
 
             if args.load:
                 bench.load(path=args.load)
 
-            if args.runs > 0:
+            if args.runs > 0 or args.analyse:
                 print_status("Preparing", bench.name, "...")
                 bench.prepare()
 
+            if args.analyse:
+                if find_cmd("malt") is not None:
+                    print_status("Analysing {} ...".format(bench))
+
+                    malt_cmd = "malt -o output:name={}/malt.{}.%3"
+                    malt_cmd = malt_cmd.format(bench_res_dir, "{perm}")
+                    bench.run(runs=1, dry_run=True, cmd_prefix=malt_cmd)
+                else:
+                    print_error("malt not found. Skipping analyse.")
+
+            print_status("Running", bench.name, "...")
             bench.run(runs=args.runs)
 
             if need_resultdir:
@@ -164,7 +179,6 @@ def main():
                     bench.save()
 
                 if not args.nosum:
-                    os.mkdir(bench.name)
                     os.chdir(bench.name)
                     print_status("Summarizing", bench.name, "...")
                     bench.summary()
@@ -182,13 +196,6 @@ def main():
             print_error(traceback.format_exc())
             print_error("Skipping", bench, "!")
 
-            # reset LD_PRELOAD
-            if src.globalvars.facts["LD_PRELOAD"] != os.environ.get("LD_PRELOAD", None):
-                if src.globalvars.facts["LD_PRELOAD"] is None:
-                    del(os.environ["LD_PRELOAD"])
-                else:
-                    os.environ["LD_PRELOAD"] = src.globalvars.facts["LD_PRELOAD"]
-
             continue
 
 
diff --git a/doc/Benchmarks.md b/doc/Benchmarks.md
index 4ab0c02..4aeb8a4 100644
--- a/doc/Benchmarks.md
+++ b/doc/Benchmarks.md
@@ -127,7 +127,7 @@ for number_of_runs
 
 #### run hooks
 
-* `preallocator_hook((alloc_name, alloc_definition), current_run, verbose)` is called
+* `preallocator_hook((alloc_name, alloc_definition), current_run, environment, verbose)` is called
 	if available once per allocator before any command is executed. This hook may
 	be useful if you want to prepare stuff for each allocator. The mysql benchmark
 	uses this hook to start the mysql server with the current allocator.
diff --git a/src/benchmark.py b/src/benchmark.py
index 23efb1e..b78e937 100644
--- a/src/benchmark.py
+++ b/src/benchmark.py
@@ -25,6 +25,7 @@ class Benchmark (object):
         "measure_cmd": "perf stat -x, -d",
         "cmd": "true",
         "allocators": src.globalvars.allocators,
+        "server_benchmark": False,
     }
 
     @staticmethod
@@ -137,7 +138,7 @@ class Benchmark (object):
         for r in self.requirements:
             exe = find_cmd(r)
             if exe is not None:
-                self.results["facts"]["libc"][r] = src.facter.get_libc_version(bin=exe)
+                self.results["facts"]["libcs"][r] = src.facter.get_libc_version(bin=exe)
             else:
                 raise Exception("Requirement: {} not found".format(r))
 
@@ -161,12 +162,10 @@ class Benchmark (object):
             if is_fixed:
                 yield p
 
-    def run(self, runs=5):
+    def run(self, runs=5, dry_run=False, cmd_prefix=""):
         if runs < 1:
             return
 
-        print_status("Running", self.name, "...")
-
         # check if perf is allowed on this system
         if self.measure_cmd == self.defaults["measure_cmd"]:
             if Benchmark.perf_allowed == None:
@@ -175,6 +174,7 @@ class Benchmark (object):
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.PIPE,
                                      universal_newlines=True)
+
                 if res.returncode != 0:
                     print_error("Test perf run failed with:")
                     print(res.stderr, file=sys.stderr)
@@ -194,37 +194,49 @@ class Benchmark (object):
                 if tname not in self.results:
                     self.results[tname] = {}
 
-                old_ld_preload = os.environ.get("LD_PRELOAD", None)
-
-                os.environ["LD_PRELOAD"] = "build/print_status_on_exit.so "
-                os.environ["LD_PRELOAD"] += t["LD_PRELOAD"]
+                env = dict(os.environ)
+                env["LD_PRELOAD"] = env.get("LD_PRELOAD", "") + "build/print_status_on_exit.so " + t["LD_PRELOAD"]
 
                 if hasattr(self, "preallocator_hook"):
-                    self.preallocator_hook((tname, t), run,
+                    self.preallocator_hook((tname, t), run, env,
                                             verbose=src.globalvars.verbosity)
 
                 for perm in self.iterate_args():
                     i += 1
                     print_info0(i, "of", n, "\r", end='')
 
-                    perm_dict = perm._asdict()
-                    perm_dict.update(t)
-                    actual_cmd = self.cmd.format(**perm_dict)
+                    # Available substitutions in cmd
+                    substitutions = {"run": run}
+                    substitutions.update(perm._asdict())
+                    substitutions["perm"] = ("{}-"*(len(perm)-1) + "{}").format(*perm)
+                    substitutions.update(t)
+
+                    actual_cmd = self.cmd.format(**substitutions)
+                    actual_env = None
 
-                    # Find absolute path of executable
-                    binary_end = actual_cmd.find(" ")
-                    binary = subprocess.run(["whereis", actual_cmd[0:binary_end]],
-                                            stdout=subprocess.PIPE,
-                                            universal_newlines=True).stdout.split()[1]
+                    if not self.server_benchmark:
+                        # Find absolute path of executable
+                        binary_end = actual_cmd.find(" ")
+                        binary_end = None if binary_end == -1 else binary_end
+                        cmd_start = len(actual_cmd) if binary_end == None else binary_end
 
-                    actual_cmd = binary + actual_cmd[binary_end:]
+                        binary = subprocess.run(["whereis", actual_cmd[0:binary_end]],
+                                                stdout=subprocess.PIPE,
+                                                universal_newlines=True).stdout.split()[1]
 
-                    actual_cmd = t["cmd_prefix"] + " " + actual_cmd
+                        actual_cmd = "{} {} {} {}{}".format(self.measure_cmd,
+                                                            t["cmd_prefix"],
+                                                            cmd_prefix,
+                                                            binary,
+                                                            actual_cmd[cmd_start:])
+                        # substitute again
+                        actual_cmd = actual_cmd.format(**substitutions)
 
-                    actual_cmd = self.measure_cmd + " " + actual_cmd
+                        actual_env = env
 
                     print_debug("Cmd:", actual_cmd)
                     res = subprocess.run(actual_cmd.split(),
+                                         env=actual_env,
                                          stderr=subprocess.PIPE,
                                          stdout=subprocess.PIPE,
                                          universal_newlines=True)
@@ -233,23 +245,26 @@ class Benchmark (object):
                         print()
                         print_debug("Stdout:\n" + res.stdout)
                         print_debug("Stderr:\n" + res.stderr)
-                        raise Exception("{} failed with exit code {} for {}".format(actual_cmd, res.returncode, tname))
+                        print_error("{} failed with exit code {} for {}".format(actual_cmd, res.returncode, tname))
+                        break
 
                     if "ERROR: ld.so" in res.stderr:
                         print()
                         print_debug("Stderr:\n" + res.stderr)
-                        raise Exception("Preloading of {} failed for {}".format(t["LD_PRELOAD"], tname))
+                        print_error("Preloading of {} failed for {}".format(t["LD_PRELOAD"], tname))
+                        break
 
                     result = {}
 
-                    # Read VmHWM from status file. If our benchmark didn't fork
-                    # the first occurance of VmHWM is from our benchmark
-                    with open("status", "r") as f:
-                        for l in f.readlines():
-                            if l.startswith("VmHWM:"):
-                                result["VmHWM"] = l.split()[1]
-                                break
-                    os.remove("status")
+                    if not self.server_benchmark:
+                        # Read VmHWM from status file. If our benchmark didn't fork
+                        # the first occurance of VmHWM is from our benchmark
+                        with open("status", "r") as f:
+                            for l in f.readlines():
+                                if l.startswith("VmHWM:"):
+                                    result["VmHWM"] = l.split()[1]
+                                    break
+                        os.remove("status")
 
                     if hasattr(self, "process_output"):
                         self.process_output(result, res.stdout, res.stderr,
@@ -268,36 +283,33 @@ class Benchmark (object):
                                 print_warn("Exception", e, "occured on", row, "for",
                                       tname, "and", perm)
 
-                    if run == 1:
-                        self.results[tname][perm] = []
-                    self.results[tname][perm].append(result)
+                    if not dry_run:
+                        if not perm in self.results[tname]:
+                            self.results[tname][perm] = []
+                        self.results[tname][perm].append(result)
 
-                    if run == runs:
-                        self.results["mean"][tname][perm] = {}
-                        self.results["std"][tname][perm] = {}
+                        if run == runs:
+                            self.results["mean"][tname][perm] = {}
+                            self.results["std"][tname][perm] = {}
 
-                        for datapoint in self.results[tname][perm][0]:
-                            try:
-                                d = [np.float(m[datapoint]) for m in self.results[tname][perm]]
-                            except ValueError:
-                                d = np.NaN
-                            self.results["mean"][tname][perm][datapoint] = np.mean(d)
-                            self.results["std"][tname][perm][datapoint] = np.std(d)
-
-                if old_ld_preload == None:
-                    del(os.environ["LD_PRELOAD"])
-                else:
-                    os.environ["LD_PRELOAD"] = old_ld_preload
+                            for datapoint in self.results[tname][perm][0]:
+                                try:
+                                    d = [np.float(m[datapoint]) for m in self.results[tname][perm]]
+                                except ValueError:
+                                    d = np.NaN
+                                self.results["mean"][tname][perm][datapoint] = np.mean(d)
+                                self.results["std"][tname][perm][datapoint] = np.std(d)
 
                 if hasattr(self, "postallocator_hook"):
                     self.postallocator_hook((tname, t), run,
                                             verbose=src.globalvars.verbosity)
             print()
+
         # Reset PATH
         os.environ["PATH"] = os.environ["PATH"].replace(":build/" + self.name, "")
 
     def plot_single_arg(self, yval, ylabel="'y-label'", xlabel="'x-label'",
-                        autoticks=True, title="default title", filepostfix="",
+                        autoticks=True, title="'default title'", filepostfix="",
                         sumdir="", arg="", scale=None, file_ext="png"):
 
         args = self.results["args"]
diff --git a/src/benchmarks/mysql.py b/src/benchmarks/mysql.py
index e3f1f0f..0ac3f75 100644
--- a/src/benchmarks/mysql.py
+++ b/src/benchmarks/mysql.py
@@ -51,12 +51,12 @@ class Benchmark_MYSQL(Benchmark):
 
         super().__init__()
 
-    def start_and_wait_for_server(self, cmd_prefix=""):
+    def start_and_wait_for_server(self, cmd_prefix="", env=None):
         actual_cmd = cmd_prefix.split() + server_cmd
         print_info("Starting server with:", actual_cmd)
 
         self.server = subprocess.Popen(actual_cmd, stdout=PIPE, stderr=PIPE,
-                                       universal_newlines=True)
+                                       env=env, universal_newlines=True)
         # TODO make sure server comes up !
         sleep(10)
         if self.server.poll() is not None:
@@ -135,8 +135,8 @@ class Benchmark_MYSQL(Benchmark):
             print_status("Delete mysqld directory")
             shutil.rmtree("mysql_test", ignore_errors=True)
 
-    def preallocator_hook(self, allocator, run, verbose):
-        if not self.start_and_wait_for_server(cmd_prefix=allocator[1]["cmd_prefix"]):
+    def preallocator_hook(self, allocator, run, env, verbose):
+        if not self.start_and_wait_for_server(cmd_prefix=allocator[1]["cmd_prefix"], env=env):
             print_debug(allocator[1]["cmd_prefix"], file=sys.stderr)
             raise Exception("Starting mysql server for {} failed with".format(allocator[0], self.server.returncode))