intruduce new way of handling analyse and cleanup mysql

chattymalloc writes now to chattymalloc.data

1 files changed, 70 insertions, 64 deletions

diff --git a/mysql.py b/mysql.py
index d1871f7..005c4b0 100644
--- a/mysql.py
+++ b/mysql.py
@@ -1,9 +1,9 @@
 import copy
 import matplotlib.pyplot as plt
+import multiprocessing
 import numpy as np
 import os
 import pickle
-import psutil
 import re
 import shutil
 import subprocess
@@ -33,17 +33,18 @@ class Benchmark_MYSQL( Benchmark ):
         del(self.targets["hoard"])
         if "klmalloc" in self.targets:
             del(self.targets["klmalloc"])
-        self.nthreads = range(1, psutil.cpu_count() * 4 + 1, 2)
+        self.nthreads = range(1, multiprocessing.cpu_count() * 4 + 1, 2)
 
         self.results = {"args": {"nthreads" : self.nthreads},
                         "targets" : self.targets}
+        self.results.update({t : {} for t in self.targets})
 
     def start_and_wait_for_server(self, verbose, log=None):
         if not log:
             log = os.devnull
 
         with open(log, "wb") as f:
-            self.server = psutil.Popen(server_cmd, env=os.environ,
+            self.server = subprocess.Popen(server_cmd, env=os.environ,
                                                stdout=f,
                                                stderr=f,
                                                universal_newlines=True)
@@ -52,7 +53,7 @@ class Benchmark_MYSQL( Benchmark ):
         return self.server.poll() == None
 
     def prepare(self, verbose=False):
-        # Setup mysqld
+        # Setup Test Environment
         if not os.path.exists("mysql_test"):
             print("Prepare mysqld directory and database")
             os.makedirs("mysql_test")
@@ -61,11 +62,11 @@ class Benchmark_MYSQL( Benchmark ):
             if b"MariaDB" in subprocess.run(["mysqld", "--version"],
                                             stdout=PIPE).stdout:
                 init_db_cmd = ["mysql_install_db", "--basedir=/usr",
-                                "--datadir={}/mysql_test".format(os.getcwd())]
+                                "--datadir={}/mysql_test".format(cwd)]
                 if verbose:
                     print("MariaDB detected")
             else:
-                init_db_cmd = ["mysqld", "-h", "{}/mysql_test".format(os.getcwd()),
+                init_db_cmd = ["mysqld", "-h", "{}/mysql_test".format(cwd),
                                 "--initialize-insecure"]
                 if verbose:
                     print("Oracle MySQL detected")
@@ -80,6 +81,7 @@ class Benchmark_MYSQL( Benchmark ):
                 print("Starting mysqld failed")
                 return False
 
+            # Create sbtest TABLE
             p = subprocess.run("mysql -u root -S {}/mysql_test/socket".format(cwd).split(" "),
                 input = b"CREATE DATABASE sbtest;\n", stdout=PIPE, stderr=PIPE)
 
@@ -109,32 +111,19 @@ class Benchmark_MYSQL( Benchmark ):
             shutil.rmtree("mysql_test")
 
     def run(self, verbose=False, runs=3):
-        cwd = os.getcwd()
         for run in range(1, runs + 1):
             print(str(run) + ". run")
 
             # run cmd for each target
             n = len(self.nthreads)
             for tname, t in self.targets.items():
-                os.environ["LD_PRELOAD"] = t["LD_PRELOAD"] # set LD_PRELOAD
+                os.environ["LD_PRELOAD"] = "build/print_status_on_exit.so " + t["LD_PRELOAD"]
 
-                if not tname in self.results:
-                    self.results[tname] = {"heap_start" : []}
-
-                log = "mysqld.log"
-                if tname == "chattymalloc":
-                    log = "chattymalloc.data"
-                if not self.start_and_wait_for_server(verbose, log):
+                if not self.start_and_wait_for_server(verbose, "mysqld.log"):
                     print("Can't start server for", tname + ".")
                     print("Aborting Benchmark.")
                     return False
 
-                # Get initial heap size
-                for m in self.server.memory_maps():
-                    if "[heap]" in m:
-                        self.results[tname]["heap_start"].append(m.size)
-                        break
-
                 for i, thread in enumerate(self.nthreads):
                     print(tname + ":", i + 1, "of", n, "\r", end='')
 
@@ -153,38 +142,18 @@ class Benchmark_MYSQL( Benchmark ):
 
                     result = {}
 
-                    if tname == "chattymalloc":
-                        with open(log, "rb") as f:
-                            hist = {}
-                            for l in f.readlines():
-                                l = str(l, "utf-8").replace("chattymalloc: ", "")
-                                try:
-                                    n = int(l)
-                                except ValueError:
-                                    continue
-
-                                if not n in hist:
-                                    hist[n] = 0
-                                hist[n] += 1
-                        result["hist"] = hist
-                    else:
-                        result["transactions"] = re.search("transactions:\s*(\d*)", p.stdout).group(1)
-                        result["queries"] = re.search("queries:\s*(\d*)", p.stdout).group(1)
-                        # Latency
-                        result["min"] = re.search("min:\s*(\d*.\d*)", p.stdout).group(1)
-                        result["avg"] = re.search("avg:\s*(\d*.\d*)", p.stdout).group(1)
-                        result["max"] = re.search("max:\s*(\d*.\d*)", p.stdout).group(1)
-
-                        # Get memory footprint
-                        for m in self.server.memory_maps():
-                            if "[heap]" in m:
-                                result["heap_end"] = m.size
-
-                        with open("/proc/"+str(self.server.pid)+"/status", "r") as f:
-                            for l in f.readlines():
-                                if l.startswith("VmHWM:"):
-                                    result["rssmax"] = l.replace("VmHWM:", "").strip().split()[0]
-                                    break
+                    result["transactions"] = re.search("transactions:\s*(\d*)", p.stdout).group(1)
+                    result["queries"] = re.search("queries:\s*(\d*)", p.stdout).group(1)
+                    # Latency
+                    result["min"] = re.search("min:\s*(\d*.\d*)", p.stdout).group(1)
+                    result["avg"] = re.search("avg:\s*(\d*.\d*)", p.stdout).group(1)
+                    result["max"] = re.search("max:\s*(\d*.\d*)", p.stdout).group(1)
+
+                    with open("/proc/"+str(self.server.pid)+"/status", "r") as f:
+                        for l in f.readlines():
+                            if l.startswith("VmHWM:"):
+                                result["rssmax"] = l.replace("VmHWM:", "").strip().split()[0]
+                                break
 
                     if not thread in self.results:
                         self.results[tname][thread] = [result]
@@ -198,6 +167,44 @@ class Benchmark_MYSQL( Benchmark ):
 
         return True
 
+    def analyse(self, verbose=False):
+        if not self.start_and_wait_for_server(verbose, "mysqld.log"):
+            print("Can't start server.")
+            print("Aborting analysing.")
+            return False
+
+        self.results["hist"] = {}
+        runs = len(self.nthreads)
+        for i, t in enumerate(self.nthreads):
+            print("analysing", i + 1, "of", runs, "\r", end='')
+
+            target_cmd = cmd.format(t, cwd).split(" ")
+            p = subprocess.run(target_cmd, stderr=PIPE, stdout=PIPE,
+                                universal_newlines=True)
+
+            if p.returncode != 0:
+                print("\n" + " ".join(target_cmd), "exited with", p.returncode, ".\n Aborting analysing.")
+                print(p.stderr)
+                print(p.stdout)
+                self.server.kill()
+                self.server.wait()
+                return False
+
+            with open("chattymalloc.data", "r") as f:
+                hist = {}
+                for l in f.readlines():
+                    n = int(l)
+
+                    if not n in hist:
+                        hist[n] = 0
+                    hist[n] += 1
+
+                self.results["hist"][t] = hist
+
+        print()
+        self.server.kill()
+        self.server.wait()
+
     def summary(self, sd=None):
         # linear plot
         nthreads = self.results["args"]["nthreads"]
@@ -210,7 +217,7 @@ class Benchmark_MYSQL( Benchmark ):
             if target == "chattymalloc":
                 continue
             y_vals = [0] * len(nthreads)
-            for thread, measures in [(t, d) for t, d in self.results[target].items() if t != "heap_start"]:
+            for thread, measures in self.results[target].items():
                 d = [int(m["transactions"]) for m in measures]
                 y_vals[y_mapping[thread]] = np.mean(d)
             plt.plot(nthreads, y_vals, label=target, linestyle='-', marker='.', color=targets[target]["color"])
@@ -232,7 +239,7 @@ class Benchmark_MYSQL( Benchmark ):
                 continue
             x_vals = [x-i/8 for x in range(1, len(nthreads) + 1)]
             y_vals = [0] * len(nthreads)
-            for thread, measures in [(t, d) for t, d in self.results[target].items() if t != "heap_start"]:
+            for thread, measures in self.results[target].items():
                 d = [int(m["transactions"]) for m in measures]
                 y_vals[y_mapping[thread]] = np.mean(d)
             plt.bar(x_vals, y_vals, width=0.2, label=target, align="center",  color=targets[target]["color"])
@@ -246,19 +253,18 @@ class Benchmark_MYSQL( Benchmark ):
         plt.clf()
 
         # Histogram
-        del(self.results["chattymalloc"]["heap_start"])
-        for thread, measures in self.results["chattymalloc"].items():
-            print(measures)
-            hist = [(n, s) for s, n in measures[0]["hist"].items()]
-            hist.sort()
-            print("Histogram for", thread, "threads:")
-            print(hist)
+        if "hist" in self.results:
+            for thread, hist in self.results["hist"].items():
+                s = [(n, s) for s, n in hist.items()]
+                s.sort()
+                print("Histogram for", thread, "threads:")
+                print(s)
 
         # Memusage
         y_mapping = {v : i for i, v in enumerate(nthreads)}
-        for target in [t for t in targets if t != "chattymalloc"]:
+        for target in targets:
             y_vals = [0] * len(nthreads)
-            for thread, measures in [(t, d) for t, d in self.results[target].items() if t != "heap_start"]:
+            for thread, measures in self.results[target].items():
                 d = [int(m["rssmax"]) for m in measures]
                 y_vals[y_mapping[thread]] = np.mean(d)
             plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"])