intruduce new way of handling analyse and cleanup mysql

chattymalloc writes now to chattymalloc.data

5 files changed, 205 insertions, 123 deletions

diff --git a/Makefile b/Makefile
index fa40390..1f5cebf 100644
--- a/Makefile
+++ b/Makefile
@@ -29,7 +29,12 @@ MAKEFILE_LIST = Makefile
 
 TARGETS = $(OBJPRE:.o=)
 
-all: $(TARGETS) $(OBJDIR)/chattymalloc.so
+all: $(TARGETS) $(OBJDIR)/chattymalloc.so $(OBJDIR)/print_status_on_exit.so
+
+$(OBJDIR)/print_status_on_exit.so: print_status_on_exit.c $(MAKEFILE_LIST)
+	@echo "cc		$@"
+	@if test \( ! \( -d $(@D) \) \) ;then mkdir -p $(@D);fi
+	$(VERBOSE) $(CC) -shared $(CFLAGS) -o $@ $< -ldl
 
 $(OBJDIR)/chattymalloc.so: chattymalloc.c $(MAKEFILE_LIST)
 	@echo "cc		$@"
diff --git a/bench.py b/bench.py
index d47c6b3..570258a 100755
--- a/bench.py
+++ b/bench.py
@@ -36,15 +36,16 @@ def main():
         if args.load:
             bench.load()
 
-        if args.analyse and bench.name == "mysql":
-            bench.targets.update(common_targets.analyse_targets)
-
-        print("Preparing", bench.name)
+        print("Preparing", bench.name, "...")
         if not bench.prepare():
             print("Preparing", bench.name, "failed!")
             return
 
-        print("Running", bench.name)
+        if args.analyse and hasattr(bench, "analyse") and callable(bench.analyse):
+            print("Analysing", bench.name, "...")
+            bench.analyse(verbose=args.verbose)
+
+        print("Running", bench.name, "...")
         if not bench.run(runs=args.runs, verbose=args.verbose):
             continue
 
@@ -52,11 +53,11 @@ def main():
             bench.save()
 
         if not args.nosum:
-            print("Summarizing", bench.name)
+            print("Summarizing", bench.name, "...")
             bench.summary(args.summarydir)
 
         if hasattr(bench, "cleanup"):
-            print("Cleaning after", bench.name)
+            print("Cleaning up", bench.name, "...")
             bench.cleanup()
 
 if __name__ == "__main__":
diff --git a/chattymalloc.c b/chattymalloc.c
index f94afde..f3db0d2 100644
--- a/chattymalloc.c
+++ b/chattymalloc.c
@@ -3,33 +3,34 @@
 #include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
 char tmpbuff[1024];
 unsigned long tmppos = 0;
 unsigned long tmpallocs = 0;
 
-void *memset(void*,int,size_t);
-void *memmove(void *to, const void *from, size_t size);
+FILE* out = NULL;
 
 /*=========================================================
  *  * interception points
  *  */
 
-static void * (*myfn_calloc)(size_t nmemb, size_t size);
 static void * (*myfn_malloc)(size_t size);
-static void   (*myfn_free)(void *ptr);
-static void * (*myfn_realloc)(void *ptr, size_t size);
-static void * (*myfn_memalign)(size_t blocksize, size_t bytes);
+static void * (*myfn_free)(void* ptr);
 
 static void init()
 {
+	out = fopen("chattymalloc.data", "w");
+	if (out == NULL)
+	{
+		fprintf(stderr, "failed to open output file\n");
+		exit(1);
+	}
+
 	myfn_malloc     = dlsym(RTLD_NEXT, "malloc");
 	myfn_free       = dlsym(RTLD_NEXT, "free");
-	myfn_calloc     = dlsym(RTLD_NEXT, "calloc");
-	myfn_realloc    = dlsym(RTLD_NEXT, "realloc");
-	myfn_memalign   = dlsym(RTLD_NEXT, "memalign");
 
-	if (!myfn_malloc || !myfn_free || !myfn_calloc || !myfn_realloc || !myfn_memalign)
+	if (!myfn_malloc || !myfn_free)
 	{
 		fprintf(stderr, "Error in `dlsym`: %s\n", dlerror());
 		exit(1);
@@ -39,6 +40,8 @@ static void init()
 void *malloc(size_t size)
 {
 	static int initializing = 0;
+	static int in_fprintf = 0;
+
 	if (myfn_malloc == NULL)
 	{
 		if (!initializing)
@@ -47,7 +50,6 @@ void *malloc(size_t size)
 			init();
 			initializing = 0;
 
-			fprintf(stdout, "jcheck: allocated %lu bytes of temp memory in %lu chunks during initialization\n", tmppos, tmpallocs);
 		}
 		else
 		{
@@ -60,13 +62,18 @@ void *malloc(size_t size)
 			}
 			else
 			{
-				fprintf(stdout, "jcheck: too much memory requested during initialisation - increase tmpbuff size\n");
+				fprintf(stderr, "jcheck: too much memory requested during initialisation - increase tmpbuff size\n");
 				exit(1);
 			}
 		}
 	}
 
-	fprintf(stderr, "chattymalloc: %d\n", size);
+	if (!in_fprintf)
+	{
+		in_fprintf = 1;
+		fprintf(out, "%d\n", size);
+		in_fprintf = 0;
+	}
 	void *ptr = myfn_malloc(size);
 	return ptr;
 }
@@ -76,43 +83,6 @@ void free(void *ptr)
 	// something wrong if we call free before one of the allocators!
 	if (myfn_malloc == NULL)
 		init();
-	if (ptr >= (void*) tmpbuff && ptr <= (void*)(tmpbuff + tmppos))
-		fprintf(stdout, "freeing temp memory\n");
-	else
+	if (!(ptr >= (void*) tmpbuff && ptr <= (void*)(tmpbuff + tmppos)))
 		myfn_free(ptr);
 }
-
-void *realloc(void *ptr, size_t size)
-{
-	if (myfn_malloc == NULL)
-	{
-		void *nptr = malloc(size);
-		if (nptr && ptr)
-		{
-			memmove(nptr, ptr, size);
-			free(ptr);
-		}
-		return nptr;
-	}
-	void *nptr = myfn_realloc(ptr, size);
-	return nptr;
-}
-
-void *calloc(size_t nmemb, size_t size)
-{
-	if (myfn_malloc == NULL)
-	{
-		void *ptr = malloc(nmemb*size);
-		if (ptr)
-		memset(ptr, 0, nmemb*size);
-		return ptr;
-	}
-	void *ptr = myfn_calloc(nmemb, size);
-	return ptr;
-}
-
-void *memalign(size_t blocksize, size_t bytes)
-{
-	void *ptr = myfn_memalign(blocksize, bytes);
-	return ptr;
-}
diff --git a/mysql.py b/mysql.py
index d1871f7..005c4b0 100644
--- a/mysql.py
+++ b/mysql.py
@@ -1,9 +1,9 @@
 import copy
 import matplotlib.pyplot as plt
+import multiprocessing
 import numpy as np
 import os
 import pickle
-import psutil
 import re
 import shutil
 import subprocess
@@ -33,17 +33,18 @@ class Benchmark_MYSQL( Benchmark ):
         del(self.targets["hoard"])
         if "klmalloc" in self.targets:
             del(self.targets["klmalloc"])
-        self.nthreads = range(1, psutil.cpu_count() * 4 + 1, 2)
+        self.nthreads = range(1, multiprocessing.cpu_count() * 4 + 1, 2)
 
         self.results = {"args": {"nthreads" : self.nthreads},
                         "targets" : self.targets}
+        self.results.update({t : {} for t in self.targets})
 
     def start_and_wait_for_server(self, verbose, log=None):
         if not log:
             log = os.devnull
 
         with open(log, "wb") as f:
-            self.server = psutil.Popen(server_cmd, env=os.environ,
+            self.server = subprocess.Popen(server_cmd, env=os.environ,
                                                stdout=f,
                                                stderr=f,
                                                universal_newlines=True)
@@ -52,7 +53,7 @@ class Benchmark_MYSQL( Benchmark ):
         return self.server.poll() == None
 
     def prepare(self, verbose=False):
-        # Setup mysqld
+        # Setup Test Environment
         if not os.path.exists("mysql_test"):
             print("Prepare mysqld directory and database")
             os.makedirs("mysql_test")
@@ -61,11 +62,11 @@ class Benchmark_MYSQL( Benchmark ):
             if b"MariaDB" in subprocess.run(["mysqld", "--version"],
                                             stdout=PIPE).stdout:
                 init_db_cmd = ["mysql_install_db", "--basedir=/usr",
-                                "--datadir={}/mysql_test".format(os.getcwd())]
+                                "--datadir={}/mysql_test".format(cwd)]
                 if verbose:
                     print("MariaDB detected")
             else:
-                init_db_cmd = ["mysqld", "-h", "{}/mysql_test".format(os.getcwd()),
+                init_db_cmd = ["mysqld", "-h", "{}/mysql_test".format(cwd),
                                 "--initialize-insecure"]
                 if verbose:
                     print("Oracle MySQL detected")
@@ -80,6 +81,7 @@ class Benchmark_MYSQL( Benchmark ):
                 print("Starting mysqld failed")
                 return False
 
+            # Create sbtest TABLE
             p = subprocess.run("mysql -u root -S {}/mysql_test/socket".format(cwd).split(" "),
                 input = b"CREATE DATABASE sbtest;\n", stdout=PIPE, stderr=PIPE)
 
@@ -109,32 +111,19 @@ class Benchmark_MYSQL( Benchmark ):
             shutil.rmtree("mysql_test")
 
     def run(self, verbose=False, runs=3):
-        cwd = os.getcwd()
         for run in range(1, runs + 1):
             print(str(run) + ". run")
 
             # run cmd for each target
             n = len(self.nthreads)
             for tname, t in self.targets.items():
-                os.environ["LD_PRELOAD"] = t["LD_PRELOAD"] # set LD_PRELOAD
+                os.environ["LD_PRELOAD"] = "build/print_status_on_exit.so " + t["LD_PRELOAD"]
 
-                if not tname in self.results:
-                    self.results[tname] = {"heap_start" : []}
-
-                log = "mysqld.log"
-                if tname == "chattymalloc":
-                    log = "chattymalloc.data"
-                if not self.start_and_wait_for_server(verbose, log):
+                if not self.start_and_wait_for_server(verbose, "mysqld.log"):
                     print("Can't start server for", tname + ".")
                     print("Aborting Benchmark.")
                     return False
 
-                # Get initial heap size
-                for m in self.server.memory_maps():
-                    if "[heap]" in m:
-                        self.results[tname]["heap_start"].append(m.size)
-                        break
-
                 for i, thread in enumerate(self.nthreads):
                     print(tname + ":", i + 1, "of", n, "\r", end='')
 
@@ -153,38 +142,18 @@ class Benchmark_MYSQL( Benchmark ):
 
                     result = {}
 
-                    if tname == "chattymalloc":
-                        with open(log, "rb") as f:
-                            hist = {}
-                            for l in f.readlines():
-                                l = str(l, "utf-8").replace("chattymalloc: ", "")
-                                try:
-                                    n = int(l)
-                                except ValueError:
-                                    continue
-
-                                if not n in hist:
-                                    hist[n] = 0
-                                hist[n] += 1
-                        result["hist"] = hist
-                    else:
-                        result["transactions"] = re.search("transactions:\s*(\d*)", p.stdout).group(1)
-                        result["queries"] = re.search("queries:\s*(\d*)", p.stdout).group(1)
-                        # Latency
-                        result["min"] = re.search("min:\s*(\d*.\d*)", p.stdout).group(1)
-                        result["avg"] = re.search("avg:\s*(\d*.\d*)", p.stdout).group(1)
-                        result["max"] = re.search("max:\s*(\d*.\d*)", p.stdout).group(1)
-
-                        # Get memory footprint
-                        for m in self.server.memory_maps():
-                            if "[heap]" in m:
-                                result["heap_end"] = m.size
-
-                        with open("/proc/"+str(self.server.pid)+"/status", "r") as f:
-                            for l in f.readlines():
-                                if l.startswith("VmHWM:"):
-                                    result["rssmax"] = l.replace("VmHWM:", "").strip().split()[0]
-                                    break
+                    result["transactions"] = re.search("transactions:\s*(\d*)", p.stdout).group(1)
+                    result["queries"] = re.search("queries:\s*(\d*)", p.stdout).group(1)
+                    # Latency
+                    result["min"] = re.search("min:\s*(\d*.\d*)", p.stdout).group(1)
+                    result["avg"] = re.search("avg:\s*(\d*.\d*)", p.stdout).group(1)
+                    result["max"] = re.search("max:\s*(\d*.\d*)", p.stdout).group(1)
+
+                    with open("/proc/"+str(self.server.pid)+"/status", "r") as f:
+                        for l in f.readlines():
+                            if l.startswith("VmHWM:"):
+                                result["rssmax"] = l.replace("VmHWM:", "").strip().split()[0]
+                                break
 
                     if not thread in self.results:
                         self.results[tname][thread] = [result]
@@ -198,6 +167,44 @@ class Benchmark_MYSQL( Benchmark ):
 
         return True
 
+    def analyse(self, verbose=False):
+        if not self.start_and_wait_for_server(verbose, "mysqld.log"):
+            print("Can't start server.")
+            print("Aborting analysing.")
+            return False
+
+        self.results["hist"] = {}
+        runs = len(self.nthreads)
+        for i, t in enumerate(self.nthreads):
+            print("analysing", i + 1, "of", runs, "\r", end='')
+
+            target_cmd = cmd.format(t, cwd).split(" ")
+            p = subprocess.run(target_cmd, stderr=PIPE, stdout=PIPE,
+                                universal_newlines=True)
+
+            if p.returncode != 0:
+                print("\n" + " ".join(target_cmd), "exited with", p.returncode, ".\n Aborting analysing.")
+                print(p.stderr)
+                print(p.stdout)
+                self.server.kill()
+                self.server.wait()
+                return False
+
+            with open("chattymalloc.data", "r") as f:
+                hist = {}
+                for l in f.readlines():
+                    n = int(l)
+
+                    if not n in hist:
+                        hist[n] = 0
+                    hist[n] += 1
+
+                self.results["hist"][t] = hist
+
+        print()
+        self.server.kill()
+        self.server.wait()
+
     def summary(self, sd=None):
         # linear plot
         nthreads = self.results["args"]["nthreads"]
@@ -210,7 +217,7 @@ class Benchmark_MYSQL( Benchmark ):
             if target == "chattymalloc":
                 continue
             y_vals = [0] * len(nthreads)
-            for thread, measures in [(t, d) for t, d in self.results[target].items() if t != "heap_start"]:
+            for thread, measures in self.results[target].items():
                 d = [int(m["transactions"]) for m in measures]
                 y_vals[y_mapping[thread]] = np.mean(d)
             plt.plot(nthreads, y_vals, label=target, linestyle='-', marker='.', color=targets[target]["color"])
@@ -232,7 +239,7 @@ class Benchmark_MYSQL( Benchmark ):
                 continue
             x_vals = [x-i/8 for x in range(1, len(nthreads) + 1)]
             y_vals = [0] * len(nthreads)
-            for thread, measures in [(t, d) for t, d in self.results[target].items() if t != "heap_start"]:
+            for thread, measures in self.results[target].items():
                 d = [int(m["transactions"]) for m in measures]
                 y_vals[y_mapping[thread]] = np.mean(d)
             plt.bar(x_vals, y_vals, width=0.2, label=target, align="center",  color=targets[target]["color"])
@@ -246,19 +253,18 @@ class Benchmark_MYSQL( Benchmark ):
         plt.clf()
 
         # Histogram
-        del(self.results["chattymalloc"]["heap_start"])
-        for thread, measures in self.results["chattymalloc"].items():
-            print(measures)
-            hist = [(n, s) for s, n in measures[0]["hist"].items()]
-            hist.sort()
-            print("Histogram for", thread, "threads:")
-            print(hist)
+        if "hist" in self.results:
+            for thread, hist in self.results["hist"].items():
+                s = [(n, s) for s, n in hist.items()]
+                s.sort()
+                print("Histogram for", thread, "threads:")
+                print(s)
 
         # Memusage
         y_mapping = {v : i for i, v in enumerate(nthreads)}
-        for target in [t for t in targets if t != "chattymalloc"]:
+        for target in targets:
             y_vals = [0] * len(nthreads)
-            for thread, measures in [(t, d) for t, d in self.results[target].items() if t != "heap_start"]:
+            for thread, measures in self.results[target].items():
                 d = [int(m["rssmax"]) for m in measures]
                 y_vals[y_mapping[thread]] = np.mean(d)
             plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"])
diff --git a/print_status_on_exit.c b/print_status_on_exit.c
new file mode 100644
index 0000000..686817e
--- /dev/null
+++ b/print_status_on_exit.c
@@ -0,0 +1,100 @@
+#define _GNU_SOURCE
+#include <dlfcn.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+char tmpbuff[1024];
+unsigned long tmppos = 0;
+unsigned long tmpallocs = 0;
+
+/*=========================================================
+ *  * interception points
+ *  */
+
+static void * (*myfn_malloc)(size_t size);
+static void   (*myfn_free)(void *ptr);
+
+static void print_status(void)
+{
+	char buf[4096];
+
+	FILE* status = fopen("/proc/self/status", "r");
+	if (status == NULL)
+	{
+		perror("fopen status");
+		exit(1);
+	}
+
+	FILE* output = fopen("status", "w");
+	if (output == NULL)
+	{
+		perror("fopen output file");
+		exit(1);
+	}
+
+	while (!feof(status))
+	{
+		fgets(&buf, 4096, status);
+		fprintf(output, "%s", buf);
+	}
+	fclose(status);
+}
+
+static void init()
+{
+	myfn_malloc     = dlsym(RTLD_NEXT, "malloc");
+	myfn_free       = dlsym(RTLD_NEXT, "free");
+
+	if (!myfn_malloc || !myfn_free)
+	{
+		fprintf(stderr, "Error in `dlsym`: %s\n", dlerror());
+		exit(1);
+	}
+	atexit(print_status);
+}
+
+void *malloc(size_t size)
+{
+	static int initializing = 0;
+	if (myfn_malloc == NULL)
+	{
+		if (!initializing)
+		{
+			initializing = 1;
+			init();
+			initializing = 0;
+
+			fprintf(stdout, "jcheck: allocated %lu bytes of temp memory in %lu chunks during initialization\n", tmppos, tmpallocs);
+		}
+		else
+		{
+			if (tmppos + size < sizeof(tmpbuff))
+			{
+				void *retptr = tmpbuff + tmppos;
+				tmppos += size;
+				++tmpallocs;
+				return retptr;
+			}
+			else
+			{
+				fprintf(stdout, "jcheck: too much memory requested during initialisation - increase tmpbuff size\n");
+				exit(1);
+			}
+		}
+	}
+
+	return myfn_malloc(size);
+}
+
+void free(void *ptr)
+{
+	// something wrong if we call free before one of the allocators!
+	if (myfn_malloc == NULL)
+		init();
+	if (ptr >= (void*) tmpbuff && ptr <= (void*)(tmpbuff + tmppos))
+		fprintf(stdout, "freeing temp memory\n");
+	else
+		myfn_free(ptr);
+}
+