diff options
| -rw-r--r-- | benchmark.py | 70 | ||||
| -rw-r--r-- | chattymalloc.c | 469 | ||||
| -rw-r--r-- | common_targets.py | 7 | ||||
| -rw-r--r-- | mysql.py | 229 |
4 files changed, 180 insertions, 595 deletions
diff --git a/benchmark.py b/benchmark.py index 92a670d..2d5fc94 100644 --- a/benchmark.py +++ b/benchmark.py @@ -42,6 +42,7 @@ class Benchmark (object): self.requirements = [] def save(self, path=None, verbose=False): + print(self.results) f = path if path else self.name + ".save" if verbose: print("Saving results to:", self.name + ".save") @@ -115,15 +116,19 @@ class Benchmark (object): def analyse(self, verbose=False): for perm in self.iterate_args(): - file_name = ".".join(list(self.name, *p.items())) - - actual_cmd = analyse_cmd.format(file_name + ".png") - if "binary_suffix" in cmd: - p["binary_suffix"] = "" - actual_cmd += cmd.format(**perm._asdict()) + perm = perm._asdict() + file_name = self.name + "." + file_name += ".".join([str(x) for x in perm.values()]) + file_name += ".memusage" + + actual_cmd = self.analyse_cmd.format(file_name + ".png") + if "binary_suffix" in self.cmd: + perm["binary_suffix"] = "" + actual_cmd += self.cmd.format(**perm) with open(file_name + ".hist", "w") as f: res = subprocess.run(actual_cmd.split(), + stdout=subprocess.PIPE, stderr=f, universal_newlines=True) @@ -133,18 +138,34 @@ class Benchmark (object): print("You may look at", file_name + ".hist", "to fix this.") return + def parse_chattymalloc_data(self, path="chattymalloc.data"): + hist = {} + with open(path, "r") as f: + for l in f.readlines(): + n = int(l) + hist[n] = hist.get(n, 0) + 1 + return hist def run(self, verbose=False, runs=5): - n = len(list(self.iterate_args())) + n = len(list(self.iterate_args())) * len(self.targets) + i = 0 for run in range(1, runs + 1): print(str(run) + ". run") - for i, perm in enumerate(self.iterate_args()): - print(i + 1, "of", n, "\r", end='') + for tname, t in self.targets.items(): + if not tname in self.results: + self.results[tname] = {} + + os.environ["LD_PRELOAD"] = "build/print_status_on_exit.so " + os.environ["LD_PRELOAD"] += t["LD_PRELOAD"] + + if hasattr(self, "pretarget_hook"): + if self.pretarget_hook((tname, t), run, verbose): + return False - for tname, t in self.targets.items(): - if not tname in self.results: - self.results[tname] = {} + for perm in self.iterate_args(): + i += 1 + print(i + 1, "of", n, "\r", end='') actual_cmd = self.perf_cmd @@ -152,16 +173,13 @@ class Benchmark (object): perm_dict.update(t) actual_cmd += self.cmd.format(**perm_dict) - os.environ["LD_PRELOAD"] = "build/print_status_on_exit.so " - os.environ["LD_PRELOAD"] += t["LD_PRELOAD"] - res = subprocess.run(actual_cmd.split(), stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) if res.returncode != 0: - print("\n" + actual_cmd, "exited with", res.returncode) + print("\n" + actual_cmd, "exited with", res.returncode, "for", tname) print("Aborting Benchmark.") print("Stdout:\n" + res.stdout) print("Stderr:\n" + res.stderr) @@ -175,7 +193,7 @@ class Benchmark (object): result = {} - # Read VmHWM from status file # If our benchmark didn't fork + # Read VmHWM from status file. If our benchmark didn't fork # the first occurance of VmHWM is from our benchmark with open("status", "r") as f: for l in f.readlines(): @@ -184,15 +202,23 @@ class Benchmark (object): break os.remove("status") - # Parse perf output - csvreader = csv.reader(res.stderr.splitlines(), delimiter=',') - for row in csvreader: - # Split of the user/kernel space info to be better portable - result[row[2].split(":")[0]] = row[0] + if hasattr(self, "process_stdout"): + self.process_stdout(result, res.stdout) + + # Parse perf output if available + if self.perf_cmd != "": + csvreader = csv.reader(res.stderr.splitlines(), delimiter=',') + for row in csvreader: + # Split of the user/kernel space info to be better portable + result[row[2].split(":")[0]] = row[0] if run == 1: self.results[tname][perm] = [] self.results[tname][perm].append(result) + + if hasattr(self, "posttarget_hook"): + if self.posttarget_hook((tname, t), run, verbose): + return False print() return True diff --git a/chattymalloc.c b/chattymalloc.c index 301e0f2..f3db0d2 100644 --- a/chattymalloc.c +++ b/chattymalloc.c @@ -1,278 +1,60 @@ #define _GNU_SOURCE #include <dlfcn.h> -#include <stdbool.h> #include <stddef.h> -#include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> -#include <time.h> -#include <signal.h> -#include <sys/syscall.h> -#include <unistd.h> -static void * (*myfn_malloc)(size_t size); -static void * (*myfn_free)(void* ptr); -static void * (*myfn_calloc)(size_t nmemb, size_t size); -static void * (*myfn_realloc)(void *ptr, size_t size); -static void * (*myfn_memalign)(size_t blocksize, size_t bytes); - -static int initializing = 0; +char tmpbuff[1024]; +unsigned long tmppos = 0; +unsigned long tmpallocs = 0; -// Memory we give out during initialisation -static char tmpbuff[4096]; -static unsigned long tmppos = 0; -static unsigned long tmpallocs = 0; - -/* Options that can be set in the CHATTYMALLOC_OPTS environment var. - * - * store: store call data in memory or dump it with each allocator call */ -static bool store = true; -/* sigusr1: dump in memory log in SIGUSR1 handler */ -static bool sigusr1 = false; -/* timestamp: get timestamp of each allocator call */ -static bool timestamp = false; -/* timestamp: get timestamp of each allocator call */ -static char* path = "chattymalloc.data"; - -void parse_options() -{ - char* options = getenv("CHATTYMALLOC_OPTS"); - if (!options) - return; - - char* opt = options; - char* val; - char c; - - for (int i = 0;; i++) - { - c = options[i]; - if (c == ':' || c == '\0') - { - if (strncmp(opt, "store", 5) == 0) - { - if (strncmp(val, "false", 5) == 0) - store = false; - } - else if (strncmp(opt, "sigusr1", 7) == 0) - { - if (strncmp(val, "true", 4) == 0) - sigusr1 = true; - } - else if (strncmp(opt, "timestamp", 9) == 0) - { - if (strncmp(val, "true", 4) == 0) - timestamp = true; - } - else - fprintf(stderr, "Unknown option \n"); +FILE* out = NULL; - if (c == '\0') - return; +/*========================================================= + * * interception points + * */ - opt = options + i + 1; - } - else if (c == '=') - { - val = options + i + 1; - } - } -} - -typedef enum { MALLOC, FREE, CALLOC, REALLOC, MEMALIGN } Func; - -// Entry in our in memory log -typedef struct Log_entry -{ - time_t time; - Func function; - uintptr_t args[3]; -} Log_entry; - -// Chunk of thread local log entries -typedef struct Log_chunk -{ - struct Log_chunk* next; - pid_t tid; - size_t n; - Log_entry entries[10000]; -} Log_chunk; - -// List of chunks -static struct Log_chunk* log_start = NULL; -static struct Log_chunk* log_end = NULL; - -static __thread pid_t tid; -static __thread Log_chunk* cur_log; - -// Flag to prevent recursion -static __thread int prevent_recursion = 0; - -static void (*old_signalhandler)(int); - -FILE* out; - -#ifdef SYS_gettid -static pid_t gettid() -{ - if (!tid) - tid = syscall(SYS_gettid); - return tid; -} -#else -#error "SYS_gettid unavailable on this system" -#endif - -static void open_output() -{ - out = fopen(path, "w"); - if (!out) - { - perror("failed to open output"); - exit(1); - } -} +static void * (*myfn_malloc)(size_t size); +static void * (*myfn_free)(void* ptr); -static void new_log_chunk (void) +static void init() { - cur_log = myfn_malloc(sizeof(Log_chunk)); - if (!cur_log) + out = fopen("chattymalloc.data", "w"); + if (out == NULL) { - perror("can't malloc chunk"); + fprintf(stderr, "failed to open output file\n"); exit(1); } - cur_log->tid = gettid(); - cur_log->n = 0; - cur_log->next = NULL; - -chain: ; - Log_chunk* old_end = log_end; - if (!__sync_bool_compare_and_swap(&log_end, old_end, cur_log)) - goto chain; - - if (old_end) - old_end->next = cur_log; -} - -static void write_log (void) -{ - Log_chunk* chunk = log_start; - - prevent_recursion = 1; - - open_output(); - - while (chunk) - { - for (size_t i = 0; i < chunk->n; i++) - { - Log_entry entry = chunk->entries[i]; - fprintf(out, "%lu %d ", entry.time, chunk->tid); - switch (entry.function) - { - case MALLOC: - fprintf(out, "ma %lu %p\n", entry.args[0], entry.args[1]); - break; - case FREE: - fprintf(out, "f %p\n", entry.args[0]); - break; - case CALLOC: - fprintf(out, "c %lu %lu %p\n", entry.args[0], entry.args[1], entry.args[2]); - break; - case REALLOC: - fprintf(out, "r %p %lu %p\n", entry.args[0], entry.args[1], entry.args[2]); - break; - case MEMALIGN: - fprintf(out, "mm %lu %lu %p\n", entry.args[0], entry.args[1], entry.args[2]); - break; - } - } - chunk = chunk->next; - } - - fclose(out); -} - -static void sigusr1_handler -(int __attribute__((__unused__)) sig) -{ - write_log(); - pid_t ppid = getppid(); - kill(ppid, SIGUSR1); - - // we are done writing the log - prevent_recursion = 0; -} - -static void init (void) -{ - initializing = 1; - myfn_malloc = dlsym(RTLD_NEXT, "malloc"); myfn_free = dlsym(RTLD_NEXT, "free"); - myfn_calloc = dlsym(RTLD_NEXT, "calloc"); - myfn_realloc = dlsym(RTLD_NEXT, "realloc"); - myfn_memalign = dlsym(RTLD_NEXT, "memalign"); - if (!myfn_malloc || !myfn_free || !myfn_calloc || !myfn_realloc || !myfn_memalign) + if (!myfn_malloc || !myfn_free) { fprintf(stderr, "Error in `dlsym`: %s\n", dlerror()); exit(1); } - - initializing = 0; - // now we can use the real allocator - prevent_recursion = 1; - - // parse chattymalloc options - parse_options(); - - if (store) - { - new_log_chunk(); - log_start = cur_log; - atexit(write_log); - } - else - open_output(); - - if (sigusr1) - { - // register USR1 signal handler to dump log - struct sigaction old; - struct sigaction sa; - - sigaction(SIGUSR1, NULL, &old); - old_signalhandler = old.sa_handler; - - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = sigusr1_handler; - sa.sa_mask = old.sa_mask; - - if (sigaction(SIGUSR1, &sa, NULL) == -1) - { - perror("Can't register our SIGUSR1 handler"); - exit(1); - } - } - - prevent_recursion = 0; } -void *malloc (size_t size) +void *malloc(size_t size) { + static int initializing = 0; + static int in_fprintf = 0; if (myfn_malloc == NULL) { if (!initializing) + { + initializing = 1; init(); + initializing = 0; + + } else { if (tmppos + size < sizeof(tmpbuff)) { - /* fprintf(stderr, "jcheck: %lu requested during init\n", size); */ void *retptr = tmpbuff + tmppos; tmppos += size; ++tmpallocs; @@ -281,217 +63,26 @@ void *malloc (size_t size) else { fprintf(stderr, "jcheck: too much memory requested during initialisation - increase tmpbuff size\n"); - *((int*) NULL) = 1; exit(1); } } } - void *ptr = myfn_malloc(size); - - if (!prevent_recursion) + if (!in_fprintf) { - struct timespec ts; - ts.tv_sec = 0; - ts.tv_nsec = 0; - - if (timestamp) - clock_gettime(CLOCK_MONOTONIC, &ts); - - if (store) - { - if (cur_log == NULL || cur_log->n == 100) - new_log_chunk(); - - Log_entry* cur_entry = &cur_log->entries[cur_log->n]; - - cur_entry->time = ts.tv_nsec; - cur_entry->function = MALLOC; - cur_entry->args[0] = (uintptr_t)size; - cur_entry->args[1] = (uintptr_t)ptr; - cur_log->n++; - } - else - { - prevent_recursion = 1; - fprintf(out, "%lu %d ma %u %p\n", ts.tv_nsec, gettid(), size, ptr); - prevent_recursion = 0; - } + in_fprintf = 1; + fprintf(out, "%d\n", size); + in_fprintf = 0; } - + void *ptr = myfn_malloc(size); return ptr; } -void free (void *ptr) +void free(void *ptr) { - if (myfn_free == NULL) + // something wrong if we call free before one of the allocators! + if (myfn_malloc == NULL) init(); if (!(ptr >= (void*) tmpbuff && ptr <= (void*)(tmpbuff + tmppos))) - { - if (!prevent_recursion) - { - struct timespec ts; - ts.tv_sec = 0; - ts.tv_nsec = 0; - - if (timestamp) - clock_gettime(CLOCK_MONOTONIC, &ts); - - if (store) - { - if (cur_log == NULL || cur_log->n == 100) - new_log_chunk(); - - Log_entry* cur_entry = &cur_log->entries[cur_log->n]; - - cur_entry->time = ts.tv_nsec; - cur_entry->function = FREE; - cur_entry->args[0] = (uintptr_t)ptr; - cur_log->n++; - } - else - { - prevent_recursion = 1; - fprintf(out, "%lu %d f %p\n", ts.tv_nsec, gettid(), ptr); - prevent_recursion = 0; - } - } - myfn_free(ptr); - } } - -void *realloc (void *ptr, size_t size) -{ - if (myfn_realloc == NULL) - { - void *nptr = malloc(size); - if (nptr && ptr) - { - memmove(nptr, ptr, size); - free(ptr); - } - return nptr; - } - - void *nptr = myfn_realloc(ptr, size); - - if (!prevent_recursion) - { - struct timespec ts; - ts.tv_sec = 0; - ts.tv_nsec = 0; - - if (timestamp) - clock_gettime(CLOCK_MONOTONIC, &ts); - - if (store) - { - if (cur_log == NULL || cur_log->n == 100) - new_log_chunk(); - - Log_entry* cur_entry = &cur_log->entries[cur_log->n]; - - cur_entry->time = ts.tv_nsec; - cur_entry->function = REALLOC; - cur_entry->args[0] = (uintptr_t)ptr; - cur_entry->args[1] = (uintptr_t)size; - cur_entry->args[2] = (uintptr_t)nptr; - cur_log->n++; - } - else - { - prevent_recursion = 1; - fprintf(out, "%lu %d r %p %u %p\n", ts.tv_nsec, gettid(), ptr, size, nptr); - prevent_recursion = 0; - } - } - - return nptr; -} - -void *calloc (size_t nmemb, size_t size) -{ - if (myfn_calloc == NULL) - { - void *ptr = malloc(nmemb*size); - if (ptr) - memset(ptr, 0, nmemb*size); - return ptr; - } - - void *ptr = myfn_calloc(nmemb, size); - - if (!prevent_recursion) - { - struct timespec ts; - ts.tv_sec = 0; - ts.tv_nsec = 0; - - if (timestamp) - clock_gettime(CLOCK_MONOTONIC, &ts); - - if (store) - { - if (cur_log == NULL || cur_log->n == 100) - new_log_chunk(); - - Log_entry* cur_entry = &cur_log->entries[cur_log->n]; - - cur_entry->time = ts.tv_nsec; - cur_entry->function = CALLOC; - cur_entry->args[0] = (uintptr_t)nmemb; - cur_entry->args[1] = (uintptr_t)size; - cur_entry->args[2] = (uintptr_t)ptr; - cur_log->n++; - } - else - { - prevent_recursion = 1; - fprintf(out, "%lu %d c %u %u %p\n", ts.tv_nsec, gettid(), nmemb, size, ptr); - prevent_recursion = 0; - } - } - - return ptr; -} - -void *memalign (size_t alignment, size_t size) -{ - // Hopefully this gets never called during init() - void *ptr = myfn_memalign(alignment, size); - - if (!prevent_recursion) - { - struct timespec ts; - ts.tv_sec = 0; - ts.tv_nsec = 0; - - if (timestamp) - clock_gettime(CLOCK_MONOTONIC, &ts); - - if (store) - { - if (cur_log == NULL || cur_log->n == 100) - new_log_chunk(); - - Log_entry* cur_entry = &cur_log->entries[cur_log->n]; - - cur_entry->time = ts.tv_nsec; - cur_entry->function = MEMALIGN; - cur_entry->args[0] = (uintptr_t)alignment; - cur_entry->args[1] = (uintptr_t)size; - cur_entry->args[2] = (uintptr_t)ptr; - cur_log->n++; - } - else - { - prevent_recursion = 1; - fprintf(out, "%lu %d mm %u %u %p\n", ts.tv_nsec, gettid(), alignment, size, ptr); - prevent_recursion = 0; - } - } - - return ptr; -} - diff --git a/common_targets.py b/common_targets.py index 6d7d539..277c3bf 100644 --- a/common_targets.py +++ b/common_targets.py @@ -1,7 +1,6 @@ import subprocess -glibc_path = "/home/cip/2014/aj46ezos/BA/glibc/glibc-install/lib" -glibc_path_notc = "/home/cip/2014/aj46ezos/BA/glibc/glibc-install-notc/lib" +glibc_path_notc = "../glibc/glibc-install-notc/lib" library_path = "" p = subprocess.run(["ldconfig", "-v"], stdout=subprocess.PIPE, @@ -38,9 +37,9 @@ common_targets = { "color" : "C4" }, "glibc-notc" : { - "cmd_prefix" : glibc_path+"/ld-2.28.9000.so " + "cmd_prefix" : glibc_path_notc+"/ld-linux-x86-64.so.2 " + "--library-path " - + glibc_path + ":" + + glibc_path_notc + ":" + library_path, "binary_suffix" : "-glibc-notc", "LD_PRELOAD" : "/usr/lib/libstdc++.so /usr/lib/libgcc_s.so.1", @@ -11,48 +11,51 @@ from time import sleep from benchmark import Benchmark from common_targets import common_targets -import chattyparser cwd = os.getcwd() prepare_cmd = ("sysbench oltp_read_only --db-driver=mysql --mysql-user=root " "--mysql-socket="+cwd+"/mysql_test/socket --tables=5 --table-size=1000000 prepare").split() -cmd = ("sysbench oltp_read_only --threads={} --time=60 --tables=5 " - "--db-driver=mysql --mysql-user=root --mysql-socket={}/mysql_test/socket run") +cmd = ("sysbench oltp_read_only --threads={nthreads} --time=1 --tables=5 " + "--db-driver=mysql --mysql-user=root --mysql-socket="+cwd+"/mysql_test/socket run") -server_cmd = ("mysqld -h {0}/mysql_test --socket={0}/mysql_test/socket " - "--secure-file-priv=").format(cwd).split() +server_cmd = (shutil.which("mysqld")+" -h "+cwd+"/mysql_test --socket="+cwd+"/mysql_test/socket " + "--secure-file-priv= ").format(cwd).split() class Benchmark_MYSQL( Benchmark ): def __init__(self): self.name = "mysql" self.descrition = """See sysbench documentation.""" + + # mysqld fails with hoard somehow self.targets = copy.copy(common_targets) del(self.targets["hoard"]) - if "klmalloc" in self.targets: - del(self.targets["klmalloc"]) - self.nthreads = range(1, multiprocessing.cpu_count() * 4 + 1, 2) - - self.results = {"args": {"nthreads" : self.nthreads}, - "targets" : self.targets} - self.results.update({t : {} for t in self.targets}) - - def start_and_wait_for_server(self, verbose, log=None): - if not log: - log = os.devnull - - with open(log, "wb") as f: - self.server = subprocess.Popen(server_cmd, env=os.environ, - stdout=f, - stderr=f, - universal_newlines=True) - #TODO make sure server comes up !!!! + + self.args = {"nthreads" : range(1, multiprocessing.cpu_count() * 4 + 1, 2)} + self.cmd = cmd + self.perf_cmd = "" + + self.requirements = ["mysqld", "sysbench"] + super().__init__() + + def start_and_wait_for_server(self, verbose, cmd_prefix=""): + actual_cmd = cmd_prefix.split() + server_cmd + if verbose: + print("Starting server with:", actual_cmd) + + self.server = subprocess.Popen(actual_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + #TODO make sure server comes up ! sleep(5) return self.server.poll() == None def prepare(self, verbose=False): + if not super().prepare(verbose=verbose): + return False # Setup Test Environment if not os.path.exists("mysql_test"): print("Prepare mysqld directory and database") @@ -62,11 +65,11 @@ class Benchmark_MYSQL( Benchmark ): if b"MariaDB" in subprocess.run(["mysqld", "--version"], stdout=PIPE).stdout: init_db_cmd = ["mysql_install_db", "--basedir=/usr", - "--datadir={}/mysql_test".format(cwd)] + "--datadir="+cwd+"/mysql_test"] if verbose: print("MariaDB detected") else: - init_db_cmd = ["mysqld", "-h", "{}/mysql_test".format(cwd), + init_db_cmd = ["mysqld", "-h", cwd+"/mysql_test", "--initialize-insecure"] if verbose: print("Oracle MySQL detected") @@ -77,7 +80,7 @@ class Benchmark_MYSQL( Benchmark ): print(p.stderr) return False - if not self.start_and_wait_for_server(verbose, "mysqld.log"): + if not self.start_and_wait_for_server(verbose): print("Starting mysqld failed") return False @@ -110,120 +113,88 @@ class Benchmark_MYSQL( Benchmark ): print("Delete mysqld directory") shutil.rmtree("mysql_test") - def run(self, verbose=False, runs=3): - for run in range(1, runs + 1): - print(str(run) + ". run") - - # run cmd for each target - n = len(self.nthreads) - for tname, t in self.targets.items(): - os.environ["LD_PRELOAD"] = "build/print_status_on_exit.so " + t["LD_PRELOAD"] - - if not self.start_and_wait_for_server(verbose, "mysqld.log"): - print("Can't start server for", tname + ".") - print("Aborting Benchmark.") - return False - - for i, thread in enumerate(self.nthreads): - print(tname + ":", i + 1, "of", n, "\r", end='') - - target_cmd = cmd.format(thread, cwd).split(" ") - p = subprocess.run(target_cmd, stderr=PIPE, stdout=PIPE, - universal_newlines=True) - - if p.returncode != 0: - print("\n" + " ".join(target_cmd), "exited with", p.returncode, ".\n Aborting Benchmark.") - print(tname, t) - print(p.stderr) - print(p.stdout) - self.server.kill() - self.server.wait() - return False - - result = {} - - result["transactions"] = re.search("transactions:\s*(\d*)", p.stdout).group(1) - result["queries"] = re.search("queries:\s*(\d*)", p.stdout).group(1) - # Latency - result["min"] = re.search("min:\s*(\d*.\d*)", p.stdout).group(1) - result["avg"] = re.search("avg:\s*(\d*.\d*)", p.stdout).group(1) - result["max"] = re.search("max:\s*(\d*.\d*)", p.stdout).group(1) - - with open("/proc/"+str(self.server.pid)+"/status", "r") as f: - for l in f.readlines(): - if l.startswith("VmHWM:"): - result["rssmax"] = l.replace("VmHWM:", "").strip().split()[0] - break - - if not thread in self.results: - self.results[tname][thread] = [result] - else: - self.results[tname][thread].append(result) + def pretarget_hook(self, target, run, verbose): + if not self.start_and_wait_for_server(verbose, + cmd_prefix=target[1]["cmd_prefix"]): + print("Can't start server for", target[0] + ".") + print("Aborting Benchmark.") + print(target[1]["cmd_prefix"]) + print(self.server.stderr.read()) + return False - print() + def posttarget_hook(self, target, run, verbose): + self.server.kill() + self.server.wait() - self.server.kill() - self.server.wait() + def process_stdout(self, result, stdout): + result["transactions"] = re.search("transactions:\s*(\d*)", stdout).group(1) + result["queries"] = re.search("queries:\s*(\d*)", stdout).group(1) + # Latency + result["min"] = re.search("min:\s*(\d*.\d*)", stdout).group(1) + result["avg"] = re.search("avg:\s*(\d*.\d*)", stdout).group(1) + result["max"] = re.search("max:\s*(\d*.\d*)", stdout).group(1) - return True + with open("/proc/"+str(self.server.pid)+"/status", "r") as f: + for l in f.readlines(): + if l.startswith("VmHWM:"): + result["rssmax"] = l.replace("VmHWM:", "").strip().split()[0] + break def analyse(self, verbose=False): + nthreads = [0] + list(self.args["nthreads"]) + failed = False + self.results["hist"] = {} os.environ["LD_PRELOAD"] = "build/chattymalloc.so" - os.environ["CHATTYMALLOC_OPTS"] = "store=false" - if not self.start_and_wait_for_server(verbose, "mysqld.log"): - print("Can't start server.") - print("Aborting analysing.") - return False - - self.server.kill() - self.server.wait() - self.results["hist"][0] = chattyparser.hist() - - runs = len(self.nthreads) - for i, t in enumerate(self.nthreads): + runs = len(nthreads) + for i, t in enumerate(nthreads): print("analysing", i + 1, "of", runs, "\r", end='') - os.environ["LD_PRELOAD"] = "build/chattymalloc.so" - os.environ["CHATTYMALLOC_OPTS"] = "store=false" - if not self.start_and_wait_for_server(verbose, "mysqld.log"): + if not self.start_and_wait_for_server(verbose): print("Can't start server.") print("Aborting analysing.") return False - os.environ["LD_PRELOAD"] = "" - - target_cmd = cmd.format(t, cwd).split(" ") - p = subprocess.run(target_cmd, stderr=PIPE, stdout=PIPE, - universal_newlines=True) + if t != 0: + target_cmd = self.cmd.format(nthreads=t).split(" ") + p = subprocess.run(target_cmd, + stderr=PIPE, + stdout=PIPE, + universal_newlines=True) - if p.returncode != 0: - print("\n" + " ".join(target_cmd), "exited with", p.returncode, ".\n Aborting analysing.") - print(p.stderr) - print(p.stdout) - return False + if p.returncode != 0: + print("\n" + " ".join(target_cmd), "exited with", p.returncode, ".\n Aborting analysing.") + print(p.stderr) + print(p.stdout) + failed = True self.server.kill() self.server.wait() - self.results["hist"][t] = chattyparser.hist(path="chattymalloc.data") + + self.results["hist"][t] = self.parse_chattymalloc_data() + + if failed: + print(self.server.stdout.read()) + print(self.server.stderr.read()) + return False print() def summary(self, sd=None): - # linear plot - nthreads = self.results["args"]["nthreads"] - targets = self.results["targets"] - y_mapping = {v: i for i, v in enumerate(nthreads)} - sd = sd or "" + targets = self.results["targets"] + nthreads = list(self.results["args"]["nthreads"]) + # linear plot for target in targets: - y_vals = [0] * len(nthreads) - for thread, measures in self.results[target].items(): - d = [int(m["transactions"]) for m in measures] - y_vals[y_mapping[thread]] = np.mean(d) - plt.plot(nthreads, y_vals, label=target, linestyle='-', marker='.', color=targets[target]["color"]) + y_vals = [] + print(self.results[target]) + for perm in self.iterate_args(): + d = [int(m["transactions"]) for m in self.results[target][perm]] + y_vals.append(np.mean(d)) + plt.plot(nthreads, y_vals, label=target, linestyle='-', + marker='.', color=targets[target]["color"]) plt.legend() plt.xlabel("threads") @@ -233,17 +204,14 @@ class Benchmark_MYSQL( Benchmark ): plt.clf() # bar plot - nthreads = list(self.results["args"]["nthreads"]) - targets = self.results["targets"] - y_mapping = {v: i for i, v in enumerate(nthreads)} - for i, target in enumerate(targets): + y_vals = [] + for perm in self.iterate_args(): + d = [int(m["transactions"]) for m in self.results[target][perm]] + y_vals.append(np.mean(d)) x_vals = [x-i/8 for x in range(1, len(nthreads) + 1)] - y_vals = [0] * len(nthreads) - for thread, measures in self.results[target].items(): - d = [int(m["transactions"]) for m in measures] - y_vals[y_mapping[thread]] = np.mean(d) - plt.bar(x_vals, y_vals, width=0.2, label=target, align="center", color=targets[target]["color"]) + plt.bar(x_vals, y_vals, width=0.2, label=target, align="center", + color=targets[target]["color"]) plt.legend() plt.xlabel("threads") @@ -276,14 +244,15 @@ class Benchmark_MYSQL( Benchmark ): plt.savefig(os.path.join(sd, self.name + ".hist." + str(t) + ".png")) plt.clf() + # Memusage - y_mapping = {v : i for i, v in enumerate(nthreads)} for target in targets: - y_vals = [0] * len(nthreads) - for thread, measures in self.results[target].items(): - d = [int(m["rssmax"]) for m in measures] - y_vals[y_mapping[thread]] = np.mean(d) - plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"]) + y_vals = [] + for perm in self.iterate_args(): + d = [int(m["rssmax"]) for m in self.results[target][perm]] + y_vals.append(np.mean(d)) + plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target, + color=targets[target]["color"]) plt.legend() plt.xlabel("threads") |