diff options
Diffstat (limited to 'mysql.py')
| -rw-r--r-- | mysql.py | 134 |
1 files changed, 70 insertions, 64 deletions
@@ -1,9 +1,9 @@ import copy import matplotlib.pyplot as plt +import multiprocessing import numpy as np import os import pickle -import psutil import re import shutil import subprocess @@ -33,17 +33,18 @@ class Benchmark_MYSQL( Benchmark ): del(self.targets["hoard"]) if "klmalloc" in self.targets: del(self.targets["klmalloc"]) - self.nthreads = range(1, psutil.cpu_count() * 4 + 1, 2) + self.nthreads = range(1, multiprocessing.cpu_count() * 4 + 1, 2) self.results = {"args": {"nthreads" : self.nthreads}, "targets" : self.targets} + self.results.update({t : {} for t in self.targets}) def start_and_wait_for_server(self, verbose, log=None): if not log: log = os.devnull with open(log, "wb") as f: - self.server = psutil.Popen(server_cmd, env=os.environ, + self.server = subprocess.Popen(server_cmd, env=os.environ, stdout=f, stderr=f, universal_newlines=True) @@ -52,7 +53,7 @@ class Benchmark_MYSQL( Benchmark ): return self.server.poll() == None def prepare(self, verbose=False): - # Setup mysqld + # Setup Test Environment if not os.path.exists("mysql_test"): print("Prepare mysqld directory and database") os.makedirs("mysql_test") @@ -61,11 +62,11 @@ class Benchmark_MYSQL( Benchmark ): if b"MariaDB" in subprocess.run(["mysqld", "--version"], stdout=PIPE).stdout: init_db_cmd = ["mysql_install_db", "--basedir=/usr", - "--datadir={}/mysql_test".format(os.getcwd())] + "--datadir={}/mysql_test".format(cwd)] if verbose: print("MariaDB detected") else: - init_db_cmd = ["mysqld", "-h", "{}/mysql_test".format(os.getcwd()), + init_db_cmd = ["mysqld", "-h", "{}/mysql_test".format(cwd), "--initialize-insecure"] if verbose: print("Oracle MySQL detected") @@ -80,6 +81,7 @@ class Benchmark_MYSQL( Benchmark ): print("Starting mysqld failed") return False + # Create sbtest TABLE p = subprocess.run("mysql -u root -S {}/mysql_test/socket".format(cwd).split(" "), input = b"CREATE DATABASE sbtest;\n", stdout=PIPE, stderr=PIPE) @@ -109,32 +111,19 @@ class Benchmark_MYSQL( Benchmark ): shutil.rmtree("mysql_test") def run(self, verbose=False, runs=3): - cwd = os.getcwd() for run in range(1, runs + 1): print(str(run) + ". run") # run cmd for each target n = len(self.nthreads) for tname, t in self.targets.items(): - os.environ["LD_PRELOAD"] = t["LD_PRELOAD"] # set LD_PRELOAD + os.environ["LD_PRELOAD"] = "build/print_status_on_exit.so " + t["LD_PRELOAD"] - if not tname in self.results: - self.results[tname] = {"heap_start" : []} - - log = "mysqld.log" - if tname == "chattymalloc": - log = "chattymalloc.data" - if not self.start_and_wait_for_server(verbose, log): + if not self.start_and_wait_for_server(verbose, "mysqld.log"): print("Can't start server for", tname + ".") print("Aborting Benchmark.") return False - # Get initial heap size - for m in self.server.memory_maps(): - if "[heap]" in m: - self.results[tname]["heap_start"].append(m.size) - break - for i, thread in enumerate(self.nthreads): print(tname + ":", i + 1, "of", n, "\r", end='') @@ -153,38 +142,18 @@ class Benchmark_MYSQL( Benchmark ): result = {} - if tname == "chattymalloc": - with open(log, "rb") as f: - hist = {} - for l in f.readlines(): - l = str(l, "utf-8").replace("chattymalloc: ", "") - try: - n = int(l) - except ValueError: - continue - - if not n in hist: - hist[n] = 0 - hist[n] += 1 - result["hist"] = hist - else: - result["transactions"] = re.search("transactions:\s*(\d*)", p.stdout).group(1) - result["queries"] = re.search("queries:\s*(\d*)", p.stdout).group(1) - # Latency - result["min"] = re.search("min:\s*(\d*.\d*)", p.stdout).group(1) - result["avg"] = re.search("avg:\s*(\d*.\d*)", p.stdout).group(1) - result["max"] = re.search("max:\s*(\d*.\d*)", p.stdout).group(1) - - # Get memory footprint - for m in self.server.memory_maps(): - if "[heap]" in m: - result["heap_end"] = m.size - - with open("/proc/"+str(self.server.pid)+"/status", "r") as f: - for l in f.readlines(): - if l.startswith("VmHWM:"): - result["rssmax"] = l.replace("VmHWM:", "").strip().split()[0] - break + result["transactions"] = re.search("transactions:\s*(\d*)", p.stdout).group(1) + result["queries"] = re.search("queries:\s*(\d*)", p.stdout).group(1) + # Latency + result["min"] = re.search("min:\s*(\d*.\d*)", p.stdout).group(1) + result["avg"] = re.search("avg:\s*(\d*.\d*)", p.stdout).group(1) + result["max"] = re.search("max:\s*(\d*.\d*)", p.stdout).group(1) + + with open("/proc/"+str(self.server.pid)+"/status", "r") as f: + for l in f.readlines(): + if l.startswith("VmHWM:"): + result["rssmax"] = l.replace("VmHWM:", "").strip().split()[0] + break if not thread in self.results: self.results[tname][thread] = [result] @@ -198,6 +167,44 @@ class Benchmark_MYSQL( Benchmark ): return True + def analyse(self, verbose=False): + if not self.start_and_wait_for_server(verbose, "mysqld.log"): + print("Can't start server.") + print("Aborting analysing.") + return False + + self.results["hist"] = {} + runs = len(self.nthreads) + for i, t in enumerate(self.nthreads): + print("analysing", i + 1, "of", runs, "\r", end='') + + target_cmd = cmd.format(t, cwd).split(" ") + p = subprocess.run(target_cmd, stderr=PIPE, stdout=PIPE, + universal_newlines=True) + + if p.returncode != 0: + print("\n" + " ".join(target_cmd), "exited with", p.returncode, ".\n Aborting analysing.") + print(p.stderr) + print(p.stdout) + self.server.kill() + self.server.wait() + return False + + with open("chattymalloc.data", "r") as f: + hist = {} + for l in f.readlines(): + n = int(l) + + if not n in hist: + hist[n] = 0 + hist[n] += 1 + + self.results["hist"][t] = hist + + print() + self.server.kill() + self.server.wait() + def summary(self, sd=None): # linear plot nthreads = self.results["args"]["nthreads"] @@ -210,7 +217,7 @@ class Benchmark_MYSQL( Benchmark ): if target == "chattymalloc": continue y_vals = [0] * len(nthreads) - for thread, measures in [(t, d) for t, d in self.results[target].items() if t != "heap_start"]: + for thread, measures in self.results[target].items(): d = [int(m["transactions"]) for m in measures] y_vals[y_mapping[thread]] = np.mean(d) plt.plot(nthreads, y_vals, label=target, linestyle='-', marker='.', color=targets[target]["color"]) @@ -232,7 +239,7 @@ class Benchmark_MYSQL( Benchmark ): continue x_vals = [x-i/8 for x in range(1, len(nthreads) + 1)] y_vals = [0] * len(nthreads) - for thread, measures in [(t, d) for t, d in self.results[target].items() if t != "heap_start"]: + for thread, measures in self.results[target].items(): d = [int(m["transactions"]) for m in measures] y_vals[y_mapping[thread]] = np.mean(d) plt.bar(x_vals, y_vals, width=0.2, label=target, align="center", color=targets[target]["color"]) @@ -246,19 +253,18 @@ class Benchmark_MYSQL( Benchmark ): plt.clf() # Histogram - del(self.results["chattymalloc"]["heap_start"]) - for thread, measures in self.results["chattymalloc"].items(): - print(measures) - hist = [(n, s) for s, n in measures[0]["hist"].items()] - hist.sort() - print("Histogram for", thread, "threads:") - print(hist) + if "hist" in self.results: + for thread, hist in self.results["hist"].items(): + s = [(n, s) for s, n in hist.items()] + s.sort() + print("Histogram for", thread, "threads:") + print(s) # Memusage y_mapping = {v : i for i, v in enumerate(nthreads)} - for target in [t for t in targets if t != "chattymalloc"]: + for target in targets: y_vals = [0] * len(nthreads) - for thread, measures in [(t, d) for t, d in self.results[target].items() if t != "heap_start"]: + for thread, measures in self.results[target].items(): d = [int(m["rssmax"]) for m in measures] y_vals[y_mapping[thread]] = np.mean(d) plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"]) |