From ddbaf12a94fd43481fc9dd3b98920fc784ebada3 Mon Sep 17 00:00:00 2001 From: Florian Fischer Date: Sat, 2 Feb 2019 14:56:51 +0100 Subject: fix some flake8 linting errors --- src/benchmark.py | 97 ++++++++++++++++++++++++++++------------------------- src/chattyparser.py | 43 ++++++++++++++---------- src/dj_trace.py | 70 +++++++++++++++++++------------------- src/falsesharing.py | 41 +++++++++++----------- src/larson.py | 35 ++++++++++--------- src/loop.py | 34 ++++++++++--------- src/mysql.py | 59 ++++++++++++++++---------------- 7 files changed, 202 insertions(+), 177 deletions(-) (limited to 'src') diff --git a/src/benchmark.py b/src/benchmark.py index 67e8157..e29df66 100644 --- a/src/benchmark.py +++ b/src/benchmark.py @@ -1,5 +1,4 @@ from collections import namedtuple -import copy import csv import itertools import matplotlib.pyplot as plt @@ -11,16 +10,18 @@ import subprocess from src.targets import targets + class Benchmark (object): defaults = { - "name" : "default_benchmark", - "description" : "This is the default benchmark description please add your own useful one.", - - "measure_cmd" : "perf stat -x, -d", - "analyse_cmd" : "memusage -p {} -t", - "cmd" : "true", - "targets" : targets, + "name": "default_benchmark", + "description": ("This is the default benchmark description please add" + "your own useful one."), + + "measure_cmd": "perf stat -x, -d", + "analyse_cmd": "memusage -p {} -t", + "cmd": "true", + "targets": targets, } def __init__(self): @@ -39,7 +40,7 @@ class Benchmark (object): self.results = {} self.results["args"] = self.args self.results["targets"] = self.targets - self.results.update({t : {} for t in self.targets}) + self.results.update({t: {} for t in self.targets}) if not hasattr(self, "requirements"): self.requirements = [] @@ -53,10 +54,10 @@ class Benchmark (object): save_data = {} save_data.update(self.results) for target in self.results["targets"]: - l = [] + tmp_list = [] for ntuple, measures in self.results[target].items(): - l.append((ntuple._asdict(), measures)) - save_data[target] = l + tmp_list.append((ntuple._asdict(), measures)) + save_data[target] = tmp_list with open(f, "wb") as f: pickle.dump(save_data, f) @@ -127,11 +128,10 @@ class Benchmark (object): if is_fixed: yield p - def analyse(self, verbose=False, nolibmemusage=True): if not nolibmemusage and not shutil.which("memusage"): print("memusage not found. Using chattymalloc.") - libmemusage = False + nolibmemusage = True if nolibmemusage: import chattyparser @@ -155,9 +155,9 @@ class Benchmark (object): actual_cmd += self.cmd.format(**perm) res = subprocess.run(actual_cmd.split(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) if res.returncode != 0: print(actual_cmd, "failed.") @@ -169,9 +169,9 @@ class Benchmark (object): if nolibmemusage: try: chattyparser.plot() - except MemoryError as memerr: - print("Can't Analyse", actual_cmd, "with chattymalloc because", - "to much memory would be needed.") + except MemoryError: + print("Can't Analyse", actual_cmd, "with chattymalloc", + "because to much memory would be needed.") continue else: with open(file_name + ".hist", "w") as f: @@ -190,7 +190,7 @@ class Benchmark (object): i = 0 for tname, t in self.targets.items(): - if not tname in self.results: + if tname not in self.results: self.results[tname] = {} os.environ["LD_PRELOAD"] = "build/print_status_on_exit.so " @@ -202,7 +202,7 @@ class Benchmark (object): for perm in self.iterate_args(): i += 1 - print(i, "of", n,"\r", end='') + print(i, "of", n, "\r", end='') actual_cmd = self.measure_cmd + " " @@ -211,19 +211,21 @@ class Benchmark (object): actual_cmd += self.cmd.format(**perm_dict) res = subprocess.run(actual_cmd.split(), - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - universal_newlines=True) + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) if res.returncode != 0: - print("\n" + actual_cmd, "exited with", res.returncode, "for", tname) + print("\n" + actual_cmd, "exited with", res.returncode, + "for", tname) print("Aborting Benchmark.") print("Stdout:\n" + res.stdout) print("Stderr:\n" + res.stderr) return False if "ERROR: ld.so" in res.stderr: - print("\nPreloading of", t["LD_PRELOAD"], "failed for", tname) + print("\nPreloading of", t["LD_PRELOAD"], + "failed for", tname) print("Stderr:\n" + res.stderr) print("Aborting Benchmark.") return False @@ -241,17 +243,19 @@ class Benchmark (object): if hasattr(self, "process_output"): self.process_output(result, res.stdout, res.stderr, - tname, perm, verbose) + tname, perm, verbose) # Parse perf output if available if self.measure_cmd == self.defaults["measure_cmd"]: - csvreader = csv.reader(res.stderr.splitlines(), delimiter=',') + csvreader = csv.reader(res.stderr.splitlines(), + delimiter=',') for row in csvreader: # Split of the user/kernel space info to be better portable try: result[row[2].split(":")[0]] = row[0] except Exception as e: - print("Exception", e, "occured on", row, "for", tname, "and", perm) + print("Exception", e, "occured on", row, "for", + tname, "and", perm) if run == 1: self.results[tname][perm] = [] @@ -261,11 +265,12 @@ class Benchmark (object): if self.posttarget_hook((tname, t), run, verbose): return False print() - os.environ["PATH"] = os.environ["PATH"].replace(":build/"+self.name, "") + os.environ["PATH"] = os.environ["PATH"].replace(":build/" + self.name, "") return True - def plot_single_arg(self, yval, ylabel="'y-label'", xlabel="'x-label'", autoticks=True, - title="default title", filepostfix="", sumdir="", arg=""): + def plot_single_arg(self, yval, ylabel="'y-label'", xlabel="'x-label'", + autoticks=True, title="default title", filepostfix="", + sumdir="", arg=""): args = self.results["args"] targets = self.results["targets"] @@ -284,7 +289,7 @@ class Benchmark (object): else: x_vals = args[arg] plt.plot(x_vals, y_vals, marker='.', linestyle='-', - label=target, color=targets[target]["color"]) + label=target, color=targets[target]["color"]) plt.legend() if not autoticks: @@ -295,8 +300,9 @@ class Benchmark (object): plt.savefig(os.path.join(sumdir, ".".join([self.name, filepostfix, "png"]))) plt.clf() - def plot_fixed_arg(self, yval, ylabel="'y-label'", xlabel="loose_arg", autoticks=True, - title="'default title'", filepostfix="", sumdir="", fixed=[]): + def plot_fixed_arg(self, yval, ylabel="'y-label'", xlabel="loose_arg", + autoticks=True, title="'default title'", filepostfix="", + sumdir="", fixed=[]): args = self.results["args"] targets = self.results["targets"] @@ -306,7 +312,7 @@ class Benchmark (object): for arg_value in args[arg]: for target in targets: y_vals = [] - for perm in self.iterate_args_fixed({arg : arg_value}, args=args): + for perm in self.iterate_args_fixed({arg: arg_value}, args=args): d = [] for m in self.results[target][perm]: d.append(eval(yval.format(**m))) @@ -316,7 +322,7 @@ class Benchmark (object): else: x_vals = args[loose_arg] plt.plot(x_vals, y_vals, marker='.', linestyle='-', - label=target, color=targets[target]["color"]) + label=target, color=targets[target]["color"]) plt.legend() if not autoticks: @@ -325,10 +331,11 @@ class Benchmark (object): plt.ylabel(eval(ylabel)) plt.title(eval(title)) plt.savefig(os.path.join(sumdir, ".".join([self.name, arg, - str(arg_value), filepostfix, "png"]))) + str(arg_value), filepostfix, "png"]))) plt.clf() - def write_best_doublearg_tex_table(self, evaluation, sort=">", filepostfix="", sumdir="", std=False): + def write_best_doublearg_tex_table(self, evaluation, sort=">", + filepostfix="", sumdir="", std=False): args = self.results["args"] keys = list(args.keys()) targets = self.results["targets"] @@ -353,7 +360,8 @@ class Benchmark (object): if not best_val: best = [target] best_val = mean - elif (sort == ">" and mean > best_val) or (sort == "<" and mean < best_val): + elif ((sort == ">" and mean > best_val) + or (sort == "<" and mean < best_val)): best = [target] best_val = mean elif mean == best_val: @@ -362,11 +370,10 @@ class Benchmark (object): row.append("{}: {:.3f}".format(best[0], best_val)) cell_text.append(row) - fname = os.path.join(sumdir, ".".join([self.name, filepostfix, "tex"])) - with open(fname , "w") as f: + with open(fname, "w") as f: print("\\begin{tabular}{|", end="", file=f) - print(" l |" * len(headers),"}", file=f) + print(" l |" * len(headers), "}", file=f) print(header_arg+"/"+row_arg, end=" & ", file=f) for header in headers[:-1]: @@ -376,6 +383,6 @@ class Benchmark (object): for i, row in enumerate(cell_text): print(rows[i], end=" & ", file=f) for e in row[:-1]: - print(e, end=" & ",file=f) + print(e, end=" & ", file=f) print(row[-1], "\\\\", file=f) print("\\end{tabular}", file=f) diff --git a/src/chattyparser.py b/src/chattyparser.py index 3406b44..b1b6cd0 100644 --- a/src/chattyparser.py +++ b/src/chattyparser.py @@ -1,6 +1,5 @@ import re import matplotlib.pyplot as plt -import numpy as np ptr = "(?:0x)?(?P(?:\w+)|(?:\(nil\)))" size = "(?P\d+)" @@ -11,7 +10,9 @@ calloc_re = re.compile("^c (?P\d+) {} {}$".format(size, ptr)) realloc_re = re.compile("^r {} {} {}$".format(ptr, size, ptr.replace("ptr", "nptr"))) memalign_re = re.compile("^mm (?P\d+) {} {}$".format(size, ptr)) -def record_allocation(hist, total_size, allocations, ptr, size, coll_size, req_size, nohist, optr=None, add=True): + +def record_allocation(hist, total_size, allocations, ptr, size, coll_size, + req_size, nohist, optr=None, add=True): size = int(size) if add: if optr and optr in allocations: @@ -44,8 +45,9 @@ def record_allocation(hist, total_size, allocations, ptr, size, coll_size, req_s elif coll_size: total_size.append(total_size[-1]) + def parse(path="chattymalloc.data", coll_size=True, req_size=None, nohist=False): - tmalloc, tcalloc, trealloc, tfree, tmemalign= 0, 0, 0, 0, 0 + tmalloc, tcalloc, trealloc, tfree, tmemalign = 0, 0, 0, 0, 0 allocations = {} requested_size = [0] hist = {} @@ -55,50 +57,53 @@ def parse(path="chattymalloc.data", coll_size=True, req_size=None, nohist=False) for i, l in enumerate(f.readlines()): ln += 1 res = malloc_re.match(l) - if res != None: + if res is not None: res = res.groupdict() record_allocation(hist, requested_size, allocations, res["ptr"], - res["size"], coll_size, req_size, nohist) + res["size"], coll_size, req_size, nohist) tmalloc += 1 continue res = free_re.match(l) - if res != None: + if res is not None: res = res.groupdict() record_allocation(hist, requested_size, allocations, res["ptr"], - 0, coll_size, req_size, nohist, add=False) - tfree +=1 + 0, coll_size, req_size, nohist, add=False) + tfree += 1 continue res = calloc_re.match(l) - if res != None: + if res is not None: res = res.groupdict() size = int(res["nmemb"]) * int(res["size"]) record_allocation(hist, requested_size, allocations, res["ptr"], - size, coll_size, req_size, nohist) + size, coll_size, req_size, nohist) tcalloc += 1 continue res = realloc_re.match(l) - if res != None: + if res is not None: res = res.groupdict() record_allocation(hist, requested_size, allocations, res["nptr"], - res["size"], coll_size, req_size, nohist, optr=res["ptr"]) + res["size"], coll_size, req_size, nohist, + optr=res["ptr"]) trealloc += 1 continue res = memalign_re.match(l) - if res != None: + if res is not None: res = res.groupdict() record_allocation(hist, requested_size, allocations, res["ptr"], - res["size"], coll_size, req_size, nohist) + res["size"], coll_size, req_size, nohist) tmemalign += 1 continue print("\ninvalid line at", ln, ":", l) - calls = {"malloc": tmalloc, "free": tfree, "calloc": tcalloc, "realloc": trealloc, "memalign": tmemalign} + calls = {"malloc": tmalloc, "free": tfree, "calloc": tcalloc, + "realloc": trealloc, "memalign": tmemalign} return hist, calls, requested_size + def plot(path): hist, calls, _ = parse(req_size=None) plot_hist_ascii(path+".hist", hist, calls) @@ -113,7 +118,8 @@ def plot_profile(path, top5): _, calls, total_size = parse(nohist=True) x_vals = range(0, sum(calls.values()) + 1) - plt.plot(x_vals, total_size, marker='', linestyle='-', label="Total requested") + plt.plot(x_vals, total_size, marker='', + linestyle='-', label="Total requested") for s in top5: _, calls, total_size = parse(nohist=True, req_size=s) @@ -126,6 +132,7 @@ def plot_profile(path, top5): plt.savefig(path) plt.clf() + def plot_hist_ascii(path, hist, calls): bins = {} for size in sorted(hist): @@ -142,8 +149,8 @@ def plot_hist_ascii(path, hist, calls): print("memalign:", calls["memalign"], file=f) print(file=f) - print("< 1024", sum([n for s,n in hist.items() if s < 1024]), file=f) - print("< 4096", sum([n for s,n in hist.items() if s < 4096]), file=f) + print("< 1024", sum([n for s, n in hist.items() if s < 1024]), file=f) + print("< 4096", sum([n for s, n in hist.items() if s < 4096]), file=f) print(file=f) print("Histogram of sizes:", file=f) diff --git a/src/dj_trace.py b/src/dj_trace.py index f4265ea..784b16f 100644 --- a/src/dj_trace.py +++ b/src/dj_trace.py @@ -1,12 +1,9 @@ import matplotlib.pyplot as plt -import matplotlib.ticker as ticker -import multiprocessing import numpy as np import os from urllib.request import urlretrieve import sys import re -import shutil from src.benchmark import Benchmark @@ -24,21 +21,22 @@ calloc_re = re.compile("^Avg calloc time:\s*{} in.*calls$".format(time_re)) realloc_re = re.compile("^Avg realloc time:\s*{} in.*calls$".format(time_re)) free_re = re.compile("^Avg free time:\s*{} in.*calls$".format(time_re)) -class Benchmark_DJ_Trace( Benchmark ): + +class Benchmark_DJ_Trace(Benchmark): def __init__(self): self.name = "dj_trace" self.descrition = """This benchmark uses the workload simulator written - by DJ Delorie to simulate workloads provided by him - under https://delorie.com/malloc. Those workloads - are generated from traces of real aaplications and are - also used by delorie to measure improvements in the - glibc allocator.""", + by DJ Delorie to simulate workloads provided by + him under https://delorie.com/malloc. Those + workloads are generated from traces of real + aplications and are also used by delorie to + measure improvements in the glibc allocator.""" self.cmd = "trace_run{binary_suffix} dj_workloads/{workload}.wl" self.measure_cmd = "" self.args = { - "workload" : [ + "workload": [ "389-ds-2", "dj", "dj2", @@ -53,31 +51,33 @@ class Benchmark_DJ_Trace( Benchmark ): self.results = { "389-ds-2": { "malloc": 170500018, "calloc": 161787184, - "realloc": 404134, "free": 314856324, "threads": 41}, + "realloc": 404134, "free": 314856324, + "threads": 41}, "dj": { "malloc": 2000000, "calloc": 200, "realloc": 0, "free": 2003140, "threads": 201}, "dj2": { - "malloc":29263321, "calloc": 3798404, "realloc":122956, - "free": 32709054, "threads":36}, + "malloc": 29263321, "calloc": 3798404, + "realloc": 122956, "free": 32709054, + "threads": 36}, "mt_test_one_alloc": { - "malloc":524290, "calloc": 1, "realloc":0, - "free":594788, "threads":2}, + "malloc": 524290, "calloc": 1, "realloc": 0, + "free": 594788, "threads": 2}, "oocalc": { - "malloc":6731734, "calloc": 38421, "realloc":14108, - "free":6826686, "threads":88}, + "malloc": 6731734, "calloc": 38421, + "realloc": 14108, "free": 6826686, "threads": 88}, "qemu-virtio": { - "malloc":1772163, "calloc": 146634, - "realloc":59813, "free":1954732, "threads":3}, + "malloc": 1772163, "calloc": 146634, + "realloc": 59813, "free": 1954732, "threads": 3}, "qemu-win7": { - "malloc":980904, "calloc": 225420, - "realloc":89880, "free":1347825, "threads":6}, + "malloc": 980904, "calloc": 225420, + "realloc": 89880, "free": 1347825, "threads": 6}, "proprietary-1": { - "malloc":316032131, "calloc": 5642, "realloc":84, - "free":319919727, "threads":20}, + "malloc": 316032131, "calloc": 5642, "realloc": 84, + "free": 319919727, "threads": 20}, "proprietary-2": { - "malloc":9753948, "calloc": 4693, - "realloc":117, "free":10099261, "threads": 19}, + "malloc": 9753948, "calloc": 4693, + "realloc": 117, "free": 10099261, "threads": 19}, } self.requirements = ["trace_run"] @@ -91,9 +91,9 @@ class Benchmark_DJ_Trace( Benchmark ): if totalsize > 0: percent = readsofar * 1e2 / totalsize s = "\r%5.1f%% %*d / %d" % ( - percent, len(str(totalsize)), readsofar, totalsize) + percent, len(str(totalsize)), readsofar, totalsize) sys.stderr.write(s) - else: # total size is unknown + else: # total size is unknown sys.stderr.write("\rdownloaded %d" % (readsofar,)) if not os.path.isdir("dj_workloads"): @@ -113,8 +113,8 @@ class Benchmark_DJ_Trace( Benchmark ): def to_int(s): return int(s.replace(',', "")) - regexs = {7:malloc_re ,8:calloc_re, 9:realloc_re, 10:free_re} - functions = {7:"malloc", 8:"calloc", 9:"realloc", 10:"free"} + regexs = {7: malloc_re, 8: calloc_re, 9: realloc_re, 10: free_re} + functions = {7: "malloc", 8: "calloc", 9: "realloc", 10: "free"} for i, l in enumerate(stdout.splitlines()): if i == 2: result["cputime"] = to_int(cpu_time_re.match(l).group("time")) @@ -161,10 +161,11 @@ class Benchmark_DJ_Trace( Benchmark ): label=target, color=targets[target]["color"]) plt.legend(loc="best") - plt.xticks(xa + 1/len(targets)*2, ["malloc\n" + str(self.results[perm.workload]["malloc"]) + "\ncalls", - "calloc\n" + str(self.results[perm.workload]["calloc"]) + "\ncalls", - "realloc\n" + str(self.results[perm.workload]["realloc"]) + "\ncalls", - "free\n" + str(self.results[perm.workload]["free"]) + "\ncalls"]) + plt.xticks(xa + 1/len(targets)*2, + ["malloc\n" + str(self.results[perm.workload]["malloc"]) + "\ncalls", + "calloc\n" + str(self.results[perm.workload]["calloc"]) + "\ncalls", + "realloc\n" + str(self.results[perm.workload]["realloc"]) + "\ncalls", + "free\n" + str(self.results[perm.workload]["free"]) + "\ncalls"]) plt.ylabel("Durchschnittliche Zeit in cycles") plt.title("Durchscnittliche Laufzeiten der API Funktionen") plt.savefig(".".join([self.name, perm.workload, "apitimes", "png"])) @@ -208,7 +209,7 @@ class Benchmark_DJ_Trace( Benchmark ): fname = ".".join([self.name, perm.workload, "table.tex"]) with open(fname, "w") as f: - print("\\begin{tabular}{| l | l | l |}" , file=f) + print("\\begin{tabular}{| l | l | l |}", file=f) print("& Zeit (ms) / $\\sigma$ (\\%) & VmHWM (KB) / $\\sigma$ (\\%) \\\\", file=f) print("\\hline", file=f) @@ -238,4 +239,5 @@ class Benchmark_DJ_Trace( Benchmark ): print("\end{tabular}", file=f) + dj_trace = Benchmark_DJ_Trace() diff --git a/src/falsesharing.py b/src/falsesharing.py index 6c4ddc0..79de436 100644 --- a/src/falsesharing.py +++ b/src/falsesharing.py @@ -1,26 +1,26 @@ import matplotlib.pyplot as plt import multiprocessing import numpy as np -import os import re from src.benchmark import Benchmark time_re = re.compile("^Time elapsed = (?P