From 4507160a1dba320b71dab245de6624401794ce7b Mon Sep 17 00:00:00 2001 From: Florian Fischer Date: Fri, 31 Aug 2018 13:18:50 +0200 Subject: put common run logic into benchmark.py --- loop.py | 238 +++++++++++++++++----------------------------------------------- 1 file changed, 61 insertions(+), 177 deletions(-) (limited to 'loop.py') diff --git a/loop.py b/loop.py index 2709e68..c6cef00 100644 --- a/loop.py +++ b/loop.py @@ -1,18 +1,9 @@ -import csv -import pickle import matplotlib.pyplot as plt import multiprocessing import numpy as np import os -import subprocess -from subprocess import PIPE from benchmark import Benchmark -from common_targets import common_targets - -perf_cmd = ("perf stat -x\; -d -e cpu-clock,cache-references,cache-misses,cycles," - "instructions,branches,faults,migrations ") -cmd = "build/bench_loop{} 1.2 {} 1000000 {} 10" class Benchmark_Loop( Benchmark ): def __init__(self): @@ -20,182 +11,75 @@ class Benchmark_Loop( Benchmark ): self.descrition = """This benchmark makes n allocations in t concurrent threads. How allocations are freed can be changed with the benchmark version""", - self.targets = common_targets - self.maxsize = [2 ** x for x in range(6, 16)] - self.nthreads = range(1, multiprocessing.cpu_count() * 2 + 1) - - self.results = {"args" : {"nthreads" : self.nthreads, "maxsize": self.maxsize}, - "targets" : self.targets} - - def prepare(self, verbose=False): - req = ["build/bench_loop"] - for r in req: - if not os.path.isfile(r): - print(r, "not found") - return False - if not os.access(r, os.X_OK): - print(r, "not found") - return False - if verbose: - print(r, "found and executable.") - return True - - - def run(self, verbose=False, runs=3): - args_permutations = [(x,y) for x in self.nthreads for y in self.maxsize] - n = len(args_permutations) - - for run in range(1, runs + 1): - print(str(run) + ". run") - - for i, args in enumerate(args_permutations): - print(i + 1, "of", n, "\r", end='') - - # run cmd for each target - for tname, t in self.targets.items(): - if not tname in self.results: - self.results[tname] = {} - - result = {} - - os.environ["LD_PRELOAD"] = t["LD_PRELOAD"] - - cur_cmd = cmd.format(t["binary_suffix"], *args) - - # Collect memory consumtion on first run - if run == 1: - os.environ["LD_PRELOAD"] = "build/print_status_on_exit.so " + os.environ["LD_PRELOAD"] - subprocess.run(cur_cmd.split(), env=os.environ) - with open("status", "r") as f: - for l in f.readlines(): - if l.startswith("VmHWM:"): - result["rssmax"] = l.split()[1] - - os.environ["LD_PRELOAD"] = t["LD_PRELOAD"] - - target_cmd = perf_cmd + cur_cmd - if verbose: - print("\n" + tname, t, "\n", target_cmd, "\n") - - p = subprocess.run(target_cmd.split(), - env=os.environ, - stderr=PIPE, - stdout=PIPE, - universal_newlines=True) - - - output = p.stderr - - if p.returncode != 0: - print("\n" + target_cmd, "exited with", p.returncode, ".\n Aborting Benchmark.") - print(tname, t) - print(output) - print(p.stdout) - return False - - if "ERROR: ld.so" in output: - print("\nPreloading of", t["LD_PRELOAD"], "failed for", tname, ".\n Aborting Benchmark.") - print(output) - return False - # Handle perf output - csvreader = csv.reader(output.splitlines(), delimiter=';') - for row in csvreader: - result[row[2].replace("\\", "")] = row[0].replace("\\", "") + self.cmd = "build/bench_loop{binary_suffix} 1.2 {nthreads} 1000000 {maxsize} 10" - if not args in self.results[tname]: - self.results[tname][args] = [result] - else: - self.results[tname][args].append(result) + self.args = { + "maxsize" : [2 ** x for x in range(6, 16)], + "nthreads" : range(1, multiprocessing.cpu_count() * 2 + 1) + } - print() - return True + self.requirements = ["build/bench_loop"] + print(type(self)) + super().__init__() def summary(self, sd=None): - nthreads = self.results["args"]["nthreads"] - maxsize = self.results["args"]["maxsize"] + args = self.results["args"] targets = self.results["targets"] sd = sd or "" - # MAXSIZE fixed - y_mapping = {v : i for i, v in enumerate(nthreads)} - for size in maxsize: - for target in targets: - y_vals = [0] * len(nthreads) - for margs, measures in [(a, m) for a, m in self.results[target].items() if a[1] == size]: - d = [] - for m in measures: - # nthreads/time = MOPS/s - for e in m: - if "cpu-clock" in e: - d.append(margs[0]/float(m[e])) - y_vals[y_mapping[margs[0]]] = np.mean(d) - plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"]) - - plt.legend() - plt.xlabel("threads") - plt.ylabel("MOPS/s") - plt.title("Loop: " + str(size) + "B") - plt.savefig(os.path.join(sd, self.name + "." + str(size) + "B.png")) - plt.clf() - - # NTHREADS fixed - y_mapping = {v : i for i, v in enumerate(maxsize)} - x_vals = [i + 1 for i in range(0, len(maxsize))] - for n in nthreads: - for target in targets: - y_vals = [0] * len(maxsize) - for margs, measures in [(a, m) for a, m in self.results[target].items() if a[0] == n]: - d = [] - for m in measures: - # nthreads/time = MOPS/S - for e in m: - if "cpu-clock" in e: - d.append(margs[0]/float(m[e])) - y_vals[y_mapping[margs[1]]] = np.mean(d) - plt.plot(x_vals, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"]) - - plt.legend() - plt.xticks(x_vals, maxsize) - plt.xlabel("size in B") - plt.ylabel("MOPS/s") - plt.title("Loop: " + str(n) + "thread(s)") - plt.savefig(os.path.join(sd, self.name + "." + str(n) + "threads.png")) - plt.clf() - - #Memusage - y_mapping = {v : i for i, v in enumerate(nthreads)} - for size in maxsize: - for target in targets: - y_vals = [0] * len(nthreads) - for margs, measures in [(a, m) for a, m in self.results[target].items() if a[1] == size]: - y_vals[y_mapping[margs[0]]] = int(measures[0]["rssmax"]) - plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"]) - - plt.legend() - plt.xlabel("threads") - plt.ylabel("kb") - plt.title("Memusage Loop: " + str(size) + "B") - plt.savefig(os.path.join(sd, self.name + "." + str(size) + "B.mem.png")) - plt.clf() - - # NTHREADS fixed - y_mapping = {v : i for i, v in enumerate(maxsize)} - x_vals = [i + 1 for i in range(0, len(maxsize))] - for n in nthreads: - for target in targets: - y_vals = [0] * len(maxsize) - for margs, measures in [(a, m) for a, m in self.results[target].items() if a[0] == n]: - y_vals[y_mapping[margs[1]]] = int(measures[0]["rssmax"]) - plt.plot(x_vals, y_vals, marker='.', linestyle='-', label=target, color=targets[target]["color"]) - - plt.legend() - plt.xticks(x_vals, maxsize) - plt.xlabel("size in B") - plt.ylabel("kb") - plt.title("Memusage Loop: " + str(n) + "thread(s)") - plt.savefig(os.path.join(sd, self.name + "." + str(n) + "threads.mem.png")) - plt.clf() + # Speed + for arg in args: + loose_arg = [a for a in args if a != arg][0] + for arg_value in args[arg]: + for target in targets: + y_vals = [] + for perm in self.iterate_args_fixed({arg : arg_value}): + d = [] + for measure in self.results[target][perm]: + # nthreads/time = MOPS/s + for e in measure: + if "task-clock" in e: + d.append(perm.nthreads/float(measure[e])) + y_vals.append(np.mean(d)) + + x_vals = list(range(1, len(y_vals) + 1)) + + plt.plot(x_vals, y_vals, marker='.', linestyle='-', + label=target, color=targets[target]["color"]) + + plt.legend() + plt.xticks(x_vals, args[loose_arg]) + plt.xlabel(loose_arg) + plt.ylabel("MOPS/s") + plt.title("Loop: " + arg + " " + str(arg_value)) + plt.savefig(os.path.join(sd, ".".join([self.name, arg, str(arg_value), "png"]))) + plt.clf() + + # Memusage + for arg in args: + loose_arg = [a for a in args if a != arg][0] + for arg_value in args[arg]: + for target in targets: + y_vals = [] + for perm in self.iterate_args_fixed({arg : arg_value}): + d = [] + for measure in self.results[target][perm]: + d.append(int(measure["VmHWM"])) + y_vals.append(np.mean(d)) + + x_vals = list(range(1, len(y_vals) + 1)) + + plt.plot(x_vals, y_vals, marker='.', linestyle='-', + label=target, color=targets[target]["color"]) + + plt.legend() + plt.xticks(x_vals, args[loose_arg]) + plt.xlabel(loose_arg) + plt.ylabel("VmHWM") + plt.title("Loop Memusage: " + arg + " " + str(arg_value)) + plt.savefig(os.path.join(sd, ".".join([self.name, arg, str(arg_value), "mem", "png"]))) + plt.clf() loop = Benchmark_Loop() -- cgit v1.2.3