[benchmarks] reintroduce prepare

Always running the possibly very expensive preparation code when
a benchmark object is created is not reasonable when benchmarks
should only be summarized.

10 files changed, 84 insertions, 56 deletions

diff --git a/allocbench/benchmark.py b/allocbench/benchmark.py
index 642737a..c312233 100644
--- a/allocbench/benchmark.py
+++ b/allocbench/benchmark.py
@@ -210,8 +210,6 @@ class Benchmark:
 
         if not hasattr(self, "requirements"):
             self.requirements = []
-        else:
-            self.check_requirements()
 
         print_debug("Creating benchmark", self.name)
         print_debug("Cmd:", self.cmd)
@@ -221,6 +219,9 @@ class Benchmark:
         print_debug("Results dictionary:", self.results)
         print_debug("Results directory:", self.result_dir)
 
+    def prepare(self):
+        self.check_requirements()
+
     def save(self, path=None):
         """Save benchmark results to a json file"""
         if not path:
diff --git a/allocbench/benchmarks/dj_trace.py b/allocbench/benchmarks/dj_trace.py
index e96ab5d..5edb56d 100644
--- a/allocbench/benchmarks/dj_trace.py
+++ b/allocbench/benchmarks/dj_trace.py
@@ -131,12 +131,16 @@ class BenchmarkDJTrace(Benchmark):
         self.requirements = ["trace_run"]
         super().__init__(name)
 
-        workloads = ArchiveArtifact(
+        self.workloads = ArchiveArtifact(
             "dj_workloads",
             "https://www4.cs.fau.de/~flow/allocbench/dj_workloads.tar.xz",
             "tar", "c9bc499eeba8023bca28a755fffbaf9200a335ad")
 
-        self.workload_dir = workloads.provide()
+        self.workload_dir = None
+
+    def prepare(self):
+        """Download and extract workload files"""
+        self.workload_dir = self.workloads.provide()
 
     @staticmethod
     def process_output(result, stdout, stderr, allocator, perm):  # pylint: disable=too-many-arguments, unused-argument
diff --git a/allocbench/benchmarks/fd.py b/allocbench/benchmarks/fd.py
index 300359d..4d61319 100644
--- a/allocbench/benchmarks/fd.py
+++ b/allocbench/benchmarks/fd.py
@@ -22,6 +22,12 @@ from allocbench.artifact import ArchiveArtifact, GitArtifact
 from allocbench.benchmark import Benchmark
 import allocbench.plots as plt
 
+LINUX_VERSION = 'v5.3'
+FD_VERSION = 'v7.4.0'
+
+FD_URL = ("https://github.com/sharkdp/fd/releases/latest/download/"
+          f"fd-{FD_VERSION}-x86_64-unknown-linux-gnu.tar.gz")
+
 
 class BenchmarkFd(Benchmark):
     """fd benchmark
@@ -31,33 +37,30 @@ class BenchmarkFd(Benchmark):
         self.cmd = "fd -HI -e c '.*[0-9].*' {linux_files}"
         super().__init__(name)
 
-        linux = GitArtifact(
+        self.linux_artifact = GitArtifact(
             "linux",
             "git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git")
-        linux_version = "v5.3"
-        self.linux_files = linux.provide(linux_version)
+        self.linux_files = None
 
-        if os.path.exists(self.build_dir):
-            return
+        self.results["facts"]["versions"]["fd"] = FD_VERSION
+        self.fd_artifact = ArchiveArtifact(
+            "fd", FD_URL, "tar", "a5d8e7c8484449aa324a46abfdfaf026d7de77ee")
 
-        fd_version = "v7.4.0"
-        self.results["facts"]["versions"]["fd"] = fd_version
-        fd_url = ("https://github.com/sharkdp/fd/releases/latest/download/"
-                  f"fd-{fd_version}-x86_64-unknown-linux-gnu.tar.gz")
+    def prepare(self):
+        """Checkout the linux sources and download fd binary"""
+        self.linux_files = self.linux_artifact.provide(LINUX_VERSION)
 
-        fd_artifact = ArchiveArtifact(
-            "fd", fd_url, "tar", "a5d8e7c8484449aa324a46abfdfaf026d7de77ee")
+        if os.path.exists(self.build_dir):
+            return
 
         fd_dir = os.path.join(self.build_dir, "fd_sources")
-        fd_artifact.provide(fd_dir)
-
-        # create symlinks
-        for exe in ["fd"]:
-            src = os.path.join(fd_dir,
-                               f"fd-{fd_version}-x86_64-unknown-linux-gnu",
-                               exe)
-            dest = os.path.join(self.build_dir, exe)
-            os.link(src, dest)
+        self.fd_artifact.provide(fd_dir)
+
+        # create symlink
+        src = os.path.join(fd_dir, f"fd-{FD_VERSION}-x86_64-unknown-linux-gnu",
+                           'fd')
+        dest = os.path.join(self.build_dir, 'fd')
+        os.link(src, dest)
 
     def summary(self):
         plt.plot(self,
diff --git a/allocbench/benchmarks/httpd.py b/allocbench/benchmarks/httpd.py
index ac06459..3f0567a 100644
--- a/allocbench/benchmarks/httpd.py
+++ b/allocbench/benchmarks/httpd.py
@@ -44,6 +44,9 @@ class BenchmarkHTTPD(Benchmark):
 
         super().__init__(name)
 
+    def prepare(self):
+        """Retrieve nginx and ab versions"""
+        super().prepare()
         self.results["facts"]["versions"]["nginx"] = facter.exe_version(
             "nginx", "-v")
         self.results["facts"]["versions"]["ab"] = facter.exe_version(
diff --git a/allocbench/benchmarks/keydb.py b/allocbench/benchmarks/keydb.py
index 3aeb6f1..ee6855c 100644
--- a/allocbench/benchmarks/keydb.py
+++ b/allocbench/benchmarks/keydb.py
@@ -27,6 +27,9 @@ from allocbench.benchmark import Benchmark
 import allocbench.plots as plt
 from allocbench.util import run_cmd
 
+KEYDB_VERSION = "v5.3.1"
+MEMTIER_VERSION = "1.2.17"
+
 
 class BenchmarkKeyDB(Benchmark):
     """Definition of the keydb benchmark"""
@@ -48,8 +51,9 @@ class BenchmarkKeyDB(Benchmark):
 
         super().__init__(name)
 
-        keydb_version = "v5.3.1"
-        self.results["facts"]["versions"]["keydb"] = keydb_version
+    def prepare(self):
+        """Build keydb and memtier if necessary"""
+        self.results["facts"]["versions"]["keydb"] = KEYDB_VERSION
         keydb_dir = os.path.join(self.build_dir, "keydb")
 
         if not os.path.exists(os.path.join(self.build_dir, "keydb-server")):
@@ -59,7 +63,7 @@ class BenchmarkKeyDB(Benchmark):
             os.makedirs(self.build_dir, exist_ok=True)
 
             # checkout sources
-            keydb_artifact.provide(keydb_version, keydb_dir)
+            keydb_artifact.provide(KEYDB_VERSION, keydb_dir)
 
             # building keyDB
             run_cmd(["make", "-C", keydb_dir, "MALLOC=libc"])
@@ -71,8 +75,7 @@ class BenchmarkKeyDB(Benchmark):
                 if not os.path.exists(dest):
                     os.link(src, dest)
 
-        memtier_version = "1.2.17"
-        self.results["facts"]["versions"]["memtier"] = memtier_version
+        self.results["facts"]["versions"]["memtier"] = MEMTIER_VERSION
         memtier_dir = os.path.join(self.build_dir, "memtier")
 
         if not os.path.exists(os.path.join(self.build_dir,
@@ -80,7 +83,7 @@ class BenchmarkKeyDB(Benchmark):
             memtier = GitArtifact(
                 "memtier", "https://github.com/RedisLabs/memtier_benchmark")
 
-            memtier.provide(memtier_version, memtier_dir)
+            memtier.provide(MEMTIER_VERSION, memtier_dir)
 
             # building memtier
             run_cmd(["autoreconf", "-ivf"], cwd=memtier_dir)
diff --git a/allocbench/benchmarks/lld.py b/allocbench/benchmarks/lld.py
index 80bc5c2..0395843 100644
--- a/allocbench/benchmarks/lld.py
+++ b/allocbench/benchmarks/lld.py
@@ -228,17 +228,24 @@ class BenchmarkLld(Benchmark):
         self.measure_cmd = "perf stat -x, -d time -f %M,KB,VmHWM"
         self.measure_cmd_csv = True
         self.requirements = ["ld.lld"]
+
+        self.tests_artifact = ArchiveArtifact(
+            "lld-speed-test",
+            "https://s3-us-west-2.amazonaws.com/linker-tests/lld-speed-test.tar.xz",
+            "tar", "2d449a11109c7363f67fd45513b42270f5ba2a92")
+        self.test_dir = None
+
         super().__init__(name)
 
+    def prepare(self):
+        """Download and extract lld test files"""
+        super().prepare()
+
         # save lld version
         self.results["facts"]["versions"]["lld"] = facter.exe_version(
             "ld.lld", "-v")
 
-        tests = ArchiveArtifact(
-            "lld-speed-test",
-            "https://s3-us-west-2.amazonaws.com/linker-tests/lld-speed-test.tar.xz",
-            "tar", "2d449a11109c7363f67fd45513b42270f5ba2a92")
-        self.test_dir = tests.provide()
+        self.test_dir = self.tests_artifact.provide()
 
     def cleanup(self):
         for perm in self.iterate_args():
diff --git a/allocbench/benchmarks/mysql.py b/allocbench/benchmarks/mysql.py
index 611405d..d06fc2c 100644
--- a/allocbench/benchmarks/mysql.py
+++ b/allocbench/benchmarks/mysql.py
@@ -119,6 +119,14 @@ class BenchmarkMYSQL(Benchmark):
 
         super().__init__(name)
 
+    def reset_preparations(self):
+        """Reset self.build_dir if preparing fails"""
+        if os.path.exists(self.build_dir):
+            print_warn("Reset mysql test directory")
+            shutil.rmtree(self.build_dir, ignore_errors=True)
+
+    def prepare(self):
+        """Setup mysql database containing random test data"""
         self.results["facts"]["runtime [s]"] = RUN_TIME
 
         # save mysqld and sysbench versions
@@ -178,12 +186,6 @@ class BenchmarkMYSQL(Benchmark):
 
             self.shutdown_servers()
 
-    def reset_preparations(self):
-        """Reset self.build_dir if preparing fails"""
-        if os.path.exists(self.build_dir):
-            print_warn("Reset mysql test directory")
-            shutil.rmtree(self.build_dir, ignore_errors=True)
-
     @staticmethod
     def process_output(result, stdout, stderr, allocator, perm):  # pylint: disable=too-many-arguments, unused-argument
         result["transactions"] = re.search("transactions:\\s*(\\d*)",
diff --git a/allocbench/benchmarks/raxmlng.py b/allocbench/benchmarks/raxmlng.py
index be2a080..645d903 100644
--- a/allocbench/benchmarks/raxmlng.py
+++ b/allocbench/benchmarks/raxmlng.py
@@ -26,6 +26,8 @@ from allocbench.util import run_cmd
 
 RUNTIME_RE = re.compile("Elapsed time: (?P<runtime>(\\d*.\\d*)) seconds")
 
+RAXMLNG_VERSION = "0.9.0"
+
 
 class BenchmarkRaxmlng(Benchmark):
     """RAxML-ng benchmark
@@ -39,16 +41,17 @@ class BenchmarkRaxmlng(Benchmark):
             f"raxml-ng --msa {self.build_dir}/data/prim.phy --model GTR+G"
             " --redo --threads 2 --seed 2")
 
+    def prepare(self):
+        """Build raxml-ng and download test data if necessary"""
         if os.path.exists(self.build_dir):
             return
 
         raxmlng_sources = GitArtifact("raxml-ng",
                                       "https://github.com/amkozlov/raxml-ng")
-        raxmlng_version = "0.9.0"
         raxmlng_dir = os.path.join(self.build_dir, "raxml-ng-git")
         raxmlng_builddir = os.path.join(raxmlng_dir, "build")
-        self.results["facts"]["versions"]["raxml-ng"] = raxmlng_version
-        raxmlng_sources.provide(raxmlng_version, raxmlng_dir)
+        self.results["facts"]["versions"]["raxml-ng"] = RAXMLNG_VERSION
+        raxmlng_sources.provide(RAXMLNG_VERSION, raxmlng_dir)
 
         # Create builddir
         os.makedirs(raxmlng_builddir, exist_ok=True)
diff --git a/allocbench/benchmarks/redis.py b/allocbench/benchmarks/redis.py
index 8db69c3..3d09f9c 100644
--- a/allocbench/benchmarks/redis.py
+++ b/allocbench/benchmarks/redis.py
@@ -31,6 +31,8 @@ from allocbench.util import run_cmd
 
 REQUESTS_RE = re.compile("(?P<requests>(\\d*.\\d*)) requests per second")
 
+REDIS_VERSION = "5.0.5"
+
 
 class BenchmarkRedis(Benchmark):
     """Definition of the redis benchmark"""
@@ -47,14 +49,15 @@ class BenchmarkRedis(Benchmark):
 
         super().__init__(name)
 
-        redis_version = "5.0.5"
-        self.results["facts"]["versions"]["redis"] = redis_version
+    def prepare(self):
+        """Build redis and memtier if necessary"""
+        self.results["facts"]["versions"]["redis"] = REDIS_VERSION
         redis_artifact = ArchiveArtifact(
             "redis",
-            f"http://download.redis.io/releases/redis-{redis_version}.tar.gz",
+            f"http://download.redis.io/releases/redis-{REDIS_VERSION}.tar.gz",
             "tar", "71e38ae09ac70012b5bc326522b976bcb8e269d6")
 
-        redis_dir = os.path.join(self.build_dir, f"redis-{redis_version}")
+        redis_dir = os.path.join(self.build_dir, f"redis-{REDIS_VERSION}")
 
         redis_artifact.provide(self.build_dir)
 
diff --git a/bench.py b/bench.py
index adecc89..6a23cd0 100755
--- a/bench.py
+++ b/bench.py
@@ -181,14 +181,13 @@ def main():
             print_error(f"Skipping {bench}! Loading failed.")
             continue
 
-            try:
-                print_status("Preparing", bench, "...")
-                bench = member()
-                break
-            except Exception:
-                print_error(traceback.format_exc())
-                print_error(f"Skipping {bench}! Preparing failed.")
-                continue
+        try:
+            print_status("Preparing", bench, "...")
+            bench.prepare()
+        except Exception:
+            print_error(traceback.format_exc())
+            print_error(f"Skipping {bench}! Preparing failed.")
+            continue
 
         if args.analyze:
             analyze_bench(bench)