Merge branch 'artifact'

26 files changed, 448 insertions, 475 deletions

diff --git a/.gitignore b/.gitignore
index 663dfcc..cba62c4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 build/*
+cache/*
 results/*
 allocators/*.so
 dj_workloads/*
diff --git a/src/allocator.py b/src/allocator.py
index 41d66c7..b88b046 100644
--- a/src/allocator.py
+++ b/src/allocator.py
@@ -25,6 +25,7 @@ import shutil
 import subprocess
 import sys
 
+from src.artifact import ArchiveArtifact, GitArtifact
 import src.globalvars
 from src.util import print_status, print_debug, print_error, print_info2
 
@@ -37,99 +38,30 @@ for line in subprocess.run(["ldconfig", "-v"], stdout=subprocess.PIPE,
     if not line.startswith('\t'):
         LIBRARY_PATH += line
 
-BUILDDIR = os.path.join(src.globalvars.builddir, "allocators")
+BUILDDIR = src.globalvars.allocbuilddir
 SRCDIR = os.path.join(src.globalvars.allocbuilddir, "src")
 
 if not os.path.isdir(SRCDIR):
     os.makedirs(SRCDIR)
 
 
-class AllocatorSources:
-    """Class representing sources an allocator is build from
-
-    AllocatorSources can retrieve, prepare and reset their managed sources
-    """
-    def __init__(self, name, retrieve_cmds=None, prepare_cmds=None, reset_cmds=None):
-        self.name = name
-        self.dir = os.path.join(SRCDIR, self.name)
-        self.patchdir = os.path.join(src.globalvars.allocsrcdir, self.name)
-        self.retrieve_cmds = retrieve_cmds or []
-        self.prepare_cmds = prepare_cmds or []
-        self.reset_cmds = reset_cmds or []
-
-    def run_cmds(self, function, cwd=SRCDIR):
-        """Helper to run retrieve, prepare or reset commands"""
-        print_status(function, self.name, "...")
-
-        cmds = getattr(self, function+"_cmds")
-
-        stdout = subprocess.PIPE if src.globalvars.verbosity < 2 else None
-
-        for cmd in cmds:
-            proc = subprocess.run(cmd, shell=True, cwd=cwd,
-                                  stderr=subprocess.PIPE, stdout=stdout,
-                                  universal_newlines=True)
-
-            if proc.returncode:
-                print_error(function, self.name, "failed with", proc.returncode,
-                            file=sys.stderr)
-                print_debug(proc.stderr, file=sys.stderr)
-                return False
-        return True
-
-    def prepare(self):
-        """Prepare the managed sources for building
-
-        If the sources aren't available yet they are retrieved.
-        Otherwise they are reset.
-        """
-        if not os.path.isdir(self.dir):
-            if (not self.run_cmds("retrieve") or
-                    not self.run_cmds("prepare", cwd=self.dir)):
-
-                shutil.rmtree(self.dir, ignore_errors=True)
-                exit(1)
-        else:
-            self.reset()
-
-    def reset(self):
-        """Reset the managed sources"""
-        if not self.run_cmds("reset", cwd=self.dir):
-            exit(1)
-
-    def patch(self, patches):
-        """Patch the managed sources using patch(1)"""
-        if not patches:
-            return
-
-        stdout = subprocess.PIPE if src.globalvars.verbosity < 2 else None
-        cwd = os.path.join(SRCDIR, self.name)
-
-        print_status("Patching", self.name, "...")
-        for patch in patches:
-            with open(patch.format(patchdir=self.patchdir), "rb") as patch_file:
-                proc = subprocess.run("patch -p1", shell=True, cwd=cwd,
-                                      stderr=subprocess.PIPE, stdout=stdout,
-                                      input=patch_file.read())
-
-                if proc.returncode:
-                    print_error("Patching of", self.name, "failed.",
-                                file=sys.stderr)
-                    print_debug(proc.stderr, file=sys.stderr)
-                    exit(1)
-
-
 class Allocator:
     """Allocator base class
 
-    It builds the allocator and produces a for allocbench usable allocator dict"""
-    allowed_attributes = ["binary_suffix", "version", "sources", "build_cmds",
-                          "LD_PRELOAD", "cmd_prefix", "color", "patches",
-                          "LD_LIBRARY_PATH"]
+    An Allocator can contain an Artifact which provides the allocator sources,
+    patches, and instructions to build the allocator.
+    Allocator.build will compile the allocator and produce a for allocbench usable
+    allocator dict"""
+    allowed_attributes = ["binary_suffix", "cmd_prefix", "LD_PRELOAD", "LD_LIBRARY_PATH", "color",
+                          "sources", "version", "patches", "prepare_cmds",
+                          "build_cmds"]
 
     def __init__(self, name, **kwargs):
+        self.class_file = inspect.getfile(self.__class__)
         self.name = name
+        self.srcdir = os.path.join(SRCDIR, self.name)
         self.dir = os.path.join(BUILDDIR, self.name)
+        self.patchdir = os.path.join(os.path.splitext(self.class_file)[0])
         # Update attributes
         self.__dict__.update((k, v) for k, v in kwargs.items()
                              if k in self.allowed_attributes)
@@ -139,51 +71,82 @@ class Allocator:
             if not hasattr(self, attr):
                 setattr(self, attr, None)
 
+    def prepare(self):
+        """Prepare the allocators sources"""
+        if not self.sources and os.path.exists(self.srcdir):
+            return
+
+        print_status("Preparing", self.name, "...")
+        if type(self.sources) == GitArtifact:
+            self.sources.provide(self.version, self.srcdir)
+        elif type(self.sources) == ArchiveArtifact:
+            self.sources.provide(self.srcdir)
+
+        if self.patches:
+            stdout = subprocess.PIPE if src.globalvars.verbosity < 2 else None
+            cwd = os.path.join(SRCDIR, self.name)
+
+            print_status("Patching", self.name, "...")
+            for patch in self.patches:
+                with open(patch.format(patchdir=self.patchdir), "rb") as patch_file:
+                    proc = subprocess.run("patch -p1", shell=True, cwd=cwd,
+                                          stderr=subprocess.PIPE, stdout=stdout,
+                                          input=patch_file.read())
+
+                    if proc.returncode:
+                        print_debug(proc.stderr, file=sys.stderr)
+                        raise Exception(f"Patching of {self.name} failed.")
+
+        if self.prepare_cmds:
+            print_status("Run prepare commands", self.name, "...")
+            stdout = subprocess.PIPE if src.globalvars.verbosity < 2 else None
+
+            for cmd in self.prepare_cmds:
+                proc = subprocess.run(cmd, shell=True, cwd=self.srcdir,
+                                      stderr=subprocess.PIPE, stdout=stdout,
+                                      universal_newlines=True)
+
+                if proc.returncode:
+                    print_debug(proc.stderr, file=sys.stderr)
+                    raise Exception(f'Prepare command "{cmd}" failed with exitcode: {proc.returncode}.')
+
     def build(self):
-        """Build the allocator if needed and produce allocator dict"""
+        """Build the allocator if needed and produce an allocator dict"""
         build_needed = not os.path.isdir(self.dir)
         buildtimestamp_path = os.path.join(self.dir, ".buildtime")
 
-        print_info2(f"Building {self.name}...")
+        print_status("Building", self.name, "...")
         if not build_needed:
             print_info2("Old build found. Comparing build time with mtime")
 
             with open(buildtimestamp_path, "r") as buildtimestamp_file:
                 timestamp = datetime.fromtimestamp(float(buildtimestamp_file.read()))
 
-            modtime = os.stat(inspect.getfile(self.__class__)).st_mtime
+            modtime = os.stat(self.class_file).st_mtime
             modtime = datetime.fromtimestamp(modtime)
 
             build_needed = timestamp < modtime
 
             print_debug("Time of last build:", timestamp.isoformat())
-            print_debug("Last modification of allocators file:",
-                        modtime.isoformat())
+            print_debug("Last modification of allocators file:", modtime.isoformat())
             print_info2("" if build_needed else "No " + "build needed")
 
         if build_needed:
-            if self.sources:
-                self.sources.prepare()
-                self.sources.patch(self.patches)
+            self.prepare()
 
             if self.build_cmds:
-                print_status("Building", self.name, "...")
-
                 stdout = subprocess.PIPE if src.globalvars.verbosity < 2 else None
 
                 for cmd in self.build_cmds:
-                    cmd = cmd.format(**{"dir": self.dir,
-                                        "srcdir": self.sources.dir})
+                    cmd = cmd.format(dir=self.dir, srcdir=self.srcdir)
 
                     proc = subprocess.run(cmd, cwd=BUILDDIR, shell=True,
                                           stderr=subprocess.PIPE, stdout=stdout,
                                           universal_newlines=True)
                     if proc.returncode:
-                        print_error(cmd, "failed with:", proc.returncode)
                         print_debug(proc.stderr, file=sys.stderr)
-                        print_error("Building", self.name, "failed ...")
                         shutil.rmtree(self.dir, ignore_errors=True)
-                        exit(2)
+                        raise Exception(f'Build command "{cmd}" failed with exitcode: {proc.returncode}')
 
                 with open(buildtimestamp_path, "w") as buildtimestamp_file:
                     print_info2("Save build time to:", buildtimestamp_path)
@@ -196,13 +159,10 @@ class Allocator:
                     "LD_LIBRARY_PATH": self.LD_LIBRARY_PATH or "",
                     "color": self.color}
 
-        paths = {"dir": self.dir}
-        paths["srcdir"] = self.sources.dir if self.sources is not None else ""
-
         for attr in ["LD_PRELOAD", "LD_LIBRARY_PATH", "cmd_prefix"]:
             value = getattr(self, attr, "") or ""
             if value != "":
-                value = value.format(**paths)
+                value = value.format(dir=self.dir, srcdir=self.srcdir)
                 res_dict[attr] = value
 
         print_debug("Resulting dictionary:", res_dict)
diff --git a/src/allocators/all.py b/src/allocators/all.py
index 6b3b46a..0593c81 100644
--- a/src/allocators/all.py
+++ b/src/allocators/all.py
@@ -31,5 +31,5 @@ from src.allocators.snmalloc import snmalloc
 
 
 allocators = [*src.allocators.glibcs.allocators, tcmalloc, tcmalloc_nofs,
-              jemalloc, hoard, mesh, supermalloc, scalloc, llalloc, tbbmalloc,
+              jemalloc, hoard, mesh, supermalloc, scalloc, tbbmalloc, llalloc, # streamflow,
               mimalloc, snmalloc]
diff --git a/src/allocators/glibc.py b/src/allocators/glibc.py
index c322427..3c739fe 100644
--- a/src/allocators/glibc.py
+++ b/src/allocators/glibc.py
@@ -17,52 +17,48 @@
 
 """Glibc definitions"""
 
-from src.allocator import Allocator, AllocatorSources, LIBRARY_PATH
-
-VERSION = 2.29
-
-GLIBC_SRC = AllocatorSources("glibc",
-                      retrieve_cmds=["git clone git://sourceware.org/git/glibc.git"],
-                      prepare_cmds=[f"git checkout glibc-{VERSION}"],
-                      reset_cmds=["git reset --hard"])
+from src.allocator import Allocator, LIBRARY_PATH
+from src.artifact import GitArtifact
 
 
 class Glibc(Allocator):
     """Glibc definition for allocbench
 
     Glibcs are loaded using their own supplied loader"""
+
+    sources = GitArtifact("glibc", "git://sourceware.org/git/glibc.git")
+
     def __init__(self, name, **kwargs):
 
-        kwargs["sources"] = GLIBC_SRC
 
         configure_args = ""
         if "configure_args" in kwargs:
             configure_args = kwargs["configure_args"]
-            del(kwargs["configure_args"])
-
-        kwargs["build_cmds"] = ["mkdir -p glibc-build",
-                                "cd glibc-build; {srcdir}/configure --prefix={dir} " + configure_args,
-                                "cd glibc-build; make",
-                                "cd glibc-build; make install"]
+            del kwargs["configure_args"]
 
-        kwargs["cmd_prefix"] = ("{dir}/lib/ld-linux-x86-64.so.2 --library-path {dir}/lib:"
-                                + LIBRARY_PATH)
+        self.build_cmds = ["mkdir -p glibc-build",
+                           "cd glibc-build; {srcdir}/configure --prefix={dir} " + configure_args,
+                           "cd glibc-build; make",
+                           "cd glibc-build; make install"]
 
-        # kwargs["LD_LIBRARY_PATH"] = "{dir}/lib:" + library_path
+        self.cmd_prefix = "{dir}/lib/ld-linux-x86-64.so.2 --library-path {dir}/lib:" + LIBRARY_PATH
 
         super().__init__(name, **kwargs)
 
 
-glibc = Glibc("glibc", color="xkcd:red")
+glibc = Glibc("glibc", version="glibc-2.29", color="xkcd:red")
 
 glibc_notc = Glibc("glibc-noThreadCache",
                    configure_args="--disable-experimental-malloc",
+                   version="glibc-2.29",
                    color="xkcd:maroon")
 
 glibc_nofs = Glibc("glibc-noFalsesharing",
                    patches=["{patchdir}/glibc_2.29_no_passive_falsesharing.patch"],
+                   version="glibc-2.29",
                    color="xkcd:pink")
 
 glibc_nofs_fancy = Glibc("glibc-noFalsesharingClever",
                          patches=["{patchdir}/glibc_2.29_no_passive_falsesharing_fancy.patch"],
+                         version="glibc-2.29",
                          color="xkcd:orange")
diff --git a/src/allocators/hoard.py b/src/allocators/hoard.py
index cc2061a..7e94d79 100644
--- a/src/allocators/hoard.py
+++ b/src/allocators/hoard.py
@@ -17,26 +17,23 @@
 
 """Hoard allocator definition for allocbench"""
 
-from src.allocator import Allocator, AllocatorSources
-
-
-sources = AllocatorSources("Hoard",
-            retrieve_cmds=["git clone https://github.com/emeryberger/Hoard.git"],
-            reset_cmds=["git reset --hard"])
+from src.allocator import Allocator
+from src.artifact import GitArtifact
 
 
 class Hoard(Allocator):
     """Hoard allocator"""
-    def __init__(self, name, **kwargs):
 
-        kwargs["sources"] = sources
-        kwargs["LD_PRELOAD"] = "{dir}/libhoard.so"
-        kwargs["build_cmds"] = ["cd {srcdir}/src; make",
-                                "mkdir -p {dir}",
-                                "ln -f -s {srcdir}/src/libhoard.so {dir}/libhoard.so"]
-        kwargs["requirements"] = ["clang"]
+    sources = GitArtifact("Hoard", "https://github.com/emeryberger/Hoard.git")
+
+    def __init__(self, name, **kwargs):
+        self.LD_PRELOAD = "{dir}/libhoard.so"
+        self.build_cmds = ["cd {srcdir}/src; make",
+                           "mkdir -p {dir}",
+                           "ln -f -s {srcdir}/src/libhoard.so {dir}/libhoard.so"]
+        self.requirements = ["clang"]
 
         super().__init__(name, **kwargs)
 
 
-hoard = Hoard("Hoard", color="xkcd:brown")
+hoard = Hoard("Hoard", version="aa6d31700d5368a9f5ede3d62731247c8d9f0ebb", color="xkcd:brown")
diff --git a/src/allocators/installed_allocators.py b/src/allocators/installed_allocators.py
index 9a11d04..d45755a 100644
--- a/src/allocators/installed_allocators.py
+++ b/src/allocators/installed_allocators.py
@@ -22,11 +22,11 @@ import subprocess
 # TODO: add more allocators
 MAYBE_ALLOCATORS = ["tcmalloc", "jemalloc", "hoard"]
 
-allocators = {"libc": {"cmd_prefix":    "",
-                       "binary_suffix": "",
-                       "LD_PRELOAD":    "",
-                       "LD_LIBRARY_PATH":    "",
-                       "color":         "C1"}}
+allocators = {"libc": {"cmd_prefix":      "",
+                       "binary_suffix":   "",
+                       "LD_PRELOAD":      "",
+                       "LD_LIBRARY_PATH": "",
+                       "color":           "C1"}}
 
 for i, t in enumerate(MAYBE_ALLOCATORS):
     try:
diff --git a/src/allocators/jemalloc.py b/src/allocators/jemalloc.py
index 4484362..18e3055 100644
--- a/src/allocators/jemalloc.py
+++ b/src/allocators/jemalloc.py
@@ -17,26 +17,24 @@
 
 """jemalloc definition for allocbench"""
 
-from src.allocator import Allocator, AllocatorSources
-
-VERSION = "5.1.0"
-
-sources = AllocatorSources("jemalloc",
-            retrieve_cmds=["git clone https://github.com/jemalloc/jemalloc.git"],
-            prepare_cmds=[f"git checkout {VERSION}", "./autogen.sh"])
+from src.allocator import Allocator
+from src.artifact import GitArtifact
 
 
 class Jemalloc(Allocator):
     """jemalloc allocator"""
-    def __init__(self, name, **kwargs):
 
-        kwargs["sources"] = sources
-        kwargs["LD_PRELOAD"] = "{srcdir}/lib/libjemalloc.so"
-        kwargs["build_cmds"] = ["cd {srcdir}; ./configure --prefix={dir}",
-                                "cd {srcdir}; make -j4",
-                                "mkdir -p {dir}"]
+    sources = GitArtifact("jemalloc", "https://github.com/jemalloc/jemalloc.git")
+
+    def __init__(self, name, **kwargs):
+        self.LD_PRELOAD = "{dir}/libjemalloc.so"
+        self.prepare_cmds = ["./autogen.sh"]
+        self.build_cmds = ["cd {srcdir}; ./configure --prefix={dir}",
+                           "cd {srcdir}; make -j4",
+                           "mkdir -p {dir}",
+                           "ln -f -s {srcdir}/lib/libjemalloc.so {dir}/libjemalloc.so"]
 
         super().__init__(name, **kwargs)
 
 
-jemalloc = Jemalloc("jemalloc", color="xkcd:yellow")
+jemalloc = Jemalloc("jemalloc", version="5.1.0", color="xkcd:yellow")
diff --git a/src/allocators/llalloc.py b/src/allocators/llalloc.py
index b0da505..8f43f59 100644
--- a/src/allocators/llalloc.py
+++ b/src/allocators/llalloc.py
@@ -17,25 +17,24 @@
 
 """Lockless allocator definition for allocbench"""
 
-from src.allocator import Allocator, AllocatorSources
-
-
-LLALLOC_SOURCE = AllocatorSources("lockless_allocator",
-                      retrieve_cmds=["wget https://locklessinc.com/downloads/lockless_allocator_src.tgz",
-                                     "tar xf lockless_allocator_src.tgz"],
-                      prepare_cmds=[],
-                      reset_cmds=[])
+from src.allocator import Allocator
+from src.artifact import ArchiveArtifact
 
 
 class LocklessAllocator(Allocator):
     """Lockless allocator"""
     def __init__(self, name, **kwargs):
 
-        kwargs["sources"] = LLALLOC_SOURCE
+        self.sources = ArchiveArtifact("llalloc",
+                                  "https://locklessinc.com/downloads/lockless_allocator_src.tgz",
+                                  "tar",
+                                  "c6cb5a57882fa4775b5227a322333a6126b61f7c")
 
-        kwargs["build_cmds"] = ["cd {srcdir}; make", "mkdir -p {dir}"]
+        self.build_cmds = ["cd {srcdir}/lockless_allocator; make",
+                           "mkdir -p {dir}",
+                           "ln -f -s {srcdir}/lockless_allocator/libllalloc.so.1.3 {dir}/libllalloc.so"]
 
-        kwargs["LD_PRELOAD"] = "{srcdir}/libllalloc.so.1.3"
+        self.LD_PRELOAD = "{dir}/libllalloc.so"
 
         super().__init__(name, **kwargs)
 
diff --git a/src/allocators/malt.py b/src/allocators/malt.py
index 599e8ff..34fd9cc 100644
--- a/src/allocators/malt.py
+++ b/src/allocators/malt.py
@@ -24,6 +24,4 @@ of a program. See https://github.com/memtt/malt for more details
 from src.allocator import Allocator
 
 # result_dir and perm are substituted during Benchmark.run
-cmd = "malt -q -o output:name={{result_dir}}/malt.{{perm}}.%3"
-
-malt = Allocator("malt", cmd_prefix=cmd)
+malt = Allocator("malt", cmd_prefix="malt -q -o output:name={{result_dir}}/malt.{{perm}}.%3")
diff --git a/src/allocators/mesh.py b/src/allocators/mesh.py
index f8b7c16..2391e23 100644
--- a/src/allocators/mesh.py
+++ b/src/allocators/mesh.py
@@ -17,29 +17,23 @@
 
 """Mesh definition for allocbench"""
 
-from src.allocator import Allocator, AllocatorSources
-
-sources = AllocatorSources("Mesh",
-            retrieve_cmds=["git clone https://github.com/plasma-umass/Mesh"],
-            reset_cmds=["git reset --hard"])
-
-# sources = src.allocator.GitAllocatorSources("Mesh",
-#             "https://github.com/plasma-umass/Mesh",
-#             "adsf0982345")
+from src.allocator import Allocator
+from src.artifact import GitArtifact
 
 
 class Mesh(Allocator):
     """Mesh allocator"""
-    def __init__(self, name, **kwargs):
 
-        kwargs["sources"] = sources
-        kwargs["LD_PRELOAD"] = "{srcdir}/libmesh.so"
-        kwargs["build_cmds"] = ["cd {srcdir}; git submodule update --init",
-                                "cd {srcdir}; ./configure",
-                                "cd {srcdir}; make -j 4",
-                                "mkdir -p {dir}"]
+    sources = GitArtifact("Mesh", "https://github.com/plasma-umass/Mesh")
+
+    def __init__(self, name, **kwargs):
+        self.LD_PRELOAD = "{dir}/libmesh.so"
+        self.build_cmds = ["cd {srcdir}; ./configure",
+                           "cd {srcdir}; make -j 4",
+                           "mkdir -p {dir}",
+                           "ln -f -s {srcdir}/libmesh.so {dir}/libmesh.so"]
 
         super().__init__(name, **kwargs)
 
 
-mesh = Mesh("Mesh", color="xkcd:mint")
+mesh = Mesh("Mesh", version="4a1012cee990cb98cc1ea0294a836f467b29be02", color="xkcd:mint")
diff --git a/src/allocators/mimalloc.py b/src/allocators/mimalloc.py
index 2f84fe1..6931217 100644
--- a/src/allocators/mimalloc.py
+++ b/src/allocators/mimalloc.py
@@ -17,28 +17,23 @@
 
 """mimalloc definition for allocbench"""
 
-from src.allocator import Allocator, AllocatorSources
-
-VERSION = "master"
-
-MIMALLOC_SRC = AllocatorSources("mimalloc",
-                         ["git clone https://github.com/microsoft/mimalloc"],
-                         [f"git checkout {VERSION}"],
-                         ["git reset --hard"])
+from src.allocator import Allocator
+from src.artifact import GitArtifact
 
 
 class Mimalloc(Allocator):
     """mimalloc allocator"""
-    def __init__(self, name, **kwargs):
 
-        kwargs["sources"] = MIMALLOC_SRC
-        kwargs["LD_PRELOAD"] = "{dir}/libmimalloc.so"
-        kwargs["build_cmds"] = ["mkdir -p {dir}",
-                                "cd {dir}; cmake {srcdir}",
-                                "cd {dir}; make"]
-        kwargs["requirements"] = ["cmake"]
+    sources = GitArtifact("mimalloc", "https://github.com/microsoft/mimalloc")
+
+    def __init__(self, name, **kwargs):
+        self.LD_PRELOAD = "{dir}/libmimalloc.so"
+        self.build_cmds = ["mkdir -p {dir}",
+                           "cd {dir}; cmake {srcdir}",
+                           "cd {dir}; make"]
+        self.requirements = ["cmake"]
 
         super().__init__(name, **kwargs)
 
 
-mimalloc = Mimalloc("mimalloc")
+mimalloc = Mimalloc("mimalloc", version="v1.0.8")
diff --git a/src/allocators/scalloc.py b/src/allocators/scalloc.py
index 0b3d3cf..82ef8d1 100644
--- a/src/allocators/scalloc.py
+++ b/src/allocators/scalloc.py
@@ -17,32 +17,25 @@
 
 """Scalloc definition for allocbench"""
 
-from src.allocator import Allocator, AllocatorSources
-from src.util import print_error
-
-
-VERSION = "v1.0.0"
-
-SCALLOC_SRC = AllocatorSources("scalloc",
-                      retrieve_cmds=["git clone https://github.com/cksystemsgroup/scalloc"],
-                      prepare_cmds=[f"git checkout {VERSION}",
-                                    "cd {srcdir}; tools/make_deps.sh",
-                                    "cd {srcdir}; build/gyp/gyp --depth=. scalloc.gyp"],
-                      reset_cmds=["git reset --hard"])
+from src.allocator import Allocator
+from src.artifact import GitArtifact
 
 
 class Scalloc(Allocator):
     """Scalloc allocator"""
-    def __init__(self, name, **kwargs):
 
-        kwargs["sources"] = SCALLOC_SRC
+    sources = GitArtifact("scalloc", "https://github.com/cksystemsgroup/scalloc")
+
+    def __init__(self, name, **kwargs):
+        self.prepare_cmds = ["tools/make_deps.sh",
+                             "build/gyp/gyp --depth=. scalloc.gyp"]
 
-        kwargs["build_cmds"] = ["cd {srcdir}; BUILDTYPE=Release make",
-                                "mkdir -p {dir}"]
+        self.build_cmds = ["cd {srcdir}; BUILDTYPE=Release make",
+                           "mkdir -p {dir}"]
 
-        kwargs["LD_PRELOAD"] = "{srcdir}/out/Release/lib.target/libscalloc.so"
+        self.LD_PRELOAD = "{srcdir}/out/Release/lib.target/libscalloc.so"
 
-        kwargs["patches"] = ["{patchdir}/scalloc_fix_log.patch"]
+        self.patches = ["{patchdir}/scalloc_fix_log.patch"]
 
         super().__init__(name, **kwargs)
 
@@ -57,4 +50,4 @@ sysctl vm.overcommit_memory=1
         return super().build()
 
 
-scalloc = Scalloc("scalloc", color="xkcd:magenta")
+scalloc = Scalloc("scalloc", color="xkcd:magenta", version="v1.0.0")
diff --git a/src/allocators/snmalloc.py b/src/allocators/snmalloc.py
index 0496196..7f27e4b 100644
--- a/src/allocators/snmalloc.py
+++ b/src/allocators/snmalloc.py
@@ -17,28 +17,23 @@
 
 """Snmalloc definition for allocbench"""
 
-from src.allocator import Allocator, AllocatorSources
-
-VERSION = "master"
-
-SNMALLOC_SRC = AllocatorSources("snmalloc",
-                         ["git clone https://github.com/microsoft/snmalloc"],
-                         [f"git checkout {VERSION}"],
-                         ["git reset --hard"])
+from src.allocator import Allocator
+from src.artifact import GitArtifact
 
 
 class Snmalloc(Allocator):
     """snmalloc allocator"""
-    def __init__(self, name, **kwargs):
 
-        kwargs["sources"] = SNMALLOC_SRC
-        kwargs["LD_PRELOAD"] = "{dir}/libsnmallocshim.so"
-        kwargs["build_cmds"] = ["mkdir -p {dir}",
-                                "cd {dir}; cmake -G Ninja {srcdir} -DCMAKE_BUILD_TYPE=Release",
-                                "cd {dir}; ninja"]
-        kwargs["requirements"] = ["cmake", "ninja", "clang"]
+    sources = GitArtifact("snmalloc", "https://github.com/microsoft/snmalloc")
+
+    def __init__(self, name, **kwargs):
+        self.LD_PRELOAD = "{dir}/libsnmallocshim.so"
+        self.build_cmds = ["mkdir -p {dir}",
+                           "cd {dir}; cmake -G Ninja {srcdir} -DCMAKE_BUILD_TYPE=Release",
+                           "cd {dir}; ninja"]
+        self.requirements = ["cmake", "ninja", "clang"]
 
         super().__init__(name, **kwargs)
 
 
-snmalloc = Snmalloc("snmalloc")
+snmalloc = Snmalloc("snmalloc", version="0.2")
diff --git a/src/allocators/speedymalloc.py b/src/allocators/speedymalloc.py
index 9df3d7d..0cf6836 100644
--- a/src/allocators/speedymalloc.py
+++ b/src/allocators/speedymalloc.py
@@ -26,4 +26,4 @@ import os
 from src.allocator import Allocator, BUILDDIR
 
 speedymalloc = Allocator("speedymalloc", LD_PRELOAD=os.path.join(BUILDDIR, "speedymalloc.so"),
-                    color="xkcd:dark")
+                         color="xkcd:dark")
diff --git a/src/allocators/streamflow.py b/src/allocators/streamflow.py
index 73aede9..005b9cb 100644
--- a/src/allocators/streamflow.py
+++ b/src/allocators/streamflow.py
@@ -17,25 +17,24 @@
 
 """Streamflow allocator definition for allocbench"""
 
-from src.allocator import Allocator, AllocatorSources
-
-
-sources = AllocatorSources("streamflow",
-            retrieve_cmds=["git clone https://github.com/scotts/streamflow"],
-            reset_cmds=["git reset --hard"])
+from src.allocator import Allocator
+from src.artifact import GitArtifact
 
 
 class Streamflow(Allocator):
     """Streamflow allocator"""
+
+    sources = GitArtifact("streamflow", "https://github.com/scotts/streamflow")
+
     def __init__(self, name, **kwargs):
 
-        kwargs["sources"] = sources
-        kwargs["LD_PRELOAD"] = "{dir}/libstreamflow.so"
-        kwargs["build_cmds"] = ["cd {srcdir}; make",
-                                "mkdir -p {dir}",
-                                "ln -f -s {srcdir}/libstreamflow.so {dir}/libstreamflow.so"]
+        self.LD_PRELOAD = "{dir}/libstreamflow.so"
+        self.build_cmds = ["cd {srcdir}; make",
+                           "mkdir -p {dir}",
+                           "ln -f -s {srcdir}/libstreamflow.so {dir}/libstreamflow.so"]
 
         super().__init__(name, **kwargs)
 
 
-streamflow = Streamflow("Streamflow", color="xkcd:light brown")
+streamflow = Streamflow("Streamflow", version="8ac345c0f69ec9e7af02f3555c2c97eaa07a442e",
+                        color="xkcd:light brown")
diff --git a/src/allocators/supermalloc.py b/src/allocators/supermalloc.py
index 130dd47..f0a192a 100644
--- a/src/allocators/supermalloc.py
+++ b/src/allocators/supermalloc.py
@@ -17,29 +17,23 @@
 
 """SuperMalloc definition for allocbench"""
 
-from src.allocator import Allocator, AllocatorSources
-
-import src.allocator
-
-VERSION = "709663fb81ba091b0a78058869a644a272f4163d"
-
-sources = AllocatorSources("SuperMalloc",
-            retrieve_cmds=["git clone https://github.com/kuszmaul/SuperMalloc"],
-            prepare_cmds=[f"git checkout {VERSION}"],
-            reset_cmds=["git reset --hard"])
-
+from src.allocator import Allocator
+from src.artifact import GitArtifact
 
 class SuperMalloc(Allocator):
     """SuperMalloc allocator"""
-    def __init__(self, name, **kwargs):
 
-        kwargs["sources"] = sources
-        kwargs["LD_PRELOAD"] = "{srcdir}/release/lib/libsupermalloc.so"
-        kwargs["build_cmds"] = ["cd {srcdir}/release; make",
-                                "mkdir -p {dir}"]
-        kwargs["patches"] = ["{patchdir}/remove_faulty_aligned_alloc_test.patch"]
+    sources = GitArtifact("SuperMalloc", "https://github.com/kuszmaul/SuperMalloc")
+
+    def __init__(self, name, **kwargs):
+        self.LD_PRELOAD = "{dir}/libsupermalloc.so"
+        self.build_cmds = ["cd {srcdir}/release; make",
+                           "mkdir -p {dir}",
+                           "ln -f -s {srcdir}/release/lib/libsupermalloc.so {dir}/libsupermalloc.so"]
+        self.patches = ["{patchdir}/remove_faulty_aligned_alloc_test.patch"]
 
         super().__init__(name, **kwargs)
 
 
-supermalloc = SuperMalloc("SuperMalloc", color="xkcd:lime")
+supermalloc = SuperMalloc("SuperMalloc", color="xkcd:lime",
+                          version="709663fb81ba091b0a78058869a644a272f4163d")
diff --git a/src/allocators/SuperMalloc/remove_faulty_aligned_alloc_test.patch b/src/allocators/supermalloc/remove_faulty_aligned_alloc_test.patch
index dc7f7c4..dc7f7c4 100644
--- a/src/allocators/SuperMalloc/remove_faulty_aligned_alloc_test.patch
+++ b/src/allocators/supermalloc/remove_faulty_aligned_alloc_test.patch
diff --git a/src/allocators/tbbmalloc.py b/src/allocators/tbbmalloc.py
index aa50253..a7af5a5 100644
--- a/src/allocators/tbbmalloc.py
+++ b/src/allocators/tbbmalloc.py
@@ -17,27 +17,22 @@
 
 """tbbmalloc definition for allocbench"""
 
-from src.allocator import Allocator, AllocatorSources
-
-VERSION = "2019_U8"
-
-source = AllocatorSources("tbb",
-                         ["git clone https://github.com/intel/tbb.git"],
-                         [f"git checkout {VERSION}"],
-                         ["git reset --hard"])
+from src.allocator import Allocator
+from src.artifact import GitArtifact
 
 
 class TBBMalloc(Allocator):
     """tbbmalloc allocator"""
-    def __init__(self, name, **kwargs):
 
-        kwargs["sources"] = source
-        kwargs["LD_PRELOAD"] = "{dir}/libtbbmalloc.so"
-        kwargs["build_cmds"] = ["cd {srcdir}; make tbbmalloc -j4",
-                                "mkdir -p {dir}",
-                                'ln -f -s $(find {srcdir} -executable -name "*libtbbmalloc_proxy.so*" | head -1) {dir}/libtbbmalloc.so']
+    sources = GitArtifact("tbb", "https://github.com/intel/tbb.git")
+
+    def __init__(self, name, **kwargs):
+        self.LD_PRELOAD = "{dir}/libtbbmalloc.so"
+        self.build_cmds = ["cd {srcdir}; make tbbmalloc -j4",
+                           "mkdir -p {dir}",
+                           'ln -f -s $(find {srcdir} -executable -name "*libtbbmalloc_proxy.so*" | head -1) {dir}/libtbbmalloc.so']
 
         super().__init__(name, **kwargs)
 
 
-tbbmalloc = TBBMalloc("tbbmalloc", color="xkcd:green")
+tbbmalloc = TBBMalloc("tbbmalloc", color="xkcd:green", version="2019_U8")
diff --git a/src/allocators/tcmalloc.py b/src/allocators/tcmalloc.py
index 916da88..f81f88b 100644
--- a/src/allocators/tcmalloc.py
+++ b/src/allocators/tcmalloc.py
@@ -17,30 +17,28 @@
 
 """TCMalloc definition for allocbench"""
 
-from src.allocator import Allocator, AllocatorSources
-
-VERSION = 2.7
-
-TCMALLOC_SRC = AllocatorSources("tcmalloc",
-                         ["git clone https://github.com/gperftools/gperftools.git tcmalloc"],
-                         [f"git checkout gperftools-{VERSION}", "./autogen.sh"],
-                         ["git reset --hard"])
+from src.allocator import Allocator
+from src.artifact import GitArtifact
 
 
 class TCMalloc(Allocator):
     """TCMalloc allocator"""
+
+    sources = GitArtifact("tcmalloc", "https://github.com/gperftools/gperftools.git")
+
     def __init__(self, name, **kwargs):
 
-        kwargs["sources"] = TCMALLOC_SRC
-        kwargs["LD_PRELOAD"] = "{dir}/lib/libtcmalloc.so"
-        kwargs["build_cmds"] = ["cd {srcdir}; ./configure --prefix={dir}",
-                                "cd {srcdir}; make install -j4"]
+        self.LD_PRELOAD = "{dir}/lib/libtcmalloc.so"
+        self.prepare_cmds = ["./autogen.sh"]
+        self.build_cmds = ["cd {srcdir}; ./configure --prefix={dir}",
+                           "cd {srcdir}; make install -j4"]
 
         super().__init__(name, **kwargs)
 
 
-tcmalloc = TCMalloc("TCMalloc", color="xkcd:blue")
+tcmalloc = TCMalloc("TCMalloc", color="xkcd:blue", version="gperftools-2.7")
 
 tcmalloc_nofs = TCMalloc("TCMalloc-NoFalsesharing",
                          patches=["{patchdir}/tcmalloc_2.7_no_active_falsesharing.patch"],
+                         version="gperftools-2.7",
                          color="xkcd:navy")
diff --git a/src/artifact.py b/src/artifact.py
new file mode 100644
index 0000000..010bc6b
--- /dev/null
+++ b/src/artifact.py
@@ -0,0 +1,142 @@
+# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
+#
+# This file is part of allocbench.
+#
+# allocbench is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# allocbench is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with allocbench.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Artifact classes
+
+An Artifact is a external ressource downloaded from the internet.
+There are two flavours of artifacts available: archives and git repositories.
+Both flavours are version controlled archive with a checksum and git repositories
+with a specific checkout.
+"""
+
+import os
+import subprocess
+
+import src.globalvars
+from src.util import print_info, print_debug, sha1sum
+
+ARTIFACT_STORE_DIR = os.path.join(src.globalvars.allocbenchdir, "cache")
+
+class Artifact:
+    """Base class for external ressources"""
+    store = {}
+
+    def __init__(self, name):
+        if name in Artifact.store:
+            raise Exception(f'duplicate artifact "{name}"')
+
+        Artifact.store[name] = self
+        self.name = name
+        self.basedir = os.path.join(ARTIFACT_STORE_DIR, name)
+
+    def retrieve(self, cmd):
+        """Run cmd to retrieve the artifact"""
+        os.makedirs(self.basedir, exist_ok=True)
+
+        print_info(f'Retrieving artifact "{self.name}"')
+        print_debug(f"By running: {cmd} in {self.basedir}")
+        proc = subprocess.run(cmd, cwd=self.basedir,
+                              # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                              universal_newlines=True)
+        if proc.returncode != 0:
+            raise Exception(f"Failed to retrieve {self.name}")
+
+
+class GitArtifact(Artifact):
+    """External git repository"""
+    def __init__(self, name, url):
+        super().__init__(name)
+        self.url = url
+        self.repo = os.path.join(self.basedir, "repo")
+
+    def retrieve(self):
+        """clone the git repo"""
+        super().retrieve(["git", "clone", "--recursive", "--bare", self.url, "repo"])
+
+    def provide(self, checkout, location=None):
+        """checkout new worktree at location"""
+        if not location:
+            location = os.path.join(self.basedir, checkout)
+
+        # check if we have already provided this checkout
+        if os.path.exists(location):
+            return
+
+        # check if we have already retrieved the repo
+        if not os.path.exists(self.repo):
+            self.retrieve()
+
+        print_debug("create new worktree. By running: ",
+                    ["git", "worktree", "add", location, checkout], f"in {self.repo}")
+        proc = subprocess.run(["git", "worktree", "add", location, checkout], cwd=self.repo,
+                              # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                              universal_newlines=True)
+        if proc.returncode != 0:
+            raise Exception(f"Failed to provide {self.name}")
+
+        print_debug("update submodules in worktree. By running: ",
+                    ["git", "submodule", "update", "--init", "--recursive"], f"in {self.repo}")
+        proc = subprocess.run(["git", "submodule", "update", "--init", "--recursive"], cwd=location,
+                              # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                              universal_newlines=True)
+
+class ArchiveArtifact(Artifact):
+    """External archive"""
+    supported_formats = ["tar"]
+    def __init__(self, name, url, format, checksum):
+        super().__init__(name)
+        self.url = url
+        if format not in self.supported_formats:
+            raise Exception(f'Archive format "{format}" not in supported list {self.supported_formats}')
+        self.format = format
+        self.archive = os.path.join(self.basedir, f"{self.name}.{self.format}")
+        self.checksum = checksum
+
+    def retrieve(self):
+        """download the archive"""
+        super().retrieve(["wget", "-O", self.archive, self.url])
+
+    def provide(self, location=None):
+        """extract the archive"""
+
+        # Download archive
+        if not os.path.exists(self.archive):
+            self.retrieve()
+
+        # compare checksums
+        if sha1sum(self.archive) != self.checksum:
+            raise Exception(f"Archive {self.archive} does not match provided checksum")
+
+        if not location:
+            location = os.path.join(self.basedir, "content")
+
+        # Check if we already provided the archive at location
+        if os.path.exists(location):
+            return
+
+        os.makedirs(location, exist_ok=True)
+
+        # Extract archive
+        if self.format == "tar":
+            cmd = ["tar", "Cxf", location, self.archive]
+
+        print_debug(f"extract archive by running: {cmd} in {self.basedir}")
+        proc = subprocess.run(cmd, cwd=self.basedir,
+                              # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                              universal_newlines=True)
+        if proc.returncode != 0:
+            raise Exception(f"Failed to extract {self.name}")
diff --git a/src/benchmarks/dj_trace.py b/src/benchmarks/dj_trace.py
index f475223..5cb1b99 100644
--- a/src/benchmarks/dj_trace.py
+++ b/src/benchmarks/dj_trace.py
@@ -25,8 +25,9 @@ from urllib.request import urlretrieve
 import matplotlib.pyplot as plt
 import numpy as np
 
+from src.artifact import ArchiveArtifact
 from src.benchmark import Benchmark
-from src.util import print_status, download_reporthook
+from src.util import print_status
 
 
 COMMA_SEP_NUMBER_RE = "(?:\\d*(?:,\\d*)?)*"
@@ -57,7 +58,7 @@ class BenchmarkDJTrace(Benchmark):
     def __init__(self):
         name = "dj_trace"
 
-        self.cmd = "trace_run{binary_suffix} dj_workloads/{workload}.wl"
+        self.cmd = "trace_run{binary_suffix} {build_dir}/dj_workloads/{workload}.wl"
         self.measure_cmd = ""
 
         self.args = {"workload": ["389-ds-2",
@@ -106,29 +107,11 @@ class BenchmarkDJTrace(Benchmark):
     def prepare(self):
         super().prepare()
 
-        workload_dir = "dj_workloads"
-        workload_archive = f"{workload_dir}.tar.xz"
-
-        if not os.path.isdir(workload_dir):
-            if not os.path.isfile(workload_archive):
-                choice = input("Download missing workloads (367M / ~6GB unpacked) [Y/n] ")
-                if not choice in ['', 'Y', 'y']:
-                    return
-
-                url = f"https://www4.cs.fau.de/~flow/allocbench/{workload_archive}"
-                urlretrieve(url, workload_archive, download_reporthook)
-                sys.stderr.write("\n")
-
-            # Extract workloads
-            proc = subprocess.run(["tar", "xf", workload_archive], stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE, universal_newlines=True)
-
-            # delete archive
-            if proc.returncode == 0:
-                os.remove(workload_archive)
-
-        for workload in os.listdir(workload_dir):
-            self.args["workload"].append(os.path.splitext(workload)[0])
+        workloads = ArchiveArtifact("dj_workloads",
+                                    "https://www4.cs.fau.de/~flow/allocbench/dj_workloads.tar.xz",
+                                    "tar",
+                                    "c9bc499eeba8023bca28a755fffbaf9200a335ad")
+        workloads.provide(self.build_dir)
 
     @staticmethod
     def process_output(result, stdout, stderr, allocator, perm):
diff --git a/src/benchmarks/fd.py b/src/benchmarks/fd.py
index d78988a..21a987d 100644
--- a/src/benchmarks/fd.py
+++ b/src/benchmarks/fd.py
@@ -23,8 +23,9 @@ import subprocess
 import sys
 from urllib.request import urlretrieve
 
+from src.artifact import ArchiveArtifact, GitArtifact
 from src.benchmark import Benchmark
-from src.util import print_info, download_reporthook
+from src.util import print_info
 
 
 class BenchmarkFd(Benchmark):
@@ -33,57 +34,36 @@ class BenchmarkFd(Benchmark):
 
     def __init__(self):
         name = "fd"
-
         super().__init__(name)
         
-        self.cmd = "fd -HI -e c \"\" {build_dir}/linux"
+        self.cmd = "fd -HI -e c '.*[0-9].*' {build_dir}/linux"
 
     def prepare(self):
         super().prepare()
 
-        fd_tag = "v7.4.0"
-        fd_release =  f"fd-{fd_tag}-x86_64-unknown-linux-gnu"
-        fd_archive = f"{fd_release}.tar.gz"
-        fd_url = f"https://github.com/sharkdp/fd/releases/latest/download/{fd_archive}"
-        fd_dir = os.path.join(self.build_dir, fd_release)
-
-        self.results["facts"]["versions"]["fd"] = fd_tag
-
-        # Create builddir
-        os.makedirs(self.build_dir, exist_ok=True)
-
-        if not os.path.isdir(fd_dir):
-            if not os.path.isfile(fd_archive):
-                print(f"Downloading fd {fd_tag}...")
-                urlretrieve(fd_url, fd_archive, download_reporthook)
-                sys.stderr.write("\n")
-            
-            
-            # Extract redis
-            proc = subprocess.run(["tar", "Cxf", self.build_dir, fd_archive],
-                                  # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                  universal_newlines=True)
-            
-            # delete archive
-            if proc.returncode == 0:
-                os.remove(fd_archive)
-
-            
-            # create symlinks
-            for exe in ["fd"]:
-                src = os.path.join(fd_dir, exe)
-                dest = os.path.join(self.build_dir, exe)
-                os.link(src, dest)
+        if os.path.exists(self.build_dir):
+            return
+
+        fd_version = "v7.4.0"
+        self.results["facts"]["versions"]["fd"] = fd_version
+        fd_url = ("https://github.com/sharkdp/fd/releases/latest/download/"
+                  f"fd-{fd_version}-x86_64-unknown-linux-gnu.tar.gz")
+
+        fd = ArchiveArtifact("fd", fd_url, "tar", "a5d8e7c8484449aa324a46abfdfaf026d7de77ee")
+
+        fd_dir = os.path.join(self.build_dir, "fd_sources")
+        fd.provide(fd_dir)
+
+        # create symlinks
+        for exe in ["fd"]:
+            src = os.path.join(fd_dir, f"fd-{fd_version}-x86_64-unknown-linux-gnu", exe)
+            dest = os.path.join(self.build_dir, exe)
+            os.link(src, dest)
         
-        linux_version = "v5.3"
-        linux_url = f"git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git"
-        linux_dir = os.path.join(self.build_dir, "linux")
-        if not os.path.isdir(linux_dir):
-            # Extract redis
-            proc = subprocess.run(["git", "clone", linux_url, linux_dir],
-                                  # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                  universal_newlines=True)
 
+        linux = GitArtifact("linux", "git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git")
+        linux_version = "v5.3"
+        linux.provide(linux_version, os.path.join(self.build_dir, "linux"))
 
     def summary(self):
         self.barplot_single_arg("{task-clock}",
diff --git a/src/benchmarks/lld.py b/src/benchmarks/lld.py
index 7ebca13..0603037 100644
--- a/src/benchmarks/lld.py
+++ b/src/benchmarks/lld.py
@@ -24,9 +24,9 @@ import sys
 
 import matplotlib.pyplot as plt
 
+from src.artifact import ArchiveArtifact
 from src.benchmark import Benchmark
 import src.facter
-from src.util import download_reporthook
 
 
 class BenchmarkLld(Benchmark):
@@ -38,7 +38,7 @@ class BenchmarkLld(Benchmark):
     def __init__(self):
         name = "lld"
 
-        self.run_dir = "lld-speed-test/{test}"
+        self.run_dir = "{build_dir}/lld-speed-test/{test}"
         # TODO: don't hardcode ld.lld location
         self.cmd = "/usr/bin/ld.lld @response.txt"
 
@@ -55,28 +55,11 @@ class BenchmarkLld(Benchmark):
         # save lld version
         self.results["facts"]["versions"]["lld"] = src.facter.exe_version("ld.lld", "-v")
 
-        test_dir = "lld-speed-test"
-        test_archive = f"{test_dir}.tar.xz"
-        if not os.path.isdir(test_dir):
-            if not os.path.isfile(test_archive):
-                choice = input("Download missing test archive (1.1GB) [Y/n/x] ")
-                if not choice in ['', 'Y', 'y']:
-                    return
-
-                url = f"https://s3-us-west-2.amazonaws.com/linker-tests/{test_archive}"
-                urlretrieve(url, test_archive, download_reporthook)
-                sys.stderr.write("\n")
-
-            # Extract tests
-            proc = subprocess.run(["tar", "xf", test_archive], stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE, universal_newlines=True)
-
-            # delete archive
-            if proc.returncode == 0:
-                os.remove(test_archive)
-
-        self.args["test"] = os.listdir(test_dir)
-
+        tests = ArchiveArtifact("lld-speed-test",
+                                "https://s3-us-west-2.amazonaws.com/linker-tests/lld-speed-test.tar.xz",
+                                "tar",
+                                "2d449a11109c7363f67fd45513b42270f5ba2a92")
+        tests.provide(self.build_dir)
 
     def cleanup(self):
         for perm in self.iterate_args():
@@ -84,7 +67,6 @@ class BenchmarkLld(Benchmark):
             if os.path.isfile(a_out):
                 os.remove(a_out)
 
-
     def summary(self):
         args = self.results["args"]
         allocators = self.results["allocators"]
diff --git a/src/benchmarks/raxmlng.py b/src/benchmarks/raxmlng.py
index 119bab7..811721a 100644
--- a/src/benchmarks/raxmlng.py
+++ b/src/benchmarks/raxmlng.py
@@ -23,8 +23,9 @@ import subprocess
 import sys
 from urllib.request import urlretrieve
 
+from src.artifact import GitArtifact
 from src.benchmark import Benchmark
-from src.util import print_info, download_reporthook
+from src.util import print_info
 
 
 RUNTIME_RE = re.compile("Elapsed time: (?P<runtime>(\\d*.\\d*)) seconds")
@@ -39,64 +40,51 @@ class BenchmarkRaxmlng(Benchmark):
 
         super().__init__(name)
         
-        self.cmd = (f"raxml-ng --msa {self.build_dir}/ng-tutorial/prim.phy --model GTR+G"
+        self.cmd = (f"raxml-ng --msa {self.build_dir}/data/prim.phy --model GTR+G"
                     " --redo --threads 2 --seed 2")
 
     def prepare(self):
         super().prepare()
 
-        # git clone --recursive 
-        # cd raxml-ng
-        # mkdir build && cd build
-        # cmake ..
-        # make
+        if os.path.exists(self.build_dir):
+            return
 
-        version = "0.9"
-        
-        url = "https://github.com/amkozlov/raxml-ng"
-        data_url = "https://github.com/amkozlov/ng-tutorial"
+        raxmlng_sources = GitArtifact("raxml-ng", "https://github.com/amkozlov/raxml-ng")
+        raxmlng_version = "0.9.0"
         raxmlng_dir = os.path.join(self.build_dir, "raxml-ng-git")
         raxmlng_builddir = os.path.join(raxmlng_dir, "build")
-
-        self.results["facts"]["versions"]["raxml-ng"] = version
-
-        if not os.path.isdir(raxmlng_dir):
-            proc = subprocess.run(["git", "clone", "--recursive", url, raxmlng_dir],
-                                  # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                  universal_newlines=True)
+        self.results["facts"]["versions"]["raxml-ng"] = raxmlng_version
+        raxmlng_sources.provide(raxmlng_version, raxmlng_dir)
             
-            # Create builddir
-            os.makedirs(raxmlng_builddir, exist_ok=True)
-
-            # building raxml-ng
-            proc = subprocess.run(["cmake", ".."],
-                                  cwd=raxmlng_builddir,
-                                  # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                  universal_newlines=True)
-
-            proc = subprocess.run(["make"],
-                                  cwd=raxmlng_builddir,
-                                  # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                  universal_newlines=True)
-            
-            # create symlinks
-            for exe in ["raxml-ng"]:
-                src = os.path.join(raxmlng_dir, "bin", exe)
-                dest = os.path.join(self.build_dir,exe)
-                os.link(src, dest)
-
-            # clone test data
-            proc = subprocess.run(["git", "clone", data_url],
-                                  cwd=self.build_dir,
-                                  # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                  universal_newlines=True)
+        # Create builddir
+        os.makedirs(raxmlng_builddir, exist_ok=True)
+
+        # building raxml-ng
+        proc = subprocess.run(["cmake", ".."],
+                              cwd=raxmlng_builddir,
+                              # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                              universal_newlines=True)
+
+        proc = subprocess.run(["make"],
+                              cwd=raxmlng_builddir,
+                              # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                              universal_newlines=True)
+
+        # create symlinks
+        for exe in ["raxml-ng"]:
+            src = os.path.join(raxmlng_dir, "bin", exe)
+            dest = os.path.join(self.build_dir,exe)
+            os.link(src, dest)
+
+        raxmlng_data = GitArtifact("raxml-ng-data", "https://github.com/amkozlov/ng-tutorial")
+        raxmlng_data_dir = os.path.join(self.build_dir, "data")
+        raxmlng_data.provide("f8f0b6a057a11397b4dad308440746e3436db8b4", raxmlng_data_dir)
 
     def cleanup(self):
         for direntry in os.listdir():
             if direntry.startswith("prim.raxml"):
                 os.remove(direntry)
 
-
     @staticmethod
     def process_output(result, stdout, stderr, allocator, perm):
         result["runtime"] = RUNTIME_RE.search(stdout).group("runtime")
diff --git a/src/benchmarks/redis.py b/src/benchmarks/redis.py
index d67559a..80bd244 100644
--- a/src/benchmarks/redis.py
+++ b/src/benchmarks/redis.py
@@ -23,8 +23,9 @@ import subprocess
 import sys
 from urllib.request import urlretrieve
 
+from src.artifact import ArchiveArtifact
 from src.benchmark import Benchmark
-from src.util import print_info, download_reporthook
+from src.util import print_info
 
 
 REQUESTS_RE = re.compile("(?P<requests>(\\d*.\\d*)) requests per second")
@@ -51,40 +52,26 @@ class BenchmarkRedis(Benchmark):
         super().prepare()
 
         redis_version = "5.0.5"
-        redis_archive = f"redis-{redis_version}.tar.gz"
-        redis_url = f"http://download.redis.io/releases/{redis_archive}"
+        self.results["facts"]["versions"]["redis"] = redis_version
+        redis = ArchiveArtifact("redis",
+                                f"http://download.redis.io/releases/redis-{redis_version}.tar.gz",
+                                "tar",
+                                "71e38ae09ac70012b5bc326522b976bcb8e269d6")
+
         redis_dir = os.path.join(self.build_dir, f"redis-{redis_version}")
 
-        self.results["facts"]["versions"]["redis"] = redis_version
+        redis.provide(redis_dir)
+
+        # building redis
+        proc = subprocess.run(["make", "-C", redis_dir],
+                              # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                              universal_newlines=True)
 
-        if not os.path.isdir(redis_dir):
-            if not os.path.isfile(redis_archive):
-                print(f"Downloading redis-{redis_version}...")
-                urlretrieve(redis_url, redis_archive, download_reporthook)
-                sys.stderr.write("\n")
-
-            # Create build_dir
-            os.mkdir(self.build_dir)
-
-            # Extract redis
-            proc = subprocess.run(["tar", "Cxf", self.build_dir, redis_archive],
-                                  # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                  universal_newlines=True)
-
-            # delete archive
-            if proc.returncode == 0:
-                os.remove(redis_archive)
-
-            # building redis
-            proc = subprocess.run(["make", "-C", redis_dir],
-                                  # stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                  universal_newlines=True)
-
-            # create symlinks
-            for exe in ["redis-cli", "redis-server", "redis-benchmark"]:
-                src = os.path.join(redis_dir, "src", exe)
-                dest = os.path.join(self.build_dir, exe)
-                os.link(src, dest)
+        # create symlinks
+        for exe in ["redis-cli", "redis-server", "redis-benchmark"]:
+            src = os.path.join(redis_dir, "src", exe)
+            dest = os.path.join(self.build_dir, exe)
+            os.link(src, dest)
 
     @staticmethod
     def process_output(result, stdout, stderr, allocator, perm):
diff --git a/src/util.py b/src/util.py
index b44a263..c5fa162 100644
--- a/src/util.py
+++ b/src/util.py
@@ -17,24 +17,13 @@
 
 """Helper functions for allocbench"""
 
+import hashlib
 import os
 import subprocess
 import sys
 
 import src.globalvars
 
-def download_reporthook(blocknum, blocksize, totalsize):
-    """Status report hook for urlretrieve"""
-    readsofar = blocknum * blocksize
-    if totalsize > 0:
-        percent = readsofar * 100 / totalsize
-        status = "\r%5.1f%% %*d / %d" % (
-                  percent, len(str(totalsize)), readsofar, totalsize)
-        sys.stderr.write(status)
-    else:  # total size is unknown
-        sys.stderr.write(f"\rdownloaded {readsofar}")
-
-
 def is_exe(fpath):
     """Check if the given path is an exexutable file"""
     return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
@@ -173,3 +162,13 @@ def print_version_and_exit():
 
     print(f"{commit}{dirty}")
     exit(0)
+
+def sha1sum(filename):
+    """Return sha1sum of a file"""
+    sha1  = hashlib.sha1()
+    barray  = bytearray(64*1024)
+    view = memoryview(barray)
+    with open(filename, 'rb', buffering=0) as f:
+        for n in iter(lambda : f.readinto(view), 0):
+            sha1.update(view[:n])
+    return sha1.hexdigest()