diff options
26 files changed, 448 insertions, 475 deletions
@@ -1,4 +1,5 @@ build/* +cache/* results/* allocators/*.so dj_workloads/* diff --git a/src/allocator.py b/src/allocator.py index 41d66c7..b88b046 100644 --- a/src/allocator.py +++ b/src/allocator.py @@ -25,6 +25,7 @@ import shutil import subprocess import sys +from src.artifact import ArchiveArtifact, GitArtifact import src.globalvars from src.util import print_status, print_debug, print_error, print_info2 @@ -37,99 +38,30 @@ for line in subprocess.run(["ldconfig", "-v"], stdout=subprocess.PIPE, if not line.startswith('\t'): LIBRARY_PATH += line -BUILDDIR = os.path.join(src.globalvars.builddir, "allocators") +BUILDDIR = src.globalvars.allocbuilddir SRCDIR = os.path.join(src.globalvars.allocbuilddir, "src") if not os.path.isdir(SRCDIR): os.makedirs(SRCDIR) -class AllocatorSources: - """Class representing sources an allocator is build from - - AllocatorSources can retrieve, prepare and reset their managed sources - """ - def __init__(self, name, retrieve_cmds=None, prepare_cmds=None, reset_cmds=None): - self.name = name - self.dir = os.path.join(SRCDIR, self.name) - self.patchdir = os.path.join(src.globalvars.allocsrcdir, self.name) - self.retrieve_cmds = retrieve_cmds or [] - self.prepare_cmds = prepare_cmds or [] - self.reset_cmds = reset_cmds or [] - - def run_cmds(self, function, cwd=SRCDIR): - """Helper to run retrieve, prepare or reset commands""" - print_status(function, self.name, "...") - - cmds = getattr(self, function+"_cmds") - - stdout = subprocess.PIPE if src.globalvars.verbosity < 2 else None - - for cmd in cmds: - proc = subprocess.run(cmd, shell=True, cwd=cwd, - stderr=subprocess.PIPE, stdout=stdout, - universal_newlines=True) - - if proc.returncode: - print_error(function, self.name, "failed with", proc.returncode, - file=sys.stderr) - print_debug(proc.stderr, file=sys.stderr) - return False - return True - - def prepare(self): - """Prepare the managed sources for building - - If the sources aren't available yet they are retrieved. - Otherwise they are reset. - """ - if not os.path.isdir(self.dir): - if (not self.run_cmds("retrieve") or - not self.run_cmds("prepare", cwd=self.dir)): - - shutil.rmtree(self.dir, ignore_errors=True) - exit(1) - else: - self.reset() - - def reset(self): - """Reset the managed sources""" - if not self.run_cmds("reset", cwd=self.dir): - exit(1) - - def patch(self, patches): - """Patch the managed sources using patch(1)""" - if not patches: - return - - stdout = subprocess.PIPE if src.globalvars.verbosity < 2 else None - cwd = os.path.join(SRCDIR, self.name) - - print_status("Patching", self.name, "...") - for patch in patches: - with open(patch.format(patchdir=self.patchdir), "rb") as patch_file: - proc = subprocess.run("patch -p1", shell=True, cwd=cwd, - stderr=subprocess.PIPE, stdout=stdout, - input=patch_file.read()) - - if proc.returncode: - print_error("Patching of", self.name, "failed.", - file=sys.stderr) - print_debug(proc.stderr, file=sys.stderr) - exit(1) - - class Allocator: """Allocator base class - It builds the allocator and produces a for allocbench usable allocator dict""" - allowed_attributes = ["binary_suffix", "version", "sources", "build_cmds", - "LD_PRELOAD", "cmd_prefix", "color", "patches", - "LD_LIBRARY_PATH"] + An Allocator can contain an Artifact which provides the allocator sources, + patches, and instructions to build the allocator. + Allocator.build will compile the allocator and produce a for allocbench usable + allocator dict""" + allowed_attributes = ["binary_suffix", "cmd_prefix", "LD_PRELOAD", "LD_LIBRARY_PATH", "color", + "sources", "version", "patches", "prepare_cmds", + "build_cmds"] def __init__(self, name, **kwargs): + self.class_file = inspect.getfile(self.__class__) self.name = name + self.srcdir = os.path.join(SRCDIR, self.name) self.dir = os.path.join(BUILDDIR, self.name) + self.patchdir = os.path.join(os.path.splitext(self.class_file)[0]) # Update attributes self.__dict__.update((k, v) for k, v in kwargs.items() if k in self.allowed_attributes) @@ -139,51 +71,82 @@ class Allocator: if not hasattr(self, attr): setattr(self, attr, None) + def prepare(self): + """Prepare the allocators sources""" + if not self.sources and os.path.exists(self.srcdir): + return + + print_status("Preparing", self.name, "...") + if type(self.sources) == GitArtifact: + self.sources.provide(self.version, self.srcdir) + elif type(self.sources) == ArchiveArtifact: + self.sources.provide(self.srcdir) + + if self.patches: + stdout = subprocess.PIPE if src.globalvars.verbosity < 2 else None + cwd = os.path.join(SRCDIR, self.name) + + print_status("Patching", self.name, "...") + for patch in self.patches: + with open(patch.format(patchdir=self.patchdir), "rb") as patch_file: + proc = subprocess.run("patch -p1", shell=True, cwd=cwd, + stderr=subprocess.PIPE, stdout=stdout, + input=patch_file.read()) + + if proc.returncode: + print_debug(proc.stderr, file=sys.stderr) + raise Exception(f"Patching of {self.name} failed.") + + if self.prepare_cmds: + print_status("Run prepare commands", self.name, "...") + stdout = subprocess.PIPE if src.globalvars.verbosity < 2 else None + + for cmd in self.prepare_cmds: + proc = subprocess.run(cmd, shell=True, cwd=self.srcdir, + stderr=subprocess.PIPE, stdout=stdout, + universal_newlines=True) + + if proc.returncode: + print_debug(proc.stderr, file=sys.stderr) + raise Exception(f'Prepare command "{cmd}" failed with exitcode: {proc.returncode}.') + def build(self): - """Build the allocator if needed and produce allocator dict""" + """Build the allocator if needed and produce an allocator dict""" build_needed = not os.path.isdir(self.dir) buildtimestamp_path = os.path.join(self.dir, ".buildtime") - print_info2(f"Building {self.name}...") + print_status("Building", self.name, "...") if not build_needed: print_info2("Old build found. Comparing build time with mtime") with open(buildtimestamp_path, "r") as buildtimestamp_file: timestamp = datetime.fromtimestamp(float(buildtimestamp_file.read())) - modtime = os.stat(inspect.getfile(self.__class__)).st_mtime + modtime = os.stat(self.class_file).st_mtime modtime = datetime.fromtimestamp(modtime) build_needed = timestamp < modtime print_debug("Time of last build:", timestamp.isoformat()) - print_debug("Last modification of allocators file:", - modtime.isoformat()) + print_debug("Last modification of allocators file:", modtime.isoformat()) print_info2("" if build_needed else "No " + "build needed") if build_needed: - if self.sources: - self.sources.prepare() - self.sources.patch(self.patches) + self.prepare() if self.build_cmds: - print_status("Building", self.name, "...") - stdout = subprocess.PIPE if src.globalvars.verbosity < 2 else None for cmd in self.build_cmds: - cmd = cmd.format(**{"dir": self.dir, - "srcdir": self.sources.dir}) + cmd = cmd.format(dir=self.dir, srcdir=self.srcdir) proc = subprocess.run(cmd, cwd=BUILDDIR, shell=True, stderr=subprocess.PIPE, stdout=stdout, universal_newlines=True) if proc.returncode: - print_error(cmd, "failed with:", proc.returncode) print_debug(proc.stderr, file=sys.stderr) - print_error("Building", self.name, "failed ...") shutil.rmtree(self.dir, ignore_errors=True) - exit(2) + raise Exception(f'Build command "{cmd}" failed with exitcode: {proc.returncode}') with open(buildtimestamp_path, "w") as buildtimestamp_file: print_info2("Save build time to:", buildtimestamp_path) @@ -196,13 +159,10 @@ class Allocator: "LD_LIBRARY_PATH": self.LD_LIBRARY_PATH or "", "color": self.color} - paths = {"dir": self.dir} - paths["srcdir"] = self.sources.dir if self.sources is not None else "" - for attr in ["LD_PRELOAD", "LD_LIBRARY_PATH", "cmd_prefix"]: value = getattr(self, attr, "") or "" if value != "": - value = value.format(**paths) + value = value.format(dir=self.dir, srcdir=self.srcdir) res_dict[attr] = value print_debug("Resulting dictionary:", res_dict) diff --git a/src/allocators/all.py b/src/allocators/all.py index 6b3b46a..0593c81 100644 --- a/src/allocators/all.py +++ b/src/allocators/all.py @@ -31,5 +31,5 @@ from src.allocators.snmalloc import snmalloc allocators = [*src.allocators.glibcs.allocators, tcmalloc, tcmalloc_nofs, - jemalloc, hoard, mesh, supermalloc, scalloc, llalloc, tbbmalloc, + jemalloc, hoard, mesh, supermalloc, scalloc, tbbmalloc, llalloc, # streamflow, mimalloc, snmalloc] diff --git a/src/allocators/glibc.py b/src/allocators/glibc.py index c322427..3c739fe 100644 --- a/src/allocators/glibc.py +++ b/src/allocators/glibc.py @@ -17,52 +17,48 @@ """Glibc definitions""" -from src.allocator import Allocator, AllocatorSources, LIBRARY_PATH - -VERSION = 2.29 - -GLIBC_SRC = AllocatorSources("glibc", - retrieve_cmds=["git clone git://sourceware.org/git/glibc.git"], - prepare_cmds=[f"git checkout glibc-{VERSION}"], - reset_cmds=["git reset --hard"]) +from src.allocator import Allocator, LIBRARY_PATH +from src.artifact import GitArtifact class Glibc(Allocator): """Glibc definition for allocbench Glibcs are loaded using their own supplied loader""" + + sources = GitArtifact("glibc", "git://sourceware.org/git/glibc.git") + def __init__(self, name, **kwargs): - kwargs["sources"] = GLIBC_SRC configure_args = "" if "configure_args" in kwargs: configure_args = kwargs["configure_args"] - del(kwargs["configure_args"]) - - kwargs["build_cmds"] = ["mkdir -p glibc-build", - "cd glibc-build; {srcdir}/configure --prefix={dir} " + configure_args, - "cd glibc-build; make", - "cd glibc-build; make install"] + del kwargs["configure_args"] - kwargs["cmd_prefix"] = ("{dir}/lib/ld-linux-x86-64.so.2 --library-path {dir}/lib:" - + LIBRARY_PATH) + self.build_cmds = ["mkdir -p glibc-build", + "cd glibc-build; {srcdir}/configure --prefix={dir} " + configure_args, + "cd glibc-build; make", + "cd glibc-build; make install"] - # kwargs["LD_LIBRARY_PATH"] = "{dir}/lib:" + library_path + self.cmd_prefix = "{dir}/lib/ld-linux-x86-64.so.2 --library-path {dir}/lib:" + LIBRARY_PATH super().__init__(name, **kwargs) -glibc = Glibc("glibc", color="xkcd:red") +glibc = Glibc("glibc", version="glibc-2.29", color="xkcd:red") glibc_notc = Glibc("glibc-noThreadCache", configure_args="--disable-experimental-malloc", + version="glibc-2.29", color="xkcd:maroon") glibc_nofs = Glibc("glibc-noFalsesharing", patches=["{patchdir}/glibc_2.29_no_passive_falsesharing.patch"], + version="glibc-2.29", color="xkcd:pink") glibc_nofs_fancy = Glibc("glibc-noFalsesharingClever", patches=["{patchdir}/glibc_2.29_no_passive_falsesharing_fancy.patch"], + version="glibc-2.29", color="xkcd:orange") diff --git a/src/allocators/hoard.py b/src/allocators/hoard.py index cc2061a..7e94d79 100644 --- a/src/allocators/hoard.py +++ b/src/allocators/hoard.py @@ -17,26 +17,23 @@ """Hoard allocator definition for allocbench""" -from src.allocator import Allocator, AllocatorSources - - -sources = AllocatorSources("Hoard", - retrieve_cmds=["git clone https://github.com/emeryberger/Hoard.git"], - reset_cmds=["git reset --hard"]) +from src.allocator import Allocator +from src.artifact import GitArtifact class Hoard(Allocator): """Hoard allocator""" - def __init__(self, name, **kwargs): - kwargs["sources"] = sources - kwargs["LD_PRELOAD"] = "{dir}/libhoard.so" - kwargs["build_cmds"] = ["cd {srcdir}/src; make", - "mkdir -p {dir}", - "ln -f -s {srcdir}/src/libhoard.so {dir}/libhoard.so"] - kwargs["requirements"] = ["clang"] + sources = GitArtifact("Hoard", "https://github.com/emeryberger/Hoard.git") + + def __init__(self, name, **kwargs): + self.LD_PRELOAD = "{dir}/libhoard.so" + self.build_cmds = ["cd {srcdir}/src; make", + "mkdir -p {dir}", + "ln -f -s {srcdir}/src/libhoard.so {dir}/libhoard.so"] + self.requirements = ["clang"] super().__init__(name, **kwargs) -hoard = Hoard("Hoard", color="xkcd:brown") +hoard = Hoard("Hoard", version="aa6d31700d5368a9f5ede3d62731247c8d9f0ebb", color="xkcd:brown") diff --git a/src/allocators/installed_allocators.py b/src/allocators/installed_allocators.py index 9a11d04..d45755a 100644 --- a/src/allocators/installed_allocators.py +++ b/src/allocators/installed_allocators.py @@ -22,11 +22,11 @@ import subprocess # TODO: add more allocators MAYBE_ALLOCATORS = ["tcmalloc", "jemalloc", "hoard"] -allocators = {"libc": {"cmd_prefix": "", - "binary_suffix": "", - "LD_PRELOAD": "", - "LD_LIBRARY_PATH": "", - "color": "C1"}} +allocators = {"libc": {"cmd_prefix": "", + "binary_suffix": "", + "LD_PRELOAD": "", + "LD_LIBRARY_PATH": "", + "color": "C1"}} for i, t in enumerate(MAYBE_ALLOCATORS): try: diff --git a/src/allocators/jemalloc.py b/src/allocators/jemalloc.py index 4484362..18e3055 100644 --- a/src/allocators/jemalloc.py +++ b/src/allocators/jemalloc.py @@ -17,26 +17,24 @@ """jemalloc definition for allocbench""" -from src.allocator import Allocator, AllocatorSources - -VERSION = "5.1.0" - -sources = AllocatorSources("jemalloc", - retrieve_cmds=["git clone https://github.com/jemalloc/jemalloc.git"], - prepare_cmds=[f"git checkout {VERSION}", "./autogen.sh"]) +from src.allocator import Allocator +from src.artifact import GitArtifact class Jemalloc(Allocator): """jemalloc allocator""" - def __init__(self, name, **kwargs): - kwargs["sources"] = sources - kwargs["LD_PRELOAD"] = "{srcdir}/lib/libjemalloc.so" - kwargs["build_cmds"] = ["cd {srcdir}; ./configure --prefix={dir}", - "cd {srcdir}; make -j4", - "mkdir -p {dir}"] + sources = GitArtifact("jemalloc", "https://github.com/jemalloc/jemalloc.git") + + def __init__(self, name, **kwargs): + self.LD_PRELOAD = "{dir}/libjemalloc.so" + self.prepare_cmds = ["./autogen.sh"] + self.build_cmds = ["cd {srcdir}; ./configure --prefix={dir}", + "cd {srcdir}; make -j4", + "mkdir -p {dir}", + "ln -f -s {srcdir}/lib/libjemalloc.so {dir}/libjemalloc.so"] super().__init__(name, **kwargs) -jemalloc = Jemalloc("jemalloc", color="xkcd:yellow") +jemalloc = Jemalloc("jemalloc", version="5.1.0", color="xkcd:yellow") diff --git a/src/allocators/llalloc.py b/src/allocators/llalloc.py index b0da505..8f43f59 100644 --- a/src/allocators/llalloc.py +++ b/src/allocators/llalloc.py @@ -17,25 +17,24 @@ """Lockless allocator definition for allocbench""" -from src.allocator import Allocator, AllocatorSources - - -LLALLOC_SOURCE = AllocatorSources("lockless_allocator", - retrieve_cmds=["wget https://locklessinc.com/downloads/lockless_allocator_src.tgz", - "tar xf lockless_allocator_src.tgz"], - prepare_cmds=[], - reset_cmds=[]) +from src.allocator import Allocator +from src.artifact import ArchiveArtifact class LocklessAllocator(Allocator): """Lockless allocator""" def __init__(self, name, **kwargs): - kwargs["sources"] = LLALLOC_SOURCE + self.sources = ArchiveArtifact("llalloc", + "https://locklessinc.com/downloads/lockless_allocator_src.tgz", + "tar", + "c6cb5a57882fa4775b5227a322333a6126b61f7c") - kwargs["build_cmds"] = ["cd {srcdir}; make", "mkdir -p {dir}"] + self.build_cmds = ["cd {srcdir}/lockless_allocator; make", + "mkdir -p {dir}", + "ln -f -s {srcdir}/lockless_allocator/libllalloc.so.1.3 {dir}/libllalloc.so"] - kwargs["LD_PRELOAD"] = "{srcdir}/libllalloc.so.1.3" + self.LD_PRELOAD = "{dir}/libllalloc.so" super().__init__(name, **kwargs) diff --git a/src/allocators/malt.py b/src/allocators/malt.py index 599e8ff..34fd9cc 100644 --- a/src/allocators/malt.py +++ b/src/allocators/malt.py @@ -24,6 +24,4 @@ of a program. See https://github.com/memtt/malt for more details from src.allocator import Allocator # result_dir and perm are substituted during Benchmark.run -cmd = "malt -q -o output:name={{result_dir}}/malt.{{perm}}.%3" - -malt = Allocator("malt", cmd_prefix=cmd) +malt = Allocator("malt", cmd_prefix="malt -q -o output:name={{result_dir}}/malt.{{perm}}.%3") diff --git a/src/allocators/mesh.py b/src/allocators/mesh.py index f8b7c16..2391e23 100644 --- a/src/allocators/mesh.py +++ b/src/allocators/mesh.py @@ -17,29 +17,23 @@ """Mesh definition for allocbench""" -from src.allocator import Allocator, AllocatorSources - -sources = AllocatorSources("Mesh", - retrieve_cmds=["git clone https://github.com/plasma-umass/Mesh"], - reset_cmds=["git reset --hard"]) - -# sources = src.allocator.GitAllocatorSources("Mesh", -# "https://github.com/plasma-umass/Mesh", -# "adsf0982345") +from src.allocator import Allocator +from src.artifact import GitArtifact class Mesh(Allocator): """Mesh allocator""" - def __init__(self, name, **kwargs): - kwargs["sources"] = sources - kwargs["LD_PRELOAD"] = "{srcdir}/libmesh.so" - kwargs["build_cmds"] = ["cd {srcdir}; git submodule update --init", - "cd {srcdir}; ./configure", - "cd {srcdir}; make -j 4", - "mkdir -p {dir}"] + sources = GitArtifact("Mesh", "https://github.com/plasma-umass/Mesh") + + def __init__(self, name, **kwargs): + self.LD_PRELOAD = "{dir}/libmesh.so" + self.build_cmds = ["cd {srcdir}; ./configure", + "cd {srcdir}; make -j 4", + "mkdir -p {dir}", + "ln -f -s {srcdir}/libmesh.so {dir}/libmesh.so"] super().__init__(name, **kwargs) -mesh = Mesh("Mesh", color="xkcd:mint") +mesh = Mesh("Mesh", version="4a1012cee990cb98cc1ea0294a836f467b29be02", color="xkcd:mint") diff --git a/src/allocators/mimalloc.py b/src/allocators/mimalloc.py index 2f84fe1..6931217 100644 --- a/src/allocators/mimalloc.py +++ b/src/allocators/mimalloc.py @@ -17,28 +17,23 @@ """mimalloc definition for allocbench""" -from src.allocator import Allocator, AllocatorSources - -VERSION = "master" - -MIMALLOC_SRC = AllocatorSources("mimalloc", - ["git clone https://github.com/microsoft/mimalloc"], - [f"git checkout {VERSION}"], - ["git reset --hard"]) +from src.allocator import Allocator +from src.artifact import GitArtifact class Mimalloc(Allocator): """mimalloc allocator""" - def __init__(self, name, **kwargs): - kwargs["sources"] = MIMALLOC_SRC - kwargs["LD_PRELOAD"] = "{dir}/libmimalloc.so" - kwargs["build_cmds"] = ["mkdir -p {dir}", - "cd {dir}; cmake {srcdir}", - "cd {dir}; make"] - kwargs["requirements"] = ["cmake"] + sources = GitArtifact("mimalloc", "https://github.com/microsoft/mimalloc") + + def __init__(self, name, **kwargs): + self.LD_PRELOAD = "{dir}/libmimalloc.so" + self.build_cmds = ["mkdir -p {dir}", + "cd {dir}; cmake {srcdir}", + "cd {dir}; make"] + self.requirements = ["cmake"] super().__init__(name, **kwargs) -mimalloc = Mimalloc("mimalloc") +mimalloc = Mimalloc("mimalloc", version="v1.0.8") diff --git a/src/allocators/scalloc.py b/src/allocators/scalloc.py index 0b3d3cf..82ef8d1 100644 --- a/src/allocators/scalloc.py +++ b/src/allocators/scalloc.py @@ -17,32 +17,25 @@ """Scalloc definition for allocbench""" -from src.allocator import Allocator, AllocatorSources -from src.util import print_error - - -VERSION = "v1.0.0" - -SCALLOC_SRC = AllocatorSources("scalloc", - retrieve_cmds=["git clone https://github.com/cksystemsgroup/scalloc"], - prepare_cmds=[f"git checkout {VERSION}", - "cd {srcdir}; tools/make_deps.sh", - "cd {srcdir}; build/gyp/gyp --depth=. scalloc.gyp"], - reset_cmds=["git reset --hard"]) +from src.allocator import Allocator +from src.artifact import GitArtifact class Scalloc(Allocator): """Scalloc allocator""" - def __init__(self, name, **kwargs): - kwargs["sources"] = SCALLOC_SRC + sources = GitArtifact("scalloc", "https://github.com/cksystemsgroup/scalloc") + + def __init__(self, name, **kwargs): + self.prepare_cmds = ["tools/make_deps.sh", + "build/gyp/gyp --depth=. scalloc.gyp"] - kwargs["build_cmds"] = ["cd {srcdir}; BUILDTYPE=Release make", - "mkdir -p {dir}"] + self.build_cmds = ["cd {srcdir}; BUILDTYPE=Release make", + "mkdir -p {dir}"] - kwargs["LD_PRELOAD"] = "{srcdir}/out/Release/lib.target/libscalloc.so" + self.LD_PRELOAD = "{srcdir}/out/Release/lib.target/libscalloc.so" - kwargs["patches"] = ["{patchdir}/scalloc_fix_log.patch"] + self.patches = ["{patchdir}/scalloc_fix_log.patch"] super().__init__(name, **kwargs) @@ -57,4 +50,4 @@ sysctl vm.overcommit_memory=1 return super().build() -scalloc = Scalloc("scalloc", color="xkcd:magenta") +scalloc = Scalloc("scalloc", color="xkcd:magenta", version="v1.0.0") diff --git a/src/allocators/snmalloc.py b/src/allocators/snmalloc.py index 0496196..7f27e4b 100644 --- a/src/allocators/snmalloc.py +++ b/src/allocators/snmalloc.py @@ -17,28 +17,23 @@ """Snmalloc definition for allocbench""" -from src.allocator import Allocator, AllocatorSources - -VERSION = "master" - -SNMALLOC_SRC = AllocatorSources("snmalloc", - ["git clone https://github.com/microsoft/snmalloc"], - [f"git checkout {VERSION}"], - ["git reset --hard"]) +from src.allocator import Allocator +from src.artifact import GitArtifact class Snmalloc(Allocator): """snmalloc allocator""" - def __init__(self, name, **kwargs): - kwargs["sources"] = SNMALLOC_SRC - kwargs["LD_PRELOAD"] = "{dir}/libsnmallocshim.so" - kwargs["build_cmds"] = ["mkdir -p {dir}", - "cd {dir}; cmake -G Ninja {srcdir} -DCMAKE_BUILD_TYPE=Release", - "cd {dir}; ninja"] - kwargs["requirements"] = ["cmake", "ninja", "clang"] + sources = GitArtifact("snmalloc", "https://github.com/microsoft/snmalloc") + + def __init__(self, name, **kwargs): + self.LD_PRELOAD = "{dir}/libsnmallocshim.so" + self.build_cmds = ["mkdir -p {dir}", + "cd {dir}; cmake -G Ninja {srcdir} -DCMAKE_BUILD_TYPE=Release", + "cd {dir}; ninja"] + self.requirements = ["cmake", "ninja", "clang"] super().__init__(name, **kwargs) -snmalloc = Snmalloc("snmalloc") +snmalloc = Snmalloc("snmalloc", version="0.2") diff --git a/src/allocators/speedymalloc.py b/src/allocators/speedymalloc.py index 9df3d7d..0cf6836 100644 --- a/src/allocators/speedymalloc.py +++ b/src/allocators/speedymalloc.py @@ -26,4 +26,4 @@ import os from src.allocator import Allocator, BUILDDIR speedymalloc = Allocator("speedymalloc", LD_PRELOAD=os.path.join(BUILDDIR, "speedymalloc.so"), - color="xkcd:dark") + color="xkcd:dark") diff --git a/src/allocators/streamflow.py b/src/allocators/streamflow.py index 73aede9..005b9cb 100644 --- a/src/allocators/streamflow.py +++ b/src/allocators/streamflow.py @@ -17,25 +17,24 @@ """Streamflow allocator definition for allocbench""" -from src.allocator import Allocator, AllocatorSources - - -sources = AllocatorSources("streamflow", - retrieve_cmds=["git clone https://github.com/scotts/streamflow"], - reset_cmds=["git reset --hard"]) +from src.allocator import Allocator +from src.artifact import GitArtifact class Streamflow(Allocator): """Streamflow allocator""" + + sources = GitArtifact("streamflow", "https://github.com/scotts/streamflow") + def __init__(self, name, **kwargs): - kwargs["sources"] = sources - kwargs["LD_PRELOAD"] = "{dir}/libstreamflow.so" - kwargs["build_cmds"] = ["cd {srcdir}; make", - "mkdir -p {dir}", - "ln -f -s {srcdir}/libstreamflow.so {dir}/libstreamflow.so"] + self.LD_PRELOAD = "{dir}/libstreamflow.so" + self.build_cmds = ["cd {srcdir}; make", + "mkdir -p {dir}", + "ln -f -s {srcdir}/libstreamflow.so {dir}/libstreamflow.so"] super().__init__(name, **kwargs) -streamflow = Streamflow("Streamflow", color="xkcd:light brown") +streamflow = Streamflow("Streamflow", version="8ac345c0f69ec9e7af02f3555c2c97eaa07a442e", + color="xkcd:light brown") diff --git a/src/allocators/supermalloc.py b/src/allocators/supermalloc.py index 130dd47..f0a192a 100644 --- a/src/allocators/supermalloc.py +++ b/src/allocators/supermalloc.py @@ -17,29 +17,23 @@ """SuperMalloc definition for allocbench""" -from src.allocator import Allocator, AllocatorSources - -import src.allocator - -VERSION = "709663fb81ba091b0a78058869a644a272f4163d" - -sources = AllocatorSources("SuperMalloc", - retrieve_cmds=["git clone https://github.com/kuszmaul/SuperMalloc"], - prepare_cmds=[f"git checkout {VERSION}"], - reset_cmds=["git reset --hard"]) - +from src.allocator import Allocator +from src.artifact import GitArtifact class SuperMalloc(Allocator): """SuperMalloc allocator""" - def __init__(self, name, **kwargs): - kwargs["sources"] = sources - kwargs["LD_PRELOAD"] = "{srcdir}/release/lib/libsupermalloc.so" - kwargs["build_cmds"] = ["cd {srcdir}/release; make", - "mkdir -p {dir}"] - kwargs["patches"] = ["{patchdir}/remove_faulty_aligned_alloc_test.patch"] + sources = GitArtifact("SuperMalloc", "https://github.com/kuszmaul/SuperMalloc") + + def __init__(self, name, **kwargs): + self.LD_PRELOAD = "{dir}/libsupermalloc.so" + self.build_cmds = ["cd {srcdir}/release; make", + "mkdir -p {dir}", + "ln -f -s {srcdir}/release/lib/libsupermalloc.so {dir}/libsupermalloc.so"] + self.patches = ["{patchdir}/remove_faulty_aligned_alloc_test.patch"] super().__init__(name, **kwargs) -supermalloc = SuperMalloc("SuperMalloc", color="xkcd:lime") +supermalloc = SuperMalloc("SuperMalloc", color="xkcd:lime", + version="709663fb81ba091b0a78058869a644a272f4163d") diff --git a/src/allocators/SuperMalloc/remove_faulty_aligned_alloc_test.patch b/src/allocators/supermalloc/remove_faulty_aligned_alloc_test.patch index dc7f7c4..dc7f7c4 100644 --- a/src/allocators/SuperMalloc/remove_faulty_aligned_alloc_test.patch +++ b/src/allocators/supermalloc/remove_faulty_aligned_alloc_test.patch diff --git a/src/allocators/tbbmalloc.py b/src/allocators/tbbmalloc.py index aa50253..a7af5a5 100644 --- a/src/allocators/tbbmalloc.py +++ b/src/allocators/tbbmalloc.py @@ -17,27 +17,22 @@ """tbbmalloc definition for allocbench""" -from src.allocator import Allocator, AllocatorSources - -VERSION = "2019_U8" - -source = AllocatorSources("tbb", - ["git clone https://github.com/intel/tbb.git"], - [f"git checkout {VERSION}"], - ["git reset --hard"]) +from src.allocator import Allocator +from src.artifact import GitArtifact class TBBMalloc(Allocator): """tbbmalloc allocator""" - def __init__(self, name, **kwargs): - kwargs["sources"] = source - kwargs["LD_PRELOAD"] = "{dir}/libtbbmalloc.so" - kwargs["build_cmds"] = ["cd {srcdir}; make tbbmalloc -j4", - "mkdir -p {dir}", - 'ln -f -s $(find {srcdir} -executable -name "*libtbbmalloc_proxy.so*" | head -1) {dir}/libtbbmalloc.so'] + sources = GitArtifact("tbb", "https://github.com/intel/tbb.git") + + def __init__(self, name, **kwargs): + self.LD_PRELOAD = "{dir}/libtbbmalloc.so" + self.build_cmds = ["cd {srcdir}; make tbbmalloc -j4", + "mkdir -p {dir}", + 'ln -f -s $(find {srcdir} -executable -name "*libtbbmalloc_proxy.so*" | head -1) {dir}/libtbbmalloc.so'] super().__init__(name, **kwargs) -tbbmalloc = TBBMalloc("tbbmalloc", color="xkcd:green") +tbbmalloc = TBBMalloc("tbbmalloc", color="xkcd:green", version="2019_U8") diff --git a/src/allocators/tcmalloc.py b/src/allocators/tcmalloc.py index 916da88..f81f88b 100644 --- a/src/allocators/tcmalloc.py +++ b/src/allocators/tcmalloc.py @@ -17,30 +17,28 @@ """TCMalloc definition for allocbench""" -from src.allocator import Allocator, AllocatorSources - -VERSION = 2.7 - -TCMALLOC_SRC = AllocatorSources("tcmalloc", - ["git clone https://github.com/gperftools/gperftools.git tcmalloc"], - [f"git checkout gperftools-{VERSION}", "./autogen.sh"], - ["git reset --hard"]) +from src.allocator import Allocator +from src.artifact import GitArtifact class TCMalloc(Allocator): """TCMalloc allocator""" + + sources = GitArtifact("tcmalloc", "https://github.com/gperftools/gperftools.git") + def __init__(self, name, **kwargs): - kwargs["sources"] = TCMALLOC_SRC - kwargs["LD_PRELOAD"] = "{dir}/lib/libtcmalloc.so" - kwargs["build_cmds"] = ["cd {srcdir}; ./configure --prefix={dir}", - "cd {srcdir}; make install -j4"] + self.LD_PRELOAD = "{dir}/lib/libtcmalloc.so" + self.prepare_cmds = ["./autogen.sh"] + self.build_cmds = ["cd {srcdir}; ./configure --prefix={dir}", + "cd {srcdir}; make install -j4"] super().__init__(name, **kwargs) -tcmalloc = TCMalloc("TCMalloc", color="xkcd:blue") +tcmalloc = TCMalloc("TCMalloc", color="xkcd:blue", version="gperftools-2.7") tcmalloc_nofs = TCMalloc("TCMalloc-NoFalsesharing", patches=["{patchdir}/tcmalloc_2.7_no_active_falsesharing.patch"], + version="gperftools-2.7", color="xkcd:navy") diff --git a/src/artifact.py b/src/artifact.py new file mode 100644 index 0000000..010bc6b --- /dev/null +++ b/src/artifact.py @@ -0,0 +1,142 @@ +# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de> +# +# This file is part of allocbench. +# +# allocbench is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# allocbench is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with allocbench. If not, see <http://www.gnu.org/licenses/>. + +"""Artifact classes + +An Artifact is a external ressource downloaded from the internet. +There are two flavours of artifacts available: archives and git repositories. +Both flavours are version controlled archive with a checksum and git repositories +with a specific checkout. +""" + +import os +import subprocess + +import src.globalvars +from src.util import print_info, print_debug, sha1sum + +ARTIFACT_STORE_DIR = os.path.join(src.globalvars.allocbenchdir, "cache") + +class Artifact: + """Base class for external ressources""" + store = {} + + def __init__(self, name): + if name in Artifact.store: + raise Exception(f'duplicate artifact "{name}"') + + Artifact.store[name] = self + self.name = name + self.basedir = os.path.join(ARTIFACT_STORE_DIR, name) + + def retrieve(self, cmd): + """Run cmd to retrieve the artifact""" + os.makedirs(self.basedir, exist_ok=True) + + print_info(f'Retrieving artifact "{self.name}"') + print_debug(f"By running: {cmd} in {self.basedir}") + proc = subprocess.run(cmd, cwd=self.basedir, + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + if proc.returncode != 0: + raise Exception(f"Failed to retrieve {self.name}") + + +class GitArtifact(Artifact): + """External git repository""" + def __init__(self, name, url): + super().__init__(name) + self.url = url + self.repo = os.path.join(self.basedir, "repo") + + def retrieve(self): + """clone the git repo""" + super().retrieve(["git", "clone", "--recursive", "--bare", self.url, "repo"]) + + def provide(self, checkout, location=None): + """checkout new worktree at location""" + if not location: + location = os.path.join(self.basedir, checkout) + + # check if we have already provided this checkout + if os.path.exists(location): + return + + # check if we have already retrieved the repo + if not os.path.exists(self.repo): + self.retrieve() + + print_debug("create new worktree. By running: ", + ["git", "worktree", "add", location, checkout], f"in {self.repo}") + proc = subprocess.run(["git", "worktree", "add", location, checkout], cwd=self.repo, + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + if proc.returncode != 0: + raise Exception(f"Failed to provide {self.name}") + + print_debug("update submodules in worktree. By running: ", + ["git", "submodule", "update", "--init", "--recursive"], f"in {self.repo}") + proc = subprocess.run(["git", "submodule", "update", "--init", "--recursive"], cwd=location, + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + +class ArchiveArtifact(Artifact): + """External archive""" + supported_formats = ["tar"] + def __init__(self, name, url, format, checksum): + super().__init__(name) + self.url = url + if format not in self.supported_formats: + raise Exception(f'Archive format "{format}" not in supported list {self.supported_formats}') + self.format = format + self.archive = os.path.join(self.basedir, f"{self.name}.{self.format}") + self.checksum = checksum + + def retrieve(self): + """download the archive""" + super().retrieve(["wget", "-O", self.archive, self.url]) + + def provide(self, location=None): + """extract the archive""" + + # Download archive + if not os.path.exists(self.archive): + self.retrieve() + + # compare checksums + if sha1sum(self.archive) != self.checksum: + raise Exception(f"Archive {self.archive} does not match provided checksum") + + if not location: + location = os.path.join(self.basedir, "content") + + # Check if we already provided the archive at location + if os.path.exists(location): + return + + os.makedirs(location, exist_ok=True) + + # Extract archive + if self.format == "tar": + cmd = ["tar", "Cxf", location, self.archive] + + print_debug(f"extract archive by running: {cmd} in {self.basedir}") + proc = subprocess.run(cmd, cwd=self.basedir, + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + if proc.returncode != 0: + raise Exception(f"Failed to extract {self.name}") diff --git a/src/benchmarks/dj_trace.py b/src/benchmarks/dj_trace.py index f475223..5cb1b99 100644 --- a/src/benchmarks/dj_trace.py +++ b/src/benchmarks/dj_trace.py @@ -25,8 +25,9 @@ from urllib.request import urlretrieve import matplotlib.pyplot as plt import numpy as np +from src.artifact import ArchiveArtifact from src.benchmark import Benchmark -from src.util import print_status, download_reporthook +from src.util import print_status COMMA_SEP_NUMBER_RE = "(?:\\d*(?:,\\d*)?)*" @@ -57,7 +58,7 @@ class BenchmarkDJTrace(Benchmark): def __init__(self): name = "dj_trace" - self.cmd = "trace_run{binary_suffix} dj_workloads/{workload}.wl" + self.cmd = "trace_run{binary_suffix} {build_dir}/dj_workloads/{workload}.wl" self.measure_cmd = "" self.args = {"workload": ["389-ds-2", @@ -106,29 +107,11 @@ class BenchmarkDJTrace(Benchmark): def prepare(self): super().prepare() - workload_dir = "dj_workloads" - workload_archive = f"{workload_dir}.tar.xz" - - if not os.path.isdir(workload_dir): - if not os.path.isfile(workload_archive): - choice = input("Download missing workloads (367M / ~6GB unpacked) [Y/n] ") - if not choice in ['', 'Y', 'y']: - return - - url = f"https://www4.cs.fau.de/~flow/allocbench/{workload_archive}" - urlretrieve(url, workload_archive, download_reporthook) - sys.stderr.write("\n") - - # Extract workloads - proc = subprocess.run(["tar", "xf", workload_archive], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=True) - - # delete archive - if proc.returncode == 0: - os.remove(workload_archive) - - for workload in os.listdir(workload_dir): - self.args["workload"].append(os.path.splitext(workload)[0]) + workloads = ArchiveArtifact("dj_workloads", + "https://www4.cs.fau.de/~flow/allocbench/dj_workloads.tar.xz", + "tar", + "c9bc499eeba8023bca28a755fffbaf9200a335ad") + workloads.provide(self.build_dir) @staticmethod def process_output(result, stdout, stderr, allocator, perm): diff --git a/src/benchmarks/fd.py b/src/benchmarks/fd.py index d78988a..21a987d 100644 --- a/src/benchmarks/fd.py +++ b/src/benchmarks/fd.py @@ -23,8 +23,9 @@ import subprocess import sys from urllib.request import urlretrieve +from src.artifact import ArchiveArtifact, GitArtifact from src.benchmark import Benchmark -from src.util import print_info, download_reporthook +from src.util import print_info class BenchmarkFd(Benchmark): @@ -33,57 +34,36 @@ class BenchmarkFd(Benchmark): def __init__(self): name = "fd" - super().__init__(name) - self.cmd = "fd -HI -e c \"\" {build_dir}/linux" + self.cmd = "fd -HI -e c '.*[0-9].*' {build_dir}/linux" def prepare(self): super().prepare() - fd_tag = "v7.4.0" - fd_release = f"fd-{fd_tag}-x86_64-unknown-linux-gnu" - fd_archive = f"{fd_release}.tar.gz" - fd_url = f"https://github.com/sharkdp/fd/releases/latest/download/{fd_archive}" - fd_dir = os.path.join(self.build_dir, fd_release) - - self.results["facts"]["versions"]["fd"] = fd_tag - - # Create builddir - os.makedirs(self.build_dir, exist_ok=True) - - if not os.path.isdir(fd_dir): - if not os.path.isfile(fd_archive): - print(f"Downloading fd {fd_tag}...") - urlretrieve(fd_url, fd_archive, download_reporthook) - sys.stderr.write("\n") - - - # Extract redis - proc = subprocess.run(["tar", "Cxf", self.build_dir, fd_archive], - # stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) - - # delete archive - if proc.returncode == 0: - os.remove(fd_archive) - - - # create symlinks - for exe in ["fd"]: - src = os.path.join(fd_dir, exe) - dest = os.path.join(self.build_dir, exe) - os.link(src, dest) + if os.path.exists(self.build_dir): + return + + fd_version = "v7.4.0" + self.results["facts"]["versions"]["fd"] = fd_version + fd_url = ("https://github.com/sharkdp/fd/releases/latest/download/" + f"fd-{fd_version}-x86_64-unknown-linux-gnu.tar.gz") + + fd = ArchiveArtifact("fd", fd_url, "tar", "a5d8e7c8484449aa324a46abfdfaf026d7de77ee") + + fd_dir = os.path.join(self.build_dir, "fd_sources") + fd.provide(fd_dir) + + # create symlinks + for exe in ["fd"]: + src = os.path.join(fd_dir, f"fd-{fd_version}-x86_64-unknown-linux-gnu", exe) + dest = os.path.join(self.build_dir, exe) + os.link(src, dest) - linux_version = "v5.3" - linux_url = f"git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" - linux_dir = os.path.join(self.build_dir, "linux") - if not os.path.isdir(linux_dir): - # Extract redis - proc = subprocess.run(["git", "clone", linux_url, linux_dir], - # stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) + linux = GitArtifact("linux", "git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git") + linux_version = "v5.3" + linux.provide(linux_version, os.path.join(self.build_dir, "linux")) def summary(self): self.barplot_single_arg("{task-clock}", diff --git a/src/benchmarks/lld.py b/src/benchmarks/lld.py index 7ebca13..0603037 100644 --- a/src/benchmarks/lld.py +++ b/src/benchmarks/lld.py @@ -24,9 +24,9 @@ import sys import matplotlib.pyplot as plt +from src.artifact import ArchiveArtifact from src.benchmark import Benchmark import src.facter -from src.util import download_reporthook class BenchmarkLld(Benchmark): @@ -38,7 +38,7 @@ class BenchmarkLld(Benchmark): def __init__(self): name = "lld" - self.run_dir = "lld-speed-test/{test}" + self.run_dir = "{build_dir}/lld-speed-test/{test}" # TODO: don't hardcode ld.lld location self.cmd = "/usr/bin/ld.lld @response.txt" @@ -55,28 +55,11 @@ class BenchmarkLld(Benchmark): # save lld version self.results["facts"]["versions"]["lld"] = src.facter.exe_version("ld.lld", "-v") - test_dir = "lld-speed-test" - test_archive = f"{test_dir}.tar.xz" - if not os.path.isdir(test_dir): - if not os.path.isfile(test_archive): - choice = input("Download missing test archive (1.1GB) [Y/n/x] ") - if not choice in ['', 'Y', 'y']: - return - - url = f"https://s3-us-west-2.amazonaws.com/linker-tests/{test_archive}" - urlretrieve(url, test_archive, download_reporthook) - sys.stderr.write("\n") - - # Extract tests - proc = subprocess.run(["tar", "xf", test_archive], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=True) - - # delete archive - if proc.returncode == 0: - os.remove(test_archive) - - self.args["test"] = os.listdir(test_dir) - + tests = ArchiveArtifact("lld-speed-test", + "https://s3-us-west-2.amazonaws.com/linker-tests/lld-speed-test.tar.xz", + "tar", + "2d449a11109c7363f67fd45513b42270f5ba2a92") + tests.provide(self.build_dir) def cleanup(self): for perm in self.iterate_args(): @@ -84,7 +67,6 @@ class BenchmarkLld(Benchmark): if os.path.isfile(a_out): os.remove(a_out) - def summary(self): args = self.results["args"] allocators = self.results["allocators"] diff --git a/src/benchmarks/raxmlng.py b/src/benchmarks/raxmlng.py index 119bab7..811721a 100644 --- a/src/benchmarks/raxmlng.py +++ b/src/benchmarks/raxmlng.py @@ -23,8 +23,9 @@ import subprocess import sys from urllib.request import urlretrieve +from src.artifact import GitArtifact from src.benchmark import Benchmark -from src.util import print_info, download_reporthook +from src.util import print_info RUNTIME_RE = re.compile("Elapsed time: (?P<runtime>(\\d*.\\d*)) seconds") @@ -39,64 +40,51 @@ class BenchmarkRaxmlng(Benchmark): super().__init__(name) - self.cmd = (f"raxml-ng --msa {self.build_dir}/ng-tutorial/prim.phy --model GTR+G" + self.cmd = (f"raxml-ng --msa {self.build_dir}/data/prim.phy --model GTR+G" " --redo --threads 2 --seed 2") def prepare(self): super().prepare() - # git clone --recursive - # cd raxml-ng - # mkdir build && cd build - # cmake .. - # make + if os.path.exists(self.build_dir): + return - version = "0.9" - - url = "https://github.com/amkozlov/raxml-ng" - data_url = "https://github.com/amkozlov/ng-tutorial" + raxmlng_sources = GitArtifact("raxml-ng", "https://github.com/amkozlov/raxml-ng") + raxmlng_version = "0.9.0" raxmlng_dir = os.path.join(self.build_dir, "raxml-ng-git") raxmlng_builddir = os.path.join(raxmlng_dir, "build") - - self.results["facts"]["versions"]["raxml-ng"] = version - - if not os.path.isdir(raxmlng_dir): - proc = subprocess.run(["git", "clone", "--recursive", url, raxmlng_dir], - # stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) + self.results["facts"]["versions"]["raxml-ng"] = raxmlng_version + raxmlng_sources.provide(raxmlng_version, raxmlng_dir) - # Create builddir - os.makedirs(raxmlng_builddir, exist_ok=True) - - # building raxml-ng - proc = subprocess.run(["cmake", ".."], - cwd=raxmlng_builddir, - # stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) - - proc = subprocess.run(["make"], - cwd=raxmlng_builddir, - # stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) - - # create symlinks - for exe in ["raxml-ng"]: - src = os.path.join(raxmlng_dir, "bin", exe) - dest = os.path.join(self.build_dir,exe) - os.link(src, dest) - - # clone test data - proc = subprocess.run(["git", "clone", data_url], - cwd=self.build_dir, - # stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) + # Create builddir + os.makedirs(raxmlng_builddir, exist_ok=True) + + # building raxml-ng + proc = subprocess.run(["cmake", ".."], + cwd=raxmlng_builddir, + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + + proc = subprocess.run(["make"], + cwd=raxmlng_builddir, + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + + # create symlinks + for exe in ["raxml-ng"]: + src = os.path.join(raxmlng_dir, "bin", exe) + dest = os.path.join(self.build_dir,exe) + os.link(src, dest) + + raxmlng_data = GitArtifact("raxml-ng-data", "https://github.com/amkozlov/ng-tutorial") + raxmlng_data_dir = os.path.join(self.build_dir, "data") + raxmlng_data.provide("f8f0b6a057a11397b4dad308440746e3436db8b4", raxmlng_data_dir) def cleanup(self): for direntry in os.listdir(): if direntry.startswith("prim.raxml"): os.remove(direntry) - @staticmethod def process_output(result, stdout, stderr, allocator, perm): result["runtime"] = RUNTIME_RE.search(stdout).group("runtime") diff --git a/src/benchmarks/redis.py b/src/benchmarks/redis.py index d67559a..80bd244 100644 --- a/src/benchmarks/redis.py +++ b/src/benchmarks/redis.py @@ -23,8 +23,9 @@ import subprocess import sys from urllib.request import urlretrieve +from src.artifact import ArchiveArtifact from src.benchmark import Benchmark -from src.util import print_info, download_reporthook +from src.util import print_info REQUESTS_RE = re.compile("(?P<requests>(\\d*.\\d*)) requests per second") @@ -51,40 +52,26 @@ class BenchmarkRedis(Benchmark): super().prepare() redis_version = "5.0.5" - redis_archive = f"redis-{redis_version}.tar.gz" - redis_url = f"http://download.redis.io/releases/{redis_archive}" + self.results["facts"]["versions"]["redis"] = redis_version + redis = ArchiveArtifact("redis", + f"http://download.redis.io/releases/redis-{redis_version}.tar.gz", + "tar", + "71e38ae09ac70012b5bc326522b976bcb8e269d6") + redis_dir = os.path.join(self.build_dir, f"redis-{redis_version}") - self.results["facts"]["versions"]["redis"] = redis_version + redis.provide(redis_dir) + + # building redis + proc = subprocess.run(["make", "-C", redis_dir], + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) - if not os.path.isdir(redis_dir): - if not os.path.isfile(redis_archive): - print(f"Downloading redis-{redis_version}...") - urlretrieve(redis_url, redis_archive, download_reporthook) - sys.stderr.write("\n") - - # Create build_dir - os.mkdir(self.build_dir) - - # Extract redis - proc = subprocess.run(["tar", "Cxf", self.build_dir, redis_archive], - # stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) - - # delete archive - if proc.returncode == 0: - os.remove(redis_archive) - - # building redis - proc = subprocess.run(["make", "-C", redis_dir], - # stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) - - # create symlinks - for exe in ["redis-cli", "redis-server", "redis-benchmark"]: - src = os.path.join(redis_dir, "src", exe) - dest = os.path.join(self.build_dir, exe) - os.link(src, dest) + # create symlinks + for exe in ["redis-cli", "redis-server", "redis-benchmark"]: + src = os.path.join(redis_dir, "src", exe) + dest = os.path.join(self.build_dir, exe) + os.link(src, dest) @staticmethod def process_output(result, stdout, stderr, allocator, perm): diff --git a/src/util.py b/src/util.py index b44a263..c5fa162 100644 --- a/src/util.py +++ b/src/util.py @@ -17,24 +17,13 @@ """Helper functions for allocbench""" +import hashlib import os import subprocess import sys import src.globalvars -def download_reporthook(blocknum, blocksize, totalsize): - """Status report hook for urlretrieve""" - readsofar = blocknum * blocksize - if totalsize > 0: - percent = readsofar * 100 / totalsize - status = "\r%5.1f%% %*d / %d" % ( - percent, len(str(totalsize)), readsofar, totalsize) - sys.stderr.write(status) - else: # total size is unknown - sys.stderr.write(f"\rdownloaded {readsofar}") - - def is_exe(fpath): """Check if the given path is an exexutable file""" return os.path.isfile(fpath) and os.access(fpath, os.X_OK) @@ -173,3 +162,13 @@ def print_version_and_exit(): print(f"{commit}{dirty}") exit(0) + +def sha1sum(filename): + """Return sha1sum of a file""" + sha1 = hashlib.sha1() + barray = bytearray(64*1024) + view = memoryview(barray) + with open(filename, 'rb', buffering=0) as f: + for n in iter(lambda : f.readinto(view), 0): + sha1.update(view[:n]) + return sha1.hexdigest() |