diff options
| -rw-r--r-- | src/allocator.py | 6 | ||||
| -rw-r--r-- | src/allocators/all.py | 4 | ||||
| -rw-r--r-- | src/allocators/llalloc.py | 21 | ||||
| -rw-r--r-- | src/artifact.py | 51 | ||||
| -rw-r--r-- | src/util.py | 11 |
5 files changed, 78 insertions, 15 deletions
diff --git a/src/allocator.py b/src/allocator.py index 23b5147..b88b046 100644 --- a/src/allocator.py +++ b/src/allocator.py @@ -25,6 +25,7 @@ import shutil import subprocess import sys +from src.artifact import ArchiveArtifact, GitArtifact import src.globalvars from src.util import print_status, print_debug, print_error, print_info2 @@ -76,7 +77,10 @@ class Allocator: return print_status("Preparing", self.name, "...") - self.sources.provide(self.version, self.srcdir) + if type(self.sources) == GitArtifact: + self.sources.provide(self.version, self.srcdir) + elif type(self.sources) == ArchiveArtifact: + self.sources.provide(self.srcdir) if self.patches: stdout = subprocess.PIPE if src.globalvars.verbosity < 2 else None diff --git a/src/allocators/all.py b/src/allocators/all.py index a2b502a..0593c81 100644 --- a/src/allocators/all.py +++ b/src/allocators/all.py @@ -24,12 +24,12 @@ from src.allocators.hoard import hoard from src.allocators.mesh import mesh from src.allocators.scalloc import scalloc from src.allocators.supermalloc import supermalloc -# from src.allocators.llalloc import llalloc +from src.allocators.llalloc import llalloc from src.allocators.tbbmalloc import tbbmalloc from src.allocators.mimalloc import mimalloc from src.allocators.snmalloc import snmalloc allocators = [*src.allocators.glibcs.allocators, tcmalloc, tcmalloc_nofs, - jemalloc, hoard, mesh, supermalloc, scalloc, tbbmalloc, # llalloc, # streamflow, + jemalloc, hoard, mesh, supermalloc, scalloc, tbbmalloc, llalloc, # streamflow, mimalloc, snmalloc] diff --git a/src/allocators/llalloc.py b/src/allocators/llalloc.py index b0da505..8f43f59 100644 --- a/src/allocators/llalloc.py +++ b/src/allocators/llalloc.py @@ -17,25 +17,24 @@ """Lockless allocator definition for allocbench""" -from src.allocator import Allocator, AllocatorSources - - -LLALLOC_SOURCE = AllocatorSources("lockless_allocator", - retrieve_cmds=["wget https://locklessinc.com/downloads/lockless_allocator_src.tgz", - "tar xf lockless_allocator_src.tgz"], - prepare_cmds=[], - reset_cmds=[]) +from src.allocator import Allocator +from src.artifact import ArchiveArtifact class LocklessAllocator(Allocator): """Lockless allocator""" def __init__(self, name, **kwargs): - kwargs["sources"] = LLALLOC_SOURCE + self.sources = ArchiveArtifact("llalloc", + "https://locklessinc.com/downloads/lockless_allocator_src.tgz", + "tar", + "c6cb5a57882fa4775b5227a322333a6126b61f7c") - kwargs["build_cmds"] = ["cd {srcdir}; make", "mkdir -p {dir}"] + self.build_cmds = ["cd {srcdir}/lockless_allocator; make", + "mkdir -p {dir}", + "ln -f -s {srcdir}/lockless_allocator/libllalloc.so.1.3 {dir}/libllalloc.so"] - kwargs["LD_PRELOAD"] = "{srcdir}/libllalloc.so.1.3" + self.LD_PRELOAD = "{dir}/libllalloc.so" super().__init__(name, **kwargs) diff --git a/src/artifact.py b/src/artifact.py index d880bb7..49530a3 100644 --- a/src/artifact.py +++ b/src/artifact.py @@ -27,7 +27,7 @@ import os import subprocess import src.globalvars -from src.util import print_info, print_debug +from src.util import print_info, print_debug, sha1sum ARTIFACT_STORE_DIR = os.path.join(src.globalvars.allocbenchdir, "cache") @@ -87,3 +87,52 @@ class GitArtifact(Artifact): universal_newlines=True) if proc.returncode != 0: raise Exception(f"Failed to provide {self.name}") + + print_debug("update submodules in worktree. By running: ", + ["git", "submodule", "update", "--init", "--recursive"], f"in {self.repo}") + proc = subprocess.run(["git", "submodule", "update", "--init", "--recursive"], cwd=location, + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + +class ArchiveArtifact(Artifact): + """External archive""" + supported_formats = ["tar"] + def __init__(self, name, url, format, checksum): + super().__init__(name) + self.url = url + if format not in self.supported_formats: + raise Exception(f'Archive format "{format}" not in supported list {self.supported_formats}') + self.format = format + self.archive = os.path.join(self.basedir, f"{self.name}.{self.format}") + self.checksum = checksum + + def retrieve(self): + """download the archive""" + super().retrieve(["wget", "-O", self.archive, self.url]) + + def provide(self, location=None): + """extract the archive""" + + # Download archive + if not os.path.exists(self.archive): + self.retrieve() + + # compare checksums + if sha1sum(self.archive) != self.checksum: + raise Exception(f"Archive {self.archive} does not match provided checksum") + + if not location: + location = os.path.join(self.basedir, "content") + + os.makedirs(location, exist_ok=True) + + # Extract archive + if self.format == "tar": + cmd = ["tar", "Cxf", location, self.archive] + + print_debug(f"extract archive by running: {cmd} in {self.basedir}") + proc = subprocess.run(cmd, cwd=self.basedir, + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + if proc.returncode != 0: + raise Exception(f"Failed to extract {self.name}") diff --git a/src/util.py b/src/util.py index 027cbcb..8b9e585 100644 --- a/src/util.py +++ b/src/util.py @@ -17,6 +17,7 @@ """Helper functions for allocbench""" +import hashlib import os import subprocess import sys @@ -173,3 +174,13 @@ def print_version_and_exit(): print(f"{commit}{dirty}") exit(0) + +def sha1sum(filename): + """Return sha1sum of a file""" + sha1 = hashlib.sha1() + barray = bytearray(64*1024) + view = memoryview(barray) + with open(filename, 'rb', buffering=0) as f: + for n in iter(lambda : f.readinto(view), 0): + sha1.update(view[:n]) + return sha1.hexdigest() |