From 84eec0baba49bff0a5c2b64834bc2d75d687e7b5 Mon Sep 17 00:00:00 2001 From: Florian Fischer Date: Sat, 21 Sep 2019 14:04:26 +0200 Subject: add artifacts Artifacts are downloadable external ressources. They provide their ressource (vcs commit, extracted archive) using Artifact.provide. GitArtifact download a bare git repository and provides specific commits using worktrees. Use GitArtifact for the linux kernel source --- src/artifact.py | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 src/artifact.py (limited to 'src/artifact.py') diff --git a/src/artifact.py b/src/artifact.py new file mode 100644 index 0000000..d880bb7 --- /dev/null +++ b/src/artifact.py @@ -0,0 +1,89 @@ +# Copyright 2018-2019 Florian Fischer +# +# This file is part of allocbench. +# +# allocbench is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# allocbench is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with allocbench. If not, see . + +"""Artifact classes + +An Artifact is a external ressource downloaded from the internet. +There are two flavours of artifacts available: archives and git repositories. +Both flavours are version controlled archive with a checksum and git repositories +with a specific checkout. +""" + +import os +import subprocess + +import src.globalvars +from src.util import print_info, print_debug + +ARTIFACT_STORE_DIR = os.path.join(src.globalvars.allocbenchdir, "cache") + +class Artifact: + """Base class for external ressources""" + store = {} + + def __init__(self, name): + if name in Artifact.store: + raise Exception(f'duplicate artifact "{name}"') + + Artifact.store[name] = self + self.name = name + self.basedir = os.path.join(ARTIFACT_STORE_DIR, name) + + def retrieve(self, cmd): + """Run cmd to retrieve the artifact""" + os.makedirs(self.basedir, exist_ok=True) + + print_info(f'Retrieving artifact "{self.name}"') + print_debug(f"By running: {cmd} in {self.basedir}") + proc = subprocess.run(cmd, cwd=self.basedir, + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + if proc.returncode != 0: + raise Exception(f"Failed to retrieve {self.name}") + + +class GitArtifact(Artifact): + """External git repository""" + def __init__(self, name, url): + super().__init__(name) + self.url = url + self.repo = os.path.join(self.basedir, "repo") + + def retrieve(self): + """clone the git repo""" + super().retrieve(["git", "clone", "--recursive", "--bare", self.url, "repo"]) + + def provide(self, checkout, location=None): + """checkout new worktree at location""" + if not location: + location = os.path.join(self.basedir, checkout) + + # check if we have already provided this checkout + if os.path.exists(location): + return + + # check if we have already retrieved the repo + if not os.path.exists(self.repo): + self.retrieve() + + print_debug("create new worktree. By running: ", + ["git", "worktree", "add", location, checkout], f"in {self.repo}") + proc = subprocess.run(["git", "worktree", "add", location, checkout], cwd=self.repo, + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + if proc.returncode != 0: + raise Exception(f"Failed to provide {self.name}") -- cgit v1.2.3 From cac85412d757b6054436dc9c63d30db645bc93ea Mon Sep 17 00:00:00 2001 From: Florian Fischer Date: Sun, 22 Sep 2019 18:35:47 +0200 Subject: Add ArchiveArtifacts ArchiveArtifacts check a downloaded archive against a provided checksum. The Archive is downloaded to cache//.. The only suported format is tar. ArchiveArtifacts can be used as sources of an Allocator. --- src/artifact.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) (limited to 'src/artifact.py') diff --git a/src/artifact.py b/src/artifact.py index d880bb7..49530a3 100644 --- a/src/artifact.py +++ b/src/artifact.py @@ -27,7 +27,7 @@ import os import subprocess import src.globalvars -from src.util import print_info, print_debug +from src.util import print_info, print_debug, sha1sum ARTIFACT_STORE_DIR = os.path.join(src.globalvars.allocbenchdir, "cache") @@ -87,3 +87,52 @@ class GitArtifact(Artifact): universal_newlines=True) if proc.returncode != 0: raise Exception(f"Failed to provide {self.name}") + + print_debug("update submodules in worktree. By running: ", + ["git", "submodule", "update", "--init", "--recursive"], f"in {self.repo}") + proc = subprocess.run(["git", "submodule", "update", "--init", "--recursive"], cwd=location, + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + +class ArchiveArtifact(Artifact): + """External archive""" + supported_formats = ["tar"] + def __init__(self, name, url, format, checksum): + super().__init__(name) + self.url = url + if format not in self.supported_formats: + raise Exception(f'Archive format "{format}" not in supported list {self.supported_formats}') + self.format = format + self.archive = os.path.join(self.basedir, f"{self.name}.{self.format}") + self.checksum = checksum + + def retrieve(self): + """download the archive""" + super().retrieve(["wget", "-O", self.archive, self.url]) + + def provide(self, location=None): + """extract the archive""" + + # Download archive + if not os.path.exists(self.archive): + self.retrieve() + + # compare checksums + if sha1sum(self.archive) != self.checksum: + raise Exception(f"Archive {self.archive} does not match provided checksum") + + if not location: + location = os.path.join(self.basedir, "content") + + os.makedirs(location, exist_ok=True) + + # Extract archive + if self.format == "tar": + cmd = ["tar", "Cxf", location, self.archive] + + print_debug(f"extract archive by running: {cmd} in {self.basedir}") + proc = subprocess.run(cmd, cwd=self.basedir, + # stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + if proc.returncode != 0: + raise Exception(f"Failed to extract {self.name}") -- cgit v1.2.3 From 6e019b83c2f82552704feb4d8ce18e0d3a19d75a Mon Sep 17 00:00:00 2001 From: Florian Fischer Date: Mon, 23 Sep 2019 13:42:05 +0200 Subject: use ArchiveArtifacts for all benchmark resources --- src/artifact.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/artifact.py') diff --git a/src/artifact.py b/src/artifact.py index 49530a3..010bc6b 100644 --- a/src/artifact.py +++ b/src/artifact.py @@ -124,6 +124,10 @@ class ArchiveArtifact(Artifact): if not location: location = os.path.join(self.basedir, "content") + # Check if we already provided the archive at location + if os.path.exists(location): + return + os.makedirs(location, exist_ok=True) # Extract archive -- cgit v1.2.3