path: root/src/allocator.py

# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
#
# This file is part of allocbench.
#
# allocbench is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# allocbench is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with allocbench.  If not, see <http://www.gnu.org/licenses/>.
"""Alloactor related class definitions and helpers"""

from datetime import datetime
import inspect
import importlib
import os
import shutil
from subprocess import CalledProcessError
import sys

from src.artifact import ArchiveArtifact, GitArtifact
import src.globalvars
from src.util import print_status, print_debug, print_error, print_info2, run_cmd

LIBRARY_PATH = ""
for line in run_cmd(["ldconfig", "-v", "-N"],
                    capture=True).stdout.splitlines():

    if not line.startswith('\t'):
        LIBRARY_PATH += line

BUILDDIR = src.globalvars.allocbuilddir
SRCDIR = os.path.join(src.globalvars.allocbuilddir, "src")

if not os.path.isdir(SRCDIR):
    os.makedirs(SRCDIR)


class Allocator:
    """Allocator base class

    An Allocator can contain an Artifact which provides the allocator sources,
    patches, and instructions to build the allocator.
    Allocator.build will compile the allocator and produce a for allocbench usable
    allocator dict"""
    allowed_attributes = [
        "binary_suffix", "cmd_prefix", "LD_PRELOAD", "LD_LIBRARY_PATH",
        "color", "sources", "version", "patches", "prepare_cmds", "build_cmds"
    ]

    binary_suffix = None
    cmd_prefix = None
    LD_PRELOAD = None
    LD_LIBRARY_PATH = None
    color = None
    sources = None
    version = None
    patches = []
    prepare_cmds = []
    build_cmds = []

    def __init__(self, name, **kwargs):
        self.class_file = inspect.getfile(self.__class__)
        self.name = name
        self.srcdir = os.path.join(SRCDIR, self.name)
        self.dir = os.path.join(BUILDDIR, self.name)
        self.patchdir = os.path.join(os.path.splitext(self.class_file)[0])

        # Update attributes
        for attr, value in kwargs.items():
            setattr(self, attr, value)

    def prepare(self):
        """Prepare the allocators sources"""
        if not self.sources and os.path.exists(self.srcdir):
            return

        print_status("Preparing", self.name, "...")
        if isinstance(self.sources, GitArtifact):
            self.sources.provide(self.version, self.srcdir)
        elif isinstance(self.sources, ArchiveArtifact):
            self.sources.provide(self.srcdir)

        if self.patches:
            cwd = os.path.join(SRCDIR, self.name)

            print_status(f"Patching {self.name} ...")
            for patch in self.patches:
                with open(patch.format(patchdir=self.patchdir),
                          "r") as patch_file:
                    patch_content = patch_file.read()

                # check if patch is already applied
                not_patched = run_cmd(
                    ["patch", "-R", "-p0", "-s", "-f", "--dry-run", "--verbose"],
                    cwd=cwd,
                    input=patch_content,
                    check=False).returncode
                if not_patched:
                    try:
                        run_cmd(["patch", "-p0", "--verbose"], cwd=cwd, input=patch_content)
                    except CalledProcessError as e:
                        print_debug(e.stderr, file=sys.stderr)
                        print_error(f"Patching of {self.name} failed.")
                        raise e

        if self.prepare_cmds:
            print_status(f"Run prepare commands {self.name} ...")
            for cmd in self.prepare_cmds:
                try:
                    run_cmd(cmd, shell=True, cwd=self.srcdir)
                except CalledProcessError as e:
                    print_debug(e.stderr, file=sys.stderr)
                    print_error(f"Prepare {self.name} failed")
                    raise e

    def build(self):
        """Build the allocator if needed and produce an allocator dict"""
        build_needed = not os.path.isdir(self.dir)
        buildtimestamp_path = os.path.join(self.dir, ".buildtime")

        if not build_needed:
            print_info2("Old build found. Comparing build time with mtime")

            with open(buildtimestamp_path, "r") as buildtimestamp_file:
                timestamp = datetime.fromtimestamp(
                    float(buildtimestamp_file.read()))

            modtime = os.stat(self.class_file).st_mtime
            modtime = datetime.fromtimestamp(modtime)

            build_needed = timestamp < modtime

            print_debug("Time of last build:", timestamp.isoformat())
            print_debug("Last modification of allocators file:",
                        modtime.isoformat())
            print_info2("" if build_needed else "No " + "build needed")

        if build_needed:
            self.prepare()
            print_status("Building", self.name, "...")

            if self.build_cmds:
                for cmd in self.build_cmds:
                    cmd = cmd.format(dir=self.dir, srcdir=self.srcdir)

                    try:
                        run_cmd(cmd, cwd=BUILDDIR, shell=True)
                    except CalledProcessError as e:
                        print_debug(e.stderr, file=sys.stderr)
                        print_error(f"Builing {self.name} failed")
                        shutil.rmtree(self.dir, ignore_errors=True)
                        raise e

                with open(buildtimestamp_path, "w") as buildtimestamp_file:
                    print_info2("Save build time to:", buildtimestamp_path)
                    buildtimestamp_file.write(str(datetime.now().timestamp()))

        print_info2("Create allocator dictionary")
        res_dict = {
            "cmd_prefix": self.cmd_prefix or "",
            "binary_suffix": self.binary_suffix or "",
            "LD_PRELOAD": self.LD_PRELOAD or "",
            "LD_LIBRARY_PATH": self.LD_LIBRARY_PATH or "",
            "color": self.color
        }

        for attr in ["LD_PRELOAD", "LD_LIBRARY_PATH", "cmd_prefix"]:
            value = getattr(self, attr, "") or ""
            if value != "":
                value = value.format(dir=self.dir, srcdir=self.srcdir)
                res_dict[attr] = value

        print_debug("Resulting dictionary:", res_dict)
        return res_dict


def read_allocators_collection_file(alloc_path):
    """Read and evaluate a python file looking for an exported dict called allocators"""

    exec_globals = {"__file__": alloc_path}
    with open(alloc_path, "r") as alloc_file:
        exec(compile(alloc_file.read()), exec_globals)

    if "allocators" in exec_globals:
        return exec_globals["allocators"]

    print_error("No global dictionary 'allocators' in", alloc_path)
    return {}


def collect_allocators(allocators):
    """Collect allocators to benchmark

    If allocators is None we use either the allocators exported in the default
    allocators file at build/allocators/allocators.py or the ones installed.

    Otherwise allocators is interpreted as a list of names or files. If an entry in
    allocators is a file it is handled as a allocator collection file exporting
    a allocators variable. If the entry is no file it is interpreted as an allocator
    name and is searched for in our allocator definitions located at src/allocators.
    """

    # Default allocators definition file
    default_allocators_file = "build/allocators/allocators.py"

    if allocators is None and os.path.isfile(default_allocators_file):
        return read_allocators_collection_file(default_allocators_file)

    ret = {}
    for name in allocators:
        if name == "installed":
            print_status("Using system-wide installed allocators ...")
            importlib.import_module('src.allocators.installed_allocators')
            ret.update(src.allocators.installed_allocators.allocators)
        # file exists -> interpret as python file with a global variable allocators
        elif os.path.isfile(name):
            print_status("Sourcing allocators definitions at", name, "...")
            ret.update(read_allocators_collection_file(name))

        # file is one of our allocator definitions import it
        elif os.path.isfile("src/allocators/" + name.split('_')[0] + ".py"):
            module = importlib.import_module('src.allocators.' + name.split('_')[0])
            # name is collection
            if hasattr(module, "allocators"):
                for alloc in module.allocators:
                    ret[alloc.name] = alloc.build()
            # name is single allocator
            elif issubclass(
                    getattr(module, name).__class__, src.allocator.Allocator):
                ret[name] = getattr(module, name).build()
        else:
            print_error(
                name,
                "is neither a python file or a known allocator definition.")
    return ret