diff options
| -rw-r--r-- | src/allocator.py | 80 | ||||
| -rw-r--r-- | src/allocators/all.py | 45 | ||||
| -rw-r--r-- | src/allocators/glibcs.py | 21 | ||||
| -rw-r--r-- | src/allocators/installed_allocators.py | 50 | ||||
| -rw-r--r-- | src/allocators/speedymallocs.py | 26 | ||||
| -rw-r--r-- | src/allocators/tcmallocs.py | 25 |
6 files changed, 62 insertions, 185 deletions
diff --git a/src/allocator.py b/src/allocator.py index 00d62ac..dd8473e 100644 --- a/src/allocator.py +++ b/src/allocator.py @@ -17,9 +17,11 @@ """Alloactor related class definitions and helpers""" from datetime import datetime +import fnmatch import inspect import importlib import os +from pathlib import Path import shutil from subprocess import CalledProcessError import sys @@ -179,16 +181,64 @@ class Allocator: print_debug("Resulting dictionary:", res_dict) return res_dict +def collect_installed_allocators(): + """Collect allocators using installed system libraries""" + + # TODO: add more allocators + MAYBE_ALLOCATORS = ["tcmalloc", "jemalloc", "hoard"] + + allocators = { + "libc": { + "cmd_prefix": "", + "binary_suffix": "", + "LD_PRELOAD": "", + "LD_LIBRARY_PATH": "", + "color": "C1" + } + } + + for i, alloc in enumerate(MAYBE_ALLOCATORS): + try: + path = run_cmd(f'whereis lib{alloc} | cut -d":" -f2', + shell=True, + capture=True).stdout.strip() + except CalledProcessError: + continue + + if path != "": + allocators[alloc] = { + "cmd_prefix": "", + "binary_suffix": "", + "LD_PRELOAD": path, + "LD_LIBRARY_PATH": "", + "color": None, + } + + return allocators + +def collect_available_allocators(): + """Collect all allocator definitions shipped with allocbench""" + + available_allocators = {} + + for alloc_def_path in Path(ALLOCDEFDIR).glob('*.py'): + alloc_module_name = '.'.join(alloc_def_path.parts[:-1] + (alloc_def_path.stem,)) + module = importlib.import_module(alloc_module_name) + for name, obj in module.__dict__.items(): + if issubclass(obj.__class__, src.allocator.Allocator): + available_allocators[name] = obj + + return available_allocators def read_allocators_collection_file(alloc_path): """Read and evaluate a python file looking for an exported dict called allocators""" exec_globals = {"__file__": alloc_path} with open(alloc_path, "r") as alloc_file: - exec(compile(alloc_file.read()), exec_globals) + exec(compile(alloc_file.read(), alloc_path, 'exec'), exec_globals) if "allocators" in exec_globals: - return exec_globals["allocators"] + return {a.name: a.build() for a in exec_globals["allocators"]} print_error("No global dictionary 'allocators' in", alloc_path) return {} @@ -216,26 +266,20 @@ def collect_allocators(allocators): for name in allocators: if name == "installed": print_status("Using system-wide installed allocators ...") - importlib.import_module('src.allocators.installed_allocators') - ret.update(src.allocators.installed_allocators.allocators) + ret.update(collect_installed_allocators()) # file exists -> interpret as python file with a global variable allocators elif os.path.isfile(name): print_status("Sourcing allocators definitions at", name, "...") ret.update(read_allocators_collection_file(name)) - # file is one of our allocator definitions import it - elif os.path.isfile("src/allocators/" + name.split('_')[0] + ".py"): - module = importlib.import_module('src.allocators.' + name.split('_')[0]) - # name is collection - if hasattr(module, "allocators"): - for alloc in module.allocators: - ret[alloc.name] = alloc.build() - # name is single allocator - elif issubclass( - getattr(module, name).__class__, src.allocator.Allocator): - ret[name] = getattr(module, name).build() + # interpret name as allocator name or wildcard else: - print_error( - name, - "is neither a python file or a known allocator definition.") + available_allocators = collect_available_allocators() + matched_allocators = fnmatch.filter(available_allocators.keys(), name) + if matched_allocators: + ret.update({a: available_allocators[a].build() for a in matched_allocators}) + else: + print_error( + name, + "is neither a python file or a known allocator definition.") return ret diff --git a/src/allocators/all.py b/src/allocators/all.py deleted file mode 100644 index ed8d01e..0000000 --- a/src/allocators/all.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de> -# -# This file is part of allocbench. -# -# allocbench is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# allocbench is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with allocbench. If not, see <http://www.gnu.org/licenses/>. -"""Collection containing all available allocators""" - -import src.allocators.glibcs -import src.allocators.tcmallocs -from src.allocators.jemalloc import jemalloc -from src.allocators.hoard import hoard -from src.allocators.mesh import mesh -from src.allocators.scalloc import scalloc -from src.allocators.supermalloc import supermalloc -from src.allocators.llalloc import llalloc -from src.allocators.tbbmalloc import tbbmalloc -from src.allocators.mimalloc import mimalloc -from src.allocators.snmalloc import snmalloc -from src.allocators.rpmalloc import rpmalloc - -allocators = [ - *src.allocators.glibcs.allocators, - *src.allocators.tcmallocs.allocators, - jemalloc, - hoard, - mesh, - supermalloc, - scalloc, - tbbmalloc, - llalloc, # streamflow, - mimalloc, - snmalloc, - rpmalloc -] diff --git a/src/allocators/glibcs.py b/src/allocators/glibcs.py deleted file mode 100644 index 8c26493..0000000 --- a/src/allocators/glibcs.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de> -# -# This file is part of allocbench. -# -# allocbench is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# allocbench is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with allocbench. If not, see <http://www.gnu.org/licenses/>. -"""Collection containing all glibc variants""" - -from src.allocators.glibc import glibc, glibc_notc, glibc_nofs, glibc_nofs_fancy - -allocators = [glibc, glibc_notc, glibc_nofs, glibc_nofs_fancy] diff --git a/src/allocators/installed_allocators.py b/src/allocators/installed_allocators.py deleted file mode 100644 index 5e3c78d..0000000 --- a/src/allocators/installed_allocators.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de> -# -# This file is part of allocbench. -# -# allocbench is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# allocbench is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with allocbench. -"""Collect allocators using installed system libraries""" - -import subprocess - -# TODO: add more allocators -MAYBE_ALLOCATORS = ["tcmalloc", "jemalloc", "hoard"] - -allocators = { - "libc": { - "cmd_prefix": "", - "binary_suffix": "", - "LD_PRELOAD": "", - "LD_LIBRARY_PATH": "", - "color": "C1" - } -} - -for i, t in enumerate(MAYBE_ALLOCATORS): - try: - path = subprocess.run('whereis lib{} | cut -d":" -f2'.format(t), - shell=True, - stdout=subprocess.PIPE, - universal_newlines=True).stdout.strip() - except: - continue - - if path != "": - allocators[t] = { - "cmd_prefix": "", - "binary_suffix": "", - "LD_PRELOAD": path, - "LD_LIBRARY_PATH": "", - "color": "C" + str(i + 2) - } diff --git a/src/allocators/speedymallocs.py b/src/allocators/speedymallocs.py deleted file mode 100644 index c3012dd..0000000 --- a/src/allocators/speedymallocs.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de> -# -# This file is part of allocbench. -# -# allocbench is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# allocbench is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with allocbench. If not, see <http://www.gnu.org/licenses/>. -"""Collection containing all glibc variants""" - -import src.allocators.speedymalloc as sm - -allocators = [ - sm.speedymalloc, sm.speedymalloc_no_madv_free, - sm.speedymalloc_no_madv_willneed, sm.speedymalloc_4095_sc_32, - sm.speedymalloc_4095_sc_128, - sm.speedymalloc_no_glab, sm.speedymalloc_70d9d160 -] diff --git a/src/allocators/tcmallocs.py b/src/allocators/tcmallocs.py deleted file mode 100644 index 0fe5d5b..0000000 --- a/src/allocators/tcmallocs.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de> -# -# This file is part of allocbench. -# -# allocbench is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# allocbench is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with allocbench. If not, see <http://www.gnu.org/licenses/>. -"""Collection containing all glibc variants""" - -import src.allocators.tcmalloc as tcm - -allocators = [ - tcm.tcmalloc, tcm.tcmalloc_align, tcm.tcmalloc_gperftools, - tcm.tcmalloc_gperftools_nofs, tcm.tcmalloc_gperftools_align, - tcm.tcmalloc_gperftools_cacheline_exclusive -] |