implement wildcard search for allocator selection

This makes most allocator collections, which only bundle all allocator
variants from one allocator obsolete.
They can be replaces with 'allocator*' on the command line.

6 files changed, 62 insertions, 185 deletions

diff --git a/src/allocator.py b/src/allocator.py
index 00d62ac..dd8473e 100644
--- a/src/allocator.py
+++ b/src/allocator.py
@@ -17,9 +17,11 @@
 """Alloactor related class definitions and helpers"""
 
 from datetime import datetime
+import fnmatch
 import inspect
 import importlib
 import os
+from pathlib import Path
 import shutil
 from subprocess import CalledProcessError
 import sys
@@ -179,16 +181,64 @@ class Allocator:
         print_debug("Resulting dictionary:", res_dict)
         return res_dict
 
+def collect_installed_allocators():
+    """Collect allocators using installed system libraries"""
+
+    # TODO: add more allocators
+    MAYBE_ALLOCATORS = ["tcmalloc", "jemalloc", "hoard"]
+
+    allocators = {
+        "libc": {
+            "cmd_prefix": "",
+            "binary_suffix": "",
+            "LD_PRELOAD": "",
+            "LD_LIBRARY_PATH": "",
+            "color": "C1"
+        }
+    }
+
+    for i, alloc in enumerate(MAYBE_ALLOCATORS):
+        try:
+            path = run_cmd(f'whereis lib{alloc} | cut -d":" -f2',
+                                  shell=True,
+                                  capture=True).stdout.strip()
+        except CalledProcessError:
+            continue
+
+        if path != "":
+            allocators[alloc] = {
+                "cmd_prefix": "",
+                "binary_suffix": "",
+                "LD_PRELOAD": path,
+                "LD_LIBRARY_PATH": "",
+                "color": None,
+            }
+
+    return allocators
+
+def collect_available_allocators():
+    """Collect all allocator definitions shipped with allocbench"""
+
+    available_allocators = {}
+
+    for alloc_def_path in Path(ALLOCDEFDIR).glob('*.py'):
+            alloc_module_name = '.'.join(alloc_def_path.parts[:-1] + (alloc_def_path.stem,))
+            module = importlib.import_module(alloc_module_name)
+            for name, obj in module.__dict__.items():
+                if issubclass(obj.__class__, src.allocator.Allocator):
+                    available_allocators[name] = obj
+
+    return available_allocators
 
 def read_allocators_collection_file(alloc_path):
     """Read and evaluate a python file looking for an exported dict called allocators"""
 
     exec_globals = {"__file__": alloc_path}
     with open(alloc_path, "r") as alloc_file:
-        exec(compile(alloc_file.read()), exec_globals)
+        exec(compile(alloc_file.read(), alloc_path, 'exec'), exec_globals)
 
     if "allocators" in exec_globals:
-        return exec_globals["allocators"]
+        return {a.name: a.build() for a in exec_globals["allocators"]}
 
     print_error("No global dictionary 'allocators' in", alloc_path)
     return {}
@@ -216,26 +266,20 @@ def collect_allocators(allocators):
     for name in allocators:
         if name == "installed":
             print_status("Using system-wide installed allocators ...")
-            importlib.import_module('src.allocators.installed_allocators')
-            ret.update(src.allocators.installed_allocators.allocators)
+            ret.update(collect_installed_allocators())
         # file exists -> interpret as python file with a global variable allocators
         elif os.path.isfile(name):
             print_status("Sourcing allocators definitions at", name, "...")
             ret.update(read_allocators_collection_file(name))
 
-        # file is one of our allocator definitions import it
-        elif os.path.isfile("src/allocators/" + name.split('_')[0] + ".py"):
-            module = importlib.import_module('src.allocators.' + name.split('_')[0])
-            # name is collection
-            if hasattr(module, "allocators"):
-                for alloc in module.allocators:
-                    ret[alloc.name] = alloc.build()
-            # name is single allocator
-            elif issubclass(
-                    getattr(module, name).__class__, src.allocator.Allocator):
-                ret[name] = getattr(module, name).build()
+        # interpret name as allocator name or wildcard
         else:
-            print_error(
-                name,
-                "is neither a python file or a known allocator definition.")
+            available_allocators = collect_available_allocators()
+            matched_allocators = fnmatch.filter(available_allocators.keys(), name)
+            if matched_allocators:
+                ret.update({a: available_allocators[a].build() for a in matched_allocators})
+            else:
+                print_error(
+                    name,
+                    "is neither a python file or a known allocator definition.")
     return ret
diff --git a/src/allocators/all.py b/src/allocators/all.py
deleted file mode 100644
index ed8d01e..0000000
--- a/src/allocators/all.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
-#
-# This file is part of allocbench.
-#
-# allocbench is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# allocbench is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with allocbench.  If not, see <http://www.gnu.org/licenses/>.
-"""Collection containing  all available allocators"""
-
-import src.allocators.glibcs
-import src.allocators.tcmallocs
-from src.allocators.jemalloc import jemalloc
-from src.allocators.hoard import hoard
-from src.allocators.mesh import mesh
-from src.allocators.scalloc import scalloc
-from src.allocators.supermalloc import supermalloc
-from src.allocators.llalloc import llalloc
-from src.allocators.tbbmalloc import tbbmalloc
-from src.allocators.mimalloc import mimalloc
-from src.allocators.snmalloc import snmalloc
-from src.allocators.rpmalloc import rpmalloc
-
-allocators = [
-    *src.allocators.glibcs.allocators,
-    *src.allocators.tcmallocs.allocators,
-    jemalloc,
-    hoard,
-    mesh,
-    supermalloc,
-    scalloc,
-    tbbmalloc,
-    llalloc,  # streamflow,
-    mimalloc,
-    snmalloc,
-    rpmalloc
-]
diff --git a/src/allocators/glibcs.py b/src/allocators/glibcs.py
deleted file mode 100644
index 8c26493..0000000
--- a/src/allocators/glibcs.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
-#
-# This file is part of allocbench.
-#
-# allocbench is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# allocbench is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with allocbench.  If not, see <http://www.gnu.org/licenses/>.
-"""Collection containing all glibc variants"""
-
-from src.allocators.glibc import glibc, glibc_notc, glibc_nofs, glibc_nofs_fancy
-
-allocators = [glibc, glibc_notc, glibc_nofs, glibc_nofs_fancy]
diff --git a/src/allocators/installed_allocators.py b/src/allocators/installed_allocators.py
deleted file mode 100644
index 5e3c78d..0000000
--- a/src/allocators/installed_allocators.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
-#
-# This file is part of allocbench.
-#
-# allocbench is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# allocbench is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with allocbench.
-"""Collect allocators using installed system libraries"""
-
-import subprocess
-
-# TODO: add more allocators
-MAYBE_ALLOCATORS = ["tcmalloc", "jemalloc", "hoard"]
-
-allocators = {
-    "libc": {
-        "cmd_prefix": "",
-        "binary_suffix": "",
-        "LD_PRELOAD": "",
-        "LD_LIBRARY_PATH": "",
-        "color": "C1"
-    }
-}
-
-for i, t in enumerate(MAYBE_ALLOCATORS):
-    try:
-        path = subprocess.run('whereis lib{} | cut -d":" -f2'.format(t),
-                              shell=True,
-                              stdout=subprocess.PIPE,
-                              universal_newlines=True).stdout.strip()
-    except:
-        continue
-
-    if path != "":
-        allocators[t] = {
-            "cmd_prefix": "",
-            "binary_suffix": "",
-            "LD_PRELOAD": path,
-            "LD_LIBRARY_PATH": "",
-            "color": "C" + str(i + 2)
-        }
diff --git a/src/allocators/speedymallocs.py b/src/allocators/speedymallocs.py
deleted file mode 100644
index c3012dd..0000000
--- a/src/allocators/speedymallocs.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
-#
-# This file is part of allocbench.
-#
-# allocbench is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# allocbench is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with allocbench.  If not, see <http://www.gnu.org/licenses/>.
-"""Collection containing all glibc variants"""
-
-import src.allocators.speedymalloc as sm
-
-allocators = [
-    sm.speedymalloc, sm.speedymalloc_no_madv_free,
-    sm.speedymalloc_no_madv_willneed, sm.speedymalloc_4095_sc_32,
-    sm.speedymalloc_4095_sc_128,
-    sm.speedymalloc_no_glab, sm.speedymalloc_70d9d160
-]
diff --git a/src/allocators/tcmallocs.py b/src/allocators/tcmallocs.py
deleted file mode 100644
index 0fe5d5b..0000000
--- a/src/allocators/tcmallocs.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
-#
-# This file is part of allocbench.
-#
-# allocbench is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# allocbench is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with allocbench.  If not, see <http://www.gnu.org/licenses/>.
-"""Collection containing all glibc variants"""
-
-import src.allocators.tcmalloc as tcm
-
-allocators = [
-    tcm.tcmalloc, tcm.tcmalloc_align, tcm.tcmalloc_gperftools,
-    tcm.tcmalloc_gperftools_nofs, tcm.tcmalloc_gperftools_align,
-    tcm.tcmalloc_gperftools_cacheline_exclusive
-]