reduce code duplication by giving each known allocator its own class

also move allocator related code to src/allocators

15 files changed, 327 insertions, 0 deletions

diff --git a/src/allocators/BA_allocators.py b/src/allocators/BA_allocators.py
new file mode 100644
index 0000000..5f4577e
--- /dev/null
+++ b/src/allocators/BA_allocators.py
@@ -0,0 +1,20 @@
+from src.allocators.glibc import Glibc
+from src.allocators.tcmalloc import TCMalloc
+from src.allocators.jemalloc import Jemalloc
+from src.allocators.hoard import Hoard
+
+glibc = Glibc("glibc", color="C1")
+
+glibc_notc = Glibc("glibc-notc",
+                   configure_args="--disable-experimental-malloc",
+                   color="C2")
+
+tcmalloc = TCMalloc("tcmalloc", color="C3")
+
+jemalloc = Jemalloc("jemalloc", color="C4")
+
+hoard = Hoard("Hoard", color="C5", patches=["allocators/hoard_make.patch"])
+
+allocators_to_build = [glibc, glibc_notc, tcmalloc, jemalloc, hoard]
+
+allocators = {a.name: a.build() for a in allocators_to_build}
diff --git a/src/allocators/Hoard/hoard_make.patch b/src/allocators/Hoard/hoard_make.patch
new file mode 100644
index 0000000..4848c57
--- /dev/null
+++ b/src/allocators/Hoard/hoard_make.patch
@@ -0,0 +1,16 @@
+diff --git a/src/GNUmakefile b/src/GNUmakefile
+index 2b2958e..7fa433a 100644
+--- a/src/GNUmakefile
++++ b/src/GNUmakefile
+@@ -1,9 +1,9 @@
+ # Commands to compile Hoard for various targets.
+ # Run make (with no arguments) to see the complete target list.
+ 
+-CPPFLAGS = -std=c++14 -O3 -DNDEBUG -ffast-math -fno-builtin-malloc -Wall -Wextra -Wshadow -Wconversion -Wuninitialized
++CPPFLAGS = -std=c++14 -O2 -DNDEBUG -ffast-math -fno-builtin-malloc -Wall -Wextra -Wshadow -Wconversion -Wuninitialized
+ #CPPFLAGS = -std=c++14 -g -O0 -ffast-math -fno-builtin-malloc -Wall -Wextra -Wshadow -Wconversion -Wuninitialized
+-CXX = clang++
++CXX = g++
+ 
+ # Compute platform (OS and architecture) and build accordingly.
+ 
diff --git a/src/allocators/__init__.py b/src/allocators/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/allocators/__init__.py
diff --git a/src/allocators/all.py b/src/allocators/all.py
new file mode 100644
index 0000000..3973c37
--- /dev/null
+++ b/src/allocators/all.py
@@ -0,0 +1,44 @@
+from src.allocator import Allocator as Alloc
+from src.allocator import Allocator_Sources as Alloc_Src
+
+from src.allocators.glibc import Glibc
+from src.allocators.tcmalloc import TCMalloc
+from src.allocators.jemalloc import Jemalloc
+from src.allocators.hoard import Hoard
+
+glibc = Glibc("glibc", color="C1")
+
+glibc_notc = Glibc("glibc-notc",
+                configure_args="--disable-experimental-malloc",
+                color="C2")
+
+glibc_nofs = Glibc("glibc_nofs",
+                   patches=["allocators/glibc_2.28_no_passive_falsesharing.patch"],
+                   color="C3")
+
+glibc_nofs_fancy = Glibc("glibc_nofs_fancy",
+                         patches=["allocators/glibc_2.28_no_passive_falsesharing_fancy.patch"],
+                         color="C4")
+
+tcmalloc = TCMalloc("tcmalloc", color="C5")
+
+tcmalloc_nofs = TCMalloc("tcmalloc_nofs",
+                         patches= ["{patchdir}/tcmalloc_2.7_no_active_falsesharing.patch"],
+                         color="C5")
+
+jemalloc = Jemalloc("jemalloc", color="C6")
+
+hoard = Hoard("Hoard", color="C7", patches=["allocators/hoard_make.patch"])
+
+mesh = Alloc("Mesh", sources=Alloc_Src("Mesh",
+                                        retrieve_cmds=["git clone https://github.com/plasma-umass/Mesh"],
+                                        reset_cmds=["git stash"]),
+                            LD_PRELOAD="{srcdir}/libmesh.so",
+                            build_cmds=["cd {srcdir}; git submodule update --init",
+                                        "cd {srcdir}; ./configure",
+                                        "cd {srcdir}; make -j 4",
+                                        "mkdir {dir}"])
+
+allocators_to_build = [glibc, glibc_notc, glibc_nofs, glibc_nofs_fancy, tcmalloc, tcmalloc_nofs, jemalloc, hoard, mesh]
+
+allocators = {a.name: a.build() for a in allocators_to_build}
diff --git a/src/allocators/glibc.py b/src/allocators/glibc.py
new file mode 100644
index 0000000..eff71dc
--- /dev/null
+++ b/src/allocators/glibc.py
@@ -0,0 +1,29 @@
+import src.allocator
+
+
+version = 2.29
+
+glibc_src = src.allocator.Allocator_Sources("glibc",
+                      retrieve_cmds=["git clone git://sourceware.org/git/glibc.git"],
+                      prepare_cmds=["git checkout release/{}/master".format(version)],
+                      reset_cmds=["git stash"])
+
+class Glibc (src.allocator.Allocator):
+    """Glibc definition for allocbench"""
+    def __init__(self, name, **kwargs):
+
+        kwargs["sources"] = glibc_src
+
+        configure_args = ""
+        if "configure_args" in kwargs:
+            configure_args = kwargs["configure_args"]
+            del(kwargs["configure_args"])
+
+        kwargs["build_cmds"] = ["mkdir -p glibc-build",
+                                "cd glibc-build; {srcdir}/configure --prefix={dir} " + configure_args,
+                                "cd glibc-build; make",
+                                "cd glibc-build; make install"]
+        kwargs["cmd_prefix"] = ("{dir}/lib/ld-linux-x86-64.so.2 --library-path {dir}/lib:"
+                              + src.allocator.library_path)
+
+        super().__init__(name, **kwargs)
diff --git a/src/allocators/glibc/glibc_2.28_no_passive_falsesharing.patch b/src/allocators/glibc/glibc_2.28_no_passive_falsesharing.patch
new file mode 100644
index 0000000..fcc695c
--- /dev/null
+++ b/src/allocators/glibc/glibc_2.28_no_passive_falsesharing.patch
@@ -0,0 +1,22 @@
+diff --git a/malloc/malloc.c b/malloc/malloc.c
+index 27cf6137c2..3aadaddd1d 100644
+--- a/malloc/malloc.c
++++ b/malloc/malloc.c
+@@ -4172,6 +4172,9 @@ _int_free (mstate av, mchunkptr p, int have_lock)
+ 
+ #if USE_TCACHE
+   {
++    /* Check if chunk is from our own arena.  */
++    if (av == thread_arena)
++      {
+     size_t tc_idx = csize2tidx (size);
+     if (tcache != NULL && tc_idx < mp_.tcache_bins)
+       {
+@@ -4201,6 +4204,7 @@ _int_free (mstate av, mchunkptr p, int have_lock)
+ 	    return;
+ 	  }
+       }
++      }
+   }
+ #endif
+ 
diff --git a/src/allocators/glibc/glibc_2.28_no_passive_falsesharing_fancy.patch b/src/allocators/glibc/glibc_2.28_no_passive_falsesharing_fancy.patch
new file mode 100644
index 0000000..044909b
--- /dev/null
+++ b/src/allocators/glibc/glibc_2.28_no_passive_falsesharing_fancy.patch
@@ -0,0 +1,25 @@
+diff --git a/malloc/malloc.c b/malloc/malloc.c
+index 27cf6137c2..fbd311801d 100644
+--- a/malloc/malloc.c
++++ b/malloc/malloc.c
+@@ -4172,6 +4172,12 @@ _int_free (mstate av, mchunkptr p, int have_lock)
+ 
+ #if USE_TCACHE
+   {
++    /* Check if chunk is from our own arena or false sharing is not possible
++       because the chunk is cache line aligned and it's size is a multiple
++       of a cacheline  */
++    if (av == thread_arena
++        || (((size_t)p & 63) == 0 && ((size + 2*SIZE_SZ) % 64) == 0))
++      {
+     size_t tc_idx = csize2tidx (size);
+     if (tcache != NULL && tc_idx < mp_.tcache_bins)
+       {
+@@ -4201,6 +4207,7 @@ _int_free (mstate av, mchunkptr p, int have_lock)
+ 	    return;
+ 	  }
+       }
++      }
+   }
+ #endif
+ 
diff --git a/src/allocators/glibcs.py b/src/allocators/glibcs.py
new file mode 100644
index 0000000..891bf15
--- /dev/null
+++ b/src/allocators/glibcs.py
@@ -0,0 +1,37 @@
+import os
+import subprocess
+
+from src.allocator import *
+from src.allocator import Allocator as Alloc
+from src.allocator import Allocator_Sources as Alloc_Src
+
+optimisation_flag = "-O2"
+
+glibc_src = Alloc_Src("glibc",
+                      retrieve_cmds=["git clone git://sourceware.org/git/glibc.git"],
+                      prepare_cmds=["git checkout release/2.29/master"],
+                      reset_cmds=["git stash"])
+
+glibc = Alloc("glibc", sources=glibc_src,
+              build_cmds=["mkdir -p glibc-build",
+                          "cd glibc-build; {srcdir}/configure --prefix={dir}",
+                          "cd glibc-build; make",
+                          "cd glibc-build; make install"],
+              cmd_prefix="{dir}/lib/ld-linux-x86-64.so.2 --library-path {dir}/lib:"+library_path)
+
+glibc_notc = Alloc("glibc_notc", sources=glibc_src,
+              build_cmds=["mkdir -p glibc-build",
+                          "cd glibc-build; {srcdir}/configure --prefix={dir} --disable-experimental-malloc",
+                          "cd glibc-build; make",
+                          "cd glibc-build; make install"],
+              cmd_prefix="{dir}/lib/ld-linux-x86-64.so.2 --library-path {dir}/lib:"+library_path)
+
+glibc_nofs = patch_alloc("glibc_nofs", glibc,
+                           ["allocators/glibc_2.28_no_passive_falsesharing.patch"])
+
+glibc_nofs_fancy = patch_alloc("glibc_nofs_fancy", glibc,
+                           ["allocators/glibc_2.28_no_passive_falsesharing_fancy.patch"])
+
+allocators_to_build = [glibc, glibc_notc, glibc_nofs, glibc_nofs_fancy]
+
+allocators = {a.name: a.build() for a in allocators_to_build}
diff --git a/src/allocators/hoard.py b/src/allocators/hoard.py
new file mode 100644
index 0000000..c50a616
--- /dev/null
+++ b/src/allocators/hoard.py
@@ -0,0 +1,18 @@
+import src.allocator
+
+
+version = 2.7
+sources = src.allocator.Allocator_Sources("Hoard",
+            retrieve_cmds=["git clone https://github.com/emeryberger/Hoard.git"],
+            reset_cmds=["git stash"])
+
+class Hoard (src.allocator.Allocator):
+    """Hoard definition for allocbench"""
+    def __init__(self, name, **kwargs):
+
+        kwargs["sources"] = sources
+        kwargs["LD_PRELOAD"] = "{srcdir}/src/libhoard.so"
+        kwargs["build_cmds"] = ["cd {srcdir}/src; make", "mkdir {dir}"]
+        kwargs["patches"] = ["{patchdir}/hoard_make.patch"]
+
+        super().__init__(name, **kwargs)
diff --git a/src/allocators/je_tc_super.py b/src/allocators/je_tc_super.py
new file mode 100644
index 0000000..2315da3
--- /dev/null
+++ b/src/allocators/je_tc_super.py
@@ -0,0 +1,19 @@
+from src.allocator import Allocator as Alloc
+from src.allocator import Allocator_Sources as Alloc_Src
+
+from src.allocators.glibc import Glibc
+from src.allocators.tcmalloc import TCMalloc
+from src.allocators.jemalloc import Jemalloc
+from src.allocators.supermalloc import SuperMalloc
+
+glibc = Glibc("glibc", color="C1")
+
+tcmalloc = TCMalloc("tcmalloc", color="C5")
+
+jemalloc = Jemalloc("jemalloc", color="C6")
+
+supermalloc = SuperMalloc("SuperMalloc", color="C8")
+
+allocators_to_build = [glibc, tcmalloc, jemalloc, supermalloc]
+
+allocators = {a.name: a.build() for a in allocators_to_build}
diff --git a/src/allocators/jemalloc.py b/src/allocators/jemalloc.py
new file mode 100644
index 0000000..c64f5af
--- /dev/null
+++ b/src/allocators/jemalloc.py
@@ -0,0 +1,20 @@
+import src.allocator
+
+version = "5.1.0"
+
+sources=src.allocator.Allocator_Sources("jemalloc",
+            retrieve_cmds=["git clone https://github.com/jemalloc/jemalloc.git"],
+            prepare_cmds=["git checkout {}".format(version), "./autogen.sh"])
+
+
+class Jemalloc (src.allocator.Allocator):
+    """jemalloc definition for allocbench"""
+    def __init__(self, name, **kwargs):
+
+        kwargs["sources"] = sources
+        kwargs["LD_PRELOAD"] = "{srcdir}/lib/libjemalloc.so"
+        kwargs["build_cmds"] = ["cd {srcdir}; ./configure --prefix={dir} CFLAGS=-O2",
+                                "cd {srcdir}; make -j4",
+                                "mkdir {dir}"]
+
+        super().__init__(name, **kwargs)
diff --git a/src/allocators/no_falsesharing.py b/src/allocators/no_falsesharing.py
new file mode 100644
index 0000000..fee631c
--- /dev/null
+++ b/src/allocators/no_falsesharing.py
@@ -0,0 +1,18 @@
+from src.allocators.tcmalloc import TCMalloc
+from src.allocators.glibc import Glibc
+
+glibc = Glibc("glibc", color="C1")
+
+glibc_nofs = Glibc("glibc_nofs",
+                   patches=["{patchdir}/glibc_2.28_no_passive_falsesharing.patch"],
+                   color="C2")
+
+tcmalloc = TCMalloc("tcmalloc", color="C3")
+
+tcmalloc_nofs = TCMalloc("tcmalloc_nofs",
+                         patches= ["{patchdir}/tcmalloc_2.7_no_active_falsesharing.patch"],
+                         color="C4")
+
+allocators_to_build = [glibc, glibc_nofs, tcmalloc, tcmalloc_nofs]
+
+allocators = {a.name: a.build() for a in allocators_to_build}
diff --git a/src/allocators/supermalloc.py b/src/allocators/supermalloc.py
new file mode 100644
index 0000000..7b96b80
--- /dev/null
+++ b/src/allocators/supermalloc.py
@@ -0,0 +1,22 @@
+import src.allocator
+
+version = "709663fb81ba091b0a78058869a644a272f4163d"
+
+sources=src.allocator.Allocator_Sources("SuperMalloc",
+            retrieve_cmds=["git clone https://github.com/kuszmaul/SuperMalloc"],
+            prepare_cmds=["git checkout {}".format(version)])
+
+
+class SuperMalloc (src.allocator.Allocator):
+    """jemalloc definition for allocbench"""
+    def __init__(self, name, **kwargs):
+
+        kwargs["sources"] = sources
+        kwargs["LD_PRELOAD"] = "{srcdir}/release/lib/libsupermalloc.so"
+        kwargs["build_cmds"] = ["cd {srcdir}/release; make CFLAGS=-O2",
+                                "mkdir {dir}"]
+
+        super().__init__(name, **kwargs)
+
+supermalloc = SuperMalloc("SuperMalloc", color="C1")
+allocators = {supermalloc.name: supermalloc.build()}
diff --git a/src/allocators/tcmalloc.py b/src/allocators/tcmalloc.py
new file mode 100644
index 0000000..1eb0db4
--- /dev/null
+++ b/src/allocators/tcmalloc.py
@@ -0,0 +1,20 @@
+import src.allocator
+
+
+version = 2.7
+
+tcmalloc_src = src.allocator.Allocator_Sources("tcmalloc",
+                         ["git clone https://github.com/gperftools/gperftools.git"],
+                         ["git checkout gperftools-{}".format(version), "./autogen.sh"],
+                         ["git stash"])
+
+class TCMalloc (src.allocator.Allocator):
+    """TCMalloc definition for allocbench"""
+    def __init__(self, name, **kwargs):
+
+        kwargs["sources"] = tcmalloc_src
+        kwargs["LD_PRELOAD"] = "{dir}/lib/libtcmalloc.so"
+        kwargs["build_cmds"] = ["cd {srcdir}; ./configure --prefix={dir} CXXFLAGS=-O2",
+                                "cd {srcdir}; make install -j4"]
+
+        super().__init__(name, **kwargs)
diff --git a/src/allocators/tcmalloc/tcmalloc_2.7_no_active_falsesharing.patch b/src/allocators/tcmalloc/tcmalloc_2.7_no_active_falsesharing.patch
new file mode 100644
index 0000000..c14172d
--- /dev/null
+++ b/src/allocators/tcmalloc/tcmalloc_2.7_no_active_falsesharing.patch
@@ -0,0 +1,17 @@
+diff --git a/src/thread_cache.cc b/src/thread_cache.cc
+index 6d2f832..2074f4a 100644
+--- a/src/thread_cache.cc
++++ b/src/thread_cache.cc
+@@ -120,7 +120,11 @@ void* ThreadCache::FetchFromCentralCache(uint32 cl, int32_t byte_size,
+   ASSERT(list->empty());
+   const int batch_size = Static::sizemap()->num_objects_to_move(cl);
+ 
+-  const int num_to_move = min<int>(list->max_length(), batch_size);
++  int num_to_move = min<int>(list->max_length(), batch_size);
++  const int cache_line_parts = 64 / byte_size;
++  if (cache_line_parts > 0 && cache_line_parts > num_to_move)
++      num_to_move = cache_line_parts;
++
+   void *start, *end;
+   int fetch_count = Static::central_cache()[cl].RemoveRange(
+       &start, &end, num_to_move);