diff options
| -rw-r--r-- | src/Makefile | 6 | ||||
| -rw-r--r-- | src/allocators/chattymalloc.py | 41 | ||||
| -rw-r--r-- | src/chattymalloc.c | 400 | ||||
| -rw-r--r-- | src/chattymalloc.h | 43 | ||||
| -rwxr-xr-x | src/chattyparser.py | 420 |
5 files changed, 32 insertions, 878 deletions
diff --git a/src/Makefile b/src/Makefile index 84179e2..caceb19 100644 --- a/src/Makefile +++ b/src/Makefile @@ -14,7 +14,7 @@ MEMSIZE_KB=$(shell free -t | tail -1 | tr -s ' ' | cut -d ' ' -f 2) MEMSIZE="$(MEMSIZE_KB) * 1024l" TOOLS = print_status_on_exit.so exec sig_handlers.so -ALLOCS = chattymalloc.so bumpptr_alloc.so align_to_cl.so +ALLOCS = bumpptr_alloc.so align_to_cl.so TARGETS = $(addprefix $(OBJDIR)/allocators/,$(ALLOCS)) $(addprefix $(OBJDIR)/,$(TOOLS)) .PHONY: all clean @@ -29,10 +29,6 @@ $(OBJDIR)/allocators/align_to_cl.so: align_to_cl.c Makefile @if test \( ! \( -d $(@D) \) \) ;then mkdir -p $(@D);fi $(CC) $(LDFLAGS) -shared $(CFLAGS) -o $@ $< -ldl -$(OBJDIR)/allocators/chattymalloc.so: chattymalloc.h chattymalloc.c Makefile - @if test \( ! \( -d $(@D) \) \) ;then mkdir -p $(@D);fi - $(CC) $(LDFLAGS) -shared $(CFLAGS) -o $@ chattymalloc.c -ldl - $(OBJDIR)/print_status_on_exit.so: print_status_on_exit.c Makefile @if test \( ! \( -d $(@D) \) \) ;then mkdir -p $(@D);fi $(CC) $(LDFLAGS) -shared $(CFLAGS) -o $@ $< diff --git a/src/allocators/chattymalloc.py b/src/allocators/chattymalloc.py index 0f893bb..ff4a3ba 100644 --- a/src/allocators/chattymalloc.py +++ b/src/allocators/chattymalloc.py @@ -1,4 +1,4 @@ -# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de> +# Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de> # # This file is part of allocbench. # @@ -16,16 +16,37 @@ # along with allocbench. If not, see <http://www.gnu.org/licenses/>. """chattymalloc allocator -This shared library is no functional allocator. It is used to retrieve a trace -of the allocator usage of the executed programm. It overrides the malloc API -and writes each call and its result to an output file. -See src/chattymalloc.c and chattyparser.py for its implementation and usage. +This shared library is not a functional allocator. It is used to trace +the allocator usage and the executed program. It overrides the malloc API +and saves each call and its result to a memory mapped output file. """ import os -from src.allocator import Allocator, BUILDDIR -chattymalloc = Allocator( - "chattymalloc", - LD_PRELOAD=os.path.join(BUILDDIR, "chattymalloc.so"), - cmd_prefix="env CHATTYMALLOC_FILE={{result_dir}}/{{perm}}.trace") +from src.artifact import GitArtifact +from src.allocator import Allocator + +VERSION = "1a09b144eb18919014ecf86da3442344b0eaa5b2" + +class Chattymalloc(Allocator): + """Chattymalloc definition for allocbench""" + + sources = GitArtifact("chattymalloc", + "https://github.com/fischerling/chattymalloc") + + def __init__(self, name, **kwargs): + + configuration = "--buildtype=release " + for option, value in kwargs.get("options", {}).items(): + configuration += f"-D{option}={value} " + + self.build_cmds = [ + f"meson {{srcdir}} {{dir}} {configuration}", "ninja -C {dir}" + ] + + self.LD_PRELOAD = "{dir}/libchattymalloc.so" + self.cmd_prefix="env CHATTYMALLOC_FILE={{result_dir}}/{{perm}}.trace" + super().__init__(name, **kwargs) + + +chattymalloc = Chattymalloc("chattymalloc", version=VERSION) diff --git a/src/chattymalloc.c b/src/chattymalloc.c deleted file mode 100644 index 14bde06..0000000 --- a/src/chattymalloc.c +++ /dev/null @@ -1,400 +0,0 @@ -/* -Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de> - -This file is part of allocbench. - -allocbench is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -allocbench is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with allocbench. If not, see <http://www.gnu.org/licenses/>. -*/ - -#define _GNU_SOURCE -#include <dlfcn.h> -#include <errno.h> -#include <fcntl.h> -#include <inttypes.h> -#include <pthread.h> -#include <stdarg.h> -#include <stddef.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sys/mman.h> -#include <sys/syscall.h> -#include <unistd.h> - -#include "chattymalloc.h" - -#define unlikely(x) __builtin_expect((x),0) - -#define GROWTH_THRESHOLD 4096 -#define GROWTH_ENTRIES 100000 - -// flag to stop recursion during bootstrap -static int initializing = 0; - -// memory to bootstrap malloc -static char tmpbuff[4096]; -static unsigned long tmppos = 0; -static unsigned long tmpallocs = 0; - -// global log file descriptor -static int out_fd = -1; -// memory mapping of our output file -static volatile trace_t* out[2] = {NULL, NULL}; -// next free index into the mapped buffer -static volatile uint64_t next_entry = 0; -// current size of our log file / mapping -static volatile uint64_t total_entries = 0; - -// pthread mutex and cond to protect growth of the buffer -static pthread_cond_t growing; -static pthread_mutex_t growth_mutex; - -static __thread pid_t tid = 0; - -// thread specific key to register a destructor -static pthread_key_t tls_key; -static pthread_once_t tls_key_once = PTHREAD_ONCE_INIT; - -// log_thread_termination forward declaration because it uses trace_write -static void log_thread_termination(void* key __attribute__ ((unused))); - -static void make_tls_key() -{ - int err = pthread_key_create(&tls_key, log_thread_termination); - if (err) { - abort(); - } -} - -static void init_thread() -{ - tid = syscall(SYS_gettid); - - // init our thread destructor - int err = pthread_once(&tls_key_once, make_tls_key); - if (err) { - abort(); - } - - // set the key to something != NULL to execute the destructor on thread exit - // NOLINTNEXTLINE(readability-magic-numbers) - err = pthread_setspecific(tls_key, (void*)42); - if (err) { - abort(); - } -} - -/*========================================================= - * intercepted functions - */ - -static void* (*next_malloc)(size_t size); -static void (*next_free)(void* ptr); -static void* (*next_calloc)(size_t nmemb, size_t size); -static void* (*next_realloc)(void* ptr, size_t size); -static void* (*next_memalign)(size_t alignment, size_t size); -static int (*next_posix_memalign)(void** memptr, size_t alignment, size_t size); -static void* (*next_valloc)(size_t size); -static void* (*next_pvalloc)(size_t size); -static void* (*next_aligned_alloc)(size_t alignment, size_t size); -static int (*next_malloc_stats)(); - -static void -grow_trace() -{ - pthread_mutex_lock(&growth_mutex); - - size_t old_buf_idx; - if (unlikely(total_entries == 0)) - old_buf_idx = 0; - else - old_buf_idx = ((total_entries) / GROWTH_ENTRIES) % 2; - size_t new_buf_size = (total_entries + GROWTH_ENTRIES) * sizeof(trace_t); - - /* remap old buffer - * hopefully no thread uses the old buffer anymore! - */ - if (out[old_buf_idx] == NULL) { - out[old_buf_idx] = (trace_t*) mmap(NULL, new_buf_size, PROT_WRITE, - MAP_FILE | MAP_SHARED, out_fd, 0); - if (out[old_buf_idx] == MAP_FAILED) { - perror("mapping new buf failed"); - exit(1); - } - } else { - size_t old_buf_size = (total_entries - GROWTH_ENTRIES) * sizeof(trace_t); - out[old_buf_idx] = (trace_t*) mremap((void*) out[old_buf_idx], old_buf_size, - new_buf_size, MREMAP_MAYMOVE); - if (out[old_buf_idx] == MAP_FAILED) { - perror("remapping old buf failed"); - exit(1); - } - } - - - if(ftruncate(out_fd, new_buf_size) != 0) { - perror("extending file failed"); - exit(1); - } - - total_entries += GROWTH_ENTRIES; - pthread_cond_broadcast(&growing); - pthread_mutex_unlock(&growth_mutex); -} - -static void -write_trace(char func, void* ptr, size_t size, size_t var_arg) -{ - - if (unlikely(tid == 0)) { - init_thread(); - } - - uint64_t idx = __atomic_fetch_add (&next_entry, 1, __ATOMIC_SEQ_CST); - if (idx == total_entries - GROWTH_THRESHOLD) { - grow_trace(); - // wait for growth completion - } else if (idx >= total_entries) { - pthread_mutex_lock(&growth_mutex); - while(idx >= total_entries) - pthread_cond_wait(&growing, &growth_mutex); - pthread_mutex_unlock(&growth_mutex); - } - - volatile trace_t* trace = &out[(idx / GROWTH_ENTRIES) % 2][idx]; - - trace->tid = tid; - trace->func = func; - trace->ptr = ptr; - trace->size = size; - trace->var_arg = var_arg; -} - -static void log_thread_termination(void* key __attribute__ ((unused))) -{ - write_trace(THREAD_TERMINATION, NULL, 0, 0); -} - -static void -trim_trace() -{ - uint64_t cur_size = next_entry * sizeof(trace_t); - if(ftruncate(out_fd, cur_size) != 0) { - perror("trimming file failed"); - } - close(out_fd); -} - -static void __attribute__((constructor)) -init() -{ - initializing = 1; - char* fname = getenv("CHATTYMALLOC_FILE"); - if (fname == NULL) - fname = "chattymalloc.trace"; - - out_fd = open(fname, - O_RDWR | O_TRUNC | O_CREAT, - S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); - if (out_fd == -1) { - perror("opening output file"); - exit(1); - } - - pthread_cond_init(&growing, NULL); - pthread_mutex_init(&growth_mutex, NULL); - - // init trace buffer - grow_trace(); - - next_malloc = dlsym(RTLD_NEXT, "malloc"); - next_free = dlsym(RTLD_NEXT, "free"); - next_calloc = dlsym(RTLD_NEXT, "calloc"); - next_realloc = dlsym(RTLD_NEXT, "realloc"); - next_memalign = dlsym(RTLD_NEXT, "memalign"); - next_posix_memalign = dlsym(RTLD_NEXT, "posix_memalign"); - next_valloc = dlsym(RTLD_NEXT, "valloc"); - next_pvalloc = dlsym(RTLD_NEXT, "pvalloc"); - next_aligned_alloc = dlsym(RTLD_NEXT, "aligned_alloc"); - next_malloc_stats = dlsym(RTLD_NEXT, "malloc_stats"); - - if (!next_malloc || !next_free || !next_calloc || !next_realloc || - !next_memalign) { - fprintf(stderr, "Can't load core functions with `dlsym`: %s\n", dlerror()); - exit(1); - } - if (!next_posix_memalign) - fprintf(stderr, "Can't load posix_memalign with `dlsym`: %s\n", dlerror()); - if (!next_valloc) - fprintf(stderr, "Can't load valloc with `dlsym`: %s\n", dlerror()); - if (!next_pvalloc) - fprintf(stderr, "Can't load pvalloc with `dlsym`: %s\n", dlerror()); - if (!next_aligned_alloc) - fprintf(stderr, "Can't load aligned_alloc with `dlsym`: %s\n", dlerror()); - if (!next_malloc_stats) - fprintf(stderr, "Can't load malloc_stats with `dlsym`: %s\n", dlerror()); - - atexit(trim_trace); - initializing = 0; -} - -void* -malloc(size_t size) -{ - if (next_malloc == NULL) { - if (!initializing) { - init(); - - } else { - void* retptr = tmpbuff + tmppos; - tmppos += size; - ++tmpallocs; - - if (tmppos < sizeof(tmpbuff)) { - return retptr; - } else { - fprintf(stderr, "%d in %d allocs\n", tmppos, tmpallocs); - fprintf(stderr, - "jcheck: too much memory requested during initialisation - " - "increase tmpbuff size\n"); - exit(1); - } - } - } - - void* ptr = next_malloc(size); - write_trace(MALLOC, ptr, size, 0); - return ptr; -} - -void -free(void* ptr) -{ - // something wrong if we call free before one of the allocators! - if (next_malloc == NULL) - init(); - if (!(ptr >= (void*)tmpbuff && ptr <= (void*)(tmpbuff + tmppos))) { - write_trace(FREE, ptr, 0, 0); - next_free(ptr); - } -} - -void* -realloc(void* ptr, size_t size) -{ - if (next_realloc == NULL) { - void* nptr = malloc(size); - if (nptr && ptr) { - memmove(nptr, ptr, size); - free(ptr); - } - return nptr; - } - - void* nptr = next_realloc(ptr, size); - write_trace(REALLOC, nptr, size, (size_t)ptr); - return nptr; -} - -void* -calloc(size_t nmemb, size_t size) -{ - if (next_calloc == NULL) { - void* ptr = malloc(nmemb * size); - if (ptr) - memset(ptr, 0, nmemb * size); - return ptr; - } - - void* ptr = next_calloc(nmemb, size); - write_trace(CALLOC, ptr, size, nmemb); - return ptr; -} - -void* -memalign(size_t alignment, size_t size) -{ - if (next_memalign == NULL) { - fprintf(stderr, "called memalign before or during init\n"); - exit(1); - } - - void* ptr = next_memalign(alignment, size); - write_trace(MEMALIGN, ptr, size, alignment); - return ptr; -} - -int -posix_memalign(void** memptr, size_t alignment, size_t size) -{ - if (next_posix_memalign == NULL) { - fprintf(stderr, "called posix_memalign before or during init\n"); - exit(1); - } - - int ret = next_posix_memalign(memptr, alignment, size); - write_trace(POSIX_MEMALIGN, *memptr, size, alignment); - return ret; -} - -void* -valloc(size_t size) -{ - if (next_valloc == NULL) { - fprintf(stderr, "called valloc before or during init"); - exit(1); - } - - void* ptr = next_valloc(size); - write_trace(VALLOC, ptr, size, 0); - return ptr; -} - -void* -pvalloc(size_t size) -{ - if (next_pvalloc == NULL) { - fprintf(stderr, "called pvalloc before or during init\n"); - exit(1); - } - - void* ptr = next_pvalloc(size); - write_trace(PVALLOC, ptr, size, 0); - return ptr; -} - -void* -aligned_alloc(size_t alignment, size_t size) -{ - if (next_aligned_alloc == NULL) { - fprintf(stderr, "called aligned_alloc before or during init\n"); - exit(1); - } - - void* ptr = next_aligned_alloc(alignment, size); - write_trace(ALIGNED_ALLOC, ptr, size, alignment); - return ptr; -} - -int -malloc_stats() -{ - if (next_malloc_stats == NULL) { - fprintf(stderr, "called malloc_stats before or during init\n"); - exit(1); - } - - fprintf(stderr, "chattymalloc by muhq\n"); - return next_malloc_stats(); -} diff --git a/src/chattymalloc.h b/src/chattymalloc.h deleted file mode 100644 index 5120193..0000000 --- a/src/chattymalloc.h +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de> - -This file is part of allocbench. - -allocbench is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -allocbench is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with allocbench. If not, see <http://www.gnu.org/licenses/>. -*/ - -#include <stdint.h> // uint8_t -#include <sys/types.h> // pid_t - -enum functions { - UNINITIALIZED, - MALLOC, - FREE, - REALLOC, - CALLOC, - MEMALIGN, - POSIX_MEMALIGN, - VALLOC, - PVALLOC, - ALIGNED_ALLOC, - THREAD_TERMINATION}; - -typedef struct trace { - void* ptr; - size_t size; - size_t var_arg; - pid_t tid; - char func; -} __attribute__((packed)) trace_t; - diff --git a/src/chattyparser.py b/src/chattyparser.py deleted file mode 100755 index 169463f..0000000 --- a/src/chattyparser.py +++ /dev/null @@ -1,420 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de> -# -# This file is part of allocbench. -# -# allocbench is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# allocbench is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with allocbench. If not, see <http://www.gnu.org/licenses/>. -"""Parser and Plotter for the traces produced by chattymalloc""" - -import argparse -from enum import Enum -import os -import struct -import sys - -import matplotlib.pyplot as plt -import numpy as np - -CHECK_ALIGNMENT = None -EXPORT_TXT = False - - -class Function(Enum): - """Enum holding all trace events of chattymalloc""" - uninitialized = 0 - malloc = 1 - free = 2 - realloc = 3 - calloc = 4 - memalign = 5 - posix_memalign = 6 - valloc = 7 - pvalloc = 8 - aligned_alloc = 9 - thread_termination = 10 - - -class Trace: - """Class representing the chattymalloc trace_t struct""" - - fmt = 'Pnnib' - size = struct.calcsize(fmt) - - def __init__(self, ptr, size, var_arg, tid, func): - self.ptr = ptr - self.size = size - self.var_arg = var_arg - self.tid = tid - self.func = Function(func) - - @classmethod - def unpack(cls, buf): - """Create a new Trace object from bytes""" - return Trace(*struct.unpack(Trace.fmt, buf)) - - @classmethod - def iter_unpack(cls, buf): - """Create a iterator returning Trace object from bytes""" - for values in struct.iter_unpack(Trace.fmt, buf): - yield Trace(*values) - - def __str__(self): - if self.func == Function.realloc: - var_arg = hex(self.var_arg) - else: - var_arg = self.var_arg - return f"{self.tid}: {self.depth} {self.func.name} {hex(self.ptr)} {self.size} {var_arg}" - -def update_cache_lines(cache_lines, trace, size): - """mark or unmark all cache lines spanned by this allocation""" - if cache_lines is None: - return "" - - start = trace.ptr - end = start + abs(size) - msg = "" - - cache_line = start & ~(64 - 1) - assert cache_line % 64 == 0 - while cache_line < end: - if trace.func != Function.free: - if cache_line not in cache_lines or cache_lines[cache_line] == []: - cache_lines[cache_line] = [trace.tid] - # false sharing - else: - if trace.tid not in cache_lines[cache_line]: - msg += (f"WARNING: cache line {hex(cache_line)} is shared " - f"between {set(cache_lines[cache_line] + [trace.tid])}\n") - cache_lines[cache_line].append(trace.tid) - else: - if trace.tid in cache_lines[cache_line]: - cache_lines[cache_line].remove(trace.tid) - else: - #If cache line is only owned by one thread it should be save to remove it - if len(cache_lines[cache_line]) == 1: - del cache_lines[cache_line] - elif len(cache_lines[cache_line]) == 0: - msg += f"INTERNAL ERROR: freeing not owned cache line\n" - #TODO fix passed allocations - else: - pass - - # print(hex(cache_line), cache_lines[cache_line], end=" ") - cache_line += 64 - - # print() - return msg - - -def record_allocation(trace, context): - """add allocation to histogram or total requested memory - - trace - Trace object ro record - context - dict holding all data structures used for parsing - allocations - dict of life allocations mapping their pointer to their size - hist - dict mapping allocation sizes to their occurrence - realloc_hist - dict mapping the two realloc sizes to their occurence - total_size - list of total requested memory till last recorded function call - cache_lines - dict of cache lines mapped to the owning tids - req_size - dict mapping sizes to their individual total requested memory - """ - - # mandatory - allocations = context.setdefault("allocations", []) - - # optional - hist = context.get("hist", None) - realloc_hist = context.get("realloc_hist", None) - total_size = context.get("total_size", None) - cache_lines = context.get("cache_lines", None) - req_sizes = context.get("req_sizes", {}) - - size = 0 - msg = "" - - if trace.func == Function.thread_termination: - return "" - - if trace.func == Function.uninitialized: - return "WARNING: empty entry\n" - - # (potential) free of a pointer - if trace.func in (Function.free, Function.realloc): - if trace.func == Function.realloc: - freed_ptr = trace.var_arg - else: - freed_ptr = trace.ptr - - # get size and delete old pointer - if freed_ptr != 0: - if freed_ptr not in allocations: - msg = f"WARNING: free of invalid pointer {freed_ptr:x}\n" - else: - size = allocations.pop(freed_ptr) * -1 - msg = update_cache_lines(cache_lines, trace, size) - - # allocations - if trace.func != Function.free and trace.ptr != 0: - # check for alignment - if CHECK_ALIGNMENT: - if (trace.ptr - CHECK_ALIGNMENT[1]) % CHECK_ALIGNMENT[0] != 0: - msg += (f"WARNING: ptr: {trace.ptr:x} is not aligned to" - f" {CHECK_ALIGNMENT[0]} with offset {CHECK_ALIGNMENT[1]}\n") - - if trace.func == Function.calloc: - allocation_size = trace.var_arg * trace.size - else: - allocation_size = trace.size - - # realloc returning the same pointer will not be reported because it has been freed already - if trace.ptr in allocations: - msg += f"WARNING: returned ptr {trace.ptr:x} is already a live allocation\n" - - allocations[trace.ptr] = allocation_size - - msg += update_cache_lines(cache_lines, trace, allocation_size) - - # update hist - if hist is not None and trace.func != Function.free: - hist[allocation_size] = hist.get(allocation_size, 0) + 1 - - # special case realloc - if trace.func == Function.realloc: - if realloc_hist is not None: - realloc_hist[(size, allocation_size)] = realloc_hist.get( - (size, allocation_size), 0) - - size += allocation_size - - # update total size - if total_size is not None: - total_size.append(total_size[-1] + size) - - for req_size in req_sizes: - if size == req_size: - req_sizes[req_size].append(req_sizes[req_size][-1] + size) - else: - req_sizes[req_size].append(req_sizes[req_size][-1]) - - return msg - - -def parse(path="chattymalloc.txt", - hist=True, - track_total=True, - track_calls=True, - realloc_hist=True, - cache_lines=True, - req_sizes=None): - """parse a chattymalloc trace - - :returns: a context dict containing the histogram, a realloc histogram, - a function call histogram, total live memory per function call, - a dict mapping cache_lines to their owning TIDs - """ - # context dictionary holding our parsed information - context = {} - - # Dictionary to track all live allocations - context["allocations"] = {} - - if track_calls: - # function call histogram - context["calls"] = {f: 0 for f in Function} - - if track_total: - # List of total live memory per operation - context["total_size"] = [0] - - if req_sizes: - # allocation sizes to track - context["req_sizes"] = req_sizes - - if hist: - # Dictionary mapping allocation sizes to the count - context["hist"] = {} - - if realloc_hist: - # Dictionary mapping realloc sizes to their count - context["realloc_hist"] = {} - - if cache_lines: - # Dictionary mapping cache lines to their owning TIDs - context["cache_lines"] = {} - - if EXPORT_TXT: - plain_file = open(path+".txt", "w") - - with open(path, "rb") as trace_file: - total_entries = os.stat(trace_file.fileno()).st_size // Trace.size - update_interval = int(total_entries * 0.0005) - if update_interval == 0: - update_interval = 1 - - i = 0 - entry = trace_file.read(Trace.size) - while entry != b'': - # print process - if i % update_interval == 0: - print(f"\r[{i} / {total_entries}] {(i/total_entries)*100:.2f}% parsed ...", end="") - - try: - trace = Trace.unpack(entry) - - if track_calls: - context["calls"][trace.func] += 1 - msg = record_allocation(trace, context) - if msg: - print(f"entry {i}: {msg}", file=sys.stderr, end="") - - if EXPORT_TXT: - print(trace, file=plain_file) - - except ValueError as err: - print(f"ERROR: {err} in entry {i}: {entry}", file=sys.stderr) - - - i += 1 - entry = trace_file.read(Trace.size) - - print(f"\r[{i} / {total_entries}] {(i / total_entries) * 100:.2f}% parsed ...") - if EXPORT_TXT: - plain_file.close() - return context - - -def plot(path): - """Plot a histogram and a memory profile of the given chattymalloc trace""" - result = parse(path=path) - hist = result["hist"] - - plot_hist_ascii(f"{path}.hist", hist, result["calls"]) - - top5 = [t[1] for t in sorted([(n, s) for s, n in hist.items()])[-5:]] - - plot_profile(path, path + ".profile.png", top5) - - -def plot_profile(trace_path, plot_path, sizes): - """Plot a memory profile of the total memory and the top 5 sizes""" - - res = parse(path=trace_path, - hist=False, - realloc_hist=False, - cache_lines=False, - req_sizes={s: [0] for s in sizes}) - - total_size = np.array(res["total_size"]) - del res["total_size"] - - x_vals = range(0, len(total_size)) - - plt.plot(x_vals, - total_size / 1000, - marker='', - linestyle='-', - label="Total requested") - - for size in sizes: - req_size = np.array(res["req_sizes"][size]) - del res["req_sizes"][size] - plt.plot(x_vals, req_size / 1000, label=size) - - plt.legend(loc="lower center") - plt.xlabel("Allocations") - plt.ylabel("mem in kb") - plt.title("Memusage profile") - plt.savefig(plot_path) - plt.clf() - - -def plot_hist_ascii(path, hist, calls): - """Plot an ascii histogram""" - bins = {} - for size in sorted(hist): - size_class = size // 16 - bins[size_class] = bins.get(size_class, 0) + hist[size] - - with open(path, "w") as hist_file: - print("Total function calls:", sum(calls.values()), file=hist_file) - for func, func_calls in calls.items(): - print(func.name, func_calls, file=hist_file) - - print(file=hist_file) - - total = sum(hist.values()) - top10 = [t[1] for t in sorted([(n, s) for s, n in hist.items()])[-10:]] - top10_total = sum([hist[size] for size in top10]) - - print( - f"Top 10 allocation sizes {(top10_total/total)*100:.2f}% of all allocations", - file=hist_file) - for i, size in enumerate(reversed(top10)): - print(f"{i+1}. {size} B occurred {hist[size]} times", - file=hist_file) - print(file=hist_file) - - for i in [64, 1024, 4096]: - allocations = sum([n for s, n in hist.items() if s <= i]) - print( - f"allocations <= {i}: {allocations} {(allocations/total)*100:.2f}%", - file=hist_file) - print(file=hist_file) - - print("Histogram of sizes:", file=hist_file) - sbins = sorted(bins) - binmaxlength = len(str(sbins[-1])) + 1 - amountmaxlength = str(len(str(sorted(bins.values())[-1]))) - for current_bin in sbins: - perc = bins[current_bin] / total * 100 - binsize = f"{{:<{binmaxlength}}} - {{:>{binmaxlength}}}" - print(binsize.format(current_bin * 16, (current_bin + 1) * 16 - 1), - end=" ", - file=hist_file) - amount = "{:<" + amountmaxlength + "} {:.2f}% {}" - print(amount.format(bins[current_bin], perc, '*' * int(perc / 2)), - file=hist_file) - - -if __name__ == "__main__": - # Code duplication with src.util.print_license_and_exit - # to keep chattyparser independent from allocbench - if "--license" in sys.argv: - print("Copyright (C) 2018-2019 Florian Fischer") - print( - "License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>" - ) - sys.exit(0) - - parser = argparse.ArgumentParser(description="parse and analyse chattymalloc traces") - parser.add_argument("trace", - help="binary trace file created by chattymalloc") - parser.add_argument("--alignment", - nargs=2, - help="export to plain text format") - parser.add_argument("--txt", - help="export to plain text format", - action="store_true") - parser.add_argument("-v", "--verbose", help="more output", action='count') - parser.add_argument("--license", - help="print license info and exit", - action='store_true') - - args = parser.parse_args() - - if args.alignment: - CHECK_ALIGNMENT = [int(x) for x in args.alignment] - EXPORT_TXT = args.txt - plot(args.trace) |