diff --git a/Jamfile b/Jamfile index dedd5bd7d..4b8819b9d 100755 --- a/Jamfile +++ b/Jamfile @@ -153,7 +153,8 @@ rule building ( properties * ) feature disk-stats : off on : composite propagated link-incompatible ; feature.compose on : TORRENT_DISK_STATS ; -feature memdebug : off on : propagated ; +feature memdebug : off on : composite propagated ; +feature.compose on : TORRENT_MEMDEBUG ; feature logging : none default verbose : composite propagated link-incompatible ; feature.compose default : TORRENT_LOGGING ; diff --git a/parse_memory_log.py b/parse_memory_log.py new file mode 100644 index 000000000..2528bee76 --- /dev/null +++ b/parse_memory_log.py @@ -0,0 +1,125 @@ +import os, sys, time + +# usage: memory.log memory_index.log + +lines = open(sys.argv[1], 'rb').readlines() +index = open(sys.argv[2], 'rb').readlines() + +# logfile format: +# #
+# example: +# #12 38 A 0xd902a0 16 16 0 16 + +allocation_points_to_print = 30 + +def print_allocation_point(ap): + print 'space_time: %d kBms' % (ap['spacetime'] / 1024) + print 'allocations: %d' % ap['allocations'] + print 'peak: %d kB' % (ap['peak'] / 1024) + print 'stack: ' + counter = 0 + for e in ap['stack']: + print '#%d %s' % (counter, e) + counter += 1 + +allocation_points = [] +for l in index: + l = l.split('#') + l.pop(0) + ap = { 'allocations': 0, 'peak': 0, 'spacetime': 0, 'allocation_point': len(allocation_points), 'stack': l} + allocation_points.append(ap); + +for l in lines: + l = l.lstrip('#').rstrip('\n').split(' ') + if len(l) != 8: + print l + continue + try: + ap = int(l[0]) + allocation_points[ap]['allocations'] += 1 + allocation_points[ap]['peak'] = int(l[7]) + allocation_points[ap]['spacetime'] = int(l[6]) + except Exception, e: + print type(e), e, l + +print '=== space time ===' + +hot_ap = [] +allocation_points.sort(key = lambda x:x['spacetime'], reverse=True); +counter = 0 +for ap in allocation_points[0:allocation_points_to_print]: + print '== %d ==' % counter + counter += 1 + print_allocation_point(ap) + hot_ap.append(ap['allocation_point']); + +print '=== allocations ===' + +allocation_points.sort(key = lambda x:x['allocations'], reverse=True); +for ap in allocation_points[0:allocation_points_to_print]: + print_allocation_point(ap) + +print '=== peak ===' + +allocation_points.sort(key = lambda x:x['peak'], reverse=True); +for ap in allocation_points[0:allocation_points_to_print]: + print_allocation_point(ap) + +# generate graph +lines = open(sys.argv[1], 'rb').readlines() + +out = open('memory.dat', 'wb') +cur_line = [0] * allocation_points_to_print +prev_line = [0] * allocation_points_to_print +last_time = 0 + +for l in lines: + l = l.lstrip('#').rstrip('\n').split(' ') + if len(l) != 8: + print l + continue + try: + time = int(l[1]) + if time != last_time: + print >>out, last_time, '\t', + for i in range(allocation_points_to_print): + if cur_line[i] == -1: + print >>out, prev_line[i], '\t', + else: + print >>out, cur_line[i], '\t', + prev_line[i] = cur_line[i] + print >>out + cur_line = [-1] * allocation_points_to_print + last_time = time + + size = int(l[5]) + ap = int(l[0]) + if ap in hot_ap: + index = hot_ap.index(ap) + cur_line[index] = max(cur_line[index], size) + + except Exception, e: + print type(e), e, l + +out.close() + +out = open('memory.gnuplot', 'wb') +print >>out, "set term png size 1200,700" +print >>out, 'set output "memory.png"' +print >>out, 'set xrange [0:*]' +print >>out, 'set xlabel "time (ms)"' +print >>out, 'set ylabel "bytes (B)"' +print >>out, "set style data lines" +print >>out, "set key box" +print >>out, 'plot', +for k in range(allocation_points_to_print): + print >>out, ' "memory.dat" using 1:(', + for i in range(k, allocation_points_to_print): + if i == k: print >>out, '$%d' % (i + 2), + else: print >>out, '+$%d' % (i + 2), + print >>out, ') title "%d" with filledcurves x1, \\' % k +print >>out, 'x=0' +out.close() + +os.system('gnuplot memory.gnuplot'); + diff --git a/src/memdebug.cpp b/src/memdebug.cpp index c31ec30f8..bdc5aa3f2 100644 --- a/src/memdebug.cpp +++ b/src/memdebug.cpp @@ -41,122 +41,184 @@ POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include +#include "libtorrent/time.hpp" +#include "libtorrent/assert.hpp" -namespace +using boost::multi_index_container; +using namespace boost::multi_index; +using libtorrent::time_now; + +struct memdebug { + memdebug() + { + malloc_log.open("memory.log"); + malloc_index_log.open("memory_index.log"); + + assert(old_malloc_hook == 0); + assert(old_free_hook == 0); + old_malloc_hook = __malloc_hook; + old_free_hook = __free_hook; + __malloc_hook = my_malloc_hook; + __free_hook = my_free_hook; + } - std::ofstream malloc_log; - std::ofstream malloc_index_log; + static void my_free_hook(void *ptr, const void *caller); + static void* my_malloc_hook(size_t size, const void *caller); - using boost::multi_index_container; - using namespace boost::multi_index; + static boost::mutex mutex; + static std::ofstream malloc_log; + static std::ofstream malloc_index_log; + + // the original library functions + static void* (*old_malloc_hook)(size_t, const void *); + static void (*old_free_hook)(void*, const void *); struct allocation_point_t { - allocation_point_t(): allocated(0) {} + allocation_point_t() + : allocated(0) + , peak_allocated(0) + , spacetime(0) + , last_update(time_now()) {} + int index; - boost::int64_t allocated; + // total number of bytes allocated from this point + int allocated; + // the maximum total number of bytes allocated + // from this point + int peak_allocated; + // total number of bytes allocated times the number of + // milliseconds they were allocated from this point + boost::int64_t spacetime; + // the last malloc or free operation on + // this allocation point. The spacetime + // should be updated from this point to + // the current operation + libtorrent::ptime last_update; }; - typedef boost::array stacktrace_t; + typedef boost::array stacktrace_t; typedef std::map allocation_map_t; - allocation_map_t allocation_points; - std::map > allocations; - int allocation_point_index = 0; + static allocation_map_t allocation_points; + static std::map > allocations; + static int allocation_point_index; + static libtorrent::ptime start_time; +}; - // the original library functions - void* (*old_malloc_hook)(size_t, const void *); - void (*old_free_hook)(void*, const void *); +boost::mutex memdebug::mutex; +int memdebug::allocation_point_index = 0; +std::ofstream memdebug::malloc_log; +std::ofstream memdebug::malloc_index_log; +void* (*memdebug::old_malloc_hook)(size_t, const void *) = 0; +void (*memdebug::old_free_hook)(void*, const void *) = 0; +memdebug::allocation_map_t memdebug::allocation_points; +std::map > memdebug::allocations; +libtorrent::ptime memdebug::start_time = time_now(); - void my_free_hook(void *ptr, const void *caller); +void* memdebug::my_malloc_hook(size_t size, const void *caller) +{ + boost::mutex::scoped_lock l(mutex); + /* Restore all old hooks */ + __malloc_hook = old_malloc_hook; + __free_hook = old_free_hook; + /* Call recursively */ + void* result = malloc(size); + /* Save underlying hooks */ + old_malloc_hook = __malloc_hook; + old_free_hook = __free_hook; - void* my_malloc_hook(size_t size, const void *caller) + stacktrace_t stack; + int stacksize = backtrace(&stack[0], stack.size()); + libtorrent::ptime now = time_now(); + + allocation_map_t::iterator i = allocation_points.lower_bound(stack); + if (i == allocation_points.end() || i->first != stack) { - void *result; - /* Restore all old hooks */ - __malloc_hook = old_malloc_hook; - __free_hook = old_free_hook; - /* Call recursively */ - result = malloc (size); - /* Save underlying hooks */ - old_malloc_hook = __malloc_hook; - old_free_hook = __free_hook; + i = allocation_points.insert(i, std::make_pair(stack, allocation_point_t())); + i->second.index = allocation_point_index++; + i->second.allocated = size; - stacktrace_t stack; - int stacksize = backtrace(&stack[0], 10); - - allocation_map_t::iterator i = allocation_points.lower_bound(stack); - if (i == allocation_points.end() || i->first != stack) - { - i = allocation_points.insert(i, std::make_pair(stack, allocation_point_t())); - i->second.index = allocation_point_index++; - i->second.allocated = size; - - malloc_index_log << "#" << i->second.index << " "; - char** symbols = backtrace_symbols(&stack[0], stacksize); - for (int j = 0; j < stacksize; ++j) - malloc_index_log << symbols[j] << " "; - malloc_index_log << std::endl; - } - else - { - i->second.allocated += size; - } - - allocations[result] = std::make_pair(i, size); - malloc_log << "#" << i->second.index << " " << time(0) << " MALLOC " - << result << " " << size << " (" << i->second.allocated << ")" << std::endl; - - /* Restore our own hooks */ - __malloc_hook = my_malloc_hook; - __free_hook = my_free_hook; - return result; + malloc_index_log << i->second.index << "#"; + char** symbols = backtrace_symbols(&stack[0], stacksize); + for (int j = 2; j < stacksize; ++j) + malloc_index_log << demangle(symbols[j]) << "#"; + malloc_index_log << std::endl; } - - void my_free_hook(void *ptr, const void *caller) + else { - /* Restore all old hooks */ - __malloc_hook = old_malloc_hook; - __free_hook = old_free_hook; - /* Call recursively */ - free (ptr); - /* Save underlying hooks */ - old_malloc_hook = __malloc_hook; - old_free_hook = __free_hook; - - std::map >::iterator i - = allocations.find(ptr); - - if (i != allocations.end()) - { - allocation_point_t& ap = i->second.first->second; - int size = i->second.second; - ap.allocated -= size; - malloc_log << "#" << ap.index << " " << time(0) << " FREE " - << ptr << " " << size << " (" << ap.allocated << ")" << std::endl; - allocations.erase(i); - } - - /* Restore our own hooks */ - __malloc_hook = my_malloc_hook; - __free_hook = my_free_hook; + allocation_point_t& ap = i->second; + ap.spacetime += libtorrent::total_milliseconds(now - ap.last_update) * ap.allocated; + ap.allocated += size; + if (ap.allocated > ap.peak_allocated) ap.peak_allocated = ap.allocated; + ap.last_update = now; } + allocation_point_t& ap = i->second; - void my_init_hook(void) - { - old_malloc_hook = __malloc_hook; - old_free_hook = __free_hook; - __malloc_hook = my_malloc_hook; - __free_hook = my_free_hook; - - malloc_log.open("memory.log"); - malloc_index_log.open("memory_index.log"); - } + allocations[result] = std::make_pair(i, size); + malloc_log << "#" << ap.index << " " + << libtorrent::total_milliseconds(time_now() - start_time) << " A " + << result << " " << size << " " << ap.allocated << " " << ap.spacetime + << " " << ap.peak_allocated << std::endl; + /* Restore our own hooks */ + __malloc_hook = my_malloc_hook; + __free_hook = my_free_hook; + return result; } -// Override initializing hook from the C library. -void (*__malloc_initialize_hook) (void) = my_init_hook; +void memdebug::my_free_hook(void *ptr, const void *caller) +{ + boost::mutex::scoped_lock l(mutex); + /* Restore all old hooks */ + __malloc_hook = old_malloc_hook; + __free_hook = old_free_hook; + /* Call recursively */ + free(ptr); + /* Save underlying hooks */ + old_malloc_hook = __malloc_hook; + old_free_hook = __free_hook; + + std::map >::iterator i + = allocations.find(ptr); + + if (i != allocations.end()) + { + allocation_point_t& ap = i->second.first->second; + int size = i->second.second; + ap.allocated -= size; + malloc_log << "#" << ap.index << " " + << libtorrent::total_milliseconds(time_now() - start_time) << " F " + << ptr << " " << size << " " << ap.allocated << " " << ap.spacetime + << " " << ap.peak_allocated << std::endl; + + allocations.erase(i); + } + + /* Restore our own hooks */ + __malloc_hook = my_malloc_hook; + __free_hook = my_free_hook; +} + +static int ref_count = 0; + +void start_malloc_debug() +{ + boost::mutex::scoped_lock l(memdebug::mutex); + static memdebug mi; + ++ref_count; +} + +void stop_malloc_debug() +{ + boost::mutex::scoped_lock l(memdebug::mutex); + if (--ref_count == 0) + { + __malloc_hook = memdebug::old_malloc_hook; + __free_hook = memdebug::old_free_hook; + } +} #endif diff --git a/src/session.cpp b/src/session.cpp index 331ffa377..9663d0900 100755 --- a/src/session.cpp +++ b/src/session.cpp @@ -80,6 +80,11 @@ using boost::bind; using boost::mutex; using libtorrent::aux::session_impl; +#ifdef TORRENT_MEMDEBUG +void start_malloc_debug(); +void stop_malloc_debug(); +#endif + namespace libtorrent { @@ -117,6 +122,9 @@ namespace libtorrent #endif )) { +#ifdef TORRENT_MEMDEBUG + start_malloc_debug(); +#endif // turn off the filename checking in boost.filesystem TORRENT_ASSERT(listen_port_range.first > 0); TORRENT_ASSERT(listen_port_range.first < listen_port_range.second); @@ -140,6 +148,9 @@ namespace libtorrent : m_impl(new session_impl(std::make_pair(0, 0), id, "0.0.0.0")) #endif { +#ifdef TORRENT_MEMDEBUG + start_malloc_debug(); +#endif #ifndef NDEBUG boost::function0 test = boost::ref(*m_impl); TORRENT_ASSERT(!test.empty()); @@ -148,6 +159,9 @@ namespace libtorrent session::~session() { +#ifdef TORRENT_MEMDEBUG + stop_malloc_debug(); +#endif TORRENT_ASSERT(m_impl); // if there is at least one destruction-proxy // abort the session and let the destructor