made memdebug work. include script to generate graph

Jamfile

@@ -153,7 +153,8 @@ rule building ( properties * )
 feature disk-stats : off on : composite propagated link-incompatible ;
 feature.compose <disk-stats>on : <define>TORRENT_DISK_STATS ;
 
-feature memdebug : off on : propagated ;
+feature memdebug : off on : composite propagated ;
+feature.compose <memdebug>on : <define>TORRENT_MEMDEBUG ;
 
 feature logging : none default verbose : composite propagated link-incompatible ;
 feature.compose <logging>default : <define>TORRENT_LOGGING ;

parse_memory_log.py

@@ -0,0 +1,125 @@
+import os, sys, time
+
+# usage: memory.log memory_index.log
+
+lines = open(sys.argv[1], 'rb').readlines()
+index = open(sys.argv[2], 'rb').readlines()
+
+# logfile format:
+# #<allocation-point> <time(ms)> <key ('A' | 'F')> <address> <size> <total-size> <total-space-time> <peak-total-size>
+# example:
+# #12 38 A 0xd902a0 16 16 0 16
+
+allocation_points_to_print = 30
+
+def print_allocation_point(ap):
+   print 'space_time: %d kBms' % (ap['spacetime'] / 1024)
+   print 'allocations: %d' % ap['allocations']
+   print 'peak: %d kB' % (ap['peak'] / 1024)
+   print 'stack: '
+   counter = 0
+   for e in ap['stack']:
+      print '#%d %s' % (counter, e)
+      counter += 1
+
+allocation_points = []
+for l in index:
+   l = l.split('#')
+   l.pop(0)
+   ap = { 'allocations': 0, 'peak': 0, 'spacetime': 0, 'allocation_point': len(allocation_points), 'stack': l}
+   allocation_points.append(ap);
+
+for l in lines:
+   l = l.lstrip('#').rstrip('\n').split(' ')
+   if len(l) != 8:
+      print l
+      continue
+   try:
+      ap = int(l[0])
+      allocation_points[ap]['allocations'] += 1
+      allocation_points[ap]['peak'] = int(l[7])
+      allocation_points[ap]['spacetime'] = int(l[6])
+   except Exception, e:
+      print type(e), e, l
+
+print '=== space time ==='
+
+hot_ap = []
+allocation_points.sort(key = lambda x:x['spacetime'], reverse=True);
+counter = 0
+for ap in allocation_points[0:allocation_points_to_print]:
+   print '== %d ==' % counter
+   counter += 1
+   print_allocation_point(ap)
+   hot_ap.append(ap['allocation_point']);
+
+print '=== allocations ==='
+
+allocation_points.sort(key = lambda x:x['allocations'], reverse=True);
+for ap in allocation_points[0:allocation_points_to_print]:
+   print_allocation_point(ap)
+
+print '=== peak ==='
+
+allocation_points.sort(key = lambda x:x['peak'], reverse=True);
+for ap in allocation_points[0:allocation_points_to_print]:
+   print_allocation_point(ap)
+
+# generate graph
+lines = open(sys.argv[1], 'rb').readlines()
+
+out = open('memory.dat', 'wb')
+cur_line = [0] * allocation_points_to_print
+prev_line = [0] * allocation_points_to_print
+last_time = 0
+
+for l in lines:
+   l = l.lstrip('#').rstrip('\n').split(' ')
+   if len(l) != 8:
+      print l
+      continue
+   try:
+      time = int(l[1])
+      if time != last_time:
+         print >>out, last_time, '\t',
+         for i in range(allocation_points_to_print):
+	    if cur_line[i] == -1:
+	       print >>out, prev_line[i], '\t',
+	    else:
+	       print >>out, cur_line[i], '\t',
+	       prev_line[i] = cur_line[i]
+         print >>out
+	 cur_line = [-1] * allocation_points_to_print
+         last_time = time
+ 
+      size = int(l[5])
+      ap = int(l[0])
+      if ap in hot_ap:
+         index = hot_ap.index(ap)
+         cur_line[index] = max(cur_line[index], size)
+
+   except Exception, e:
+      print type(e), e, l
+
+out.close()
+
+out = open('memory.gnuplot', 'wb')
+print >>out, "set term png size 1200,700"
+print >>out, 'set output "memory.png"'
+print >>out, 'set xrange [0:*]'
+print >>out, 'set xlabel "time (ms)"'
+print >>out, 'set ylabel "bytes (B)"'
+print >>out, "set style data lines"
+print >>out, "set key box"
+print >>out, 'plot',
+for k in range(allocation_points_to_print):
+   print >>out, ' "memory.dat" using 1:(',
+   for i in range(k, allocation_points_to_print):
+      if i == k: print >>out, '$%d' % (i + 2),
+      else: print >>out, '+$%d' % (i + 2),
+   print >>out, ') title "%d" with filledcurves x1, \\' % k
+print >>out, 'x=0'
+out.close()
+
+os.system('gnuplot memory.gnuplot');
+

src/memdebug.cpp

@@ -41,122 +41,184 @@ POSSIBILITY OF SUCH DAMAGE.
 #include <map>
 #include <boost/cstdint.hpp>
 #include <boost/multi_index_container.hpp>
+#include <boost/thread/mutex.hpp>
+#include "libtorrent/time.hpp"
+#include "libtorrent/assert.hpp"
 
-namespace
+using boost::multi_index_container;
+using namespace boost::multi_index;
+using libtorrent::time_now;
+
+struct memdebug
 {
+	memdebug()
+	{
+		malloc_log.open("memory.log");
+		malloc_index_log.open("memory_index.log");
+	
+		assert(old_malloc_hook == 0);
+		assert(old_free_hook == 0);
+		old_malloc_hook = __malloc_hook;
+		old_free_hook = __free_hook;
+		__malloc_hook = my_malloc_hook;
+		__free_hook = my_free_hook;
+	}
 
-	std::ofstream malloc_log;
+	static void my_free_hook(void *ptr, const void *caller);
-	std::ofstream malloc_index_log;
+	static void* my_malloc_hook(size_t size, const void *caller);
 
-	using boost::multi_index_container;
+	static boost::mutex mutex;
-	using namespace boost::multi_index;
+	static std::ofstream malloc_log;
+	static std::ofstream malloc_index_log;
+
+	// the original library functions
+	static void* (*old_malloc_hook)(size_t, const void *);
+	static void (*old_free_hook)(void*, const void *);
 
 	struct allocation_point_t
 	{
-		allocation_point_t(): allocated(0) {}
+		allocation_point_t()
+			: allocated(0)
+			, peak_allocated(0)
+			, spacetime(0)
+			, last_update(time_now()) {}
+
 		int index;
-		boost::int64_t allocated;
+		// total number of bytes allocated from this point
+		int allocated;
+		// the maximum total number of bytes allocated
+		// from this point
+		int peak_allocated;
+		// total number of bytes allocated times the number of
+		// milliseconds they were allocated from this point
+		boost::int64_t spacetime;
+		// the last malloc or free operation on
+		// this allocation point. The spacetime
+		// should be updated from this point to
+		// the current operation
+		libtorrent::ptime last_update;
 	};
 
-	typedef boost::array<void*, 10> stacktrace_t;
+	typedef boost::array<void*, 15> stacktrace_t;
 	typedef std::map<stacktrace_t, allocation_point_t> allocation_map_t;
-	allocation_map_t allocation_points;
+	static allocation_map_t allocation_points;
-	std::map<void*, std::pair<allocation_map_t::iterator, int> > allocations;
+	static std::map<void*, std::pair<allocation_map_t::iterator, int> > allocations;
-	int allocation_point_index = 0;
+	static int allocation_point_index;
+	static libtorrent::ptime start_time;
+};
 
-	// the original library functions
+boost::mutex memdebug::mutex;
-	void* (*old_malloc_hook)(size_t, const void *);
+int memdebug::allocation_point_index = 0;
-	void (*old_free_hook)(void*, const void *);
+std::ofstream memdebug::malloc_log;
+std::ofstream memdebug::malloc_index_log;
+void* (*memdebug::old_malloc_hook)(size_t, const void *) = 0;
+void (*memdebug::old_free_hook)(void*, const void *) = 0;
+memdebug::allocation_map_t memdebug::allocation_points;
+std::map<void*, std::pair<memdebug::allocation_map_t::iterator, int> > memdebug::allocations;
+libtorrent::ptime memdebug::start_time = time_now();
 
-	void my_free_hook(void *ptr, const void *caller);
+void* memdebug::my_malloc_hook(size_t size, const void *caller)
+{
+	boost::mutex::scoped_lock l(mutex);
+	/* Restore all old hooks */
+	__malloc_hook = old_malloc_hook;
+	__free_hook = old_free_hook;
+	/* Call recursively */
+	void* result = malloc(size);
+	/* Save underlying hooks */
+	old_malloc_hook = __malloc_hook;
+	old_free_hook = __free_hook;
 
-	void* my_malloc_hook(size_t size, const void *caller)
+	stacktrace_t stack;
+	int stacksize = backtrace(&stack[0], stack.size());
+	libtorrent::ptime now = time_now();
+
+	allocation_map_t::iterator i = allocation_points.lower_bound(stack);
+	if (i == allocation_points.end() || i->first != stack)
 	{
-		void *result;
+		i = allocation_points.insert(i, std::make_pair(stack, allocation_point_t()));
-		/* Restore all old hooks */
+		i->second.index = allocation_point_index++;
-		__malloc_hook = old_malloc_hook;
+		i->second.allocated = size;
-		__free_hook = old_free_hook;
-		/* Call recursively */
-		result = malloc (size);
-		/* Save underlying hooks */
-		old_malloc_hook = __malloc_hook;
-		old_free_hook = __free_hook;
 
-		stacktrace_t stack;
+		malloc_index_log << i->second.index << "#";
-		int stacksize = backtrace(&stack[0], 10);
+		char** symbols = backtrace_symbols(&stack[0], stacksize);
-
+		for (int j = 2; j < stacksize; ++j)
-		allocation_map_t::iterator i = allocation_points.lower_bound(stack);
+			malloc_index_log << demangle(symbols[j]) << "#";
-		if (i == allocation_points.end() || i->first != stack)
+		malloc_index_log << std::endl;
-		{
-			i = allocation_points.insert(i, std::make_pair(stack, allocation_point_t()));
-			i->second.index = allocation_point_index++;
-			i->second.allocated = size;
-
-			malloc_index_log << "#" << i->second.index << " ";
-			char** symbols = backtrace_symbols(&stack[0], stacksize);
-			for (int j = 0; j < stacksize; ++j)
-				malloc_index_log << symbols[j] << " ";
-			malloc_index_log << std::endl;
-		}
-		else
-		{
-			i->second.allocated += size;
-		}
-
-		allocations[result] = std::make_pair(i, size);
-		malloc_log << "#" << i->second.index << " " << time(0) << " MALLOC "
-			<< result << " " << size << " (" << i->second.allocated << ")" << std::endl;
-
-		/* Restore our own hooks */
-		__malloc_hook = my_malloc_hook;
-		__free_hook = my_free_hook;
-		return result;
 	}
-
+	else
-	void my_free_hook(void *ptr, const void *caller)
 	{
-		/* Restore all old hooks */
+		allocation_point_t& ap = i->second;
-		__malloc_hook = old_malloc_hook;
+		ap.spacetime += libtorrent::total_milliseconds(now - ap.last_update) * ap.allocated;
-		__free_hook = old_free_hook;
+		ap.allocated += size;
-		/* Call recursively */
+		if (ap.allocated > ap.peak_allocated) ap.peak_allocated = ap.allocated;
-		free (ptr);
+		ap.last_update = now;
-		/* Save underlying hooks */
-		old_malloc_hook = __malloc_hook;
-		old_free_hook = __free_hook;
-
-		std::map<void*, std::pair<allocation_map_t::iterator, int> >::iterator i
-			= allocations.find(ptr);
-
-		if (i != allocations.end())
-		{
-			allocation_point_t& ap = i->second.first->second;
-			int size = i->second.second;
-			ap.allocated -= size;
-			malloc_log << "#" << ap.index << " " << time(0) << " FREE "
-				<< ptr << " " << size << " (" << ap.allocated << ")" << std::endl;
-			allocations.erase(i);
-		}
-
-		/* Restore our own hooks */
-		__malloc_hook = my_malloc_hook;
-		__free_hook = my_free_hook;
 	}
+	allocation_point_t& ap = i->second;
 
-	void my_init_hook(void)
+	allocations[result] = std::make_pair(i, size);
-	{
+	malloc_log << "#" << ap.index << " "
-		old_malloc_hook = __malloc_hook;
+		<< libtorrent::total_milliseconds(time_now() - start_time) << " A "
-		old_free_hook = __free_hook;
+		<< result << " " << size << " " << ap.allocated << " " << ap.spacetime
-		__malloc_hook = my_malloc_hook;
+		<< " " << ap.peak_allocated << std::endl;
-		__free_hook = my_free_hook;
-
-		malloc_log.open("memory.log");
-		malloc_index_log.open("memory_index.log");
-	}
 
+	/* Restore our own hooks */
+	__malloc_hook = my_malloc_hook;
+	__free_hook = my_free_hook;
+	return result;
 }
 
-// Override initializing hook from the C library.
+void memdebug::my_free_hook(void *ptr, const void *caller)
-void (*__malloc_initialize_hook) (void) = my_init_hook;
+{
+	boost::mutex::scoped_lock l(mutex);
+	/* Restore all old hooks */
+	__malloc_hook = old_malloc_hook;
+	__free_hook = old_free_hook;
+	/* Call recursively */
+	free(ptr);
+	/* Save underlying hooks */
+	old_malloc_hook = __malloc_hook;
+	old_free_hook = __free_hook;
+
+	std::map<void*, std::pair<allocation_map_t::iterator, int> >::iterator i
+		= allocations.find(ptr);
+
+	if (i != allocations.end())
+	{
+		allocation_point_t& ap = i->second.first->second;
+		int size = i->second.second;
+		ap.allocated -= size;
+		malloc_log << "#" << ap.index << " "
+			<< libtorrent::total_milliseconds(time_now() - start_time) << " F "
+			<< ptr << " " << size << " " << ap.allocated << " " << ap.spacetime
+			<< " " << ap.peak_allocated << std::endl;
+
+		allocations.erase(i);
+	}
+
+	/* Restore our own hooks */
+	__malloc_hook = my_malloc_hook;
+	__free_hook = my_free_hook;
+}
+
+static int ref_count = 0;
+
+void start_malloc_debug()
+{
+	boost::mutex::scoped_lock l(memdebug::mutex);
+	static memdebug mi;
+	++ref_count;
+}
+
+void stop_malloc_debug()
+{
+	boost::mutex::scoped_lock l(memdebug::mutex);
+	if (--ref_count == 0)
+	{
+		__malloc_hook = memdebug::old_malloc_hook;
+		__free_hook = memdebug::old_free_hook;
+	}
+}
 
 #endif
 

src/session.cpp

@@ -80,6 +80,11 @@ using boost::bind;
 using boost::mutex;
 using libtorrent::aux::session_impl;
 
+#ifdef TORRENT_MEMDEBUG
+void start_malloc_debug();
+void stop_malloc_debug();
+#endif
+
 namespace libtorrent
 {
 
@@ -117,6 +122,9 @@ namespace libtorrent
 #endif
 					))
 	{
+#ifdef TORRENT_MEMDEBUG
+		start_malloc_debug();
+#endif
 		// turn off the filename checking in boost.filesystem
 		TORRENT_ASSERT(listen_port_range.first > 0);
 		TORRENT_ASSERT(listen_port_range.first < listen_port_range.second);
@@ -140,6 +148,9 @@ namespace libtorrent
 		: m_impl(new session_impl(std::make_pair(0, 0), id, "0.0.0.0"))
 #endif
 	{
+#ifdef TORRENT_MEMDEBUG
+		start_malloc_debug();
+#endif
 #ifndef NDEBUG
 		boost::function0<void> test = boost::ref(*m_impl);
 		TORRENT_ASSERT(!test.empty());
@@ -148,6 +159,9 @@ namespace libtorrent
 
 	session::~session()
 	{
+#ifdef TORRENT_MEMDEBUG
+		stop_malloc_debug();
+#endif
 		TORRENT_ASSERT(m_impl);
 		// if there is at least one destruction-proxy
 		// abort the session and let the destructor