added a plugin 'smart_ban' that can ban peers sending bad data with higher accuracy

Jamfile

@@ -238,11 +238,8 @@ SOURCES =
 	http_tracker_connection
 	udp_tracker_connection
 	sha1
-	metadata_transfer
 	udp_socket
 	upnp
-	ut_pex
-	ut_metadata
 	logger
 	file_pool
 	lsd
@@ -250,6 +247,12 @@ SOURCES =
 	enum_net
 	broadcast_socket
 	magnet_uri
+
+# -- extensions --
+	metadata_transfer
+	ut_pex
+	ut_metadata
+	smart_ban
 	;
 
 KADEMLIA_SOURCES =

docs/manual.html

@@ -712,18 +712,37 @@ object that is called with a <tt class="docutils literal"><span class="pre">torr
 <dd>Allows peers to download the metadata (.torren files) from the swarm
 directly. Makes it possible to join a swarm with just a tracker and
 info-hash.</dd>
+</dl>
+<pre class="literal-block">
+#include &lt;libtorrent/extensions/metadata_transfer.hpp&gt;
+ses.add_extension(&amp;libtorrent::create_metadata_plugin);
+</pre>
+<dl class="docutils">
+<dt>uTorrent metadata</dt>
+<dd>Same as <tt class="docutils literal"><span class="pre">metadata</span> <span class="pre">extension</span></tt> but compatible with uTorrent.</dd>
+</dl>
+<pre class="literal-block">
+#include &lt;libtorrent/extensions/ut_metadata.hpp&gt;
+ses.add_extension(&amp;libtorrent::create_ut_metadata_plugin);
+</pre>
+<dl class="docutils">
 <dt>uTorrent peer exchange</dt>
 <dd>Exchanges peers between clients.</dd>
 </dl>
-<p>To use these, imclude <tt class="docutils literal"><span class="pre">&lt;libtorrent/extensions/metadata_transfer.hpp&gt;</span></tt>
-or <tt class="docutils literal"><span class="pre">&lt;libtorrent/extensions/ut_pex.hpp&gt;</span></tt>. The functions to pass in to
-<tt class="docutils literal"><span class="pre">add_extension()</span></tt> are <tt class="docutils literal"><span class="pre">libtorrent::create_metadata_plugin</span></tt> and
-<tt class="docutils literal"><span class="pre">libtorrent::create_ut_pex_plugin</span></tt> respectively.</p>
-<p>e.g.</p>
 <pre class="literal-block">
-ses.add_extension(&amp;libtorrent::create_metadata_plugin);
+#include &lt;libtorrent/extensions/ut_pex.hpp&gt;
 ses.add_extension(&amp;libtorrent::create_ut_pex_plugin);
 </pre>
+<dl class="docutils">
+<dt>smart ban plugin</dt>
+<dd>A plugin that, with a small overhead, can ban peers
+that sends bad data with very high accuracy. Should
+eliminate most problems on poisoned torrents.</dd>
+</dl>
+<pre class="literal-block">
+#include &lt;libtorrent/extensions/smart_ban.hpp&gt;
+ses.add_extension(&amp;libtorrent::create_smart_ban_plugin);
+</pre>
 </div>
 <div class="section">
 <h2><a id="set-settings-set-pe-settings" name="set-settings-set-pe-settings">set_settings() set_pe_settings()</a></h2>

docs/manual.rst

@@ -565,21 +565,38 @@ metadata extension
 	directly. Makes it possible to join a swarm with just a tracker and
 	info-hash.
 
-uTorrent peer exchange
+::
-	Exchanges peers between clients.
 
-To use these, imclude ``<libtorrent/extensions/metadata_transfer.hpp>``
+	#include <libtorrent/extensions/metadata_transfer.hpp>
-or ``<libtorrent/extensions/ut_pex.hpp>``. The functions to pass in to
+	ses.add_extension(&libtorrent::create_metadata_plugin);
-``add_extension()`` are ``libtorrent::create_metadata_plugin`` and
-``libtorrent::create_ut_pex_plugin`` respectively.
 
-e.g.
+uTorrent metadata
+	Same as ``metadata extension`` but compatible with uTorrent.
 
 ::
 
-	ses.add_extension(&libtorrent::create_metadata_plugin);
+	#include <libtorrent/extensions/ut_metadata.hpp>
+	ses.add_extension(&libtorrent::create_ut_metadata_plugin);
+
+uTorrent peer exchange
+	Exchanges peers between clients.
+
+::
+
+	#include <libtorrent/extensions/ut_pex.hpp>
 	ses.add_extension(&libtorrent::create_ut_pex_plugin);
 
+smart ban plugin
+	A plugin that, with a small overhead, can ban peers
+	that sends bad data with very high accuracy. Should
+	eliminate most problems on poisoned torrents.
+
+::
+
+	#include <libtorrent/extensions/smart_ban.hpp>
+	ses.add_extension(&libtorrent::create_smart_ban_plugin);
+
+
 .. _`libtorrent plugins`: libtorrent_plugins.html
 
 set_settings() set_pe_settings()

examples/client_test.cpp

@@ -56,6 +56,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/extensions/metadata_transfer.hpp"
 #include "libtorrent/extensions/ut_metadata.hpp"
 #include "libtorrent/extensions/ut_pex.hpp"
+#include "libtorrent/extensions/smart_ban.hpp"
 
 #include "libtorrent/entry.hpp"
 #include "libtorrent/bencode.hpp"
@@ -483,8 +484,7 @@ void scan_dir(path const& dir_path
 
 		// the file has been added to the dir, start
 		// downloading it.
-		add_torrent(ses, handles, file, preferred_ratio, compact_mode ? storage_mode_compact
+		add_torrent(ses, handles, file, preferred_ratio, compact_mode
-			: storage_mode_sparse
 			, save_path, true);
 		valid.insert(file);
 	}
@@ -571,7 +571,7 @@ int main(int ac, char* av[])
 		("ip-filter,f", po::value<std::string>(&ip_filter_file)->default_value("")
 			, "sets the path to the ip-filter file used to block access from certain "
 			"ips. ")
-		("allocation-mode,a", po::value<std::string>(&allocation_mode)->default_value("compact")
+		("allocation-mode,a", po::value<std::string>(&allocation_mode)->default_value("full")
 			, "sets mode used for allocating the downloaded files on disk. "
 			"Possible options are [full | compact]")
 		("input-file,i", po::value<std::vector<std::string> >()
@@ -709,6 +709,7 @@ int main(int ac, char* av[])
 		ses.add_extension(&create_metadata_plugin);
 		ses.add_extension(&create_ut_pex_plugin);
 		ses.add_extension(&create_ut_metadata_plugin);
+		ses.add_extension(&create_smart_ban_plugin);
 
 		ses.set_max_uploads(upload_slots_limit);
 		ses.set_max_half_open_connections(half_open_limit);

include/Makefile.am

@@ -72,6 +72,7 @@ libtorrent/extensions/metadata_transfer.hpp \
 libtorrent/extensions/ut_pex.hpp \
 libtorrent/extensions/ut_metadata.hpp \
 libtorrent/extensions/logger.hpp \
+libtorrent/extensions/smart_ban.hpp \
 \
 libtorrent/kademlia/closest_nodes.hpp \
 libtorrent/kademlia/dht_tracker.hpp \

include/libtorrent/extensions/smart_ban.hpp

@@ -0,0 +1,55 @@
+/*
+
+Copyright (c) 2007, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#ifndef TORRENT_SMART_BAN_HPP_INCLUDED
+#define TORRENT_SMART_BAN_HPP_INCLUDED
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#endif
+
+#include <boost/shared_ptr.hpp>
+#include "libtorrent/config.hpp"
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+namespace libtorrent
+{
+	struct torrent_plugin;
+	class torrent;
+	TORRENT_EXPORT boost::shared_ptr<torrent_plugin> create_smart_ban_plugin(torrent*, void*);
+}
+
+#endif // TORRENT_SMART_BAN_HPP_INCLUDED
+

include/libtorrent/peer_connection.hpp

@@ -404,6 +404,10 @@ namespace libtorrent
 		bool packet_finished() const
 		{ return m_packet_size <= m_recv_pos; }
 
+#ifndef NDEBUG
+		bool piece_failed;
+#endif
+
 	protected:
 
 		virtual void get_specific_peer_info(peer_info& p) const = 0;

src/Makefile.am

@@ -22,7 +22,7 @@ http_tracker_connection.cpp udp_tracker_connection.cpp \
 alert.cpp identify_client.cpp ip_filter.cpp file.cpp metadata_transfer.cpp \
 logger.cpp file_pool.cpp ut_pex.cpp lsd.cpp upnp.cpp instantiate_connection.cpp \
 socks5_stream.cpp socks4_stream.cpp http_stream.cpp connection_queue.cpp \
-disk_io_thread.cpp ut_metadata.cpp magnet_uri.cpp udp_socket.cpp \
+disk_io_thread.cpp ut_metadata.cpp magnet_uri.cpp udp_socket.cpp smart_ban.cpp \
 $(kademlia_sources)
 
 noinst_HEADERS = \

src/peer_connection.cpp

@@ -133,6 +133,7 @@ namespace libtorrent
 		(*m_logger) << "*** OUTGOING CONNECTION\n";
 #endif
 #ifndef NDEBUG
+		piece_failed = false;
 		m_requested_read_quota = false;
 		m_requested_write_quota = false;
 #endif
@@ -220,6 +221,7 @@ namespace libtorrent
 #endif
 		
 #ifndef NDEBUG
+		piece_failed = false;
 		m_requested_read_quota = false;
 		m_requested_write_quota = false;
 #endif
@@ -2935,7 +2937,7 @@ namespace libtorrent
 					complete = false;
 					break;
 				}
-				if (complete)
+				if (complete && !piece_failed)
 				{
 					disk_io_job ret = m_ses.m_disk_thread.find_job(
 						&t->filesystem(), -1, i->index);

src/smart_ban.cpp

@@ -0,0 +1,299 @@
+/*
+
+Copyright (c) 2007, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "libtorrent/pch.hpp"
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#endif
+
+#include <boost/shared_ptr.hpp>
+#include <boost/enable_shared_from_this.hpp>
+#include <boost/lexical_cast.hpp>
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+#include <vector>
+#include <map>
+#include <utility>
+#include <numeric>
+#include <cstdio>
+
+#include "libtorrent/peer_connection.hpp"
+#include "libtorrent/bt_peer_connection.hpp"
+#include "libtorrent/hasher.hpp"
+#include "libtorrent/bencode.hpp"
+#include "libtorrent/torrent.hpp"
+#include "libtorrent/extensions.hpp"
+#include "libtorrent/extensions/smart_ban.hpp"
+#include "libtorrent/alert_types.hpp"
+#include "libtorrent/disk_io_thread.hpp"
+#include "libtorrent/aux_/session_impl.hpp"
+
+namespace libtorrent { namespace
+{
+
+	struct smart_ban_plugin : torrent_plugin, boost::enable_shared_from_this<smart_ban_plugin>
+	{
+		smart_ban_plugin(torrent& t)
+			: m_torrent(t)
+			, m_salt(rand())
+		{
+		}
+
+		void on_piece_pass(int p)
+		{
+#ifdef TORRENT_LOGGING
+			(*m_torrent.session().m_logger) << time_now_string() << " PIECE PASS [ p: " << p
+				<< " | block_crc_size: " << m_block_crc.size() << " ]\n";
+#endif
+			// has this piece failed earlier? If it has, go through the
+			// CRCs from the time it failed and ban the peers that
+			// sent bad blocks
+			std::map<piece_block, block_entry>::iterator i = m_block_crc.lower_bound(piece_block(p, 0));
+			if (i == m_block_crc.end() || i->first.piece_index != p) return;
+
+			int size = m_torrent.torrent_file().piece_size(p);
+			peer_request r = {p, 0, std::min(16*1024, size)};
+			piece_block pb(p, 0);
+			while (size > 0)
+			{
+				if (i->first.block_index == pb.block_index)
+				{
+					m_torrent.filesystem().async_read(r, bind(&smart_ban_plugin::on_read_ok_block
+						, shared_from_this(), *i, _1, _2));
+					m_block_crc.erase(i++);
+				}
+				else
+				{
+					TORRENT_ASSERT(i->first.block_index > pb.block_index);
+				}
+
+				if (i == m_block_crc.end() || i->first.piece_index != p)
+					break;
+
+				r.start += 16*1024;
+				size -= 16*1024;
+				r.length = std::min(16*1024, size);
+				++pb.block_index;
+			}
+
+#ifndef NDEBUG
+			// make sure we actually removed all the entries for piece 'p'
+			i = m_block_crc.lower_bound(piece_block(p, 0));
+			TORRENT_ASSERT(i == m_block_crc.end() || i->first.piece_index != p);
+#endif
+
+			if (m_torrent.is_seed())
+			{
+				std::map<piece_block, block_entry>().swap(m_block_crc);
+				return;
+			}
+		}
+
+		void on_piece_failed(int p)
+		{
+			// The piece failed the hash check. Record
+			// the CRC and origin peer of every block
+
+			std::vector<void*> downloaders;
+			m_torrent.picker().get_downloaders(downloaders, p);
+
+			int size = m_torrent.torrent_file().piece_size(p);
+			peer_request r = {p, 0, std::min(16*1024, size)};
+			piece_block pb(p, 0);
+			for (std::vector<void*>::iterator i = downloaders.begin()
+				, end(downloaders.end()); i != end; ++i)
+			{
+				if (*i != 0)
+				{
+					m_torrent.filesystem().async_read(r, bind(&smart_ban_plugin::on_read_failed_block
+						, shared_from_this(), pb, (policy::peer*)*i, _1, _2));
+				}
+
+				r.start += 16*1024;
+				size -= 16*1024;
+				r.length = std::min(16*1024, size);
+				++pb.block_index;
+			}
+			TORRENT_ASSERT(size <= 0);
+		}
+
+	private:
+
+		// this entry ties a specific block CRC to
+		// a peer.
+		struct block_entry
+		{
+			policy::peer* peer;
+			unsigned long crc;
+		};
+
+		void on_read_failed_block(piece_block b, policy::peer* p, int ret, disk_io_job const& j)
+		{
+			TORRENT_ASSERT(p);
+			// ignore read errors
+			if (ret != j.buffer_size) return;
+
+			adler32_crc crc;
+			crc.update(j.buffer, j.buffer_size);
+			crc.update((char const*)&m_salt, sizeof(m_salt));
+
+			block_entry e = {p, crc.final()};
+
+			// since this callback is called directory from the disk io
+			// thread, the session mutex is not locked when we get here
+			aux::session_impl::mutex_t::scoped_lock l(m_torrent.session().m_mutex);
+			
+			std::map<piece_block, block_entry>::iterator i = m_block_crc.lower_bound(b);
+			if (i->first == b && i->second.peer == p)
+			{
+				// this peer has sent us this block before
+				if (i->second.crc != e.crc)
+				{
+					// this time the crc of the block is different
+					// from the first time it sent it
+					// at least one of them must be bad
+
+					// TODO: make sure p is still a valid pointer
+					if (p == 0) return;
+
+#ifdef TORRENT_LOGGING
+					char const* client = "-";
+					peer_info info;
+					if (p->connection)
+					{
+						p->connection->get_peer_info(info);
+						client = info.client.c_str();
+					}
+					(*m_torrent.session().m_logger) << time_now_string() << " BANNING PEER [ p: " << b.piece_index
+						<< " | b: " << b.block_index
+						<< " | c: " << client
+						<< " | crc1: " << i->second.crc
+						<< " | crc2: " << e.crc
+						<< " | ip: " << p->ip << " ]\n";
+#endif
+					p->banned = true;
+					if (p->connection) p->connection->disconnect();
+				}
+				// we already have this exact entry in the map
+				// we don't have to insert it
+				return;
+			}
+			
+			m_block_crc.insert(i, std::make_pair(b, e));
+
+#ifdef TORRENT_LOGGING
+			char const* client = "-";
+			peer_info info;
+			if (p->connection)
+			{
+				p->connection->get_peer_info(info);
+				client = info.client.c_str();
+			}
+			(*m_torrent.session().m_logger) << time_now_string() << " STORE BLOCK CRC [ p: " << b.piece_index
+				<< " | b: " << b.block_index
+				<< " | c: " << client
+				<< " | crc: " << e.crc
+				<< " | ip: " << p->ip << " ]\n";
+#endif
+		}
+		
+		void on_read_ok_block(std::pair<piece_block, block_entry> b, int ret, disk_io_job const& j)
+		{
+			// since this callback is called directory from the disk io
+			// thread, the session mutex is not locked when we get here
+			aux::session_impl::mutex_t::scoped_lock l(m_torrent.session().m_mutex);
+
+			// ignore read errors
+			if (ret != j.buffer_size) return;
+
+			adler32_crc crc;
+			crc.update(j.buffer, j.buffer_size);
+			crc.update((char const*)&m_salt, sizeof(m_salt));
+			unsigned long ok_crc = crc.final();
+
+			if (b.second.crc == ok_crc) return;
+
+			policy::peer* p = b.second.peer;
+
+			// TODO: make sure p is still a valid pointer
+			if (p == 0) return;
+
+#ifdef TORRENT_LOGGING
+			char const* client = "-";
+			peer_info info;
+			if (p->connection)
+			{
+				p->connection->get_peer_info(info);
+				client = info.client.c_str();
+			}
+			(*m_torrent.session().m_logger) << time_now_string() << " BANNING PEER [ p: " << b.first.piece_index
+				<< " | b: " << b.first.block_index
+				<< " | c: " << client
+				<< " | ok_crc: " << ok_crc
+				<< " | bad_crc: " << b.second.crc
+				<< " | ip: " << p->ip << " ]\n";
+#endif
+			p->banned = true;
+			if (p->connection) p->connection->disconnect();
+		}
+		
+		torrent& m_torrent;
+
+		// This table maps a piece_block (piece and block index
+		// pair) to a peer and the block CRC. The CRC is calculated
+		// from the data in the block + the salt
+		std::map<piece_block, block_entry> m_block_crc;
+
+		// This salt is a random value used to calculate the block CRCs
+		// Since the CRC function that is used is not a one way function
+		// the salt is required to avoid attacks where bad data is sent
+		// that is forged to match the CRC of the good data.
+		int m_salt;
+	};
+
+} }
+
+namespace libtorrent
+{
+
+	boost::shared_ptr<torrent_plugin> create_smart_ban_plugin(torrent* t, void*)
+	{
+		return boost::shared_ptr<torrent_plugin>(new smart_ban_plugin(*t));
+	}
+
+}
+
+

src/torrent.cpp

@@ -1030,6 +1030,18 @@ namespace libtorrent
 		std::set<void*> peers;
 		std::copy(downloaders.begin(), downloaders.end(), std::inserter(peers, peers.begin()));
 
+#ifndef NDEBUG
+		for (std::vector<void*>::iterator i = downloaders.begin()
+			, end(downloaders.end()); i != end; ++i)
+		{
+			policy::peer* p = (policy::peer*)*i;
+			if (p && p->connection)
+			{
+				p->connection->piece_failed = true;
+			}
+		}
+#endif
+
 #ifndef TORRENT_DISABLE_EXTENSIONS
 		for (extension_list_t::iterator i = m_extensions.begin()
 			, end(m_extensions.end()); i != end; ++i)
@@ -1089,6 +1101,18 @@ namespace libtorrent
 		m_storage->mark_failed(index);
 
 		TORRENT_ASSERT(m_have_pieces[index] == false);
+
+#ifndef NDEBUG
+		for (std::vector<void*>::iterator i = downloaders.begin()
+			, end(downloaders.end()); i != end; ++i)
+		{
+			policy::peer* p = (policy::peer*)*i;
+			if (p && p->connection)
+			{
+				p->connection->piece_failed = false;
+			}
+		}
+#endif
 	}
 
 	void torrent::abort()