improve web seed hash failure mode

ChangeLog

@@ -1,3 +1,4 @@
+	* improve web seed hash failure case
 	* improve DHT lookup times
 	* uTP path MTU discovery improvements
 	* optimized the torrent creator optimizer to scale significantly better with more files

docs/make_torrent.rst

@@ -173,12 +173,29 @@ file structure. Its synopsis::
 		int piece_length() const;
 		int piece_size(int index) const;
 
+		// index accessors
 		sha1_hash const& hash(int index) const;
 		std::string const& symlink(int index) const;
 		time_t mtime(int index) const;
 		int file_index(int index) const;
 		size_type file_base(int index) const;
 		void set_file_base(int index, size_type off);
+		std::string file_path(int index) const;
+		std::string file_name(int index) const;
+		size_type file_size(int index) const;
+		size_type file_offset(int index) const;
+
+		// iterator accessors
+		sha1_hash hash(internal_file_entry const& fe) const;
+		std::string const& symlink(internal_file_entry const& fe) const;
+		time_t mtime(internal_file_entry const& fe) const;
+		int file_index(internal_file_entry const& fe) const;
+		size_type file_base(internal_file_entry const& fe) const;
+		void set_file_base(internal_file_entry const& fe, size_type off);
+		std::string file_path(internal_file_entry const& fe) const;
+		std::string file_name(internal_file_entry const& fe) const;
+		size_type file_size(internal_file_entry const& fe) const;
+		size_type file_offset(internal_file_entry const& fe) const;
 
 		void set_name(std::string const& n);
 		void set_name(std::wstring const& n);

include/libtorrent/file_storage.hpp

@@ -255,6 +255,7 @@ namespace libtorrent
 		std::string file_path(int index) const;
 		std::string file_name(int index) const;
 		size_type file_size(int index) const;
+		size_type file_offset(int index) const;
 
 		sha1_hash hash(internal_file_entry const& fe) const;
 		std::string const& symlink(internal_file_entry const& fe) const;
@@ -265,6 +266,7 @@ namespace libtorrent
 		std::string file_path(internal_file_entry const& fe) const;
 		std::string file_name(internal_file_entry const& fe) const;
 		size_type file_size(internal_file_entry const& fe) const;
+		size_type file_offset(internal_file_entry const& fe) const;
 
 #if !defined TORRENT_VERBOSE_LOGGING \
 	&& !defined TORRENT_LOGGING \

include/libtorrent/peer_connection.hpp

@@ -388,8 +388,10 @@ namespace libtorrent
 		void add_free_upload(size_type free_upload);
 
 		// trust management.
-		void received_valid_data(int index);
-		void received_invalid_data(int index);
+		virtual void received_valid_data(int index);
+		// returns false if the peer should not be
+		// disconnected
+		virtual bool received_invalid_data(int index, bool single_peer);
 
 		size_type share_diff() const;
 

include/libtorrent/web_peer_connection.hpp

@@ -103,6 +103,8 @@ namespace libtorrent
 
 		void write_request(peer_request const& r);
 
+		virtual bool received_invalid_data(int index, bool single_peer);
+
 	private:
 
 		bool maybe_harvest_block();

src/file_storage.cpp

@@ -439,6 +439,12 @@ namespace libtorrent
 		return m_files[index].size;
 	}
 
+	size_type file_storage::file_offset(int index) const
+	{
+		TORRENT_ASSERT(index >= 0 && index < int(m_files.size()));
+		return m_files[index].offset;
+	}
+
 	sha1_hash file_storage::hash(internal_file_entry const& fe) const
 	{
 		int index = &fe - &m_files[0];
@@ -498,6 +504,11 @@ namespace libtorrent
 		return fe.size;
 	}
 
+	size_type file_storage::file_offset(internal_file_entry const& fe) const
+	{
+		return fe.offset;
+	}
+
 	bool compare_file_entry_size(internal_file_entry const& fe1, internal_file_entry const& fe2)
 	{ return fe1.size < fe2.size; }
 

src/peer_connection.cpp

@@ -933,7 +933,7 @@ namespace libtorrent
 #endif
 	}
 
-	void peer_connection::received_invalid_data(int index)
+	bool peer_connection::received_invalid_data(int index, bool single_peer)
 	{
 		INVARIANT_CHECK;
 
@@ -946,6 +946,7 @@ namespace libtorrent
 			} TORRENT_CATCH(std::exception&) {}
 		}
 #endif
+		return true;
 	}
 	
 	size_type peer_connection::total_free_upload() const

src/torrent.cpp

@@ -3398,16 +3398,24 @@ namespace libtorrent
 		}
 #endif
 
+		// did we receive this piece from a single peer?
+		bool single_peer = peers.size() == 1;
+
 		for (std::set<void*>::iterator i = peers.begin()
 			, end(peers.end()); i != end; ++i)
 		{
 			policy::peer* p = static_cast<policy::peer*>(*i);
 			if (p == 0) continue;
 			TORRENT_ASSERT(p->in_use);
+			bool allow_disconnect = true;
 			if (p->connection)
 			{
 				TORRENT_ASSERT(p->connection->m_in_use == 1337);
-				p->connection->received_invalid_data(index);
+
+				// the peer implementation can ask not to be disconnected.
+				// this is used for web seeds for instance, to instead of
+				// disconnecting, mark the file as not being haved.
+				allow_disconnect = p->connection->received_invalid_data(index, single_peer);
 			}
 
 			if (m_ses.settings().use_parole_mode)
@@ -3427,8 +3435,10 @@ namespace libtorrent
 
 			// either, we have received too many failed hashes
 			// or this was the only peer that sent us this piece.
+			// if we have failed more than 3 pieces from this peer,
+			// don't trust it regardless.
 			if (p->trust_points <= -7
-				|| peers.size() == 1)
+				|| (single_peer && allow_disconnect))
 			{
 				// we don't trust this peer anymore
 				// ban it.

src/web_peer_connection.cpp

@@ -315,6 +315,51 @@ namespace libtorrent
 		return true;
 	}
 
+	bool web_peer_connection::received_invalid_data(int index, bool single_peer)
+	{
+		if (!single_peer) return peer_connection::received_invalid_data(index, single_peer);
+
+		// when a web seed fails a hash check, do the following:
+		// 1. if the whole piece only overlaps a single file, mark that file as not
+		//    have for this peer
+		// 2. if the piece overlaps more than one file, mark the piece as not have
+		//    for this peer
+		// 3. if it's a single file torrent, just ban it right away
+		// this handles the case where web seeds may have some files updated but not other
+
+		boost::shared_ptr<torrent> t = associated_torrent().lock();
+		file_storage const& fs = t->torrent_file().files();
+
+		// single file torrent
+		if (fs.num_files() == 1) return peer_connection::received_invalid_data(index, single_peer);
+
+		std::vector<file_slice> files = fs.map_block(index, 0, fs.piece_size(index));
+
+		if (files.size() == 1)
+		{
+			// assume the web seed has a different copy of this specific file
+			// than what we expect, and pretend not to have it.
+			int fi = files[0].file_index;
+			int first_piece = fs.file_offset(fi) / fs.piece_length();
+			// one past last piece
+			int end_piece = (fs.file_offset(fi) + fs.file_size(fi) + 1) / fs.piece_length();
+			for (int i = first_piece; i < end_piece; ++i)
+				incoming_dont_have(i);
+		}
+		else
+		{
+			incoming_dont_have(index);
+		}
+
+		peer_connection::received_invalid_data(index, single_peer);
+
+		// if we don't think we have any of the files, allow banning the web seed
+		if (num_have_pieces() == 0) return true;
+
+		// don't disconnect, we won't request anything from this file again
+		return false;
+	}
+
 	void web_peer_connection::on_receive(error_code const& error
 		, std::size_t bytes_transferred)
 	{