rewrote most of the web seed downloader to report its progress accurately (to prevent inconsistencies in the core). Not well tested with multi file torrents yet

include/libtorrent/http_tracker_connection.hpp

@@ -72,11 +72,13 @@ namespace libtorrent
 		std::string const& protocol() const { return m_protocol; }
 		int status_code() const { return m_status_code; }
 		std::string message() const { return m_server_message; }
-		buffer::const_interval get_body();
+		buffer::const_interval get_body() const;
 		bool header_finished() const { return m_state == read_body; }
 		bool finished() const { return m_finished; }
 		boost::tuple<int, int> incoming(buffer::const_interval recv_buffer);
 		int body_start() const { return m_body_start_pos; }
+
+		void reset();
 	private:
 		int m_recv_pos;
 		int m_status_code;

src/http_tracker_connection.cpp

@@ -98,9 +98,13 @@ namespace libtorrent
 
 	boost::tuple<int, int> http_parser::incoming(buffer::const_interval recv_buffer)
 	{
-		m_recv_buffer = recv_buffer;
+		assert(recv_buffer.left() >= m_recv_buffer.left());
 		boost::tuple<int, int> ret(0, 0);
 
+		// early exit if there's nothing new in the receive buffer
+		if (recv_buffer.left() == m_recv_buffer.left()) return ret;
+		m_recv_buffer = recv_buffer;
+
 		char const* pos = recv_buffer.begin + m_recv_pos;
 		if (m_state == read_status)
 		{
@@ -200,20 +204,29 @@ namespace libtorrent
 		return ret;
 	}
 	
-	buffer::const_interval http_parser::get_body()
+	buffer::const_interval http_parser::get_body() const
 	{
-		char const* body_begin = m_recv_buffer.begin + m_body_start_pos;
+		assert(m_state == read_body);
-		char const* body_end = m_recv_buffer.begin + m_recv_pos;
+		if (m_content_length >= 0)
+			return buffer::const_interval(m_recv_buffer.begin + m_body_start_pos
+				, m_recv_buffer.begin + std::min(m_recv_pos
+				, m_body_start_pos + m_content_length));
+		else
+			return buffer::const_interval(m_recv_buffer.begin + m_body_start_pos
+				, m_recv_buffer.begin + m_recv_pos);
+	}
 	
+	void http_parser::reset()
+	{
 		m_recv_pos = 0;
 		m_body_start_pos = 0;
 		m_status_code = -1;
 		m_content_length = -1;
 		m_finished = false;
 		m_state = read_status;
+		m_recv_buffer.begin = 0;
+		m_recv_buffer.end = 0;
 		m_header.clear();
-		
-		return buffer::const_interval(body_begin, body_end);
 	}
 	
 	http_tracker_connection::http_tracker_connection(

src/peer_connection.cpp

@@ -1088,12 +1088,6 @@ namespace libtorrent
 		bool was_finished = picker.num_filtered() + t->num_pieces()
 			== t->torrent_file().num_pieces();
 
-		for (std::vector<piece_block>::iterator i = finished_blocks.begin()
-			, end(finished_blocks.end()); i != end; ++i)
-		{
-
-		}
-
 		// did we just finish the piece?
 		if (picker.is_piece_finished(p.piece))
 		{
@@ -1134,30 +1128,7 @@ namespace libtorrent
 				t->completed();
 			}
 
-#ifndef NDEBUG
-
-			size_type total_done, total_wanted;
-			boost::tie(total_done, total_wanted) = t->bytes_done();
-			if (t->is_seed())
-				assert(total_done == t->torrent_file().total_size());
-			else
-				assert(total_done != t->torrent_file().total_size());
-				
-			t->check_invariant();
-#endif
-
 		}
-#ifndef NDEBUG
-
-		size_type total_done, total_wanted;
-		boost::tie(total_done, total_wanted) = t->bytes_done();
-		if (t->is_seed())
-			assert(total_done == t->torrent_file().total_size());
-		else
-			assert(total_done != t->torrent_file().total_size());
-			
-		t->check_invariant();
-#endif
 	}
 
 	// -----------------------------

src/torrent.cpp

@@ -765,8 +765,8 @@ namespace libtorrent
 				wanted_done += corr;
 		}
 
-		assert(total_done <= m_torrent_file.total_size());
+		assert(total_done < m_torrent_file.total_size());
-		assert(wanted_done <= m_torrent_file.total_size());
+		assert(wanted_done < m_torrent_file.total_size());
 
 		std::map<piece_block, int> downloading_piece;
 		for (const_peer_iterator i = begin(); i != end(); ++i)
@@ -794,7 +794,15 @@ namespace libtorrent
 				{
 					downloading_piece[block] = p->bytes_downloaded;
 				}
+#ifndef NDEBUG
 				assert(p->bytes_downloaded <= p->full_block_bytes);
+				int last_piece = m_torrent_file.num_pieces() - 1;
+				if (p->piece_index == last_piece
+					&& p->block_index == m_torrent_file.piece_size(last_piece) / block_size())
+					assert(p->full_block_bytes == m_torrent_file.piece_size(last_piece) % block_size());
+				else
+					assert(p->full_block_bytes == block_size());
+#endif
 			}
 		}
 		for (std::map<piece_block, int>::iterator i = downloading_piece.begin();
@@ -805,8 +813,43 @@ namespace libtorrent
 				wanted_done += i->second;
 		}
 
-		assert(total_done <= m_torrent_file.total_size());
+#ifndef NDEBUG
-		assert(wanted_done <= m_torrent_file.total_size());
+
+		if (total_done >= m_torrent_file.total_size())
+		{
+			std::copy(m_have_pieces.begin(), m_have_pieces.end()
+				, std::ostream_iterator<bool>(std::cerr, " "));
+			std::cerr << std::endl;
+			std::cerr << "num_pieces: " << m_num_pieces << std::endl;
+			
+			std::cerr << "unfinished:" << std::endl;
+			
+			for (std::vector<piece_picker::downloading_piece>::const_iterator i =
+				dl_queue.begin(); i != dl_queue.end(); ++i)
+			{
+				std::cerr << "   " << i->index << " ";
+				for (int j = 0; j < blocks_per_piece; ++j)
+				{
+					std::cerr << i->finished_blocks[j];
+				}
+				std::cerr << std::endl;
+			}
+			
+			std::cerr << "downloading pieces:" << std::endl;
+
+			for (std::map<piece_block, int>::iterator i = downloading_piece.begin();
+				i != downloading_piece.end(); ++i)
+			{
+				std::cerr << "   " << i->first.piece_index << ":" << i->first.block_index
+					<< "  " << i->second << std::endl;
+			}
+
+		}
+
+		assert(total_done < m_torrent_file.total_size());
+		assert(wanted_done < m_torrent_file.total_size());
+
+#endif
 
 		assert(total_done >= wanted_done);
 		return make_tuple(total_done, wanted_done);
@@ -1726,8 +1769,8 @@ namespace libtorrent
 			assert(total_done != m_torrent_file.total_size());
 
 // This check is very expensive.
-//		assert(m_num_pieces
+		assert(m_num_pieces
-//			== std::count(m_have_pieces.begin(), m_have_pieces.end(), true));
+			== std::count(m_have_pieces.begin(), m_have_pieces.end(), true));
 		assert(m_priority >= 0.f && m_priority < 1.f);
 		assert(!valid_metadata() || m_block_size > 0);
 		assert(!valid_metadata() || (m_torrent_file.piece_length() % m_block_size) == 0);

src/web_peer_connection.cpp

@@ -107,15 +107,17 @@ namespace libtorrent
 		boost::shared_ptr<torrent> t = associated_torrent().lock();
 		assert(t);
 
-		int body_start = m_parser.body_start();
+		buffer::const_interval http_body = m_parser.get_body();
-		buffer::const_interval recv_buffer = receive_buffer();
-		assert(body_start <= recv_buffer.left());
 		piece_block_progress ret;
 
 		ret.piece_index = m_requests.front().piece;
-		ret.block_index = m_requests.front().start / t->block_size();
+		ret.bytes_downloaded = http_body.left() % t->block_size();
-		ret.bytes_downloaded = recv_buffer.left() - body_start;
+		ret.block_index = (m_requests.front().start + ret.bytes_downloaded) / t->block_size();
-		ret.full_block_bytes = m_requests.front().length;
+		ret.full_block_bytes = t->block_size();
+		const int last_piece = t->torrent_file().num_pieces() - 1;
+		if (ret.piece_index == last_piece && ret.block_index
+			== t->torrent_file().piece_size(last_piece) / t->block_size())
+			ret.full_block_bytes = t->torrent_file().piece_size(last_piece) % t->block_size();
 		return ret;
 	}
 
@@ -150,7 +152,16 @@ namespace libtorrent
 		
 		std::string request;
 
-		m_requests.push_back(r);
+		int size = r.length;
+		const int block_size = t->block_size();
+		while (size > 0)
+		{
+			int request_size = std::min(block_size, size);
+			peer_request pr = {r.piece, r.start + r.length - size
+				,  request_size};
+			m_requests.push_back(pr);
+			size -= request_size;
+		}
 
 		bool using_proxy = false;
 		if (!m_ses.settings().proxy_ip.empty())
@@ -281,7 +292,9 @@ namespace libtorrent
 				throw std::runtime_error("HTTP server does not support byte range requests");
 			}
 
-			if (!m_parser.finished()) break;
+			if (!m_parser.header_finished()) break;
+
+			buffer::const_interval http_body = m_parser.get_body();
 
 			std::string server_version = m_parser.header<std::string>("server");
 			if (!server_version.empty())
@@ -314,21 +327,69 @@ namespace libtorrent
 				throw std::runtime_error("unexpected HTTP response");
 
 			int file_index = m_file_requests.front();
-			m_file_requests.pop_front();
+			peer_request in_range = info.map_file(file_index, range_start
-
-			peer_request r = info.map_file(file_index, range_start
 				, range_end - range_start);
 
-			buffer::const_interval http_body = m_parser.get_body();
+			peer_request front_request = m_requests.front();
-
+			if (in_range.piece != front_request.piece
-			if (r == m_requests.front())
+				|| in_range.start > front_request.start)
 			{
-				m_requests.pop_front();
+				throw std::runtime_error("invalid range in HTTP response");
-				incoming_piece(r, http_body.begin);
-				cut_receive_buffer(http_body.end - recv_buffer.begin, 512*1024+1024);
-				return;
 			}
 
+			// skip the http header and the blocks we've already read. The
+			// http_body.begin is now in sync with the request at the front
+			// of the request queue
+			assert(in_range.start <= front_request.start);
+			http_body.begin += front_request.start - in_range.start;
+
+			if ((in_range.start + in_range.length - front_request.start
+				< front_request.length)
+				&& (http_body.left() > front_request.length - m_piece.size()))
+			{
+				// the start of the next block to receive is stored
+				// in m_piece. We need to append the rest of that
+				// block from the http receive buffer and then
+				// (if it completed) call incoming_piece() with
+				// m_piece as buffer.
+				
+				m_piece.reserve(info.piece_length());
+				char const* start = http_body.begin;
+				int copy_size = std::min(front_request.length - int(m_piece.size())
+					, http_body.left());
+				std::copy(start, start + copy_size, std::back_inserter(m_piece));
+				http_body.begin += copy_size;
+				if (m_piece.size() == front_request.length)
+				{
+					m_requests.pop_front();
+					incoming_piece(front_request, &m_piece[0]);
+					m_piece.clear();
+				}
+			}
+
+			// report all received blocks to the bittorrent engine
+			if (m_piece.empty())
+			{
+				while (!m_requests.empty()
+					&& http_body.left() >= m_requests.front().length)
+				{
+					peer_request r = m_requests.front();
+					m_requests.pop_front();
+					incoming_piece(r, http_body.begin);
+					http_body.begin += r.length;
+				}
+			}
+
+			if (m_parser.finished())
+			{
+				m_file_requests.pop_front();
+				assert(http_body.left() == 0);
+				m_parser.reset();
+				cut_receive_buffer(http_body.end - recv_buffer.begin, 512*1024+1024);
+				continue;
+			}
+			break;
+/*
 			if (!m_piece.empty())
 			{
 				// this is not the first partial request we get
@@ -368,6 +429,7 @@ namespace libtorrent
 			}
 
 			cut_receive_buffer(http_body.end - recv_buffer.begin, 512*1024+1024);
+*/
 		}
 	}