From 3a7eeb39660bb149b7191302c704dd4c0ee60bce Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Mon, 18 Dec 2006 01:23:30 +0000 Subject: [PATCH] rewrote most of the web seed downloader to report its progress accurately (to prevent inconsistencies in the core). Not well tested with multi file torrents yet --- .../libtorrent/http_tracker_connection.hpp | 4 +- src/http_tracker_connection.cpp | 27 +++-- src/peer_connection.cpp | 29 ------ src/torrent.cpp | 55 +++++++++-- src/web_peer_connection.cpp | 98 +++++++++++++++---- 5 files changed, 152 insertions(+), 61 deletions(-) diff --git a/include/libtorrent/http_tracker_connection.hpp b/include/libtorrent/http_tracker_connection.hpp index 9dffbe049..db71237f9 100755 --- a/include/libtorrent/http_tracker_connection.hpp +++ b/include/libtorrent/http_tracker_connection.hpp @@ -72,11 +72,13 @@ namespace libtorrent std::string const& protocol() const { return m_protocol; } int status_code() const { return m_status_code; } std::string message() const { return m_server_message; } - buffer::const_interval get_body(); + buffer::const_interval get_body() const; bool header_finished() const { return m_state == read_body; } bool finished() const { return m_finished; } boost::tuple incoming(buffer::const_interval recv_buffer); int body_start() const { return m_body_start_pos; } + + void reset(); private: int m_recv_pos; int m_status_code; diff --git a/src/http_tracker_connection.cpp b/src/http_tracker_connection.cpp index fb90ab67b..3a1246d6d 100755 --- a/src/http_tracker_connection.cpp +++ b/src/http_tracker_connection.cpp @@ -98,9 +98,13 @@ namespace libtorrent boost::tuple http_parser::incoming(buffer::const_interval recv_buffer) { - m_recv_buffer = recv_buffer; + assert(recv_buffer.left() >= m_recv_buffer.left()); boost::tuple ret(0, 0); + // early exit if there's nothing new in the receive buffer + if (recv_buffer.left() == m_recv_buffer.left()) return ret; + m_recv_buffer = recv_buffer; + char const* pos = recv_buffer.begin + m_recv_pos; if (m_state == read_status) { @@ -200,20 +204,29 @@ namespace libtorrent return ret; } - buffer::const_interval http_parser::get_body() + buffer::const_interval http_parser::get_body() const + { + assert(m_state == read_body); + if (m_content_length >= 0) + return buffer::const_interval(m_recv_buffer.begin + m_body_start_pos + , m_recv_buffer.begin + std::min(m_recv_pos + , m_body_start_pos + m_content_length)); + else + return buffer::const_interval(m_recv_buffer.begin + m_body_start_pos + , m_recv_buffer.begin + m_recv_pos); + } + + void http_parser::reset() { - char const* body_begin = m_recv_buffer.begin + m_body_start_pos; - char const* body_end = m_recv_buffer.begin + m_recv_pos; - m_recv_pos = 0; m_body_start_pos = 0; m_status_code = -1; m_content_length = -1; m_finished = false; m_state = read_status; + m_recv_buffer.begin = 0; + m_recv_buffer.end = 0; m_header.clear(); - - return buffer::const_interval(body_begin, body_end); } http_tracker_connection::http_tracker_connection( diff --git a/src/peer_connection.cpp b/src/peer_connection.cpp index 2004c2b1f..941e15a6a 100755 --- a/src/peer_connection.cpp +++ b/src/peer_connection.cpp @@ -1088,12 +1088,6 @@ namespace libtorrent bool was_finished = picker.num_filtered() + t->num_pieces() == t->torrent_file().num_pieces(); - for (std::vector::iterator i = finished_blocks.begin() - , end(finished_blocks.end()); i != end; ++i) - { - - } - // did we just finish the piece? if (picker.is_piece_finished(p.piece)) { @@ -1134,30 +1128,7 @@ namespace libtorrent t->completed(); } -#ifndef NDEBUG - - size_type total_done, total_wanted; - boost::tie(total_done, total_wanted) = t->bytes_done(); - if (t->is_seed()) - assert(total_done == t->torrent_file().total_size()); - else - assert(total_done != t->torrent_file().total_size()); - - t->check_invariant(); -#endif - } -#ifndef NDEBUG - - size_type total_done, total_wanted; - boost::tie(total_done, total_wanted) = t->bytes_done(); - if (t->is_seed()) - assert(total_done == t->torrent_file().total_size()); - else - assert(total_done != t->torrent_file().total_size()); - - t->check_invariant(); -#endif } // ----------------------------- diff --git a/src/torrent.cpp b/src/torrent.cpp index 002216e3d..72ddba0e2 100755 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -765,8 +765,8 @@ namespace libtorrent wanted_done += corr; } - assert(total_done <= m_torrent_file.total_size()); - assert(wanted_done <= m_torrent_file.total_size()); + assert(total_done < m_torrent_file.total_size()); + assert(wanted_done < m_torrent_file.total_size()); std::map downloading_piece; for (const_peer_iterator i = begin(); i != end(); ++i) @@ -794,7 +794,15 @@ namespace libtorrent { downloading_piece[block] = p->bytes_downloaded; } +#ifndef NDEBUG assert(p->bytes_downloaded <= p->full_block_bytes); + int last_piece = m_torrent_file.num_pieces() - 1; + if (p->piece_index == last_piece + && p->block_index == m_torrent_file.piece_size(last_piece) / block_size()) + assert(p->full_block_bytes == m_torrent_file.piece_size(last_piece) % block_size()); + else + assert(p->full_block_bytes == block_size()); +#endif } } for (std::map::iterator i = downloading_piece.begin(); @@ -805,8 +813,43 @@ namespace libtorrent wanted_done += i->second; } - assert(total_done <= m_torrent_file.total_size()); - assert(wanted_done <= m_torrent_file.total_size()); +#ifndef NDEBUG + + if (total_done >= m_torrent_file.total_size()) + { + std::copy(m_have_pieces.begin(), m_have_pieces.end() + , std::ostream_iterator(std::cerr, " ")); + std::cerr << std::endl; + std::cerr << "num_pieces: " << m_num_pieces << std::endl; + + std::cerr << "unfinished:" << std::endl; + + for (std::vector::const_iterator i = + dl_queue.begin(); i != dl_queue.end(); ++i) + { + std::cerr << " " << i->index << " "; + for (int j = 0; j < blocks_per_piece; ++j) + { + std::cerr << i->finished_blocks[j]; + } + std::cerr << std::endl; + } + + std::cerr << "downloading pieces:" << std::endl; + + for (std::map::iterator i = downloading_piece.begin(); + i != downloading_piece.end(); ++i) + { + std::cerr << " " << i->first.piece_index << ":" << i->first.block_index + << " " << i->second << std::endl; + } + + } + + assert(total_done < m_torrent_file.total_size()); + assert(wanted_done < m_torrent_file.total_size()); + +#endif assert(total_done >= wanted_done); return make_tuple(total_done, wanted_done); @@ -1726,8 +1769,8 @@ namespace libtorrent assert(total_done != m_torrent_file.total_size()); // This check is very expensive. -// assert(m_num_pieces -// == std::count(m_have_pieces.begin(), m_have_pieces.end(), true)); + assert(m_num_pieces + == std::count(m_have_pieces.begin(), m_have_pieces.end(), true)); assert(m_priority >= 0.f && m_priority < 1.f); assert(!valid_metadata() || m_block_size > 0); assert(!valid_metadata() || (m_torrent_file.piece_length() % m_block_size) == 0); diff --git a/src/web_peer_connection.cpp b/src/web_peer_connection.cpp index e9a10d205..082fb8cb4 100755 --- a/src/web_peer_connection.cpp +++ b/src/web_peer_connection.cpp @@ -107,15 +107,17 @@ namespace libtorrent boost::shared_ptr t = associated_torrent().lock(); assert(t); - int body_start = m_parser.body_start(); - buffer::const_interval recv_buffer = receive_buffer(); - assert(body_start <= recv_buffer.left()); + buffer::const_interval http_body = m_parser.get_body(); piece_block_progress ret; ret.piece_index = m_requests.front().piece; - ret.block_index = m_requests.front().start / t->block_size(); - ret.bytes_downloaded = recv_buffer.left() - body_start; - ret.full_block_bytes = m_requests.front().length; + ret.bytes_downloaded = http_body.left() % t->block_size(); + ret.block_index = (m_requests.front().start + ret.bytes_downloaded) / t->block_size(); + ret.full_block_bytes = t->block_size(); + const int last_piece = t->torrent_file().num_pieces() - 1; + if (ret.piece_index == last_piece && ret.block_index + == t->torrent_file().piece_size(last_piece) / t->block_size()) + ret.full_block_bytes = t->torrent_file().piece_size(last_piece) % t->block_size(); return ret; } @@ -150,7 +152,16 @@ namespace libtorrent std::string request; - m_requests.push_back(r); + int size = r.length; + const int block_size = t->block_size(); + while (size > 0) + { + int request_size = std::min(block_size, size); + peer_request pr = {r.piece, r.start + r.length - size + , request_size}; + m_requests.push_back(pr); + size -= request_size; + } bool using_proxy = false; if (!m_ses.settings().proxy_ip.empty()) @@ -281,7 +292,9 @@ namespace libtorrent throw std::runtime_error("HTTP server does not support byte range requests"); } - if (!m_parser.finished()) break; + if (!m_parser.header_finished()) break; + + buffer::const_interval http_body = m_parser.get_body(); std::string server_version = m_parser.header("server"); if (!server_version.empty()) @@ -314,21 +327,69 @@ namespace libtorrent throw std::runtime_error("unexpected HTTP response"); int file_index = m_file_requests.front(); - m_file_requests.pop_front(); - - peer_request r = info.map_file(file_index, range_start + peer_request in_range = info.map_file(file_index, range_start , range_end - range_start); - buffer::const_interval http_body = m_parser.get_body(); - - if (r == m_requests.front()) + peer_request front_request = m_requests.front(); + if (in_range.piece != front_request.piece + || in_range.start > front_request.start) { - m_requests.pop_front(); - incoming_piece(r, http_body.begin); - cut_receive_buffer(http_body.end - recv_buffer.begin, 512*1024+1024); - return; + throw std::runtime_error("invalid range in HTTP response"); } + // skip the http header and the blocks we've already read. The + // http_body.begin is now in sync with the request at the front + // of the request queue + assert(in_range.start <= front_request.start); + http_body.begin += front_request.start - in_range.start; + + if ((in_range.start + in_range.length - front_request.start + < front_request.length) + && (http_body.left() > front_request.length - m_piece.size())) + { + // the start of the next block to receive is stored + // in m_piece. We need to append the rest of that + // block from the http receive buffer and then + // (if it completed) call incoming_piece() with + // m_piece as buffer. + + m_piece.reserve(info.piece_length()); + char const* start = http_body.begin; + int copy_size = std::min(front_request.length - int(m_piece.size()) + , http_body.left()); + std::copy(start, start + copy_size, std::back_inserter(m_piece)); + http_body.begin += copy_size; + if (m_piece.size() == front_request.length) + { + m_requests.pop_front(); + incoming_piece(front_request, &m_piece[0]); + m_piece.clear(); + } + } + + // report all received blocks to the bittorrent engine + if (m_piece.empty()) + { + while (!m_requests.empty() + && http_body.left() >= m_requests.front().length) + { + peer_request r = m_requests.front(); + m_requests.pop_front(); + incoming_piece(r, http_body.begin); + http_body.begin += r.length; + } + } + + if (m_parser.finished()) + { + m_file_requests.pop_front(); + assert(http_body.left() == 0); + m_parser.reset(); + cut_receive_buffer(http_body.end - recv_buffer.begin, 512*1024+1024); + continue; + } + break; +/* if (!m_piece.empty()) { // this is not the first partial request we get @@ -368,6 +429,7 @@ namespace libtorrent } cut_receive_buffer(http_body.end - recv_buffer.begin, 512*1024+1024); +*/ } }