From d07ccaf6b8881f67e113b12d41484f3da6012ebb Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Mon, 5 Jan 2009 01:08:09 +0000 Subject: [PATCH] workaround for sparse files issue on Windows vista --- ChangeLog | 1 + docs/manual.rst | 17 ++++++++++ examples/client_test.cpp | 3 +- include/libtorrent/piece_picker.hpp | 4 +++ include/libtorrent/session_settings.hpp | 16 ++++++++++ include/libtorrent/torrent.hpp | 2 ++ include/libtorrent/torrent_handle.hpp | 4 +++ src/piece_picker.cpp | 23 ++++++++++++++ src/torrent.cpp | 41 +++++++++++++++++++++++++ 9 files changed, 110 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index afd8c7d30..dd418a178 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ + * added workaround for sparse file issue on Windows Vista * added new lt_trackers extension to exchange trackers between peers * added support for BEP 17 http seeds diff --git a/docs/manual.rst b/docs/manual.rst index c3122961c..99b7ef113 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -2613,6 +2613,8 @@ It contains the following fields:: int last_scrape; bool has_incoming; + + int sparse_regions; }; ``progress`` is a value in the range [0, 1], that represents the progress of the @@ -2800,6 +2802,9 @@ If it has never done that, this value is -1. ``has_incoming`` is true if there has ever been an incoming connection attempt to this torrent.' +``sparse_regions`` the number of regions of non-downloaded pieces in the +torrent. This is an interesting metric on windows vista, since there is +a limit on the number of sparse regions in a single file there. peer_info ========= @@ -3217,6 +3222,8 @@ that will be sent to the tracker. The user-agent is a good way to identify your bool strict_super_seeding; int seeding_piece_quota; + + int max_sparse_regions; }; ``user_agent`` this is the client identification to the tracker. @@ -3526,6 +3533,16 @@ It defaults to 3 pieces, which means that when seeding, any peer we've sent more than this number of pieces to will be unchoked in favour of a choked peer. +``max_sparse_regions`` is a limit of the number of *sparse regions* in +a torrent. A sparse region is defined as a hole of pieces we have not +yet downloaded, in between pieces that have been downloaded. This is +used as a hack for windows vista which has a bug where you cannot +write files with more than a certain number of sparse regions. This +limit is not hard, it will be exceeded. Once it's exceeded, pieces +that will maintain or decrease the number of sparse regions are +prioritized. To disable this functionality, set this to 0. It defaults +to 0 on all platforms except windows. + pe_settings =========== diff --git a/examples/client_test.cpp b/examples/client_test.cpp index 2f9a42ed6..ff8d33a01 100644 --- a/examples/client_test.cpp +++ b/examples/client_test.cpp @@ -1382,7 +1382,8 @@ int main(int ac, char* av[]) out << " peers: " << esc("37") << s.num_peers << esc("0") << " (" << esc("37") << s.connect_candidates << esc("0") << ") " << "seeds: " << esc("37") << s.num_seeds << esc("0") << " " << "distributed copies: " << esc("37") << s.distributed_copies << esc("0") -// << " magnet-link: " << make_magnet_uri(h) << "\n" + << " sparse regions: " << s.sparse_regions +// << " magnet-link: " << make_magnet_uri(h) << "\n" << " download: " << esc("32") << (s.download_rate > 0 ? add_suffix(s.download_rate) + "/s ": " ") << esc("0"); boost::posix_time::time_duration t = s.next_announce; out << " next announce: " << esc("37") diff --git a/include/libtorrent/piece_picker.hpp b/include/libtorrent/piece_picker.hpp index 1cdc84bec..cd52ecc4e 100644 --- a/include/libtorrent/piece_picker.hpp +++ b/include/libtorrent/piece_picker.hpp @@ -182,6 +182,7 @@ namespace libtorrent int cursor() const { return m_cursor; } int reverse_cursor() const { return m_reverse_cursor; } + int sparse_regions() const { return m_sparse_regions; } // sets all pieces to dont-have void init(int blocks_per_piece, int total_num_blocks); @@ -523,6 +524,9 @@ namespace libtorrent // all the subsequent pieces int m_reverse_cursor; + // the number of regions of pieces we don't have. + int m_sparse_regions; + // if this is set to true, it means update_pieces() // has to be called before accessing m_pieces. mutable bool m_dirty; diff --git a/include/libtorrent/session_settings.hpp b/include/libtorrent/session_settings.hpp index 0f8979643..2782567a5 100644 --- a/include/libtorrent/session_settings.hpp +++ b/include/libtorrent/session_settings.hpp @@ -148,6 +148,11 @@ namespace libtorrent , prefer_udp_trackers(true) , strict_super_seeding(false) , seeding_piece_quota(3) +#ifdef TORRENT_WINDOWS + , max_sparse_regions(30000) +#else + , max_sparse_regions(0) +#endif {} // this is the user agent that will be sent to the tracker @@ -478,6 +483,17 @@ namespace libtorrent // the number of pieces to send to each peer when seeding // before rotating to a new peer int seeding_piece_quota; + + // the maximum number of sparse regions before starting + // to prioritize pieces close to other pieces (to maintain + // the number of sparse regions). This is set to 30000 on + // windows because windows vista has a new limit on the + // numbers of sparse regions one file may have + // if it is set to 0 this behavior is disabled + // this is a hack to avoid a terrible bug on windows + // don't use unless you have to, it screws with rarest-first + // piece selection, and reduces swarm performance + int max_sparse_regions; }; #ifndef TORRENT_DISABLE_DHT diff --git a/include/libtorrent/torrent.hpp b/include/libtorrent/torrent.hpp index b18464870..f98d51ff3 100644 --- a/include/libtorrent/torrent.hpp +++ b/include/libtorrent/torrent.hpp @@ -451,6 +451,8 @@ namespace libtorrent // -------------------------------------------- // PIECE MANAGEMENT + void update_sparse_piece_prio(int piece, int cursor, int reverse_cursor); + bool super_seeding() const { return m_super_seeding; } diff --git a/include/libtorrent/torrent_handle.hpp b/include/libtorrent/torrent_handle.hpp index d2cdd82cc..57a067ab1 100644 --- a/include/libtorrent/torrent_handle.hpp +++ b/include/libtorrent/torrent_handle.hpp @@ -120,6 +120,7 @@ namespace libtorrent , seed_rank(0) , last_scrape(0) , has_incoming(false) + , sparse_regions(0) {} enum state_t @@ -270,6 +271,9 @@ namespace libtorrent // true if there are incoming connections to this // torrent bool has_incoming; + + // the number of "holes" in the torrent + int sparse_regions; }; struct TORRENT_EXPORT block_info diff --git a/src/piece_picker.cpp b/src/piece_picker.cpp index ab30440b0..d1634f1aa 100644 --- a/src/piece_picker.cpp +++ b/src/piece_picker.cpp @@ -65,6 +65,7 @@ namespace libtorrent , m_num_have(0) , m_cursor(0) , m_reverse_cursor(0) + , m_sparse_regions(1) , m_dirty(false) { #ifdef TORRENT_PICKER_LOG @@ -1073,6 +1074,28 @@ namespace libtorrent , has_index(index)) == m_downloads.end()); if (p.have()) return; + +// maintain sparse_regions + if (index == 0) + { + if (index == m_piece_map.size() - 1 + || m_piece_map[index + 1].have()) + --m_sparse_regions; + } + else if (index == int(m_piece_map.size() - 1)) + { + if (index == 0 + || m_piece_map[index - 1].have()) + --m_sparse_regions; + } + else + { + bool have_before = m_piece_map[index-1].have(); + bool have_after = m_piece_map[index+1].have(); + if (have_after && have_before) --m_sparse_regions; + else if (!have_after && !have_before) ++m_sparse_regions; + } + if (p.filtered()) { --m_num_filtered; diff --git a/src/torrent.cpp b/src/torrent.cpp index 265531708..a7107371e 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -1547,6 +1547,19 @@ namespace libtorrent } } + void torrent::update_sparse_piece_prio(int i, int start, int end) + { + TORRENT_ASSERT(m_picker); + if (m_picker->have_piece(i) || m_picker->piece_priority(i) == 0) + return; + bool have_before = i == 0 || m_picker->have_piece(i - 1); + bool have_after = i == end - 1 || m_picker->have_piece(i + 1); + if (have_after && have_before) + m_picker->set_piece_priority(i, 7); + else if (have_after || have_before) + m_picker->set_piece_priority(i, 6); + } + void torrent::piece_passed(int index) { // INVARIANT_CHECK; @@ -1591,6 +1604,19 @@ namespace libtorrent if (p->connection) p->connection->received_valid_data(index); } + if (settings().max_sparse_regions > 0 + && m_picker->sparse_regions() > settings().max_sparse_regions) + { + // we have too many sparse regions. Prioritize pieces + // that won't introduce new sparse regions + // prioritize pieces that will reduce the number of sparse + // regions even higher + int start = m_picker->cursor(); + int end = m_picker->reverse_cursor(); + if (index > start) update_sparse_piece_prio(index - 1, start, end); + if (index < end - 1) update_sparse_piece_prio(index + 1, start, end); + } + #ifndef TORRENT_DISABLE_EXTENSIONS for (extension_list_t::iterator i = m_extensions.begin() , end(m_extensions.end()); i != end; ++i) @@ -4654,6 +4680,20 @@ namespace libtorrent if (m_time_scaler <= 0) { m_time_scaler = 10; + + if (settings().max_sparse_regions > 0 + && m_picker + && m_picker->sparse_regions() > settings().max_sparse_regions) + { + // we have too many sparse regions. Prioritize pieces + // that won't introduce new sparse regions + // prioritize pieces that will reduce the number of sparse + // regions even higher + int start = m_picker->cursor(); + int end = m_picker->reverse_cursor(); + for (int i = start; i < end; ++i) + update_sparse_piece_prio(i, start, end); + } m_policy.pulse(); } } @@ -5059,6 +5099,7 @@ namespace libtorrent if (has_picker()) { + st.sparse_regions = m_picker->sparse_regions(); int num_pieces = m_picker->num_pieces(); st.pieces.resize(num_pieces, false); for (int i = 0; i < num_pieces; ++i)