From f09774607f3c0d01496df962d50667bc60cb254c Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Thu, 18 Jun 2009 22:32:55 +0000 Subject: [PATCH] added upload mode --- ChangeLog | 3 +- docs/manual.rst | 78 ++++++++++++++++++++++--- include/libtorrent/peer_connection.hpp | 8 ++- include/libtorrent/session.hpp | 2 + include/libtorrent/session_settings.hpp | 12 ++-- include/libtorrent/torrent.hpp | 9 +++ include/libtorrent/torrent_handle.hpp | 7 +++ src/peer_connection.cpp | 56 +++++++++++++++++- src/policy.cpp | 3 +- src/torrent.cpp | 62 +++++++++++++++----- src/torrent_handle.cpp | 6 ++ test/test_transfer.cpp | 17 +----- 12 files changed, 209 insertions(+), 54 deletions(-) diff --git a/ChangeLog b/ChangeLog index 60770eb9a..b877f914f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,5 @@ - * by default, all piece priorities are set to 0 when a write fails. + * introduced an upload mode, which torrents are switched into when + it hits a disk write error, instead of stopping the torrent. this lets libtorrent keep uploading the parts it has when it encounters a disk-full error for instance * improved disk error handling and expanded use of error_code in diff --git a/docs/manual.rst b/docs/manual.rst index b8c137d08..86f216f45 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -38,6 +38,34 @@ For a description on how to create torrent files, see make_torrent_. .. _make_torrent: make_torrent.html +things to keep in mind +====================== + +A common problem developers are facing is torrents stopping without explanation. +Here is a description on which conditions libtorrent will stop your torrents, +how to find out about it and what to do about it. + +Make sure to keep track of the paused state, the error state and the upload +mode of your torrents. By default, torrents are auto-managed, which means +libtorrent will pause them, unpause them, scrape them and take them out +of upload-mode automatically. + +Whenever a torrent encounters a fatal error, it will be stopped, and the +``session_status::error`` will describe the error that caused it. If a torrent +is auto managed, it is scraped periodically and paused or resumed based on +the number of downloaders per seed. This will effectively seed torrents that +are in the greatest need of seeds. + +If a torrent hits a disk write error, it will be put into upload mode. This +means it will not download anything, but only upload. The assumption is that +the write error is caused by a full disk or write permission errors. If the +torrent is auto-managed, it will periodically be taken out of the upload +mode, trying to write things to the disk again. This means torrent will recover +from certain disk errors if the problem is resolved. If the torrent is not +auto managed, you have to call `set_upload_mode()`_ to turn +downloading back on again. + + network primitives ================== @@ -306,6 +334,7 @@ add_torrent() void* userdata; bool seed_mode; bool override_resume_data; + bool upload_mode; }; torrent_handle add_torrent(add_torrent_params const& params); @@ -413,6 +442,11 @@ If ``override_resume_data`` is set to true, the ``paused`` and ``auto_managed`` state of the torrent are not loaded from the resume data, but the states requested by this ``add_torrent_params`` will override it. +If ``upload_mode`` is set to true, the torrent will be initialized in upload-mode, +which means it will not make any piece requests. This state is typically entered +on disk I/O errors, and if the torrent is also auto managed, it will be taken out +of this state periodically. This mode can be used to avoid race conditions when +adjusting priorities of pieces before allowing the torrent to start downloading. remove_torrent() ---------------- @@ -1841,6 +1875,7 @@ Its declaration looks like this:: bool is_seed() const; void force_recheck() const; void clear_error() const; + void set_upload_mode(bool m) const; void resolve_countries(bool r); bool resolve_countries() const; @@ -2273,6 +2308,23 @@ clear_error() If the torrent is in an error state (i.e. ``torrent_status::error`` is non-empty), this will clear the error and start the torrent again. +set_upload_mode() +----------------- + +:: + + void set_upload_mode(bool m) const; + +Explicitly sets the upload mode of the torrent. In upload mode, the torrent will not +request any pieces. If the torrent is auto managed, it will automatically be taken out +of upload mode periodically (see ``session_settings::optimistic_disk_retry``). Torrents +are automatically put in upload mode whenever they encounter a disk write error. + +``m`` should be true to enter upload mode, and false to leave it. + +To test if a torrent is in upload mode, call ``torrent_handle::status()`` and inspect +``torrent_status::upload_mode``. + resolve_countries() ------------------- @@ -2776,6 +2828,8 @@ It contains the following fields:: int sparse_regions; bool seed_mode; + + bool upload_mode; }; ``progress`` is a value in the range [0, 1], that represents the progress of the @@ -2974,6 +3028,13 @@ a limit on the number of sparse regions in a single file there. started in seed mode, it will leave seed mode once all pieces have been checked or as soon as one piece fails the hash check. +``upload_mode`` is true if the torrent is blocked from downloading. This +typically happens when a disk write operation fails. If the torrent is +auto-managed, it will periodically be taken out of this state, in the +hope that the disk condition (be it disk full or permission errors) has +been resolved. If the torrent is not auto-managed, you have to explicitly +take it out of the upload mode by calling `set_upload_mode()`_ on the +torrent_handle_. peer_info ========= @@ -3451,7 +3512,7 @@ session_settings int read_cache_line_size; int write_cache_line_size; - bool adjust_priority_on_disk_failure; + int optimistic_disk_retry; }; ``user_agent`` this is the client identification to the tracker. @@ -3857,15 +3918,14 @@ When a piece in the write cache has ``write_cache_line_size`` contiguous blocks in it, they will be flushed. Setting this to 1 effectively disables the write cache. -``adjust_priority_on_disk_failure`` specifies what libtorrent should do -on disk failures. If this is set to true, instead of pausing the torrent -and setting it to an error state when it fails to write to disk, the -priorities of all pieces are set to 0. This effectively means the client -can keep seeding the parts that were already downloaded, instead of -leaving the swarm because of the error. +``optimistic_disk_retry`` is the number of seconds from a disk write +errors occur on a torrent until libtorrent will take it out of the +upload mode, to test if the error condition has been fixed. -If a read operation fails, it will still set an error on the torrent -and pause it. +libtorrent will only do this automatically for auto managed torrents. + +You can explicitly take a torrent out of upload only mode using +`set_upload_mode()`_. pe_settings =========== diff --git a/include/libtorrent/peer_connection.hpp b/include/libtorrent/peer_connection.hpp index f0dcd4efc..9124c2c36 100644 --- a/include/libtorrent/peer_connection.hpp +++ b/include/libtorrent/peer_connection.hpp @@ -427,7 +427,13 @@ namespace libtorrent void make_time_critical(piece_block const& block); // adds a block to the request queue - void add_request(piece_block const& b, bool time_critical = false); + // returns true if successful, false otherwise + bool add_request(piece_block const& b, bool time_critical = false); + + // clears the request queue and sends cancels for all messages + // in the download queue + void cancel_all_requests(); + // removes a block from the request queue or download queue // sends a cancel message if appropriate // refills the request queue, and possibly ignoring pieces requested diff --git a/include/libtorrent/session.hpp b/include/libtorrent/session.hpp index 328cfcbb1..18b7a080c 100644 --- a/include/libtorrent/session.hpp +++ b/include/libtorrent/session.hpp @@ -174,6 +174,7 @@ namespace libtorrent , userdata(0) , seed_mode(false) , override_resume_data(false) + , upload_mode(false) {} boost::intrusive_ptr ti; @@ -190,6 +191,7 @@ namespace libtorrent void* userdata; bool seed_mode; bool override_resume_data; + bool upload_mode; }; class TORRENT_EXPORT session: public boost::noncopyable, aux::eh_initializer diff --git a/include/libtorrent/session_settings.hpp b/include/libtorrent/session_settings.hpp index f0c8e584b..61daba2e8 100644 --- a/include/libtorrent/session_settings.hpp +++ b/include/libtorrent/session_settings.hpp @@ -173,7 +173,7 @@ namespace libtorrent , disk_cache_algorithm(largest_contiguous) , read_cache_line_size(16) , write_cache_line_size(32) - , adjust_priority_on_disk_failure(true) + , optimistic_disk_retry(10 * 60) {} // this is the user agent that will be sent to the tracker @@ -600,13 +600,9 @@ namespace libtorrent // is flushed immediately int write_cache_line_size; - // if this is set to true, piece priorities - // will be automatically changed if there is - // an error writing to the disk, attempting - // to avoid hitting the error again. If there - // is an error reading, the torrent will be - // paused - bool adjust_priority_on_disk_failure; + // this is the number of seconds a disk failure + // occurs until libtorrent will re-try. + int optimistic_disk_retry; }; #ifndef TORRENT_DISABLE_DHT diff --git a/include/libtorrent/torrent.hpp b/include/libtorrent/torrent.hpp index e3356ba10..242ab3f25 100644 --- a/include/libtorrent/torrent.hpp +++ b/include/libtorrent/torrent.hpp @@ -159,6 +159,9 @@ namespace libtorrent void start_announcing(); void stop_announcing(); + void set_upload_mode(bool b); + bool upload_mode() const { return m_upload_mode; } + int seed_rank(session_settings const& s) const; enum flags_t { overwrite_existing = 1 }; @@ -730,6 +733,9 @@ namespace libtorrent // one of the trackers in this torrent ptime m_last_scrape; + // the time when we switched to upload mode + ptime m_upload_mode_time; + boost::intrusive_ptr m_torrent_file; void parse_response(const entry& e, std::vector& peer_list); @@ -995,6 +1001,9 @@ namespace libtorrent // is true if this torrent has been paused bool m_paused:1; + // set to true when this torrent may not download anything + bool m_upload_mode:1; + // if this is true, libtorrent may pause and resume // this torrent depending on queuing rules. Torrents // started with auto_managed flag set may be added in diff --git a/include/libtorrent/torrent_handle.hpp b/include/libtorrent/torrent_handle.hpp index 286af7643..3d4cd1182 100644 --- a/include/libtorrent/torrent_handle.hpp +++ b/include/libtorrent/torrent_handle.hpp @@ -273,6 +273,11 @@ namespace libtorrent // is true if this torrent is (still) in seed_mode bool seed_mode; + + // this is set to true when the torrent is blocked + // from downloading, typically caused by a file + // write operation failing + bool upload_mode; }; struct TORRENT_EXPORT block_info @@ -397,6 +402,8 @@ namespace libtorrent bool is_paused() const; void pause() const; void resume() const; + void set_upload_mode(bool b) const; + void force_recheck() const; void save_resume_data() const; diff --git a/src/peer_connection.cpp b/src/peer_connection.cpp index eb695f3da..d1dfa45c4 100644 --- a/src/peer_connection.cpp +++ b/src/peer_connection.cpp @@ -2374,7 +2374,7 @@ namespace libtorrent ++m_queued_time_critical; } - void peer_connection::add_request(piece_block const& block, bool time_critical) + bool peer_connection::add_request(piece_block const& block, bool time_critical) { // INVARIANT_CHECK; @@ -2393,6 +2393,8 @@ namespace libtorrent TORRENT_ASSERT(std::find(m_request_queue.begin(), m_request_queue.end() , block) == m_request_queue.end()); + if (t->upload_mode()) return false; + piece_picker::piece_state_t state; peer_speed_t speed = peer_speed(); char const* speedmsg = 0; @@ -2413,7 +2415,7 @@ namespace libtorrent } if (!t->picker().mark_as_downloading(block, peer_info_struct(), state)) - return; + return false; if (t->alerts().should_post()) { @@ -2431,6 +2433,53 @@ namespace libtorrent { m_request_queue.push_back(block); } + return true; + } + + void peer_connection::cancel_all_requests() + { + INVARIANT_CHECK; + + boost::shared_ptr t = m_torrent.lock(); + // this peer might be disconnecting + if (!t) return; + + TORRENT_ASSERT(t->valid_metadata()); + +#ifdef TORRENT_VERBOSE_LOGGING + (*m_logger) << time_now_string() << " *** CANCEL ALL REQUESTS\n"; +#endif + + while (!m_request_queue.empty()) + { + t->picker().abort_download(m_request_queue.back()); + m_request_queue.pop_back(); + } + + for (std::vector::iterator i = m_download_queue.begin() + , end(m_download_queue.end()); i != end; ++i) + { + piece_block b = i->block; + + int block_offset = b.block_index * t->block_size(); + int block_size + = (std::min)(t->torrent_file().piece_size(b.piece_index)-block_offset, + t->block_size()); + TORRENT_ASSERT(block_size > 0); + TORRENT_ASSERT(block_size <= t->block_size()); + + peer_request r; + r.piece = b.piece_index; + r.start = block_offset; + r.length = block_size; + +#ifdef TORRENT_VERBOSE_LOGGING + (*m_logger) << time_now_string() + << " ==> CANCEL [ piece: " << b.piece_index << " | s: " + << block_offset << " | l: " << block_size << " | " << b.block_index << " ]\n"; +#endif + write_cancel(r); + } } void peer_connection::cancel_request(piece_block const& block) @@ -2588,7 +2637,8 @@ namespace libtorrent boost::shared_ptr t = m_torrent.lock(); TORRENT_ASSERT(t); - if ((int)m_download_queue.size() >= m_desired_queue_size) return; + if ((int)m_download_queue.size() >= m_desired_queue_size + || t->upload_mode()) return; bool empty_download_queue = m_download_queue.empty(); diff --git a/src/policy.cpp b/src/policy.cpp index 70cc24871..83165ed88 100644 --- a/src/policy.cpp +++ b/src/policy.cpp @@ -193,6 +193,7 @@ namespace libtorrent { if (t.is_seed()) return; if (c.no_download()) return; + if (t.upload_mode()) return; TORRENT_ASSERT(t.valid_metadata()); TORRENT_ASSERT(c.peer_info_struct() != 0 || !dynamic_cast(&c)); @@ -313,7 +314,7 @@ namespace libtorrent // ok, we found a piece that's not being downloaded // by somebody else. request it from this peer // and return - c.add_request(*i); + if (!c.add_request(*i)) continue; TORRENT_ASSERT(p.num_peers(*i) == 1); TORRENT_ASSERT(p.is_requested(*i)); num_requests--; diff --git a/src/torrent.cpp b/src/torrent.cpp index 17ed06600..ee2a58552 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -144,6 +144,7 @@ namespace libtorrent , m_total_downloaded(0) , m_started(time_now()) , m_last_scrape(min_time()) + , m_upload_mode_time(time_now()) , m_torrent_file(p.ti ? p.ti : new torrent_info(p.info_hash)) , m_storage(0) , m_host_resolver(ses.m_io_service) @@ -180,6 +181,7 @@ namespace libtorrent , m_time_scaler(0) , m_abort(false) , m_paused(p.paused) + , m_upload_mode(p.upload_mode) , m_auto_managed(p.auto_managed) #ifndef TORRENT_DISABLE_RESOLVE_COUNTRIES , m_resolving_country(false) @@ -341,6 +343,36 @@ namespace libtorrent } } + void torrent::set_upload_mode(bool b) + { + if (b == m_upload_mode) return; + + m_upload_mode = b; + + if (m_upload_mode) + { + // clear request queues of all peers + for (std::set::iterator i = m_connections.begin() + , end(m_connections.end()); i != end; ++i) + { + peer_connection* p = (*i); + p->cancel_all_requests(); + } + // this is used to try leaving upload only mode periodically + m_upload_mode_time = time_now(); + } + else + { + // send_block_requests on all peers + for (std::set::iterator i = m_connections.begin() + , end(m_connections.end()); i != end; ++i) + { + peer_connection* p = (*i); + p->send_block_requests(); + } + } + } + void torrent::handle_disk_error(disk_io_job const& j, peer_connection* c) { if (!j.error) return; @@ -389,20 +421,8 @@ namespace libtorrent // and the filesystem doesn't support sparse files, only zero the priorities // of the pieces that are at the tails of all files, leaving everything // up to the highest written piece in each file - if (m_ses.settings().adjust_priority_on_disk_failure - && has_picker()) - { - bool filter_updated = false; - bool was_finished = is_finished(); - const int num_pieces = m_torrent_file->num_pieces(); - for (int i = 0; i < num_pieces; ++i) - { - filter_updated |= m_picker->set_piece_priority(i, 0); - TORRENT_ASSERT(num_have() >= m_picker->num_have_filtered()); - } - if (filter_updated) update_peer_interest(was_finished); - return; - } + set_upload_mode(true); + return; } // put the torrent in an error-state @@ -4897,6 +4917,8 @@ namespace libtorrent { INVARIANT_CHECK; + ptime now = time_now(); + #ifndef TORRENT_DISABLE_EXTENSIONS for (extension_list_t::iterator i = m_extensions.begin() , end(m_extensions.end()); i != end; ++i) @@ -4932,6 +4954,15 @@ namespace libtorrent m_policy.pulse(); } + // if we're in upload only mode and we're auto-managed + // leave upload mode every 10 minutes hoping that the error + // condition has been fixed + if (m_upload_mode && m_auto_managed && now - m_upload_mode_time + > seconds(m_settings.optimistic_disk_retry)) + { + set_upload_mode(false); + } + if (is_paused()) { // let the stats fade out to 0 @@ -4965,8 +4996,6 @@ namespace libtorrent if (is_seed()) m_seeding_time += since_last_tick; m_active_time += since_last_tick; - ptime now = time_now(); - // ---- TIME CRITICAL PIECES ---- if (!m_time_critical_pieces.empty()) @@ -5437,6 +5466,7 @@ namespace libtorrent { st.last_scrape = total_seconds(now - m_last_scrape); } + st.upload_mode = m_upload_mode; st.up_bandwidth_queue = 0; st.down_bandwidth_queue = 0; diff --git a/src/torrent_handle.cpp b/src/torrent_handle.cpp index 1afc40695..d6e07ae7a 100644 --- a/src/torrent_handle.cpp +++ b/src/torrent_handle.cpp @@ -290,6 +290,12 @@ namespace libtorrent TORRENT_FORWARD(pause()); } + void torrent_handle::set_upload_mode(bool b) const + { + INVARIANT_CHECK; + TORRENT_FORWARD(set_upload_mode(b)); + } + void torrent_handle::save_resume_data() const { INVARIANT_CHECK; diff --git a/test/test_transfer.cpp b/test/test_transfer.cpp index bc77cc95c..1a767b275 100644 --- a/test/test_transfer.cpp +++ b/test/test_transfer.cpp @@ -271,24 +271,11 @@ void test_transfer(bool test_disk_full = false) std::cerr << "moving storage" << std::endl; } - if (test_disk_full && tor2.is_finished()) + if (test_disk_full && st2.upload_mode) { test_disk_full = false; - std::vector priorities2 = tor2.piece_priorities(); - std::cerr << "piece priorities: "; - for (std::vector::iterator i = priorities2.begin() - , end(priorities2.end()); i != end; ++i) - { - TEST_CHECK(*i == 0); - std::cerr << *i << ", "; - } - std::cerr << std::endl; - ((test_storage*)tor2.get_storage_impl())->m_limit = 16 * 1024 * 1024; - tor2.prioritize_pieces(priorities); - std::cerr << "setting priorities: "; - std::copy(priorities.begin(), priorities.end(), std::ostream_iterator(std::cerr, ", ")); - std::cerr << std::endl; + tor2.set_upload_mode(false); continue; }