diff --git a/ChangeLog b/ChangeLog index eccf3f7e0..060a3879d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -90,6 +90,8 @@ 1.1.9 release + * save both file and piece priorities in resume file + * added missing stats_metric python binding * uTP connections are no longer exempt from rate limits by default * fix exporting files from partfile while seeding * fix potential deadlock on Windows, caused by performing restricted diff --git a/bindings/python/src/session.cpp b/bindings/python/src/session.cpp index 970824132..2a0196506 100644 --- a/bindings/python/src/session.cpp +++ b/bindings/python/src/session.cpp @@ -1138,6 +1138,11 @@ void bind_session() .def_readonly("type", &stats_metric::type) ; + enum_("metric_type_t") + .value("counter", metric_type_t::counter) + .value("gauge", metric_type_t::gauge) + ; + def("session_stats_metrics", session_stats_metrics); def("find_metric_idx", find_metric_idx_wrap); diff --git a/bindings/python/test.py b/bindings/python/test.py index c133a503a..386af39f9 100644 --- a/bindings/python/test.py +++ b/bindings/python/test.py @@ -264,10 +264,12 @@ class test_torrent_handle(unittest.TestCase): self.assertEqual(trackers[0].get('tier'), 0) self.assertEqual(self.h.get_file_priorities(), [1, 1]) self.assertEqual(self.h.http_seeds(), ['http://test.com/file3']) - # url_seeds was already set, test that it did not got overwritten + # url_seeds was already set, test that it did not get overwritten self.assertEqual(self.h.url_seeds(), ['http://test.com/announce-url/', 'http://test.com/file/']) - self.assertEqual(self.h.get_piece_priorities(), [4]) + # piece priorities weren't set explicitly, but they were updated by the + # file priorities being set + self.assertEqual(self.h.get_piece_priorities(), [1]) self.assertEqual(self.ti.merkle_tree(), []) self.assertEqual(self.st.verified_pieces, []) @@ -592,6 +594,11 @@ class test_session(unittest.TestCase): self.assertTrue('connection_speed' in seed_mode) self.assertTrue('file_pool_size' in seed_mode) + def test_default_settings(self): + + default = lt.default_settings() + print(default) + class test_example_client(unittest.TestCase): diff --git a/include/libtorrent/torrent.hpp b/include/libtorrent/torrent.hpp index 0c4795213..d112bd7a7 100644 --- a/include/libtorrent/torrent.hpp +++ b/include/libtorrent/torrent.hpp @@ -570,7 +570,8 @@ namespace libtorrent { void set_piece_deadline(piece_index_t piece, int t, deadline_flags_t flags); void reset_piece_deadline(piece_index_t piece); void clear_time_critical(); - void update_piece_priorities(); + void update_piece_priorities( + aux::vector const& file_prios); void status(torrent_status* st, status_flags_t flags); diff --git a/include/libtorrent/torrent_handle.hpp b/include/libtorrent/torrent_handle.hpp index 2975c059d..34520656b 100644 --- a/include/libtorrent/torrent_handle.hpp +++ b/include/libtorrent/torrent_handle.hpp @@ -949,9 +949,8 @@ TORRENT_IPV6_NAMESPACE_END // The default priority of pieces is 4. // // Piece priorities can not be changed for torrents that have not - // downloaded the metadata yet. For instance, magnet links and torrents - // added by URL won't have metadata immediately. see the - // metadata_received_alert. + // downloaded the metadata yet. Magnet links won't have metadata + // immediately. see the metadata_received_alert. // // ``piece_priority`` sets or gets the priority for an individual piece, // specified by ``index``. @@ -968,6 +967,10 @@ TORRENT_IPV6_NAMESPACE_END // // ``get_piece_priorities`` returns a vector with one element for each piece // in the torrent. Each element is the current priority of that piece. + // + // It's possible to cancel the effect of *file* priorities by setting the + // priorities for the affected pieces. Care has to be taken when mixing + // usage of file- and piece priorities. void piece_priority(piece_index_t index, download_priority_t priority) const; download_priority_t piece_priority(piece_index_t index) const; void prioritize_pieces(std::vector const& pieces) const; @@ -1005,6 +1008,16 @@ TORRENT_IPV6_NAMESPACE_END // You cannot set the file priorities on a torrent that does not yet have // metadata or a torrent that is a seed. ``file_priority(int, int)`` and // prioritize_files() are both no-ops for such torrents. + // + // Since changing file priorities may involve disk operations (of moving + // files in- and out of the part file), the internal accounting of file + // priorities happen asynchronously. i.e. setting file priorities and then + // immediately querying them may not yield the same priorities just set. + // However, the *piece* priorities are updated immediately. + // + // when combining file- and piece priorities, the resume file will record + // both. When loading the resume data, the file priorities will be applied + // first, then the piece priorities. void file_priority(file_index_t index, download_priority_t priority) const; download_priority_t file_priority(file_index_t index) const; void prioritize_files(std::vector const& files) const; diff --git a/src/torrent.cpp b/src/torrent.cpp index 4de5dbe47..f16a5d0af 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -1769,9 +1769,12 @@ bool is_downloading_state(int const st) } } - // if we've already loaded file priorities, don't load piece priorities, - // they will interfere. - if (m_add_torrent_params && m_file_priority.empty()) + // in case file priorities were passed in via the add_torrent_params + // and also in the case of share mode, we need to update the priorities + // this has to be applied before piece priority + if (!m_file_priority.empty()) update_piece_priorities(m_file_priority); + + if (m_add_torrent_params) { piece_index_t idx(0); for (auto prio : m_add_torrent_params->piece_priorities) @@ -1786,13 +1789,6 @@ bool is_downloading_state(int const st) update_gauge(); } - // in case file priorities were passed in via the add_torrent_params - // and also in the case of share mode, we need to update the priorities - if (!m_file_priority.empty() && std::find(m_file_priority.begin() - , m_file_priority.end(), dont_download) != m_file_priority.end()) - { - update_piece_priorities(); - } if (m_seed_mode) { @@ -5008,7 +5004,6 @@ bool is_downloading_state(int const st) if (m_file_priority != prios) { m_file_priority = std::move(prios); - update_piece_priorities(); if (m_share_mode) recalc_share_mode(); } @@ -5035,6 +5030,13 @@ bool is_downloading_state(int const st) // storage may be NULL during shutdown if (m_storage) { + // the update of m_file_priority is deferred until the disk job comes + // back, but to preserve sanity and consistency, the piece priorities are + // updated immediately. If, on the off-chance, there's a disk failure, the + // piece priorities still stay the same, but the file priorities are + // possibly not fully updated. + update_piece_priorities(new_priority); + ADD_OUTSTANDING_ASYNC("file_priority"); m_ses.disk_thread().async_set_file_priority(m_storage , std::move(new_priority), std::bind(&torrent::on_file_priority, shared_from_this(), _1, _2)); @@ -5072,6 +5074,12 @@ bool is_downloading_state(int const st) // storage may be nullptr during shutdown if (m_storage) { + // the update of m_file_priority is deferred until the disk job comes + // back, but to preserve sanity and consistency, the piece priorities are + // updated immediately. If, on the off-chance, there's a disk failure, the + // piece priorities still stay the same, but the file priorities are + // possibly not fully updated. + update_piece_priorities(new_priority); ADD_OUTSTANDING_ASYNC("file_priority"); m_ses.disk_thread().async_set_file_priority(m_storage , std::move(new_priority), std::bind(&torrent::on_file_priority, shared_from_this(), _1, _2)); @@ -5118,7 +5126,8 @@ bool is_downloading_state(int const st) files->resize(m_torrent_file->num_files(), default_priority); } - void torrent::update_piece_priorities() + void torrent::update_piece_priorities( + aux::vector const& file_prios) { INVARIANT_CHECK; @@ -5140,8 +5149,8 @@ bool is_downloading_state(int const st) // pad files always have priority 0 download_priority_t const file_prio = fs.pad_file_at(i) ? dont_download - : i >= m_file_priority.end_index() ? default_priority - : m_file_priority[i]; + : i >= file_prios.end_index() ? default_priority + : file_prios[i]; if (file_prio == dont_download) { @@ -6331,20 +6340,15 @@ bool is_downloading_state(int const st) // piece priorities and file priorities are mutually exclusive. If there // are file priorities set, don't save piece priorities. - if (!m_file_priority.empty()) + // when in seed mode (i.e. the client promises that we have all files) + // it does not make sense to save file priorities. + if (!m_file_priority.empty() && !m_seed_mode) { - // when in seed mode (i.e. the client promises that we have all files) - // it does not make sense to save file priorities. - if (!m_seed_mode) - { - // write file priorities - ret.file_priorities.clear(); - ret.file_priorities.reserve(m_file_priority.size()); - for (auto const prio : m_file_priority) - ret.file_priorities.push_back(prio); - } + // write file priorities + ret.file_priorities = m_file_priority; } - else if (has_picker()) + + if (has_picker()) { // write piece priorities // but only if they are not set to the default diff --git a/test/setup_transfer.cpp b/test/setup_transfer.cpp index 652735b26..6b9a6c389 100644 --- a/test/setup_transfer.cpp +++ b/test/setup_transfer.cpp @@ -67,6 +67,30 @@ using namespace lt; #include #endif +std::shared_ptr generate_torrent() +{ + file_storage fs; + fs.add_file("test_resume/tmp1", 128 * 1024 * 8); + fs.add_file("test_resume/tmp2", 128 * 1024); + fs.add_file("test_resume/tmp3", 128 * 1024); + lt::create_torrent t(fs, 128 * 1024, 6); + + t.add_tracker("http://torrent_file_tracker.com/announce"); + t.add_url_seed("http://torrent_file_url_seed.com/"); + + TEST_CHECK(t.num_pieces() > 0); + for (auto const i : fs.piece_range()) + { + sha1_hash ph; + aux::random_bytes(ph); + t.set_hash(i, ph); + } + + std::vector buf; + bencode(std::back_inserter(buf), t.generate()); + return std::make_shared(buf, from_span); +} + namespace { std::uint32_t g_addr = 0x92343023; } diff --git a/test/setup_transfer.hpp b/test/setup_transfer.hpp index 523227453..ed637a888 100644 --- a/test/setup_transfer.hpp +++ b/test/setup_transfer.hpp @@ -33,12 +33,15 @@ POSSIBILITY OF SUCH DAMAGE. #ifndef SETUP_TRANSFER_HPP #define SETUP_TRANSFER_HPP +#include #include #include "test.hpp" #include "libtorrent/session.hpp" #include "libtorrent/units.hpp" #include "libtorrent/fwd.hpp" +EXPORT std::shared_ptr generate_torrent(); + EXPORT int print_failures(); EXPORT unsigned char random_byte(); diff --git a/test/test_priority.cpp b/test/test_priority.cpp index 4f3bfe1bc..975c18128 100644 --- a/test/test_priority.cpp +++ b/test/test_priority.cpp @@ -529,3 +529,43 @@ TORRENT_TEST(export_file_while_seed) TEST_CHECK(exists("temporary")); } +TORRENT_TEST(test_piece_priority_after_resume) +{ + auto const new_prio = lt::low_priority; + + add_torrent_params p; + auto ti = generate_torrent(); + { + auto const prio = top_priority; + + p.save_path = "."; + p.ti = ti; + p.file_priorities.resize(1, prio); + + lt::session ses(settings()); + torrent_handle h = ses.add_torrent(p); + + TEST_EQUAL(h.piece_priority(piece_index_t{0}), prio); + + using prio_vec = std::vector>; + h.prioritize_pieces(prio_vec{{piece_index_t{0}, new_prio}}); + TEST_EQUAL(h.piece_priority(piece_index_t{0}), new_prio); + + ses.pause(); + h.save_resume_data(); + + alert const* a = wait_for_alert(ses, save_resume_data_alert::alert_type); + save_resume_data_alert const* rd = alert_cast(a); + + p = rd->params; + } + { + p.save_path = "."; + p.ti = ti; + + lt::session ses(settings()); + torrent_handle h = ses.add_torrent(p); + + TEST_EQUAL(h.piece_priority(piece_index_t{0}), new_prio); + } +} diff --git a/test/test_resume.cpp b/test/test_resume.cpp index 9362574b3..04374cf6e 100644 --- a/test/test_resume.cpp +++ b/test/test_resume.cpp @@ -64,30 +64,6 @@ torrent_flags_t const flags_mask | torrent_flags::upload_mode | torrent_flags::apply_ip_filter; -std::shared_ptr generate_torrent() -{ - file_storage fs; - fs.add_file("test_resume/tmp1", 128 * 1024 * 8); - fs.add_file("test_resume/tmp2", 128 * 1024); - fs.add_file("test_resume/tmp3", 128 * 1024); - lt::create_torrent t(fs, 128 * 1024, 6); - - t.add_tracker("http://torrent_file_tracker.com/announce"); - t.add_url_seed("http://torrent_file_url_seed.com/"); - - TEST_CHECK(t.num_pieces() > 0); - for (auto const i : fs.piece_range()) - { - sha1_hash ph; - aux::random_bytes(ph); - t.set_hash(i, ph); - } - - std::vector buf; - bencode(std::back_inserter(buf), t.generate()); - return std::make_shared(buf, from_span); -} - std::vector generate_resume_data(torrent_info* ti , char const* file_priorities = "") { @@ -838,7 +814,7 @@ TORRENT_TEST(file_priorities_seed_mode) namespace { -void test_zero_file_prio(bool test_deprecated = false) +void test_zero_file_prio(bool test_deprecated = false, bool mix_prios = false) { std::printf("test_file_prio\n"); @@ -855,17 +831,14 @@ void test_zero_file_prio(bool test_deprecated = false) rd["info-hash"] = ti->info_hash().to_string(); rd["blocks per piece"] = std::max(1, ti->piece_length() / 0x4000); - entry::list_type& file_prio = rd["file_priority"].list(); - for (int i = 0; i < 100; ++i) - { - file_prio.push_back(entry(0)); - } + // set file priorities to 0 + rd["file_priority"] = entry::list_type(100, entry(0)); - std::string pieces(std::size_t(ti->num_pieces()), '\x01'); - rd["pieces"] = pieces; + rd["pieces"] = std::string(std::size_t(ti->num_pieces()), '\x01'); - std::string pieces_prio(std::size_t(ti->num_pieces()), '\x01'); - rd["piece_priority"] = pieces_prio; + // but set the piece priorities to 1. these take precedence + if (mix_prios) + rd["piece_priority"] = std::string(std::size_t(ti->num_pieces()), '\x01'); std::vector resume_data; bencode(back_inserter(resume_data), rd); @@ -888,7 +861,14 @@ void test_zero_file_prio(bool test_deprecated = false) torrent_handle h = ses.add_torrent(p); torrent_status s = h.status(); - TEST_EQUAL(s.total_wanted, 0); + if (mix_prios) + { + TEST_EQUAL(s.total_wanted, ti->total_size()); + } + else + { + TEST_EQUAL(s.total_wanted, 0); + } } } // anonymous namespace @@ -899,6 +879,12 @@ TORRENT_TEST(zero_file_prio_deprecated) test_zero_file_prio(true); } +TORRENT_TEST(mixing_file_and_piece_prio_deprecated) +{ + test_zero_file_prio(true, true); +} + + TORRENT_TEST(backwards_compatible_resume_info_dict) { // make sure the "info" dictionary is picked up correctly from the @@ -951,6 +937,11 @@ TORRENT_TEST(zero_file_prio) test_zero_file_prio(); } +TORRENT_TEST(mixing_file_and_piece_prio) +{ + test_zero_file_prio(false, true); +} + using test_mode_t = flags::bitfield_flag; namespace test_mode { diff --git a/test/test_torrent.cpp b/test/test_torrent.cpp index 7271523d6..9deaf0b64 100644 --- a/test/test_torrent.cpp +++ b/test/test_torrent.cpp @@ -225,13 +225,14 @@ TORRENT_TEST(total_wanted) torrent_handle h = ses.add_torrent(std::move(p)); torrent_status st = h.status(); - std::cout << "total_wanted: " << st.total_wanted << " : " << 1024 << std::endl; TEST_EQUAL(st.total_wanted, 1024); - std::cout << "total_wanted_done: " << st.total_wanted_done << " : 0" << std::endl; TEST_EQUAL(st.total_wanted_done, 0); + // make sure that selecting and unseleting a file quickly still end up with + // the last set priority h.file_priority(file_index_t{1}, default_priority); h.file_priority(file_index_t{1}, dont_download); + TEST_EQUAL(h.status({}).total_wanted, 0); TEST_CHECK(wait_priority(h, aux::vector(static_cast(fs.num_files())))); TEST_EQUAL(h.status({}).total_wanted, 0); }