merged RC_1_1 into master

ChangeLog

@@ -90,6 +90,8 @@
 
 1.1.9 release
 
+	* save both file and piece priorities in resume file
+	* added missing stats_metric python binding
 	* uTP connections are no longer exempt from rate limits by default
 	* fix exporting files from partfile while seeding
 	* fix potential deadlock on Windows, caused by performing restricted

bindings/python/src/session.cpp

@@ -1138,6 +1138,11 @@ void bind_session()
 		.def_readonly("type", &stats_metric::type)
 	;
 
+	enum_<metric_type_t>("metric_type_t")
+		.value("counter", metric_type_t::counter)
+		.value("gauge", metric_type_t::gauge)
+		;
+
     def("session_stats_metrics", session_stats_metrics);
     def("find_metric_idx", find_metric_idx_wrap);
 

bindings/python/test.py

@@ -264,10 +264,12 @@ class test_torrent_handle(unittest.TestCase):
         self.assertEqual(trackers[0].get('tier'), 0)
         self.assertEqual(self.h.get_file_priorities(), [1, 1])
         self.assertEqual(self.h.http_seeds(), ['http://test.com/file3'])
-        # url_seeds was already set, test that it did not got overwritten
+        # url_seeds was already set, test that it did not get overwritten
         self.assertEqual(self.h.url_seeds(),
                          ['http://test.com/announce-url/', 'http://test.com/file/'])
-        self.assertEqual(self.h.get_piece_priorities(), [4])
+        # piece priorities weren't set explicitly, but they were updated by the
+        # file priorities being set
+        self.assertEqual(self.h.get_piece_priorities(), [1])
         self.assertEqual(self.ti.merkle_tree(), [])
         self.assertEqual(self.st.verified_pieces, [])
 
@@ -592,6 +594,11 @@ class test_session(unittest.TestCase):
         self.assertTrue('connection_speed' in seed_mode)
         self.assertTrue('file_pool_size' in seed_mode)
 
+    def test_default_settings(self):
+
+        default = lt.default_settings()
+        print(default)
+
 
 class test_example_client(unittest.TestCase):
 

include/libtorrent/torrent.hpp

@@ -570,7 +570,8 @@ namespace libtorrent {
 		void set_piece_deadline(piece_index_t piece, int t, deadline_flags_t flags);
 		void reset_piece_deadline(piece_index_t piece);
 		void clear_time_critical();
-		void update_piece_priorities();
+		void update_piece_priorities(
+			aux::vector<download_priority_t, file_index_t> const& file_prios);
 
 		void status(torrent_status* st, status_flags_t flags);
 

include/libtorrent/torrent_handle.hpp

@@ -949,9 +949,8 @@ TORRENT_IPV6_NAMESPACE_END
 		// The default priority of pieces is 4.
 		//
 		// Piece priorities can not be changed for torrents that have not
-		// downloaded the metadata yet. For instance, magnet links and torrents
-		// added by URL won't have metadata immediately. see the
-		// metadata_received_alert.
+		// downloaded the metadata yet. Magnet links won't have metadata
+		// immediately. see the metadata_received_alert.
 		//
 		// ``piece_priority`` sets or gets the priority for an individual piece,
 		// specified by ``index``.
@@ -968,6 +967,10 @@ TORRENT_IPV6_NAMESPACE_END
 		//
 		// ``get_piece_priorities`` returns a vector with one element for each piece
 		// in the torrent. Each element is the current priority of that piece.
+		//
+		// It's possible to cancel the effect of *file* priorities by setting the
+		// priorities for the affected pieces. Care has to be taken when mixing
+		// usage of file- and piece priorities.
 		void piece_priority(piece_index_t index, download_priority_t priority) const;
 		download_priority_t piece_priority(piece_index_t index) const;
 		void prioritize_pieces(std::vector<download_priority_t> const& pieces) const;
@@ -1005,6 +1008,16 @@ TORRENT_IPV6_NAMESPACE_END
 		// You cannot set the file priorities on a torrent that does not yet have
 		// metadata or a torrent that is a seed. ``file_priority(int, int)`` and
 		// prioritize_files() are both no-ops for such torrents.
+		//
+		// Since changing file priorities may involve disk operations (of moving
+		// files in- and out of the part file), the internal accounting of file
+		// priorities happen asynchronously. i.e. setting file priorities and then
+		// immediately querying them may not yield the same priorities just set.
+		// However, the *piece* priorities are updated immediately.
+		//
+		// when combining file- and piece priorities, the resume file will record
+		// both. When loading the resume data, the file priorities will be applied
+		// first, then the piece priorities.
 		void file_priority(file_index_t index, download_priority_t priority) const;
 		download_priority_t file_priority(file_index_t index) const;
 		void prioritize_files(std::vector<download_priority_t> const& files) const;

src/torrent.cpp

@@ -1769,9 +1769,12 @@ bool is_downloading_state(int const st)
 			}
 		}
 
-		// if we've already loaded file priorities, don't load piece priorities,
-		// they will interfere.
-		if (m_add_torrent_params && m_file_priority.empty())
+		// in case file priorities were passed in via the add_torrent_params
+		// and also in the case of share mode, we need to update the priorities
+		// this has to be applied before piece priority
+		if (!m_file_priority.empty()) update_piece_priorities(m_file_priority);
+
+		if (m_add_torrent_params)
 		{
 			piece_index_t idx(0);
 			for (auto prio : m_add_torrent_params->piece_priorities)
@@ -1786,13 +1789,6 @@ bool is_downloading_state(int const st)
 			update_gauge();
 		}
 
-		// in case file priorities were passed in via the add_torrent_params
-		// and also in the case of share mode, we need to update the priorities
-		if (!m_file_priority.empty() && std::find(m_file_priority.begin()
-			, m_file_priority.end(), dont_download) != m_file_priority.end())
-		{
-			update_piece_priorities();
-		}
 
 		if (m_seed_mode)
 		{
@@ -5008,7 +5004,6 @@ bool is_downloading_state(int const st)
 		if (m_file_priority != prios)
 		{
 			m_file_priority = std::move(prios);
-			update_piece_priorities();
 			if (m_share_mode)
 				recalc_share_mode();
 		}
@@ -5035,6 +5030,13 @@ bool is_downloading_state(int const st)
 		// storage may be NULL during shutdown
 		if (m_storage)
 		{
+			// the update of m_file_priority is deferred until the disk job comes
+			// back, but to preserve sanity and consistency, the piece priorities are
+			// updated immediately. If, on the off-chance, there's a disk failure, the
+			// piece priorities still stay the same, but the file priorities are
+			// possibly not fully updated.
+			update_piece_priorities(new_priority);
+
 			ADD_OUTSTANDING_ASYNC("file_priority");
 			m_ses.disk_thread().async_set_file_priority(m_storage
 				, std::move(new_priority), std::bind(&torrent::on_file_priority, shared_from_this(), _1, _2));
@@ -5072,6 +5074,12 @@ bool is_downloading_state(int const st)
 		// storage may be nullptr during shutdown
 		if (m_storage)
 		{
+			// the update of m_file_priority is deferred until the disk job comes
+			// back, but to preserve sanity and consistency, the piece priorities are
+			// updated immediately. If, on the off-chance, there's a disk failure, the
+			// piece priorities still stay the same, but the file priorities are
+			// possibly not fully updated.
+			update_piece_priorities(new_priority);
 			ADD_OUTSTANDING_ASYNC("file_priority");
 			m_ses.disk_thread().async_set_file_priority(m_storage
 				, std::move(new_priority), std::bind(&torrent::on_file_priority, shared_from_this(), _1, _2));
@@ -5118,7 +5126,8 @@ bool is_downloading_state(int const st)
 		files->resize(m_torrent_file->num_files(), default_priority);
 	}
 
-	void torrent::update_piece_priorities()
+	void torrent::update_piece_priorities(
+		aux::vector<download_priority_t, file_index_t> const& file_prios)
 	{
 		INVARIANT_CHECK;
 
@@ -5140,8 +5149,8 @@ bool is_downloading_state(int const st)
 			// pad files always have priority 0
 			download_priority_t const file_prio
 				= fs.pad_file_at(i) ? dont_download
-				: i >= m_file_priority.end_index() ? default_priority
-				: m_file_priority[i];
+				: i >= file_prios.end_index() ? default_priority
+				: file_prios[i];
 
 			if (file_prio == dont_download)
 			{
@@ -6331,20 +6340,15 @@ bool is_downloading_state(int const st)
 
 		// piece priorities and file priorities are mutually exclusive. If there
 		// are file priorities set, don't save piece priorities.
-		if (!m_file_priority.empty())
+		// when in seed mode (i.e. the client promises that we have all files)
+		// it does not make sense to save file priorities.
+		if (!m_file_priority.empty() && !m_seed_mode)
 		{
-			// when in seed mode (i.e. the client promises that we have all files)
-			// it does not make sense to save file priorities.
-			if (!m_seed_mode)
-			{
-				// write file priorities
-				ret.file_priorities.clear();
-				ret.file_priorities.reserve(m_file_priority.size());
-				for (auto const prio : m_file_priority)
-					ret.file_priorities.push_back(prio);
-			}
+			// write file priorities
+			ret.file_priorities = m_file_priority;
 		}
-		else if (has_picker())
+
+		if (has_picker())
 		{
 			// write piece priorities
 			// but only if they are not set to the default

test/setup_transfer.cpp

@@ -67,6 +67,30 @@ using namespace lt;
 #include <conio.h>
 #endif
 
+std::shared_ptr<torrent_info> generate_torrent()
+{
+	file_storage fs;
+	fs.add_file("test_resume/tmp1", 128 * 1024 * 8);
+	fs.add_file("test_resume/tmp2", 128 * 1024);
+	fs.add_file("test_resume/tmp3", 128 * 1024);
+	lt::create_torrent t(fs, 128 * 1024, 6);
+
+	t.add_tracker("http://torrent_file_tracker.com/announce");
+	t.add_url_seed("http://torrent_file_url_seed.com/");
+
+	TEST_CHECK(t.num_pieces() > 0);
+	for (auto const i : fs.piece_range())
+	{
+		sha1_hash ph;
+		aux::random_bytes(ph);
+		t.set_hash(i, ph);
+	}
+
+	std::vector<char> buf;
+	bencode(std::back_inserter(buf), t.generate());
+	return std::make_shared<torrent_info>(buf, from_span);
+}
+
 namespace {
 	std::uint32_t g_addr = 0x92343023;
 }

test/setup_transfer.hpp

@@ -33,12 +33,15 @@ POSSIBILITY OF SUCH DAMAGE.
 #ifndef SETUP_TRANSFER_HPP
 #define SETUP_TRANSFER_HPP
 
+#include <memory>
 #include <tuple>
 #include "test.hpp"
 #include "libtorrent/session.hpp"
 #include "libtorrent/units.hpp"
 #include "libtorrent/fwd.hpp"
 
+EXPORT std::shared_ptr<lt::torrent_info> generate_torrent();
+
 EXPORT int print_failures();
 EXPORT unsigned char random_byte();
 

test/test_priority.cpp

@@ -529,3 +529,43 @@ TORRENT_TEST(export_file_while_seed)
 	TEST_CHECK(exists("temporary"));
 }
 
+TORRENT_TEST(test_piece_priority_after_resume)
+{
+	auto const new_prio = lt::low_priority;
+
+	add_torrent_params p;
+	auto ti = generate_torrent();
+	{
+		auto const prio = top_priority;
+
+		p.save_path = ".";
+		p.ti = ti;
+		p.file_priorities.resize(1, prio);
+
+		lt::session ses(settings());
+		torrent_handle h = ses.add_torrent(p);
+
+		TEST_EQUAL(h.piece_priority(piece_index_t{0}), prio);
+
+		using prio_vec = std::vector<std::pair<lt::piece_index_t, lt::download_priority_t>>;
+		h.prioritize_pieces(prio_vec{{piece_index_t{0}, new_prio}});
+		TEST_EQUAL(h.piece_priority(piece_index_t{0}), new_prio);
+
+		ses.pause();
+		h.save_resume_data();
+
+		alert const* a = wait_for_alert(ses, save_resume_data_alert::alert_type);
+		save_resume_data_alert const* rd = alert_cast<save_resume_data_alert>(a);
+
+		p = rd->params;
+	}
+	{
+		p.save_path = ".";
+		p.ti = ti;
+
+		lt::session ses(settings());
+		torrent_handle h = ses.add_torrent(p);
+
+		TEST_EQUAL(h.piece_priority(piece_index_t{0}), new_prio);
+	}
+}

test/test_resume.cpp

@@ -64,30 +64,6 @@ torrent_flags_t const flags_mask
 	| torrent_flags::upload_mode
 	| torrent_flags::apply_ip_filter;
 
-std::shared_ptr<torrent_info> generate_torrent()
-{
-	file_storage fs;
-	fs.add_file("test_resume/tmp1", 128 * 1024 * 8);
-	fs.add_file("test_resume/tmp2", 128 * 1024);
-	fs.add_file("test_resume/tmp3", 128 * 1024);
-	lt::create_torrent t(fs, 128 * 1024, 6);
-
-	t.add_tracker("http://torrent_file_tracker.com/announce");
-	t.add_url_seed("http://torrent_file_url_seed.com/");
-
-	TEST_CHECK(t.num_pieces() > 0);
-	for (auto const i : fs.piece_range())
-	{
-		sha1_hash ph;
-		aux::random_bytes(ph);
-		t.set_hash(i, ph);
-	}
-
-	std::vector<char> buf;
-	bencode(std::back_inserter(buf), t.generate());
-	return std::make_shared<torrent_info>(buf, from_span);
-}
-
 std::vector<char> generate_resume_data(torrent_info* ti
 	, char const* file_priorities = "")
 {
@@ -838,7 +814,7 @@ TORRENT_TEST(file_priorities_seed_mode)
 
 namespace {
 
-void test_zero_file_prio(bool test_deprecated = false)
+void test_zero_file_prio(bool test_deprecated = false, bool mix_prios = false)
 {
 	std::printf("test_file_prio\n");
 
@@ -855,17 +831,14 @@ void test_zero_file_prio(bool test_deprecated = false)
 	rd["info-hash"] = ti->info_hash().to_string();
 	rd["blocks per piece"] = std::max(1, ti->piece_length() / 0x4000);
 
-	entry::list_type& file_prio = rd["file_priority"].list();
-	for (int i = 0; i < 100; ++i)
-	{
-		file_prio.push_back(entry(0));
-	}
+	// set file priorities to 0
+	rd["file_priority"] = entry::list_type(100, entry(0));
 
-	std::string pieces(std::size_t(ti->num_pieces()), '\x01');
-	rd["pieces"] = pieces;
+	rd["pieces"] = std::string(std::size_t(ti->num_pieces()), '\x01');
 
-	std::string pieces_prio(std::size_t(ti->num_pieces()), '\x01');
-	rd["piece_priority"] = pieces_prio;
+	// but set the piece priorities to 1. these take precedence
+	if (mix_prios)
+		rd["piece_priority"] = std::string(std::size_t(ti->num_pieces()), '\x01');
 
 	std::vector<char> resume_data;
 	bencode(back_inserter(resume_data), rd);
@@ -888,7 +861,14 @@ void test_zero_file_prio(bool test_deprecated = false)
 	torrent_handle h = ses.add_torrent(p);
 
 	torrent_status s = h.status();
-	TEST_EQUAL(s.total_wanted, 0);
+	if (mix_prios)
+	{
+		TEST_EQUAL(s.total_wanted, ti->total_size());
+	}
+	else
+	{
+		TEST_EQUAL(s.total_wanted, 0);
+	}
 }
 
 } // anonymous namespace
@@ -899,6 +879,12 @@ TORRENT_TEST(zero_file_prio_deprecated)
 	test_zero_file_prio(true);
 }
 
+TORRENT_TEST(mixing_file_and_piece_prio_deprecated)
+{
+	test_zero_file_prio(true, true);
+}
+
+
 TORRENT_TEST(backwards_compatible_resume_info_dict)
 {
 	// make sure the "info" dictionary is picked up correctly from the
@@ -951,6 +937,11 @@ TORRENT_TEST(zero_file_prio)
 	test_zero_file_prio();
 }
 
+TORRENT_TEST(mixing_file_and_piece_prio)
+{
+	test_zero_file_prio(false, true);
+}
+
 using test_mode_t = flags::bitfield_flag<std::uint8_t, struct test_mode_tag>;
 
 namespace test_mode {

test/test_torrent.cpp

@@ -225,13 +225,14 @@ TORRENT_TEST(total_wanted)
 	torrent_handle h = ses.add_torrent(std::move(p));
 
 	torrent_status st = h.status();
-	std::cout << "total_wanted: " << st.total_wanted << " : " << 1024 << std::endl;
 	TEST_EQUAL(st.total_wanted, 1024);
-	std::cout << "total_wanted_done: " << st.total_wanted_done << " : 0" << std::endl;
 	TEST_EQUAL(st.total_wanted_done, 0);
 
+	// make sure that selecting and unseleting a file quickly still end up with
+	// the last set priority
 	h.file_priority(file_index_t{1}, default_priority);
 	h.file_priority(file_index_t{1}, dont_download);
+	TEST_EQUAL(h.status({}).total_wanted, 0);
 	TEST_CHECK(wait_priority(h, aux::vector<download_priority_t, file_index_t>(static_cast<std::size_t>(fs.num_files()))));
 	TEST_EQUAL(h.status({}).total_wanted, 0);
 }