merged RC_1_1 into master

ChangeLog

@@ -82,6 +82,9 @@
 	* resume data no longer has timestamps of files
 	* require C++11 to build libtorrent
 
+	* fix backwards compatibility to downloads without partfiles
+	* improve part-file related error messages
+	* fix reporting &redundant= in tracker announces
 	* fix tie-break in duplicate peer connection disconnect logic
 	* fix issue with SSL tracker connections left in CLOSE_WAIT state
 	* defer truncating existing files until the first time we write to them

bindings/python/test.py

@@ -79,6 +79,10 @@ class test_torrent_handle(unittest.TestCase):
         # also test the overload that takes a list of piece->priority mappings
         self.h.prioritize_pieces([(0, 1)])
         self.assertEqual(self.h.get_piece_priorities(), [1])
+        self.h.connect_peer(('127.0.0.1', 6881))
+        self.h.connect_peer(('127.0.0.2', 6881), source=4)
+        self.h.connect_peer(('127.0.0.3', 6881), flags=2)
+        self.h.connect_peer(('127.0.0.4', 6881), flags=2, source=4)
 
         print(self.h.queue_position())
 

docs/hunspell/libtorrent.dic

@@ -189,6 +189,8 @@ src
 'fingerprints'
 'query'
 'ro'
+pre-partfile
+pre
 GCC
 prioritization
 nullptr

include/libtorrent/storage.hpp

@@ -439,16 +439,23 @@ namespace libtorrent {
 		file_handle open_file(file_index_t file, open_mode_t mode, storage_error& ec) const;
 		file_handle open_file_impl(file_index_t file, open_mode_t mode, error_code& ec) const;
 
+		bool use_partfile(file_index_t index) const;
+		void use_partfile(file_index_t index, bool b);
+
 		aux::vector<download_priority_t, file_index_t> m_file_priority;
 		std::string m_save_path;
 		std::string m_part_file_name;
 
-		// if this is false, we're not using a part file to store priority-0
-		// pieces, but we instead look for them under their actual file names
-		// this defaults to true, but when checking resume data for a torrent
-		// where we would expect to have a part file, but there isn't one, we set
-		// this to false.
-		bool m_use_part_file = true;
+		// this this is an array indexed by file-index. Each slot represents
+		// whether this file has the part-file enabled for it. This is used for
+		// backwards compatibility with pre-partfile versions of libtorrent. If
+		// this vector is empty, the default is that files *do* use the partfile.
+		// on startup, any 0-priority file that's found in it's original location
+		// is expected to be an old-style (pre-partfile) torrent storage, and
+		// those files have their slot set to false in this vector.
+		// note that the vector is *sparse*, it's only allocated if a file has its
+		// entry set to false, and only indices up to that entry.
+		aux::vector<bool, file_index_t> m_use_partfile;
 
 		// the file pool is a member of the disk_io_thread
 		// to make all storage instances share the pool

include/libtorrent/torrent_status.hpp

@@ -151,6 +151,9 @@ namespace libtorrent {
 		// or a torrent log alert may provide more information.
 		static constexpr file_index_t error_file_exception{-5};
 
+		// the error occurred with the partfile
+		static constexpr file_index_t error_file_partfile{-6};
+
 		// the path to the directory where this torrent's files are stored.
 		// It's typically the path as was given to async_add_torrent() or
 		// add_torrent() when this torrent was started. This field is only

simulation/create_torrent.hpp

@@ -30,8 +30,8 @@ POSSIBILITY OF SUCH DAMAGE.
 
 */
 
-#ifndef TORRENT_CREATE_TORRENT_HPP_INCLUDED
-#define TORRENT_CREATE_TORRENT_HPP_INCLUDED
+#ifndef TORRENT_SIM_CREATE_TORRENT_HPP_INCLUDED
+#define TORRENT_SIM_CREATE_TORRENT_HPP_INCLUDED
 
 #include <string>
 #include "libtorrent/add_torrent_params.hpp"

simulation/test_checking.cpp

@@ -35,15 +35,20 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/address.hpp"
 #include "libtorrent/torrent_status.hpp"
 #include "libtorrent/torrent_info.hpp"
+#include "libtorrent/file.hpp"
+#include "libtorrent/create_torrent.hpp"
+#include "libtorrent/hex.hpp"
 #include "simulator/simulator.hpp"
 #include "simulator/utils.hpp"
 
 #include "test.hpp"
 #include "settings.hpp"
+#include "setup_transfer.hpp" // for create_random_files
 #include "create_torrent.hpp"
 #include "utils.hpp"
 
 #include <fstream>
+#include <iostream>
 
 template <typename Setup, typename Test>
 void run_test(Setup const& setup, Test const& test)
@@ -85,7 +90,7 @@ TORRENT_TEST(no_truncate_checking)
 	int size = 0;
 	run_test(
 		[&](lt::add_torrent_params& atp, lt::settings_pack& p) {
-			filename = atp.save_path + "/" + atp.ti->files().file_path(lt::file_index_t{0});
+			filename = lt::combine_path(atp.save_path, atp.ti->files().file_path(lt::file_index_t{0}));
 			std::ofstream f(filename);
 			// create a file that's 100 bytes larger
 			size = int(atp.ti->files().file_size(lt::file_index_t{0}) + 100);
@@ -101,6 +106,105 @@ TORRENT_TEST(no_truncate_checking)
 	TEST_EQUAL(f.tellg(), std::fstream::pos_type(size));
 }
 
+std::shared_ptr<lt::torrent_info> create_multifile_torrent()
+{
+	// the two first files are exactly the size of a piece
+	static const int file_sizes[] = { 0x40000, 0x40000, 4300, 0, 400, 4300, 6, 4};
+	const int num_files = sizeof(file_sizes)/sizeof(file_sizes[0]);
+
+	lt::file_storage fs;
+	create_random_files("test_torrent_dir", file_sizes, num_files, &fs);
+	lt::create_torrent t(fs, 0x40000, -1, {});
+
+	// calculate the hash for all pieces
+	lt::error_code ec;
+	set_piece_hashes(t, ".");
+
+	std::vector<char> buf;
+	lt::bencode(std::back_inserter(buf), t.generate());
+	return std::make_shared<lt::torrent_info>(buf, lt::from_span);
+}
+
+TORRENT_TEST(aligned_zero_priority)
+{
+	run_test(
+		[&](lt::add_torrent_params& atp, lt::settings_pack& p) {
+			atp.file_priorities.push_back(lt::download_priority_t{1});
+			atp.file_priorities.push_back(lt::download_priority_t{0});
+			atp.ti = create_multifile_torrent();
+			atp.save_path = ".";
+		},
+		[](lt::session& ses) {
+			std::vector<lt::torrent_handle> tor = ses.get_torrents();
+			TEST_EQUAL(tor.size(), 1);
+			TEST_EQUAL(tor[0].status().is_finished, true);
+		}
+	);
+}
+
+// we have a zero-priority file that also does not exist on disk. It does not
+// overlap any piece in another file, so we don't need a partfile
+TORRENT_TEST(aligned_zero_priority_no_file)
+{
+	std::string partfile;
+	run_test(
+		[&](lt::add_torrent_params& atp, lt::settings_pack& p) {
+			atp.ti = create_multifile_torrent();
+			atp.save_path = ".";
+			atp.file_priorities.push_back(lt::download_priority_t{1});
+			atp.file_priorities.push_back(lt::download_priority_t{0});
+			std::string filename = lt::combine_path(lt::current_working_directory()
+				, lt::combine_path(atp.save_path, atp.ti->files().file_path(lt::file_index_t{1})));
+			partfile = lt::combine_path(lt::current_working_directory()
+				, lt::combine_path(atp.save_path, "." + lt::aux::to_hex(atp.ti->info_hash().to_string()) + ".parts"));
+			lt::error_code ec;
+			lt::remove(filename, ec);
+			TEST_CHECK(!ec);
+		},
+		[](lt::session& ses) {
+			std::vector<lt::torrent_handle> tor = ses.get_torrents();
+			TEST_EQUAL(tor.size(), 1);
+			TEST_EQUAL(tor[0].status().is_finished, true);
+		}
+	);
+
+	// the part file should not have been created. There is no need for a
+	// partfile
+	lt::error_code ec;
+	lt::file_status fs;
+	stat_file(partfile, &fs, ec);
+	TEST_EQUAL(ec, boost::system::errc::no_such_file_or_directory);
+}
+
+// we have a file whose priority is 0, we don't have the file on disk nor a
+// part-file for it. The checking should complete and enter download state.
+TORRENT_TEST(zero_priority_missing_partfile)
+{
+	std::shared_ptr<lt::torrent_info> ti = create_multifile_torrent();
+	run_test(
+		[&](lt::add_torrent_params& atp, lt::settings_pack& p) {
+			atp.ti = ti;
+			atp.save_path = ".";
+			atp.file_priorities.push_back(lt::download_priority_t{1});
+			atp.file_priorities.push_back(lt::download_priority_t{1});
+			atp.file_priorities.push_back(lt::download_priority_t{0});
+			std::string const filename = lt::combine_path(lt::current_working_directory()
+				, lt::combine_path(atp.save_path, atp.ti->files().file_path(lt::file_index_t{2})));
+
+			std::cout << "removing: " << filename << "\n";
+			lt::error_code ec;
+			lt::remove(filename, ec);
+			TEST_CHECK(!ec);
+		},
+		[&](lt::session& ses) {
+			std::vector<lt::torrent_handle> tor = ses.get_torrents();
+			TEST_EQUAL(tor.size(), 1);
+			TEST_EQUAL(tor[0].status().num_pieces, ti->num_pieces() - 1);
+			TEST_EQUAL(tor[0].status().is_finished, false);
+		}
+	);
+}
+
 TORRENT_TEST(cache_after_checking)
 {
 	run_test(

src/create_torrent.cpp

@@ -251,7 +251,13 @@ namespace {
 		, std::function<void(piece_index_t)> const& f, error_code& ec)
 	{
 		// optimized path
+#ifdef TORRENT_BUILD_SIMULATOR
+		sim::default_config conf;
+		sim::simulation sim{conf};
+		io_service ios{sim};
+#else
 		io_service ios;
+#endif
 
 #if TORRENT_USE_UNC_PATHS
 		std::string const path = canonicalize_path(p);
@@ -305,7 +311,13 @@ namespace {
 			if (st.piece_counter >= t.files().end_piece()) break;
 		}
 		disk_thread.submit_jobs();
+
+#ifdef TORRENT_BUILD_SIMULATOR
+		sim.run();
+#else
 		ios.run(ec);
+#endif
+		disk_thread.abort(true);
 	}
 
 	create_torrent::~create_torrent() = default;

src/storage.cpp

@@ -91,12 +91,26 @@ namespace libtorrent {
 		m_part_file_name = "." + aux::to_hex(params.info_hash) + ".parts";
 
 		file_storage const& fs = files();
+		// if some files have priority 0, we need to check if they exist on the
+		// filesystem, in which case we won't use a partfile for them.
+		// this is to be backwards compatible with previous versions of
+		// libtorrent, when part files were not supported.
 		for (file_index_t i(0); i < m_file_priority.end_index(); ++i)
 		{
-			if (m_file_priority[i] == dont_download && !fs.pad_file_at(i))
+			if (m_file_priority[i] != dont_download || fs.pad_file_at(i))
+				continue;
+
+			file_status s;
+			std::string const file_path = files().file_path(i, m_save_path);
+			error_code ec;
+			stat_file(file_path, &s, ec);
+			if (!ec)
+			{
+				use_partfile(i, false);
+			}
+			else
 			{
 				need_partfile();
-				break;
 			}
 		}
 	}
@@ -132,30 +146,39 @@ namespace libtorrent {
 		file_storage const& fs = files();
 		for (file_index_t i(0); i < prio.end_index(); ++i)
 		{
+			// pad files always have priority 0.
+			if (fs.pad_file_at(i)) continue;
+
 			download_priority_t const old_prio = m_file_priority[i];
 			download_priority_t new_prio = prio[i];
 			if (old_prio == dont_download && new_prio != dont_download)
 			{
 				// move stuff out of the part file
 				file_handle f = open_file(i, open_mode::read_write, ec);
-				if (ec) return;
-
-				need_partfile();
-
-				m_part_file->export_file([&f, &ec](std::int64_t file_offset, span<char> buf)
-				{
-					iovec_t const v = {buf.data(), buf.size()};
-					std::int64_t const ret = f->writev(file_offset, v, ec.ec);
-					TORRENT_UNUSED(ret);
-					TORRENT_ASSERT(ec || ret == std::int64_t(v.size()));
-				}, fs.file_offset(i), fs.file_size(i), ec.ec);
-
 				if (ec)
 				{
 					ec.file(i);
-					ec.operation = operation_t::partfile_write;
+					ec.operation = operation_t::file_open;
 					return;
 				}
+
+				if (m_part_file)
+				{
+					m_part_file->export_file([&f, &ec](std::int64_t file_offset, span<char> buf)
+					{
+						iovec_t const v = {buf.data(), buf.size()};
+						std::int64_t const ret = f->writev(file_offset, v, ec.ec);
+						TORRENT_UNUSED(ret);
+						TORRENT_ASSERT(ec || ret == std::int64_t(v.size()));
+					}, fs.file_offset(i), fs.file_size(i), ec.ec);
+
+					if (ec)
+					{
+						ec.file(i);
+						ec.operation = operation_t::partfile_write;
+						return;
+					}
+				}
 			}
 			else if (old_prio != dont_download && new_prio == dont_download)
 			{
@@ -195,17 +218,32 @@ namespace libtorrent {
 			ec.ec.clear();
 			m_file_priority[i] = new_prio;
 
-			if (m_file_priority[i] == dont_download && !fs.pad_file_at(i))
+			if (m_file_priority[i] == dont_download && use_partfile(i))
+			{
 				need_partfile();
+			}
 		}
 		if (m_part_file) m_part_file->flush_metadata(ec.ec);
 		if (ec)
 		{
-			ec.file(file_index_t(-1));
+			ec.file(torrent_status::error_file_partfile);
 			ec.operation = operation_t::partfile_write;
 		}
 	}
 
+	bool default_storage::use_partfile(file_index_t const index) const
+	{
+		TORRENT_ASSERT_VAL(index >= file_index_t{}, index);
+		if (index >= m_use_partfile.end_index()) return true;
+		return m_use_partfile[index];
+	}
+
+	void default_storage::use_partfile(file_index_t const index, bool const b)
+	{
+		if (index >= m_use_partfile.end_index()) m_use_partfile.resize(static_cast<int>(index) + 1, true);
+		m_use_partfile[index] = b;
+	}
+
 	void default_storage::initialize(storage_error& ec)
 	{
 		m_stat_cache.reserve(files().num_files());
@@ -307,7 +345,7 @@ namespace libtorrent {
 
 		if (ec)
 		{
-			ec.file(file_index_t(-1));
+			ec.file(torrent_status::error_file_partfile);
 			ec.operation = operation_t::file_stat;
 			return false;
 		}
@@ -460,7 +498,7 @@ namespace libtorrent {
 
 			if (file_index < m_file_priority.end_index()
 				&& m_file_priority[file_index] == dont_download
-				&& m_use_part_file)
+				&& use_partfile(file_index))
 			{
 				TORRENT_ASSERT(m_part_file);
 
@@ -498,6 +536,7 @@ namespace libtorrent {
 
 			if (e)
 			{
+				ec.file(torrent_status::error_file_partfile);
 				ec.ec = e;
 				ec.file(file_index);
 				return -1;
@@ -524,7 +563,7 @@ namespace libtorrent {
 
 			if (file_index < m_file_priority.end_index()
 				&& m_file_priority[file_index] == dont_download
-				&& m_use_part_file)
+				&& use_partfile(file_index))
 			{
 				TORRENT_ASSERT(m_part_file);
 

src/storage_utils.cpp

@@ -317,7 +317,7 @@ namespace libtorrent { namespace aux {
 			if (e)
 			{
 				ec.ec = e;
-				ec.file(file_index_t(-1));
+				ec.file(torrent_status::error_file_partfile);
 				ec.operation = operation_t::partfile_move;
 			}
 		}

src/torrent.cpp

@@ -2722,6 +2722,8 @@ namespace libtorrent {
 		// exclude redundant bytes if we should
 		if (!settings().get_bool(settings_pack::report_true_downloaded))
 			req.downloaded -= m_total_redundant_bytes;
+		if (settings().get_bool(settings_pack::report_redundant_bytes))
+			req.redundant = m_total_redundant_bytes;
 		if (req.downloaded < 0) req.downloaded = 0;
 
 		req.event = e;
@@ -8186,11 +8188,12 @@ namespace libtorrent {
 	{
 		if (file == torrent_status::error_file_none) return "";
 #ifndef TORRENT_NO_DEPRECATE
-		// deprecated in 1.2
 		if (file == torrent_status::error_file_url) return m_url;
 #endif
 		if (file == torrent_status::error_file_ssl_ctx) return "SSL Context";
 		if (file == torrent_status::error_file_exception) return "exception";
+		if (file == torrent_status::error_file_partfile) return "partfile";
+		if (file == torrent_status::error_file_metadata) return "metadata (from user load function)";
 
 		if (m_storage && file >= file_index_t(0))
 		{

src/torrent_status.cpp

@@ -37,8 +37,9 @@ namespace libtorrent {
 	file_index_t constexpr torrent_status::error_file_none;
 	file_index_t constexpr torrent_status::error_file_url;
 	file_index_t constexpr torrent_status::error_file_ssl_ctx;
-	file_index_t constexpr torrent_status::error_file_metadata;
 	file_index_t constexpr torrent_status::error_file_exception;
+	file_index_t constexpr torrent_status::error_file_partfile;
+	file_index_t constexpr torrent_status::error_file_metadata;
 
 	torrent_status::torrent_status() noexcept {}
 	torrent_status::~torrent_status() = default;

test/setup_transfer.cpp

@@ -650,7 +650,10 @@ void create_random_files(std::string const& path, const int file_sizes[], int nu
 		std::snprintf(dirname, sizeof(dirname), "test_dir%d", i / 5);
 
 		std::string full_path = combine_path(path, dirname);
-		create_directory(full_path, ec);
+		lt::create_directories(full_path, ec);
+		if (ec) fprintf(stderr, "create_directory(%s) failed: (%d) %s\n"
+			, full_path.c_str(), ec.value(), ec.message().c_str());
+
 		full_path = combine_path(full_path, filename);
 
 		int to_write = file_sizes[i];