diff --git a/ChangeLog b/ChangeLog index aa8d5e500..5e243b27d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -82,6 +82,9 @@ * resume data no longer has timestamps of files * require C++11 to build libtorrent + * fix backwards compatibility to downloads without partfiles + * improve part-file related error messages + * fix reporting &redundant= in tracker announces * fix tie-break in duplicate peer connection disconnect logic * fix issue with SSL tracker connections left in CLOSE_WAIT state * defer truncating existing files until the first time we write to them diff --git a/bindings/python/test.py b/bindings/python/test.py index ea288e377..7cf31e49b 100644 --- a/bindings/python/test.py +++ b/bindings/python/test.py @@ -79,6 +79,10 @@ class test_torrent_handle(unittest.TestCase): # also test the overload that takes a list of piece->priority mappings self.h.prioritize_pieces([(0, 1)]) self.assertEqual(self.h.get_piece_priorities(), [1]) + self.h.connect_peer(('127.0.0.1', 6881)) + self.h.connect_peer(('127.0.0.2', 6881), source=4) + self.h.connect_peer(('127.0.0.3', 6881), flags=2) + self.h.connect_peer(('127.0.0.4', 6881), flags=2, source=4) print(self.h.queue_position()) diff --git a/docs/hunspell/libtorrent.dic b/docs/hunspell/libtorrent.dic index 779c7e882..a07174774 100644 --- a/docs/hunspell/libtorrent.dic +++ b/docs/hunspell/libtorrent.dic @@ -189,6 +189,8 @@ src 'fingerprints' 'query' 'ro' +pre-partfile +pre GCC prioritization nullptr diff --git a/include/libtorrent/storage.hpp b/include/libtorrent/storage.hpp index 9b9a3a550..95e0bd2dc 100644 --- a/include/libtorrent/storage.hpp +++ b/include/libtorrent/storage.hpp @@ -439,16 +439,23 @@ namespace libtorrent { file_handle open_file(file_index_t file, open_mode_t mode, storage_error& ec) const; file_handle open_file_impl(file_index_t file, open_mode_t mode, error_code& ec) const; + bool use_partfile(file_index_t index) const; + void use_partfile(file_index_t index, bool b); + aux::vector m_file_priority; std::string m_save_path; std::string m_part_file_name; - // if this is false, we're not using a part file to store priority-0 - // pieces, but we instead look for them under their actual file names - // this defaults to true, but when checking resume data for a torrent - // where we would expect to have a part file, but there isn't one, we set - // this to false. - bool m_use_part_file = true; + // this this is an array indexed by file-index. Each slot represents + // whether this file has the part-file enabled for it. This is used for + // backwards compatibility with pre-partfile versions of libtorrent. If + // this vector is empty, the default is that files *do* use the partfile. + // on startup, any 0-priority file that's found in it's original location + // is expected to be an old-style (pre-partfile) torrent storage, and + // those files have their slot set to false in this vector. + // note that the vector is *sparse*, it's only allocated if a file has its + // entry set to false, and only indices up to that entry. + aux::vector m_use_partfile; // the file pool is a member of the disk_io_thread // to make all storage instances share the pool diff --git a/include/libtorrent/torrent_status.hpp b/include/libtorrent/torrent_status.hpp index ce0e05f3c..2417cde7f 100644 --- a/include/libtorrent/torrent_status.hpp +++ b/include/libtorrent/torrent_status.hpp @@ -151,6 +151,9 @@ namespace libtorrent { // or a torrent log alert may provide more information. static constexpr file_index_t error_file_exception{-5}; + // the error occurred with the partfile + static constexpr file_index_t error_file_partfile{-6}; + // the path to the directory where this torrent's files are stored. // It's typically the path as was given to async_add_torrent() or // add_torrent() when this torrent was started. This field is only diff --git a/simulation/create_torrent.hpp b/simulation/create_torrent.hpp index f3c289d3a..b0e8e3be2 100644 --- a/simulation/create_torrent.hpp +++ b/simulation/create_torrent.hpp @@ -30,8 +30,8 @@ POSSIBILITY OF SUCH DAMAGE. */ -#ifndef TORRENT_CREATE_TORRENT_HPP_INCLUDED -#define TORRENT_CREATE_TORRENT_HPP_INCLUDED +#ifndef TORRENT_SIM_CREATE_TORRENT_HPP_INCLUDED +#define TORRENT_SIM_CREATE_TORRENT_HPP_INCLUDED #include #include "libtorrent/add_torrent_params.hpp" diff --git a/simulation/test_checking.cpp b/simulation/test_checking.cpp index da023b6c4..c9707d83f 100644 --- a/simulation/test_checking.cpp +++ b/simulation/test_checking.cpp @@ -35,15 +35,20 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/address.hpp" #include "libtorrent/torrent_status.hpp" #include "libtorrent/torrent_info.hpp" +#include "libtorrent/file.hpp" +#include "libtorrent/create_torrent.hpp" +#include "libtorrent/hex.hpp" #include "simulator/simulator.hpp" #include "simulator/utils.hpp" #include "test.hpp" #include "settings.hpp" +#include "setup_transfer.hpp" // for create_random_files #include "create_torrent.hpp" #include "utils.hpp" #include +#include template void run_test(Setup const& setup, Test const& test) @@ -85,7 +90,7 @@ TORRENT_TEST(no_truncate_checking) int size = 0; run_test( [&](lt::add_torrent_params& atp, lt::settings_pack& p) { - filename = atp.save_path + "/" + atp.ti->files().file_path(lt::file_index_t{0}); + filename = lt::combine_path(atp.save_path, atp.ti->files().file_path(lt::file_index_t{0})); std::ofstream f(filename); // create a file that's 100 bytes larger size = int(atp.ti->files().file_size(lt::file_index_t{0}) + 100); @@ -101,6 +106,105 @@ TORRENT_TEST(no_truncate_checking) TEST_EQUAL(f.tellg(), std::fstream::pos_type(size)); } +std::shared_ptr create_multifile_torrent() +{ + // the two first files are exactly the size of a piece + static const int file_sizes[] = { 0x40000, 0x40000, 4300, 0, 400, 4300, 6, 4}; + const int num_files = sizeof(file_sizes)/sizeof(file_sizes[0]); + + lt::file_storage fs; + create_random_files("test_torrent_dir", file_sizes, num_files, &fs); + lt::create_torrent t(fs, 0x40000, -1, {}); + + // calculate the hash for all pieces + lt::error_code ec; + set_piece_hashes(t, "."); + + std::vector buf; + lt::bencode(std::back_inserter(buf), t.generate()); + return std::make_shared(buf, lt::from_span); +} + +TORRENT_TEST(aligned_zero_priority) +{ + run_test( + [&](lt::add_torrent_params& atp, lt::settings_pack& p) { + atp.file_priorities.push_back(lt::download_priority_t{1}); + atp.file_priorities.push_back(lt::download_priority_t{0}); + atp.ti = create_multifile_torrent(); + atp.save_path = "."; + }, + [](lt::session& ses) { + std::vector tor = ses.get_torrents(); + TEST_EQUAL(tor.size(), 1); + TEST_EQUAL(tor[0].status().is_finished, true); + } + ); +} + +// we have a zero-priority file that also does not exist on disk. It does not +// overlap any piece in another file, so we don't need a partfile +TORRENT_TEST(aligned_zero_priority_no_file) +{ + std::string partfile; + run_test( + [&](lt::add_torrent_params& atp, lt::settings_pack& p) { + atp.ti = create_multifile_torrent(); + atp.save_path = "."; + atp.file_priorities.push_back(lt::download_priority_t{1}); + atp.file_priorities.push_back(lt::download_priority_t{0}); + std::string filename = lt::combine_path(lt::current_working_directory() + , lt::combine_path(atp.save_path, atp.ti->files().file_path(lt::file_index_t{1}))); + partfile = lt::combine_path(lt::current_working_directory() + , lt::combine_path(atp.save_path, "." + lt::aux::to_hex(atp.ti->info_hash().to_string()) + ".parts")); + lt::error_code ec; + lt::remove(filename, ec); + TEST_CHECK(!ec); + }, + [](lt::session& ses) { + std::vector tor = ses.get_torrents(); + TEST_EQUAL(tor.size(), 1); + TEST_EQUAL(tor[0].status().is_finished, true); + } + ); + + // the part file should not have been created. There is no need for a + // partfile + lt::error_code ec; + lt::file_status fs; + stat_file(partfile, &fs, ec); + TEST_EQUAL(ec, boost::system::errc::no_such_file_or_directory); +} + +// we have a file whose priority is 0, we don't have the file on disk nor a +// part-file for it. The checking should complete and enter download state. +TORRENT_TEST(zero_priority_missing_partfile) +{ + std::shared_ptr ti = create_multifile_torrent(); + run_test( + [&](lt::add_torrent_params& atp, lt::settings_pack& p) { + atp.ti = ti; + atp.save_path = "."; + atp.file_priorities.push_back(lt::download_priority_t{1}); + atp.file_priorities.push_back(lt::download_priority_t{1}); + atp.file_priorities.push_back(lt::download_priority_t{0}); + std::string const filename = lt::combine_path(lt::current_working_directory() + , lt::combine_path(atp.save_path, atp.ti->files().file_path(lt::file_index_t{2}))); + + std::cout << "removing: " << filename << "\n"; + lt::error_code ec; + lt::remove(filename, ec); + TEST_CHECK(!ec); + }, + [&](lt::session& ses) { + std::vector tor = ses.get_torrents(); + TEST_EQUAL(tor.size(), 1); + TEST_EQUAL(tor[0].status().num_pieces, ti->num_pieces() - 1); + TEST_EQUAL(tor[0].status().is_finished, false); + } + ); +} + TORRENT_TEST(cache_after_checking) { run_test( diff --git a/src/create_torrent.cpp b/src/create_torrent.cpp index 63a68e05d..7a1be4fac 100644 --- a/src/create_torrent.cpp +++ b/src/create_torrent.cpp @@ -251,7 +251,13 @@ namespace { , std::function const& f, error_code& ec) { // optimized path +#ifdef TORRENT_BUILD_SIMULATOR + sim::default_config conf; + sim::simulation sim{conf}; + io_service ios{sim}; +#else io_service ios; +#endif #if TORRENT_USE_UNC_PATHS std::string const path = canonicalize_path(p); @@ -305,7 +311,13 @@ namespace { if (st.piece_counter >= t.files().end_piece()) break; } disk_thread.submit_jobs(); + +#ifdef TORRENT_BUILD_SIMULATOR + sim.run(); +#else ios.run(ec); +#endif + disk_thread.abort(true); } create_torrent::~create_torrent() = default; diff --git a/src/storage.cpp b/src/storage.cpp index d0f2daaf0..570932d84 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -91,12 +91,26 @@ namespace libtorrent { m_part_file_name = "." + aux::to_hex(params.info_hash) + ".parts"; file_storage const& fs = files(); + // if some files have priority 0, we need to check if they exist on the + // filesystem, in which case we won't use a partfile for them. + // this is to be backwards compatible with previous versions of + // libtorrent, when part files were not supported. for (file_index_t i(0); i < m_file_priority.end_index(); ++i) { - if (m_file_priority[i] == dont_download && !fs.pad_file_at(i)) + if (m_file_priority[i] != dont_download || fs.pad_file_at(i)) + continue; + + file_status s; + std::string const file_path = files().file_path(i, m_save_path); + error_code ec; + stat_file(file_path, &s, ec); + if (!ec) + { + use_partfile(i, false); + } + else { need_partfile(); - break; } } } @@ -132,30 +146,39 @@ namespace libtorrent { file_storage const& fs = files(); for (file_index_t i(0); i < prio.end_index(); ++i) { + // pad files always have priority 0. + if (fs.pad_file_at(i)) continue; + download_priority_t const old_prio = m_file_priority[i]; download_priority_t new_prio = prio[i]; if (old_prio == dont_download && new_prio != dont_download) { // move stuff out of the part file file_handle f = open_file(i, open_mode::read_write, ec); - if (ec) return; - - need_partfile(); - - m_part_file->export_file([&f, &ec](std::int64_t file_offset, span buf) - { - iovec_t const v = {buf.data(), buf.size()}; - std::int64_t const ret = f->writev(file_offset, v, ec.ec); - TORRENT_UNUSED(ret); - TORRENT_ASSERT(ec || ret == std::int64_t(v.size())); - }, fs.file_offset(i), fs.file_size(i), ec.ec); - if (ec) { ec.file(i); - ec.operation = operation_t::partfile_write; + ec.operation = operation_t::file_open; return; } + + if (m_part_file) + { + m_part_file->export_file([&f, &ec](std::int64_t file_offset, span buf) + { + iovec_t const v = {buf.data(), buf.size()}; + std::int64_t const ret = f->writev(file_offset, v, ec.ec); + TORRENT_UNUSED(ret); + TORRENT_ASSERT(ec || ret == std::int64_t(v.size())); + }, fs.file_offset(i), fs.file_size(i), ec.ec); + + if (ec) + { + ec.file(i); + ec.operation = operation_t::partfile_write; + return; + } + } } else if (old_prio != dont_download && new_prio == dont_download) { @@ -195,17 +218,32 @@ namespace libtorrent { ec.ec.clear(); m_file_priority[i] = new_prio; - if (m_file_priority[i] == dont_download && !fs.pad_file_at(i)) + if (m_file_priority[i] == dont_download && use_partfile(i)) + { need_partfile(); + } } if (m_part_file) m_part_file->flush_metadata(ec.ec); if (ec) { - ec.file(file_index_t(-1)); + ec.file(torrent_status::error_file_partfile); ec.operation = operation_t::partfile_write; } } + bool default_storage::use_partfile(file_index_t const index) const + { + TORRENT_ASSERT_VAL(index >= file_index_t{}, index); + if (index >= m_use_partfile.end_index()) return true; + return m_use_partfile[index]; + } + + void default_storage::use_partfile(file_index_t const index, bool const b) + { + if (index >= m_use_partfile.end_index()) m_use_partfile.resize(static_cast(index) + 1, true); + m_use_partfile[index] = b; + } + void default_storage::initialize(storage_error& ec) { m_stat_cache.reserve(files().num_files()); @@ -307,7 +345,7 @@ namespace libtorrent { if (ec) { - ec.file(file_index_t(-1)); + ec.file(torrent_status::error_file_partfile); ec.operation = operation_t::file_stat; return false; } @@ -460,7 +498,7 @@ namespace libtorrent { if (file_index < m_file_priority.end_index() && m_file_priority[file_index] == dont_download - && m_use_part_file) + && use_partfile(file_index)) { TORRENT_ASSERT(m_part_file); @@ -498,6 +536,7 @@ namespace libtorrent { if (e) { + ec.file(torrent_status::error_file_partfile); ec.ec = e; ec.file(file_index); return -1; @@ -524,7 +563,7 @@ namespace libtorrent { if (file_index < m_file_priority.end_index() && m_file_priority[file_index] == dont_download - && m_use_part_file) + && use_partfile(file_index)) { TORRENT_ASSERT(m_part_file); diff --git a/src/storage_utils.cpp b/src/storage_utils.cpp index 298409bfd..4bdfb7177 100644 --- a/src/storage_utils.cpp +++ b/src/storage_utils.cpp @@ -317,7 +317,7 @@ namespace libtorrent { namespace aux { if (e) { ec.ec = e; - ec.file(file_index_t(-1)); + ec.file(torrent_status::error_file_partfile); ec.operation = operation_t::partfile_move; } } diff --git a/src/torrent.cpp b/src/torrent.cpp index 1f3d23801..7688d6b57 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -2722,6 +2722,8 @@ namespace libtorrent { // exclude redundant bytes if we should if (!settings().get_bool(settings_pack::report_true_downloaded)) req.downloaded -= m_total_redundant_bytes; + if (settings().get_bool(settings_pack::report_redundant_bytes)) + req.redundant = m_total_redundant_bytes; if (req.downloaded < 0) req.downloaded = 0; req.event = e; @@ -8186,11 +8188,12 @@ namespace libtorrent { { if (file == torrent_status::error_file_none) return ""; #ifndef TORRENT_NO_DEPRECATE - // deprecated in 1.2 if (file == torrent_status::error_file_url) return m_url; #endif if (file == torrent_status::error_file_ssl_ctx) return "SSL Context"; if (file == torrent_status::error_file_exception) return "exception"; + if (file == torrent_status::error_file_partfile) return "partfile"; + if (file == torrent_status::error_file_metadata) return "metadata (from user load function)"; if (m_storage && file >= file_index_t(0)) { diff --git a/src/torrent_status.cpp b/src/torrent_status.cpp index 80c8976d1..8039cc052 100644 --- a/src/torrent_status.cpp +++ b/src/torrent_status.cpp @@ -37,8 +37,9 @@ namespace libtorrent { file_index_t constexpr torrent_status::error_file_none; file_index_t constexpr torrent_status::error_file_url; file_index_t constexpr torrent_status::error_file_ssl_ctx; - file_index_t constexpr torrent_status::error_file_metadata; file_index_t constexpr torrent_status::error_file_exception; + file_index_t constexpr torrent_status::error_file_partfile; + file_index_t constexpr torrent_status::error_file_metadata; torrent_status::torrent_status() noexcept {} torrent_status::~torrent_status() = default; diff --git a/test/setup_transfer.cpp b/test/setup_transfer.cpp index 90a4deee9..adf05b7bb 100644 --- a/test/setup_transfer.cpp +++ b/test/setup_transfer.cpp @@ -650,7 +650,10 @@ void create_random_files(std::string const& path, const int file_sizes[], int nu std::snprintf(dirname, sizeof(dirname), "test_dir%d", i / 5); std::string full_path = combine_path(path, dirname); - create_directory(full_path, ec); + lt::create_directories(full_path, ec); + if (ec) fprintf(stderr, "create_directory(%s) failed: (%d) %s\n" + , full_path.c_str(), ec.value(), ec.message().c_str()); + full_path = combine_path(full_path, filename); int to_write = file_sizes[i];