diff --git a/ChangeLog b/ChangeLog index daa762535..c7d3a1b63 100644 --- a/ChangeLog +++ b/ChangeLog @@ -59,6 +59,10 @@ * resume data no longer has timestamps of files * require C++11 to build libtorrent + * add feature to periodically close files (to make windows clear disk cache) + * fix bug in torrent_handle::file_status + * fix issue with peers not updated on metadata from magnet links + 1.1.2 release * default TOS marking to 0x20 diff --git a/Makefile.am b/Makefile.am index 82cea584c..179b3f046 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,20 +10,16 @@ DOCS_IMAGES = \ docs/cwnd_thumb.png \ docs/delays.png \ docs/delays_thumb.png \ - docs/disk_io.png \ docs/hacking.html \ - docs/im_thumb.jpg \ docs/merkle_tree.png \ docs/our_delay_base.png \ docs/our_delay_base_thumb.png \ docs/read_disk_buffers.png \ docs/read_disk_buffers.diagram \ - docs/session_stats_peers.png \ docs/storage.png \ docs/todo.html \ docs/write_disk_buffers.png \ docs/write_disk_buffers.diagram \ - docs/ziptorrent_thumb.gif \ docs/ip_id_v4.png \ docs/ip_id_v6.png \ docs/hash_distribution.png \ diff --git a/docs/disk_io.png b/docs/disk_io.png deleted file mode 100644 index 0ec82d167..000000000 Binary files a/docs/disk_io.png and /dev/null differ diff --git a/docs/im_thumb.jpg b/docs/im_thumb.jpg deleted file mode 100644 index 1dcfd75d3..000000000 Binary files a/docs/im_thumb.jpg and /dev/null differ diff --git a/docs/session_stats_peers.png b/docs/session_stats_peers.png deleted file mode 100644 index e134f310f..000000000 Binary files a/docs/session_stats_peers.png and /dev/null differ diff --git a/docs/ziptorrent_thumb.gif b/docs/ziptorrent_thumb.gif deleted file mode 100644 index 11f0d99f2..000000000 Binary files a/docs/ziptorrent_thumb.gif and /dev/null differ diff --git a/examples/Makefile.am b/examples/Makefile.am index beb13c8dc..a78b29ac6 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -30,7 +30,7 @@ upnp_test_SOURCES = upnp_test.cpp LDADD = $(top_builddir)/src/libtorrent-rasterbar.la -AM_CPPFLAGS = -ftemplate-depth-50 -I$(top_srcdir)/include @DEBUGFLAGS@ +AM_CPPFLAGS = -ftemplate-depth-50 -I$(top_srcdir)/include @DEBUGFLAGS@ @OPENSSL_INCLUDES@ AM_LDFLAGS = @BOOST_SYSTEM_LIB@ @OPENSSL_LDFLAGS@ @OPENSSL_LIBS@ diff --git a/include/libtorrent/aux_/session_impl.hpp b/include/libtorrent/aux_/session_impl.hpp index a49208e09..842581d85 100644 --- a/include/libtorrent/aux_/session_impl.hpp +++ b/include/libtorrent/aux_/session_impl.hpp @@ -700,6 +700,7 @@ namespace libtorrent void update_privileged_ports(); void update_auto_sequential(); void update_max_failcount(); + void update_close_file_interval(); void update_upnp(); void update_natpmp(); @@ -976,6 +977,7 @@ namespace libtorrent int m_peak_down_rate = 0; void on_tick(error_code const& e); + void on_close_file(error_code const& e); void try_connect_more_peers(); void auto_manage_checking_torrents(std::vector& list @@ -1157,6 +1159,11 @@ namespace libtorrent // by Local service discovery deadline_timer m_lsd_announce_timer; + // this is the timer used to call ``close_oldest`` on the ``file_pool`` + // object. This closes the file that's been opened the longest every + // time it's called, to force the windows disk cache to be flushed + deadline_timer m_close_file_timer; + resolver m_host_resolver; // the index of the torrent that will be offered to diff --git a/include/libtorrent/file_pool.hpp b/include/libtorrent/file_pool.hpp index 6623c995c..9ff39990c 100644 --- a/include/libtorrent/file_pool.hpp +++ b/include/libtorrent/file_pool.hpp @@ -114,6 +114,12 @@ namespace libtorrent void set_low_prio_io(bool b) { m_low_prio_io = b; } std::vector get_status(storage_index_t st) const; + // close the file that was opened least recently (i.e. not *accessed* + // least recently). The purpose is to make the OS (really just windows) + // clear and flush its disk cache associated with this file. We don't want + // any file to stay open for too long, allowing the disk cache to accrue. + void close_oldest(); + #if TORRENT_USE_ASSERTS bool assert_idle_files(storage_index_t st) const; @@ -133,7 +139,8 @@ namespace libtorrent struct lru_file_entry { file_handle file_ptr; - time_point last_use{aux::time_now()}; + time_point const opened{aux::time_now()}; + time_point last_use{opened}; int mode = 0; }; diff --git a/include/libtorrent/settings_pack.hpp b/include/libtorrent/settings_pack.hpp index e44119377..e0a2e9ad2 100644 --- a/include/libtorrent/settings_pack.hpp +++ b/include/libtorrent/settings_pack.hpp @@ -1618,6 +1618,15 @@ namespace libtorrent // time to wait until a new retry of a web seed name lookup web_seed_name_lookup_retry, + // the number of seconds between closing the file opened the longest + // ago. 0 means to disable the feature. The purpose of this is to + // periodically close files to trigger the operating system flushing + // disk cache. Specifically it has been observed to be required on + // windows to not have the disk cache grow indefinitely. + // This defaults to 120 seconds on windows, and disabled on other + // systems. + close_file_interval, + max_int_setting_internal }; diff --git a/simulation/Jamfile b/simulation/Jamfile index 72dd0984a..92e1c02a6 100644 --- a/simulation/Jamfile +++ b/simulation/Jamfile @@ -50,6 +50,7 @@ alias libtorrent-sims : [ run test_ip_filter.cpp ] [ run test_dht_rate_limit.cpp ] [ run test_fast_extensions.cpp ] + [ run test_file_pool.cpp ] [ run test_save_resume.cpp ] ; diff --git a/simulation/test_file_pool.cpp b/simulation/test_file_pool.cpp new file mode 100644 index 000000000..d1f250d10 --- /dev/null +++ b/simulation/test_file_pool.cpp @@ -0,0 +1,86 @@ +/* + +Copyright (c) 2008, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "setup_swarm.hpp" +#include "test.hpp" +#include "utils.hpp" +#include "libtorrent/alert.hpp" +#include "libtorrent/alert_types.hpp" +#include "libtorrent/session.hpp" +#include "libtorrent/session_stats.hpp" +#include "libtorrent/file.hpp" + +using namespace libtorrent; + +TORRENT_TEST(close_file_interval) +{ + bool ran_to_completion = false; + + // with seed mode + setup_swarm(2, swarm_test::download + // add session + , [](lt::settings_pack& pack) { + pack.set_int(settings_pack::close_file_interval, 20); + } + // add torrent + , [](lt::add_torrent_params&) {} + // on alert + , [](lt::alert const* a, lt::session& ses) {} + // terminate + , [&](int ticks, lt::session& ses) -> bool + { + // terminate after 40 seconds + if (ticks > 24) + { + ran_to_completion = true; + return true; + } + + torrent_handle h = ses.get_torrents().front(); + std::vector file_status; + h.file_status(file_status); + printf("%d: %d files\n", ticks, int(file_status.size())); + if (ticks > 0 && ticks < 19) + { + TEST_EQUAL(file_status.size(), 1); + } + else if (ticks > 21) + { + // the close file timer shuold have kicked in at 20 seconds + // and closed the file + TEST_EQUAL(file_status.size(), 0); + } + return false; + }); + TEST_CHECK(ran_to_completion); +} + diff --git a/src/file_pool.cpp b/src/file_pool.cpp index 69cfc70ba..c1f8f20d6 100644 --- a/src/file_pool.cpp +++ b/src/file_pool.cpp @@ -291,4 +291,22 @@ namespace libtorrent remove_oldest(l); } + void file_pool::close_oldest() + { + std::unique_lock l(m_mutex); + + using value_type = decltype(m_files)::value_type; + auto const i = std::min_element(m_files.begin(), m_files.end() + , [] (value_type const& lhs, value_type const& rhs) + { return lhs.second.opened < rhs.second.opened; }); + if (i == m_files.end()) return; + + file_handle file_ptr = i->second.file_ptr; + m_files.erase(i); + + // closing a file may be long running operation (mac os x) + l.unlock(); + file_ptr.reset(); + l.lock(); + } } diff --git a/src/session_impl.cpp b/src/session_impl.cpp index f7e27488f..cf4d4433d 100644 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -469,6 +469,7 @@ namespace aux { #endif , m_timer(m_io_service) , m_lsd_announce_timer(m_io_service) + , m_close_file_timer(m_io_service) , m_host_resolver(m_io_service) { update_time_now(); @@ -901,6 +902,8 @@ namespace aux { // about to send event=stopped to m_host_resolver.abort(); + m_close_file_timer.cancel(); + // abort the main thread m_abort = true; error_code ec; @@ -3436,6 +3439,16 @@ namespace aux { // m_peer_pool.release_memory(); } + void session_impl::on_close_file(error_code const& e) + { + if (e) return; + + m_disk_thread.files().close_oldest(); + + // re-issue the timer + update_close_file_interval(); + } + namespace { // returns the index of the first set bit. int log2(std::uint32_t v) @@ -5188,6 +5201,19 @@ namespace aux { } } + void session_impl::update_close_file_interval() + { + int const interval = m_settings.get_int(settings_pack::close_file_interval); + if (interval == 0 || m_abort) + { + m_close_file_timer.cancel(); + return; + } + error_code ec; + m_close_file_timer.expires_from_now(seconds(interval), ec); + m_close_file_timer.async_wait(make_tick_handler(std::bind(&session_impl::on_close_file, this, _1))); + } + void session_impl::update_proxy() { // in case we just set a socks proxy, we might have to diff --git a/src/settings_pack.cpp b/src/settings_pack.cpp index 3488fa349..0ed847936 100644 --- a/src/settings_pack.cpp +++ b/src/settings_pack.cpp @@ -98,6 +98,12 @@ namespace libtorrent #define DEPRECATED_SET(name, default_value, fun) { #name, fun, default_value } #else #define DEPRECATED_SET(name, default_value, fun) { "", nullptr, 0 } +#endif + +#ifdef TORRENT_WINDOWS +#define CLOSE_FILE_INTERVAL 120 +#else +#define CLOSE_FILE_INTERVAL 0 #endif namespace { @@ -327,6 +333,7 @@ namespace libtorrent SET(cache_size_volatile, 256, nullptr), SET(urlseed_max_request_bytes, 16 * 1024 * 1024, 0), SET(web_seed_name_lookup_retry, 1800, nullptr), + SET(close_file_interval, CLOSE_FILE_INTERVAL, &session_impl::update_close_file_interval), }}); #undef SET diff --git a/src/web_connection_base.cpp b/src/web_connection_base.cpp index 7c448e124..529608db5 100644 --- a/src/web_connection_base.cpp +++ b/src/web_connection_base.cpp @@ -98,7 +98,11 @@ namespace libtorrent void web_connection_base::start() { - set_upload_only(true); + // avoid calling torrent::set_seed because it calls torrent::check_invariant + // which fails because the m_num_connecting count is not consistent until + // after we call peer_connection::start + m_upload_only = true; + disconnect_if_redundant(); if (is_disconnecting()) return; peer_connection::start(); } diff --git a/test/Makefile.am b/test/Makefile.am index 76e6afa34..8768fb2df 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -243,7 +243,7 @@ test_session_params_SOURCES = test_session_params.cpp LDADD = libtest.la $(top_builddir)/src/libtorrent-rasterbar.la #AM_CXXFLAGS=-ftemplate-depth-50 -I$(top_srcdir)/include -I$(top_srcdir)/include/libtorrent @DEBUGFLAGS@ @PTHREAD_CFLAGS@ -AM_CPPFLAGS=-ftemplate-depth-50 -I$(top_srcdir)/include @DEBUGFLAGS@ +AM_CPPFLAGS=-ftemplate-depth-50 -I$(top_srcdir)/include @DEBUGFLAGS@ @OPENSSL_INCLUDES@ AM_LDFLAGS=@BOOST_SYSTEM_LIB@ @PTHREAD_LIBS@ @OPENSSL_LDFLAGS@ @OPENSSL_LIBS@ diff --git a/test/test_priority.cpp b/test/test_priority.cpp index ad569cdc8..3f37e5a95 100644 --- a/test/test_priority.cpp +++ b/test/test_priority.cpp @@ -39,6 +39,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/file.hpp" #include "libtorrent/torrent_info.hpp" #include "libtorrent/read_resume_data.hpp" +#include "libtorrent/write_resume_data.hpp" #include #include @@ -256,8 +257,7 @@ void test_transfer(settings_pack const& sett, bool test_deprecated = false) std::cout << "ses2: " << a->message() << std::endl; if (alert_cast(a)) { - bencode(std::back_inserter(resume_data) - , *alert_cast(a)->resume_data); + resume_data = write_resume_data_buf(alert_cast(a)->params); std::printf("saved resume data\n"); goto done; } diff --git a/test/test_resume.cpp b/test/test_resume.cpp index 88d608dab..27d32c25a 100644 --- a/test/test_resume.cpp +++ b/test/test_resume.cpp @@ -39,6 +39,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/entry.hpp" #include "libtorrent/bencode.hpp" #include "libtorrent/read_resume_data.hpp" +#include "libtorrent/write_resume_data.hpp" #include "test.hpp" #include "settings.hpp" @@ -223,15 +224,13 @@ void test_piece_priorities(bool test_deprecated = false) TEST_CHECK(ra); if (ra) { - std::printf("%s\n", ra->resume_data->to_string().c_str()); - entry::string_type prios = (*ra->resume_data)["piece_priority"].string(); + auto const prios = ra->params.piece_priorities; TEST_EQUAL(int(prios.size()), ti->num_pieces()); TEST_EQUAL(prios[0], '\0'); TEST_EQUAL(prios[1], '\x04'); TEST_EQUAL(prios[ti->num_pieces()-1], '\0'); - std::vector resume_data; - bencode(std::back_inserter(resume_data), *ra->resume_data); + std::vector resume_data = write_resume_data_buf(ra->params); #ifndef TORRENT_NO_DEPRECATE if (test_deprecated) @@ -332,16 +331,12 @@ TORRENT_TEST(resume_save_load_deprecated) TEST_CHECK(a); if (a == nullptr) return; - TEST_CHECK(a->resume_data); - - entry& e = *a->resume_data; - entry::list_type& l = e["file_priority"].list(); - entry::list_type::iterator i = l.begin(); + auto const l = a->params.file_priority; TEST_EQUAL(l.size(), 3); - TEST_EQUAL(*i++, 1); - TEST_EQUAL(*i++, 2); - TEST_EQUAL(*i++, 3); + TEST_EQUAL(l[0], 1); + TEST_EQUAL(l[1], 2); + TEST_EQUAL(l[2], 3); } TORRENT_TEST(resume_save_load_resume_deprecated) @@ -358,16 +353,12 @@ TORRENT_TEST(resume_save_load_resume_deprecated) TEST_CHECK(a); if (a == nullptr) return; - TEST_CHECK(a->resume_data); - - entry& e = *a->resume_data; - entry::list_type& l = e["file_priority"].list(); - entry::list_type::iterator i = l.begin(); + auto const l = a->params.file_priority; TEST_EQUAL(l.size(), 3); - TEST_EQUAL(*i++, 1); - TEST_EQUAL(*i++, 2); - TEST_EQUAL(*i++, 3); + TEST_EQUAL(l[0], 1); + TEST_EQUAL(l[1], 2); + TEST_EQUAL(l[2], 3); } TORRENT_TEST(file_priorities_resume_override_deprecated) @@ -903,16 +894,12 @@ TORRENT_TEST(resume_save_load) TEST_CHECK(a); if (a == nullptr) return; - TEST_CHECK(a->resume_data); - - entry& e = *a->resume_data; - entry::list_type& l = e["file_priority"].list(); - entry::list_type::iterator i = l.begin(); + auto const l = a->params.file_priorities; TEST_EQUAL(l.size(), 3); - TEST_EQUAL(*i++, 1); - TEST_EQUAL(*i++, 2); - TEST_EQUAL(*i++, 3); + TEST_EQUAL(l[0], 1); + TEST_EQUAL(l[1], 2); + TEST_EQUAL(l[2], 3); } TORRENT_TEST(resume_save_load_resume) @@ -929,18 +916,12 @@ TORRENT_TEST(resume_save_load_resume) TEST_CHECK(a); if (a == nullptr) return; - TEST_CHECK(a->resume_data); - - entry& e = *a->resume_data; - entry::list_type& l = e["file_priority"].list(); - entry::list_type::iterator i = l.begin(); - - std::printf("%s\n", e.to_string().c_str()); + auto const l = a->params.file_priorities; TEST_EQUAL(l.size(), 3); - TEST_EQUAL(*i++, 1); - TEST_EQUAL(*i++, 2); - TEST_EQUAL(*i++, 3); + TEST_EQUAL(l[0], 1); + TEST_EQUAL(l[1], 2); + TEST_EQUAL(l[2], 3); } TORRENT_TEST(file_priorities_resume) diff --git a/test/test_settings_pack.cpp b/test/test_settings_pack.cpp index 4c48c6f18..476ce32d9 100644 --- a/test/test_settings_pack.cpp +++ b/test/test_settings_pack.cpp @@ -243,7 +243,7 @@ TORRENT_TEST(settings_pack_abi) TEST_EQUAL(settings_pack::dht_bootstrap_nodes, settings_pack::string_type_base + 11); // bool - TEST_EQUAL(settings_pack::lazy_bitfields, settings_pack::bool_type_base + 3); + TEST_EQUAL(settings_pack::use_dht_as_fallback, settings_pack::bool_type_base + 4); TEST_EQUAL(settings_pack::use_read_cache, settings_pack::bool_type_base + 7); TEST_EQUAL(settings_pack::proxy_tracker_connections, settings_pack::bool_type_base + 67); diff --git a/test/test_storage.cpp b/test/test_storage.cpp index e8ec0bcbb..c6e4e9577 100644 --- a/test/test_storage.cpp +++ b/test/test_storage.cpp @@ -44,6 +44,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/create_torrent.hpp" #include "libtorrent/torrent_info.hpp" #include "libtorrent/read_resume_data.hpp" +#include "libtorrent/write_resume_data.hpp" #include #include @@ -681,7 +682,7 @@ void test_fastresume(bool const test_deprecated) h.save_resume_data(); alert const* ra = wait_for_alert(ses, save_resume_data_alert::alert_type); TEST_CHECK(ra); - if (ra) resume = *alert_cast(ra)->resume_data; + if (ra) resume = write_resume_data(alert_cast(ra)->params); ses.remove_torrent(h, lt::session::delete_files); alert const* da = wait_for_alert(ses, torrent_deleted_alert::alert_type); TEST_CHECK(da); @@ -798,15 +799,11 @@ TORRENT_TEST(rename_file) alert const* ra = wait_for_alert(ses, save_resume_data_alert::alert_type); TEST_CHECK(ra); if (!ra) return; - entry resume = *alert_cast(ra)->resume_data; + add_torrent_params resume = alert_cast(ra)->params; - std::cout << resume.to_string() << "\n"; - - entry::list_type files = resume.dict().find("mapped_files")->second.list(); - for (entry::list_type::iterator i = files.begin(); i != files.end(); ++i) - { - TEST_EQUAL(i->string().substr(0, 14), "temp_storage__"); - } + auto const files = resume.renamed_files; + for (auto const& i : files) + TEST_EQUAL(i.second.substr(0, 14), "temp_storage__"); } void test_rename_file_fastresume(bool test_deprecated) @@ -825,7 +822,7 @@ void test_rename_file_fastresume(bool test_deprecated) file.close(); TEST_CHECK(exists(combine_path(test_path, "tmp2/temporary"))); - entry resume; + add_torrent_params resume; { settings_pack pack = settings(); lt::session ses(pack); @@ -853,14 +850,16 @@ void test_rename_file_fastresume(bool test_deprecated) h.save_resume_data(); alert const* ra = wait_for_alert(ses, save_resume_data_alert::alert_type); TEST_CHECK(ra); - if (ra) resume = *alert_cast(ra)->resume_data; + if (ra) resume = alert_cast(ra)->params; ses.remove_torrent(h); } TEST_CHECK(!exists(combine_path(test_path, "tmp2/temporary"))); TEST_CHECK(exists(combine_path(test_path, "tmp2/testing_renamed_files"))); - TEST_CHECK(resume.dict().find("mapped_files") != resume.dict().end()); + TEST_CHECK(!resume.renamed_files.empty()); - std::cout << resume.to_string() << "\n"; + entry resume_ent = write_resume_data(resume); + + std::cout << resume_ent.to_string() << "\n"; // make sure the fast resume check succeeds, even though we renamed the file { @@ -869,7 +868,7 @@ void test_rename_file_fastresume(bool test_deprecated) add_torrent_params p; std::vector resume_data; - bencode(std::back_inserter(resume_data), resume); + bencode(std::back_inserter(resume_data), resume_ent); #ifndef TORRENT_NO_DEPRECATE if (test_deprecated) @@ -901,12 +900,13 @@ void test_rename_file_fastresume(bool test_deprecated) h.save_resume_data(); alert const* ra = wait_for_alert(ses, save_resume_data_alert::alert_type); TEST_CHECK(ra); - if (ra) resume = *alert_cast(ra)->resume_data; + if (ra) resume = alert_cast(ra)->params; ses.remove_torrent(h); } - TEST_CHECK(resume.dict().find("mapped_files") != resume.dict().end()); + TEST_CHECK(!resume.renamed_files.empty()); - std::cout << resume.to_string() << "\n"; + resume_ent = write_resume_data(resume); + std::cout << resume_ent.to_string() << "\n"; remove_all(combine_path(test_path, "tmp2"), ec); if (ec && ec != boost::system::errc::no_such_file_or_directory)