From faa2029f8b1ff390b82437ff4aacbeaceeb1a69e Mon Sep 17 00:00:00 2001 From: arvidn Date: Tue, 28 Feb 2017 18:06:02 -0500 Subject: [PATCH] add feature to periodically close files (to make windows clear disk cache) --- ChangeLog | 2 + include/libtorrent/aux_/session_impl.hpp | 7 ++ include/libtorrent/file_pool.hpp | 12 +++- include/libtorrent/settings_pack.hpp | 9 +++ simulation/Jamfile | 1 + simulation/test_file_pool.cpp | 86 ++++++++++++++++++++++++ src/file_pool.cpp | 17 +++++ src/session_impl.cpp | 26 +++++++ src/settings_pack.cpp | 7 ++ src/torrent_handle.cpp | 2 +- 10 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 simulation/test_file_pool.cpp diff --git a/ChangeLog b/ChangeLog index cc9f76f34..5f9a48279 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ + * add feature to periodically close files (to make windows clear disk cache) + * fix bug in torrent_handle::file_status * fix issue with peers not updated on metadata from magnet links 1.1.2 release diff --git a/include/libtorrent/aux_/session_impl.hpp b/include/libtorrent/aux_/session_impl.hpp index 1e43d38d2..11e067604 100644 --- a/include/libtorrent/aux_/session_impl.hpp +++ b/include/libtorrent/aux_/session_impl.hpp @@ -649,6 +649,7 @@ namespace libtorrent void update_privileged_ports(); void update_auto_sequential(); void update_max_failcount(); + void update_close_file_interval(); void update_upnp(); void update_natpmp(); @@ -976,6 +977,7 @@ namespace libtorrent int m_peak_down_rate; void on_tick(error_code const& e); + void on_close_file(error_code const& e); void try_connect_more_peers(); void auto_manage_checking_torrents(std::vector& list @@ -1123,6 +1125,11 @@ namespace libtorrent // by Local service discovery deadline_timer m_lsd_announce_timer; + // this is the timer used to call ``close_oldest`` on the ``file_pool`` + // object. This closes the file that's been opened the longest every + // time it's called, to force the windows disk cache to be flushed + deadline_timer m_close_file_timer; + resolver m_host_resolver; // the index of the torrent that will be offered to diff --git a/include/libtorrent/file_pool.hpp b/include/libtorrent/file_pool.hpp index c213898b9..f4bfe2c75 100644 --- a/include/libtorrent/file_pool.hpp +++ b/include/libtorrent/file_pool.hpp @@ -109,6 +109,12 @@ namespace libtorrent void set_low_prio_io(bool b) { m_low_prio_io = b; } void get_status(std::vector* files, void* st) const; + // close the file that was opened least recently (i.e. not *accessed* + // least recently). The purpose is to make the OS (really just windows) + // clear and flush its disk cache associated with this file. We don't want + // any file to stay open for too long, allowing the disk cache to accrue. + void close_oldest(); + #if TORRENT_USE_ASSERTS bool assert_idle_files(void* st) const; @@ -127,8 +133,12 @@ namespace libtorrent struct lru_file_entry { - lru_file_entry(): last_use(aux::time_now()), mode(0) {} + lru_file_entry() + : opened(aux::time_now()) + , last_use(opened) + , mode(0) {} file_handle file_ptr; + time_point const opened; time_point last_use; int mode; }; diff --git a/include/libtorrent/settings_pack.hpp b/include/libtorrent/settings_pack.hpp index 0dc760402..6938a200d 100644 --- a/include/libtorrent/settings_pack.hpp +++ b/include/libtorrent/settings_pack.hpp @@ -1594,6 +1594,15 @@ namespace libtorrent // time to wait until a new retry of a web seed name lookup web_seed_name_lookup_retry, + // the number of seconds between closing the file opened the longest + // ago. 0 means to disable the feature. The purpose of this is to + // periodically close files to trigger the operating system flushing + // disk cache. Specifically it has been observed to be required on + // windows to not have the disk cache grow indefinitely. + // This defaults to 120 seconds on windows, and disabled on other + // systems. + close_file_interval, + max_int_setting_internal }; diff --git a/simulation/Jamfile b/simulation/Jamfile index 09b85f76b..6c7f0c74b 100644 --- a/simulation/Jamfile +++ b/simulation/Jamfile @@ -46,5 +46,6 @@ alias libtorrent-sims : [ run test_tracker.cpp ] [ run test_ip_filter.cpp ] [ run test_fast_extensions.cpp ] + [ run test_file_pool.cpp ] ; diff --git a/simulation/test_file_pool.cpp b/simulation/test_file_pool.cpp new file mode 100644 index 000000000..16d8eb051 --- /dev/null +++ b/simulation/test_file_pool.cpp @@ -0,0 +1,86 @@ +/* + +Copyright (c) 2008, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "setup_swarm.hpp" +#include "test.hpp" +#include "utils.hpp" +#include "libtorrent/alert.hpp" +#include "libtorrent/alert_types.hpp" +#include "libtorrent/session.hpp" +#include "libtorrent/session_stats.hpp" +#include "libtorrent/file.hpp" + +using namespace libtorrent; + +TORRENT_TEST(close_file_interval) +{ + bool ran_to_completion = false; + + // with seed mode + setup_swarm(2, swarm_test::download + // add session + , [](lt::settings_pack& pack) { + pack.set_int(settings_pack::close_file_interval, 20); + } + // add torrent + , [](lt::add_torrent_params&) {} + // on alert + , [](lt::alert const* a, lt::session& ses) {} + // terminate + , [&](int ticks, lt::session& ses) -> bool + { + // terminate after 40 seconds + if (ticks > 25) + { + ran_to_completion = true; + return true; + } + + torrent_handle h = ses.get_torrents().front(); + std::vector file_status; + h.file_status(file_status); + printf("%d: %d files\n", ticks, int(file_status.size())); + if (ticks > 0 && ticks < 19) + { + TEST_EQUAL(file_status.size(), 1); + } + else if (ticks > 21) + { + // the close file timer shuold have kicked in at 20 seconds + // and closed the file + TEST_EQUAL(file_status.size(), 0); + } + return false; + }); + TEST_CHECK(ran_to_completion); +} + diff --git a/src/file_pool.cpp b/src/file_pool.cpp index cf644dd39..3582798aa 100644 --- a/src/file_pool.cpp +++ b/src/file_pool.cpp @@ -329,5 +329,22 @@ namespace libtorrent remove_oldest(l); } + void file_pool::close_oldest() + { + mutex::scoped_lock l(m_mutex); + + file_set::iterator i = std::min_element(m_files.begin(), m_files.end() + , boost::bind(&lru_file_entry::opened, boost::bind(&file_set::value_type::second, _1)) + < boost::bind(&lru_file_entry::opened, boost::bind(&file_set::value_type::second, _2))); + if (i == m_files.end()) return; + + file_handle file_ptr = i->second.file_ptr; + m_files.erase(i); + + // closing a file may be long running operation (mac os x) + l.unlock(); + file_ptr.reset(); + l.lock(); + } } diff --git a/src/session_impl.cpp b/src/session_impl.cpp index bd0f29d4e..10d632263 100644 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -426,6 +426,7 @@ namespace aux { , m_boost_connections(0) , m_timer(m_io_service) , m_lsd_announce_timer(m_io_service) + , m_close_file_timer(m_io_service) , m_host_resolver(m_io_service) , m_next_downloading_connect_torrent(0) , m_next_finished_connect_torrent(0) @@ -1094,6 +1095,8 @@ namespace aux { // about to send event=stopped to m_host_resolver.abort(); + m_close_file_timer.cancel(); + // abort the main thread m_abort = true; error_code ec; @@ -3565,6 +3568,16 @@ retry: // m_peer_pool.release_memory(); } + void session_impl::on_close_file(error_code const& e) + { + if (e) return; + + m_disk_thread.files().close_oldest(); + + // re-issue the timer + update_close_file_interval(); + } + namespace { // returns the index of the first set bit. int log2(boost::uint32_t v) @@ -5454,6 +5467,19 @@ retry: } } + void session_impl::update_close_file_interval() + { + int const interval = m_settings.get_int(settings_pack::close_file_interval); + if (interval == 0 || m_abort) + { + m_close_file_timer.cancel(); + return; + } + error_code ec; + m_close_file_timer.expires_from_now(seconds(interval), ec); + m_close_file_timer.async_wait(make_tick_handler(boost::bind(&session_impl::on_close_file, this, _1))); + } + void session_impl::update_proxy() { // in case we just set a socks proxy, we might have to diff --git a/src/settings_pack.cpp b/src/settings_pack.cpp index a51419dfb..8513d5132 100644 --- a/src/settings_pack.cpp +++ b/src/settings_pack.cpp @@ -124,6 +124,12 @@ namespace libtorrent #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" +#endif + +#ifdef TORRENT_WINDOWS +#define CLOSE_FILE_INTERVAL 120 +#else +#define CLOSE_FILE_INTERVAL 0 #endif namespace { @@ -350,6 +356,7 @@ namespace libtorrent SET_NOPREV(cache_size_volatile, 256, 0), SET_NOPREV(urlseed_max_request_bytes, 16 * 1024 * 1024, 0), SET_NOPREV(web_seed_name_lookup_retry, 1800, 0), + SET_NOPREV(close_file_interval, CLOSE_FILE_INTERVAL, &session_impl::update_close_file_interval), }; #undef SET diff --git a/src/torrent_handle.cpp b/src/torrent_handle.cpp index b5f72cfbc..04d20a9be 100644 --- a/src/torrent_handle.cpp +++ b/src/torrent_handle.cpp @@ -733,7 +733,7 @@ namespace libtorrent boost::shared_ptr t = m_torrent.lock(); if (!t || !t->has_storage()) return; session_impl& ses = static_cast(t->session()); - ses.disk_thread().files().get_status(&status, &t->storage()); + ses.disk_thread().files().get_status(&status, t->get_storage()); } void torrent_handle::scrape_tracker(int idx) const