add feature to periodically close files (to make windows clear disk cache)

ChangeLog

@@ -1,3 +1,5 @@
+	* add feature to periodically close files (to make windows clear disk cache)
+	* fix bug in torrent_handle::file_status
 	* fix issue with peers not updated on metadata from magnet links
 
 1.1.2 release

include/libtorrent/aux_/session_impl.hpp

@@ -649,6 +649,7 @@ namespace libtorrent
 			void update_privileged_ports();
 			void update_auto_sequential();
 			void update_max_failcount();
+			void update_close_file_interval();
 
 			void update_upnp();
 			void update_natpmp();
@@ -976,6 +977,7 @@ namespace libtorrent
 			int m_peak_down_rate;
 
 			void on_tick(error_code const& e);
+			void on_close_file(error_code const& e);
 
 			void try_connect_more_peers();
 			void auto_manage_checking_torrents(std::vector<torrent*>& list
@@ -1123,6 +1125,11 @@ namespace libtorrent
 			// by Local service discovery
 			deadline_timer m_lsd_announce_timer;
 
+			// this is the timer used to call ``close_oldest`` on the ``file_pool``
+			// object. This closes the file that's been opened the longest every
+			// time it's called, to force the windows disk cache to be flushed
+			deadline_timer m_close_file_timer;
+
 			resolver m_host_resolver;
 
 			// the index of the torrent that will be offered to

include/libtorrent/file_pool.hpp

@@ -109,6 +109,12 @@ namespace libtorrent
 		void set_low_prio_io(bool b) { m_low_prio_io = b; }
 		void get_status(std::vector<pool_file_status>* files, void* st) const;
 
+		// close the file that was opened least recently (i.e. not *accessed*
+		// least recently). The purpose is to make the OS (really just windows)
+		// clear and flush its disk cache associated with this file. We don't want
+		// any file to stay open for too long, allowing the disk cache to accrue.
+		void close_oldest();
+
 #if TORRENT_USE_ASSERTS
 		bool assert_idle_files(void* st) const;
 
@@ -127,8 +133,12 @@ namespace libtorrent
 
 		struct lru_file_entry
 		{
-			lru_file_entry(): last_use(aux::time_now()), mode(0) {}
+			lru_file_entry()
+				: opened(aux::time_now())
+				, last_use(opened)
+				, mode(0) {}
 			file_handle file_ptr;
+			time_point const opened;
 			time_point last_use;
 			int mode;
 		};

include/libtorrent/settings_pack.hpp

@@ -1594,6 +1594,15 @@ namespace libtorrent
 			// time to wait until a new retry of a web seed name lookup
 			web_seed_name_lookup_retry,
 
+			// the number of seconds between closing the file opened the longest
+			// ago. 0 means to disable the feature. The purpose of this is to
+			// periodically close files to trigger the operating system flushing
+			// disk cache. Specifically it has been observed to be required on
+			// windows to not have the disk cache grow indefinitely.
+			// This defaults to 120 seconds on windows, and disabled on other
+			// systems.
+			close_file_interval,
+
 			max_int_setting_internal
 		};
 

simulation/Jamfile

@@ -46,5 +46,6 @@ alias libtorrent-sims :
 	[ run test_tracker.cpp ]
 	[ run test_ip_filter.cpp ]
 	[ run test_fast_extensions.cpp ]
+	[ run test_file_pool.cpp ]
 	;
 

simulation/test_file_pool.cpp

@@ -0,0 +1,86 @@
+/*
+
+Copyright (c) 2008, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "setup_swarm.hpp"
+#include "test.hpp"
+#include "utils.hpp"
+#include "libtorrent/alert.hpp"
+#include "libtorrent/alert_types.hpp"
+#include "libtorrent/session.hpp"
+#include "libtorrent/session_stats.hpp"
+#include "libtorrent/file.hpp"
+
+using namespace libtorrent;
+
+TORRENT_TEST(close_file_interval)
+{
+	bool ran_to_completion = false;
+
+	// with seed mode
+	setup_swarm(2, swarm_test::download
+		// add session
+		, [](lt::settings_pack& pack) {
+			pack.set_int(settings_pack::close_file_interval, 20);
+		}
+		// add torrent
+		, [](lt::add_torrent_params&) {}
+		// on alert
+		, [](lt::alert const* a, lt::session& ses) {}
+		// terminate
+		, [&](int ticks, lt::session& ses) -> bool
+		{
+			// terminate after 40 seconds
+			if (ticks > 25)
+			{
+				ran_to_completion = true;
+				return true;
+			}
+
+			torrent_handle h = ses.get_torrents().front();
+			std::vector<pool_file_status> file_status;
+			h.file_status(file_status);
+			printf("%d: %d files\n", ticks, int(file_status.size()));
+			if (ticks > 0 && ticks < 19)
+			{
+				TEST_EQUAL(file_status.size(), 1);
+			}
+			else if (ticks > 21)
+			{
+				// the close file timer shuold have kicked in at 20 seconds
+				// and closed the file
+				TEST_EQUAL(file_status.size(), 0);
+			}
+			return false;
+		});
+	TEST_CHECK(ran_to_completion);
+}
+

src/file_pool.cpp

@@ -329,5 +329,22 @@ namespace libtorrent
 			remove_oldest(l);
 	}
 
+	void file_pool::close_oldest()
+	{
+		mutex::scoped_lock l(m_mutex);
+
+		file_set::iterator i = std::min_element(m_files.begin(), m_files.end()
+			, boost::bind(&lru_file_entry::opened, boost::bind(&file_set::value_type::second, _1))
+				< boost::bind(&lru_file_entry::opened, boost::bind(&file_set::value_type::second, _2)));
+		if (i == m_files.end()) return;
+
+		file_handle file_ptr = i->second.file_ptr;
+		m_files.erase(i);
+
+		// closing a file may be long running operation (mac os x)
+		l.unlock();
+		file_ptr.reset();
+		l.lock();
+	}
 }
 

src/session_impl.cpp

@@ -426,6 +426,7 @@ namespace aux {
 		, m_boost_connections(0)
 		, m_timer(m_io_service)
 		, m_lsd_announce_timer(m_io_service)
+		, m_close_file_timer(m_io_service)
 		, m_host_resolver(m_io_service)
 		, m_next_downloading_connect_torrent(0)
 		, m_next_finished_connect_torrent(0)
@@ -1094,6 +1095,8 @@ namespace aux {
 		// about to send event=stopped to
 		m_host_resolver.abort();
 
+		m_close_file_timer.cancel();
+
 		// abort the main thread
 		m_abort = true;
 		error_code ec;
@@ -3565,6 +3568,16 @@ retry:
 //		m_peer_pool.release_memory();
 	}
 
+	void session_impl::on_close_file(error_code const& e)
+	{
+		if (e) return;
+
+		m_disk_thread.files().close_oldest();
+
+		// re-issue the timer
+		update_close_file_interval();
+	}
+
 	namespace {
 	// returns the index of the first set bit.
 	int log2(boost::uint32_t v)
@@ -5454,6 +5467,19 @@ retry:
 		}
 	}
 
+	void session_impl::update_close_file_interval()
+	{
+		int const interval = m_settings.get_int(settings_pack::close_file_interval);
+		if (interval == 0 || m_abort)
+		{
+			m_close_file_timer.cancel();
+			return;
+		}
+		error_code ec;
+		m_close_file_timer.expires_from_now(seconds(interval), ec);
+		m_close_file_timer.async_wait(make_tick_handler(boost::bind(&session_impl::on_close_file, this, _1)));
+	}
+
 	void session_impl::update_proxy()
 	{
 		// in case we just set a socks proxy, we might have to

src/settings_pack.cpp

@@ -124,6 +124,12 @@ namespace libtorrent
 #ifdef __GNUC__
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Winvalid-offsetof"
+#endif
+
+#ifdef TORRENT_WINDOWS
+#define CLOSE_FILE_INTERVAL 120
+#else
+#define CLOSE_FILE_INTERVAL 0
 #endif
 
 	namespace {
@@ -350,6 +356,7 @@ namespace libtorrent
 		SET_NOPREV(cache_size_volatile, 256, 0),
 		SET_NOPREV(urlseed_max_request_bytes, 16 * 1024 * 1024, 0),
 		SET_NOPREV(web_seed_name_lookup_retry, 1800, 0),
+		SET_NOPREV(close_file_interval, CLOSE_FILE_INTERVAL, &session_impl::update_close_file_interval),
 	};
 
 #undef SET

src/torrent_handle.cpp

@@ -733,7 +733,7 @@ namespace libtorrent
 		boost::shared_ptr<torrent> t = m_torrent.lock();
 		if (!t || !t->has_storage()) return;
 		session_impl& ses = static_cast<session_impl&>(t->session());
-		ses.disk_thread().files().get_status(&status, &t->storage());
+		ses.disk_thread().files().get_status(&status, t->get_storage());
 	}
 
 	void torrent_handle::scrape_tracker(int idx) const