merge RC_1_1 into master

ChangeLog

@@ -59,6 +59,10 @@
 	* resume data no longer has timestamps of files
 	* require C++11 to build libtorrent
 
+	* add feature to periodically close files (to make windows clear disk cache)
+	* fix bug in torrent_handle::file_status
+	* fix issue with peers not updated on metadata from magnet links
+
 1.1.2 release
 
 	* default TOS marking to 0x20

Makefile.am

@@ -10,20 +10,16 @@ DOCS_IMAGES = \
   docs/cwnd_thumb.png             \
   docs/delays.png                 \
   docs/delays_thumb.png           \
-  docs/disk_io.png                \
   docs/hacking.html               \
-  docs/im_thumb.jpg               \
   docs/merkle_tree.png            \
   docs/our_delay_base.png         \
   docs/our_delay_base_thumb.png   \
   docs/read_disk_buffers.png      \
   docs/read_disk_buffers.diagram  \
-  docs/session_stats_peers.png    \
   docs/storage.png                \
   docs/todo.html                  \
   docs/write_disk_buffers.png     \
   docs/write_disk_buffers.diagram \
-  docs/ziptorrent_thumb.gif       \
   docs/ip_id_v4.png               \
   docs/ip_id_v6.png               \
   docs/hash_distribution.png      \

examples/Makefile.am

@@ -30,7 +30,7 @@ upnp_test_SOURCES = upnp_test.cpp
 
 LDADD = $(top_builddir)/src/libtorrent-rasterbar.la
 
-AM_CPPFLAGS = -ftemplate-depth-50 -I$(top_srcdir)/include @DEBUGFLAGS@
+AM_CPPFLAGS = -ftemplate-depth-50 -I$(top_srcdir)/include @DEBUGFLAGS@ @OPENSSL_INCLUDES@
 
 AM_LDFLAGS = @BOOST_SYSTEM_LIB@ @OPENSSL_LDFLAGS@ @OPENSSL_LIBS@
 

include/libtorrent/aux_/session_impl.hpp

@@ -700,6 +700,7 @@ namespace libtorrent
 			void update_privileged_ports();
 			void update_auto_sequential();
 			void update_max_failcount();
+			void update_close_file_interval();
 
 			void update_upnp();
 			void update_natpmp();
@@ -976,6 +977,7 @@ namespace libtorrent
 			int m_peak_down_rate = 0;
 
 			void on_tick(error_code const& e);
+			void on_close_file(error_code const& e);
 
 			void try_connect_more_peers();
 			void auto_manage_checking_torrents(std::vector<torrent*>& list
@@ -1157,6 +1159,11 @@ namespace libtorrent
 			// by Local service discovery
 			deadline_timer m_lsd_announce_timer;
 
+			// this is the timer used to call ``close_oldest`` on the ``file_pool``
+			// object. This closes the file that's been opened the longest every
+			// time it's called, to force the windows disk cache to be flushed
+			deadline_timer m_close_file_timer;
+
 			resolver m_host_resolver;
 
 			// the index of the torrent that will be offered to

include/libtorrent/file_pool.hpp

@@ -114,6 +114,12 @@ namespace libtorrent
 		void set_low_prio_io(bool b) { m_low_prio_io = b; }
 		std::vector<pool_file_status> get_status(storage_index_t st) const;
 
+		// close the file that was opened least recently (i.e. not *accessed*
+		// least recently). The purpose is to make the OS (really just windows)
+		// clear and flush its disk cache associated with this file. We don't want
+		// any file to stay open for too long, allowing the disk cache to accrue.
+		void close_oldest();
+
 #if TORRENT_USE_ASSERTS
 		bool assert_idle_files(storage_index_t st) const;
 
@@ -133,7 +139,8 @@ namespace libtorrent
 		struct lru_file_entry
 		{
 			file_handle file_ptr;
-			time_point last_use{aux::time_now()};
+			time_point const opened{aux::time_now()};
+			time_point last_use{opened};
 			int mode = 0;
 		};
 

include/libtorrent/settings_pack.hpp

@@ -1618,6 +1618,15 @@ namespace libtorrent
 			// time to wait until a new retry of a web seed name lookup
 			web_seed_name_lookup_retry,
 
+			// the number of seconds between closing the file opened the longest
+			// ago. 0 means to disable the feature. The purpose of this is to
+			// periodically close files to trigger the operating system flushing
+			// disk cache. Specifically it has been observed to be required on
+			// windows to not have the disk cache grow indefinitely.
+			// This defaults to 120 seconds on windows, and disabled on other
+			// systems.
+			close_file_interval,
+
 			max_int_setting_internal
 		};
 

simulation/Jamfile

@@ -50,6 +50,7 @@ alias libtorrent-sims :
 	[ run test_ip_filter.cpp ]
 	[ run test_dht_rate_limit.cpp ]
 	[ run test_fast_extensions.cpp ]
+	[ run test_file_pool.cpp ]
 	[ run test_save_resume.cpp ]
 	;
 

simulation/test_file_pool.cpp

@@ -0,0 +1,86 @@
+/*
+
+Copyright (c) 2008, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "setup_swarm.hpp"
+#include "test.hpp"
+#include "utils.hpp"
+#include "libtorrent/alert.hpp"
+#include "libtorrent/alert_types.hpp"
+#include "libtorrent/session.hpp"
+#include "libtorrent/session_stats.hpp"
+#include "libtorrent/file.hpp"
+
+using namespace libtorrent;
+
+TORRENT_TEST(close_file_interval)
+{
+	bool ran_to_completion = false;
+
+	// with seed mode
+	setup_swarm(2, swarm_test::download
+		// add session
+		, [](lt::settings_pack& pack) {
+			pack.set_int(settings_pack::close_file_interval, 20);
+		}
+		// add torrent
+		, [](lt::add_torrent_params&) {}
+		// on alert
+		, [](lt::alert const* a, lt::session& ses) {}
+		// terminate
+		, [&](int ticks, lt::session& ses) -> bool
+		{
+			// terminate after 40 seconds
+			if (ticks > 24)
+			{
+				ran_to_completion = true;
+				return true;
+			}
+
+			torrent_handle h = ses.get_torrents().front();
+			std::vector<pool_file_status> file_status;
+			h.file_status(file_status);
+			printf("%d: %d files\n", ticks, int(file_status.size()));
+			if (ticks > 0 && ticks < 19)
+			{
+				TEST_EQUAL(file_status.size(), 1);
+			}
+			else if (ticks > 21)
+			{
+				// the close file timer shuold have kicked in at 20 seconds
+				// and closed the file
+				TEST_EQUAL(file_status.size(), 0);
+			}
+			return false;
+		});
+	TEST_CHECK(ran_to_completion);
+}
+

src/file_pool.cpp

@@ -291,4 +291,22 @@ namespace libtorrent
 			remove_oldest(l);
 	}
 
+	void file_pool::close_oldest()
+	{
+		std::unique_lock<std::mutex> l(m_mutex);
+
+		using value_type = decltype(m_files)::value_type;
+		auto const i = std::min_element(m_files.begin(), m_files.end()
+			, [] (value_type const& lhs, value_type const& rhs)
+				{ return lhs.second.opened < rhs.second.opened; });
+		if (i == m_files.end()) return;
+
+		file_handle file_ptr = i->second.file_ptr;
+		m_files.erase(i);
+
+		// closing a file may be long running operation (mac os x)
+		l.unlock();
+		file_ptr.reset();
+		l.lock();
+	}
 }

src/session_impl.cpp

@@ -469,6 +469,7 @@ namespace aux {
 #endif
 		, m_timer(m_io_service)
 		, m_lsd_announce_timer(m_io_service)
+		, m_close_file_timer(m_io_service)
 		, m_host_resolver(m_io_service)
 	{
 		update_time_now();
@@ -901,6 +902,8 @@ namespace aux {
 		// about to send event=stopped to
 		m_host_resolver.abort();
 
+		m_close_file_timer.cancel();
+
 		// abort the main thread
 		m_abort = true;
 		error_code ec;
@@ -3436,6 +3439,16 @@ namespace aux {
 //		m_peer_pool.release_memory();
 	}
 
+	void session_impl::on_close_file(error_code const& e)
+	{
+		if (e) return;
+
+		m_disk_thread.files().close_oldest();
+
+		// re-issue the timer
+		update_close_file_interval();
+	}
+
 	namespace {
 	// returns the index of the first set bit.
 	int log2(std::uint32_t v)
@@ -5188,6 +5201,19 @@ namespace aux {
 		}
 	}
 
+	void session_impl::update_close_file_interval()
+	{
+		int const interval = m_settings.get_int(settings_pack::close_file_interval);
+		if (interval == 0 || m_abort)
+		{
+			m_close_file_timer.cancel();
+			return;
+		}
+		error_code ec;
+		m_close_file_timer.expires_from_now(seconds(interval), ec);
+		m_close_file_timer.async_wait(make_tick_handler(std::bind(&session_impl::on_close_file, this, _1)));
+	}
+
 	void session_impl::update_proxy()
 	{
 		// in case we just set a socks proxy, we might have to

src/settings_pack.cpp

@@ -98,6 +98,12 @@ namespace libtorrent
 #define DEPRECATED_SET(name, default_value, fun) { #name, fun, default_value }
 #else
 #define DEPRECATED_SET(name, default_value, fun) { "", nullptr, 0 }
+#endif
+
+#ifdef TORRENT_WINDOWS
+#define CLOSE_FILE_INTERVAL 120
+#else
+#define CLOSE_FILE_INTERVAL 0
 #endif
 
 	namespace {
@@ -327,6 +333,7 @@ namespace libtorrent
 		SET(cache_size_volatile, 256, nullptr),
 		SET(urlseed_max_request_bytes, 16 * 1024 * 1024, 0),
 		SET(web_seed_name_lookup_retry, 1800, nullptr),
+		SET(close_file_interval, CLOSE_FILE_INTERVAL, &session_impl::update_close_file_interval),
 	}});
 
 #undef SET

src/web_connection_base.cpp

@@ -98,7 +98,11 @@ namespace libtorrent
 
 	void web_connection_base::start()
 	{
-		set_upload_only(true);
+		// avoid calling torrent::set_seed because it calls torrent::check_invariant
+		// which fails because the m_num_connecting count is not consistent until
+		// after we call peer_connection::start
+		m_upload_only = true;
+		disconnect_if_redundant();
 		if (is_disconnecting()) return;
 		peer_connection::start();
 	}

test/Makefile.am

@@ -243,7 +243,7 @@ test_session_params_SOURCES = test_session_params.cpp
 LDADD = libtest.la $(top_builddir)/src/libtorrent-rasterbar.la
 
 #AM_CXXFLAGS=-ftemplate-depth-50 -I$(top_srcdir)/include -I$(top_srcdir)/include/libtorrent @DEBUGFLAGS@ @PTHREAD_CFLAGS@
-AM_CPPFLAGS=-ftemplate-depth-50 -I$(top_srcdir)/include @DEBUGFLAGS@
+AM_CPPFLAGS=-ftemplate-depth-50 -I$(top_srcdir)/include @DEBUGFLAGS@ @OPENSSL_INCLUDES@
 
 AM_LDFLAGS=@BOOST_SYSTEM_LIB@ @PTHREAD_LIBS@ @OPENSSL_LDFLAGS@ @OPENSSL_LIBS@
 

test/test_priority.cpp

@@ -39,6 +39,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/file.hpp"
 #include "libtorrent/torrent_info.hpp"
 #include "libtorrent/read_resume_data.hpp"
+#include "libtorrent/write_resume_data.hpp"
 #include <tuple>
 #include <functional>
 
@@ -256,8 +257,7 @@ void test_transfer(settings_pack const& sett, bool test_deprecated = false)
 			std::cout << "ses2: " << a->message() << std::endl;
 			if (alert_cast<save_resume_data_alert>(a))
 			{
-				bencode(std::back_inserter(resume_data)
-					, *alert_cast<save_resume_data_alert>(a)->resume_data);
+				resume_data = write_resume_data_buf(alert_cast<save_resume_data_alert>(a)->params);
 				std::printf("saved resume data\n");
 				goto done;
 			}

test/test_resume.cpp

@@ -39,6 +39,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/entry.hpp"
 #include "libtorrent/bencode.hpp"
 #include "libtorrent/read_resume_data.hpp"
+#include "libtorrent/write_resume_data.hpp"
 
 #include "test.hpp"
 #include "settings.hpp"
@@ -223,15 +224,13 @@ void test_piece_priorities(bool test_deprecated = false)
 	TEST_CHECK(ra);
 	if (ra)
 	{
-		std::printf("%s\n", ra->resume_data->to_string().c_str());
-		entry::string_type prios = (*ra->resume_data)["piece_priority"].string();
+		auto const prios = ra->params.piece_priorities;
 		TEST_EQUAL(int(prios.size()), ti->num_pieces());
 		TEST_EQUAL(prios[0], '\0');
 		TEST_EQUAL(prios[1], '\x04');
 		TEST_EQUAL(prios[ti->num_pieces()-1], '\0');
 
-		std::vector<char> resume_data;
-		bencode(std::back_inserter(resume_data), *ra->resume_data);
+		std::vector<char> resume_data = write_resume_data_buf(ra->params);
 
 #ifndef TORRENT_NO_DEPRECATE
 		if (test_deprecated)
@@ -332,16 +331,12 @@ TORRENT_TEST(resume_save_load_deprecated)
 	TEST_CHECK(a);
 	if (a == nullptr) return;
 
-	TEST_CHECK(a->resume_data);
-
-	entry& e = *a->resume_data;
-	entry::list_type& l = e["file_priority"].list();
-	entry::list_type::iterator i = l.begin();
+	auto const l = a->params.file_priority;
 
 	TEST_EQUAL(l.size(), 3);
-	TEST_EQUAL(*i++, 1);
-	TEST_EQUAL(*i++, 2);
-	TEST_EQUAL(*i++, 3);
+	TEST_EQUAL(l[0], 1);
+	TEST_EQUAL(l[1], 2);
+	TEST_EQUAL(l[2], 3);
 }
 
 TORRENT_TEST(resume_save_load_resume_deprecated)
@@ -358,16 +353,12 @@ TORRENT_TEST(resume_save_load_resume_deprecated)
 	TEST_CHECK(a);
 	if (a == nullptr) return;
 
-	TEST_CHECK(a->resume_data);
-
-	entry& e = *a->resume_data;
-	entry::list_type& l = e["file_priority"].list();
-	entry::list_type::iterator i = l.begin();
+	auto const l = a->params.file_priority;
 
 	TEST_EQUAL(l.size(), 3);
-	TEST_EQUAL(*i++, 1);
-	TEST_EQUAL(*i++, 2);
-	TEST_EQUAL(*i++, 3);
+	TEST_EQUAL(l[0], 1);
+	TEST_EQUAL(l[1], 2);
+	TEST_EQUAL(l[2], 3);
 }
 
 TORRENT_TEST(file_priorities_resume_override_deprecated)
@@ -903,16 +894,12 @@ TORRENT_TEST(resume_save_load)
 	TEST_CHECK(a);
 	if (a == nullptr) return;
 
-	TEST_CHECK(a->resume_data);
-
-	entry& e = *a->resume_data;
-	entry::list_type& l = e["file_priority"].list();
-	entry::list_type::iterator i = l.begin();
+	auto const l = a->params.file_priorities;
 
 	TEST_EQUAL(l.size(), 3);
-	TEST_EQUAL(*i++, 1);
-	TEST_EQUAL(*i++, 2);
-	TEST_EQUAL(*i++, 3);
+	TEST_EQUAL(l[0], 1);
+	TEST_EQUAL(l[1], 2);
+	TEST_EQUAL(l[2], 3);
 }
 
 TORRENT_TEST(resume_save_load_resume)
@@ -929,18 +916,12 @@ TORRENT_TEST(resume_save_load_resume)
 	TEST_CHECK(a);
 	if (a == nullptr) return;
 
-	TEST_CHECK(a->resume_data);
-
-	entry& e = *a->resume_data;
-	entry::list_type& l = e["file_priority"].list();
-	entry::list_type::iterator i = l.begin();
-
-	std::printf("%s\n", e.to_string().c_str());
+	auto const l = a->params.file_priorities;
 
 	TEST_EQUAL(l.size(), 3);
-	TEST_EQUAL(*i++, 1);
-	TEST_EQUAL(*i++, 2);
-	TEST_EQUAL(*i++, 3);
+	TEST_EQUAL(l[0], 1);
+	TEST_EQUAL(l[1], 2);
+	TEST_EQUAL(l[2], 3);
 }
 
 TORRENT_TEST(file_priorities_resume)

test/test_settings_pack.cpp

@@ -243,7 +243,7 @@ TORRENT_TEST(settings_pack_abi)
 	TEST_EQUAL(settings_pack::dht_bootstrap_nodes, settings_pack::string_type_base + 11);
 
 	// bool
-	TEST_EQUAL(settings_pack::lazy_bitfields, settings_pack::bool_type_base + 3);
+	TEST_EQUAL(settings_pack::use_dht_as_fallback, settings_pack::bool_type_base + 4);
 	TEST_EQUAL(settings_pack::use_read_cache, settings_pack::bool_type_base + 7);
 	TEST_EQUAL(settings_pack::proxy_tracker_connections, settings_pack::bool_type_base + 67);
 

test/test_storage.cpp

@@ -44,6 +44,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/create_torrent.hpp"
 #include "libtorrent/torrent_info.hpp"
 #include "libtorrent/read_resume_data.hpp"
+#include "libtorrent/write_resume_data.hpp"
 
 #include <iostream>
 #include <fstream>
@@ -681,7 +682,7 @@ void test_fastresume(bool const test_deprecated)
 		h.save_resume_data();
 		alert const* ra = wait_for_alert(ses, save_resume_data_alert::alert_type);
 		TEST_CHECK(ra);
-		if (ra) resume = *alert_cast<save_resume_data_alert>(ra)->resume_data;
+		if (ra) resume = write_resume_data(alert_cast<save_resume_data_alert>(ra)->params);
 		ses.remove_torrent(h, lt::session::delete_files);
 		alert const* da = wait_for_alert(ses, torrent_deleted_alert::alert_type);
 		TEST_CHECK(da);
@@ -798,15 +799,11 @@ TORRENT_TEST(rename_file)
 	alert const* ra = wait_for_alert(ses, save_resume_data_alert::alert_type);
 	TEST_CHECK(ra);
 	if (!ra) return;
-	entry resume = *alert_cast<save_resume_data_alert>(ra)->resume_data;
+	add_torrent_params resume = alert_cast<save_resume_data_alert>(ra)->params;
 
-	std::cout << resume.to_string() << "\n";
-
-	entry::list_type files = resume.dict().find("mapped_files")->second.list();
-	for (entry::list_type::iterator i = files.begin(); i != files.end(); ++i)
-	{
-		TEST_EQUAL(i->string().substr(0, 14), "temp_storage__");
-	}
+	auto const files = resume.renamed_files;
+	for (auto const& i : files)
+		TEST_EQUAL(i.second.substr(0, 14), "temp_storage__");
 }
 
 void test_rename_file_fastresume(bool test_deprecated)
@@ -825,7 +822,7 @@ void test_rename_file_fastresume(bool test_deprecated)
 	file.close();
 	TEST_CHECK(exists(combine_path(test_path, "tmp2/temporary")));
 
-	entry resume;
+	add_torrent_params resume;
 	{
 		settings_pack pack = settings();
 		lt::session ses(pack);
@@ -853,14 +850,16 @@ void test_rename_file_fastresume(bool test_deprecated)
 		h.save_resume_data();
 		alert const* ra = wait_for_alert(ses, save_resume_data_alert::alert_type);
 		TEST_CHECK(ra);
-		if (ra) resume = *alert_cast<save_resume_data_alert>(ra)->resume_data;
+		if (ra) resume = alert_cast<save_resume_data_alert>(ra)->params;
 		ses.remove_torrent(h);
 	}
 	TEST_CHECK(!exists(combine_path(test_path, "tmp2/temporary")));
 	TEST_CHECK(exists(combine_path(test_path, "tmp2/testing_renamed_files")));
-	TEST_CHECK(resume.dict().find("mapped_files") != resume.dict().end());
+	TEST_CHECK(!resume.renamed_files.empty());
 
-	std::cout << resume.to_string() << "\n";
+	entry resume_ent = write_resume_data(resume);
+
+	std::cout << resume_ent.to_string() << "\n";
 
 	// make sure the fast resume check succeeds, even though we renamed the file
 	{
@@ -869,7 +868,7 @@ void test_rename_file_fastresume(bool test_deprecated)
 
 		add_torrent_params p;
 		std::vector<char> resume_data;
-		bencode(std::back_inserter(resume_data), resume);
+		bencode(std::back_inserter(resume_data), resume_ent);
 
 #ifndef TORRENT_NO_DEPRECATE
 		if (test_deprecated)
@@ -901,12 +900,13 @@ void test_rename_file_fastresume(bool test_deprecated)
 		h.save_resume_data();
 		alert const* ra = wait_for_alert(ses, save_resume_data_alert::alert_type);
 		TEST_CHECK(ra);
-		if (ra) resume = *alert_cast<save_resume_data_alert>(ra)->resume_data;
+		if (ra) resume = alert_cast<save_resume_data_alert>(ra)->params;
 		ses.remove_torrent(h);
 	}
-	TEST_CHECK(resume.dict().find("mapped_files") != resume.dict().end());
+	TEST_CHECK(!resume.renamed_files.empty());
 
-	std::cout << resume.to_string() << "\n";
+	resume_ent = write_resume_data(resume);
+	std::cout << resume_ent.to_string() << "\n";
 
 	remove_all(combine_path(test_path, "tmp2"), ec);
 	if (ec && ec != boost::system::errc::no_such_file_or_directory)