fixed bug introduced in latest storage changes (triggered when using metadata extension). improved commandline switches to client_test

ChangeLog

@@ -1,4 +1,5 @@
-
+	* greatly improved the command line control of the example client_test
+	* fixed bug where upload rate limit was not being applied
 	* files that are being checked will no longer stall files that don't need
 	  checking.
 	* changed the way libtorrent identifies support for its excentions

Jamfile

@@ -36,6 +36,7 @@ project torrent
 	<threading>multi
 	<toolset>msvc:<cxxflags>/Zc:wchar_t
 	<toolset>msvc:<cxxflags>/Zc:forScope
+	<define>TORRENT_PROFILE
 
 	: usage-requirements 
 
@@ -43,6 +44,7 @@ project torrent
 	<include>$(BOOST_ROOT)
 	<variant>release:<define>NDEBUG
 	<define>BOOST_ALL_NO_LIB
+	<define>TORRENT_PROFILE
 
 	;
 

examples/Jamfile

@@ -1,10 +1,15 @@
+import modules ;
+
+BOOST_ROOT = [ modules.peek : BOOST_ROOT ] ;
+
 use-project /torrent : .. ;
+use-project /boost : $(BOOST_ROOT) ;
 
 project client_test
    : requirements <threading>multi <library>/torrent
    ;  
 
-exe client_test : client_test.cpp ;
+exe client_test : client_test.cpp /boost/program_options /boost/regex ;
 exe simple_client : simple_client.cpp ;
 exe dump_torrent : dump_torrent.cpp ;
 exe make_torrent : make_torrent.cpp ;

examples/client_test.cpp

@@ -43,6 +43,8 @@ POSSIBILITY OF SUCH DAMAGE.
 #include <boost/filesystem/fstream.hpp>
 #include <boost/filesystem/exception.hpp>
 #include <boost/bind.hpp>
+#include <boost/program_options.hpp>
+#include <boost/regex.hpp>
 
 #ifdef _MSC_VER
 #pragma warning(pop)
@@ -77,14 +79,7 @@ bool sleep_and_input(char* c)
 	return false;
 };
 
-void set_cursor(int x, int y)
+void clear_home()
-{
-	HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE);
-	COORD c = {x, y};
-	SetConsoleCursorPosition(h, c);
-}
-
-void clear()
 {
 	CONSOLE_SCREEN_BUFFER_INFO si;
 	HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE);
@@ -92,6 +87,7 @@ void clear()
 	COORD c = {0, 0};
 	DWORD n;
 	FillConsoleOutputCharacter(h, ' ', si.dwSize.X * si.dwSize.Y, c, &n);
+	SetConsoleCursorPosition(h, c);
 }
 
 #else
@@ -139,14 +135,9 @@ bool sleep_and_input(char* c)
 	return false;
 }
 
-void set_cursor(int x, int y)
+void clear_home()
 {
-	std::cout << "\033[" << y << ";" << x << "H";
+	std::cout << "\033[2J\033[0;0H";
-}
-
-void clear()
-{
-	std::cout << "\033[2J";
 }
 
 #endif
@@ -258,27 +249,23 @@ void print_peer_info(std::ostream& out, std::vector<libtorrent::peer_info> const
 	{
 		out.fill(' ');
 		out.width(2);
-		out << esc("32") << add_suffix(i->down_speed) << "/s " << esc("0")
+		out << esc("32") << add_suffix(i->down_speed) << "/s "
-//						<< "(" << add_suffix(i->total_download) << ") "
+			<< "(" << add_suffix(i->total_download) << ") " << esc("0")
-			<< esc("31") << add_suffix(i->up_speed) << "/s " << esc("0")
+			<< esc("31") << add_suffix(i->up_speed) << "/s "
-//						<< "(" << add_suffix(i->total_upload) << ") "
+			<< "(" << add_suffix(i->total_upload) << ") " << esc("0")
-//						<< "ul:" << add_suffix(i->upload_limit) << "/s "
-//						<< "uc:" << add_suffix(i->upload_ceiling) << "/s "
-//						<< "df:" << ratio(i->total_download, i->total_upload) << " "
 			<< to_string(i->download_queue_length, 2, 2) << " "
 			<< to_string(i->upload_queue_length, 2, 2) << " "
-			<< static_cast<const char*>((i->flags & peer_info::interesting)?"I":"_")
+			<< ((i->flags & peer_info::interesting)?'I':'.')
-			<< static_cast<const char*>((i->flags & peer_info::choked)?"C":"_")
+			<< ((i->flags & peer_info::choked)?'C':'.')
-			<< static_cast<const char*>((i->flags & peer_info::remote_interested)?"i":"_")
+			<< ((i->flags & peer_info::remote_interested)?'i':'.')
-			<< static_cast<const char*>((i->flags & peer_info::remote_choked)?"c":"_")
+			<< ((i->flags & peer_info::remote_choked)?'c':'.')
-			<< static_cast<const char*>((i->flags & peer_info::supports_extensions)?"e":"_")
+			<< ((i->flags & peer_info::supports_extensions)?'e':'.')
-			<< static_cast<const char*>((i->flags & peer_info::local_connection)?"l":"r") << " ";
+			<< ((i->flags & peer_info::local_connection)?'l':'r') << " ";
 
 		if (i->downloading_piece_index >= 0)
 		{
 			out << progress_bar(
-				i->downloading_progress / static_cast<float>(i->downloading_total)
+				i->downloading_progress / float(i->downloading_total), 15);
-				, 15);
 		}
 		else
 		{
@@ -289,17 +276,107 @@ void print_peer_info(std::ostream& out, std::vector<libtorrent::peer_info> const
 	}
 }
 
-int main(int argc, char* argv[])
+void add_torrent(libtorrent::session& ses
+	, std::vector<libtorrent::torrent_handle>& handles
+	, char const* torrent
+	, float preferred_ratio
+	, boost::filesystem::path const& save_path)
 {
 	using namespace libtorrent;
 
-	if (argc < 2)
+	TORRENT_CHECKPOINT("++ load torrent");
+	std::ifstream in(torrent, std::ios_base::binary);
+	in.unsetf(std::ios_base::skipws);
+	entry e = bdecode(std::istream_iterator<char>(in), std::istream_iterator<char>());
+	torrent_info t(e);
+	TORRENT_CHECKPOINT("-- load torrent");
+	std::cout << t.name() << "\n";
+
+	TORRENT_CHECKPOINT("++ load resumedata");
+	entry resume_data;
+	try
 	{
-		std::cerr << "usage: ./client_test torrent-files ...\n"
+		std::stringstream s;
-			"to stop the client, press q.\n";
+		s << t.name() << ".fastresume";
+		boost::filesystem::ifstream resume_file(save_path / s.str(), std::ios_base::binary);
+		resume_file.unsetf(std::ios_base::skipws);
+		resume_data = bdecode(
+				std::istream_iterator<char>(resume_file)
+				, std::istream_iterator<char>());
+	}
+	catch (invalid_encoding&) {}
+	catch (boost::filesystem::filesystem_error&) {}
+	TORRENT_CHECKPOINT("-- load resumedata");
+
+	handles.push_back(ses.add_torrent(e, save_path, resume_data, true, 16 * 1024));
+	handles.back().set_max_connections(60);
+	handles.back().set_max_uploads(-1);
+	handles.back().set_ratio(preferred_ratio);
+}
+
+int main(int ac, char* av[])
+{
+	int listen_port;
+	float preferred_ratio;
+	int download_limit;
+	int upload_limit;
+	std::string save_path_str;
+	std::string log_level;
+	std::string ip_filter_file;
+
+	namespace po = boost::program_options;
+
+	po::options_description desc("supported options");
+	desc.add_options()
+		("help,h", "display this help message")
+		("port,p", po::value<int>(&listen_port)->default_value(6881)
+			, "set listening port")
+		("ratio,r", po::value<float>(&preferred_ratio)->default_value(0)
+			, "set the preferred upload/download ratio. 0 means infinite. Values "
+		 	"smaller than 1 are clamped to 1.")
+		("max-download-rate,d", po::value<int>(&download_limit)->default_value(0)
+			, "the maximum download rate given in kB/s. 0 means infinite.")
+		("max-upload-rate,u", po::value<int>(&upload_limit)->default_value(0)
+			, "the maximum upload rate given in kB/s. 0 means infinite.")
+		("save-path,s", po::value<std::string>(&save_path_str)->default_value("./")
+			, "the path where the downloaded file/folder should be placed.")
+		("log-level,l", po::value<std::string>(&log_level)->default_value("info")
+			, "sets the level at which events are logged [debug | info | warning].")
+		("ip-filter,f", po::value<std::string>(&ip_filter_file)->default_value("")
+			, "sets the path to the ip-filter file used to block access from certain "
+			"ips. ")
+		("input-file,i", po::value< std::vector<std::string> >()
+			, "adds an input .torrent file. At least one is required. arguments "
+			"without any flag are implicitly an input file. To start a torrentless "
+			"download, use <info-hash>@<tracker-url> instead of specifying a file.")
+		;
+
+	po::positional_options_description p;
+	p.add("input-file", -1);
+
+	po::variables_map vm;
+	po::store(po::command_line_parser(ac, av).
+		options(desc).positional(p).run(), vm);
+	po::notify(vm);    
+	
+	if (vm.count("help") || vm.count("input-file") == 0)
+	{
+		std::cout << desc << "\n";
 		return 1;
 	}
 
+	// make sure the arguments stays within the usable limits
+	if (listen_port < 0 || listen_port > 65525) listen_port = 6881;
+	if (preferred_ratio != 0 && preferred_ratio < 1.f) preferred_ratio = 1.f;
+	upload_limit *= 1000;
+	download_limit *= 1000;
+	if (download_limit <= 0) download_limit = -1;
+	if (upload_limit <= 0) upload_limit = -1;
+	
+	using namespace libtorrent;
+
+	std::vector<std::string> const& input = vm["input-file"].as< std::vector<std::string> >();
+
 	namespace fs = boost::filesystem;
 	fs::path::default_name_check(fs::no_check);
 	
@@ -317,11 +394,16 @@ int main(int argc, char* argv[])
 		std::vector<torrent_handle> handles;
 		session ses;
 
-		ses.listen_on(std::make_pair(6880, 6889));
+		ses.set_download_rate_limit(download_limit);
-		//ses.set_upload_rate_limit(512 * 1024);
+		ses.set_upload_rate_limit(upload_limit);
+		ses.listen_on(std::make_pair(listen_port, listen_port + 10));
 		ses.set_http_settings(settings);
-		ses.set_severity_level(alert::debug);
+		if (log_level == "debug")
-//		ses.set_severity_level(alert::info);
+			ses.set_severity_level(alert::debug);
+		else if (log_level == "warning")
+			ses.set_severity_level(alert::warning);
+		else
+			ses.set_severity_level(alert::info);
 
 		// look for ipfilter.dat
 		// poor man's parser
@@ -342,7 +424,7 @@ int main(int argc, char* argv[])
 		// here ranges may overlap, and it is the last added
 		// rule that has precedence for addresses that may fall
 		// into more than one range.
-		std::ifstream in("ipfilter.dat");
+		std::ifstream in(ip_filter_file.c_str());
 		ip_filter filter;
 		while (in.good())
 		{
@@ -366,54 +448,29 @@ int main(int argc, char* argv[])
 		}
 	
 		ses.set_ip_filter(filter);
-	
+		boost::filesystem::path save_path(save_path_str);
-		for (int i = 0; i < argc-1; ++i)
+
+		// load the torrents given on the commandline
+		boost::regex ex("([0-9A-Fa-f]{40})@(.+)");
+		for (std::vector<std::string>::const_iterator i = input.begin();
+			i != input.end(); ++i)
 		{
 			try
 			{
-				boost::filesystem::path save_path("./");
+				// first see if this is a torrentless download
-
+				boost::cmatch what;
-				if (std::string(argv[i+1]).substr(0, 7) == "http://")
+				if (boost::regex_match(i->c_str(), what, ex))
 				{
-					sha1_hash info_hash = boost::lexical_cast<sha1_hash>(argv[i+2]);
+					sha1_hash info_hash = boost::lexical_cast<sha1_hash>(what[1]);
 
-					handles.push_back(ses.add_torrent(argv[i+1], info_hash, save_path));
+					handles.push_back(ses.add_torrent(std::string(what[2]).c_str(), info_hash, save_path));
 					handles.back().set_max_connections(60);
 					handles.back().set_max_uploads(-1);
-//					handles.back().set_ratio(1.1f);
+					handles.back().set_ratio(preferred_ratio);
-					++i;
-
 					continue;
 				}
-				std::ifstream in(argv[i+1], std::ios_base::binary);
+				// if it's a torrent file, open it as usual
-				in.unsetf(std::ios_base::skipws);
+				add_torrent(ses, handles, i->c_str(), preferred_ratio, save_path);
-				entry e = bdecode(std::istream_iterator<char>(in), std::istream_iterator<char>());
-				torrent_info t(e);
-				t.print(std::cout);
-
-				entry resume_data;
-				try
-				{
-					std::stringstream s;
-					s << t.name() << ".fastresume";
-					boost::filesystem::ifstream resume_file(save_path / s.str(), std::ios_base::binary);
-					resume_file.unsetf(std::ios_base::skipws);
-					resume_data = bdecode(
-						std::istream_iterator<char>(resume_file)
-						, std::istream_iterator<char>());
-				}
-				catch (invalid_encoding&) {}
-				catch (boost::filesystem::filesystem_error&) {}
-
-				handles.push_back(ses.add_torrent(e, save_path, resume_data, true, 16 * 1024));
-				handles.back().set_max_connections(60);
-				handles.back().set_max_uploads(-1);
-//				handles.back().set_ratio(1.02f);
-				
-//				std::vector<bool> ffilter(t.num_files(), true);
-//				ffilter[0] = false;
-//				handles.back().filter_files(ffilter);
-
 			}
 			catch (std::exception& e)
 			{
@@ -423,6 +480,7 @@ int main(int argc, char* argv[])
 
 		if (handles.empty()) return 1;
 
+		// main loop
 		std::vector<peer_info> peers;
 		std::vector<partial_piece_info> queue;
 
@@ -441,9 +499,10 @@ int main(int argc, char* argv[])
 						i != handles.end(); ++i)
 					{
 						torrent_handle h = *i;
-						if (!h.get_torrent_info().is_valid()) continue;
+						if (!h.is_valid() || !h.has_metadata()) continue;
 
 						h.pause();
+
 						entry data = h.write_resume_data();
 						std::stringstream s;
 						s << h.get_torrent_info().name() << ".fastresume";
@@ -458,34 +517,27 @@ int main(int argc, char* argv[])
 				if(c == 'r')
 				{
 					// force reannounce on all torrents
-					std::for_each(
+					std::for_each(handles.begin(), handles.end()
-						handles.begin()
-						, handles.end()
 						, boost::bind(&torrent_handle::force_reannounce, _1));
 				}
 
 				if(c == 'p')
 				{
 					// pause all torrents
-					std::for_each(
+					std::for_each(handles.begin(), handles.end()
-						handles.begin()
-						, handles.end()
 						, boost::bind(&torrent_handle::pause, _1));
 				}
 
 				if(c == 'u')
 				{
 					// unpause all torrents
-					std::for_each(
+					std::for_each(handles.begin(), handles.end()
-						handles.begin()
-						, handles.end()
 						, boost::bind(&torrent_handle::resume, _1));
 				}
 
 				if (c == 'i') print_peers = !print_peers;
 				if (c == 'l') print_log = !print_log;
 				if (c == 'd') print_downloads = !print_downloads;
-
 			}
 
 			// loop through the alert queue to see if anything has happened.
@@ -543,30 +595,10 @@ int main(int argc, char* argv[])
 
 				if (s.state != torrent_status::seeding)
 				{
-					switch(s.state)
+					static char const* state_str[] =
-					{
+						{"queued", "checking", "connecting", "downloading metadata"
-						case torrent_status::queued_for_checking:
+						, "downloading", "finished", "seeding"};
-							out << "queued ";
+					out << state_str[s.state] << " ";
-							break;
-						case torrent_status::checking_files:
-							out << "checking ";
-							break;
-						case torrent_status::connecting_to_tracker:
-							out << "connecting to tracker ";
-							break;
-						case torrent_status::downloading_metadata:
-							out << "downloading metadata ";
-							break;
-						case torrent_status::downloading:
-							out << "downloading ";
-							break;
-						case torrent_status::finished:
-							out << "finished ";
-							break;
-						case torrent_status::seeding:
-							out << "seeding ";
-							break;
-					};
 				}
 
 				i->get_peer_info(peers);
@@ -658,16 +690,17 @@ int main(int argc, char* argv[])
 				}
 			}
 
-			clear();
+			clear_home();
-			set_cursor(0, 0);
 			puts(out.str().c_str());
-//			std::cout << out.str();
-//			std::cout.flush();
 		}
 	}
 	catch (std::exception& e)
 	{
   		std::cout << e.what() << "\n";
 	}
+#ifdef TORRENT_PROFILE
+	print_checkpoints();
+#endif
 	return 0;
 }
+

include/libtorrent/session.hpp

@@ -305,7 +305,7 @@ namespace libtorrent
 	{
 	public:
 
-		session(fingerprint const& print = fingerprint("LT", 0, 1, 0, 0));
+		session(fingerprint const& print = fingerprint("LT", 0, 9, 1, 0));
 		session(
 			fingerprint const& print
 			, std::pair<int, int> listen_port_range

include/libtorrent/torrent.hpp

@@ -384,7 +384,8 @@ namespace libtorrent
 		void set_max_connections(int limit);
 		bool move_storage(boost::filesystem::path const& save_path);
 
-		bool valid_metadata() const { return m_storage.get() != 0; }
+		bool valid_metadata() const
+		{ return m_storage.get() != 0 && m_connections_initialized; }
 		std::vector<char> const& metadata() const { return m_metadata; }
 
 		bool received_metadata(
@@ -546,6 +547,17 @@ namespace libtorrent
 		// defaults to 16 kiB, but can be set by the user
 		// when creating the torrent
 		const int m_default_block_size;
+
+		// this is set to false as long as the connections
+		// of this torrent hasn't been initialized. If we
+		// have metadata from the start, connections are
+		// initialized immediately, if we didn't have metadata,
+		// they are initialized right after files_checked().
+		// valid_resume_data() will return false as long as
+		// the connections aren't initialized, to avoid
+		// them from altering the piece-picker before it
+		// has been initialized with files_checked().
+		bool m_connections_initialized;
 	};
 
 	inline boost::posix_time::ptime torrent::next_announce() const

src/session.cpp

@@ -199,6 +199,13 @@ namespace libtorrent { namespace detail
 							t->torrent_ptr->get_handle()
 							, e.what()));
 				}
+				if (t->torrent_ptr->num_peers())
+				{
+					m_ses.m_torrents.insert(std::make_pair(
+						t->info_hash, t->torrent_ptr));
+					t->torrent_ptr->abort();
+				}
+
 				assert(!m_torrents.empty());
 				m_torrents.pop_front();
 			}
@@ -282,6 +289,7 @@ namespace libtorrent { namespace detail
 				// This will happen if the storage fails to initialize
 				boost::mutex::scoped_lock l(m_ses.m_mutex);
 				boost::mutex::scoped_lock l2(m_mutex);
+
 				if (m_ses.m_alerts.should_post(alert::fatal))
 				{
 					m_ses.m_alerts.post_alert(
@@ -291,6 +299,13 @@ namespace libtorrent { namespace detail
 				}
 				assert(!m_processing.empty());
 
+				if (processing->torrent_ptr->num_peers())
+				{
+					m_ses.m_torrents.insert(std::make_pair(
+						processing->info_hash, processing->torrent_ptr));
+					processing->torrent_ptr->abort();
+				}
+
 				processing.reset();
 				m_processing.pop_front();
 				if (!m_processing.empty())

src/storage.cpp

@@ -329,7 +329,7 @@ namespace libtorrent
 {
 
 	std::vector<std::pair<size_type, std::time_t> > get_filesizes(
-		const torrent_info& t, path p)
+		torrent_info const& t, path p)
 	{
 		p = complete(p);
 		std::vector<std::pair<size_type, std::time_t> > sizes;
@@ -469,7 +469,7 @@ namespace libtorrent
 
 	file_pool storage::impl::files(40);
 
-	storage::storage(const torrent_info& info, const path& path)
+	storage::storage(torrent_info const& info, path const& path)
 		: m_pimpl(new impl(info, path))
 	{
 		assert(info.begin_files() != info.end_files());
@@ -787,8 +787,8 @@ namespace libtorrent
 	public:
 
 		impl(
-			const torrent_info& info
+			torrent_info const& info
-			, const path& path);
+			, path const& path);
 
 		bool check_fastresume(
 			detail::piece_checker_data& d
@@ -870,7 +870,7 @@ namespace libtorrent
 		// a bitmask representing the pieces we have
 		std::vector<bool> m_have_piece;
 
-		const torrent_info& m_info;
+		torrent_info const& m_info;
 
 		// slots that haven't had any file storage allocated
 		std::vector<int> m_unallocated_slots;
@@ -929,8 +929,8 @@ namespace libtorrent
 	};
 
 	piece_manager::impl::impl(
-		const torrent_info& info
+		torrent_info const& info
-		, const path& save_path)
+		, path const& save_path)
 		: m_storage(info, save_path)
 		, m_compact_mode(false)
 		, m_fill_mode(true)
@@ -942,8 +942,8 @@ namespace libtorrent
 	}
 
 	piece_manager::piece_manager(
-		const torrent_info& info
+		torrent_info const& info
-	  , const path& save_path)
+		, path const& save_path)
 		: m_pimpl(new impl(info, save_path))
 	{
 	}

src/torrent.cpp

@@ -221,6 +221,7 @@ namespace libtorrent
 		, m_metadata_progress(0)
 		, m_metadata_size(0)
 		, m_default_block_size(block_size)
+		, m_connections_initialized(true)
 	{
 		m_uploads_quota.min = 2;
 		m_connections_quota.min = 2;
@@ -306,6 +307,7 @@ namespace libtorrent
 		, m_metadata_progress(0)
 		, m_metadata_size(0)
 		, m_default_block_size(block_size)
+		, m_connections_initialized(false)
 	{
 		m_uploads_quota.min = 2;
 		m_connections_quota.min = 2;
@@ -1067,7 +1069,13 @@ namespace libtorrent
 	}
 
 	bool torrent::check_fastresume(detail::piece_checker_data& data)
-	{	
+	{
+		if (!m_storage.get())
+		{
+			// this means we have received the metadata through the
+			// metadata extension, and we have to initialize
+			init();
+		}
 		assert(m_storage.get());
 		return m_storage->check_fastresume(data, m_have_pieces, m_compact_mode);
 	}
@@ -1087,6 +1095,20 @@ namespace libtorrent
 		  , true);
 
 		m_picker->files_checked(m_have_pieces, unfinished_pieces);
+		if (!m_connections_initialized)
+		{
+			m_connections_initialized = true;
+			// all peer connections have to initialize themselves now that the metadata
+			// is available
+			typedef std::map<address, peer_connection*> conn_map;
+			for (conn_map::iterator i = m_connections.begin()
+					, end(m_connections.end()); i != end; ++i)
+			{
+				try { i->second->init(); }
+				catch (std::exception&e) {}
+				// TODO: in case of an exception, close the connection
+			}
+		}
 	}
 
 	alert_manager& torrent::alerts() const
@@ -1516,7 +1538,6 @@ namespace libtorrent
 
 		entry metadata = bdecode(m_metadata.begin(), m_metadata.end());
 		m_torrent_file.parse_info_section(metadata);
-		init();
 
 		{
 			boost::mutex::scoped_lock(m_checker.m_mutex);
@@ -1543,20 +1564,6 @@ namespace libtorrent
 				get_handle(), "metadata successfully received from swarm"));
 		}
 
-		// all peer connections have to initialize themselves now that the metadata
-		// is available
-		// TODO: is it ok to initialize the connections before the file check?
-		typedef std::map<address, peer_connection*> conn_map;
-		for (conn_map::iterator i = m_connections.begin()
-			, end(m_connections.end()); i != end; ++i)
-		{
-			i->second->init();
-		}
-
-#ifndef NDEBUG
-		m_picker->integrity_check(this);
-#endif
-
 		// clear the storage for the bitfield
 		std::vector<bool>().swap(m_have_metadata);
 		std::vector<int>().swap(m_requested_metadata);