optimized session::add_torrent() by putting more info in torrent_info, to make it possible to reproduce the exact info section from an torrent_info object. The result is that the .torrent file is only parsed once.

This commit is contained in:
Arvid Norberg 2005-10-16 16:58:41 +00:00
parent f770d1438d
commit f63702c964
15 changed files with 117 additions and 74 deletions

View File

@ -1,10 +1,11 @@
* greatly improved the command line control of the example client_test * changed the interface to session::add_torrent for soem speed optimizations.
* fixed bug where upload rate limit was not being applied * greatly improved the command line control of the example client_test.
* fixed bug where upload rate limit was not being applied.
* files that are being checked will no longer stall files that don't need * files that are being checked will no longer stall files that don't need
checking. checking.
* changed the way libtorrent identifies support for its excentions * changed the way libtorrent identifies support for its excentions
to look for 'ext' at the end of the peer-id. to look for 'ext' at the end of the peer-id.
* improved performance by adding a circle buffer for the send buffer * improved performance by adding a circle buffer for the send buffer.
* fixed bugs in the http tracker connection when using an http proxy. * fixed bugs in the http tracker connection when using an http proxy.
* fixed problem with storage's file pool when creating torrents and then * fixed problem with storage's file pool when creating torrents and then
starting to seed them. starting to seed them.

View File

@ -36,7 +36,6 @@ project torrent
<threading>multi <threading>multi
<toolset>msvc:<cxxflags>/Zc:wchar_t <toolset>msvc:<cxxflags>/Zc:wchar_t
<toolset>msvc:<cxxflags>/Zc:forScope <toolset>msvc:<cxxflags>/Zc:forScope
<define>TORRENT_PROFILE
: usage-requirements : usage-requirements
@ -44,7 +43,6 @@ project torrent
<include>$(BOOST_ROOT) <include>$(BOOST_ROOT)
<variant>release:<define>NDEBUG <variant>release:<define>NDEBUG
<define>BOOST_ALL_NO_LIB <define>BOOST_ALL_NO_LIB
<define>TORRENT_PROFILE
; ;

View File

@ -555,7 +555,7 @@ class session: public boost::noncopyable
, const char* listen_interface = 0); , const char* listen_interface = 0);
torrent_handle add_torrent( torrent_handle add_torrent(
entry const&amp; e torrent_info const&amp; ti
, boost::filesystem::path const&amp; save_path , boost::filesystem::path const&amp; save_path
, entry const&amp; resume_data = entry() , entry const&amp; resume_data = entry()
, bool compact_mode = true , bool compact_mode = true
@ -630,7 +630,7 @@ timeout can be set with <tt class="docutils literal"><span class="pre">set_http_
<blockquote> <blockquote>
<pre class="literal-block"> <pre class="literal-block">
torrent_handle add_torrent( torrent_handle add_torrent(
entry const&amp; e torrent_info const&amp; ti
, boost::filesystem::path const&amp; save_path , boost::filesystem::path const&amp; save_path
, entry const&amp; resume_data = entry() , entry const&amp; resume_data = entry()
, bool compact_mode = true , bool compact_mode = true
@ -667,10 +667,10 @@ size is an even power of 2, so must the block size be. If the block size given h
out to be greater than the piece size, it will simply be clamped to the piece size.</p> out to be greater than the piece size, it will simply be clamped to the piece size.</p>
<p>The <a class="reference" href="#torrent-handle">torrent_handle</a> returned by <tt class="docutils literal"><span class="pre">add_torrent()</span></tt> can be used to retrieve information <p>The <a class="reference" href="#torrent-handle">torrent_handle</a> returned by <tt class="docutils literal"><span class="pre">add_torrent()</span></tt> can be used to retrieve information
about the torrent's progress, its peers etc. It is also used to abort a torrent.</p> about the torrent's progress, its peers etc. It is also used to abort a torrent.</p>
<p>The second overload that takes a tracker url and an info-hash instead of metadata (<tt class="docutils literal"><span class="pre">entry</span></tt>) <p>The second overload that takes a tracker url and an info-hash instead of metadata
can be used with torrents where (at least some) peers support the metadata extension. For (<tt class="docutils literal"><span class="pre">torrent_info</span></tt>) can be used with torrents where (at least some) peers support
the overload to be available, libtorrent must be built with extensions enabled the metadata extension. For the overload to be available, libtorrent must be built
(<tt class="docutils literal"><span class="pre">TORRENT_ENABLE_EXTENSIONS</span></tt> defined).</p> with extensions enabled (<tt class="docutils literal"><span class="pre">TORRENT_ENABLE_EXTENSIONS</span></tt> defined).</p>
</div> </div>
<div class="section" id="remove-torrent"> <div class="section" id="remove-torrent">
<h2><a name="remove-torrent">remove_torrent()</a></h2> <h2><a name="remove-torrent">remove_torrent()</a></h2>

View File

@ -433,7 +433,7 @@ The ``session`` class has the following synopsis::
, const char* listen_interface = 0); , const char* listen_interface = 0);
torrent_handle add_torrent( torrent_handle add_torrent(
entry const& e torrent_info const& ti
, boost::filesystem::path const& save_path , boost::filesystem::path const& save_path
, entry const& resume_data = entry() , entry const& resume_data = entry()
, bool compact_mode = true , bool compact_mode = true
@ -512,7 +512,7 @@ add_torrent()
:: ::
torrent_handle add_torrent( torrent_handle add_torrent(
entry const& e torrent_info const& ti
, boost::filesystem::path const& save_path , boost::filesystem::path const& save_path
, entry const& resume_data = entry() , entry const& resume_data = entry()
, bool compact_mode = true , bool compact_mode = true
@ -554,10 +554,10 @@ out to be greater than the piece size, it will simply be clamped to the piece si
The torrent_handle_ returned by ``add_torrent()`` can be used to retrieve information The torrent_handle_ returned by ``add_torrent()`` can be used to retrieve information
about the torrent's progress, its peers etc. It is also used to abort a torrent. about the torrent's progress, its peers etc. It is also used to abort a torrent.
The second overload that takes a tracker url and an info-hash instead of metadata (``entry``) The second overload that takes a tracker url and an info-hash instead of metadata
can be used with torrents where (at least some) peers support the metadata extension. For (``torrent_info``) can be used with torrents where (at least some) peers support
the overload to be available, libtorrent must be built with extensions enabled the metadata extension. For the overload to be available, libtorrent must be built
(``TORRENT_ENABLE_EXTENSIONS`` defined). with extensions enabled (``TORRENT_ENABLE_EXTENSIONS`` defined).
remove_torrent() remove_torrent()
---------------- ----------------

View File

@ -242,7 +242,7 @@ void print_peer_info(std::ostream& out, std::vector<libtorrent::peer_info> const
{ {
using namespace libtorrent; using namespace libtorrent;
out << " down up q r flags block progress client \n"; out << " down (total) up (total) q r flags block progress client \n";
for (std::vector<peer_info>::const_iterator i = peers.begin(); for (std::vector<peer_info>::const_iterator i = peers.begin();
i != peers.end(); ++i) i != peers.end(); ++i)
@ -289,7 +289,7 @@ void add_torrent(libtorrent::session& ses
in.unsetf(std::ios_base::skipws); in.unsetf(std::ios_base::skipws);
entry e = bdecode(std::istream_iterator<char>(in), std::istream_iterator<char>()); entry e = bdecode(std::istream_iterator<char>(in), std::istream_iterator<char>());
torrent_info t(e); torrent_info t(e);
TORRENT_CHECKPOINT("-- load torrent");
std::cout << t.name() << "\n"; std::cout << t.name() << "\n";
TORRENT_CHECKPOINT("++ load resumedata"); TORRENT_CHECKPOINT("++ load resumedata");
@ -306,12 +306,17 @@ void add_torrent(libtorrent::session& ses
} }
catch (invalid_encoding&) {} catch (invalid_encoding&) {}
catch (boost::filesystem::filesystem_error&) {} catch (boost::filesystem::filesystem_error&) {}
TORRENT_CHECKPOINT("-- load resumedata");
handles.push_back(ses.add_torrent(e, save_path, resume_data, true, 16 * 1024)); TORRENT_CHECKPOINT("++ ses::add_torrent");
handles.push_back(ses.add_torrent(t, save_path, resume_data, true, 16 * 1024));
TORRENT_CHECKPOINT("-- ses::add_torrent");
handles.back().set_max_connections(60); handles.back().set_max_connections(60);
handles.back().set_max_uploads(-1); handles.back().set_max_uploads(-1);
handles.back().set_ratio(preferred_ratio); handles.back().set_ratio(preferred_ratio);
TORRENT_CHECKPOINT("-- add_torrent");
} }
int main(int ac, char* av[]) int main(int ac, char* av[])

View File

@ -58,20 +58,16 @@ int main(int argc, char* argv[])
in.unsetf(std::ios_base::skipws); in.unsetf(std::ios_base::skipws);
entry e = bdecode(std::istream_iterator<char>(in), std::istream_iterator<char>()); entry e = bdecode(std::istream_iterator<char>(in), std::istream_iterator<char>());
std::cout << "\n\n----- raw info -----\n\n"; std::cout << "\n\n----- raw info -----\n\n";
e.print(std::cout); e.print(std::cout);
torrent_info t(e); torrent_info t(e);
// print info about torrent // print info about torrent
std::cout << "\n\n----- torrent file info -----\n\n"; std::cout << "\n\n----- torrent file info -----\n\n";
std::cout << "trackers:\n"; std::cout << "trackers:\n";
for (std::vector<announce_entry>::const_iterator i = t.trackers().begin(); for (std::vector<announce_entry>::const_iterator i = t.trackers().begin();
i != t.trackers().end(); i != t.trackers().end(); ++i)
++i)
{ {
std::cout << i->tier << ": " << i->url << "\n"; std::cout << i->tier << ": " << i->url << "\n";
} }

View File

@ -65,7 +65,7 @@ int main(int argc, char* argv[])
std::ifstream in(argv[1], std::ios_base::binary); std::ifstream in(argv[1], std::ios_base::binary);
in.unsetf(std::ios_base::skipws); in.unsetf(std::ios_base::skipws);
entry e = bdecode(std::istream_iterator<char>(in), std::istream_iterator<char>()); entry e = bdecode(std::istream_iterator<char>(in), std::istream_iterator<char>());
s.add_torrent(e, "./"); s.add_torrent(torrent_info(e), "./");
// wait for the user to end // wait for the user to end
char a; char a;

View File

@ -54,7 +54,6 @@ POSSIBILITY OF SUCH DAMAGE.
#endif #endif
#include "libtorrent/torrent_handle.hpp" #include "libtorrent/torrent_handle.hpp"
//#include "libtorrent/torrent.hpp"
#include "libtorrent/entry.hpp" #include "libtorrent/entry.hpp"
#include "libtorrent/torrent_info.hpp" #include "libtorrent/torrent_info.hpp"
#include "libtorrent/socket.hpp" #include "libtorrent/socket.hpp"
@ -317,12 +316,24 @@ namespace libtorrent
// all torrent_handles must be destructed before the session is destructed! // all torrent_handles must be destructed before the session is destructed!
torrent_handle add_torrent( torrent_handle add_torrent(
entry const& metadata torrent_info const& ti
, boost::filesystem::path const& save_path , boost::filesystem::path const& save_path
, entry const& resume_data = entry() , entry const& resume_data = entry()
, bool compact_mode = true , bool compact_mode = true
, int block_size = 16 * 1024); , int block_size = 16 * 1024);
// TODO: depricated, this is for backwards compatibility only
torrent_handle add_torrent(
entry const& e
, boost::filesystem::path const& save_path
, entry const& resume_data = entry()
, bool compact_mode = true
, int block_size = 16 * 1024)
{
return add_torrent(torrent_info(e), save_path, resume_data
, compact_mode, block_size);
}
torrent_handle add_torrent( torrent_handle add_torrent(
char const* tracker_url char const* tracker_url
, sha1_hash const& info_hash , sha1_hash const& info_hash

View File

@ -97,7 +97,6 @@ namespace libtorrent
torrent( torrent(
detail::session_impl& ses detail::session_impl& ses
, detail::checker_impl& checker , detail::checker_impl& checker
, entry const& metadata
, torrent_info const& tf , torrent_info const& tf
, boost::filesystem::path const& save_path , boost::filesystem::path const& save_path
, address const& net_interface , address const& net_interface
@ -386,7 +385,7 @@ namespace libtorrent
bool valid_metadata() const bool valid_metadata() const
{ return m_storage.get() != 0 && m_connections_initialized; } { return m_storage.get() != 0 && m_connections_initialized; }
std::vector<char> const& metadata() const { return m_metadata; } std::vector<char> const& metadata() const;
bool received_metadata( bool received_metadata(
char const* buf char const* buf
@ -524,7 +523,8 @@ namespace libtorrent
// this buffer is filled with the info-section of // this buffer is filled with the info-section of
// the metadata file while downloading it from // the metadata file while downloading it from
// peers, and while sending it. // peers, and while sending it.
std::vector<char> m_metadata; // it is mutable because it's generated lazily
mutable std::vector<char> m_metadata;
// this is a bitfield of size 256, each bit represents // this is a bitfield of size 256, each bit represents
// a piece of the metadata. It is set to one if we // a piece of the metadata. It is set to one if we

View File

@ -91,7 +91,7 @@ namespace libtorrent
torrent_info(entry const& torrent_file); torrent_info(entry const& torrent_file);
~torrent_info(); ~torrent_info();
entry create_torrent(); entry create_torrent() const;
entry create_info_metadata() const; entry create_info_metadata() const;
void set_comment(char const* str); void set_comment(char const* str);
void set_creator(char const* str); void set_creator(char const* str);
@ -168,7 +168,9 @@ namespace libtorrent
size_type m_total_size; size_type m_total_size;
// the hash that identifies this torrent // the hash that identifies this torrent
sha1_hash m_info_hash; // it is mutable because it's calculated
// lazily
mutable sha1_hash m_info_hash;
std::string m_name; std::string m_name;
@ -191,6 +193,12 @@ namespace libtorrent
// or not. e.g. test/test there's one file and one directory // or not. e.g. test/test there's one file and one directory
// and they have the same name. // and they have the same name.
bool m_multifile; bool m_multifile;
// contains any non-parsed entries from the info-section
// these are kept in order to be able to accurately
// reproduce the info-section when sending the metadata
// to peers.
entry m_extra_info;
}; };
} }

View File

@ -651,9 +651,9 @@ namespace libtorrent
assert(num_blocks > 0); assert(num_blocks > 0);
#ifdef TORRENT_VERBOSE_LOGGING #ifdef TORRENT_VERBOSE_LOGGING
std::ofstream f("piece_picker.log", std::ios_base::app); // std::ofstream f("piece_picker.log", std::ios_base::app);
f << "backup_blocks: " << backup_blocks.size() << "\n" // f << "backup_blocks: " << backup_blocks.size() << "\n"
<< "used: " << std::min(num_blocks, (int)backup_blocks.size()) << "\n----\n"; // << "used: " << std::min(num_blocks, (int)backup_blocks.size()) << "\n----\n";
#endif #endif
interesting_blocks.insert(interesting_blocks.end() interesting_blocks.insert(interesting_blocks.end()

View File

@ -382,6 +382,9 @@ namespace libtorrent { namespace detail
{ {
#if defined(TORRENT_VERBOSE_LOGGING) || defined(TORRENT_LOGGING) #if defined(TORRENT_VERBOSE_LOGGING) || defined(TORRENT_LOGGING)
m_logger = create_log("main_session", false); m_logger = create_log("main_session", false);
using boost::posix_time::second_clock;
using boost::posix_time::to_simple_string;
(*m_logger) << to_simple_string(second_clock::universal_time()) << "\n";
#endif #endif
std::fill(m_extension_enabled, m_extension_enabled std::fill(m_extension_enabled, m_extension_enabled
+ peer_connection::num_supported_extensions, true); + peer_connection::num_supported_extensions, true);
@ -1142,13 +1145,12 @@ namespace libtorrent
// current platform. // current platform.
// if the torrent already exists, this will throw duplicate_torrent // if the torrent already exists, this will throw duplicate_torrent
torrent_handle session::add_torrent( torrent_handle session::add_torrent(
entry const& metadata torrent_info const& ti
, boost::filesystem::path const& save_path , boost::filesystem::path const& save_path
, entry const& resume_data , entry const& resume_data
, bool compact_mode , bool compact_mode
, int block_size) , int block_size)
{ {
TORRENT_CHECKPOINT("++ session::add_torrent()");
// make sure the block_size is an even power of 2 // make sure the block_size is an even power of 2
#ifndef NDEBUG #ifndef NDEBUG
for (int i = 0; i < 32; ++i) for (int i = 0; i < 32; ++i)
@ -1163,7 +1165,6 @@ namespace libtorrent
assert(!save_path.empty()); assert(!save_path.empty());
torrent_info ti(metadata);
if (ti.begin_files() == ti.end_files()) if (ti.begin_files() == ti.end_files())
throw std::runtime_error("no files in torrent"); throw std::runtime_error("no files in torrent");
@ -1184,7 +1185,7 @@ namespace libtorrent
// the checker thread and store it before starting // the checker thread and store it before starting
// the thread // the thread
boost::shared_ptr<torrent> torrent_ptr( boost::shared_ptr<torrent> torrent_ptr(
new torrent(m_impl, m_checker_impl, metadata, ti, save_path new torrent(m_impl, m_checker_impl, ti, save_path
, m_impl.m_listen_interface, compact_mode, block_size)); , m_impl.m_listen_interface, compact_mode, block_size));
boost::shared_ptr<detail::piece_checker_data> d( boost::shared_ptr<detail::piece_checker_data> d(
@ -1200,7 +1201,6 @@ namespace libtorrent
// job in its queue // job in its queue
m_checker_impl.m_cond.notify_one(); m_checker_impl.m_cond.notify_one();
TORRENT_CHECKPOINT("-- session::add_torrent()");
return torrent_handle(&m_impl, &m_checker_impl, ti.info_hash()); return torrent_handle(&m_impl, &m_checker_impl, ti.info_hash());
} }

View File

@ -182,7 +182,6 @@ namespace libtorrent
torrent::torrent( torrent::torrent(
detail::session_impl& ses detail::session_impl& ses
, detail::checker_impl& checker , detail::checker_impl& checker
, entry const& metadata
, torrent_info const& tf , torrent_info const& tf
, boost::filesystem::path const& save_path , boost::filesystem::path const& save_path
, address const& net_interface , address const& net_interface
@ -246,7 +245,6 @@ namespace libtorrent
m_ul_bandwidth_quota.min = 100; m_ul_bandwidth_quota.min = 100;
m_ul_bandwidth_quota.max = resource_request::inf; m_ul_bandwidth_quota.max = resource_request::inf;
if (m_ses.m_upload_rate == -1) if (m_ses.m_upload_rate == -1)
{ {
m_ul_bandwidth_quota.given = resource_request::inf; m_ul_bandwidth_quota.given = resource_request::inf;
@ -256,13 +254,7 @@ namespace libtorrent
m_ul_bandwidth_quota.given = 400; m_ul_bandwidth_quota.given = 400;
} }
m_policy.reset(new policy(this)); m_policy.reset(new policy(this));
// if anything should be optimized in this constructor
// this encoding should be made on demand. But that would
// require the copying of the entry-tree instead, which
// probably is more expensive
bencode(std::back_inserter(m_metadata), metadata["info"]);
init(); init();
} }
@ -1340,6 +1332,14 @@ namespace libtorrent
return m_tracker_address; return m_tracker_address;
} }
std::vector<char> const& torrent::metadata() const
{
if (m_metadata.empty())
bencode(std::back_inserter(m_metadata), m_torrent_file.create_torrent());
assert(!m_metadata.empty());
return m_metadata;
}
torrent_status torrent::status() const torrent_status torrent::status() const
{ {
assert(std::accumulate( assert(std::accumulate(

View File

@ -436,8 +436,7 @@ namespace libtorrent
entry::list_type& peer_list = ret["peers"].list(); entry::list_type& peer_list = ret["peers"].list();
for (torrent::const_peer_iterator i = t->begin(); for (torrent::const_peer_iterator i = t->begin();
i != t->end(); i != t->end(); ++i)
++i)
{ {
// we cannot save remote connection // we cannot save remote connection
// since we don't know their listen port // since we don't know their listen port

View File

@ -124,6 +124,7 @@ namespace libtorrent
torrent_info::torrent_info(const entry& torrent_file) torrent_info::torrent_info(const entry& torrent_file)
: m_creation_date(date(not_a_date_time)) : m_creation_date(date(not_a_date_time))
, m_multifile(false) , m_multifile(false)
, m_extra_info(entry::dictionary_t)
{ {
try try
{ {
@ -146,6 +147,7 @@ namespace libtorrent
, m_name() , m_name()
, m_creation_date(second_clock::universal_time()) , m_creation_date(second_clock::universal_time())
, m_multifile(false) , m_multifile(false)
, m_extra_info(entry::dictionary_t)
{ {
} }
@ -156,6 +158,7 @@ namespace libtorrent
, m_name() , m_name()
, m_creation_date(second_clock::universal_time()) , m_creation_date(second_clock::universal_time())
, m_multifile(false) , m_multifile(false)
, m_extra_info(entry::dictionary_t)
{ {
} }
@ -251,6 +254,24 @@ namespace libtorrent
hash_string.begin() + i*20 hash_string.begin() + i*20
, hash_string.begin() + (i+1)*20 , hash_string.begin() + (i+1)*20
, m_piece_hash[i].begin()); , m_piece_hash[i].begin());
for (entry::dictionary_type::const_iterator i = info.dict().begin()
, end(info.dict().end()); i != end; ++i)
{
if (i->first == "pieces"
|| i->first == "piece length"
|| i->first == "length")
continue;
m_extra_info[i->first] = i->second;
}
#ifndef NDEBUG
std::vector<char> info_section_buf;
entry gen_info_section = create_info_metadata();
bencode(std::back_inserter(info_section_buf), gen_info_section);
assert(hasher(&info_section_buf[0], info_section_buf.size()).final()
== m_info_hash);
#endif
} }
// extracts information from a libtorrent file and fills in the structures in // extracts information from a libtorrent file and fills in the structures in
@ -405,14 +426,18 @@ namespace libtorrent
// you have to add files to the torrent first // you have to add files to the torrent first
assert(!m_files.empty()); assert(!m_files.empty());
entry info(entry::dictionary_t); entry info(m_extra_info);
if (!info.find_key("name"))
info["name"] = m_name; info["name"] = m_name;
if (!m_multifile) if (!m_multifile)
{ {
info["length"] = m_files.front().size; info["length"] = m_files.front().size;
} }
else else
{
if (!info.find_key("files"))
{ {
entry& files = info["files"]; entry& files = info["files"];
files = entry(entry::list_t); files = entry(entry::list_t);
@ -437,6 +462,7 @@ namespace libtorrent
} }
} }
} }
}
info["piece length"] = piece_length(); info["piece length"] = piece_length();
entry& pieces = info["pieces"]; entry& pieces = info["pieces"];
@ -445,8 +471,7 @@ namespace libtorrent
std::string& p = pieces.string(); std::string& p = pieces.string();
for (std::vector<sha1_hash>::const_iterator i = m_piece_hash.begin(); for (std::vector<sha1_hash>::const_iterator i = m_piece_hash.begin();
i != m_piece_hash.end(); i != m_piece_hash.end(); ++i)
++i)
{ {
p.append((char*)i->begin(), (char*)i->end()); p.append((char*)i->begin(), (char*)i->end());
} }
@ -454,7 +479,7 @@ namespace libtorrent
return info; return info;
} }
entry torrent_info::create_torrent() entry torrent_info::create_torrent() const
{ {
assert(m_piece_length > 0); assert(m_piece_length > 0);