diff --git a/CMakeLists.txt b/CMakeLists.txt index 75230547f..8cf75121f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,6 +57,7 @@ set(sources random receive_buffer request_blocks + resolve_links resolver rss session diff --git a/ChangeLog b/ChangeLog index ec94b2b57..f81a40006 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ + * experimental support for BEP 38, "mutable torrents" * replaced lazy_bdecode with a new bdecoder that's a lot more efficient * deprecate time functions, expose typedefs of boost::chrono in the libtorrent namespace instead diff --git a/Jamfile b/Jamfile index bc51a2856..48bc6d0ae 100755 --- a/Jamfile +++ b/Jamfile @@ -405,6 +405,9 @@ feature.compose logging : TORRENT_DHT_VERBOSE_LOGGING ; feature encryption : on off : composite propagated link-incompatible ; feature.compose off : TORRENT_DISABLE_ENCRYPTION ; +feature mutable-torrents : on off : composite propagated link-incompatible ; +feature.compose off : TORRENT_DISABLE_MUTABLE_TORRENTS ; + feature crypto : built-in openssl gcrypt : composite propagated ; feature.compose openssl : TORRENT_USE_OPENSSL ; feature.compose gcrypt : TORRENT_USE_GCRYPT ; @@ -573,6 +576,7 @@ SOURCES = puff random receive_buffer + resolve_links rss session session_impl diff --git a/docs/building.rst b/docs/building.rst index 293a7e91b..1c0f98f66 100644 --- a/docs/building.rst +++ b/docs/building.rst @@ -289,6 +289,10 @@ Build features: | | connections. The shipped public domain SHA-1 | | | implementation is used. | +--------------------------+----------------------------------------------------+ +| ``mutable-torrents`` | * ``on`` - mutable torrents are supported | +| | (`BEP 38`_) (default). | +| | * ``off`` - mutable torrents are not supported. | ++--------------------------+----------------------------------------------------+ | ``crypto`` | * ``built-in`` - (default) uses built-in SHA-1 | | | implementation. | | | * ``openssl`` - links against openssl and | @@ -578,6 +582,8 @@ defines you can use to control the build. +----------------------------------------+-------------------------------------------------+ | ``TORRENT_DISABLE_POOL_ALLOCATOR`` | Disables use of ``boost::pool<>``. | +----------------------------------------+-------------------------------------------------+ +| ``TORRENT_DISABLE_MUTABLE_TORRENTS`` | Disables mutable torrent support (`BEP 38`_) | ++----------------------------------------+-------------------------------------------------+ | ``TORRENT_LINKING_SHARED`` | If this is defined when including the | | | libtorrent headers, the classes and functions | | | will be tagged with ``__declspec(dllimport)`` | @@ -644,6 +650,7 @@ defines you can use to control the build. | | custom one. | +----------------------------------------+-------------------------------------------------+ +.. _`BEP 38`: http://www.bittorrent.org/beps/bep_0038.html If you experience that libtorrent uses unreasonable amounts of cpu, it will definitely help to define ``NDEBUG``, since it will remove the invariant checks diff --git a/examples/make_torrent.cpp b/examples/make_torrent.cpp index 7a0d8ef6b..f90504884 100644 --- a/examples/make_torrent.cpp +++ b/examples/make_torrent.cpp @@ -39,6 +39,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/create_torrent.hpp" #include "libtorrent/file.hpp" #include "libtorrent/file_pool.hpp" +#include "libtorrent/escape_string.hpp" // for from_hex #include @@ -164,29 +165,38 @@ void print_usage() "Generates a torrent file from the specified file\n" "or directory and writes it to standard out\n\n" "OPTIONS:\n" - "-m file generate a merkle hash tree torrent.\n" - " merkle torrents require client support\n" - " the resulting full merkle tree is written to\n" - " the specified file\n" - "-w url adds a web seed to the torrent with\n" - " the specified url\n" - "-t url adds the specified tracker to the\n" - " torrent. For multiple trackers, specify more\n" - " -t options\n" - "-c comment sets the comment to the specified string\n" - "-C creator sets the created-by field to the specified string\n" - "-p bytes enables padding files. Files larger\n" - " than bytes will be piece-aligned\n" - "-s bytes specifies a piece size for the torrent\n" - " This has to be a multiple of 16 kiB\n" - "-l Don't follow symlinks, instead encode them as\n" - " links in the torrent file\n" - "-o file specifies the output filename of the torrent file\n" - " If this is not specified, the torrent file is\n" - " printed to the standard out, except on windows\n" - " where the filename defaults to a.torrent\n" - "-r file add root certificate to the torrent, to verify\n" - " the HTTPS tracker\n" + "-m file generate a merkle hash tree torrent.\n" + " merkle torrents require client support\n" + " the resulting full merkle tree is written to\n" + " the specified file\n" + "-w url adds a web seed to the torrent with\n" + " the specified url\n" + "-t url adds the specified tracker to the\n" + " torrent. For multiple trackers, specify more\n" + " -t options\n" + "-c comment sets the comment to the specified string\n" + "-C creator sets the created-by field to the specified string\n" + "-p bytes enables padding files. Files larger\n" + " than bytes will be piece-aligned\n" + "-s bytes specifies a piece size for the torrent\n" + " This has to be a multiple of 16 kiB\n" + "-l Don't follow symlinks, instead encode them as\n" + " links in the torrent file\n" + "-o file specifies the output filename of the torrent file\n" + " If this is not specified, the torrent file is\n" + " printed to the standard out, except on windows\n" + " where the filename defaults to a.torrent\n" + "-r file add root certificate to the torrent, to verify\n" + " the HTTPS tracker\n" + "-S info-hash add a similar torrent by info-hash. The similar\n" + " torrent is expected to share some files with this one\n" + "-L collection add a collection name to this torrent. Other torrents\n" + " in the same collection is expected to share files\n" + " with this one.\n" + "-M make the torrent compatible with mutable torrents\n" + " this means aligning large files and pad them in order\n" + " for piece hashes to uniquely indentify a file without\n" + " overlap\n" , stderr); } @@ -209,6 +219,8 @@ int main(int argc, char* argv[]) #endif std::vector web_seeds; std::vector trackers; + std::vector collections; + std::vector similar; int pad_file_limit = -1; int piece_size = 0; int flags = 0; @@ -240,6 +252,10 @@ int main(int argc, char* argv[]) ++i; trackers.push_back(argv[i]); break; + case 'M': + flags |= create_torrent::mutable_torrent_support; + pad_file_limit = 0x4000; + break; case 'p': ++i; pad_file_limit = atoi(argv[i]); @@ -273,6 +289,30 @@ int main(int argc, char* argv[]) ++i; root_cert = argv[i]; break; + case 'S': + { + ++i; + if (strlen(argv[i]) != 40) + { + fprintf(stderr, "invalid info-hash for -S. " + "Expected 40 hex characters\n"); + print_usage(); + return 1; + } + sha1_hash ih; + if (!from_hex(argv[i], 40, (char*)&ih[0])) + { + fprintf(stderr, "invalid info-hash for -S\n"); + print_usage(); + return 1; + } + similar.push_back(ih); + } + break; + case 'L': + ++i; + collections.push_back(argv[i]); + break; default: print_usage(); return 1; @@ -314,6 +354,14 @@ int main(int argc, char* argv[]) , end(web_seeds.end()); i != end; ++i) t.add_url_seed(*i); + for (std::vector::iterator i = collections.begin() + , end(collections.end()); i != end; ++i) + t.add_collection(*i); + + for (std::vector::iterator i = similar.begin() + , end(similar.end()); i != end; ++i) + t.add_similar_torrent(*i); + error_code ec; set_piece_hashes(t, branch_path(full_path) , boost::bind(&print_progress, _1, t.num_pieces()), ec); diff --git a/include/libtorrent/Makefile.am b/include/libtorrent/Makefile.am index 15a1cb307..4b4ba7610 100644 --- a/include/libtorrent/Makefile.am +++ b/include/libtorrent/Makefile.am @@ -101,6 +101,7 @@ nobase_include_HEADERS = \ proxy_base.hpp \ puff.hpp \ random.hpp \ + resolve_links.hpp \ resolver.hpp \ resolver_interface.hpp \ rss.hpp \ diff --git a/include/libtorrent/aux_/session_impl.hpp b/include/libtorrent/aux_/session_impl.hpp index 3eb3cc3a3..86b75f6f2 100644 --- a/include/libtorrent/aux_/session_impl.hpp +++ b/include/libtorrent/aux_/session_impl.hpp @@ -268,6 +268,10 @@ namespace libtorrent boost::weak_ptr find_torrent(sha1_hash const& info_hash) const; boost::weak_ptr find_torrent(std::string const& uuid) const; +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + std::vector > find_collection( + std::string const& collection) const; +#endif boost::weak_ptr find_disconnect_candidate_torrent() const; int num_torrents() const { return m_torrents.size(); } diff --git a/include/libtorrent/aux_/session_interface.hpp b/include/libtorrent/aux_/session_interface.hpp index 5da3edeab..a590dc197 100644 --- a/include/libtorrent/aux_/session_interface.hpp +++ b/include/libtorrent/aux_/session_interface.hpp @@ -226,6 +226,11 @@ namespace libtorrent { namespace aux virtual bool verify_bound_address(address const& addr, bool utp , error_code& ec) = 0; +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + virtual std::vector > find_collection( + std::string const& collection) const = 0; +#endif + // TODO: it would be nice to not have this be part of session_interface virtual proxy_settings proxy() const = 0; diff --git a/include/libtorrent/create_torrent.hpp b/include/libtorrent/create_torrent.hpp index faa0d4a80..2b2d9cd47 100644 --- a/include/libtorrent/create_torrent.hpp +++ b/include/libtorrent/create_torrent.hpp @@ -114,8 +114,14 @@ namespace libtorrent enum flags_t { // This will insert pad files to align the files to piece boundaries, for - // optimized disk-I/O. - optimize = 1 + // optimized disk-I/O. This will minimize the number of bytes of pad- + // files, to keep the impact down for clients that don't support + // them. + optimize_alignment = 1, +#ifndef TORRENT_NO_DEPRECATED + // same as optimize_alignment, for backwards compatibility + optimize = 1, +#endif // This will create a merkle hash tree torrent. A merkle torrent cannot // be opened in clients that don't specifically support merkle torrents. @@ -125,7 +131,7 @@ namespace libtorrent // When creating merkle torrents, the full hash tree is also generated // and should be saved off separately. It is accessed through the // create_torrent::merkle_tree() function. - , merkle = 2 + merkle = 2, // This will include the file modification time as part of the torrent. // This is not enabled by default, as it might cause problems when you @@ -133,13 +139,20 @@ namespace libtorrent // yield the same info-hash. If the files have different modification times, // with this option enabled, you would get different info-hashes for the // files. - , modification_time = 4 + modification_time = 4, // If this flag is set, files that are symlinks get a symlink attribute // set on them and their data will not be included in the torrent. This // is useful if you need to reconstruct a file hierarchy which contains // symlinks. - , symlinks = 8 + symlinks = 8, + + // to create a torrent that can be updated via a *mutable torrent* + // (see BEP38_). This also needs to be enabled for torrents that update + // another torrent. + // + // .. _BEP38: http://www.bittorrent.org/beps/bep_0038.html + mutable_torrent_support = 16, }; // The ``piece_size`` is the size of each piece in bytes. It must @@ -279,6 +292,17 @@ namespace libtorrent // tree will be saved in the resume data. std::vector const& merkle_tree() const { return m_merkle_tree; } + // Add similar torrents (by info-hash) or collections of similar torrents. + // Similar torrents are expected to share some files with this torrent. + // Torrents sharing a collection name with this torrent are also expected + // to share files with this torrent. A torrent may have more than one + // collection and more than one similar torrents. For more information, + // see `BEP 38`_. + // + // .. _`BEP 38`: http://www.bittorrent.org/beps/bep_0038.html + void add_similar_torrent(sha1_hash ih); + void add_collection(std::string c); + private: file_storage& m_files; @@ -298,6 +322,9 @@ namespace libtorrent std::vector m_filehashes; + std::vector m_similar; + std::vector m_collections; + // if we're generating a merkle torrent, this is the // merkle tree we got. This should be saved in fast-resume // in order to start seeding the torrent diff --git a/include/libtorrent/disk_interface.hpp b/include/libtorrent/disk_interface.hpp index 26fdb302f..74b73d224 100644 --- a/include/libtorrent/disk_interface.hpp +++ b/include/libtorrent/disk_interface.hpp @@ -39,6 +39,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/bdecode.hpp" #include +#include namespace libtorrent { @@ -67,6 +68,7 @@ namespace libtorrent = boost::function()) = 0; virtual void async_check_fastresume(piece_manager* storage , bdecode_node const* resume_data + , std::auto_ptr >& links , boost::function const& handler) = 0; #ifndef TORRENT_NO_DEPRECATE virtual void async_finalize_file(piece_manager*, int file diff --git a/include/libtorrent/disk_io_job.hpp b/include/libtorrent/disk_io_job.hpp index b0d1db037..15675a425 100644 --- a/include/libtorrent/disk_io_job.hpp +++ b/include/libtorrent/disk_io_job.hpp @@ -96,6 +96,7 @@ namespace libtorrent , load_torrent , clear_piece , tick_storage + , resolve_links , num_job_ids }; @@ -164,6 +165,12 @@ namespace libtorrent // result for hash jobs char piece_hash[20]; + // this is used for check_fastresume to pass in a vector of hard-links + // to create. Each element corresponds to a file in the file_storage. + // The string is the absolute path of the identical file to create + // the hard link to. + std::vector* links; + struct io_args { // if this is set, the read operation is required to diff --git a/include/libtorrent/disk_io_thread.hpp b/include/libtorrent/disk_io_thread.hpp index f0000ad25..ed6832718 100644 --- a/include/libtorrent/disk_io_thread.hpp +++ b/include/libtorrent/disk_io_thread.hpp @@ -308,6 +308,7 @@ namespace libtorrent , boost::function const& handler); void async_check_fastresume(piece_manager* storage , bdecode_node const* resume_data + , std::auto_ptr >& links , boost::function const& handler); void async_save_resume_data(piece_manager* storage , boost::function const& handler); @@ -420,6 +421,7 @@ namespace libtorrent int do_load_torrent(disk_io_job* j, tailqueue& completed_jobs); int do_clear_piece(disk_io_job* j, tailqueue& completed_jobs); int do_tick(disk_io_job* j, tailqueue& completed_jobs); + int do_resolve_links(disk_io_job* j, tailqueue& completed_jobs); void call_job_handlers(void* userdata); diff --git a/include/libtorrent/error_code.hpp b/include/libtorrent/error_code.hpp index 3d35f3fc0..12be2f4fa 100644 --- a/include/libtorrent/error_code.hpp +++ b/include/libtorrent/error_code.hpp @@ -578,6 +578,8 @@ namespace libtorrent partfile_move, partfile_read, partfile_write, + check_resume, + hard_link, }; // Returns a string literal representing the file operation @@ -590,6 +592,7 @@ namespace libtorrent "", "stat", "mkdir", "open", "rename", "remove", "copy" , "read", "write", "fallocate", "allocate cache piece" , "partfile move", "partfile read", "partfile write" + , "check resume", "hard_link" }; return ops[operation]; } diff --git a/include/libtorrent/file.hpp b/include/libtorrent/file.hpp index c19da5058..c79340bf6 100644 --- a/include/libtorrent/file.hpp +++ b/include/libtorrent/file.hpp @@ -140,6 +140,11 @@ namespace libtorrent TORRENT_EXTRA_EXPORT void copy_file(std::string const& f , std::string const& newf, error_code& ec); + // file is expected to exist, link will be created to point to it. If hard + // links are not supported by the filesystem or OS, the file will be copied. + TORRENT_EXTRA_EXPORT void hard_link(std::string const& file + , std::string const& link, error_code& ec); + TORRENT_EXTRA_EXPORT std::string split_path(std::string const& f); TORRENT_EXTRA_EXPORT char const* next_path_element(char const* p); TORRENT_EXTRA_EXPORT std::string extension(std::string const& f); diff --git a/include/libtorrent/file_storage.hpp b/include/libtorrent/file_storage.hpp index 161a33822..3c11b1428 100644 --- a/include/libtorrent/file_storage.hpp +++ b/include/libtorrent/file_storage.hpp @@ -425,7 +425,11 @@ namespace libtorrent // (-1) but it may also make sense to set it to 16 kiB, or something // divisible by 16 kiB. // If pad_file_limit is 0, every file will be padded (except empty ones). - void optimize(int pad_file_limit = -1, int alignment = -1); + // ``tail_padding`` indicates whether aligned files also are padded at + // the end to make them end aligned. This is required for mutable + // torrents, since piece hashes are compared + void optimize(int pad_file_limit = -1, int alignment = -1 + , bool tail_padding = false); // These functions are used to query attributes of files at // a given index. @@ -545,6 +549,11 @@ namespace libtorrent private: + void add_pad_file(int size + , std::vector::iterator& i + , boost::int64_t& offset + , int& pad_file_counter); + // the number of bytes in a regular piece // (i.e. not the potentially truncated last piece) int m_piece_length; diff --git a/include/libtorrent/resolve_links.hpp b/include/libtorrent/resolve_links.hpp new file mode 100644 index 000000000..14e2680a1 --- /dev/null +++ b/include/libtorrent/resolve_links.hpp @@ -0,0 +1,83 @@ +/* + +Copyright (c) 2014, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef TORRENT_RESOLVE_LINKS_HPP +#define TORRENT_RESOLVE_LINKS_HPP + +#include +#include +#include +#include + +namespace libtorrent +{ + class torrent_info; + +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + // this class is used for mutable torrents, to discover identical files + // in other torrents. + struct resolve_links + { + struct link_t + { + boost::shared_ptr ti; + std::string save_path; + int file_idx; + }; + + resolve_links(boost::shared_ptr ti); + + // check to see if any files are shared with this torrent + void match(boost::shared_ptr const& ti + , std::string const& save_path); + + std::vector const& get_links() const + { return m_links; } + + private: + // this is the torrent we're trying to find files for. + boost::shared_ptr m_torrent_file; + + // each file in m_torrent_file has an entry in this vector. Any file + // that also exists somewhere else, is filled in with the corresponding + // torrent_info object and file index + std::vector m_links; + + // maps file size to file index, in m_torrent_file + boost::unordered_multimap m_file_sizes; + }; +#endif // TORRENT_DISABLE_MUTABLE_TORRENTS + +} + +#endif + diff --git a/include/libtorrent/storage.hpp b/include/libtorrent/storage.hpp index 75840b288..138fd4653 100644 --- a/include/libtorrent/storage.hpp +++ b/include/libtorrent/storage.hpp @@ -115,7 +115,9 @@ POSSIBILITY OF SUCH DAMAGE. // virtual bool rename_file(int file, std::string const& new_name) // { assert(false); return false; } // virtual bool move_storage(std::string const& save_path) { return false; } -// virtual bool verify_resume_data(bdecode_node const& rd, storage_error& error) { return false; } +// virtual bool verify_resume_data(bdecode_node const& rd +// , std::vector const* links +// , storage_error& error) { return false; } // virtual bool write_resume_data(entry& rd) const { return false; } // virtual boost::int64_t physical_offset(int slot, int offset) // { return slot * m_files.piece_length() + offset; }; @@ -307,7 +309,15 @@ namespace libtorrent // This function should verify the resume data ``rd`` with the files // on disk. If the resume data seems to be up-to-date, return true. If // not, set ``error`` to a description of what mismatched and return false. - virtual bool verify_resume_data(bdecode_node const& rd, storage_error& ec) = 0; + // + // If the ``links`` pointer is non-null, it has the same number + // of elements as there are files. Each element is either empty or contains + // the absolute path to a file identical to the corresponding file in this + // torrent. The storage must create hard links (or copy) those files. If + // any file does not exist or is inaccessible, the disk job must fail. + virtual bool verify_resume_data(bdecode_node const& rd + , std::vector const* links + , storage_error& ec) = 0; // This function should fill in resume data, the current state of the // storage, in ``rd``. The default storage adds file timestamps and @@ -423,7 +433,9 @@ namespace libtorrent void initialize(storage_error& ec); int move_storage(std::string const& save_path, int flags, storage_error& ec); int sparse_end(int start) const; - bool verify_resume_data(bdecode_node const& rd, storage_error& error); + bool verify_resume_data(bdecode_node const& rd + , std::vector const* links + , storage_error& error); void write_resume_data(entry& rd, storage_error& ec) const; bool tick(); @@ -519,7 +531,9 @@ namespace libtorrent int writev(file::iovec_t const* bufs, int num_bufs, int piece , int offset, int flags, storage_error& ec); - bool verify_resume_data(bdecode_node const&, storage_error&) { return false; } + bool verify_resume_data(bdecode_node const& + , std::vector const* links + , storage_error&) { return false; } void write_resume_data(entry&, storage_error&) const {} int m_piece_size; @@ -541,7 +555,9 @@ namespace libtorrent , storage_error&) {} virtual int move_storage(std::string const& /* save_path */ , int /* flags */, storage_error&) { return 0; } - virtual bool verify_resume_data(bdecode_node const& /* rd */, storage_error&) + virtual bool verify_resume_data(bdecode_node const& /* rd */ + , std::vector const* /* links */ + , storage_error&) { return false; } virtual void write_resume_data(entry&, storage_error&) const {} virtual void release_files(storage_error&) {} @@ -674,7 +690,9 @@ namespace libtorrent // the error message indicates that the fast resume data was rejected // if 'fatal_disk_error' is returned, the error message indicates what // when wrong in the disk access - int check_fastresume(bdecode_node const& rd, storage_error& error); + int check_fastresume(bdecode_node const& rd + , std::vector const* links + , storage_error& error); // helper functions for check_fastresume int check_no_fastresume(storage_error& error); diff --git a/include/libtorrent/torrent_info.hpp b/include/libtorrent/torrent_info.hpp index 2db9db7e8..ad428dde3 100644 --- a/include/libtorrent/torrent_info.hpp +++ b/include/libtorrent/torrent_info.hpp @@ -413,6 +413,16 @@ namespace libtorrent void add_tracker(std::string const& url, int tier = 0); std::vector const& trackers() const { return m_urls; } + // These two functions are related to BEP38_ (mutable torrents). The + // vectors returned from these correspond to the "similar" and + // "collections" keys in the .torrent file. Both info-hashes and + // collections from within the info-dict and from outside of it are + // included. + // + // .. _BEP38: http://www.bittorrent.org/beps/bep_0038.html + std::vector similar_torrents() const; + std::vector collections() const; + #ifndef TORRENT_NO_DEPRECATE // deprecated in 0.16. Use web_seeds() instead TORRENT_DEPRECATED_PREFIX @@ -713,6 +723,27 @@ namespace libtorrent std::vector m_web_seeds; nodes_t m_nodes; + // the info-hashes (20 bytes each) in the "similar" key. The pointers + // point directly into the info_section. When copied, these pointers must + // be corrected to point into the copied-to buffer + std::vector m_similar_torrents; + + // these are similar torrents from outside of the info-dict. We can't + // have non-owning pointers to those, as we only keep the info-dict + // around. + std::vector m_owned_similar_torrents; + + // these or strings of the "collections" key from the torrent file. The + // pointers point directly into the info_section buffer and when copied, + // these pointers must be corrected to point into the new buffer. The + // int is the length of the string. Strings are not NULL-terminated. + std::vector > m_collections; + + // these are the collections from outside of the info-dict. These are + // owning strings, since we only keep the info-section around, these + // cannot be pointers into that buffer. + std::vector m_owned_collections; + // if this is a merkle torrent, this is the merkle // tree. It has space for merkle_num_nodes(merkle_num_leafs(num_pieces)) // hashes diff --git a/src/Makefile.am b/src/Makefile.am index bb1e65312..027dcecfd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -102,6 +102,7 @@ libtorrent_rasterbar_la_SOURCES = \ random.cpp \ receive_buffer.cpp \ request_blocks.cpp \ + resolve_links.cpp \ resolver.cpp \ rss.cpp \ session.cpp \ diff --git a/src/create_torrent.cpp b/src/create_torrent.cpp index 4aaee9860..49bacd72d 100644 --- a/src/create_torrent.cpp +++ b/src/create_torrent.cpp @@ -326,6 +326,12 @@ namespace libtorrent piece_size = 64*1024; } + // to support mutable torrents, alignment always has to be the piece size, + // because piece hashes are compared to determine whether files are + // identical + if (flags & mutable_torrent_support) + alignment = piece_size; + // make sure the size is an even power of 2 #ifndef NDEBUG for (int i = 0; i < 32; ++i) @@ -338,8 +344,9 @@ namespace libtorrent } #endif m_files.set_piece_length(piece_size); - if (flags & optimize) - m_files.optimize(pad_file_limit, alignment); + if (flags & (optimize_alignment | mutable_torrent_support)) + m_files.optimize(pad_file_limit, alignment, flags & mutable_torrent_support); + m_files.set_num_pieces(static_cast( (m_files.total_size() + m_files.piece_length() - 1) / m_files.piece_length())); m_piece_hash.resize(m_files.num_pieces()); @@ -481,6 +488,26 @@ namespace libtorrent return dict; } + if (!m_collections.empty()) + { + entry& list = info["collections"]; + for (std::vector::const_iterator i + = m_collections.begin(); i != m_collections.end(); ++i) + { + list.list().push_back(entry(*i)); + } + } + + if (!m_similar.empty()) + { + entry& list = info["similar"]; + for (std::vector::const_iterator i + = m_similar.begin(); i != m_similar.end(); ++i) + { + list.list().push_back(entry(i->to_string())); + } + } + info["name"] = m_files.name(); if (!m_root_cert.empty()) @@ -635,6 +662,16 @@ namespace libtorrent m_root_cert = cert; } + void create_torrent::add_similar_torrent(sha1_hash ih) + { + m_similar.push_back(ih); + } + + void create_torrent::add_collection(std::string c) + { + m_collections.push_back(c); + } + void create_torrent::set_hash(int index, sha1_hash const& h) { TORRENT_ASSERT(index >= 0); diff --git a/src/disk_io_thread.cpp b/src/disk_io_thread.cpp index 32e6b5010..8f62fd4aa 100644 --- a/src/disk_io_thread.cpp +++ b/src/disk_io_thread.cpp @@ -1903,6 +1903,7 @@ namespace libtorrent void disk_io_thread::async_check_fastresume(piece_manager* storage , bdecode_node const* resume_data + , std::auto_ptr >& links , boost::function const& handler) { #ifdef TORRENT_DEBUG @@ -1914,9 +1915,11 @@ namespace libtorrent disk_io_job* j = allocate_job(disk_io_job::check_fastresume); j->storage = storage->shared_from_this(); j->buffer = (char*)resume_data; + j->d.links = links.get(); j->callback = handler; add_fence_job(storage, j); + links.release(); } void disk_io_thread::async_save_resume_data(piece_manager* storage @@ -2607,7 +2610,8 @@ namespace libtorrent bdecode_node tmp; if (rd == NULL) rd = &tmp; - return j->storage->check_fastresume(*rd, j->error); + std::auto_ptr > links(j->d.links); + return j->storage->check_fastresume(*rd, links.get(), j->error); } int disk_io_thread::do_save_resume_data(disk_io_job* j, tailqueue& completed_jobs) diff --git a/src/file.cpp b/src/file.cpp index d60701fe5..1f6a362e6 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -440,25 +440,85 @@ namespace libtorrent { ec.clear(); -#if defined TORRENT_WINDOWS && TORRENT_USE_WSTRING +#ifdef TORRENT_WINDOWS +#if TORRENT_USE_WSTRING #define CreateDirectory_ CreateDirectoryW std::wstring n = convert_to_wstring(f); #else #define CreateDirectory_ CreateDirectoryA std::string const& n = convert_to_native(f); -#endif +#endif // TORRENT_USE_WSTRING -#ifdef TORRENT_WINDOWS if (CreateDirectory_(n.c_str(), 0) == 0 && GetLastError() != ERROR_ALREADY_EXISTS) ec.assign(GetLastError(), boost::system::system_category()); #else + std::string n = convert_to_native(f); int ret = mkdir(n.c_str(), 0777); if (ret < 0 && errno != EEXIST) ec.assign(errno, generic_category()); #endif } + void hard_link(std::string const& file, std::string const& link + , error_code& ec) + { +#ifdef TORRENT_WINDOWS + +#if TORRENT_USE_WSTRING +#define CreateHardLink_ CreateHardLinkW + std::wstring n_exist = convert_to_wstring(file); + std::wstring n_link = convert_to_wstring(link); +#else +#define CreateHardLink_ CreateHardLinkA + std::string n_exist = convert_to_native(file); + std::string n_link = convert_to_native(link); +#endif + BOOL ret = CreateHardLink(n_link.c_str(), n_exist.c_str(), NULL); + if (ret) + { + ec.clear(); + return; + } + + // something failed. Does the filesystem not support hard links? + // TODO: 3 find out what error code is reported when the filesystem + // does not support hard links. + + // it's possible CreateHardLink will copy the file internally too, + // if the filesystem does not support it. + ec.assign(GetLastError(), system_category()); + return; + +#else + + std::string n_exist = convert_to_native(file); + std::string n_link = convert_to_native(link); + + // assume posix's link() function exists + int ret = ::link(n_exist.c_str(), n_link.c_str()); + + if (ret == 0) + { + ec.clear(); + return; + } + + // most errors are passed through, except for the ones that indicate that + // hard links are not supported and require a copy. + // TODO: 2 test this on a FAT volume to see what error we get! + if (errno != EMLINK || errno != EXDEV) + { + // some error happened, report up to the caller + ec.assign(errno, generic_category()); + return; + } +#endif + + // if we get here, we should copy the file + copy_file(file, link, ec); + } + bool is_directory(std::string const& f, error_code& ec) { ec.clear(); @@ -494,7 +554,8 @@ namespace libtorrent void copy_file(std::string const& inf, std::string const& newf, error_code& ec) { ec.clear(); -#if TORRENT_USE_WSTRING && defined TORRENT_WINDOWS +#ifdef TORRENT_WINDOWS +#if TORRENT_USE_WSTRING #define CopyFile_ CopyFileW std::wstring f1 = convert_to_wstring(inf); std::wstring f2 = convert_to_wstring(newf); @@ -504,17 +565,22 @@ namespace libtorrent std::string const& f2 = convert_to_native(newf); #endif -#ifdef TORRENT_WINDOWS if (CopyFile_(f1.c_str(), f2.c_str(), false) == 0) ec.assign(GetLastError(), boost::system::system_category()); #elif defined __APPLE__ && defined __MACH__ && MAC_OS_X_VERSION_MIN_REQUIRED >= 1050 + std::string f1 = convert_to_native(inf); + std::string f2 = convert_to_native(newf); + // this only works on 10.5 copyfile_state_t state = copyfile_state_alloc(); if (copyfile(f1.c_str(), f2.c_str(), state, COPYFILE_ALL) < 0) ec.assign(errno, generic_category()); copyfile_state_free(state); #else - int infd = ::open(inf.c_str(), O_RDONLY); + std::string f1 = convert_to_native(inf); + std::string f2 = convert_to_native(newf); + + int infd = ::open(f1.c_str(), O_RDONLY); if (infd < 0) { ec.assign(errno, generic_category()); @@ -527,7 +593,7 @@ namespace libtorrent | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; - int outfd = ::open(newf.c_str(), O_WRONLY | O_CREAT, permissions); + int outfd = ::open(f2.c_str(), O_WRONLY | O_CREAT, permissions); if (outfd < 0) { close(infd); diff --git a/src/file_storage.cpp b/src/file_storage.cpp index 71b94df90..356ba6ca3 100644 --- a/src/file_storage.cpp +++ b/src/file_storage.cpp @@ -861,7 +861,8 @@ namespace libtorrent #endif // TORRENT_DEPRECATED } - void file_storage::optimize(int pad_file_limit, int alignment) + void file_storage::optimize(int pad_file_limit, int alignment + , bool tail_padding) { if (alignment == -1) alignment = m_piece_length; @@ -938,40 +939,62 @@ namespace libtorrent // then swap it in place. This minimizes the amount // of copying of internal_file_entry, which is somewhat // expensive (until we have move semantics) - int cur_index = i - m_files.begin(); - int index = m_files.size(); - m_files.push_back(internal_file_entry()); - ++m_num_files; - internal_file_entry& e = m_files.back(); - // i may have been invalidated, refresh it - i = m_files.begin() + cur_index; - e.size = pad_size; - e.offset = off; - char name[30]; - snprintf(name, sizeof(name), ".____padding_file/%d", padding_file); - std::string path = combine_path(m_name, name); - e.set_name(path.c_str()); - e.pad_file = true; - off += pad_size; - ++padding_file; - - if (!m_mtime.empty()) m_mtime.resize(index + 1, 0); - if (!m_file_hashes.empty()) m_file_hashes.resize(index + 1, NULL); -#ifndef TORRENT_NO_DEPRECATE - if (!m_file_base.empty()) m_file_base.resize(index + 1, 0); -#endif - - reorder_file(index, cur_index); + add_pad_file(pad_size, i, off, padding_file); TORRENT_ASSERT((off % alignment) == 0); continue; } i->offset = off; off += i->size; + + if (tail_padding + && i->size > boost::uint32_t(pad_file_limit) + && (off % alignment) != 0) + { + // skip the file we just put in place, so we put the pad + // file after it + ++i; + + // tail-padding is enabled, and the offset after this file is not + // aligned and it's not the last file. The last file must be padded + // too, in order to match an equivalent tail-padded file. + add_pad_file(alignment - (off % alignment), i, off, padding_file); + + TORRENT_ASSERT((off % alignment) == 0); + } } m_total_size = off; } + void file_storage::add_pad_file(int size + , std::vector::iterator& i + , boost::int64_t& offset + , int& pad_file_counter) + { + int cur_index = i - m_files.begin(); + int index = m_files.size(); + m_files.push_back(internal_file_entry()); + ++m_num_files; + internal_file_entry& e = m_files.back(); + // i may have been invalidated, refresh it + i = m_files.begin() + cur_index; + e.size = size; + e.offset = offset; + char name[30]; + snprintf(name, sizeof(name), ".____padding_file/%d", pad_file_counter); + std::string path = combine_path(m_name, name); + e.set_name(path.c_str()); + e.pad_file = true; + offset += size; + ++pad_file_counter; + + if (!m_mtime.empty()) m_mtime.resize(index + 1, 0); + if (!m_file_hashes.empty()) m_file_hashes.resize(index + 1, NULL); + if (!m_file_base.empty()) m_file_base.resize(index + 1, 0); + + reorder_file(index, cur_index); + } + void file_storage::unload() { std::vector().swap(m_files); diff --git a/src/resolve_links.cpp b/src/resolve_links.cpp new file mode 100644 index 000000000..081b84d4b --- /dev/null +++ b/src/resolve_links.cpp @@ -0,0 +1,138 @@ +/* + +Copyright (c) 2014, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "resolve_links.hpp" + +#include "libtorrent/torrent_info.hpp" +#include + +namespace libtorrent +{ + +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS +resolve_links::resolve_links(boost::shared_ptr ti) + : m_torrent_file(ti) +{ + TORRENT_ASSERT(ti); + + int piece_size = ti->piece_length(); + + file_storage const& fs = ti->files(); + m_file_sizes.reserve(fs.num_files()); + for (int i = 0; i < fs.num_files(); ++i) + { + // don't match pad-files, and don't match files that aren't aligned to + // ieces. Files are matched by comparing piece hashes, so pieces must + // be aligned and the same size + if (fs.pad_file_at(i)) continue; + if ((fs.file_offset(i) % piece_size) != 0) continue; + + m_file_sizes.insert(std::make_pair(fs.file_size(i), i)); + } + + m_links.resize(m_torrent_file->num_files()); +} + +void resolve_links::match(boost::shared_ptr const& ti + , std::string const& save_path) +{ + if (!ti) return; + + // only torrents with the same + if (ti->piece_length() != m_torrent_file->piece_length()) return; + + int piece_size = ti->piece_length(); + + file_storage const& fs = ti->files(); + m_file_sizes.reserve(fs.num_files()); + for (int i = 0; i < fs.num_files(); ++i) + { + // for every file in the other torrent, see if we have one that match + // it in m_torrent_file + + // if the file base is not aligned to pieces, we're not going to match + // it anyway (we only compare piece hashes) + if ((fs.file_offset(i) % piece_size) != 0) continue; + if (fs.pad_file_at(i)) continue; + + boost::int64_t file_size = fs.file_size(i); + + typedef boost::unordered_multimap::iterator iterator; + iterator iter = m_file_sizes.find(file_size); + + // we don't have a file whose size matches, look at the next one + if (iter == m_file_sizes.end()) continue; + + TORRENT_ASSERT(iter->second < m_torrent_file->files().num_files()); + TORRENT_ASSERT(iter->second >= 0); + + // if we already have found a duplicate for this file, no need + // to keep looking + if (m_links[iter->second].ti) continue; + + // files are aligned and have the same size, now start comparing + // piece hashes, to see if the files are identical + + // the pieces of the incoming file + int their_piece = fs.map_file(i, 0, 0).piece; + // the pieces of "this" file (from m_torrent_file) + int our_piece = m_torrent_file->files().map_file( + iter->second, 0, 0).piece; + + int num_pieces = (file_size + piece_size - 1) / piece_size; + + bool match = true; + for (int p = 0; p < num_pieces; ++p, ++their_piece, ++our_piece) + { + if (m_torrent_file->hash_for_piece(our_piece) + != ti->hash_for_piece(their_piece)) + { + match = false; + break; + } + } + if (!match) continue; + + m_links[iter->second].ti = ti; + m_links[iter->second].save_path = save_path; + m_links[iter->second].file_idx = i; + + // since we have a duplicate for this file, we may as well remove + // it from the file-size map, so we won't find it again. + m_file_sizes.erase(iter); + } + +} +#endif // TORRENT_DISABLE_MUTABLE_TORRENTS + +} // namespace libtorrent + diff --git a/src/session_impl.cpp b/src/session_impl.cpp index c99aceead..2f3da3aae 100644 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -4222,6 +4222,24 @@ retry: return boost::weak_ptr(); } +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + std::vector > session_impl::find_collection( + std::string const& collection) const + { + std::vector > ret; + for (session_impl::torrent_map::const_iterator i = m_torrents.begin() + , end(m_torrents.end()); i != end; ++i) + { + boost::shared_ptr t = i->second; + if (!t) continue; + std::vector const& c = t->torrent_file().collections(); + if (std::count(c.begin(), c.end(), collection) == 0) continue; + ret.push_back(t); + } + return ret; + } +#endif //TORRENT_DISABLE_MUTABLE_TORRENTS + // returns true if lhs is a better disconnect candidate than rhs bool compare_disconnect_torrent(session_impl::torrent_map::value_type const& lhs , session_impl::torrent_map::value_type const& rhs) diff --git a/src/storage.cpp b/src/storage.cpp index 2e3c7791a..8ae171d1b 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -712,7 +712,9 @@ namespace libtorrent return int((data_start + files().piece_length() - 1) / files().piece_length()); } - bool default_storage::verify_resume_data(bdecode_node const& rd, storage_error& ec) + bool default_storage::verify_resume_data(bdecode_node const& rd + , std::vector const* links + , storage_error& ec) { // TODO: make this more generic to not just work if files have been // renamed, but also if they have been merged into a single file for instance @@ -742,6 +744,8 @@ namespace libtorrent if (file_sizes_ent == 0) { ec.ec = errors::missing_file_sizes; + ec.file = -1; + ec.operation = storage_error::check_resume; return false; } @@ -756,7 +760,7 @@ namespace libtorrent { ec.ec = errors::mismatching_number_of_files; ec.file = -1; - ec.operation = storage_error::none; + ec.operation = storage_error::check_resume; return false; } @@ -792,6 +796,8 @@ namespace libtorrent else { ec.ec = errors::missing_pieces; + ec.file = -1; + ec.operation = storage_error::check_resume; return false; } @@ -806,7 +812,7 @@ namespace libtorrent { ec.ec = errors::missing_file_sizes; ec.file = i; - ec.operation = storage_error::none; + ec.operation = storage_error::check_resume; return false; } @@ -819,7 +825,7 @@ namespace libtorrent { ec.ec = errors::mismatching_file_size; ec.file = i; - ec.operation = storage_error::none; + ec.operation = storage_error::check_resume; return false; } @@ -883,6 +889,41 @@ namespace libtorrent if (rd.dict_find_string_value("allocation") != "compact") full_allocation_mode = true; +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + if (links) + { + // if this is a mutable torrent, and we need to pick up some files + // from other torrents, do that now. Note that there is an inherent + // race condition here. We checked if the files existed on a different + // thread a while ago. These files may no longer exist or may have been + // moved. If so, we just fail. The user is responsible to not touch + // other torrents until a new mutable torrent has been completely + // added. + int idx = 0; + for (std::vector::const_iterator i = links->begin(); + i != links->end(); ++i, ++idx) + { + if (i->empty()) continue; + + error_code err; + std::string file_path = fs.file_path(idx, m_save_path); + hard_link(*i, file_path, err); + + // if the file already exists, that's not an error + // TODO: 2 is this risky? The upper layer will assume we have the + // whole file. Perhaps we should verify that at least the size + // of the file is correct + if (!err || err == boost::system::errc::file_exists) + continue; + + ec.ec = err; + ec.file = idx; + ec.operation = storage_error::hard_link; + return false; + } + } +#endif // TORRENT_DISABLE_MUTABLE_TORRENTS + return true; } @@ -1470,9 +1511,15 @@ namespace libtorrent // check if the fastresume data is up to date // if it is, use it and return true. If it // isn't return false and the full check - // will be run + // will be run. If the links pointer is non-null, it has the same number + // of elements as there are files. Each element is either empty or contains + // the absolute path to a file identical to the corresponding file in this + // torrent. The storage must create hard links (or copy) those files. If + // any file does not exist or is inaccessible, the disk job must fail. int piece_manager::check_fastresume( - bdecode_node const& rd, storage_error& ec) + bdecode_node const& rd + , std::vector const* links + , storage_error& ec) { TORRENT_ASSERT(m_files.piece_length() > 0); @@ -1494,7 +1541,7 @@ namespace libtorrent return check_no_fastresume(ec); } - if (!m_storage->verify_resume_data(rd, ec)) + if (!m_storage->verify_resume_data(rd, links, ec)) return check_no_fastresume(ec); return check_init_storage(ec); diff --git a/src/torrent.cpp b/src/torrent.cpp index 30f575e6f..531d5fa49 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -86,6 +86,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/alert_manager.hpp" // for alert_manageralert_manager #include "libtorrent/resolver_interface.hpp" #include "libtorrent/alloca.hpp" +#include "libtorrent/resolve_links.hpp" #ifdef TORRENT_USE_OPENSSL #include "libtorrent/ssl_stream.hpp" @@ -1855,8 +1856,6 @@ namespace libtorrent return; } - set_state(torrent_status::checking_resume_data); - int num_pad_files = 0; TORRENT_ASSERT(block_size() > 0); file_storage const& fs = m_torrent_file->files(); @@ -1927,10 +1926,68 @@ namespace libtorrent if (num_pad_files > 0) m_picker->set_num_pad_files(num_pad_files); + std::auto_ptr > links; +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + if (!m_torrent_file->similar_torrents().empty() + || !m_torrent_file->collections().empty()) + { + resolve_links res(m_torrent_file); + + std::vector s = m_torrent_file->similar_torrents(); + for (std::vector::iterator i = s.begin(), end(s.end()); + i != end; ++i) + { + boost::shared_ptr t = m_ses.find_torrent(*i).lock(); + if (!t) continue; + + // Only attempt to reuse files from torrents that are seeding. + // TODO: this could be optimized by looking up which files are + // complete and just look at those + if (!t->is_seed()) continue; + + res.match(t->get_torrent_copy(), t->save_path()); + } + std::vector c = m_torrent_file->collections(); + for (std::vector::iterator i = c.begin(), end(c.end()); + i != end; ++i) + { + std::vector > ts = m_ses.find_collection(*i); + + for (std::vector >::iterator k = ts.begin() + , end(ts.end()); k != end; ++k) + { + // Only attempt to reuse files from torrents that are seeding. + // TODO: this could be optimized by looking up which files are + // complete and just look at those + if (!(*k)->is_seed()) continue; + + res.match((*k)->get_torrent_copy(), (*k)->save_path()); + } + } + + std::vector const& l = res.get_links(); + if (!l.empty()) + { + links.reset(new std::vector(l.size())); + for (std::vector::const_iterator i = l.begin() + , end(l.end()); i != end; ++i) + { + if (!i->ti) continue; + + torrent_info const& ti = *i->ti; + std::string const& save_path = i->save_path; + links->push_back(combine_path(save_path + , ti.files().file_path(i->file_idx))); + } + } + } +#endif // TORRENT_DISABLE_MUTABLE_TORRENTS + inc_refcount("check_fastresume"); + // async_check_fastresume will release links m_ses.disk_thread().async_check_fastresume( m_storage.get(), m_resume_data ? &m_resume_data->node : NULL - , boost::bind(&torrent::on_resume_data_checked + , links, boost::bind(&torrent::on_resume_data_checked , shared_from_this(), _1)); #if defined TORRENT_LOGGING debug_log("init, async_check_fastresume"); @@ -2443,9 +2500,10 @@ namespace libtorrent m_resume_data.reset(); + std::auto_ptr > links; inc_refcount("force_recheck"); m_ses.disk_thread().async_check_fastresume(m_storage.get(), NULL - , boost::bind(&torrent::on_force_recheck + , links, boost::bind(&torrent::on_force_recheck , shared_from_this(), _1)); } diff --git a/src/torrent_info.cpp b/src/torrent_info.cpp index b8a2a5d05..cb183f145 100644 --- a/src/torrent_info.cpp +++ b/src/torrent_info.cpp @@ -752,6 +752,14 @@ namespace libtorrent if (m_orig_files) const_cast(*m_orig_files).apply_pointer_offset(offset); +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + for (int i = 0; i < m_collections.size(); ++i) + m_collections[i].first += offset; + + for (int i = 0; i < m_similar_torrents.size(); ++i) + m_similar_torrents[i] += offset; +#endif + if (m_info_dict) { // make this decoded object point to our copy of the info section @@ -1273,7 +1281,6 @@ namespace libtorrent std::string name; sanitize_append_path_element(name, name_ent.string_ptr() , name_ent.string_length()); - if (name.empty()) name = to_hex(m_info_hash.to_string()); // extract file list @@ -1340,6 +1347,38 @@ namespace libtorrent m_private = info.dict_find_int_value("private", 0); +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + bdecode_node similar = info.dict_find_list("similar"); + if (similar) + { + for (int i = 0; i < similar.list_size(); ++i) + { + if (similar.list_at(i).type() != bdecode_node::string_t) + continue; + + if (similar.list_at(i).string_length() != 20) + continue; + + m_similar_torrents.push_back(similar.list_at(i).string_ptr() + + info_ptr_diff); + } + } + + bdecode_node collections = info.dict_find_list("collections"); + if (collections) + { + for (int i = 0; i < collections.list_size(); ++i) + { + bdecode_node str = collections.list_at(i); + + if (str.type() != bdecode_node::string_t) continue; + + m_collections.push_back(std::make_pair(str.string_ptr() + + info_ptr_diff, str.string_length())); + } + } +#endif // TORRENT_DISABLE_MUTABLE_TORRENTS + // now, commit the files structure we just parsed out // into the torrent_info object. // if we already have an m_files that's populated, it @@ -1480,6 +1519,38 @@ namespace libtorrent if (!parse_info_section(info, ec, flags)) return false; resolve_duplicate_filenames(); +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + bdecode_node similar = torrent_file.dict_find_list("similar"); + if (similar) + { + for (int i = 0; i < similar.list_size(); ++i) + { + if (similar.list_at(i).type() != bdecode_node::string_t) + continue; + + if (similar.list_at(i).string_length() != 20) + continue; + + m_owned_similar_torrents.push_back( + sha1_hash(similar.list_at(i).string_ptr())); + } + } + + bdecode_node collections = torrent_file.dict_find_list("collections"); + if (collections) + { + for (int i = 0; i < collections.list_size(); ++i) + { + bdecode_node str = collections.list_at(i); + + if (str.type() != bdecode_node::string_t) continue; + + m_owned_collections.push_back(std::string(str.string_ptr() + , str.string_length())); + } + } +#endif // TORRENT_DISABLE_MUTABLE_TORRENTS + // extract the url of the tracker bdecode_node i = torrent_file.dict_find_list("announce-list"); if (i) @@ -1683,21 +1754,52 @@ namespace libtorrent #endif // TORRENT_NO_DEPRECATE void torrent_info::add_url_seed(std::string const& url - , std::string const& ext_auth - , web_seed_entry::headers_t const& ext_headers) + , std::string const& ext_auth + , web_seed_entry::headers_t const& ext_headers) { m_web_seeds.push_back(web_seed_entry(url, web_seed_entry::url_seed , ext_auth, ext_headers)); } void torrent_info::add_http_seed(std::string const& url - , std::string const& auth - , web_seed_entry::headers_t const& extra_headers) + , std::string const& auth + , web_seed_entry::headers_t const& extra_headers) { m_web_seeds.push_back(web_seed_entry(url, web_seed_entry::http_seed , auth, extra_headers)); } + std::vector torrent_info::similar_torrents() const + { + std::vector ret; +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + ret.reserve(m_similar_torrents.size() + m_owned_similar_torrents.size()); + + for (int i = 0; i < m_similar_torrents.size(); ++i) + ret.push_back(sha1_hash(m_similar_torrents[i])); + + for (int i = 0; i < m_owned_similar_torrents.size(); ++i) + ret.push_back(m_owned_similar_torrents[i]); +#endif + + return ret; + } + + std::vector torrent_info::collections() const + { + std::vector ret; +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + ret.reserve(m_collections.size() + m_owned_collections.size()); + + for (int i = 0; i < m_collections.size(); ++i) + ret.push_back(std::string(m_collections[i].first, m_collections[i].second)); + + for (int i = 0; i < m_owned_collections.size(); ++i) + ret.push_back(m_owned_collections[i]); +#endif // TORRENT_DISABLE_MUTABLE_TORRENTS + + return ret; + } #if !defined TORRENT_NO_DEPRECATE && TORRENT_USE_IOSTREAM // ------- start deprecation ------- diff --git a/test/Jamfile b/test/Jamfile index 62458f211..184dda665 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -95,6 +95,7 @@ feature launcher : none valgrind : composite ; feature.compose valgrind : "valgrind --tool=memcheck -v --num-callers=20 --read-var-info=yes --track-origins=yes --error-exitcode=222 --suppressions=valgrind_suppressions.txt" on ; test-suite libtorrent : + [ run test_resolve_links.cpp ] [ run test_crc32.cpp ] [ run test_resume.cpp ] [ run test_sliding_average.cpp ] diff --git a/test/Makefile.am b/test/Makefile.am index bff761098..9d3e60620 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -113,6 +113,14 @@ EXTRA_DIST = Jamfile \ test_torrents/empty_path_multi.torrent \ test_torrents/duplicate_web_seeds.torrent \ test_torrents/sample.torrent \ + mutable_test_torrents/test1.torrent \ + mutable_test_torrents/test1_pad_files.torrent \ + mutable_test_torrents/test1_single.torrent\ + mutable_test_torrents/test1_single_padded.torrent \ + mutable_test_torrents/test2.torrent \ + mutable_test_torrents/test2_pad_files.torrent \ + mutable_test_torrents/test3.torrent \ + mutable_test_torrents/test3_pad_files.torrent \ eztv.xml \ kat.xml \ cb.xml \ diff --git a/test/mutable_test_torrents/test1.torrent b/test/mutable_test_torrents/test1.torrent new file mode 100644 index 000000000..d73236c21 --- /dev/null +++ b/test/mutable_test_torrents/test1.torrent @@ -0,0 +1,3 @@ +d10:created by10:libtorrent13:creation datei1419452992e4:infod5:filesld6:lengthi51200e4:pathl1:aeed6:lengthi18e4:pathl1:beed6:lengthi19e4:pathl1:ceed6:lengthi53248e4:pathl1:deee4:name5:test112:piece lengthi16384e6:pieces140:}_ML)دym$ +ޟk}@эbƏ =vwQ\\trsx ^z57FW`nryܱtVBoHQkЂU6Q +kk tee \ No newline at end of file diff --git a/test/mutable_test_torrents/test1_pad_files.torrent b/test/mutable_test_torrents/test1_pad_files.torrent new file mode 100644 index 000000000..b0c6088b6 --- /dev/null +++ b/test/mutable_test_torrents/test1_pad_files.torrent @@ -0,0 +1,2 @@ +d10:created by10:libtorrent13:creation datei1419490165e4:infod5:filesld6:lengthi53248e4:pathl1:deed4:attr1:p6:lengthi12288e4:pathl17:.____padding_file1:0eed6:lengthi51200e4:pathl1:aeed4:attr1:p6:lengthi14336e4:pathl17:.____padding_file1:1eed6:lengthi19e4:pathl1:ceed6:lengthi18e4:pathl1:beee4:name5:test112:piece lengthi16384e6:pieces180:1Q5Oۃ@Mst0zF؉9V{0]yrvc>FO~zmN}_ML)دym$ +ޟk}@эbƏ =vwQ F?{+,/߼G:ktۉ۱geee \ No newline at end of file diff --git a/test/mutable_test_torrents/test1_single.torrent b/test/mutable_test_torrents/test1_single.torrent new file mode 100644 index 000000000..eccef4b82 --- /dev/null +++ b/test/mutable_test_torrents/test1_single.torrent @@ -0,0 +1,2 @@ +d10:created by10:libtorrent13:creation datei1419490700e4:infod6:lengthi51200e4:name1:a12:piece lengthi16384e6:pieces80:}_ML)دym$ +ޟk}@эbƏ =vwQ(R!+O0^+wIee \ No newline at end of file diff --git a/test/mutable_test_torrents/test1_single_padded.torrent b/test/mutable_test_torrents/test1_single_padded.torrent new file mode 100644 index 000000000..9e349cdbd --- /dev/null +++ b/test/mutable_test_torrents/test1_single_padded.torrent @@ -0,0 +1,2 @@ +d10:created by10:libtorrent13:creation datei1419490711e4:infod6:lengthi51200e4:name1:a12:piece lengthi16384e6:pieces80:}_ML)دym$ +ޟk}@эbƏ =vwQ F?{+,/߼Gee \ No newline at end of file diff --git a/test/mutable_test_torrents/test2.torrent b/test/mutable_test_torrents/test2.torrent new file mode 100644 index 000000000..1a6000545 --- /dev/null +++ b/test/mutable_test_torrents/test2.torrent @@ -0,0 +1 @@ +d10:created by10:libtorrent13:creation datei1419452987e4:infod5:filesld6:lengthi18e4:pathl1:aeed6:lengthi51200e4:pathl1:beed6:lengthi19e4:pathl1:ceed6:lengthi46080e4:pathl1:deee4:name5:test212:piece lengthi16384e6:pieces120:Ή!ut %˝0i/%ON\<<,3J~,tS Ę@EiQו7GFy-IǿRb0ghXBVD8-..45kqD- +fH섔僸Ky +A,@f0kHO:ktۉ۱geee \ No newline at end of file diff --git a/test/test_block_cache.cpp b/test/test_block_cache.cpp index f4885e7cd..ee4a47eb0 100644 --- a/test/test_block_cache.cpp +++ b/test/test_block_cache.cpp @@ -74,10 +74,11 @@ struct test_storage_impl : storage_interface virtual bool has_any_file(storage_error& ec) { return false; } virtual void set_file_priority(std::vector const& prio , storage_error& ec) {} - virtual int move_storage(std::string const& save_path, int flags, storage_error& ec) - { return 0; } - virtual bool verify_resume_data(bdecode_node const& rd, storage_error& ec) - { return true; } + virtual int move_storage(std::string const& save_path, int flags + , storage_error& ec) { return 0; } + virtual bool verify_resume_data(bdecode_node const& rd + , std::vector const* links + , storage_error& ec) { return true; } virtual void write_resume_data(entry& rd, storage_error& ec) const {} virtual void release_files(storage_error& ec) {} virtual void rename_file(int index, std::string const& new_filenamem diff --git a/test/test_file_storage.cpp b/test/test_file_storage.cpp index 8f4885ccc..9d0c1f3d9 100644 --- a/test/test_file_storage.cpp +++ b/test/test_file_storage.cpp @@ -207,6 +207,7 @@ int test_main() TEST_EQUAL(file_hash, path_hash); } + // TODO: test file_storage::optimize too // TODO: test map_block // TODO: test piece_size(int piece) // TODO: test file_index_at_offset diff --git a/test/test_resolve_links.cpp b/test/test_resolve_links.cpp new file mode 100644 index 000000000..c224d04a7 --- /dev/null +++ b/test/test_resolve_links.cpp @@ -0,0 +1,133 @@ +/* + +Copyright (c) 2014, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "test.hpp" +#include "libtorrent/torrent_info.hpp" +#include "libtorrent/resolve_links.hpp" +#include "libtorrent/file.hpp" // for combine_path +#include +#include + +using namespace libtorrent; + +struct test_torrent_t +{ + char const* filename1; + char const* filename2; + int expected_matches; +}; + +test_torrent_t test_torrents[] = { + // no match because shared file in test2 and test3 is not padded/aligned + { "test2", "test1_pad_files", 0}, + { "test3", "test1_pad_files", 0}, + + // in this case, test1 happens to have the shared file as the first one, + // which makes it padded, however, the tail of it isn't padded, so it + // still overlaps with the next file + { "test1", "test1_pad_files", 0}, + + // test2 and test3 don't have the shared file aligned + { "test2", "test1_pad_files", 0}, + { "test3", "test1_pad_files", 0}, + { "test2", "test1_single", 0}, + + // these are all padded. The first small file will accidentally also + // match, even though it's not tail padded, the following file is identical + { "test2_pad_files", "test1_pad_files", 2}, + { "test3_pad_files", "test1_pad_files", 2}, + { "test3_pad_files", "test2_pad_files", 2}, + { "test1_pad_files", "test2_pad_files", 2}, + { "test1_pad_files", "test3_pad_files", 2}, + { "test2_pad_files", "test3_pad_files", 2}, + + // one might expect this to work, but since the tail of the single file + // torrent is not padded, the last piece hash won't match + { "test1_pad_files", "test1_single", 0}, + + // if it's padded on the other hand, it will work + { "test1_pad_files", "test1_single_padded", 1}, + + // TODO: test files with different piece size (negative test) +}; + +// TODO: it would be nice to test resolving of more than just 2 files as well. +// like 3 single file torrents merged into one, resolving all 3 files. + +int test_main() +{ + +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + std::string path + = combine_path("..", "mutable_test_torrents"); + + for (int i = 0; i < sizeof(test_torrents)/sizeof(test_torrents[0]); ++i) + { + test_torrent_t const& e = test_torrents[i]; + + std::string p = combine_path(path, e.filename1) + ".torrent"; + fprintf(stderr, "loading %s\n", p.c_str()); + boost::shared_ptr ti1 = boost::make_shared(p); + + p = combine_path(path, e.filename2) + ".torrent"; + fprintf(stderr, "loading %s\n", p.c_str()); + boost::shared_ptr ti2 = boost::make_shared(p); + + fprintf(stderr, "resolving\n"); + resolve_links l(ti1); + l.match(ti2, "."); + + std::vector const& links = l.get_links(); + + int num_matches = std::count_if(links.begin(), links.end() + , boost::bind(&resolve_links::link_t::ti, _1)); + + // some debug output in case the test fails + if (num_matches > e.expected_matches) + { + file_storage const& fs = ti1->files(); + for (int i = 0; i < links.size(); ++i) + { + TORRENT_ASSERT(i < fs.num_files()); + fprintf(stderr, "%s --> %s : %d\n", fs.file_name(i).c_str() + , links[i].ti ? to_hex(links[i].ti->info_hash() + .to_string()).c_str() : "", links[i].file_idx); + } + } + + TEST_EQUAL(num_matches, e.expected_matches); + + } +#endif // TORRENT_DISABLE_MUTABLE_TORRENTS + return 0; +} + diff --git a/test/test_storage.cpp b/test/test_storage.cpp index 20a3510e5..5a116aae0 100644 --- a/test/test_storage.cpp +++ b/test/test_storage.cpp @@ -460,7 +460,9 @@ void test_check_files(std::string const& test_path bool done = false; bdecode_node frd; - io.async_check_fastresume(pm.get(), &frd, boost::bind(&on_check_resume_data, _1, &done)); + std::auto_ptr > links; + io.async_check_fastresume(pm.get(), &frd, links + , boost::bind(&on_check_resume_data, _1, &done)); io.submit_jobs(); ios.reset(); run_until(ios, done); diff --git a/test/test_torrent_info.cpp b/test/test_torrent_info.cpp index 255f7d0d0..db7df98a8 100644 --- a/test/test_torrent_info.cpp +++ b/test/test_torrent_info.cpp @@ -43,6 +43,46 @@ POSSIBILITY OF SUCH DAMAGE. using namespace libtorrent; +void test_mutable_torrents() +{ + file_storage fs; + + fs.add_file("test/temporary.txt", 0x4000); + + libtorrent::create_torrent t(fs, 0x4000); + + // calculate the hash for all pieces + int num = t.num_pieces(); + sha1_hash ph; + for (int i = 0; i < num; ++i) + t.set_hash(i, ph); + + t.add_collection("collection1"); + t.add_collection("collection2"); + + t.add_similar_torrent(sha1_hash("abababababababababab")); + t.add_similar_torrent(sha1_hash("babababababababababa")); + + std::vector tmp; + std::back_insert_iterator > out(tmp); + + entry tor = t.generate(); + bencode(out, tor); + + torrent_info ti(&tmp[0], tmp.size()); + + std::vector similar; + similar.push_back(sha1_hash("abababababababababab")); + similar.push_back(sha1_hash("babababababababababa")); + + std::vector collections; + collections.push_back("collection1"); + collections.push_back("collection2"); + + TEST_CHECK(similar == ti.similar_torrents()); + TEST_CHECK(collections == ti.collections()); +} + struct test_torrent_t { char const* file; @@ -654,6 +694,9 @@ int test_main() { test_resolve_duplicates(); test_copy(); +#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS + test_mutable_torrents(); +#endif test_torrent_parse(); return 0;