landed mutable torrents branch in trunk

CMakeLists.txt

@@ -57,6 +57,7 @@ set(sources
 	random
 	receive_buffer
 	request_blocks
+	resolve_links
 	resolver
 	rss
 	session

ChangeLog

@@ -1,3 +1,4 @@
+	* experimental support for BEP 38, "mutable torrents"
 	* replaced lazy_bdecode with a new bdecoder that's a lot more efficient
 	* deprecate time functions, expose typedefs of boost::chrono in the libtorrent
 	  namespace instead

Jamfile

@@ -405,6 +405,9 @@ feature.compose <dht>logging : <define>TORRENT_DHT_VERBOSE_LOGGING ;
 feature encryption : on off : composite propagated link-incompatible ;
 feature.compose <encryption>off : <define>TORRENT_DISABLE_ENCRYPTION ;
 
+feature mutable-torrents : on off : composite propagated link-incompatible ;
+feature.compose <mutable-torrents>off : <define>TORRENT_DISABLE_MUTABLE_TORRENTS ;
+
 feature crypto : built-in openssl gcrypt : composite propagated ;
 feature.compose <crypto>openssl : <define>TORRENT_USE_OPENSSL ;
 feature.compose <crypto>gcrypt : <define>TORRENT_USE_GCRYPT ;
@@ -573,6 +576,7 @@ SOURCES =
 	puff
 	random
 	receive_buffer
+	resolve_links
 	rss
 	session
 	session_impl

docs/building.rst

@@ -289,6 +289,10 @@ Build features:
 |                          |   connections. The shipped public domain SHA-1     |
 |                          |   implementation is used.                          |
 +--------------------------+----------------------------------------------------+
+| ``mutable-torrents``     | * ``on`` - mutable torrents are supported          |
+|                          |   (`BEP 38`_) (default).                           |
+|                          | * ``off`` - mutable torrents are not supported.    |
++--------------------------+----------------------------------------------------+
 | ``crypto``               | * ``built-in`` - (default) uses built-in SHA-1     |
 |                          |   implementation.                                  |
 |                          | * ``openssl`` - links against openssl and          |
@@ -578,6 +582,8 @@ defines you can use to control the build.
 +----------------------------------------+-------------------------------------------------+
 | ``TORRENT_DISABLE_POOL_ALLOCATOR``     | Disables use of ``boost::pool<>``.              |
 +----------------------------------------+-------------------------------------------------+
+| ``TORRENT_DISABLE_MUTABLE_TORRENTS``   | Disables mutable torrent support (`BEP 38`_)    |
++----------------------------------------+-------------------------------------------------+
 | ``TORRENT_LINKING_SHARED``             | If this is defined when including the           |
 |                                        | libtorrent headers, the classes and functions   |
 |                                        | will be tagged with ``__declspec(dllimport)``   |
@@ -644,6 +650,7 @@ defines you can use to control the build.
 |                                        | custom one.                                     |
 +----------------------------------------+-------------------------------------------------+
 
+.. _`BEP 38`: http://www.bittorrent.org/beps/bep_0038.html
 
 If you experience that libtorrent uses unreasonable amounts of cpu, it will
 definitely help to define ``NDEBUG``, since it will remove the invariant checks

examples/make_torrent.cpp

@@ -39,6 +39,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/create_torrent.hpp"
 #include "libtorrent/file.hpp"
 #include "libtorrent/file_pool.hpp"
+#include "libtorrent/escape_string.hpp" // for from_hex
 
 #include <boost/bind.hpp>
 
@@ -164,29 +165,38 @@ void print_usage()
 		"Generates a torrent file from the specified file\n"
 		"or directory and writes it to standard out\n\n"
 		"OPTIONS:\n"
-		"-m file     generate a merkle hash tree torrent.\n"
-		"            merkle torrents require client support\n"
-		"            the resulting full merkle tree is written to\n"
-		"            the specified file\n"
-		"-w url      adds a web seed to the torrent with\n"
-		"            the specified url\n"
-		"-t url      adds the specified tracker to the\n"
-		"            torrent. For multiple trackers, specify more\n"
-		"            -t options\n"
-		"-c comment  sets the comment to the specified string\n"
-		"-C creator  sets the created-by field to the specified string\n"
-		"-p bytes    enables padding files. Files larger\n"
-		"            than bytes will be piece-aligned\n"
-		"-s bytes    specifies a piece size for the torrent\n"
-		"            This has to be a multiple of 16 kiB\n"
-		"-l          Don't follow symlinks, instead encode them as\n"
-		"            links in the torrent file\n"
-		"-o file     specifies the output filename of the torrent file\n"
-		"            If this is not specified, the torrent file is\n"
-		"            printed to the standard out, except on windows\n"
-		"            where the filename defaults to a.torrent\n"
-		"-r file     add root certificate to the torrent, to verify\n"
-		"            the HTTPS tracker\n"
+		"-m file       generate a merkle hash tree torrent.\n"
+		"              merkle torrents require client support\n"
+		"              the resulting full merkle tree is written to\n"
+		"              the specified file\n"
+		"-w url        adds a web seed to the torrent with\n"
+		"              the specified url\n"
+		"-t url        adds the specified tracker to the\n"
+		"              torrent. For multiple trackers, specify more\n"
+		"              -t options\n"
+		"-c comment    sets the comment to the specified string\n"
+		"-C creator    sets the created-by field to the specified string\n"
+		"-p bytes      enables padding files. Files larger\n"
+		"              than bytes will be piece-aligned\n"
+		"-s bytes      specifies a piece size for the torrent\n"
+		"              This has to be a multiple of 16 kiB\n"
+		"-l            Don't follow symlinks, instead encode them as\n"
+		"              links in the torrent file\n"
+		"-o file       specifies the output filename of the torrent file\n"
+		"              If this is not specified, the torrent file is\n"
+		"              printed to the standard out, except on windows\n"
+		"              where the filename defaults to a.torrent\n"
+		"-r file       add root certificate to the torrent, to verify\n"
+		"              the HTTPS tracker\n"
+		"-S info-hash  add a similar torrent by info-hash. The similar\n"
+		"              torrent is expected to share some files with this one\n"
+		"-L collection add a collection name to this torrent. Other torrents\n"
+		"              in the same collection is expected to share files\n"
+		"              with this one.\n"
+		"-M            make the torrent compatible with mutable torrents\n"
+		"              this means aligning large files and pad them in order\n"
+		"              for piece hashes to uniquely indentify a file without\n"
+		"              overlap\n"
 		, stderr);
 }
 
@@ -209,6 +219,8 @@ int main(int argc, char* argv[])
 #endif
 		std::vector<std::string> web_seeds;
 		std::vector<std::string> trackers;
+		std::vector<std::string> collections;
+		std::vector<sha1_hash> similar;
 		int pad_file_limit = -1;
 		int piece_size = 0;
 		int flags = 0;
@@ -240,6 +252,10 @@ int main(int argc, char* argv[])
 					++i;
 					trackers.push_back(argv[i]);
 					break;
+				case 'M':
+					flags |= create_torrent::mutable_torrent_support;
+					pad_file_limit = 0x4000;
+					break;
 				case 'p':
 					++i;
 					pad_file_limit = atoi(argv[i]);
@@ -273,6 +289,30 @@ int main(int argc, char* argv[])
 					++i;
 					root_cert = argv[i];
 					break;
+				case 'S':
+					{
+					++i;
+					if (strlen(argv[i]) != 40)
+					{
+						fprintf(stderr, "invalid info-hash for -S. "
+							"Expected 40 hex characters\n");
+						print_usage();
+						return 1;
+					}
+					sha1_hash ih;
+					if (!from_hex(argv[i], 40, (char*)&ih[0]))
+					{
+						fprintf(stderr, "invalid info-hash for -S\n");
+						print_usage();
+						return 1;
+					}
+					similar.push_back(ih);
+					}
+					break;
+				case 'L':
+					++i;
+					collections.push_back(argv[i]);
+					break;
 				default:
 					print_usage();
 					return 1;
@@ -314,6 +354,14 @@ int main(int argc, char* argv[])
 			, end(web_seeds.end()); i != end; ++i)
 			t.add_url_seed(*i);
 
+		for (std::vector<std::string>::iterator i = collections.begin()
+			, end(collections.end()); i != end; ++i)
+			t.add_collection(*i);
+
+		for (std::vector<sha1_hash>::iterator i = similar.begin()
+			, end(similar.end()); i != end; ++i)
+			t.add_similar_torrent(*i);
+
 		error_code ec;
 		set_piece_hashes(t, branch_path(full_path)
 			, boost::bind(&print_progress, _1, t.num_pieces()), ec);

include/libtorrent/Makefile.am

@@ -101,6 +101,7 @@ nobase_include_HEADERS = \
   proxy_base.hpp               \
   puff.hpp                     \
   random.hpp                   \
+  resolve_links.hpp            \
   resolver.hpp                 \
   resolver_interface.hpp       \
   rss.hpp                      \

include/libtorrent/aux_/session_impl.hpp

@@ -268,6 +268,10 @@ namespace libtorrent
 
 			boost::weak_ptr<torrent> find_torrent(sha1_hash const& info_hash) const;
 			boost::weak_ptr<torrent> find_torrent(std::string const& uuid) const;
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+			std::vector<boost::shared_ptr<torrent> > find_collection(
+				std::string const& collection) const;
+#endif
 			boost::weak_ptr<torrent> find_disconnect_candidate_torrent() const;
 			int num_torrents() const { return m_torrents.size(); }
 

include/libtorrent/aux_/session_interface.hpp

@@ -226,6 +226,11 @@ namespace libtorrent { namespace aux
 		virtual bool verify_bound_address(address const& addr, bool utp
 			, error_code& ec) = 0;
 
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+		virtual std::vector<boost::shared_ptr<torrent> > find_collection(
+			std::string const& collection) const = 0;
+#endif
+
 		// TODO: it would be nice to not have this be part of session_interface
 		virtual proxy_settings proxy() const = 0;
 

include/libtorrent/create_torrent.hpp

@@ -114,8 +114,14 @@ namespace libtorrent
 		enum flags_t
 		{
 			// This will insert pad files to align the files to piece boundaries, for
-			// optimized disk-I/O.
-			optimize = 1
+			// optimized disk-I/O. This will minimize the number of bytes of pad-
+			// files, to keep the impact down for clients that don't support
+			// them.
+			optimize_alignment = 1,
+#ifndef TORRENT_NO_DEPRECATED
+			// same as optimize_alignment, for backwards compatibility
+			optimize = 1,
+#endif
 
 			// This will create a merkle hash tree torrent. A merkle torrent cannot
 			// be opened in clients that don't specifically support merkle torrents.
@@ -125,7 +131,7 @@ namespace libtorrent
 			// When creating merkle torrents, the full hash tree is also generated
 			// and should be saved off separately. It is accessed through the 
 			// create_torrent::merkle_tree() function.
-			, merkle = 2
+			merkle = 2,
 
 			// This will include the file modification time as part of the torrent.
 			// This is not enabled by default, as it might cause problems when you
@@ -133,13 +139,20 @@ namespace libtorrent
 			// yield the same info-hash. If the files have different modification times,
 			// with this option enabled, you would get different info-hashes for the
 			// files.
-			, modification_time = 4
+			modification_time = 4,
 
 			// If this flag is set, files that are symlinks get a symlink attribute
 			// set on them and their data will not be included in the torrent. This
 			// is useful if you need to reconstruct a file hierarchy which contains
 			// symlinks.
-			, symlinks = 8
+			symlinks = 8,
+
+			// to create a torrent that can be updated via a *mutable torrent*
+			// (see BEP38_). This also needs to be enabled for torrents that update
+			// another torrent.
+			// 
+			// .. _BEP38: http://www.bittorrent.org/beps/bep_0038.html
+			mutable_torrent_support = 16,
 		};
 
 		// The ``piece_size`` is the size of each piece in bytes. It must
@@ -279,6 +292,17 @@ namespace libtorrent
 		// tree will be saved in the resume data.
 		std::vector<sha1_hash> const& merkle_tree() const { return m_merkle_tree; }
 
+		// Add similar torrents (by info-hash) or collections of similar torrents.
+		// Similar torrents are expected to share some files with this torrent.
+		// Torrents sharing a collection name with this torrent are also expected
+		// to share files with this torrent. A torrent may have more than one
+		// collection and more than one similar torrents. For more information,
+		// see `BEP 38`_.
+		// 
+		// .. _`BEP 38`: http://www.bittorrent.org/beps/bep_0038.html
+		void add_similar_torrent(sha1_hash ih);
+		void add_collection(std::string c);
+
 	private:
 
 		file_storage& m_files;
@@ -298,6 +322,9 @@ namespace libtorrent
 
 		std::vector<sha1_hash> m_filehashes;
 
+		std::vector<sha1_hash> m_similar;
+		std::vector<std::string> m_collections;
+
 		// if we're generating a merkle torrent, this is the
 		// merkle tree we got. This should be saved in fast-resume
 		// in order to start seeding the torrent

include/libtorrent/disk_interface.hpp

@@ -39,6 +39,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/bdecode.hpp"
 
 #include <string>
+#include <memory>
 
 namespace libtorrent
 {
@@ -67,6 +68,7 @@ namespace libtorrent
 			= boost::function<void(disk_io_job const*)>()) = 0;
 		virtual void async_check_fastresume(piece_manager* storage
 			, bdecode_node const* resume_data
+			, std::auto_ptr<std::vector<std::string> >& links
 			, boost::function<void(disk_io_job const*)> const& handler) = 0;
 #ifndef TORRENT_NO_DEPRECATE
 		virtual void async_finalize_file(piece_manager*, int file

include/libtorrent/disk_io_job.hpp

@@ -96,6 +96,7 @@ namespace libtorrent
 			, load_torrent
 			, clear_piece
 			, tick_storage
+			, resolve_links
 
 			, num_job_ids
 		};
@@ -164,6 +165,12 @@ namespace libtorrent
 			// result for hash jobs
 			char piece_hash[20];
 
+			// this is used for check_fastresume to pass in a vector of hard-links
+			// to create. Each element corresponds to a file in the file_storage.
+			// The string is the absolute path of the identical file to create
+			// the hard link to.
+			std::vector<std::string>* links;
+
 			struct io_args
 			{
 			// if this is set, the read operation is required to

include/libtorrent/disk_io_thread.hpp

@@ -308,6 +308,7 @@ namespace libtorrent
 			, boost::function<void(disk_io_job const*)> const& handler);
 		void async_check_fastresume(piece_manager* storage
 			, bdecode_node const* resume_data
+			, std::auto_ptr<std::vector<std::string> >& links
 			, boost::function<void(disk_io_job const*)> const& handler);
 		void async_save_resume_data(piece_manager* storage
 			, boost::function<void(disk_io_job const*)> const& handler);
@@ -420,6 +421,7 @@ namespace libtorrent
 		int do_load_torrent(disk_io_job* j, tailqueue& completed_jobs);
 		int do_clear_piece(disk_io_job* j, tailqueue& completed_jobs);
 		int do_tick(disk_io_job* j, tailqueue& completed_jobs);
+		int do_resolve_links(disk_io_job* j, tailqueue& completed_jobs);
 
 		void call_job_handlers(void* userdata);
 

include/libtorrent/error_code.hpp

@@ -578,6 +578,8 @@ namespace libtorrent
 			partfile_move,
 			partfile_read,
 			partfile_write,
+			check_resume,
+			hard_link,
 		};
 
 		// Returns a string literal representing the file operation
@@ -590,6 +592,7 @@ namespace libtorrent
 				"", "stat", "mkdir", "open", "rename", "remove", "copy"
 				, "read", "write", "fallocate", "allocate cache piece"
 				, "partfile move", "partfile read", "partfile write"
+				, "check resume", "hard_link"
 			};
 			return ops[operation];
 		}

include/libtorrent/file.hpp

@@ -140,6 +140,11 @@ namespace libtorrent
 	TORRENT_EXTRA_EXPORT void copy_file(std::string const& f
 		, std::string const& newf, error_code& ec);
 
+	// file is expected to exist, link will be created to point to it. If hard
+	// links are not supported by the filesystem or OS, the file will be copied.
+	TORRENT_EXTRA_EXPORT void hard_link(std::string const& file
+		, std::string const& link, error_code& ec);
+
 	TORRENT_EXTRA_EXPORT std::string split_path(std::string const& f);
 	TORRENT_EXTRA_EXPORT char const* next_path_element(char const* p);
 	TORRENT_EXTRA_EXPORT std::string extension(std::string const& f);

include/libtorrent/file_storage.hpp

@@ -425,7 +425,11 @@ namespace libtorrent
 		// (-1) but it may also make sense to set it to 16 kiB, or something
 		// divisible by 16 kiB.
 		// If pad_file_limit is 0, every file will be padded (except empty ones).
-		void optimize(int pad_file_limit = -1, int alignment = -1);
+		// ``tail_padding`` indicates whether aligned files also are padded at
+		// the end to make them end aligned. This is required for mutable
+		// torrents, since piece hashes are compared
+		void optimize(int pad_file_limit = -1, int alignment = -1
+			, bool tail_padding = false);
 
 		// These functions are used to query attributes of files at
 		// a given index.
@@ -545,6 +549,11 @@ namespace libtorrent
 
 	private:
 
+		void add_pad_file(int size
+			, std::vector<internal_file_entry>::iterator& i
+			, boost::int64_t& offset
+			, int& pad_file_counter);
+
 		// the number of bytes in a regular piece
 		// (i.e. not the potentially truncated last piece)
 		int m_piece_length;

include/libtorrent/resolve_links.hpp

@@ -0,0 +1,83 @@
+/*
+
+Copyright (c) 2014, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#ifndef TORRENT_RESOLVE_LINKS_HPP
+#define TORRENT_RESOLVE_LINKS_HPP
+
+#include <boost/shared_ptr.hpp>
+#include <boost/unordered_map.hpp>
+#include <vector>
+#include <utility>
+
+namespace libtorrent
+{
+	class torrent_info;
+
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+	// this class is used for mutable torrents, to discover identical files
+	// in other torrents.
+	struct resolve_links
+	{
+		struct link_t
+		{
+			boost::shared_ptr<const torrent_info> ti;
+			std::string save_path;
+			int file_idx;
+		};
+
+		resolve_links(boost::shared_ptr<torrent_info> ti);
+
+		// check to see if any files are shared with this torrent
+		void match(boost::shared_ptr<const torrent_info> const& ti
+			, std::string const& save_path);
+
+		std::vector<link_t> const& get_links() const
+		{ return m_links; }
+
+	private:
+		// this is the torrent we're trying to find files for.
+		boost::shared_ptr<torrent_info> m_torrent_file;
+
+		// each file in m_torrent_file has an entry in this vector. Any file
+		// that also exists somewhere else, is filled in with the corresponding
+		// torrent_info object and file index
+		std::vector<link_t> m_links;
+
+		// maps file size to file index, in m_torrent_file
+		boost::unordered_multimap<boost::int64_t, int> m_file_sizes;
+	};
+#endif // TORRENT_DISABLE_MUTABLE_TORRENTS
+
+}
+
+#endif
+

include/libtorrent/storage.hpp

@@ -115,7 +115,9 @@ POSSIBILITY OF SUCH DAMAGE.
 //		virtual bool rename_file(int file, std::string const& new_name)
 //		{ assert(false); return false; }
 //		virtual bool move_storage(std::string const& save_path) { return false; }
-//		virtual bool verify_resume_data(bdecode_node const& rd, storage_error& error) { return false; }
+//		virtual bool verify_resume_data(bdecode_node const& rd
+//			, std::vector<std::string> const* links
+//			, storage_error& error) { return false; }
 //		virtual bool write_resume_data(entry& rd) const { return false; }
 //		virtual boost::int64_t physical_offset(int slot, int offset)
 //		{ return slot * m_files.piece_length() + offset; };
@@ -307,7 +309,15 @@ namespace libtorrent
 		// This function should verify the resume data ``rd`` with the files
 		// on disk. If the resume data seems to be up-to-date, return true. If
 		// not, set ``error`` to a description of what mismatched and return false.
-		virtual bool verify_resume_data(bdecode_node const& rd, storage_error& ec) = 0;
+		// 
+		// If the ``links`` pointer is non-null, it has the same number
+		// of elements as there are files. Each element is either empty or contains
+		// the absolute path to a file identical to the corresponding file in this
+		// torrent. The storage must create hard links (or copy) those files. If
+		// any file does not exist or is inaccessible, the disk job must fail.
+		virtual bool verify_resume_data(bdecode_node const& rd
+			, std::vector<std::string> const* links
+			, storage_error& ec) = 0;
 
 		// This function should fill in resume data, the current state of the
 		// storage, in ``rd``. The default storage adds file timestamps and
@@ -423,7 +433,9 @@ namespace libtorrent
 		void initialize(storage_error& ec);
 		int move_storage(std::string const& save_path, int flags, storage_error& ec);
 		int sparse_end(int start) const;
-		bool verify_resume_data(bdecode_node const& rd, storage_error& error);
+		bool verify_resume_data(bdecode_node const& rd
+			, std::vector<std::string> const* links
+			, storage_error& error);
 		void write_resume_data(entry& rd, storage_error& ec) const;
 		bool tick();
 
@@ -519,7 +531,9 @@ namespace libtorrent
 		int writev(file::iovec_t const* bufs, int num_bufs, int piece
 			, int offset, int flags, storage_error& ec);
 
-		bool verify_resume_data(bdecode_node const&, storage_error&) { return false; }
+		bool verify_resume_data(bdecode_node const&
+			, std::vector<std::string> const* links
+			, storage_error&) { return false; }
 		void write_resume_data(entry&, storage_error&) const {}
 
 		int m_piece_size;
@@ -541,7 +555,9 @@ namespace libtorrent
 			, storage_error&) {}
 		virtual int move_storage(std::string const& /* save_path */
 			, int /* flags */, storage_error&) { return 0; }
-		virtual bool verify_resume_data(bdecode_node const& /* rd */, storage_error&)
+		virtual bool verify_resume_data(bdecode_node const& /* rd */
+			, std::vector<std::string> const* /* links */
+			, storage_error&)
 			{ return false; }
 		virtual void write_resume_data(entry&, storage_error&) const {}
 		virtual void release_files(storage_error&) {}
@@ -674,7 +690,9 @@ namespace libtorrent
 		// the error message indicates that the fast resume data was rejected
 		// if 'fatal_disk_error' is returned, the error message indicates what
 		// when wrong in the disk access
-		int check_fastresume(bdecode_node const& rd, storage_error& error);
+		int check_fastresume(bdecode_node const& rd
+			, std::vector<std::string> const* links
+			, storage_error& error);
 
 		// helper functions for check_fastresume	
 		int check_no_fastresume(storage_error& error);

include/libtorrent/torrent_info.hpp

@@ -413,6 +413,16 @@ namespace libtorrent
 		void add_tracker(std::string const& url, int tier = 0);
 		std::vector<announce_entry> const& trackers() const { return m_urls; }
 
+		// These two functions are related to BEP38_ (mutable torrents). The
+		// vectors returned from these correspond to the "similar" and
+		// "collections" keys in the .torrent file. Both info-hashes and
+		// collections from within the info-dict and from outside of it are
+		// included.
+		// 
+		// .. _BEP38: http://www.bittorrent.org/beps/bep_0038.html
+		std::vector<sha1_hash> similar_torrents() const;
+		std::vector<std::string> collections() const;
+
 #ifndef TORRENT_NO_DEPRECATE
 		// deprecated in 0.16. Use web_seeds() instead
 		TORRENT_DEPRECATED_PREFIX
@@ -713,6 +723,27 @@ namespace libtorrent
 		std::vector<web_seed_entry> m_web_seeds;
 		nodes_t m_nodes;
 
+		// the info-hashes (20 bytes each) in the "similar" key. The pointers
+		// point directly into the info_section. When copied, these pointers must
+		// be corrected to point into the copied-to buffer
+		std::vector<char const*> m_similar_torrents;
+
+		// these are similar torrents from outside of the info-dict. We can't
+		// have non-owning pointers to those, as we only keep the info-dict
+		// around.
+		std::vector<sha1_hash> m_owned_similar_torrents;
+
+		// these or strings of the "collections" key from the torrent file. The
+		// pointers point directly into the info_section buffer and when copied,
+		// these pointers must be corrected to point into the new buffer. The
+		// int is the length of the string. Strings are not NULL-terminated.
+		std::vector<std::pair<char const*, int> > m_collections;
+
+		// these are the collections from outside of the info-dict. These are
+		// owning strings, since we only keep the info-section around, these
+		// cannot be pointers into that buffer.
+		std::vector<std::string> m_owned_collections;
+
 		// if this is a merkle torrent, this is the merkle
 		// tree. It has space for merkle_num_nodes(merkle_num_leafs(num_pieces))
 		// hashes

src/Makefile.am

@@ -102,6 +102,7 @@ libtorrent_rasterbar_la_SOURCES = \
   random.cpp                      \
   receive_buffer.cpp              \
   request_blocks.cpp              \
+  resolve_links.cpp               \
   resolver.cpp                    \
   rss.cpp                         \
   session.cpp                     \

src/create_torrent.cpp

@@ -326,6 +326,12 @@ namespace libtorrent
 			piece_size = 64*1024;
 		}
 
+		// to support mutable torrents, alignment always has to be the piece size,
+		// because piece hashes are compared to determine whether files are
+		// identical
+		if (flags & mutable_torrent_support)
+			alignment = piece_size;
+
 		// make sure the size is an even power of 2
 #ifndef NDEBUG
 		for (int i = 0; i < 32; ++i)
@@ -338,8 +344,9 @@ namespace libtorrent
 		}
 #endif
 		m_files.set_piece_length(piece_size);
-		if (flags & optimize)
-			m_files.optimize(pad_file_limit, alignment);
+		if (flags & (optimize_alignment | mutable_torrent_support))
+			m_files.optimize(pad_file_limit, alignment, flags & mutable_torrent_support);
+
 		m_files.set_num_pieces(static_cast<int>(
 			(m_files.total_size() + m_files.piece_length() - 1) / m_files.piece_length()));
 		m_piece_hash.resize(m_files.num_pieces());
@@ -481,6 +488,26 @@ namespace libtorrent
 			return dict;
 		}
 
+		if (!m_collections.empty())
+		{
+			entry& list = info["collections"];
+			for (std::vector<std::string>::const_iterator i
+				= m_collections.begin(); i != m_collections.end(); ++i)
+			{
+				list.list().push_back(entry(*i));
+			}
+		}
+
+		if (!m_similar.empty())
+		{
+			entry& list = info["similar"];
+			for (std::vector<sha1_hash>::const_iterator i
+				= m_similar.begin(); i != m_similar.end(); ++i)
+			{
+				list.list().push_back(entry(i->to_string()));
+			}
+		}
+
 		info["name"] = m_files.name();
 
 		if (!m_root_cert.empty())
@@ -635,6 +662,16 @@ namespace libtorrent
 		m_root_cert = cert;
 	}
 
+	void create_torrent::add_similar_torrent(sha1_hash ih)
+	{
+		m_similar.push_back(ih);
+	}
+
+	void create_torrent::add_collection(std::string c)
+	{
+		m_collections.push_back(c);
+	}
+
 	void create_torrent::set_hash(int index, sha1_hash const& h)
 	{
 		TORRENT_ASSERT(index >= 0);

src/disk_io_thread.cpp

@@ -1903,6 +1903,7 @@ namespace libtorrent
 
 	void disk_io_thread::async_check_fastresume(piece_manager* storage
 		, bdecode_node const* resume_data
+		, std::auto_ptr<std::vector<std::string> >& links
 		, boost::function<void(disk_io_job const*)> const& handler)
 	{
 #ifdef TORRENT_DEBUG
@@ -1914,9 +1915,11 @@ namespace libtorrent
 		disk_io_job* j = allocate_job(disk_io_job::check_fastresume);
 		j->storage = storage->shared_from_this();
 		j->buffer = (char*)resume_data;
+		j->d.links = links.get();
 		j->callback = handler;
 
 		add_fence_job(storage, j);
+		links.release();
 	}
 
 	void disk_io_thread::async_save_resume_data(piece_manager* storage
@@ -2607,7 +2610,8 @@ namespace libtorrent
 		bdecode_node tmp;
 		if (rd == NULL) rd = &tmp;
 
-		return j->storage->check_fastresume(*rd, j->error);
+		std::auto_ptr<std::vector<std::string> > links(j->d.links);
+		return j->storage->check_fastresume(*rd, links.get(), j->error);
 	}
 
 	int disk_io_thread::do_save_resume_data(disk_io_job* j, tailqueue& completed_jobs)

src/file.cpp

@@ -440,25 +440,85 @@ namespace libtorrent
 	{
 		ec.clear();
 
-#if defined TORRENT_WINDOWS && TORRENT_USE_WSTRING
+#ifdef TORRENT_WINDOWS
+#if TORRENT_USE_WSTRING
 #define CreateDirectory_ CreateDirectoryW
 		std::wstring n = convert_to_wstring(f);
 #else
 #define CreateDirectory_ CreateDirectoryA
 		std::string const& n = convert_to_native(f);
-#endif
+#endif // TORRENT_USE_WSTRING
 
-#ifdef TORRENT_WINDOWS
 		if (CreateDirectory_(n.c_str(), 0) == 0
 			&& GetLastError() != ERROR_ALREADY_EXISTS)
 			ec.assign(GetLastError(), boost::system::system_category());
 #else
+		std::string n = convert_to_native(f);
 		int ret = mkdir(n.c_str(), 0777);
 		if (ret < 0 && errno != EEXIST)
 			ec.assign(errno, generic_category());
 #endif
 	}
 
+	void hard_link(std::string const& file, std::string const& link
+		, error_code& ec)
+	{
+#ifdef TORRENT_WINDOWS
+
+#if TORRENT_USE_WSTRING
+#define CreateHardLink_ CreateHardLinkW
+		std::wstring n_exist = convert_to_wstring(file);
+		std::wstring n_link = convert_to_wstring(link);
+#else
+#define CreateHardLink_ CreateHardLinkA
+		std::string n_exist = convert_to_native(file);
+		std::string n_link = convert_to_native(link);
+#endif
+		BOOL ret = CreateHardLink(n_link.c_str(), n_exist.c_str(), NULL);
+		if (ret)
+		{
+			ec.clear();
+			return;
+		}
+
+		// something failed. Does the filesystem not support hard links?
+		// TODO: 3 find out what error code is reported when the filesystem
+		// does not support hard links.
+
+		// it's possible CreateHardLink will copy the file internally too,
+		// if the filesystem does not support it.
+		ec.assign(GetLastError(), system_category());
+		return;
+
+#else
+
+		std::string n_exist = convert_to_native(file);
+		std::string n_link = convert_to_native(link);
+
+		// assume posix's link() function exists
+		int ret = ::link(n_exist.c_str(), n_link.c_str());
+
+		if (ret == 0)
+		{
+			ec.clear();
+			return;
+		}
+
+		// most errors are passed through, except for the ones that indicate that
+		// hard links are not supported and require a copy.
+		// TODO: 2 test this on a FAT volume to see what error we get!
+		if (errno != EMLINK || errno != EXDEV)
+		{
+			// some error happened, report up to the caller
+			ec.assign(errno, generic_category());
+			return;
+		}
+#endif
+
+		// if we get here, we should copy the file
+		copy_file(file, link, ec);
+	}
+
 	bool is_directory(std::string const& f, error_code& ec)
 	{
 		ec.clear();
@@ -494,7 +554,8 @@ namespace libtorrent
 	void copy_file(std::string const& inf, std::string const& newf, error_code& ec)
 	{
 		ec.clear();
-#if TORRENT_USE_WSTRING && defined TORRENT_WINDOWS
+#ifdef TORRENT_WINDOWS
+#if TORRENT_USE_WSTRING
 #define CopyFile_ CopyFileW
 		std::wstring f1 = convert_to_wstring(inf);
 		std::wstring f2 = convert_to_wstring(newf);
@@ -504,17 +565,22 @@ namespace libtorrent
 		std::string const& f2 = convert_to_native(newf);
 #endif
 
-#ifdef TORRENT_WINDOWS
 		if (CopyFile_(f1.c_str(), f2.c_str(), false) == 0)
 			ec.assign(GetLastError(), boost::system::system_category());
 #elif defined __APPLE__ && defined __MACH__ && MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
+		std::string f1 = convert_to_native(inf);
+		std::string f2 = convert_to_native(newf);
+
 		// this only works on 10.5
 		copyfile_state_t state = copyfile_state_alloc();
 		if (copyfile(f1.c_str(), f2.c_str(), state, COPYFILE_ALL) < 0)
 			ec.assign(errno, generic_category());
 		copyfile_state_free(state);
 #else
-		int infd = ::open(inf.c_str(), O_RDONLY);
+		std::string f1 = convert_to_native(inf);
+		std::string f2 = convert_to_native(newf);
+
+		int infd = ::open(f1.c_str(), O_RDONLY);
 		if (infd < 0)
 		{
 			ec.assign(errno, generic_category());
@@ -527,7 +593,7 @@ namespace libtorrent
 			| S_IRGRP | S_IWGRP
 			| S_IROTH | S_IWOTH;
 
-		int outfd = ::open(newf.c_str(), O_WRONLY | O_CREAT, permissions);
+		int outfd = ::open(f2.c_str(), O_WRONLY | O_CREAT, permissions);
 		if (outfd < 0)
 		{
 			close(infd);

src/file_storage.cpp

@@ -861,7 +861,8 @@ namespace libtorrent
 #endif // TORRENT_DEPRECATED
 	}
 
-	void file_storage::optimize(int pad_file_limit, int alignment)
+	void file_storage::optimize(int pad_file_limit, int alignment
+		, bool tail_padding)
 	{
 		if (alignment == -1)
 			alignment = m_piece_length;
@@ -938,40 +939,62 @@ namespace libtorrent
 				// then swap it in place. This minimizes the amount
 				// of copying of internal_file_entry, which is somewhat
 				// expensive (until we have move semantics)
-				int cur_index = i - m_files.begin();
-				int index = m_files.size();
-				m_files.push_back(internal_file_entry());
-				++m_num_files;
-				internal_file_entry& e = m_files.back();
-				// i may have been invalidated, refresh it
-				i = m_files.begin() + cur_index;
-				e.size = pad_size;
-				e.offset = off;
-				char name[30];
-				snprintf(name, sizeof(name), ".____padding_file/%d", padding_file);
-				std::string path = combine_path(m_name, name);
-				e.set_name(path.c_str());
-				e.pad_file = true;
-				off += pad_size;
-				++padding_file;
-
-				if (!m_mtime.empty()) m_mtime.resize(index + 1, 0);
-				if (!m_file_hashes.empty()) m_file_hashes.resize(index + 1, NULL);
-#ifndef TORRENT_NO_DEPRECATE
-				if (!m_file_base.empty()) m_file_base.resize(index + 1, 0);
-#endif
-
-				reorder_file(index, cur_index);
+				add_pad_file(pad_size, i, off, padding_file);
 
 				TORRENT_ASSERT((off % alignment) == 0);
 				continue;
 			}
 			i->offset = off;
 			off += i->size;
+
+			if (tail_padding
+				&& i->size > boost::uint32_t(pad_file_limit)
+				&& (off % alignment) != 0)
+			{
+				// skip the file we just put in place, so we put the pad
+				// file after it
+				++i;
+
+				// tail-padding is enabled, and the offset after this file is not
+				// aligned and it's not the last file. The last file must be padded
+				// too, in order to match an equivalent tail-padded file.
+				add_pad_file(alignment - (off % alignment), i, off, padding_file);
+
+				TORRENT_ASSERT((off % alignment) == 0);
+			}
 		}
 		m_total_size = off;
 	}
 
+	void file_storage::add_pad_file(int size
+		, std::vector<internal_file_entry>::iterator& i
+		, boost::int64_t& offset
+		, int& pad_file_counter)
+	{
+		int cur_index = i - m_files.begin();
+		int index = m_files.size();
+		m_files.push_back(internal_file_entry());
+		++m_num_files;
+		internal_file_entry& e = m_files.back();
+		// i may have been invalidated, refresh it
+		i = m_files.begin() + cur_index;
+		e.size = size;
+		e.offset = offset;
+		char name[30];
+		snprintf(name, sizeof(name), ".____padding_file/%d", pad_file_counter);
+		std::string path = combine_path(m_name, name);
+		e.set_name(path.c_str());
+		e.pad_file = true;
+		offset += size;
+		++pad_file_counter;
+
+		if (!m_mtime.empty()) m_mtime.resize(index + 1, 0);
+		if (!m_file_hashes.empty()) m_file_hashes.resize(index + 1, NULL);
+		if (!m_file_base.empty()) m_file_base.resize(index + 1, 0);
+
+		reorder_file(index, cur_index);
+	}
+
 	void file_storage::unload()
 	{
 		std::vector<internal_file_entry>().swap(m_files);

src/resolve_links.cpp

@@ -0,0 +1,138 @@
+/*
+
+Copyright (c) 2014, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "resolve_links.hpp"
+
+#include "libtorrent/torrent_info.hpp"
+#include <boost/shared_ptr.hpp>
+
+namespace libtorrent
+{
+
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+resolve_links::resolve_links(boost::shared_ptr<torrent_info> ti)
+	: m_torrent_file(ti)
+{
+	TORRENT_ASSERT(ti);
+
+	int piece_size = ti->piece_length();
+
+	file_storage const& fs = ti->files();
+	m_file_sizes.reserve(fs.num_files());
+	for (int i = 0; i < fs.num_files(); ++i)
+	{
+		// don't match pad-files, and don't match files that aren't aligned to
+		// ieces. Files are matched by comparing piece hashes, so pieces must
+		// be aligned and the same size
+		if (fs.pad_file_at(i)) continue;
+		if ((fs.file_offset(i) % piece_size) != 0) continue;
+
+		m_file_sizes.insert(std::make_pair(fs.file_size(i), i));
+	}
+
+	m_links.resize(m_torrent_file->num_files());
+}
+
+void resolve_links::match(boost::shared_ptr<const torrent_info> const& ti
+	, std::string const& save_path)
+{
+	if (!ti) return;
+
+	// only torrents with the same 
+	if (ti->piece_length() != m_torrent_file->piece_length()) return;
+
+	int piece_size = ti->piece_length();
+
+	file_storage const& fs = ti->files();
+	m_file_sizes.reserve(fs.num_files());
+	for (int i = 0; i < fs.num_files(); ++i)
+	{
+		// for every file in the other torrent, see if we have one that match
+		// it in m_torrent_file
+
+		// if the file base is not aligned to pieces, we're not going to match
+		// it anyway (we only compare piece hashes)
+		if ((fs.file_offset(i) % piece_size) != 0) continue;
+		if (fs.pad_file_at(i)) continue;
+
+		boost::int64_t file_size = fs.file_size(i);
+
+		typedef boost::unordered_multimap<boost::int64_t, int>::iterator iterator;
+		iterator iter = m_file_sizes.find(file_size);
+
+		// we don't have a file whose size matches, look at the next one
+		if (iter == m_file_sizes.end()) continue;
+
+		TORRENT_ASSERT(iter->second < m_torrent_file->files().num_files());
+		TORRENT_ASSERT(iter->second >= 0);
+
+		// if we already have found a duplicate for this file, no need
+		// to keep looking
+		if (m_links[iter->second].ti) continue;
+
+		// files are aligned and have the same size, now start comparing
+		// piece hashes, to see if the files are identical
+
+		// the pieces of the incoming file
+		int their_piece = fs.map_file(i, 0, 0).piece;
+		// the pieces of "this" file (from m_torrent_file)
+		int our_piece = m_torrent_file->files().map_file(
+			iter->second, 0, 0).piece;
+
+		int num_pieces = (file_size + piece_size - 1) / piece_size;
+
+		bool match = true;
+		for (int p = 0; p < num_pieces; ++p, ++their_piece, ++our_piece)
+		{
+			if (m_torrent_file->hash_for_piece(our_piece)
+				!= ti->hash_for_piece(their_piece))
+			{
+				match = false;
+				break;
+			}
+		}
+		if (!match) continue;
+
+		m_links[iter->second].ti = ti;
+		m_links[iter->second].save_path = save_path;
+		m_links[iter->second].file_idx = i;
+
+		// since we have a duplicate for this file, we may as well remove
+		// it from the file-size map, so we won't find it again.
+		m_file_sizes.erase(iter);
+	}
+
+}
+#endif // TORRENT_DISABLE_MUTABLE_TORRENTS
+
+} // namespace libtorrent
+

src/session_impl.cpp

@@ -4222,6 +4222,24 @@ retry:
 		return boost::weak_ptr<torrent>();
 	}
 
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+	std::vector<boost::shared_ptr<torrent> > session_impl::find_collection(
+		std::string const& collection) const
+	{
+		std::vector<boost::shared_ptr<torrent> > ret;
+		for (session_impl::torrent_map::const_iterator i = m_torrents.begin()
+			, end(m_torrents.end()); i != end; ++i)
+		{
+			boost::shared_ptr<torrent> t = i->second;
+			if (!t) continue;
+			std::vector<std::string> const& c = t->torrent_file().collections();
+			if (std::count(c.begin(), c.end(), collection) == 0) continue;
+			ret.push_back(t);
+		}
+		return ret;
+	}
+#endif //TORRENT_DISABLE_MUTABLE_TORRENTS
+
 	// returns true if lhs is a better disconnect candidate than rhs
 	bool compare_disconnect_torrent(session_impl::torrent_map::value_type const& lhs
 		, session_impl::torrent_map::value_type const& rhs)

src/storage.cpp

@@ -712,7 +712,9 @@ namespace libtorrent
 		return int((data_start + files().piece_length() - 1) / files().piece_length());
 	}
 
-	bool default_storage::verify_resume_data(bdecode_node const& rd, storage_error& ec)
+	bool default_storage::verify_resume_data(bdecode_node const& rd
+		, std::vector<std::string> const* links
+		, storage_error& ec)
 	{
 		// TODO: make this more generic to not just work if files have been
 		// renamed, but also if they have been merged into a single file for instance
@@ -742,6 +744,8 @@ namespace libtorrent
 		if (file_sizes_ent == 0)
 		{
 			ec.ec = errors::missing_file_sizes;
+			ec.file = -1;
+			ec.operation = storage_error::check_resume;
 			return false;
 		}
 		
@@ -756,7 +760,7 @@ namespace libtorrent
 		{
 			ec.ec = errors::mismatching_number_of_files;
 			ec.file = -1;
-			ec.operation = storage_error::none;
+			ec.operation = storage_error::check_resume;
 			return false;
 		}
 
@@ -792,6 +796,8 @@ namespace libtorrent
 		else
 		{
 			ec.ec = errors::missing_pieces;
+			ec.file = -1;
+			ec.operation = storage_error::check_resume;
 			return false;
 		}
 
@@ -806,7 +812,7 @@ namespace libtorrent
 			{
 				ec.ec = errors::missing_file_sizes;
 				ec.file = i;
-				ec.operation = storage_error::none;
+				ec.operation = storage_error::check_resume;
 				return false;
 			}
 
@@ -819,7 +825,7 @@ namespace libtorrent
 			{
 				ec.ec = errors::mismatching_file_size;
 				ec.file = i;
-				ec.operation = storage_error::none;
+				ec.operation = storage_error::check_resume;
 				return false;
 			}
 
@@ -883,6 +889,41 @@ namespace libtorrent
 		if (rd.dict_find_string_value("allocation") != "compact")
 			full_allocation_mode = true;
 
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+		if (links)
+		{
+			// if this is a mutable torrent, and we need to pick up some files
+			// from other torrents, do that now. Note that there is an inherent
+			// race condition here. We checked if the files existed on a different
+			// thread a while ago. These files may no longer exist or may have been
+			// moved. If so, we just fail. The user is responsible to not touch
+			// other torrents until a new mutable torrent has been completely
+			// added.
+			int idx = 0;
+			for (std::vector<std::string>::const_iterator i = links->begin();
+				i != links->end(); ++i, ++idx)
+			{
+				if (i->empty()) continue;
+
+				error_code err;
+				std::string file_path = fs.file_path(idx, m_save_path);
+				hard_link(*i, file_path, err);
+
+				// if the file already exists, that's not an error
+				// TODO: 2 is this risky? The upper layer will assume we have the
+				// whole file. Perhaps we should verify that at least the size
+				// of the file is correct
+				if (!err || err == boost::system::errc::file_exists)
+					continue;
+
+				ec.ec = err;
+				ec.file = idx;
+				ec.operation = storage_error::hard_link;
+				return false;
+			}
+		}
+#endif // TORRENT_DISABLE_MUTABLE_TORRENTS
+
 		return true;
 	}
 
@@ -1470,9 +1511,15 @@ namespace libtorrent
 	// check if the fastresume data is up to date
 	// if it is, use it and return true. If it 
 	// isn't return false and the full check
-	// will be run
+	// will be run. If the links pointer is non-null, it has the same number
+	// of elements as there are files. Each element is either empty or contains
+	// the absolute path to a file identical to the corresponding file in this
+	// torrent. The storage must create hard links (or copy) those files. If
+	// any file does not exist or is inaccessible, the disk job must fail.
 	int piece_manager::check_fastresume(
-		bdecode_node const& rd, storage_error& ec)
+		bdecode_node const& rd
+		, std::vector<std::string> const* links
+		, storage_error& ec)
 	{
 		TORRENT_ASSERT(m_files.piece_length() > 0);
 		
@@ -1494,7 +1541,7 @@ namespace libtorrent
 			return check_no_fastresume(ec);
 		}
 
-		if (!m_storage->verify_resume_data(rd, ec))
+		if (!m_storage->verify_resume_data(rd, links, ec))
 			return check_no_fastresume(ec);
 
 		return check_init_storage(ec);

src/torrent.cpp

@@ -86,6 +86,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/alert_manager.hpp" // for alert_manageralert_manager
 #include "libtorrent/resolver_interface.hpp"
 #include "libtorrent/alloca.hpp"
+#include "libtorrent/resolve_links.hpp"
 
 #ifdef TORRENT_USE_OPENSSL
 #include "libtorrent/ssl_stream.hpp"
@@ -1855,8 +1856,6 @@ namespace libtorrent
 			return;
 		}
 
-		set_state(torrent_status::checking_resume_data);
-
 		int num_pad_files = 0;
 		TORRENT_ASSERT(block_size() > 0);
 		file_storage const& fs = m_torrent_file->files();
@@ -1927,10 +1926,68 @@ namespace libtorrent
 		if (num_pad_files > 0)
 			m_picker->set_num_pad_files(num_pad_files);
 
+		std::auto_ptr<std::vector<std::string> > links;
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+		if (!m_torrent_file->similar_torrents().empty()
+			|| !m_torrent_file->collections().empty())
+		{
+			resolve_links res(m_torrent_file);
+
+			std::vector<sha1_hash> s = m_torrent_file->similar_torrents();
+			for (std::vector<sha1_hash>::iterator i = s.begin(), end(s.end());
+				i != end; ++i)
+			{
+				boost::shared_ptr<torrent> t = m_ses.find_torrent(*i).lock();
+				if (!t) continue;
+
+				// Only attempt to reuse files from torrents that are seeding.
+				// TODO: this could be optimized by looking up which files are
+				// complete and just look at those
+				if (!t->is_seed()) continue;
+
+				res.match(t->get_torrent_copy(), t->save_path());
+			}
+			std::vector<std::string> c = m_torrent_file->collections();
+			for (std::vector<std::string>::iterator i = c.begin(), end(c.end());
+				i != end; ++i)
+			{
+				std::vector<boost::shared_ptr<torrent> > ts = m_ses.find_collection(*i);
+
+				for (std::vector<boost::shared_ptr<torrent> >::iterator k = ts.begin()
+					, end(ts.end()); k != end; ++k)
+				{
+					// Only attempt to reuse files from torrents that are seeding.
+					// TODO: this could be optimized by looking up which files are
+					// complete and just look at those
+					if (!(*k)->is_seed()) continue;
+
+					res.match((*k)->get_torrent_copy(), (*k)->save_path());
+				}
+			}
+
+			std::vector<resolve_links::link_t> const& l = res.get_links();
+			if (!l.empty())
+			{
+				links.reset(new std::vector<std::string>(l.size()));
+				for (std::vector<resolve_links::link_t>::const_iterator i = l.begin()
+					, end(l.end()); i != end; ++i)
+				{
+					if (!i->ti) continue;
+
+					torrent_info const& ti = *i->ti;
+					std::string const& save_path = i->save_path;
+					links->push_back(combine_path(save_path
+						, ti.files().file_path(i->file_idx)));
+				}
+			}
+		}
+#endif // TORRENT_DISABLE_MUTABLE_TORRENTS
+
 		inc_refcount("check_fastresume");
+		// async_check_fastresume will release links
 		m_ses.disk_thread().async_check_fastresume(
 			m_storage.get(), m_resume_data ? &m_resume_data->node : NULL
-			, boost::bind(&torrent::on_resume_data_checked
+			, links, boost::bind(&torrent::on_resume_data_checked
 			, shared_from_this(), _1));
 #if defined TORRENT_LOGGING
 		debug_log("init, async_check_fastresume");
@@ -2443,9 +2500,10 @@ namespace libtorrent
 
 		m_resume_data.reset();
 
+		std::auto_ptr<std::vector<std::string> > links;
 		inc_refcount("force_recheck");
 		m_ses.disk_thread().async_check_fastresume(m_storage.get(), NULL
-			, boost::bind(&torrent::on_force_recheck
+			, links, boost::bind(&torrent::on_force_recheck
 			, shared_from_this(), _1));
 	}
 

src/torrent_info.cpp

@@ -752,6 +752,14 @@ namespace libtorrent
 		if (m_orig_files)
 			const_cast<file_storage&>(*m_orig_files).apply_pointer_offset(offset);
 
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+		for (int i = 0; i < m_collections.size(); ++i)
+			m_collections[i].first += offset;
+
+		for (int i = 0; i < m_similar_torrents.size(); ++i)
+			m_similar_torrents[i] += offset;
+#endif
+
 		if (m_info_dict)
 		{
 			// make this decoded object point to our copy of the info section
@@ -1273,7 +1281,6 @@ namespace libtorrent
 		std::string name;
 		sanitize_append_path_element(name, name_ent.string_ptr()
 			, name_ent.string_length());
-	
 		if (name.empty()) name = to_hex(m_info_hash.to_string());
 
 		// extract file list
@@ -1340,6 +1347,38 @@ namespace libtorrent
 
 		m_private = info.dict_find_int_value("private", 0);
 
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+		bdecode_node similar = info.dict_find_list("similar");
+		if (similar)
+		{
+			for (int i = 0; i < similar.list_size(); ++i)
+			{
+				if (similar.list_at(i).type() != bdecode_node::string_t)
+					continue;
+
+				if (similar.list_at(i).string_length() != 20)
+					continue;
+
+				m_similar_torrents.push_back(similar.list_at(i).string_ptr()
+					+ info_ptr_diff);
+			}
+		}
+
+		bdecode_node collections = info.dict_find_list("collections");
+		if (collections)
+		{
+			for (int i = 0; i < collections.list_size(); ++i)
+			{
+				bdecode_node str = collections.list_at(i);
+
+				if (str.type() != bdecode_node::string_t) continue;
+
+				m_collections.push_back(std::make_pair(str.string_ptr()
+					+ info_ptr_diff, str.string_length()));
+			}
+		}
+#endif // TORRENT_DISABLE_MUTABLE_TORRENTS
+
 		// now, commit the files structure we just parsed out
 		// into the torrent_info object.
 		// if we already have an m_files that's populated, it
@@ -1480,6 +1519,38 @@ namespace libtorrent
 		if (!parse_info_section(info, ec, flags)) return false;
 		resolve_duplicate_filenames();
 
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+		bdecode_node similar = torrent_file.dict_find_list("similar");
+		if (similar)
+		{
+			for (int i = 0; i < similar.list_size(); ++i)
+			{
+				if (similar.list_at(i).type() != bdecode_node::string_t)
+					continue;
+
+				if (similar.list_at(i).string_length() != 20)
+					continue;
+
+				m_owned_similar_torrents.push_back(
+					sha1_hash(similar.list_at(i).string_ptr()));
+			}
+		}
+
+		bdecode_node collections = torrent_file.dict_find_list("collections");
+		if (collections)
+		{
+			for (int i = 0; i < collections.list_size(); ++i)
+			{
+				bdecode_node str = collections.list_at(i);
+
+				if (str.type() != bdecode_node::string_t) continue;
+
+				m_owned_collections.push_back(std::string(str.string_ptr()
+					, str.string_length()));
+			}
+		}
+#endif // TORRENT_DISABLE_MUTABLE_TORRENTS
+
 		// extract the url of the tracker
 		bdecode_node i = torrent_file.dict_find_list("announce-list");
 		if (i)
@@ -1683,21 +1754,52 @@ namespace libtorrent
 #endif // TORRENT_NO_DEPRECATE
 
 	void torrent_info::add_url_seed(std::string const& url
-			, std::string const& ext_auth
-			, web_seed_entry::headers_t const& ext_headers)
+		, std::string const& ext_auth
+		, web_seed_entry::headers_t const& ext_headers)
 	{
 		m_web_seeds.push_back(web_seed_entry(url, web_seed_entry::url_seed
 			, ext_auth, ext_headers));
 	}
 
 	void torrent_info::add_http_seed(std::string const& url
-			, std::string const& auth
-			, web_seed_entry::headers_t const& extra_headers)
+		, std::string const& auth
+		, web_seed_entry::headers_t const& extra_headers)
 	{
 		m_web_seeds.push_back(web_seed_entry(url, web_seed_entry::http_seed
 			, auth, extra_headers));
 	}
 
+	std::vector<sha1_hash> torrent_info::similar_torrents() const
+	{
+		std::vector<sha1_hash> ret;
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+		ret.reserve(m_similar_torrents.size() + m_owned_similar_torrents.size());
+
+		for (int i = 0; i < m_similar_torrents.size(); ++i)
+			ret.push_back(sha1_hash(m_similar_torrents[i]));
+
+		for (int i = 0; i < m_owned_similar_torrents.size(); ++i)
+			ret.push_back(m_owned_similar_torrents[i]);
+#endif
+
+		return ret;
+	}
+
+	std::vector<std::string> torrent_info::collections() const
+	{
+		std::vector<std::string> ret;
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+		ret.reserve(m_collections.size() + m_owned_collections.size());
+
+		for (int i = 0; i < m_collections.size(); ++i)
+			ret.push_back(std::string(m_collections[i].first, m_collections[i].second));
+
+		for (int i = 0; i < m_owned_collections.size(); ++i)
+			ret.push_back(m_owned_collections[i]);
+#endif // TORRENT_DISABLE_MUTABLE_TORRENTS
+
+		return ret;
+	}
 
 #if !defined TORRENT_NO_DEPRECATE && TORRENT_USE_IOSTREAM
 // ------- start deprecation -------

test/Jamfile

@@ -95,6 +95,7 @@ feature launcher : none valgrind : composite ;
 feature.compose <launcher>valgrind : <testing.launcher>"valgrind --tool=memcheck -v --num-callers=20 --read-var-info=yes --track-origins=yes --error-exitcode=222 --suppressions=valgrind_suppressions.txt" <use-valgrind>on ;
 
 test-suite libtorrent : 	
+	[ run test_resolve_links.cpp ]
 	[ run test_crc32.cpp ]
 	[ run test_resume.cpp ]
 	[ run test_sliding_average.cpp ]

test/Makefile.am

@@ -113,6 +113,14 @@ EXTRA_DIST = Jamfile \
   test_torrents/empty_path_multi.torrent \
   test_torrents/duplicate_web_seeds.torrent \
   test_torrents/sample.torrent \
+  mutable_test_torrents/test1.torrent \
+  mutable_test_torrents/test1_pad_files.torrent \
+  mutable_test_torrents/test1_single.torrent\
+  mutable_test_torrents/test1_single_padded.torrent \
+  mutable_test_torrents/test2.torrent \
+  mutable_test_torrents/test2_pad_files.torrent \
+  mutable_test_torrents/test3.torrent \
+  mutable_test_torrents/test3_pad_files.torrent \
   eztv.xml \
   kat.xml \
   cb.xml \

test/mutable_test_torrents/test1.torrent

@@ -0,0 +1,3 @@
+d10:created by10:libtorrent13:creation datei1419452992e4:infod5:filesld6:lengthi51200e4:pathl1:aeed6:lengthi18e4:pathl1:beed6:lengthi19e4:pathl1:ceed6:lengthi53248e4:pathl1:deee4:name5:test112:piece lengthi16384e6:pieces140:ÈÄÔ}„Ç_ML¤)دym$„¶
+ÞŸÏk}@ûˆÑ<CB86>¹bÆ<62>®¸ßòü<C3B2>Ý ‡=åæâvw‰<77>¯Q\¯\tÿr§Ÿsx	<09>^ñzò57¸F¬ëW`žnøryܱt§VÈÛÀÔBôo­HQ<48>Ýk¦¥•ÄЂÜ÷—½UçôÀ6¤Q
+ñkª•kØà	t¢ee

test/mutable_test_torrents/test1_pad_files.torrent

@@ -0,0 +1,2 @@
+d10:created by10:libtorrent13:creation datei1419490165e4:infod5:filesld6:lengthi53248e4:pathl1:deed4:attr1:p6:lengthi12288e4:pathl17:.____padding_file1:0eed6:lengthi51200e4:pathl1:aeed4:attr1:p6:lengthi14336e4:pathl17:.____padding_file1:1eed6:lengthi19e4:pathl1:ceed6:lengthi18e4:pathl1:beee4:name5:test112:piece lengthi16384e6:pieces180:1QìêÄô«5ðOÓÛƒ»åÌ@…´<E280A6>îÍÜMsêt0üz€<7A>«F‹؉9™V{0Š“]y¥ÅrÞöv¤c>F†Ú×O½~zmÀNÈÄÔ}„Ç_ML¤)دym$„¶
+ÞŸÏk}@ûˆÑ<CB86>¹bÆ<62>®¸ßòü<C3B2>Ý ‡=åæâvw‰<77>¯Q“
âF?ó{¶ô<C2B6>+,/ß¼Gæ:¥k†të´Û‰Û±<C39B>geee

test/mutable_test_torrents/test1_single.torrent

@@ -0,0 +1,2 @@
+d10:created by10:libtorrent13:creation datei1419490700e4:infod6:lengthi51200e4:name1:a12:piece lengthi16384e6:pieces80:ÈÄÔ}„Ç_ML¤)دym$„¶
+ÞŸÏk}@ûˆÑ<CB86>¹bÆ<62>®¸ßòü<C3B2>Ý ‡=åæâvw‰<77>¯Q¦(R!¯+O0^+w÷I•Çee

test/mutable_test_torrents/test1_single_padded.torrent

@@ -0,0 +1,2 @@
+d10:created by10:libtorrent13:creation datei1419490711e4:infod6:lengthi51200e4:name1:a12:piece lengthi16384e6:pieces80:ÈÄÔ}„Ç_ML¤)دym$„¶
+ÞŸÏk}@ûˆÑ<CB86>¹bÆ<62>®¸ßòü<C3B2>Ý ‡=åæâvw‰<77>¯Q“
âF?ó{¶ô<C2B6>+,/ß¼Gee

test/mutable_test_torrents/test2.torrent

@@ -0,0 +1 @@
+d10:created by10:libtorrent13:creation datei1419452987e4:infod5:filesld6:lengthi18e4:pathl1:aeed6:lengthi51200e4:pathl1:beed6:lengthi19e4:pathl1:ceed6:lengthi46080e4:pathl1:deee4:name5:test212:piece lengthi16384e6:pieces120:Ή!utŠ	Ë%Ë<>0½¼£½ÍîiŒÇØ/%Äá¿O¸N´\´<< ú,3J«~ÊÛ,tS’žÊ
Ęì…@EÜi»›ú½Áƒ¯ˆQ<CB86>ôו…™7GFÖ쮶yê-I›å«Ç¿RbÁ0ˆòðgï<W׉Þ5%Ðee

test/mutable_test_torrents/test3_pad_files.torrent

@@ -0,0 +1,4 @@
+d10:created by10:libtorrent13:creation datei1419490178e4:infod5:filesld6:lengthi51200e4:pathl1:ceed4:attr1:p6:lengthi14336e4:pathl17:.____padding_file1:0eed6:lengthi51200e4:pathl1:deed4:attr1:p6:lengthi14336e4:pathl17:.____padding_file1:1eed6:lengthi19e4:pathl1:aeed6:lengthi18e4:pathl1:beee4:name5:test312:piece lengthi16384e6:pieces180:ÈÄÔ}„Ç_ML¤)دym$„¶
+ÞŸÏk}@ûˆÑ<CB86>¹bÆ<62>®¸ßòü<C3B2>Ý ‡=åæâvw‰<77>¯Q“
âF?ó{¶ô<C2B6>+,/ß¼G[°nÂ=s½'ÚRÞ;œ&¡•>à’hX¹¾âÆBVD8ô-..45k×qÝDÄ-
+fH„óì„”ýŽ†åƒ¸’Ky<EFBFBD>ûñ€ª
+A•,õ@¦¤f0kHOæ:¥k†të´Û‰Û±<C39B>geee

test/test_block_cache.cpp

@@ -74,10 +74,11 @@ struct test_storage_impl : storage_interface
 	virtual bool has_any_file(storage_error& ec) { return false; }
 	virtual void set_file_priority(std::vector<boost::uint8_t> const& prio
 		, storage_error& ec) {}
-	virtual int move_storage(std::string const& save_path, int flags, storage_error& ec)
-	{ return 0; }
-	virtual bool verify_resume_data(bdecode_node const& rd, storage_error& ec)
-	{ return true; }
+	virtual int move_storage(std::string const& save_path, int flags
+		, storage_error& ec) { return 0; }
+	virtual bool verify_resume_data(bdecode_node const& rd
+		, std::vector<std::string> const* links
+		, storage_error& ec) { return true; }
 	virtual void write_resume_data(entry& rd, storage_error& ec) const {}
 	virtual void release_files(storage_error& ec) {}
 	virtual void rename_file(int index, std::string const& new_filenamem

test/test_file_storage.cpp

@@ -207,6 +207,7 @@ int test_main()
 		TEST_EQUAL(file_hash, path_hash);
 	}
 
+	// TODO: test file_storage::optimize too
 	// TODO: test map_block
 	// TODO: test piece_size(int piece)
 	// TODO: test file_index_at_offset

test/test_resolve_links.cpp

@@ -0,0 +1,133 @@
+/*
+
+Copyright (c) 2014, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "test.hpp"
+#include "libtorrent/torrent_info.hpp"
+#include "libtorrent/resolve_links.hpp"
+#include "libtorrent/file.hpp" // for combine_path
+#include <boost/make_shared.hpp>
+#include <boost/bind.hpp>
+
+using namespace libtorrent;
+
+struct test_torrent_t
+{
+	char const* filename1;
+	char const* filename2;
+	int expected_matches;
+};
+
+test_torrent_t test_torrents[] = {
+	// no match because shared file in test2 and test3 is not padded/aligned
+	{ "test2", "test1_pad_files", 0},
+	{ "test3", "test1_pad_files", 0},
+
+	// in this case, test1 happens to have the shared file as the first one,
+	// which makes it padded, however, the tail of it isn't padded, so it
+	// still overlaps with the next file
+	{ "test1", "test1_pad_files", 0},
+
+	// test2 and test3 don't have the shared file aligned
+	{ "test2", "test1_pad_files", 0},
+	{ "test3", "test1_pad_files", 0},
+	{ "test2", "test1_single", 0},
+
+	// these are all padded. The first small file will accidentally also
+	// match, even though it's not tail padded, the following file is identical
+	{ "test2_pad_files", "test1_pad_files", 2},
+	{ "test3_pad_files", "test1_pad_files", 2},
+	{ "test3_pad_files", "test2_pad_files", 2},
+	{ "test1_pad_files", "test2_pad_files", 2},
+	{ "test1_pad_files", "test3_pad_files", 2},
+	{ "test2_pad_files", "test3_pad_files", 2},
+
+	// one might expect this to work, but since the tail of the single file
+	// torrent is not padded, the last piece hash won't match
+	{ "test1_pad_files", "test1_single", 0},
+
+	// if it's padded on the other hand, it will work
+	{ "test1_pad_files", "test1_single_padded", 1},
+
+	// TODO: test files with different piece size (negative test)
+};
+
+// TODO: it would be nice to test resolving of more than just 2 files as well.
+// like 3 single file torrents merged into one, resolving all 3 files.
+
+int test_main()
+{
+
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+	std::string path
+		= combine_path("..", "mutable_test_torrents");
+
+	for (int i = 0; i < sizeof(test_torrents)/sizeof(test_torrents[0]); ++i)
+	{
+		test_torrent_t const& e = test_torrents[i];
+
+		std::string p = combine_path(path, e.filename1) + ".torrent";
+		fprintf(stderr, "loading %s\n", p.c_str());
+		boost::shared_ptr<torrent_info> ti1 = boost::make_shared<torrent_info>(p);
+
+		p = combine_path(path, e.filename2) + ".torrent";
+		fprintf(stderr, "loading %s\n", p.c_str());
+		boost::shared_ptr<torrent_info> ti2 = boost::make_shared<torrent_info>(p);
+
+		fprintf(stderr, "resolving\n");
+		resolve_links l(ti1);
+		l.match(ti2, ".");
+
+		std::vector<resolve_links::link_t> const& links = l.get_links();
+
+		int num_matches = std::count_if(links.begin(), links.end()
+			, boost::bind(&resolve_links::link_t::ti, _1));
+
+		// some debug output in case the test fails
+		if (num_matches > e.expected_matches)
+		{
+			file_storage const& fs = ti1->files();
+			for (int i = 0; i < links.size(); ++i)
+			{
+				TORRENT_ASSERT(i < fs.num_files());
+				fprintf(stderr, "%s --> %s : %d\n", fs.file_name(i).c_str()
+					, links[i].ti ? to_hex(links[i].ti->info_hash()
+						.to_string()).c_str() : "", links[i].file_idx);
+			}
+		}
+
+		TEST_EQUAL(num_matches, e.expected_matches);
+
+	}
+#endif // TORRENT_DISABLE_MUTABLE_TORRENTS
+	return 0;
+}
+

test/test_storage.cpp

@@ -460,7 +460,9 @@ void test_check_files(std::string const& test_path
 
 	bool done = false;
 	bdecode_node frd;
-	io.async_check_fastresume(pm.get(), &frd, boost::bind(&on_check_resume_data, _1, &done));
+	std::auto_ptr<std::vector<std::string> > links;
+	io.async_check_fastresume(pm.get(), &frd, links
+		, boost::bind(&on_check_resume_data, _1, &done));
 	io.submit_jobs();
 	ios.reset();
 	run_until(ios, done);

test/test_torrent_info.cpp

@@ -43,6 +43,46 @@ POSSIBILITY OF SUCH DAMAGE.
 
 using namespace libtorrent;
 
+void test_mutable_torrents()
+{
+	file_storage fs;
+
+	fs.add_file("test/temporary.txt", 0x4000);
+
+	libtorrent::create_torrent t(fs, 0x4000);
+
+	// calculate the hash for all pieces
+	int num = t.num_pieces();
+	sha1_hash ph;
+	for (int i = 0; i < num; ++i)
+		t.set_hash(i, ph);
+
+	t.add_collection("collection1");
+	t.add_collection("collection2");
+
+	t.add_similar_torrent(sha1_hash("abababababababababab"));
+	t.add_similar_torrent(sha1_hash("babababababababababa"));
+
+	std::vector<char> tmp;
+	std::back_insert_iterator<std::vector<char> > out(tmp);
+
+	entry tor = t.generate();
+	bencode(out, tor);
+
+	torrent_info ti(&tmp[0], tmp.size());
+
+	std::vector<sha1_hash> similar;
+	similar.push_back(sha1_hash("abababababababababab"));
+	similar.push_back(sha1_hash("babababababababababa"));
+
+	std::vector<std::string> collections;
+	collections.push_back("collection1");
+	collections.push_back("collection2");
+
+	TEST_CHECK(similar == ti.similar_torrents());
+	TEST_CHECK(collections == ti.collections());
+}
+
 struct test_torrent_t
 {
 	char const* file;
@@ -654,6 +694,9 @@ int test_main()
 {
 	test_resolve_duplicates();
 	test_copy();
+#ifndef TORRENT_DISABLE_MUTABLE_TORRENTS
+	test_mutable_torrents();
+#endif
 	test_torrent_parse();
 
 	return 0;