added support for generating torrent files with padding files in them

ChangeLog

@@ -22,6 +22,7 @@
 	* added monitoring of the DHT lookups
 	* added bandwidth reports for estimated TCP/IP overhead and DHT
 	* includes DHT traffic in the rate limiter
+	* added support for bitcomet padding files
 
 release 0.14.2
 

docs/make_torrent.rst

@@ -118,8 +118,8 @@ file structure. Its synopsis::
 		bool is_valid() const;
 
 		void add_file(file_entry const& e);
-		void add_file(fs::path const& p, size_type size);
-		void add_file(fs::wpath const& p, size_type size);
+		void add_file(fs::path const& p, size_type size, bool pad_file = false);
+		void add_file(fs::wpath const& p, size_type size, bool pad_file = false);
 		void rename_file(int index, std::string const& new_filename);
 		void rename_file(int index, std::wstring const& new_filename);
 
@@ -161,8 +161,7 @@ The ``create_torrent`` class has the following synopsis::
 
 	struct create_torrent
 	{
-		create_torrent(file_storage& fs, int piece_size);
-		create_torrent(file_storage& fs);
+		create_torrent(file_storage& fs, int piece_size = 0, int pad_size_limit = -1);
 		create_torrent(torrent_info const& ti);
 
 		entry generate() const;
@@ -188,15 +187,16 @@ create_torrent()
 
 	::
 
-		create_torrent(file_storage& fs, int piece_size);
-		create_torrent(file_storage& fs);
+		create_torrent(file_storage& fs, int piece_size = 0, int pad_size_limit = -1);
 		create_torrent(torrent_info const& ti);
 
 The ``piece_size`` is the size of each piece in bytes. It must
-be a multiple of 16 kiB.
+be a multiple of 16 kiB. If a piece size of 0 is specified, a
+piece_size will becalculated such that the torrent file is roughly 40 kB.
 
-The constructor that does not take a piece_size will calculate
-a piece size such that the torrent file is roughly 40 kB.
+If a ``pad_size_limit`` is specified (other than -1), any file larger than
+the specified number of bytes will be preceeded by a pad file to align it
+with the start od a piece.
 
 The overlad that takes a ``torrent_info`` object will make a verbatim
 copy of its info dictionary (to preserve the info-hash). The copy of

examples/make_torrent.cpp

@@ -65,19 +65,33 @@ void print_progress(int i, int num)
 	std::cerr << "\r" << (i+1) << "/" << num;
 }
 
+void print_usage()
+{
+	std::cerr << "usage: make_torrent FILE [OPTIONS]\n"
+		"\n"
+		"Generates a torrent file from the specified file\n"
+		"or directory and writes it to standard out\n\n"
+		"OPTIONS:\n"
+		"-w url      adds a web seed to the torrent with\n"
+		"            the specified url\n"
+		"-t url      adds the specified tracker to the\n"
+		"            torrent\n"
+		"-p bytes    enables padding files. Files larger\n"
+		"            than bytes will be piece-aligned\n"
+		"-s bytes    specifies a piece size for the torrent\n"
+		"            This has to be a multiple of 16 kiB\n";
+}
+
 int main(int argc, char* argv[])
 {
 	using namespace libtorrent;
 	using namespace boost::filesystem;
 
-	int piece_size = 256 * 1024;
 	char const* creator_str = "libtorrent";
 
-	if (argc != 4 && argc != 5)
+	if (argc < 2)
 	{
-		std::cerr << "usage: make_torrent <output torrent-file> "
-			"<announce url> <file or directory to create torrent from> "
-			"[url-seed]\n";
+		print_usage();
 		return 1;
 	}
 
@@ -85,24 +99,65 @@ int main(int argc, char* argv[])
 	try
 	{
 #endif
+		std::vector<std::string> web_seeds;
+		std::vector<std::string> trackers;
+		int pad_file_limit = -1;
+		int piece_size = 0;
+
+		for (int i = 2; i < argc; ++i)
+		{
+			if (argv[i][0] != '-')
+			{
+				print_usage();
+				return 1;
+			}
+
+			switch (argv[i][1])
+			{
+				case 'w':
+					++i;
+					web_seeds.push_back(argv[i]);
+					break;
+				case 't':
+					++i;
+					trackers.push_back(argv[i]);
+					break;
+				case 'p':
+					++i;
+					pad_file_limit = atoi(argv[i]);
+					break;
+				case 's':
+					++i;
+					piece_size = atoi(argv[i]);
+					break;
+				default:
+					print_usage();
+					return 1;
+			}
+		}
+
 		file_storage fs;
 		file_pool fp;
-		path full_path = complete(path(argv[3]));
+		path full_path = complete(path(argv[1]));
 
 		add_files(fs, full_path, file_filter);
 
-		create_torrent t(fs, piece_size);
-		t.add_tracker(argv[2]);
+		create_torrent t(fs, piece_size, pad_file_limit);
+		for (std::vector<std::string>::iterator i = trackers.begin()
+			, end(trackers.end()); i != end; ++i)
+			t.add_tracker(*i);
+
+		for (std::vector<std::string>::iterator i = web_seeds.begin()
+			, end(web_seeds.end()); i != end; ++i)
+			t.add_url_seed(*i);
+
 		set_piece_hashes(t, full_path.branch_path()
 			, boost::bind(&print_progress, _1, t.num_pieces()));
 		std::cerr << std::endl;
 		t.set_creator(creator_str);
 
-		if (argc == 5) t.add_url_seed(argv[4]);
-
-		// create the torrent and print it to out
-		ofstream out(complete(path(argv[1])), std::ios_base::binary);
-		bencode(std::ostream_iterator<char>(out), t.generate());
+		// create the torrent and print it to stdout
+		bencode(std::ostream_iterator<char>(std::cout), t.generate());
 #ifndef BOOST_NO_EXCEPTIONS
 	}
 	catch (std::exception& e)

include/libtorrent/create_torrent.hpp

@@ -68,8 +68,7 @@ namespace libtorrent
 
 	struct TORRENT_EXPORT create_torrent
 	{
-		create_torrent(file_storage& fs, int piece_size);
-		create_torrent(file_storage& fs);
+		create_torrent(file_storage& fs, int piece_size = 0, int pad_file_limit = -1);
 		create_torrent(torrent_info const& ti);
 		entry generate() const;
 

include/libtorrent/file_storage.hpp

@@ -111,7 +111,7 @@ namespace libtorrent
 			return m_files[index];
 		}
 		
-		size_type total_size() const { TORRENT_ASSERT(m_piece_length > 0); return m_total_size; }
+		size_type total_size() const { return m_total_size; }
 		void set_num_pieces(int n) { m_num_pieces = n; }
 		int num_pieces() const { TORRENT_ASSERT(m_piece_length > 0); return m_num_pieces; }
 		void set_piece_length(int l)  { m_piece_length = l; }
@@ -132,6 +132,11 @@ namespace libtorrent
 			swap(ti.m_name, m_name);
 		}
 
+		// if pad_file_limit >= 0, files larger than
+		// that limit will be padded, default is to
+		// not add any padding
+		void optimize(int pad_file_limit = -1);
+
 	private:
 		int m_piece_length;
 

include/libtorrent/storage.hpp

@@ -116,6 +116,7 @@ namespace libtorrent
 
 	struct TORRENT_EXPORT storage_interface
 	{
+		storage_interface(): m_io_thread(0) {}
 		// create directories and set file sizes
 		// if allocate_files is true. 
 		// allocate_files is true if allocation mode
@@ -167,7 +168,7 @@ namespace libtorrent
 		// non-zero return value indicates an error
 		virtual bool delete_files() = 0;
 
-		disk_io_thread& io_thread() { return *m_io_thread; }
+		disk_io_thread* io_thread() { return m_io_thread; }
 
 		void set_error(boost::filesystem::path const& file, error_code const& ec) const
 		{

src/create_torrent.cpp

@@ -42,7 +42,7 @@ namespace gr = boost::gregorian;
 
 namespace libtorrent
 {
-	create_torrent::create_torrent(file_storage& fs, int size)
+	create_torrent::create_torrent(file_storage& fs, int piece_size, int pad_file_limit)
 		: m_files(fs)
 		, m_creation_date(pt::second_clock::universal_time())
 		, m_multifile(fs.num_files() > 1)
@@ -55,47 +55,33 @@ namespace libtorrent
 		if (!m_multifile && m_files.at(0).path.has_parent_path()) m_multifile = true;
 #endif
 
+		// a piece_size of 0 means automatic
+		if (piece_size == 0)
+		{
+			const int target_size = 40 * 1024;
+			piece_size = fs.total_size() / (target_size / 20);
+	
+			for (int i = 2*1024*1024; i >= 16*1024; i /= 2)
+			{
+				if (piece_size < i) continue;
+				piece_size = i;
+				break;
+			}
+		}
+
 		// make sure the size is an even power of 2
 #ifndef NDEBUG
 		for (int i = 0; i < 32; ++i)
 		{
-			if (size & (1 << i))
+			if (piece_size & (1 << i))
 			{
-				TORRENT_ASSERT((size & ~(1 << i)) == 0);
+				TORRENT_ASSERT((piece_size & ~(1 << i)) == 0);
 				break;
 			}
 		}
 #endif
-		m_files.set_piece_length(size);
-		m_files.set_num_pieces(static_cast<int>(
-			(m_files.total_size() + m_files.piece_length() - 1) / m_files.piece_length()));
-		m_piece_hash.resize(m_files.num_pieces());
-	}
-
-	create_torrent::create_torrent(file_storage& fs)
-		: m_files(fs)
-		, m_creation_date(pt::second_clock::universal_time())
-		, m_multifile(fs.num_files() > 1)
-		, m_private(false)
-	{
-		TORRENT_ASSERT(fs.num_files() > 0);
-#if BOOST_VERSION < 103600
-		if (!m_multifile && m_files.at(0).path.has_branch_path()) m_multifile = true;
-#else
-		if (!m_multifile && m_files.at(0).path.has_parent_path()) m_multifile = true;
-#endif
-
-		const int target_size = 40 * 1024;
-		int size = fs.total_size() / (target_size / 20);
-	
-		for (int i = 4*1024*1024; i > 16*1024; i /= 2)
-		{
-			if (size < i) continue;
-			size = i;
-			break;
-		}
-
-		m_files.set_piece_length(size);
+		m_files.set_piece_length(piece_size);
+		m_files.optimize(pad_file_limit);
 		m_files.set_num_pieces(static_cast<int>(
 			(m_files.total_size() + m_files.piece_length() - 1) / m_files.piece_length()));
 		m_piece_hash.resize(m_files.num_pieces());

src/file_storage.cpp

@@ -34,6 +34,7 @@ POSSIBILITY OF SUCH DAMAGE.
 
 #include "libtorrent/file_storage.hpp"
 #include "libtorrent/utf8.hpp"
+#include <boost/bind.hpp>
 
 
 namespace libtorrent
@@ -186,5 +187,81 @@ namespace libtorrent
 	{
 		add_file(e.path, e.size, e.pad_file);
 	}
+
+	void file_storage::optimize(int pad_file_limit)
+	{
+		// the main purpuse of padding is to optimize disk
+		// I/O. This is a conservative memory page size assumption
+		int alignment = 8*1024;
+
+		// it doesn't make any sense to pad files that
+		// are smaller than one piece
+		if (pad_file_limit >= 0 && pad_file_limit < alignment)
+			pad_file_limit = alignment;
+
+		// put the largest file at the front, to make sure
+		// it's aligned
+		std::vector<file_entry>::iterator i = std::max_element(m_files.begin(), m_files.end()
+			, boost::bind(&file_entry::size, _1) < boost::bind(&file_entry::size, _2));
+
+		using std::iter_swap;
+		iter_swap(i, m_files.begin());
+
+		size_type off = 0;
+		int padding_file = 0;
+		for (std::vector<file_entry>::iterator i = m_files.begin();
+			i != m_files.end(); ++i)
+		{
+			if (pad_file_limit >= 0
+				&& (off & (alignment-1)) != 0
+				&& i->size > pad_file_limit)
+			{
+				// if we have pad files enabled, and this file is
+				// not piece-aligned and the file size exceeds the
+				// limit, so add a padding file in front of it
+				int pad_size = alignment - (off & (alignment-1));
+				
+				// find the largest file that fits in pad_size
+				std::vector<file_entry>::iterator best_match = m_files.end();
+				for (std::vector<file_entry>::iterator j = i+1; j < m_files.end(); ++j)
+				{
+					if (j->size > pad_size) continue;
+					if (best_match == m_files.end() || j->size > best_match->size)
+						best_match = j;
+				}
+
+				if (best_match != m_files.end())
+				{
+					// we found one
+					file_entry e = *best_match;
+					m_files.erase(best_match);
+					i = m_files.insert(i, e);
+					i->offset = off;
+					off += i->size;
+					continue;
+				}
+
+				// we could not find a file that fits in pad_size
+				// add a padding file
+
+				file_entry e;
+				i = m_files.insert(i, e);
+				i->size = pad_size;
+				i->offset = off;
+				i->file_base = 0;
+				char name[10];
+				sprintf(name, "%d", padding_file);
+				i->path = *(i+1)->path.begin();
+				i->path /= "_____padding_file_";
+				i->path /= name;
+				off += pad_size;
+				++padding_file;
+				++i;
+			}
+			i->offset = off;
+			off += i->size;
+		}
+		m_total_size = off;
+	}
 }
 

src/storage.cpp

@@ -454,13 +454,15 @@ namespace libtorrent
 		int slot_size = piece_size - ph.offset;
 		if (slot_size > 0)
 		{
+			int block_size = 16 * 1024;
+			if (io_thread()) block_size = io_thread()->block_size();
 			int size = slot_size;
-			int num_blocks = (size + io_thread().block_size() - 1) / io_thread().block_size();
+			int num_blocks = (size + block_size - 1) / block_size;
 			file::iovec_t* bufs = TORRENT_ALLOCA(file::iovec_t, num_blocks);
 			for (int i = 0; i < num_blocks; ++i)
 			{
-				bufs[i].iov_base = io_thread().allocate_buffer();
-				bufs[i].iov_len = (std::min)(io_thread().block_size(), size);
+				bufs[i].iov_base = io_thread()->allocate_buffer();
+				bufs[i].iov_len = (std::min)(block_size, size);
 				size -= bufs[i].iov_len;
 			}
 			readv(bufs, slot, ph.offset, num_blocks);
@@ -468,7 +470,7 @@ namespace libtorrent
 			for (int i = 0; i < num_blocks; ++i)
 			{
 				ph.h.update((char const*)bufs[i].iov_base, bufs[i].iov_len);
-				io_thread().free_buffer((char*)bufs[i].iov_base);
+				io_thread()->free_buffer((char*)bufs[i].iov_base);
 			}
 			if (error()) return sha1_hash(0);
 		}
@@ -526,7 +528,8 @@ namespace libtorrent
 			{
 				error_code ec;
 				int mode = file::read_write;
-				if (io_thread().no_buffer()
+				if (io_thread()
+					&& io_thread()->no_buffer()
 					&& ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0)
 					mode |= file::no_buffer;
 				boost::shared_ptr<file> f = m_pool.open_file(this
@@ -915,22 +918,22 @@ namespace libtorrent
 #endif
 
 #define TORRENT_ALLOCATE_BLOCKS(bufs, num_blocks, piece_size) \
-	int num_blocks = (piece_size + io_thread().block_size() - 1) / io_thread().block_size(); \
+	int num_blocks = (piece_size + io_thread()->block_size() - 1) / io_thread()->block_size(); \
 	file::iovec_t* bufs = TORRENT_ALLOCA(file::iovec_t, num_blocks); \
 	for (int i = 0, size = piece_size; i < num_blocks; ++i) \
 	{ \
-		bufs[i].iov_base = io_thread().allocate_buffer(); \
-		bufs[i].iov_len = (std::min)(io_thread().block_size(), size); \
+		bufs[i].iov_base = io_thread()->allocate_buffer(); \
+		bufs[i].iov_len = (std::min)(io_thread()->block_size(), size); \
 		size -= bufs[i].iov_len; \
 	}
 
 #define TORRENT_FREE_BLOCKS(bufs, num_blocks) \
 	for (int i = 0; i < num_blocks; ++i) \
-		io_thread().free_buffer((char*)bufs[i].iov_base);
+		io_thread()->free_buffer((char*)bufs[i].iov_base);
 
 #define TORRENT_SET_SIZE(bufs, size, num_bufs) \
-	for (num_bufs = 0; size > 0; size -= io_thread().block_size(), ++num_bufs) \
-		bufs[num_bufs].iov_len = (std::min)(io_thread().block_size(), size)
+	for (num_bufs = 0; size > 0; size -= io_thread()->block_size(), ++num_bufs) \
+		bufs[num_bufs].iov_len = (std::min)(io_thread()->block_size(), size)
 	
 
 	bool storage::move_slot(int src_slot, int dst_slot)
@@ -954,7 +957,6 @@ ret:
 		bool r = true;
 
 		// the size of the target slot is the size of the piece
-		int piece_size = m_files.piece_length();
 		int piece1_size = m_files.piece_size(slot2);
 		int piece2_size = m_files.piece_size(slot1);
 
@@ -1098,7 +1100,8 @@ ret:
 
 			error_code ec;
 			int mode = file::read_only;
-			if (io_thread().no_buffer()
+			if (io_thread()
+				&& io_thread()->no_buffer()
 				&& ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0)
 				mode |= file::no_buffer;
 			in = m_pool.open_file(this, path, mode, ec);
@@ -1241,7 +1244,8 @@ ret:
 
 			error_code ec;
 			int mode = file::read_write;
-			if (io_thread().no_buffer()
+			if (io_thread()
+				&& io_thread()->no_buffer()
 				&& ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0)
 				mode |= file::no_buffer;
 			out = m_pool.open_file(this, path, mode, ec);

src/torrent_info.cpp

@@ -219,11 +219,7 @@ namespace
 
 		// bitcomet pad file
 
-#if BOOST_VERSION < 103600
-		if (target.path.leaf().substr(0, 18) == "_____padding_file_")
-#else
-		if (target.path.filename().substr(0, 18) == "_____padding_file_")
-#endif
+		if (target.path.string().find("_____padding_file_") != std::string::npos)
 			target.pad_file = true;
 
 		return true;