From eea890de11ae1339c36237c53881bda270bf5593 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Sun, 11 Jan 2009 10:32:57 +0000 Subject: [PATCH] added support for generating torrent files with padding files in them --- ChangeLog | 1 + docs/make_torrent.rst | 18 +++--- examples/make_torrent.cpp | 81 ++++++++++++++++++++++----- include/libtorrent/create_torrent.hpp | 3 +- include/libtorrent/file_storage.hpp | 7 ++- include/libtorrent/storage.hpp | 3 +- src/create_torrent.cpp | 52 +++++++---------- src/file_storage.cpp | 77 +++++++++++++++++++++++++ src/storage.cpp | 32 ++++++----- src/torrent_info.cpp | 6 +- 10 files changed, 202 insertions(+), 78 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1e9a8eef1..62c48a7d1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,7 @@ * added monitoring of the DHT lookups * added bandwidth reports for estimated TCP/IP overhead and DHT * includes DHT traffic in the rate limiter + * added support for bitcomet padding files release 0.14.2 diff --git a/docs/make_torrent.rst b/docs/make_torrent.rst index e33a2893d..9fbdc98ba 100644 --- a/docs/make_torrent.rst +++ b/docs/make_torrent.rst @@ -118,8 +118,8 @@ file structure. Its synopsis:: bool is_valid() const; void add_file(file_entry const& e); - void add_file(fs::path const& p, size_type size); - void add_file(fs::wpath const& p, size_type size); + void add_file(fs::path const& p, size_type size, bool pad_file = false); + void add_file(fs::wpath const& p, size_type size, bool pad_file = false); void rename_file(int index, std::string const& new_filename); void rename_file(int index, std::wstring const& new_filename); @@ -161,8 +161,7 @@ The ``create_torrent`` class has the following synopsis:: struct create_torrent { - create_torrent(file_storage& fs, int piece_size); - create_torrent(file_storage& fs); + create_torrent(file_storage& fs, int piece_size = 0, int pad_size_limit = -1); create_torrent(torrent_info const& ti); entry generate() const; @@ -188,15 +187,16 @@ create_torrent() :: - create_torrent(file_storage& fs, int piece_size); - create_torrent(file_storage& fs); + create_torrent(file_storage& fs, int piece_size = 0, int pad_size_limit = -1); create_torrent(torrent_info const& ti); The ``piece_size`` is the size of each piece in bytes. It must -be a multiple of 16 kiB. +be a multiple of 16 kiB. If a piece size of 0 is specified, a +piece_size will becalculated such that the torrent file is roughly 40 kB. -The constructor that does not take a piece_size will calculate -a piece size such that the torrent file is roughly 40 kB. +If a ``pad_size_limit`` is specified (other than -1), any file larger than +the specified number of bytes will be preceeded by a pad file to align it +with the start od a piece. The overlad that takes a ``torrent_info`` object will make a verbatim copy of its info dictionary (to preserve the info-hash). The copy of diff --git a/examples/make_torrent.cpp b/examples/make_torrent.cpp index 1b9ff7cdb..21133a1fe 100644 --- a/examples/make_torrent.cpp +++ b/examples/make_torrent.cpp @@ -65,19 +65,33 @@ void print_progress(int i, int num) std::cerr << "\r" << (i+1) << "/" << num; } +void print_usage() +{ + std::cerr << "usage: make_torrent FILE [OPTIONS]\n" + "\n" + "Generates a torrent file from the specified file\n" + "or directory and writes it to standard out\n\n" + "OPTIONS:\n" + "-w url adds a web seed to the torrent with\n" + " the specified url\n" + "-t url adds the specified tracker to the\n" + " torrent\n" + "-p bytes enables padding files. Files larger\n" + " than bytes will be piece-aligned\n" + "-s bytes specifies a piece size for the torrent\n" + " This has to be a multiple of 16 kiB\n"; +} + int main(int argc, char* argv[]) { using namespace libtorrent; using namespace boost::filesystem; - int piece_size = 256 * 1024; char const* creator_str = "libtorrent"; - if (argc != 4 && argc != 5) + if (argc < 2) { - std::cerr << "usage: make_torrent " - " " - "[url-seed]\n"; + print_usage(); return 1; } @@ -85,24 +99,65 @@ int main(int argc, char* argv[]) try { #endif + std::vector web_seeds; + std::vector trackers; + int pad_file_limit = -1; + int piece_size = 0; + + for (int i = 2; i < argc; ++i) + { + if (argv[i][0] != '-') + { + print_usage(); + return 1; + } + + switch (argv[i][1]) + { + case 'w': + ++i; + web_seeds.push_back(argv[i]); + break; + case 't': + ++i; + trackers.push_back(argv[i]); + break; + case 'p': + ++i; + pad_file_limit = atoi(argv[i]); + break; + case 's': + ++i; + piece_size = atoi(argv[i]); + break; + default: + print_usage(); + return 1; + } + } + file_storage fs; file_pool fp; - path full_path = complete(path(argv[3])); + path full_path = complete(path(argv[1])); add_files(fs, full_path, file_filter); - create_torrent t(fs, piece_size); - t.add_tracker(argv[2]); + create_torrent t(fs, piece_size, pad_file_limit); + for (std::vector::iterator i = trackers.begin() + , end(trackers.end()); i != end; ++i) + t.add_tracker(*i); + + for (std::vector::iterator i = web_seeds.begin() + , end(web_seeds.end()); i != end; ++i) + t.add_url_seed(*i); + set_piece_hashes(t, full_path.branch_path() , boost::bind(&print_progress, _1, t.num_pieces())); std::cerr << std::endl; t.set_creator(creator_str); - if (argc == 5) t.add_url_seed(argv[4]); - - // create the torrent and print it to out - ofstream out(complete(path(argv[1])), std::ios_base::binary); - bencode(std::ostream_iterator(out), t.generate()); + // create the torrent and print it to stdout + bencode(std::ostream_iterator(std::cout), t.generate()); #ifndef BOOST_NO_EXCEPTIONS } catch (std::exception& e) diff --git a/include/libtorrent/create_torrent.hpp b/include/libtorrent/create_torrent.hpp index f7899e56b..58a2155ce 100644 --- a/include/libtorrent/create_torrent.hpp +++ b/include/libtorrent/create_torrent.hpp @@ -68,8 +68,7 @@ namespace libtorrent struct TORRENT_EXPORT create_torrent { - create_torrent(file_storage& fs, int piece_size); - create_torrent(file_storage& fs); + create_torrent(file_storage& fs, int piece_size = 0, int pad_file_limit = -1); create_torrent(torrent_info const& ti); entry generate() const; diff --git a/include/libtorrent/file_storage.hpp b/include/libtorrent/file_storage.hpp index 9087e65d2..1a4b79dca 100644 --- a/include/libtorrent/file_storage.hpp +++ b/include/libtorrent/file_storage.hpp @@ -111,7 +111,7 @@ namespace libtorrent return m_files[index]; } - size_type total_size() const { TORRENT_ASSERT(m_piece_length > 0); return m_total_size; } + size_type total_size() const { return m_total_size; } void set_num_pieces(int n) { m_num_pieces = n; } int num_pieces() const { TORRENT_ASSERT(m_piece_length > 0); return m_num_pieces; } void set_piece_length(int l) { m_piece_length = l; } @@ -132,6 +132,11 @@ namespace libtorrent swap(ti.m_name, m_name); } + // if pad_file_limit >= 0, files larger than + // that limit will be padded, default is to + // not add any padding + void optimize(int pad_file_limit = -1); + private: int m_piece_length; diff --git a/include/libtorrent/storage.hpp b/include/libtorrent/storage.hpp index 45930e284..33d9111f2 100644 --- a/include/libtorrent/storage.hpp +++ b/include/libtorrent/storage.hpp @@ -116,6 +116,7 @@ namespace libtorrent struct TORRENT_EXPORT storage_interface { + storage_interface(): m_io_thread(0) {} // create directories and set file sizes // if allocate_files is true. // allocate_files is true if allocation mode @@ -167,7 +168,7 @@ namespace libtorrent // non-zero return value indicates an error virtual bool delete_files() = 0; - disk_io_thread& io_thread() { return *m_io_thread; } + disk_io_thread* io_thread() { return m_io_thread; } void set_error(boost::filesystem::path const& file, error_code const& ec) const { diff --git a/src/create_torrent.cpp b/src/create_torrent.cpp index 9eb617dad..f6b1068a1 100644 --- a/src/create_torrent.cpp +++ b/src/create_torrent.cpp @@ -42,7 +42,7 @@ namespace gr = boost::gregorian; namespace libtorrent { - create_torrent::create_torrent(file_storage& fs, int size) + create_torrent::create_torrent(file_storage& fs, int piece_size, int pad_file_limit) : m_files(fs) , m_creation_date(pt::second_clock::universal_time()) , m_multifile(fs.num_files() > 1) @@ -55,47 +55,33 @@ namespace libtorrent if (!m_multifile && m_files.at(0).path.has_parent_path()) m_multifile = true; #endif + // a piece_size of 0 means automatic + if (piece_size == 0) + { + const int target_size = 40 * 1024; + piece_size = fs.total_size() / (target_size / 20); + + for (int i = 2*1024*1024; i >= 16*1024; i /= 2) + { + if (piece_size < i) continue; + piece_size = i; + break; + } + } + // make sure the size is an even power of 2 #ifndef NDEBUG for (int i = 0; i < 32; ++i) { - if (size & (1 << i)) + if (piece_size & (1 << i)) { - TORRENT_ASSERT((size & ~(1 << i)) == 0); + TORRENT_ASSERT((piece_size & ~(1 << i)) == 0); break; } } #endif - m_files.set_piece_length(size); - m_files.set_num_pieces(static_cast( - (m_files.total_size() + m_files.piece_length() - 1) / m_files.piece_length())); - m_piece_hash.resize(m_files.num_pieces()); - } - - create_torrent::create_torrent(file_storage& fs) - : m_files(fs) - , m_creation_date(pt::second_clock::universal_time()) - , m_multifile(fs.num_files() > 1) - , m_private(false) - { - TORRENT_ASSERT(fs.num_files() > 0); -#if BOOST_VERSION < 103600 - if (!m_multifile && m_files.at(0).path.has_branch_path()) m_multifile = true; -#else - if (!m_multifile && m_files.at(0).path.has_parent_path()) m_multifile = true; -#endif - - const int target_size = 40 * 1024; - int size = fs.total_size() / (target_size / 20); - - for (int i = 4*1024*1024; i > 16*1024; i /= 2) - { - if (size < i) continue; - size = i; - break; - } - - m_files.set_piece_length(size); + m_files.set_piece_length(piece_size); + m_files.optimize(pad_file_limit); m_files.set_num_pieces(static_cast( (m_files.total_size() + m_files.piece_length() - 1) / m_files.piece_length())); m_piece_hash.resize(m_files.num_pieces()); diff --git a/src/file_storage.cpp b/src/file_storage.cpp index f34f15346..30e7b0656 100644 --- a/src/file_storage.cpp +++ b/src/file_storage.cpp @@ -34,6 +34,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/file_storage.hpp" #include "libtorrent/utf8.hpp" +#include namespace libtorrent @@ -186,5 +187,81 @@ namespace libtorrent { add_file(e.path, e.size, e.pad_file); } + + void file_storage::optimize(int pad_file_limit) + { + // the main purpuse of padding is to optimize disk + // I/O. This is a conservative memory page size assumption + int alignment = 8*1024; + + // it doesn't make any sense to pad files that + // are smaller than one piece + if (pad_file_limit >= 0 && pad_file_limit < alignment) + pad_file_limit = alignment; + + // put the largest file at the front, to make sure + // it's aligned + std::vector::iterator i = std::max_element(m_files.begin(), m_files.end() + , boost::bind(&file_entry::size, _1) < boost::bind(&file_entry::size, _2)); + + using std::iter_swap; + iter_swap(i, m_files.begin()); + + size_type off = 0; + int padding_file = 0; + for (std::vector::iterator i = m_files.begin(); + i != m_files.end(); ++i) + { + if (pad_file_limit >= 0 + && (off & (alignment-1)) != 0 + && i->size > pad_file_limit) + { + // if we have pad files enabled, and this file is + // not piece-aligned and the file size exceeds the + // limit, so add a padding file in front of it + int pad_size = alignment - (off & (alignment-1)); + + // find the largest file that fits in pad_size + std::vector::iterator best_match = m_files.end(); + for (std::vector::iterator j = i+1; j < m_files.end(); ++j) + { + if (j->size > pad_size) continue; + if (best_match == m_files.end() || j->size > best_match->size) + best_match = j; + } + + if (best_match != m_files.end()) + { + // we found one + file_entry e = *best_match; + m_files.erase(best_match); + i = m_files.insert(i, e); + i->offset = off; + off += i->size; + continue; + } + + // we could not find a file that fits in pad_size + // add a padding file + + file_entry e; + i = m_files.insert(i, e); + i->size = pad_size; + i->offset = off; + i->file_base = 0; + char name[10]; + sprintf(name, "%d", padding_file); + i->path = *(i+1)->path.begin(); + i->path /= "_____padding_file_"; + i->path /= name; + off += pad_size; + ++padding_file; + ++i; + } + i->offset = off; + off += i->size; + } + m_total_size = off; + } } diff --git a/src/storage.cpp b/src/storage.cpp index 42c83f339..0fadab6a7 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -454,13 +454,15 @@ namespace libtorrent int slot_size = piece_size - ph.offset; if (slot_size > 0) { + int block_size = 16 * 1024; + if (io_thread()) block_size = io_thread()->block_size(); int size = slot_size; - int num_blocks = (size + io_thread().block_size() - 1) / io_thread().block_size(); + int num_blocks = (size + block_size - 1) / block_size; file::iovec_t* bufs = TORRENT_ALLOCA(file::iovec_t, num_blocks); for (int i = 0; i < num_blocks; ++i) { - bufs[i].iov_base = io_thread().allocate_buffer(); - bufs[i].iov_len = (std::min)(io_thread().block_size(), size); + bufs[i].iov_base = io_thread()->allocate_buffer(); + bufs[i].iov_len = (std::min)(block_size, size); size -= bufs[i].iov_len; } readv(bufs, slot, ph.offset, num_blocks); @@ -468,7 +470,7 @@ namespace libtorrent for (int i = 0; i < num_blocks; ++i) { ph.h.update((char const*)bufs[i].iov_base, bufs[i].iov_len); - io_thread().free_buffer((char*)bufs[i].iov_base); + io_thread()->free_buffer((char*)bufs[i].iov_base); } if (error()) return sha1_hash(0); } @@ -526,7 +528,8 @@ namespace libtorrent { error_code ec; int mode = file::read_write; - if (io_thread().no_buffer() + if (io_thread() + && io_thread()->no_buffer() && ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0) mode |= file::no_buffer; boost::shared_ptr f = m_pool.open_file(this @@ -915,22 +918,22 @@ namespace libtorrent #endif #define TORRENT_ALLOCATE_BLOCKS(bufs, num_blocks, piece_size) \ - int num_blocks = (piece_size + io_thread().block_size() - 1) / io_thread().block_size(); \ + int num_blocks = (piece_size + io_thread()->block_size() - 1) / io_thread()->block_size(); \ file::iovec_t* bufs = TORRENT_ALLOCA(file::iovec_t, num_blocks); \ for (int i = 0, size = piece_size; i < num_blocks; ++i) \ { \ - bufs[i].iov_base = io_thread().allocate_buffer(); \ - bufs[i].iov_len = (std::min)(io_thread().block_size(), size); \ + bufs[i].iov_base = io_thread()->allocate_buffer(); \ + bufs[i].iov_len = (std::min)(io_thread()->block_size(), size); \ size -= bufs[i].iov_len; \ } #define TORRENT_FREE_BLOCKS(bufs, num_blocks) \ for (int i = 0; i < num_blocks; ++i) \ - io_thread().free_buffer((char*)bufs[i].iov_base); + io_thread()->free_buffer((char*)bufs[i].iov_base); #define TORRENT_SET_SIZE(bufs, size, num_bufs) \ - for (num_bufs = 0; size > 0; size -= io_thread().block_size(), ++num_bufs) \ - bufs[num_bufs].iov_len = (std::min)(io_thread().block_size(), size) + for (num_bufs = 0; size > 0; size -= io_thread()->block_size(), ++num_bufs) \ + bufs[num_bufs].iov_len = (std::min)(io_thread()->block_size(), size) bool storage::move_slot(int src_slot, int dst_slot) @@ -954,7 +957,6 @@ ret: bool r = true; // the size of the target slot is the size of the piece - int piece_size = m_files.piece_length(); int piece1_size = m_files.piece_size(slot2); int piece2_size = m_files.piece_size(slot1); @@ -1098,7 +1100,8 @@ ret: error_code ec; int mode = file::read_only; - if (io_thread().no_buffer() + if (io_thread() + && io_thread()->no_buffer() && ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0) mode |= file::no_buffer; in = m_pool.open_file(this, path, mode, ec); @@ -1241,7 +1244,8 @@ ret: error_code ec; int mode = file::read_write; - if (io_thread().no_buffer() + if (io_thread() + && io_thread()->no_buffer() && ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0) mode |= file::no_buffer; out = m_pool.open_file(this, path, mode, ec); diff --git a/src/torrent_info.cpp b/src/torrent_info.cpp index c7846c720..d3b4b7b1d 100644 --- a/src/torrent_info.cpp +++ b/src/torrent_info.cpp @@ -219,11 +219,7 @@ namespace // bitcomet pad file -#if BOOST_VERSION < 103600 - if (target.path.leaf().substr(0, 18) == "_____padding_file_") -#else - if (target.path.filename().substr(0, 18) == "_____padding_file_") -#endif + if (target.path.string().find("_____padding_file_") != std::string::npos) target.pad_file = true; return true;