From 7592ad4aee8a4f198e8607a020828c7f879ae49a Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Sun, 11 Jan 2009 02:02:34 +0000 Subject: [PATCH] updated disk IO to support unbuffered files --- ChangeLog | 1 + docs/manual.rst | 53 ++- include/libtorrent/disk_buffer_holder.hpp | 11 +- include/libtorrent/disk_io_thread.hpp | 18 +- include/libtorrent/file.hpp | 47 +- include/libtorrent/session_settings.hpp | 5 + include/libtorrent/storage.hpp | 19 +- src/disk_buffer_holder.cpp | 25 +- src/disk_io_thread.cpp | 80 +++- src/file.cpp | 526 ++++++++++++++++------ src/file_pool.cpp | 8 +- src/kademlia/dht_tracker.cpp | 2 +- src/storage.cpp | 287 +++++++----- src/torrent_info.cpp | 9 +- 14 files changed, 785 insertions(+), 306 deletions(-) diff --git a/ChangeLog b/ChangeLog index 62417f87a..1e9a8eef1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ + * added support for unbuffered I/O for aligned files * added workaround for sparse file issue on Windows Vista * added new lt_trackers extension to exchange trackers between peers diff --git a/docs/manual.rst b/docs/manual.rst index f8e142767..b4f9498ba 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -3194,6 +3194,7 @@ that will be sent to the tracker. The user-agent is a good way to identify your int cache_size; int cache_expiry; bool use_read_cache; + bool disk_io_no_buffer; std::pair outgoing_ports; char peer_tos; @@ -3424,6 +3425,15 @@ in the write cache, to when it's forcefully flushed to disk. Default is 60 secon ``use_read_cache``, is set to true (default), the disk cache is also used to cache pieces read from disk. Blocks for writing pieces takes presedence. +``disk_io_no_buffer`` defaults to true. When set to true, files are preferred +to be opened in unbuffered mode. This helps the operating system from growing +its file cache indefinitely. Currently only files whose offset in the torrent +is page aligned are opened in unbuffered mode. A page is typically 4096 bytes +and since blocks in bittorrent are 16kB, any file that is aligned to a block +or piece will get the benefit of be opened in unbuffered mode. It is therefore +recommended to make the largest file in a torrent the first file (with offset 0) +or use pad files to align all files to piece boundries. + ``outgoing_ports``, if set to something other than (0, 0) is a range of ports used to bind outgoing sockets to. This may be useful for users whose router allows them to assign QoS classes to traffic based on its local port. It is @@ -4949,8 +4959,8 @@ this:: struct storage_interface { virtual bool initialize(bool allocate_files) = 0; - virtual int read(char* buf, int slot, int offset, int size) = 0; - virtual int write(const char* buf, int slot, int offset, int size) = 0; + virtual int readv(file::iovec_t const* bufs, int slot, int offset, int num_bufs) = 0; + virtual int writev(file::iovec_t const* bufs, int slot, int offset, int num_bufs) = 0; virtual bool move_storage(fs::path save_path) = 0; virtual bool verify_resume_data(lazy_entry const& rd, std::string& error) = 0; virtual bool write_resume_data(entry& rd) const = 0; @@ -4978,31 +4988,56 @@ it will also ``ftruncate`` all files to their target size. Returning ``true`` indicates an error occurred. -read() + +readv() ------ :: - int read(char* buf, int slot, int offset, int size) = 0; + int readv(file::iovec_t const* buf, int slot, int offset, int num_bufs) = 0; -This function should read the data in the given slot and at the given offset -and ``size`` number of bytes. The data is to be copied to ``buf``. +This function should read the data in the given ``slot`` and at the given ``offset``. +It should read ``num_bufs`` buffers, where the size of each buffer is specified in the +buffer array ``bufs``. The file::iovec_t type has the following members:: + + struct iovec_t + { + void* iov_base; + size_t iov_len; + }; The return value is the number of bytes actually read. +Every buffer in ``bufs`` can be assumed to be page aligned and be of a page aligned size, +except for the last buffer of the torrent. The buffer can be assumed to fit a fully page +aligned number of bytes though. -write() + +writev() ------- :: int write(const char* buf, int slot, int offset, int size) = 0; -This function should write the data in ``buf`` to the given slot (``slot``) at offset -``offset`` in that slot. The buffer size is ``size``. +This function should write the data to the given ``slot`` and at the given ``offset``. +It should write ``num_bufs`` buffers, where the size of each buffer is specified in the +buffer array ``bufs``. The file::iovec_t type has the following members:: + + struct iovec_t + { + void* iov_base; + size_t iov_len; + }; The return value is the number of bytes actually written. +Every buffer in ``bufs`` can be assumed to be page aligned and be of a page aligned size, +except for the last buffer of the torrent. The buffer can be assumed to fit a fully page +aligned number of bytes though. +This function should write the data in ``buf`` to the given slot (``slot``) at offset +``offset`` in that slot. The buffer size is ``size``. + move_storage() -------------- diff --git a/include/libtorrent/disk_buffer_holder.hpp b/include/libtorrent/disk_buffer_holder.hpp index 31be6b3c7..e356fb2b8 100644 --- a/include/libtorrent/disk_buffer_holder.hpp +++ b/include/libtorrent/disk_buffer_holder.hpp @@ -34,6 +34,8 @@ POSSIBILITY OF SUCH DAMAGE. #define TORRENT_DISK_BUFFER_HOLDER_HPP_INCLUDED #include "libtorrent/config.hpp" +#include "libtorrent/assert.hpp" +#include namespace libtorrent { @@ -45,10 +47,16 @@ namespace libtorrent { disk_buffer_holder(aux::session_impl& ses, char* buf); disk_buffer_holder(disk_io_thread& iothread, char* buf); + disk_buffer_holder(disk_io_thread& iothread, char* buf, int num_blocks); ~disk_buffer_holder(); char* release(); char* get() const { return m_buf; } - void reset(char* buf = 0); + void reset(char* buf = 0, int num_blocks = 1); + void swap(disk_buffer_holder& h) + { + TORRENT_ASSERT(&h.m_iothread == &m_iothread); + std::swap(h.m_buf, m_buf); + } typedef char* (disk_buffer_holder::*unspecified_bool_type)(); operator unspecified_bool_type() const @@ -57,6 +65,7 @@ namespace libtorrent private: disk_io_thread& m_iothread; char* m_buf; + int m_num_blocks; }; } diff --git a/include/libtorrent/disk_io_thread.hpp b/include/libtorrent/disk_io_thread.hpp index 193c2a5b1..c8e3f217f 100644 --- a/include/libtorrent/disk_io_thread.hpp +++ b/include/libtorrent/disk_io_thread.hpp @@ -53,6 +53,14 @@ POSSIBILITY OF SUCH DAMAGE. namespace libtorrent { + struct page_aligned_allocator + { + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + + static char* malloc(const size_type bytes); + static void free(char* const block); + }; struct cached_piece_info { @@ -197,10 +205,16 @@ namespace libtorrent char* allocate_buffer(); void free_buffer(char* buf); + char* allocate_buffers(int blocks); + void free_buffers(char* buf, int blocks); + #ifdef TORRENT_DEBUG void check_invariant() const; #endif + int block_size() const { return m_block_size; } + bool no_buffer() const { return m_disk_io_no_buffer; } + private: struct cached_piece_entry @@ -282,14 +296,16 @@ namespace libtorrent // falls back to writing each block separately. bool m_coalesce_writes; bool m_coalesce_reads; + bool m_use_read_cache; + bool m_disk_io_no_buffer; // this only protects the pool allocator mutable mutex_t m_pool_mutex; #ifndef TORRENT_DISABLE_POOL_ALLOCATOR // memory pool for read and write operations // and disk cache - boost::pool<> m_pool; + boost::pool m_pool; #endif // number of bytes per block. The BitTorrent diff --git a/include/libtorrent/file.hpp b/include/libtorrent/file.hpp index dbc4b3760..9f2a1dc6d 100644 --- a/include/libtorrent/file.hpp +++ b/include/libtorrent/file.hpp @@ -59,6 +59,15 @@ POSSIBILITY OF SUCH DAMAGE. #else // posix part #define _FILE_OFFSET_BITS 64 + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 600 +#endif + #include #include #include @@ -75,18 +84,26 @@ namespace libtorrent enum { + // when a file is opened with no_buffer + // file offsets have to be aligned to + // pages and buffer addresses and sizes + // have to be page aligned too #ifdef TORRENT_WINDOWS read_only = GENERIC_READ, write_only = GENERIC_WRITE, read_write = GENERIC_READ | GENERIC_WRITE, - begin = FILE_BEGIN, - end = FILE_END, + rw_mask = GENERIC_READ | GENERIC_WRITE, + no_buffer = 1 #else - begin = SEEK_SET, - end = SEEK_END, read_only = O_RDONLY, write_only = O_WRONLY | O_CREAT, read_write = O_RDWR | O_CREAT, + rw_mask = O_RDONLY | O_WRONLY | O_RDWR | O_CREAT, +#if defined O_DIRECT + no_buffer = O_DIRECT +#else + no_buffer = O_SYNC +#endif #endif }; @@ -109,26 +126,28 @@ namespace libtorrent void close(); bool set_size(size_type size, error_code& ec); - size_type writev(iovec_t const* bufs, int num_bufs, error_code& ec); - size_type readv(iovec_t const* bufs, int num_bufs, error_code& ec); + size_type writev(size_type file_offset, iovec_t const* bufs, int num_bufs, error_code& ec); + size_type readv(size_type file_offset, iovec_t const* bufs, int num_bufs, error_code& ec); - size_type write(char const*, size_type num_bytes, error_code& ec); - size_type read(char*, size_type num_bytes, error_code& ec); - - size_type seek(size_type pos, int m, error_code& ec); - size_type tell(error_code& ec); + size_type get_size(error_code& ec); private: #ifdef TORRENT_WINDOWS HANDLE m_file_handle; +#ifdef TORRENT_USE_WPATH + std::wstring m_path; +#else + std::string m_path; +#endif #else int m_fd; #endif -#ifdef TORRENT_DEBUG - int m_open_mode; +#if defined TORRENT_WINDOWS || defined TORRENT_DEBUG + void init_file(); + static int m_page_size; #endif - + int m_open_mode; }; } diff --git a/include/libtorrent/session_settings.hpp b/include/libtorrent/session_settings.hpp index dd24c8f7c..dc0910d7f 100644 --- a/include/libtorrent/session_settings.hpp +++ b/include/libtorrent/session_settings.hpp @@ -125,6 +125,7 @@ namespace libtorrent , cache_size(512) , cache_expiry(60) , use_read_cache(true) + , disk_io_no_buffer(true) , outgoing_ports(0,0) , peer_tos(0) , active_downloads(8) @@ -377,6 +378,10 @@ namespace libtorrent // cache for caching blocks read from disk too bool use_read_cache; + // when set to true, files that can be opened with + // no OS buffering will be opened that way + bool disk_io_no_buffer; + // if != (0, 0), this is the range of ports that // outgoing connections will be bound to. This // is useful for users that have routers that diff --git a/include/libtorrent/storage.hpp b/include/libtorrent/storage.hpp index b0e5d4f74..45930e284 100644 --- a/include/libtorrent/storage.hpp +++ b/include/libtorrent/storage.hpp @@ -57,8 +57,8 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/peer_request.hpp" #include "libtorrent/hasher.hpp" #include "libtorrent/config.hpp" -#include "libtorrent/buffer.hpp" #include "libtorrent/file.hpp" +#include "libtorrent/disk_buffer_holder.hpp" namespace libtorrent { @@ -72,7 +72,6 @@ namespace libtorrent class session; struct file_pool; struct disk_io_job; - struct disk_buffer_holder; enum storage_mode_t { @@ -124,8 +123,8 @@ namespace libtorrent // false return value indicates an error virtual bool initialize(bool allocate_files) = 0; - virtual int readv(file::iovec_t* bufs, int slot, int offset, int num_bufs); - virtual int writev(file::iovec_t* buf, int slot, int offset, int num_bufs); + virtual int readv(file::iovec_t const* bufs, int slot, int offset, int num_bufs); + virtual int writev(file::iovec_t const* bufs, int slot, int offset, int num_bufs); // negative return value indicates an error virtual int read(char* buf, int slot, int offset, int size) = 0; @@ -168,6 +167,8 @@ namespace libtorrent // non-zero return value indicates an error virtual bool delete_files() = 0; + disk_io_thread& io_thread() { return *m_io_thread; } + void set_error(boost::filesystem::path const& file, error_code const& ec) const { m_error_file = file.string(); @@ -182,6 +183,8 @@ namespace libtorrent mutable std::string m_error_file; virtual ~storage_interface() {} + + disk_io_thread* m_io_thread; }; typedef storage_interface* (&storage_constructor_type)( @@ -321,7 +324,7 @@ namespace libtorrent // -1=error 0=ok 1=skip int check_one_piece(int& have_piece); int identify_data( - const std::vector& piece_data + char const* piece_data , int current_slot); void switch_to_full_mode(); @@ -394,8 +397,8 @@ namespace libtorrent // used to move pieces while expanding // the storage from compact allocation // to full allocation - buffer m_scratch_buffer; - buffer m_scratch_buffer2; + disk_buffer_holder m_scratch_buffer; + disk_buffer_holder m_scratch_buffer2; // the piece that is in the scratch buffer int m_scratch_piece; @@ -404,7 +407,7 @@ namespace libtorrent storage_constructor_type m_storage_constructor; // temporary buffer used while checking - std::vector m_piece_data; + disk_buffer_holder m_piece_data; // this maps a piece hash to piece index. It will be // build the first time it is used (to save time if it diff --git a/src/disk_buffer_holder.cpp b/src/disk_buffer_holder.cpp index 1ee83e93e..64776b3d5 100644 --- a/src/disk_buffer_holder.cpp +++ b/src/disk_buffer_holder.cpp @@ -38,21 +38,32 @@ namespace libtorrent { disk_buffer_holder::disk_buffer_holder(aux::session_impl& ses, char* buf) - : m_iothread(ses.m_disk_thread), m_buf(buf) + : m_iothread(ses.m_disk_thread), m_buf(buf), m_num_blocks(1) { TORRENT_ASSERT(buf == 0 || m_iothread.is_disk_buffer(buf)); } disk_buffer_holder::disk_buffer_holder(disk_io_thread& iothread, char* buf) - : m_iothread(iothread), m_buf(buf) + : m_iothread(iothread), m_buf(buf), m_num_blocks(1) { TORRENT_ASSERT(buf == 0 || m_iothread.is_disk_buffer(buf)); } - void disk_buffer_holder::reset(char* buf) + disk_buffer_holder::disk_buffer_holder(disk_io_thread& iothread, char* buf, int num_blocks) + : m_iothread(iothread), m_buf(buf), m_num_blocks(num_blocks) { - if (m_buf) m_iothread.free_buffer(m_buf); + TORRENT_ASSERT(buf == 0 || m_iothread.is_disk_buffer(buf)); + } + + void disk_buffer_holder::reset(char* buf, int num_blocks) + { + if (m_buf) + { + if (m_num_blocks == 1) m_iothread.free_buffer(m_buf); + else m_iothread.free_buffers(m_buf, m_num_blocks); + } m_buf = buf; + m_num_blocks = num_blocks; } char* disk_buffer_holder::release() @@ -64,7 +75,11 @@ namespace libtorrent disk_buffer_holder::~disk_buffer_holder() { - if (m_buf) m_iothread.free_buffer(m_buf); + if (m_buf) + { + if (m_num_blocks == 1) m_iothread.free_buffer(m_buf); + else m_iothread.free_buffers(m_buf, m_num_blocks); + } } } diff --git a/src/disk_io_thread.cpp b/src/disk_io_thread.cpp index 1bcc22c74..824ba6d18 100644 --- a/src/disk_io_thread.cpp +++ b/src/disk_io_thread.cpp @@ -34,13 +34,14 @@ POSSIBILITY OF SUCH DAMAGE. #include #include "libtorrent/disk_io_thread.hpp" #include "libtorrent/disk_buffer_holder.hpp" +#include "libtorrent/alloca.hpp" +#include "libtorrent/invariant_check.hpp" #include -#ifdef _WIN32 -#include -#ifndef alloca -#define alloca(s) _alloca(s) -#endif +#ifdef TORRENT_WINDOWS +#include +#else +#include #endif #ifdef TORRENT_DISK_STATS @@ -50,23 +51,33 @@ POSSIBILITY OF SUCH DAMAGE. namespace libtorrent { + char* page_aligned_allocator::malloc(const size_type bytes) + { +#ifdef TORRENT_WINDOWS + return reinterpret_cast(VirtualAlloc(0, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE)); +#else + return reinterpret_cast(valloc(bytes)); +#endif + } + + void page_aligned_allocator::free(char* const block) + { +#ifdef TORRENT_WINDOWS + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + disk_io_thread::disk_io_thread(asio::io_service& ios, int block_size) : m_abort(false) , m_queue_buffer_size(0) , m_cache_size(512) // 512 * 16kB = 8MB , m_cache_expiry(60) // 1 minute -// the file class doesn't support proper writev -// and readv on windows, so it's more efficient -// to coalesce reads and writes into a bigger -// buffer first -#ifdef TORRENT_WINDOWS - , m_coalesce_writes(true) - , m_coalesce_reads(true) -#else , m_coalesce_writes(false) , m_coalesce_reads(false) -#endif , m_use_read_cache(true) + , m_disk_io_no_buffer(true) #ifndef TORRENT_DISABLE_POOL_ALLOCATOR , m_pool(block_size) #endif @@ -330,11 +341,10 @@ namespace libtorrent int offset = 0; boost::scoped_array buf; - boost::scoped_array iov; + file::iovec_t* iov = 0; int iov_counter = 0; if (m_coalesce_writes) buf.reset(new (std::nothrow) char[piece_size]); - // TOOD: replace with alloca - else iov.reset(new file::iovec_t[blocks_in_piece]); + else iov = TORRENT_ALLOCA(file::iovec_t, blocks_in_piece); for (int i = 0; i <= blocks_in_piece; ++i) { @@ -346,8 +356,7 @@ namespace libtorrent l.unlock(); if (iov) { - TORRENT_ASSERT(iov); - p.storage->write_impl(iov.get(), p.piece, (std::min)( + p.storage->write_impl(iov, p.piece, (std::min)( i * m_block_size, piece_size) - buffer_size, iov_counter); iov_counter = 0; } @@ -762,7 +771,7 @@ namespace libtorrent ++m_allocations; #endif #ifdef TORRENT_DISABLE_POOL_ALLOCATOR - return (char*)malloc(m_block_size); + return page_aligned_allocator::malloc(m_block_sizes); #else return (char*)m_pool.ordered_malloc(); #endif @@ -776,12 +785,40 @@ namespace libtorrent --m_allocations; #endif #ifdef TORRENT_DISABLE_POOL_ALLOCATOR - free(buf); + page_aligned_allocator::free(buf); #else m_pool.ordered_free(buf); #endif } + char* disk_io_thread::allocate_buffers(int num_blocks) + { + mutex_t::scoped_lock l(m_pool_mutex); +#ifdef TORRENT_STATS + m_allocations += num_blocks; +#endif +#ifdef TORRENT_DISABLE_POOL_ALLOCATOR + return page_aligned_allocator::malloc(m_block_size * num_blocks); +#else + return (char*)m_pool.ordered_malloc(num_blocks); +#endif + } + + void disk_io_thread::free_buffers(char* buf, int num_blocks) + { + TORRENT_ASSERT(buf); + TORRENT_ASSERT(num_blocks >= 1); + mutex_t::scoped_lock l(m_pool_mutex); +#ifdef TORRENT_STATS + m_allocations -= num_blocks; +#endif +#ifdef TORRENT_DISABLE_POOL_ALLOCATOR + page_aligned_allocator::free(buf); +#else + m_pool.ordered_free(buf, num_blocks); +#endif + } + bool disk_io_thread::test_error(disk_io_job& j) { error_code const& ec = j.storage->error(); @@ -870,6 +907,7 @@ namespace libtorrent m_cache_size = s.cache_size; m_cache_expiry = s.cache_expiry; m_use_read_cache = s.use_read_cache; + m_disk_io_no_buffer = s.disk_io_no_buffer; } case disk_io_job::abort_torrent: { diff --git a/src/file.cpp b/src/file.cpp index 69fba4206..eaf3f6a7d 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -32,6 +32,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/pch.hpp" #include "libtorrent/config.hpp" +#include "libtorrent/alloca.hpp" #include #ifdef TORRENT_WINDOWS @@ -70,6 +71,8 @@ BOOST_STATIC_ASSERT(sizeof(lseek(0, 0, 0)) >= 8); #include "libtorrent/assert.hpp" +BOOST_STATIC_ASSERT((libtorrent::file::rw_mask & libtorrent::file::no_buffer) == 0); + namespace { #ifdef TORRENT_WINDOWS @@ -107,9 +110,7 @@ namespace libtorrent #else : m_fd(-1) #endif -#ifdef TORRENT_DEBUG , m_open_mode(0) -#endif {} file::file(fs::path const& path, int mode, error_code& ec) @@ -118,9 +119,7 @@ namespace libtorrent #else : m_fd(-1) #endif -#ifdef TORRENT_DEBUG , m_open_mode(0) -#endif { open(path, mode, ec); } @@ -136,18 +135,19 @@ namespace libtorrent #ifdef TORRENT_WINDOWS #ifdef TORRENT_USE_WPATH - std::wstring file_path(safe_convert(path.external_file_string())); + m_path = safe_convert(path.external_file_string()); #else - std::string file_path = utf8_native(path.external_file_string()); + m_path = utf8_native(path.external_file_string()); #endif m_file_handle = CreateFile( - file_path.c_str() - , mode + m_path.c_str() + , mode & rw_mask , FILE_SHARE_READ , 0 - , (mode == read_write || mode == write_only)?OPEN_ALWAYS:OPEN_EXISTING - , FILE_ATTRIBUTE_NORMAL + , ((mode & rw_mask) == read_write || (mode & rw_mask) == write_only)?OPEN_ALWAYS:OPEN_EXISTING + , FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS + | (mode & no_buffer?FILE_FLAG_OVERLAPPED | FILE_FLAG_NO_BUFFERING:0) , 0); if (m_file_handle == INVALID_HANDLE_VALUE) @@ -157,7 +157,7 @@ namespace libtorrent } // try to make the file sparse if supported - if (mode == write_only || mode == read_write) + if ((mode & rw_mask) == write_only || (mode & rw_mask) == read_write) { DWORD temp; ::DeviceIoControl(m_file_handle, FSCTL_SET_SPARSE, 0, 0 @@ -178,10 +178,23 @@ namespace libtorrent ec = error_code(errno, get_posix_category()); return false; } + +#ifdef F_NOCACHE + if (mode & no_buffer) + { + int yes = 1; + fcntl(m_fd, F_NOCACHE, &yes); + } +#endif + +#ifdef POSIX_FADV_RANDOM + // disable read-ahead + posix_fadvise(m_fd, 0, 0, POSIX_FADV_RANDOM); +#endif + #endif -#ifdef TORRENT_DEBUG m_open_mode = mode; -#endif + TORRENT_ASSERT(is_open()); return true; } @@ -201,119 +214,401 @@ namespace libtorrent if (m_file_handle == INVALID_HANDLE_VALUE) return; CloseHandle(m_file_handle); m_file_handle = INVALID_HANDLE_VALUE; + m_path.clear(); #else if (m_fd == -1) return; ::close(m_fd); m_fd = -1; #endif -#ifdef TORRENT_DEBUG m_open_mode = 0; -#endif } - size_type file::read(char* buf, size_type num_bytes, error_code& ec) + // defined in storage.cpp + int bufs_size(file::iovec_t const* bufs, int num_bufs); + +#if defined TORRENT_WINDOWS || defined TORRENT_DEBUG + + int file::m_page_size = 0; + + void file::init_file() { - TORRENT_ASSERT(m_open_mode == read_only || m_open_mode == read_write); - TORRENT_ASSERT(buf); - TORRENT_ASSERT(num_bytes >= 0); - TORRENT_ASSERT(is_open()); + if (m_page_size != 0) return; #ifdef TORRENT_WINDOWS - TORRENT_ASSERT(DWORD(num_bytes) == num_bytes); - DWORD ret = 0; - if (num_bytes != 0) - { - if (ReadFile(m_file_handle, buf, (DWORD)num_bytes, &ret, 0) == FALSE) - { - ec = error_code(GetLastError(), get_system_category()); - return -1; - } - } + SYSTEM_INFO si; + GetSystemInfo(&si); + m_page_size = si.dwPageSize; #else - size_type ret = ::read(m_fd, buf, num_bytes); - if (ret == -1) ec = error_code(errno, get_posix_category()); + m_page_size = sysconf(_SC_PAGESIZE); #endif - return ret; } - size_type file::readv(iovec_t const* bufs, int num_bufs, error_code& ec) +#endif + + size_type file::readv(size_type file_offset, iovec_t const* bufs, int num_bufs, error_code& ec) { - TORRENT_ASSERT(m_open_mode == read_only || m_open_mode == read_write); + TORRENT_ASSERT((m_open_mode & rw_mask) == read_only || (m_open_mode & rw_mask) == read_write); TORRENT_ASSERT(bufs); - TORRENT_ASSERT(num_bufs >= 0); + TORRENT_ASSERT(num_bufs > 0); TORRENT_ASSERT(is_open()); -#ifdef TORRENT_WINDOWS - // TODO: Replace with ReadFileScatter if possible - size_type ret = 0; - for (iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i) - { - if (i->iov_len <= 0) continue; - DWORD intermediate = 0; - if (ReadFile(m_file_handle, i->iov_base, (DWORD)i->iov_len, &intermediate, 0) == FALSE) - { - ec = error_code(GetLastError(), get_system_category()); - return -1; - } - ret += intermediate; - } -#else - size_type ret = ::readv(m_fd, bufs, num_bufs); - if (ret == -1) ec = error_code(errno, get_posix_category()); +#if defined TORRENT_WINDOWS || defined TORRENT_DEBUG + // make sure m_page_size is initialized + init_file(); #endif - return ret; - } - size_type file::writev(iovec_t const* bufs, int num_bufs, error_code& ec) - { - TORRENT_ASSERT(m_open_mode == write_only || m_open_mode == read_write); - TORRENT_ASSERT(bufs); - TORRENT_ASSERT(num_bufs >= 0); - TORRENT_ASSERT(is_open()); - -#ifdef TORRENT_WINDOWS - // Replace by WriteFileGather if possible - size_type ret = 0; - for (iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i) +#ifdef TORRENT_DEBUG + if (m_open_mode & no_buffer) { - if (i->iov_len <= 0) continue; - DWORD intermediate = 0; - if (WriteFile(m_file_handle, i->iov_base, (DWORD)i->iov_len, &intermediate, 0) == FALSE) + bool eof = false; + int size = 0; + TORRENT_ASSERT((file_offset & (m_page_size-1)) == 0); + for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i) { - ec = error_code(GetLastError(), get_system_category()); - return -1; + TORRENT_ASSERT((int(i->iov_base) & (m_page_size-1)) == 0); + // every buffer must be a multiple of the page size + // except for the last one + TORRENT_ASSERT((i->iov_len & (m_page_size-1)) == 0 || i == end-1); + if ((i->iov_len & (m_page_size-1)) != 0) eof = true; + size += i->iov_len; } - ret += intermediate; + error_code code; + if (eof) TORRENT_ASSERT(file_offset + size >= get_size(code)); } -#else - size_type ret = ::writev(m_fd, bufs, num_bufs); - if (ret == -1) ec = error_code(errno, get_posix_category()); #endif - return ret; - } - - size_type file::write(const char* buf, size_type num_bytes, error_code& ec) - { - TORRENT_ASSERT(m_open_mode == write_only || m_open_mode == read_write); - TORRENT_ASSERT(buf); - TORRENT_ASSERT(num_bytes >= 0); - TORRENT_ASSERT(is_open()); #ifdef TORRENT_WINDOWS + DWORD ret = 0; - if (num_bytes != 0) + + // since the ReadFileScatter requires the file to be opened + // with no buffering, and no buffering requires page aligned + // buffers, open the file in non-buffered mode in case the + // buffer is not aligned. Most of the times the buffer should + // be aligned though + + if ((m_open_mode & no_buffer) == 0) { - if (WriteFile(m_file_handle, buf, (DWORD)num_bytes, &ret, 0) == FALSE) + // this means the buffer base or the buffer size is not aligned + // to the page size. Use a regular file for this operation. + + LARGE_INTEGER offs; + offs.QuadPart = file_offset; + if (SetFilePointerEx(m_file_handle, offs, &offs, FILE_BEGIN) == FALSE) { ec = error_code(GetLastError(), get_system_category()); return -1; } + + for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i) + { + DWORD intermediate = 0; + if (ReadFile(m_file_handle, (char*)bufs->iov_base + , (DWORD)bufs->iov_len, &intermediate, 0) == FALSE) + { + ec = error_code(GetLastError(), get_system_category()); + return -1; + } + ret += intermediate; + } + return ret; + } + + int size = bufs_size(bufs, num_bufs); + // number of pages for the read. round up + int num_pages = (size + m_page_size - 1) / m_page_size; + // allocate array of FILE_SEGMENT_ELEMENT for ReadFileScatter + FILE_SEGMENT_ELEMENT* segment_array = TORRENT_ALLOCA(FILE_SEGMENT_ELEMENT, num_pages + 1); + FILE_SEGMENT_ELEMENT* cur_seg = segment_array; + + for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i) + { + for (int k = 0; k < i->iov_len; k += m_page_size) + { + cur_seg->Buffer = ((char*)i->iov_base) + k; + ++cur_seg; + } + } + // terminate the array + cur_seg->Buffer = 0; + + OVERLAPPED ol; + ol.Internal = 0; + ol.InternalHigh = 0; + ol.OffsetHigh = file_offset >> 32; + ol.Offset = file_offset & 0xffffffff; + ol.Pointer = 0; + ol.hEvent = CreateEvent(0, true, false, 0); + + ret += size; + size = num_pages * m_page_size; + if (ReadFileScatter(m_file_handle, segment_array, size, 0, &ol) == 0) + { + DWORD last_error = GetLastError(); + if (last_error != ERROR_IO_PENDING && last_error != ERROR_HANDLE_EOF) + { + ec = error_code(GetLastError(), get_system_category()); + CloseHandle(ol.hEvent); + return -1; + } + if (GetOverlappedResult(m_file_handle, &ol, &ret, true) == 0) + { + ec = error_code(GetLastError(), get_system_category()); + CloseHandle(ol.hEvent); + return -1; + } } -#else - size_type ret = ::write(m_fd, buf, num_bytes); - if (ret == -1) ec = error_code(errno, get_posix_category()); -#endif return ret; +#else + size_type ret = lseek(m_fd, file_offset, SEEK_SET); + if (ret < 0) + { + ec = error_code(errno, get_posix_category()); + return -1; + } +#ifdef TORRENT_LINUX + bool aligned = false; + int size = 0; + // if we're not opened in no-buffer mode, we don't need alignment + if ((m_open_mode & no_buffer) == 0) aligned = true; + if (!aligned) + { + size = bufs_size(bufs, num_bufs); + if (size & (m_page_size-1) == 0) aligned = true; + } + if (aligned) +#endif + { + ret = ::readv(m_fd, bufs, num_bufs); + if (ret < 0) + { + ec = error_code(errno, get_posix_category()); + return -1; + } + return ret; + } +#ifdef TORRENT_LINUX + file::iovec_t* temp_bufs = TORRENT_ALLOCA(file::iovec_t, num_bufs); + memcpy(temp_bufs, bufs, sizeof(file::iovec_t) * num_bufs); + iovec_t& last = temp_bufs[num_bufs-1]; + last.iov_len = (last.iov_len & ~(m_page_size-1)) + m_page_size; + ret = ::readv(m_fd, temp_bufs, num_bufs); + if (ret < 0) + { + ec = error_code(errno, get_posix_category()); + return -1; + } + return (std::min)(ret, size_type(size)); +#endif +#endif + } + + size_type file::writev(size_type file_offset, iovec_t const* bufs, int num_bufs, error_code& ec) + { + TORRENT_ASSERT((m_open_mode & rw_mask) == write_only || (m_open_mode & rw_mask) == read_write); + TORRENT_ASSERT(bufs); + TORRENT_ASSERT(num_bufs > 0); + TORRENT_ASSERT(is_open()); + +#if defined TORRENT_WINDOWS || defined TORRENT_DEBUG + // make sure m_page_size is initialized + init_file(); +#endif + +#ifdef TORRENT_DEBUG + if (m_open_mode & no_buffer) + { + bool eof = false; + int size = 0; + TORRENT_ASSERT((file_offset & (m_page_size-1)) == 0); + for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i) + { + TORRENT_ASSERT((int(i->iov_base) & (m_page_size-1)) == 0); + // every buffer must be a multiple of the page size + // except for the last one + TORRENT_ASSERT((i->iov_len & (m_page_size-1)) == 0 || i == end-1); + if ((i->iov_len & (m_page_size-1)) != 0) eof = true; + size += i->iov_len; + } + error_code code; + if (eof) TORRENT_ASSERT(file_offset + size >= get_size(code)); + } +#endif + +#ifdef TORRENT_WINDOWS + + DWORD ret = 0; + + // since the ReadFileScatter requires the file to be opened + // with no buffering, and no buffering requires page aligned + // buffers, open the file in non-buffered mode in case the + // buffer is not aligned. Most of the times the buffer should + // be aligned though + + if ((m_open_mode & no_buffer) == 0) + { + // this means the buffer base or the buffer size is not aligned + // to the page size. Use a regular file for this operation. + + LARGE_INTEGER offs; + offs.QuadPart = file_offset; + if (SetFilePointerEx(m_file_handle, offs, &offs, SEEK_SET) == FALSE) + { + ec = error_code(GetLastError(), get_system_category()); + return -1; + } + + for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i) + { + DWORD intermediate = 0; + if (WriteFile(m_file_handle, (char const*)bufs->iov_base + , (DWORD)bufs->iov_len, &intermediate, 0) == FALSE) + { + ec = error_code(GetLastError(), get_system_category()); + return -1; + } + ret += intermediate; + } + return ret; + } + + int size = bufs_size(bufs, num_bufs); + // number of pages for the write. round up + int num_pages = (size + m_page_size - 1) / m_page_size; + // allocate array of FILE_SEGMENT_ELEMENT for WriteFileGather + FILE_SEGMENT_ELEMENT* segment_array = TORRENT_ALLOCA(FILE_SEGMENT_ELEMENT, num_pages + 1); + FILE_SEGMENT_ELEMENT* cur_seg = segment_array; + + for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i) + { + for (int k = 0; k < i->iov_len; k += m_page_size) + { + cur_seg->Buffer = ((char*)i->iov_base) + k; + ++cur_seg; + } + } + // terminate the array + cur_seg->Buffer = 0; + + OVERLAPPED ol; + ol.Internal = 0; + ol.InternalHigh = 0; + ol.OffsetHigh = file_offset >> 32; + ol.Offset = file_offset & 0xffffffff; + ol.Pointer = 0; + ol.hEvent = CreateEvent(0, true, false, 0); + + ret += size; + // if file_size is > 0, the file will be opened in unbuffered + // mode after the write completes, and truncate the file to + // file_size. + size_type file_size = 0; + + if (size & (m_page_size-1) != 0) + { + // if size is not an even multiple, this must be the tail + // of the file. Write the whole page and then open a new + // file without FILE_FLAG_NO_BUFFERING and set the + // file size to file_offset + size + + file_size = file_offset + size; + size = num_pages * m_page_size; + } + + if (WriteFileGather(m_file_handle, segment_array, size, 0, &ol) == 0) + { + if (GetLastError() != ERROR_IO_PENDING) + { + ec = error_code(GetLastError(), get_system_category()); + CloseHandle(ol.hEvent); + return -1; + } + if (GetOverlappedResult(m_file_handle, &ol, &ret, true) == 0) + { + ec = error_code(GetLastError(), get_system_category()); + CloseHandle(ol.hEvent); + return -1; + } + } + CloseHandle(ol.hEvent); + + if (file_size > 0) + { + HANDLE f = CreateFile(m_path.c_str(), GENERIC_WRITE + , FILE_SHARE_WRITE, 0, OPEN_EXISTING + , FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, 0); + + if (f == INVALID_HANDLE_VALUE) + { + ec = error_code(GetLastError(), get_system_category()); + return false; + } + + LARGE_INTEGER offs; + offs.QuadPart = file_size; + if (SetFilePointerEx(m_file_handle, offs, &offs, FILE_BEGIN) == FALSE) + { + CloseHandle(f); + ec = error_code(GetLastError(), get_system_category()); + return -1; + } + if (::SetEndOfFile(f) == FALSE) + { + ec = error_code(GetLastError(), get_system_category()); + CloseHandle(f); + return false; + } + CloseHandle(f); + } + + return ret; +#else + size_type ret = lseek(m_fd, file_offset, SEEK_SET); + if (ret < 0) + { + ec = error_code(errno, get_posix_category()); + return -1; + } +#ifdef TORRENT_LINUX + bool aligned = false; + int size = 0; + // if we're not opened in no-buffer mode, we don't need alignment + if ((m_open_mode & no_buffer) == 0) aligned = true; + if (!aligned) + { + size = bufs_size(bufs, num_bufs); + if (size & (m_page_size-1) == 0) aligned = true; + } + if (aligned) +#endif + { + ret = ::writev(m_fd, bufs, num_bufs); + if (ret < 0) + { + ec = error_code(errno, get_posix_category()); + return -1; + } + return ret; + } +#ifdef TORRENT_LINUX + file::iovec_t* temp_bufs = TORRENT_ALLOCA(file::iovec_t, num_bufs); + memcpy(temp_bufs, bufs, sizeof(file::iovec_t) * num_bufs); + iovec_t& last = temp_bufs[num_bufs-1]; + last.iov_len = (last.iov_len & ~(m_page_size-1)) + m_page_size; + ret = ::writev(m_fd, temp_bufs, num_bufs); + if (ret < 0) + { + ec = error_code(errno, get_posix_category()); + return -1; + } + if (ftruncate(m_fd, file_offset + size) < 0) + { + ec = error_code(errno, get_posix_category()); + return false; + } + return (std::min)(ret, size_type(size)); +#endif +#endif } bool file::set_size(size_type s, error_code& ec) @@ -322,10 +617,13 @@ namespace libtorrent TORRENT_ASSERT(s >= 0); #ifdef TORRENT_WINDOWS - size_type pos = tell(ec); - if (ec) return false; - seek(s, begin, ec); - if (ec) return false; + LARGE_INTEGER offs; + offs.QuadPart = s; + if (SetFilePointerEx(m_file_handle, offs, &offs, FILE_BEGIN) == FALSE) + { + ec = error_code(GetLastError(), get_system_category()); + return false; + } if (::SetEndOfFile(m_file_handle) == FALSE) { ec = error_code(GetLastError(), get_system_category()); @@ -341,48 +639,24 @@ namespace libtorrent return true; } - size_type file::seek(size_type offset, int m, error_code& ec) + size_type file::get_size(error_code& ec) { - TORRENT_ASSERT(is_open()); - #ifdef TORRENT_WINDOWS - LARGE_INTEGER offs; - offs.QuadPart = offset; - if (SetFilePointerEx(m_file_handle, offs, &offs, m) == FALSE) + LARGE_INTEGER file_size; + if (!GetFileSizeEx(m_file_handle, &file_size)) { ec = error_code(GetLastError(), get_system_category()); return -1; } - return offs.QuadPart; + return file_size.QuadPart; #else - size_type ret = lseek(m_fd, offset, m); - if (ret < 0) ec = error_code(errno, get_posix_category()); - return ret; -#endif - } - - size_type file::tell(error_code& ec) - { - TORRENT_ASSERT(is_open()); - -#ifdef TORRENT_WINDOWS - LARGE_INTEGER offs; - offs.QuadPart = 0; - - // is there any other way to get offset? - if (SetFilePointerEx(m_file_handle, offs, &offs - , FILE_CURRENT) == FALSE) + struct stat fs; + if (fstat(m_fd, &fs) != 0) { - ec = error_code(GetLastError(), get_system_category()); + ec = error_code(errno, get_posix_category()); return -1; } - - return offs.QuadPart; -#else - size_type ret; - ret = lseek(m_fd, 0, SEEK_CUR); - if (ret < 0) ec = error_code(errno, get_posix_category()); - return ret; + return fs.st_size; #endif } } diff --git a/src/file_pool.cpp b/src/file_pool.cpp index 92d8ca5cb..b65cfd987 100644 --- a/src/file_pool.cpp +++ b/src/file_pool.cpp @@ -47,7 +47,8 @@ namespace libtorrent { TORRENT_ASSERT(st != 0); TORRENT_ASSERT(p.is_complete()); - TORRENT_ASSERT(m == file::read_only || m == file::read_write); + TORRENT_ASSERT((m & file::rw_mask) == file::read_only + || (m & file::rw_mask) == file::read_write); boost::mutex::scoped_lock l(m_mutex); typedef nth_index::type path_view; path_view& pt = get<0>(m_files); @@ -71,8 +72,8 @@ namespace libtorrent // if we asked for a file in write mode, // and the cached file is is not opened in // write mode, re-open it - if ((e.mode != file::read_write) - && (m == file::read_write)) + if (((e.mode & file::rw_mask) != file::read_write) + && ((m & file::rw_mask) == file::read_write)) { // close the file before we open it with // the new read/write privilages @@ -88,6 +89,7 @@ namespace libtorrent e.mode = m; } pt.replace(i, e); + TORRENT_ASSERT((e.mode & ~file::rw_mask) == (m & ~file::rw_mask)); return e.file_ptr; } // the file is not in our cache diff --git a/src/kademlia/dht_tracker.cpp b/src/kademlia/dht_tracker.cpp index a6c984a5c..e397bc094 100644 --- a/src/kademlia/dht_tracker.cpp +++ b/src/kademlia/dht_tracker.cpp @@ -844,7 +844,7 @@ namespace libtorrent { namespace dht } #ifdef TORRENT_DHT_VERBOSE_LOGGING TORRENT_LOG(dht_tracker) << log_line.str() << " ]"; - TORRENT_LOG(dht_tracker) << "ERROR: incoming error: " << std::dec << m.error_code + TORRENT_LOG(dht_tracker) << "ERROR: incoming error: " << m.error_code << " " << m.error_msg; #endif return; diff --git a/src/storage.cpp b/src/storage.cpp index e1fd991f0..42c83f339 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -317,7 +317,7 @@ namespace libtorrent // for backwards compatibility, let the default readv and // writev implementations be implemented in terms of the // old read and write - int storage_interface::readv(file::iovec_t* bufs + int storage_interface::readv(file::iovec_t const* bufs , int slot, int offset, int num_bufs) { int ret = 0; @@ -331,7 +331,7 @@ namespace libtorrent return ret; } - int storage_interface::writev(file::iovec_t* bufs, int slot + int storage_interface::writev(file::iovec_t const* bufs, int slot , int offset, int num_bufs) { int ret = 0; @@ -380,17 +380,17 @@ namespace libtorrent } } - int bufs_size(file::iovec_t *bufs, int num_bufs) + int bufs_size(file::iovec_t const* bufs, int num_bufs) { int size = 0; - for (file::iovec_t* i = bufs, *end(bufs + num_bufs); i < end; ++i) + for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i) size += i->iov_len; return size; } - void clear_bufs(file::iovec_t *bufs, int num_bufs) + void clear_bufs(file::iovec_t const* bufs, int num_bufs) { - for (file::iovec_t* i = bufs, *end(bufs + num_bufs); i < end; ++i) + for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i) std::memset(i->iov_base, 0, i->iov_len); } @@ -400,10 +400,18 @@ namespace libtorrent storage(file_storage const& fs, fs::path const& path, file_pool& fp) : m_files(fs) , m_pool(fp) + , m_page_size(4096) { TORRENT_ASSERT(m_files.begin() != m_files.end()); m_save_path = fs::complete(path); TORRENT_ASSERT(m_save_path.is_complete()); +#ifdef TORRENT_WINDOWS + SYSTEM_INFO si; + GetSystemInfo(&si); + m_page_size = si.dwPageSize; +#else + m_page_size = sysconf(_SC_PAGESIZE); +#endif } bool rename_file(int index, std::string const& new_filename); @@ -413,8 +421,8 @@ namespace libtorrent bool move_storage(fs::path save_path); int read(char* buf, int slot, int offset, int size); int write(char const* buf, int slot, int offset, int size); - int readv(file::iovec_t* bufs, int slot, int offset, int num_bufs); - int writev(file::iovec_t* buf, int slot, int offset, int num_bufs); + int readv(file::iovec_t const* bufs, int slot, int offset, int num_bufs); + int writev(file::iovec_t const* buf, int slot, int offset, int num_bufs); bool move_slot(int src_slot, int dst_slot); bool swap_slots(int slot1, int slot2); bool swap_slots3(int slot1, int slot2, int slot3); @@ -436,42 +444,35 @@ namespace libtorrent // the session, to make all storage // instances use the same pool file_pool& m_pool; - - // temporary storage for moving pieces - buffer m_scratch_buffer; + + int m_page_size; }; sha1_hash storage::hash_for_slot(int slot, partial_hash& ph, int piece_size) { TORRENT_ASSERT(!error()); -#ifdef TORRENT_DEBUG - hasher partial; - hasher whole; - int slot_size1 = piece_size; - m_scratch_buffer.resize(slot_size1); - read(&m_scratch_buffer[0], slot, 0, slot_size1); - if (error()) return sha1_hash(0); - if (ph.offset > 0) - partial.update(&m_scratch_buffer[0], ph.offset); - whole.update(&m_scratch_buffer[0], slot_size1); - hasher partial_copy = ph.h; - TORRENT_ASSERT(ph.offset == 0 || partial_copy.final() == partial.final()); -#endif int slot_size = piece_size - ph.offset; if (slot_size > 0) { - m_scratch_buffer.resize(slot_size); - read(&m_scratch_buffer[0], slot, ph.offset, slot_size); + int size = slot_size; + int num_blocks = (size + io_thread().block_size() - 1) / io_thread().block_size(); + file::iovec_t* bufs = TORRENT_ALLOCA(file::iovec_t, num_blocks); + for (int i = 0; i < num_blocks; ++i) + { + bufs[i].iov_base = io_thread().allocate_buffer(); + bufs[i].iov_len = (std::min)(io_thread().block_size(), size); + size -= bufs[i].iov_len; + } + readv(bufs, slot, ph.offset, num_blocks); + + for (int i = 0; i < num_blocks; ++i) + { + ph.h.update((char const*)bufs[i].iov_base, bufs[i].iov_len); + io_thread().free_buffer((char*)bufs[i].iov_base); + } if (error()) return sha1_hash(0); - ph.h.update(&m_scratch_buffer[0], slot_size); } -#ifdef TORRENT_DEBUG - sha1_hash ret = ph.h.final(); - TORRENT_ASSERT(ret == whole.final()); - return ret; -#else return ph.h.final(); -#endif } bool storage::initialize(bool allocate_files) @@ -524,8 +525,12 @@ namespace libtorrent || (exists(file_path) && file_size(file_path) > file_iter->size)) { error_code ec; + int mode = file::read_write; + if (io_thread().no_buffer() + && ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0) + mode |= file::no_buffer; boost::shared_ptr f = m_pool.open_file(this - , m_save_path / file_iter->path, file::read_write, ec); + , m_save_path / file_iter->path, mode, ec); if (ec) set_error(m_save_path / file_iter->path, ec); else if (f) { @@ -605,7 +610,6 @@ namespace libtorrent bool storage::release_files() { m_pool.release(this); - buffer().swap(m_scratch_buffer); return false; } @@ -613,7 +617,6 @@ namespace libtorrent { // make sure we don't have the files open m_pool.release(this); - buffer().swap(m_scratch_buffer); int error = 0; std::string error_file; @@ -911,61 +914,97 @@ namespace libtorrent */ #endif +#define TORRENT_ALLOCATE_BLOCKS(bufs, num_blocks, piece_size) \ + int num_blocks = (piece_size + io_thread().block_size() - 1) / io_thread().block_size(); \ + file::iovec_t* bufs = TORRENT_ALLOCA(file::iovec_t, num_blocks); \ + for (int i = 0, size = piece_size; i < num_blocks; ++i) \ + { \ + bufs[i].iov_base = io_thread().allocate_buffer(); \ + bufs[i].iov_len = (std::min)(io_thread().block_size(), size); \ + size -= bufs[i].iov_len; \ + } + +#define TORRENT_FREE_BLOCKS(bufs, num_blocks) \ + for (int i = 0; i < num_blocks; ++i) \ + io_thread().free_buffer((char*)bufs[i].iov_base); + +#define TORRENT_SET_SIZE(bufs, size, num_bufs) \ + for (num_bufs = 0; size > 0; size -= io_thread().block_size(), ++num_bufs) \ + bufs[num_bufs].iov_len = (std::min)(io_thread().block_size(), size) + + bool storage::move_slot(int src_slot, int dst_slot) { + bool r = true; int piece_size = m_files.piece_size(dst_slot); - m_scratch_buffer.resize(piece_size); - int ret = read(&m_scratch_buffer[0], src_slot, 0, piece_size); - if (ret != piece_size) return true; - ret = write(&m_scratch_buffer[0], dst_slot, 0, piece_size); - if (ret != piece_size) return true; - return false; + + TORRENT_ALLOCATE_BLOCKS(bufs, num_blocks, piece_size); + + readv(bufs, src_slot, 0, num_blocks); if (error()) goto ret; + writev(bufs, dst_slot, 0, num_blocks); if (error()) goto ret; + + r = false; +ret: + TORRENT_FREE_BLOCKS(bufs, num_blocks) + return r; } bool storage::swap_slots(int slot1, int slot2) { + bool r = true; + // the size of the target slot is the size of the piece int piece_size = m_files.piece_length(); int piece1_size = m_files.piece_size(slot2); int piece2_size = m_files.piece_size(slot1); - m_scratch_buffer.resize(piece_size * 2); - int ret = read(&m_scratch_buffer[0], slot1, 0, piece1_size); - if (ret != piece1_size) return true; - ret = read(&m_scratch_buffer[piece_size], slot2, 0, piece2_size); - if (ret != piece2_size) return true; - ret = write(&m_scratch_buffer[0], slot2, 0, piece1_size); - if (ret != piece1_size) return true; - ret = write(&m_scratch_buffer[piece_size], slot1, 0, piece2_size); - if (ret != piece2_size) return true; - return false; + + TORRENT_ALLOCATE_BLOCKS(bufs1, num_blocks1, piece1_size); + TORRENT_ALLOCATE_BLOCKS(bufs2, num_blocks2, piece2_size); + + readv(bufs1, slot1, 0, num_blocks1); if (error()) goto ret; + readv(bufs2, slot2, 0, num_blocks2); if (error()) goto ret; + writev(bufs1, slot2, 0, num_blocks1); if (error()) goto ret; + writev(bufs2, slot1, 0, num_blocks2); if (error()) goto ret; + + r = false; +ret: + TORRENT_FREE_BLOCKS(bufs1, num_blocks1) + TORRENT_FREE_BLOCKS(bufs2, num_blocks2) + return r; } bool storage::swap_slots3(int slot1, int slot2, int slot3) { + bool r = true; + // the size of the target slot is the size of the piece int piece_size = m_files.piece_length(); int piece1_size = m_files.piece_size(slot2); int piece2_size = m_files.piece_size(slot3); int piece3_size = m_files.piece_size(slot1); - m_scratch_buffer.resize(piece_size * 2); - int ret = 0; - ret = read(&m_scratch_buffer[0], slot1, 0, piece1_size); - if (ret != piece1_size) return true; - ret = read(&m_scratch_buffer[piece_size], slot2, 0, piece2_size); - if (ret != piece2_size) return true; - ret = write(&m_scratch_buffer[0], slot2, 0, piece1_size); - if (ret != piece1_size) return true; - ret = read(&m_scratch_buffer[0], slot3, 0, piece3_size); - if (ret != piece3_size) return true; - ret = write(&m_scratch_buffer[piece_size], slot3, 0, piece2_size); - if (ret != piece2_size) return true; - ret = write(&m_scratch_buffer[0], slot1, 0, piece3_size); - if (ret != piece3_size) return true; - return false; + + TORRENT_ALLOCATE_BLOCKS(bufs1, num_blocks1, piece_size); + TORRENT_ALLOCATE_BLOCKS(bufs2, num_blocks2, piece_size); + + int tmp1 = 0; + int tmp2 = 0; + TORRENT_SET_SIZE(bufs1, piece1_size, tmp1); + readv(bufs1, slot1, 0, tmp1); if (error()) goto ret; + TORRENT_SET_SIZE(bufs2, piece2_size, tmp2); + readv(bufs2, slot2, 0, tmp2); if (error()) goto ret; + writev(bufs1, slot2, 0, tmp1); if (error()) goto ret; + TORRENT_SET_SIZE(bufs1, piece3_size, tmp1); + readv(bufs1, slot3, 0, tmp1); if (error()) goto ret; + writev(bufs2, slot3, 0, tmp2); if (error()) goto ret; + writev(bufs1, slot1, 0, tmp1); if (error()) goto ret; +ret: + TORRENT_FREE_BLOCKS(bufs1, num_blocks1) + TORRENT_FREE_BLOCKS(bufs2, num_blocks2) + return r; } int storage::readv( - file::iovec_t* bufs + file::iovec_t const* bufs , int slot , int offset , int num_bufs) @@ -1022,7 +1061,9 @@ namespace libtorrent int counter = 0; #endif - file::iovec_t* current_buf = bufs; + file::iovec_t* tmp_bufs = TORRENT_ALLOCA(file::iovec_t, num_bufs); + file::iovec_t* current_buf = TORRENT_ALLOCA(file::iovec_t, num_bufs); + copy_bufs(bufs, size, current_buf); int read_bytes; for (;left_to_read > 0; ++file_iter, left_to_read -= read_bytes , buf_pos += read_bytes) @@ -1047,7 +1088,6 @@ namespace libtorrent if (file_iter->pad_file) { - file::iovec_t* tmp_bufs = TORRENT_ALLOCA(file::iovec_t, num_bufs); int num_tmp_bufs = copy_bufs(current_buf, file_iter->size, tmp_bufs); clear_bufs(tmp_bufs, num_tmp_bufs); advance_bufs(current_buf, file_iter->size); @@ -1057,23 +1097,21 @@ namespace libtorrent fs::path path = m_save_path / file_iter->path; error_code ec; - in = m_pool.open_file(this, path, file::read_only, ec); + int mode = file::read_only; + if (io_thread().no_buffer() + && ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0) + mode |= file::no_buffer; + in = m_pool.open_file(this, path, mode, ec); if (!in || ec) { set_error(path, ec); return -1; } - size_type pos = in->seek(file_iter->file_base + file_offset, file::begin, ec); - if (pos != file_iter->file_base + file_offset || ec) - { - set_error(m_save_path / file_iter->path, ec); - return -1; - } - file_offset = 0; - file::iovec_t* tmp_bufs = TORRENT_ALLOCA(file::iovec_t, num_bufs); int num_tmp_bufs = copy_bufs(current_buf, read_bytes, tmp_bufs); - int actual_read = int(in->readv(tmp_bufs, num_tmp_bufs, ec)); + int actual_read = int(in->readv(file_iter->file_base + + file_offset, tmp_bufs, num_tmp_bufs, ec)); + file_offset = 0; if (read_bytes != actual_read || ec) { @@ -1094,6 +1132,8 @@ namespace libtorrent , int offset , int size) { + // buffers must be page aligned + TORRENT_ASSERT((int(buf) % 4096) == 0); file::iovec_t b = { (void*)buf, size }; return writev(&b, slot, offset, 1); } @@ -1104,12 +1144,14 @@ namespace libtorrent , int offset , int size) { + // buffers must be page aligned + TORRENT_ASSERT((int(buf) % 4096) == 0); file::iovec_t b = { (void*)buf, size }; return readv(&b, slot, offset, 1); } int storage::writev( - file::iovec_t* bufs + file::iovec_t const* bufs , int slot , int offset , int num_bufs) @@ -1162,7 +1204,9 @@ namespace libtorrent int counter = 0; #endif - file::iovec_t* current_buf = bufs; + file::iovec_t* tmp_bufs = TORRENT_ALLOCA(file::iovec_t, num_bufs); + file::iovec_t* current_buf = TORRENT_ALLOCA(file::iovec_t, num_bufs); + copy_bufs(bufs, size, current_buf); int write_bytes; for (;left_to_write > 0; ++file_iter, left_to_write -= write_bytes , buf_pos += write_bytes) @@ -1186,28 +1230,32 @@ namespace libtorrent #endif if (file_iter->pad_file) + { + int actual_written = (std::min)(int(file_iter->size), left_to_write); + advance_bufs(current_buf, actual_written); + left_to_write -= actual_written; continue; + } fs::path path = m_save_path / file_iter->path; error_code ec; - out = m_pool.open_file(this, path, file::read_write, ec); + int mode = file::read_write; + if (io_thread().no_buffer() + && ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0) + mode |= file::no_buffer; + out = m_pool.open_file(this, path, mode, ec); if (!out || ec) { set_error(path, ec); return -1; } - size_type pos = out->seek(file_iter->file_base + file_offset, file::begin, ec); - if (pos != file_iter->file_base + file_offset || ec) - { - set_error(m_save_path / file_iter->path, ec); - return -1; - } - file_offset = 0; - file::iovec_t* tmp_bufs = TORRENT_ALLOCA(file::iovec_t, num_bufs); int num_tmp_bufs = copy_bufs(current_buf, write_bytes, tmp_bufs); - int actual_written = int(out->writev(tmp_bufs, num_tmp_bufs, ec)); + int actual_written = int(out->writev(file_iter->file_base + + file_offset, tmp_bufs, num_tmp_bufs, ec)); + + file_offset = 0; if (write_bytes != actual_written || ec) { @@ -1245,11 +1293,15 @@ namespace libtorrent , m_state(state_none) , m_current_slot(0) , m_out_of_place(false) + , m_scratch_buffer(io, 0) + , m_scratch_buffer2(io, 0) , m_scratch_piece(-1) , m_storage_constructor(sc) + , m_piece_data(io, 0) , m_io_thread(io) , m_torrent(torrent) { + m_storage->m_io_thread = &m_io_thread; } piece_manager::~piece_manager() @@ -1586,7 +1638,7 @@ namespace libtorrent } int piece_manager::identify_data( - const std::vector& piece_data + char const* piece_data , int current_slot) { // INVARIANT_CHECK; @@ -1595,19 +1647,17 @@ namespace libtorrent const int last_piece_size = static_cast(m_files.piece_size( m_files.num_pieces() - 1)); - TORRENT_ASSERT((int)piece_data.size() >= last_piece_size); - // calculate a small digest, with the same // size as the last piece. And a large digest // which has the same size as a normal piece hasher small_digest; - small_digest.update(&piece_data[0], last_piece_size); + small_digest.update(piece_data, last_piece_size); hasher large_digest(small_digest); TORRENT_ASSERT(piece_size - last_piece_size >= 0); if (piece_size - last_piece_size > 0) { large_digest.update( - &piece_data[last_piece_size] + piece_data + last_piece_size , piece_size - last_piece_size); } sha1_hash large_hash = large_digest.final(); @@ -1785,8 +1835,8 @@ namespace libtorrent return fatal_disk_error; } m_state = state_finished; - buffer().swap(m_scratch_buffer); - buffer().swap(m_scratch_buffer2); + m_scratch_buffer.reset(); + m_scratch_buffer2.reset(); if (m_storage_mode != storage_mode_compact) { // if no piece is out of place @@ -2035,11 +2085,16 @@ namespace libtorrent if (other_piece >= 0) { - if (m_scratch_buffer2.empty()) - m_scratch_buffer2.resize(m_files.piece_length()); + if (!m_scratch_buffer2) + { + int blocks_per_piece = (std::max)(m_files.piece_length() + / m_io_thread.block_size(), 1); + m_scratch_buffer2.reset(m_io_thread.allocate_buffers( + blocks_per_piece), blocks_per_piece); + } int piece_size = m_files.piece_size(other_piece); - if (m_storage->read(&m_scratch_buffer2[0], piece, 0, piece_size) + if (m_storage->read(m_scratch_buffer2.get(), piece, 0, piece_size) != piece_size) { error = m_storage->error().message(); @@ -2053,7 +2108,7 @@ namespace libtorrent // the slot where this piece belongs is // free. Just move the piece there. int piece_size = m_files.piece_size(piece); - if (m_storage->write(&m_scratch_buffer[0], piece, 0, piece_size) != piece_size) + if (m_storage->write(m_scratch_buffer.get(), piece, 0, piece_size) != piece_size) { error = m_storage->error().message(); TORRENT_ASSERT(!error.empty()); @@ -2091,11 +2146,15 @@ namespace libtorrent // there is another piece in the slot // where this one goes. Store it in the scratch // buffer until next iteration. - if (m_scratch_buffer.empty()) - m_scratch_buffer.resize(m_files.piece_length()); + if (!m_scratch_buffer) + { + int blocks_per_piece = (std::max)(m_files.piece_length() / m_io_thread.block_size(), 1); + m_scratch_buffer.reset(m_io_thread.allocate_buffers( + blocks_per_piece), blocks_per_piece); + } int piece_size = m_files.piece_size(other_piece); - if (m_storage->read(&m_scratch_buffer[0], piece, 0, piece_size) != piece_size) + if (m_storage->read(m_scratch_buffer.get(), piece, 0, piece_size) != piece_size) { error = m_storage->error().message(); TORRENT_ASSERT(!error.empty()); @@ -2171,7 +2230,7 @@ namespace libtorrent TORRENT_ASSERT(m_current_slot == m_files.num_pieces()); // clear the memory we've been using - std::vector().swap(m_piece_data); + m_piece_data.reset(); std::multimap().swap(m_hash_to_piece); if (m_storage_mode != storage_mode_compact) @@ -2226,16 +2285,22 @@ namespace libtorrent m_hash_to_piece.insert(std::make_pair(m_info->hash_for_piece(i), i)); } - m_piece_data.resize(int(m_files.piece_length())); + if (!m_piece_data) + { + int blocks_per_piece = (std::max)(m_files.piece_length() / m_io_thread.block_size(), 1); + m_piece_data.reset(m_io_thread.allocate_buffers(blocks_per_piece), blocks_per_piece); + } + int piece_size = m_files.piece_size(m_current_slot); - int num_read = m_storage->read(&m_piece_data[0] + int num_read = m_storage->read(m_piece_data.get() , m_current_slot, 0, piece_size); if (num_read < 0) { if (m_storage->error() #ifdef TORRENT_WINDOWS - && m_storage->error() != error_code(ERROR_FILE_NOT_FOUND, get_system_category())) + && m_storage->error() != error_code(ERROR_FILE_NOT_FOUND, get_system_category()) + && m_storage->error() != error_code(ERROR_HANDLE_EOF, get_system_category())) #else && m_storage->error() != error_code(ENOENT, get_posix_category())) #endif @@ -2247,7 +2312,7 @@ namespace libtorrent if (num_read != piece_size) return 1; - int piece_index = identify_data(m_piece_data, m_current_slot); + int piece_index = identify_data(m_piece_data.get(), m_current_slot); if (piece_index >= 0) have_piece = piece_index; diff --git a/src/torrent_info.cpp b/src/torrent_info.cpp index 4b4ca9b8e..c7846c720 100644 --- a/src/torrent_info.cpp +++ b/src/torrent_info.cpp @@ -252,16 +252,13 @@ namespace libtorrent file f; error_code ec; if (!f.open(filename, file::read_only, ec)) return -1; - f.seek(0, file::end, ec); - if (ec) return -1; - size_type s = f.tell(ec); + size_type s = f.get_size(ec); if (ec) return -1; if (s > 5000000) return -2; v.resize(s); if (s == 0) return 0; - f.seek(0, file::begin, ec); - if (ec) return -1; - size_type read = f.read(&v[0], s, ec); + file::iovec_t b = {&v[0], s}; + size_type read = f.readv(0, &b, 1, ec); if (read != s) return -3; if (ec) return -3; return 0;