From 13ca386838914e575564df739b01d96a65684b38 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Tue, 17 Feb 2009 00:11:38 +0000 Subject: [PATCH] improved support for sparse files on windows --- ChangeLog | 1 + docs/manual.rst | 12 ++++++++ include/libtorrent/file.hpp | 6 +++- include/libtorrent/storage.hpp | 5 ++++ src/file.cpp | 43 ++++++++++++++++++++++++++- src/storage.cpp | 53 ++++++++++++++++++++++++++++++++-- test/test_storage.cpp | 39 ++++++++++++++++++------- 7 files changed, 145 insertions(+), 14 deletions(-) diff --git a/ChangeLog b/ChangeLog index cd83b28c1..88e5ce347 100644 --- a/ChangeLog +++ b/ChangeLog @@ -26,6 +26,7 @@ * added bandwidth reports for estimated TCP/IP overhead and DHT * includes DHT traffic in the rate limiter * added support for bitcomet padding files + * improved support for sparse files on windows release 0.14.2 diff --git a/docs/manual.rst b/docs/manual.rst index b0bc34f26..16a5487ea 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -5000,6 +5000,7 @@ this:: virtual bool initialize(bool allocate_files) = 0; virtual int readv(file::iovec_t const* bufs, int slot, int offset, int num_bufs) = 0; virtual int writev(file::iovec_t const* bufs, int slot, int offset, int num_bufs) = 0; + virtual int sparse_end(int start) const; virtual bool move_storage(fs::path save_path) = 0; virtual bool verify_resume_data(lazy_entry const& rd, std::string& error) = 0; virtual bool write_resume_data(entry& rd) const = 0; @@ -5067,6 +5068,17 @@ exceptions when it's not. Specifically if the read cache is disabled/or full and client requests unaligned data, or the file itself is not aligned in the torrent. Most clients request aligned data. +sparse_end() +------------ + + :: + + int sparse_end(int start) const; + +This function is optional. It is supposed to return the first piece, starting at +``start`` that is fully contained within a data-region on disk (i.e. non-sparse +region). The purpose of this is to skip parts of files that can be known to contain +zeros when checking files. move_storage() -------------- diff --git a/include/libtorrent/file.hpp b/include/libtorrent/file.hpp index 1b16d0c7e..9603e38e1 100644 --- a/include/libtorrent/file.hpp +++ b/include/libtorrent/file.hpp @@ -140,7 +140,11 @@ namespace libtorrent size_type writev(size_type file_offset, iovec_t const* bufs, int num_bufs, error_code& ec); size_type readv(size_type file_offset, iovec_t const* bufs, int num_bufs, error_code& ec); - size_type get_size(error_code& ec); + size_type get_size(error_code& ec) const; + + // return the offset of the first byte that + // belongs to a data-region + size_type sparse_end(size_type start) const; private: diff --git a/include/libtorrent/storage.hpp b/include/libtorrent/storage.hpp index 57f6b6a59..ae6470c5d 100644 --- a/include/libtorrent/storage.hpp +++ b/include/libtorrent/storage.hpp @@ -134,6 +134,11 @@ namespace libtorrent // negative return value indicates an error virtual int write(const char* buf, int slot, int offset, int size) = 0; + // returns the end of the sparse region the slot 'start' + // resides in i.e. the next slot with content. If start + // is not in a sparse region, start itself is returned + virtual int sparse_end(int start) const { return start; } + // non-zero return value indicates an error virtual bool move_storage(fs::path save_path) = 0; diff --git a/src/file.cpp b/src/file.cpp index 0cfc7da6c..83d74da73 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -795,7 +795,7 @@ namespace libtorrent return true; } - size_type file::get_size(error_code& ec) + size_type file::get_size(error_code& ec) const { #ifdef TORRENT_WINDOWS LARGE_INTEGER file_size; @@ -815,5 +815,46 @@ namespace libtorrent return fs.st_size; #endif } + + size_type file::sparse_end(size_type start) const + { +#ifdef TORRENT_WINDOWS + FILE_ALLOCATED_RANGE_BUFFER buffer; + DWORD bytes_returned = 0; + FILE_ALLOCATED_RANGE_BUFFER in; + error_code ec; + size_type file_size = get_size(ec); + if (ec) return start; + in.FileOffset.QuadPart = start; + in.Length.QuadPart = file_size - start; + if (!DeviceIoControl(m_file_handle, FSCTL_QUERY_ALLOCATED_RANGES + , &in, sizeof(FILE_ALLOCATED_RANGE_BUFFER) + , &buffer, sizeof(FILE_ALLOCATED_RANGE_BUFFER), &bytes_returned, 0)) + { + int err = GetLastError(); + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) return start; + } + + // if there are no allocated regions within the rest + // of the file, return the end of the file + if (bytes_returned == 0) return file_size; + + // assume that this range overlaps the start of the + // region we were interested in, and that start actually + // resides in an allocated region. + if (buffer.FileOffset.QuadPart < start) return start; + + // return the offset to the next allocated region + return buffer.FileOffset.QuadPart; + +#elif defined SEEK_DATA + // this is supported on solaris + size_type ret = lseek(m_fd, start, SEEK_DATA); + if (ret < 0) return start; +#else + return start; +#endif + } + } diff --git a/src/storage.cpp b/src/storage.cpp index 8db6d1f1f..9974a4428 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -439,6 +439,7 @@ namespace libtorrent bool move_storage(fs::path save_path); int read(char* buf, int slot, int offset, int size); int write(char const* buf, int slot, int offset, int size); + int sparse_end(int start) const; int readv(file::iovec_t const* bufs, int slot, int offset, int num_bufs); int writev(file::iovec_t const* buf, int slot, int offset, int num_bufs); bool move_slot(int src_slot, int dst_slot); @@ -762,6 +763,43 @@ namespace libtorrent return false; } + int storage::sparse_end(int slot) const + { + TORRENT_ASSERT(slot >= 0); + TORRENT_ASSERT(slot < m_files.num_pieces()); + + size_type file_offset = (size_type)slot * m_files.piece_length(); + std::vector::const_iterator file_iter; + + for (file_iter = files().begin();;) + { + if (file_offset < file_iter->size) + break; + + file_offset -= file_iter->size; + ++file_iter; + TORRENT_ASSERT(file_iter != files().end()); + } + + fs::path path = m_save_path / file_iter->path; + error_code ec; + int mode = file::read_only; + + boost::shared_ptr file_handle; + int cache_setting = m_settings ? settings().disk_io_write_mode : 0; + if (cache_setting == session_settings::disable_os_cache + || (cache_setting == session_settings::disable_os_cache_for_aligned_files + && ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0)) + mode |= file::no_buffer; + if (!m_allocate_files) mode |= file::sparse; + + file_handle = m_pool.open_file(const_cast(this), path, mode, ec); + if (!file_handle || ec) return slot; + + size_type data_start = file_handle->sparse_end(file_offset); + return (data_start + m_files.piece_length() - 1) / m_files.piece_length(); + } + bool storage::verify_resume_data(lazy_entry const& rd, std::string& error) { lazy_entry const* file_priority = rd.dict_find_list("file_priority"); @@ -2230,7 +2268,7 @@ ret: return fatal_disk_error; } - if (skip) + if (skip > 0) { clear_error(); // skip means that the piece we checked failed to be read from disk @@ -2240,7 +2278,7 @@ ret: if (m_storage_mode == storage_mode_compact) { - for (int i = m_current_slot; i < m_current_slot + skip; ++i) + for (int i = m_current_slot; i < m_current_slot + skip - 1; ++i) { TORRENT_ASSERT(m_slot_to_piece[i] == unallocated); m_unallocated_slots.push_back(i); @@ -2573,6 +2611,17 @@ ret: TORRENT_ASSERT(m_slot_to_piece[m_current_slot] == unassigned || m_piece_to_slot[m_slot_to_piece[m_current_slot]] == m_current_slot); } + + if (piece_index == unassigned) + { + // the data did not match any piece. Maybe we're reading + // from a sparse region, see if we are and skip + if (m_current_slot == m_files.num_pieces() -1) return 0; + + int next_slot = m_storage->sparse_end(m_current_slot + 1); + if (next_slot > m_current_slot + 1) return next_slot - m_current_slot; + } + return 0; } diff --git a/test/test_storage.cpp b/test/test_storage.cpp index e4c20f82d..899856662 100644 --- a/test/test_storage.cpp +++ b/test/test_storage.cpp @@ -61,7 +61,7 @@ void on_read_piece(int ret, disk_io_job const& j, char const* data, int size) { std::cerr << "on_read_piece piece: " << j.piece << std::endl; TEST_CHECK(ret == size); - TEST_CHECK(std::equal(j.buffer, j.buffer + ret, data)); + if (ret > 0) TEST_CHECK(std::equal(j.buffer, j.buffer + ret, data)); } void on_check_resume_data(int ret, disk_io_job const& j) @@ -98,6 +98,14 @@ void on_move_storage(int ret, disk_io_job const& j, std::string path) TEST_CHECK(j.str == path); } +void print_error(int ret, boost::scoped_ptr const& s) +{ + std::cerr << "returned: " << ret + << " error: " << s->error().message() + << " file: " << s->error_file() + << std::endl; +} + void run_storage_tests(boost::intrusive_ptr info , file_storage& fs , path const& test_path @@ -125,31 +133,42 @@ void run_storage_tests(boost::intrusive_ptr info s->m_settings = &set; s->m_disk_pool = &dp; + int ret = 0; + // write piece 1 (in slot 0) - s->write(piece1, 0, 0, half); - s->write(piece1 + half, 0, half, half); + ret = s->write(piece1, 0, 0, half); + if (ret != half) print_error(ret, s); + ret = s->write(piece1 + half, 0, half, half); + if (ret != half) print_error(ret, s); // test unaligned read (where the bytes are aligned) - s->read(piece + 3, 0, 3, piece_size-9); + ret = s->read(piece + 3, 0, 3, piece_size-9); + if (ret != piece_size - 9) print_error(ret, s); TEST_CHECK(std::equal(piece+3, piece + piece_size-9, piece1+3)); // test unaligned read (where the bytes are not aligned) - s->read(piece, 0, 3, piece_size-9); + ret = s->read(piece, 0, 3, piece_size-9); + if (ret != piece_size - 9) print_error(ret, s); TEST_CHECK(std::equal(piece, piece + piece_size-9, piece1+3)); // verify piece 1 - TEST_CHECK(s->read(piece, 0, 0, piece_size) == piece_size); + ret = s->read(piece, 0, 0, piece_size); + if (ret != piece_size) print_error(ret, s); TEST_CHECK(std::equal(piece, piece + piece_size, piece1)); // do the same with piece 0 and 2 (in slot 1 and 2) - s->write(piece0, 1, 0, piece_size); - s->write(piece2, 2, 0, piece_size); + ret = s->write(piece0, 1, 0, piece_size); + if (ret != piece_size) print_error(ret, s); + ret = s->write(piece2, 2, 0, piece_size); + if (ret != piece_size) print_error(ret, s); // verify piece 0 and 2 - TEST_CHECK(s->read(piece, 1, 0, piece_size) == piece_size); + ret = s->read(piece, 1, 0, piece_size); + if (ret != piece_size) print_error(ret, s); TEST_CHECK(std::equal(piece, piece + piece_size, piece0)); - s->read(piece, 2, 0, piece_size); + ret = s->read(piece, 2, 0, piece_size); + if (ret != piece_size) print_error(ret, s); TEST_CHECK(std::equal(piece, piece + piece_size, piece2)); s->release_files();