From e45c71dd30461723a19e29b8dc9e4ff5aae68234 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Sat, 16 Feb 2013 08:26:55 +0000 Subject: [PATCH] merged sparse file fix from RC_0_16 --- ChangeLog | 1 + include/libtorrent/disk_io_thread.hpp | 2 + include/libtorrent/file.hpp | 4 - include/libtorrent/storage.hpp | 8 +- src/disk_io_thread.cpp | 10 +- src/file.cpp | 142 ++++++++++++++++++++++---- src/storage.cpp | 27 +---- src/torrent.cpp | 8 -- test/test_storage.cpp | 7 ++ test/test_web_seed.cpp | 2 + 10 files changed, 146 insertions(+), 65 deletions(-) diff --git a/ChangeLog b/ChangeLog index 623fa2c9a..8568a4f4a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,7 @@ * fix uTP edge case where udp socket buffer fills up * fix nagle implementation in uTP + * fixed sparse flag manipulation on windows * fixed streaming piece picking issue 0.16.8 release diff --git a/include/libtorrent/disk_io_thread.hpp b/include/libtorrent/disk_io_thread.hpp index 2857d909a..2952dcb83 100644 --- a/include/libtorrent/disk_io_thread.hpp +++ b/include/libtorrent/disk_io_thread.hpp @@ -101,7 +101,9 @@ namespace libtorrent , update_settings , read_and_hash , cache_piece +#ifndef TORRENT_NO_DEPRECATE , finalize_file +#endif }; action_t action; diff --git a/include/libtorrent/file.hpp b/include/libtorrent/file.hpp index 0cc08c834..3d967be8d 100644 --- a/include/libtorrent/file.hpp +++ b/include/libtorrent/file.hpp @@ -224,10 +224,6 @@ namespace libtorrent void close(); bool set_size(size_type size, error_code& ec); - // called when we're done writing to the file. - // On windows this will clear the sparse bit - void finalize(); - int open_mode() const { return m_open_mode; } // when opened in unbuffered mode, this is the diff --git a/include/libtorrent/storage.hpp b/include/libtorrent/storage.hpp index 9d1df21e6..f618d5386 100644 --- a/include/libtorrent/storage.hpp +++ b/include/libtorrent/storage.hpp @@ -160,7 +160,9 @@ namespace libtorrent // non-zero return value indicates an error virtual bool delete_files() = 0; +#ifndef TORRENT_NO_DEPRECATE virtual void finalize_file(int) {} +#endif disk_buffer_pool* disk_pool() { return m_disk_pool; } session_settings const& settings() const { return *m_settings; } @@ -187,7 +189,9 @@ namespace libtorrent , file_pool& fp, std::vector const& file_prio); ~default_storage(); +#ifndef TORRENT_NO_DEPRECATE void finalize_file(int file); +#endif bool has_any_file(); bool rename_file(int index, std::string const& new_filename); bool release_files(); @@ -307,8 +311,6 @@ namespace libtorrent boost::intrusive_ptr info() const { return m_info; } void write_resume_data(entry& rd) const; - void async_finalize_file(int file); - void async_check_fastresume(lazy_entry const* resume_data , boost::function const& handler); @@ -437,8 +439,6 @@ namespace libtorrent size_type physical_offset(int piece_index, int offset); - void finalize_file(int index); - // returns the number of pieces left in the // file currently being checked int skip_file() const; diff --git a/src/disk_io_thread.cpp b/src/disk_io_thread.cpp index 10112341b..06ab2bdf8 100644 --- a/src/disk_io_thread.cpp +++ b/src/disk_io_thread.cpp @@ -1420,7 +1420,9 @@ namespace libtorrent , cancel_on_abort // update_settings , read_operation + cancel_on_abort // read_and_hash , read_operation + cancel_on_abort // cache_piece +#ifndef TORRENT_NO_DEPRECATE , 0 // finalize_file +#endif }; bool should_cancel_on_abort(disk_io_job const& j) @@ -1963,14 +1965,10 @@ namespace libtorrent #endif break; } +#ifndef TORRENT_NO_DEPRECATE case disk_io_job::finalize_file: - { -#ifdef TORRENT_DISK_STATS - m_log << log_time() << " finalize_file " << j.piece << std::endl; -#endif - j.storage->finalize_file(j.piece); break; - } +#endif case disk_io_job::read: { if (test_error(j)) diff --git a/src/file.cpp b/src/file.cpp index 2b5670cdc..9ea4babba 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -842,6 +842,47 @@ namespace libtorrent #endif } +#ifdef TORRENT_WINDOWS + struct overlapped_t + { + overlapped_t() + { + memset(&ol, 0, sizeof(ol)); + ol.hEvent = CreateEvent(0, true, false, 0); + } + ~overlapped_t() + { + if (ol.hEvent != INVALID_HANDLE_VALUE) + CloseHandle(ol.hEvent); + } + int wait(HANDLE file, error_code& ec) + { + if (WaitForSingleObject(ol.hEvent, INFINITE) == WAIT_FAILED) + { + ec.assign(GetLastError(), get_system_category()); + return -1; + } + + DWORD ret = -1; + if (GetOverlappedResult(file, &ol, &ret, false) == 0) + { + DWORD last_error = GetLastError(); + if (last_error != ERROR_HANDLE_EOF) + { +#ifndef TORRENT_MINGW + TORRENT_ASSERT(last_error != ERROR_CANT_WAIT); +#endif + ec.assign(last_error, get_system_category()); + return -1; + } + } + return ret; + } + + OVERLAPPED ol; + }; +#endif // TORRENT_WINDOWS + file::file() #ifdef TORRENT_WINDOWS : m_file_handle(INVALID_HANDLE_VALUE) @@ -956,8 +997,13 @@ namespace libtorrent if ((mode & file::sparse) && (mode & rw_mask) != read_only) { DWORD temp; + bool use_overlapped = m_open_mode & no_buffer; + overlapped_t ol; ::DeviceIoControl(m_file_handle, FSCTL_SET_SPARSE, 0, 0 - , 0, 0, &temp, 0); + , 0, 0, &temp, use_overlapped ? &ol.ol : NULL); + error_code error; + if (use_overlapped) + ol.wait(m_file_handle, error); } #else // TORRENT_WINDOWS @@ -1157,6 +1203,50 @@ namespace libtorrent #endif } +#ifdef TORRENT_WINDOWS + bool is_sparse(HANDLE file, bool overlapped) + { + LARGE_INTEGER file_size; + if (!GetFileSizeEx(file, &file_size)) + return -1; + + overlapped_t ol; + if (ol.ol.hEvent == NULL) return -1; + +#ifdef TORRENT_MINGW +typedef struct _FILE_ALLOCATED_RANGE_BUFFER { + LARGE_INTEGER FileOffset; + LARGE_INTEGER Length; +} FILE_ALLOCATED_RANGE_BUFFER, *PFILE_ALLOCATED_RANGE_BUFFER; +#define FSCTL_QUERY_ALLOCATED_RANGES ((0x9 << 16) | (1 << 14) | (51 << 2) | 3) +#endif + FILE_ALLOCATED_RANGE_BUFFER in; + in.FileOffset.QuadPart = 0; + in.Length.QuadPart = file_size.QuadPart; + + FILE_ALLOCATED_RANGE_BUFFER out[2]; + + DWORD returned_bytes = 0; + BOOL ret = DeviceIoControl(file, FSCTL_QUERY_ALLOCATED_RANGES, (void*)&in, sizeof(in) + , out, sizeof(out), &returned_bytes, overlapped ? &ol.ol : NULL); + + if (overlapped) + { + error_code ec; + returned_bytes = ol.wait(file, ec); + if (ec) return true; + } + else if (ret == FALSE) + { + int error = GetLastError(); + return true; + } + + // if we only have a single range in the file, we're not sparse + return returned_bytes != sizeof(FILE_ALLOCATED_RANGE_BUFFER); + } +#endif + void file::close() { #if defined TORRENT_WINDOWS || defined TORRENT_LINUX @@ -1165,6 +1255,36 @@ namespace libtorrent #ifdef TORRENT_WINDOWS if (m_file_handle == INVALID_HANDLE_VALUE) return; + + // if this file is open for writing, has the sparse + // flag set, but there are no sparse regions, unset + // the flag + int rw_mode = m_open_mode & rw_mask; + bool use_overlapped = m_open_mode & no_buffer; + if ((rw_mode == read_write || rw_mode == write_only) + && (m_open_mode & sparse) + && !is_sparse(m_file_handle, use_overlapped)) + { + overlapped_t ol; + // according to MSDN, clearing the sparse flag of a file only + // works on windows vista and later +#ifdef TORRENT_MINGW + typedef struct _FILE_SET_SPARSE_BUFFER { + BOOLEAN SetSparse; + } FILE_SET_SPARSE_BUFFER, *PFILE_SET_SPARSE_BUFFER; +#endif + DWORD temp; + FILE_SET_SPARSE_BUFFER b; + b.SetSparse = FALSE; + int ret = ::DeviceIoControl(m_file_handle, FSCTL_SET_SPARSE, &b, sizeof(b) + , 0, 0, &temp, use_overlapped ? &ol.ol : NULL); + error_code ec; + if (use_overlapped) + { + ol.wait(m_file_handle, ec); + } + } + CloseHandle(m_file_handle); m_file_handle = INVALID_HANDLE_VALUE; m_path.clear(); @@ -1318,6 +1438,7 @@ namespace libtorrent cur_seg->Buffer = 0; OVERLAPPED ol; + memset(&ol, 0, sizeof(ol)); ol.Internal = 0; ol.InternalHigh = 0; ol.OffsetHigh = DWORD(file_offset >> 32); @@ -1552,6 +1673,7 @@ namespace libtorrent cur_seg->Buffer = 0; OVERLAPPED ol; + memset(&ol, 0, sizeof(ol)); ol.Internal = 0; ol.InternalHigh = 0; ol.OffsetHigh = DWORD(file_offset >> 32); @@ -1998,24 +2120,6 @@ namespace libtorrent return true; } - void file::finalize() - { -#ifdef TORRENT_WINDOWS - // according to MSDN, clearing the sparse flag of a file only - // works on windows vista and later -#ifdef TORRENT_MINGW -typedef struct _FILE_SET_SPARSE_BUFFER { - BOOLEAN SetSparse; -} FILE_SET_SPARSE_BUFFER, *PFILE_SET_SPARSE_BUFFER; -#endif - DWORD temp; - FILE_SET_SPARSE_BUFFER b; - b.SetSparse = FALSE; - ::DeviceIoControl(m_file_handle, FSCTL_SET_SPARSE, &b, sizeof(b) - , 0, 0, &temp, 0); -#endif - } - size_type file::get_size(error_code& ec) const { #ifdef TORRENT_WINDOWS diff --git a/src/storage.cpp b/src/storage.cpp index 7078f22f4..f746ab205 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -467,17 +467,9 @@ namespace libtorrent return error() ? true : false; } - void default_storage::finalize_file(int index) - { - TORRENT_ASSERT(index >= 0 && index < files().num_files()); - if (index < 0 || index >= files().num_files()) return; - - error_code ec; - boost::intrusive_ptr f = open_file(files().begin() + index, file::read_write, ec); - if (ec || !f) return; - - f->finalize(); - } +#ifndef TORRENT_NO_DEPRECATE + void default_storage::finalize_file(int index) {} +#endif bool default_storage::has_any_file() { @@ -1458,23 +1450,10 @@ ret: m_storage->m_disk_pool = &m_io_thread; } - void piece_manager::finalize_file(int index) - { m_storage->finalize_file(index); } - piece_manager::~piece_manager() { } - void piece_manager::async_finalize_file(int file) - { - disk_io_job j; - j.storage = this; - j.action = disk_io_job::finalize_file; - j.piece = file; - boost::function empty; - m_io_thread.add_job(j, empty); - } - void piece_manager::async_save_resume_data( boost::function const& handler) { diff --git a/src/torrent.cpp b/src/torrent.cpp index 84bb5642a..25277196c 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -3239,14 +3239,6 @@ namespace libtorrent { if (!m_torrent_file->files().pad_file_at(file_index)) { - // don't finalize files if we discover that they exist - // in whole (i.e. while checking). In that case, just assume - // they were finalized when they completed. - // The main purpose of finalizing files is to clear the sparse - // flag on windows, which only needs to be done once - if (m_owning_storage.get() && m_state == torrent_status::downloading) - m_storage->async_finalize_file(file_index); - if (m_ses.m_alerts.should_post()) { // this file just completed, post alert diff --git a/test/test_storage.cpp b/test/test_storage.cpp index bfe358eb3..e45bea3cf 100644 --- a/test/test_storage.cpp +++ b/test/test_storage.cpp @@ -764,6 +764,10 @@ void test_check_files(std::string const& test_path io.join(); } +#ifdef TORRENT_NO_DEPRECATE +#define storage_mode_compact storage_mode_sparse +#endif + void run_test(std::string const& test_path, bool unbuffered) { std::cerr << "\n=== " << test_path << " ===\n" << std::endl; @@ -903,6 +907,7 @@ void test_fastresume(std::string const& test_path) break; } } + // TODO: 3 don't use this deprecated function resume = h.write_resume_data(); ses.remove_torrent(h, session::delete_files); } @@ -990,6 +995,7 @@ void test_rename_file_in_fastresume(std::string const& test_path) std::cout << "stop loop" << std::endl; torrent_status s = h.status(); TEST_CHECK(s.state == torrent_status::seeding); + // TODO: 3 don't use this deprecated function resume = h.write_resume_data(); ses.remove_torrent(h); } @@ -1022,6 +1028,7 @@ void test_rename_file_in_fastresume(std::string const& test_path) torrent_status stat = h.status(); TEST_CHECK(stat.state == torrent_status::seeding); + // TODO: 3 don't use this deprecated function resume = h.write_resume_data(); ses.remove_torrent(h); } diff --git a/test/test_web_seed.cpp b/test/test_web_seed.cpp index d9af99108..32923e086 100644 --- a/test/test_web_seed.cpp +++ b/test/test_web_seed.cpp @@ -84,7 +84,9 @@ void test_transfer(boost::intrusive_ptr torrent_file p.flags &= ~add_torrent_params::flag_auto_managed; p.ti = torrent_file; p.save_path = "tmp2_web_seed"; +#ifndef TORRENT_NO_DEPRECATE p.storage_mode = storage_mode_compact; +#endif torrent_handle th = ses.add_torrent(p, ec); std::vector empty;