improved support for sparse files on windows

ChangeLog

@@ -26,6 +26,7 @@
 	* added bandwidth reports for estimated TCP/IP overhead and DHT
 	* includes DHT traffic in the rate limiter
 	* added support for bitcomet padding files
+	* improved support for sparse files on windows
 
 release 0.14.2
 

docs/manual.rst

@@ -5000,6 +5000,7 @@ this::
 		virtual bool initialize(bool allocate_files) = 0;
 		virtual int readv(file::iovec_t const* bufs, int slot, int offset, int num_bufs) = 0;
 		virtual int writev(file::iovec_t const* bufs, int slot, int offset, int num_bufs) = 0;
+		virtual int sparse_end(int start) const;
 		virtual bool move_storage(fs::path save_path) = 0;
 		virtual bool verify_resume_data(lazy_entry const& rd, std::string& error) = 0;
 		virtual bool write_resume_data(entry& rd) const = 0;
@@ -5067,6 +5068,17 @@ exceptions when it's not. Specifically if the read cache is disabled/or full and
 client requests unaligned data, or the file itself is not aligned in the torrent.
 Most clients request aligned data.
 
+sparse_end()
+------------
+
+	::
+
+		int sparse_end(int start) const;
+
+This function is optional. It is supposed to return the first piece, starting at
+``start`` that is fully contained within a data-region on disk (i.e. non-sparse
+region). The purpose of this is to skip parts of files that can be known to contain
+zeros when checking files.
 
 move_storage()
 --------------

include/libtorrent/file.hpp

@@ -140,7 +140,11 @@ namespace libtorrent
 		size_type writev(size_type file_offset, iovec_t const* bufs, int num_bufs, error_code& ec);
 		size_type readv(size_type file_offset, iovec_t const* bufs, int num_bufs, error_code& ec);
 
-		size_type get_size(error_code& ec);
+		size_type get_size(error_code& ec) const;
+
+		// return the offset of the first byte that
+		// belongs to a data-region
+		size_type sparse_end(size_type start) const;
 
 	private:
 

include/libtorrent/storage.hpp

@@ -134,6 +134,11 @@ namespace libtorrent
 		// negative return value indicates an error
 		virtual int write(const char* buf, int slot, int offset, int size) = 0;
 
+		// returns the end of the sparse region the slot 'start'
+		// resides in i.e. the next slot with content. If start
+		// is not in a sparse region, start itself is returned
+		virtual int sparse_end(int start) const { return start; }
+
 		// non-zero return value indicates an error
 		virtual bool move_storage(fs::path save_path) = 0;
 

src/file.cpp

@@ -795,7 +795,7 @@ namespace libtorrent
 		return true;
 	}
 
-	size_type file::get_size(error_code& ec)
+	size_type file::get_size(error_code& ec) const
 	{
 #ifdef TORRENT_WINDOWS
 		LARGE_INTEGER file_size;
@@ -815,5 +815,46 @@ namespace libtorrent
 		return fs.st_size;
 #endif
 	}
+
+	size_type file::sparse_end(size_type start) const
+	{
+#ifdef TORRENT_WINDOWS
+		FILE_ALLOCATED_RANGE_BUFFER buffer;
+		DWORD bytes_returned = 0;
+		FILE_ALLOCATED_RANGE_BUFFER in;
+		error_code ec;
+		size_type file_size = get_size(ec);
+		if (ec) return start;
+		in.FileOffset.QuadPart = start;
+		in.Length.QuadPart = file_size - start;
+		if (!DeviceIoControl(m_file_handle, FSCTL_QUERY_ALLOCATED_RANGES
+			, &in, sizeof(FILE_ALLOCATED_RANGE_BUFFER)
+			, &buffer, sizeof(FILE_ALLOCATED_RANGE_BUFFER), &bytes_returned, 0))
+		{
+			int err = GetLastError();
+			if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) return start;
+		}
+
+		// if there are no allocated regions within the rest
+		// of the file, return the end of the file
+		if (bytes_returned == 0) return file_size;
+
+		// assume that this range overlaps the start of the
+		// region we were interested in, and that start actually
+		// resides in an allocated region.
+		if (buffer.FileOffset.QuadPart < start) return start;
+
+		// return the offset to the next allocated region
+		return buffer.FileOffset.QuadPart;
+		
+#elif defined SEEK_DATA
+		// this is supported on solaris
+		size_type ret = lseek(m_fd, start, SEEK_DATA);
+		if (ret < 0) return start;
+#else
+		return start;
+#endif
+	}
+
 }
 

src/storage.cpp

@@ -439,6 +439,7 @@ namespace libtorrent
 		bool move_storage(fs::path save_path);
 		int read(char* buf, int slot, int offset, int size);
 		int write(char const* buf, int slot, int offset, int size);
+		int sparse_end(int start) const;
 		int readv(file::iovec_t const* bufs, int slot, int offset, int num_bufs);
 		int writev(file::iovec_t const* buf, int slot, int offset, int num_bufs);
 		bool move_slot(int src_slot, int dst_slot);
@@ -762,6 +763,43 @@ namespace libtorrent
 		return false;
 	}
 
+	int storage::sparse_end(int slot) const
+	{
+		TORRENT_ASSERT(slot >= 0);
+		TORRENT_ASSERT(slot < m_files.num_pieces());
+
+		size_type file_offset = (size_type)slot * m_files.piece_length();
+		std::vector<file_entry>::const_iterator file_iter;
+
+		for (file_iter = files().begin();;)
+		{
+			if (file_offset < file_iter->size)
+				break;
+
+			file_offset -= file_iter->size;
+			++file_iter;
+			TORRENT_ASSERT(file_iter != files().end());
+		}
+	
+		fs::path path = m_save_path / file_iter->path;
+		error_code ec;
+		int mode = file::read_only;
+
+		boost::shared_ptr<file> file_handle;
+		int cache_setting = m_settings ? settings().disk_io_write_mode : 0;
+		if (cache_setting == session_settings::disable_os_cache
+			|| (cache_setting == session_settings::disable_os_cache_for_aligned_files
+			&& ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0))
+			mode |= file::no_buffer;
+		if (!m_allocate_files) mode |= file::sparse;
+
+		file_handle = m_pool.open_file(const_cast<storage*>(this), path, mode, ec);
+		if (!file_handle || ec) return slot;
+
+		size_type data_start = file_handle->sparse_end(file_offset);
+		return (data_start + m_files.piece_length() - 1) / m_files.piece_length();
+	}
+
 	bool storage::verify_resume_data(lazy_entry const& rd, std::string& error)
 	{
 		lazy_entry const* file_priority = rd.dict_find_list("file_priority");
@@ -2230,7 +2268,7 @@ ret:
 			return fatal_disk_error;
 		}
 
-		if (skip)
+		if (skip > 0)
 		{
 			clear_error();
 			// skip means that the piece we checked failed to be read from disk
@@ -2240,7 +2278,7 @@ ret:
 
 			if (m_storage_mode == storage_mode_compact)
 			{
-				for (int i = m_current_slot; i < m_current_slot + skip; ++i)
+				for (int i = m_current_slot; i < m_current_slot + skip - 1; ++i)
 				{
 					TORRENT_ASSERT(m_slot_to_piece[i] == unallocated);
 					m_unallocated_slots.push_back(i);
@@ -2573,6 +2611,17 @@ ret:
 			TORRENT_ASSERT(m_slot_to_piece[m_current_slot] == unassigned
 				|| m_piece_to_slot[m_slot_to_piece[m_current_slot]] == m_current_slot);
 		}
+
+		if (piece_index == unassigned)
+		{
+			// the data did not match any piece. Maybe we're reading
+			// from a sparse region, see if we are and skip
+			if (m_current_slot == m_files.num_pieces() -1) return 0;
+
+			int next_slot = m_storage->sparse_end(m_current_slot + 1);
+			if (next_slot > m_current_slot + 1) return next_slot - m_current_slot;
+		}
+
 		return 0;
 	}
 

test/test_storage.cpp

@@ -61,7 +61,7 @@ void on_read_piece(int ret, disk_io_job const& j, char const* data, int size)
 {
 	std::cerr << "on_read_piece piece: " << j.piece << std::endl;
 	TEST_CHECK(ret == size);
-	TEST_CHECK(std::equal(j.buffer, j.buffer + ret, data));
+	if (ret > 0) TEST_CHECK(std::equal(j.buffer, j.buffer + ret, data));
 }
 
 void on_check_resume_data(int ret, disk_io_job const& j)
@@ -98,6 +98,14 @@ void on_move_storage(int ret, disk_io_job const& j, std::string path)
 	TEST_CHECK(j.str == path);
 }
 
+void print_error(int ret, boost::scoped_ptr<storage_interface> const& s)
+{
+	std::cerr << "returned: " << ret
+		<< " error: " << s->error().message()
+		<< " file: " << s->error_file()
+		<< std::endl;
+}
+
 void run_storage_tests(boost::intrusive_ptr<torrent_info> info
 	, file_storage& fs
 	, path const& test_path
@@ -125,31 +133,42 @@ void run_storage_tests(boost::intrusive_ptr<torrent_info> info
 	s->m_settings = &set;
 	s->m_disk_pool = &dp;
 
+	int ret = 0;
+
 	// write piece 1 (in slot 0)
-	s->write(piece1, 0, 0, half);
-	s->write(piece1 + half, 0, half, half);
+	ret = s->write(piece1, 0, 0, half);
+	if (ret != half) print_error(ret, s);
+	ret = s->write(piece1 + half, 0, half, half);
+	if (ret != half) print_error(ret, s);
 
 	// test unaligned read (where the bytes are aligned)
-	s->read(piece + 3, 0, 3, piece_size-9);
+	ret = s->read(piece + 3, 0, 3, piece_size-9);
+	if (ret != piece_size - 9) print_error(ret, s);
 	TEST_CHECK(std::equal(piece+3, piece + piece_size-9, piece1+3));
 	
 	// test unaligned read (where the bytes are not aligned)
-	s->read(piece, 0, 3, piece_size-9);
+	ret = s->read(piece, 0, 3, piece_size-9);
+	if (ret != piece_size - 9) print_error(ret, s);
 	TEST_CHECK(std::equal(piece, piece + piece_size-9, piece1+3));
 
 	// verify piece 1
-	TEST_CHECK(s->read(piece, 0, 0, piece_size) == piece_size);
+	ret = s->read(piece, 0, 0, piece_size);
+	if (ret != piece_size) print_error(ret, s);
 	TEST_CHECK(std::equal(piece, piece + piece_size, piece1));
 	
 	// do the same with piece 0 and 2 (in slot 1 and 2)
-	s->write(piece0, 1, 0, piece_size);
-	s->write(piece2, 2, 0, piece_size);
+	ret = s->write(piece0, 1, 0, piece_size);
+	if (ret != piece_size) print_error(ret, s);
+	ret = s->write(piece2, 2, 0, piece_size);
+	if (ret != piece_size) print_error(ret, s);
 
 	// verify piece 0 and 2
-	TEST_CHECK(s->read(piece, 1, 0, piece_size) == piece_size);
+	ret = s->read(piece, 1, 0, piece_size);
+	if (ret != piece_size) print_error(ret, s);
 	TEST_CHECK(std::equal(piece, piece + piece_size, piece0));
 
-	s->read(piece, 2, 0, piece_size);
+	ret = s->read(piece, 2, 0, piece_size);
+	if (ret != piece_size) print_error(ret, s);
 	TEST_CHECK(std::equal(piece, piece + piece_size, piece2));
 
 	s->release_files();