merged sparse file fix from RC_0_16

ChangeLog

@@ -13,6 +13,7 @@
 	* fix uTP edge case where udp socket buffer fills up
 	* fix nagle implementation in uTP
 
+	* fixed sparse flag manipulation on windows
 	* fixed streaming piece picking issue
 
 0.16.8 release

include/libtorrent/disk_io_thread.hpp

@@ -101,7 +101,9 @@ namespace libtorrent
 			, update_settings
 			, read_and_hash
 			, cache_piece
+#ifndef TORRENT_NO_DEPRECATE
 			, finalize_file
+#endif
 		};
 
 		action_t action;

include/libtorrent/file.hpp

@@ -224,10 +224,6 @@ namespace libtorrent
 		void close();
 		bool set_size(size_type size, error_code& ec);
 
-		// called when we're done writing to the file.
-		// On windows this will clear the sparse bit
-		void finalize();
-
 		int open_mode() const { return m_open_mode; }
 
 		// when opened in unbuffered mode, this is the

include/libtorrent/storage.hpp

@@ -160,7 +160,9 @@ namespace libtorrent
 		// non-zero return value indicates an error
 		virtual bool delete_files() = 0;
 
+#ifndef TORRENT_NO_DEPRECATE
 		virtual void finalize_file(int) {}
+#endif
 
 		disk_buffer_pool* disk_pool() { return m_disk_pool; }
 		session_settings const& settings() const { return *m_settings; }
@@ -187,7 +189,9 @@ namespace libtorrent
 			, file_pool& fp, std::vector<boost::uint8_t> const& file_prio);
 		~default_storage();
 
+#ifndef TORRENT_NO_DEPRECATE
 		void finalize_file(int file);
+#endif
 		bool has_any_file();
 		bool rename_file(int index, std::string const& new_filename);
 		bool release_files();
@@ -307,8 +311,6 @@ namespace libtorrent
 		boost::intrusive_ptr<torrent_info const> info() const { return m_info; }
 		void write_resume_data(entry& rd) const;
 
-		void async_finalize_file(int file);
-
 		void async_check_fastresume(lazy_entry const* resume_data
 			, boost::function<void(int, disk_io_job const&)> const& handler);
 		
@@ -437,8 +439,6 @@ namespace libtorrent
 
 		size_type physical_offset(int piece_index, int offset);
 
-		void finalize_file(int index);
-
 		// returns the number of pieces left in the
 		// file currently being checked
 		int skip_file() const;

src/disk_io_thread.cpp

@@ -1420,7 +1420,9 @@ namespace libtorrent
 		, cancel_on_abort // update_settings
 		, read_operation + cancel_on_abort // read_and_hash
 		, read_operation + cancel_on_abort // cache_piece
+#ifndef TORRENT_NO_DEPRECATE
 		, 0 // finalize_file
+#endif
 	};
 
 	bool should_cancel_on_abort(disk_io_job const& j)
@@ -1963,14 +1965,10 @@ namespace libtorrent
 #endif
 					break;
 				}
+#ifndef TORRENT_NO_DEPRECATE
 				case disk_io_job::finalize_file:
-				{
-#ifdef TORRENT_DISK_STATS
-					m_log << log_time() << " finalize_file " << j.piece << std::endl;
-#endif
-					j.storage->finalize_file(j.piece);
 					break;
-				}
+#endif
 				case disk_io_job::read:
 				{
 					if (test_error(j))

src/file.cpp

@@ -842,6 +842,47 @@ namespace libtorrent
 #endif
 	}
 
+#ifdef TORRENT_WINDOWS
+	struct overlapped_t
+	{
+		overlapped_t()
+		{
+			memset(&ol, 0, sizeof(ol));
+			ol.hEvent = CreateEvent(0, true, false, 0);
+		}
+		~overlapped_t()
+		{
+			if (ol.hEvent != INVALID_HANDLE_VALUE)
+				CloseHandle(ol.hEvent);
+		}
+		int wait(HANDLE file, error_code& ec)
+		{
+			if (WaitForSingleObject(ol.hEvent, INFINITE) == WAIT_FAILED)
+			{
+				ec.assign(GetLastError(), get_system_category());
+				return -1;
+			}
+
+			DWORD ret = -1;
+			if (GetOverlappedResult(file, &ol, &ret, false) == 0)
+			{
+				DWORD last_error = GetLastError();
+				if (last_error != ERROR_HANDLE_EOF)
+				{
+#ifndef TORRENT_MINGW
+					TORRENT_ASSERT(last_error != ERROR_CANT_WAIT);
+#endif
+					ec.assign(last_error, get_system_category());
+					return -1;
+				}
+			}
+			return ret;
+		}
+
+		OVERLAPPED ol;
+	};
+#endif // TORRENT_WINDOWS
+
 	file::file()
 #ifdef TORRENT_WINDOWS
 		: m_file_handle(INVALID_HANDLE_VALUE)
@@ -956,8 +997,13 @@ namespace libtorrent
 		if ((mode & file::sparse) && (mode & rw_mask) != read_only)
 		{
 			DWORD temp;
+			bool use_overlapped = m_open_mode & no_buffer;
+			overlapped_t ol;
 			::DeviceIoControl(m_file_handle, FSCTL_SET_SPARSE, 0, 0
-				, 0, 0, &temp, 0);
+				, 0, 0, &temp, use_overlapped ? &ol.ol : NULL);
+			error_code error;
+			if (use_overlapped)
+				ol.wait(m_file_handle, error);
 		}
 #else // TORRENT_WINDOWS
 
@@ -1157,6 +1203,50 @@ namespace libtorrent
 #endif
 	}
 
+#ifdef TORRENT_WINDOWS
+	bool is_sparse(HANDLE file, bool overlapped)
+	{
+		LARGE_INTEGER file_size;
+		if (!GetFileSizeEx(file, &file_size))
+			return -1;
+
+		overlapped_t ol;
+		if (ol.ol.hEvent == NULL) return -1;
+
+#ifdef TORRENT_MINGW
+typedef struct _FILE_ALLOCATED_RANGE_BUFFER {
+	LARGE_INTEGER FileOffset;
+	LARGE_INTEGER Length;
+} FILE_ALLOCATED_RANGE_BUFFER, *PFILE_ALLOCATED_RANGE_BUFFER;
+#define FSCTL_QUERY_ALLOCATED_RANGES ((0x9 << 16) | (1 << 14) | (51 << 2) | 3)
+#endif
+		FILE_ALLOCATED_RANGE_BUFFER in;
+		in.FileOffset.QuadPart = 0;
+		in.Length.QuadPart = file_size.QuadPart;
+
+		FILE_ALLOCATED_RANGE_BUFFER out[2];
+
+		DWORD returned_bytes = 0;
+		BOOL ret = DeviceIoControl(file, FSCTL_QUERY_ALLOCATED_RANGES, (void*)&in, sizeof(in)
+			, out, sizeof(out), &returned_bytes, overlapped ? &ol.ol : NULL);
+
+		if (overlapped)
+		{
+			error_code ec;
+			returned_bytes = ol.wait(file, ec);
+			if (ec) return true;
+		}
+		else if (ret == FALSE)
+		{
+			int error = GetLastError();
+			return true;
+		}
+
+		// if we only have a single range in the file, we're not sparse
+		return returned_bytes != sizeof(FILE_ALLOCATED_RANGE_BUFFER);
+	}
+#endif
+
 	void file::close()
 	{
 #if defined TORRENT_WINDOWS || defined TORRENT_LINUX
@@ -1165,6 +1255,36 @@ namespace libtorrent
 
 #ifdef TORRENT_WINDOWS
 		if (m_file_handle == INVALID_HANDLE_VALUE) return;
+
+		// if this file is open for writing, has the sparse
+		// flag set, but there are no sparse regions, unset
+		// the flag
+		int rw_mode = m_open_mode & rw_mask;
+		bool use_overlapped = m_open_mode & no_buffer;
+		if ((rw_mode == read_write || rw_mode == write_only)
+			&& (m_open_mode & sparse)
+			&& !is_sparse(m_file_handle, use_overlapped))
+		{
+			overlapped_t ol;
+			// according to MSDN, clearing the sparse flag of a file only
+			// works on windows vista and later
+#ifdef TORRENT_MINGW
+	typedef struct _FILE_SET_SPARSE_BUFFER {
+		    BOOLEAN SetSparse;
+	} FILE_SET_SPARSE_BUFFER, *PFILE_SET_SPARSE_BUFFER;
+#endif
+			DWORD temp;
+			FILE_SET_SPARSE_BUFFER b;
+			b.SetSparse = FALSE;
+			int ret = ::DeviceIoControl(m_file_handle, FSCTL_SET_SPARSE, &b, sizeof(b)
+				, 0, 0, &temp, use_overlapped ? &ol.ol : NULL);
+			error_code ec;
+			if (use_overlapped)
+			{
+				ol.wait(m_file_handle, ec);
+			}
+		}
+
 		CloseHandle(m_file_handle);
 		m_file_handle = INVALID_HANDLE_VALUE;
 		m_path.clear();
@@ -1318,6 +1438,7 @@ namespace libtorrent
 		cur_seg->Buffer = 0;
 
 		OVERLAPPED ol;
+		memset(&ol, 0, sizeof(ol));
 		ol.Internal = 0;
 		ol.InternalHigh = 0;
 		ol.OffsetHigh = DWORD(file_offset >> 32);
@@ -1552,6 +1673,7 @@ namespace libtorrent
 		cur_seg->Buffer = 0;
 
 		OVERLAPPED ol;
+		memset(&ol, 0, sizeof(ol));
 		ol.Internal = 0;
 		ol.InternalHigh = 0;
 		ol.OffsetHigh = DWORD(file_offset >> 32);
@@ -1998,24 +2120,6 @@ namespace libtorrent
 		return true;
 	}
 
-	void file::finalize()
-	{
-#ifdef TORRENT_WINDOWS
-		// according to MSDN, clearing the sparse flag of a file only
-		// works on windows vista and later
-#ifdef TORRENT_MINGW
-typedef struct _FILE_SET_SPARSE_BUFFER {
-	    BOOLEAN SetSparse;
-} FILE_SET_SPARSE_BUFFER, *PFILE_SET_SPARSE_BUFFER;
-#endif
-		DWORD temp;
-		FILE_SET_SPARSE_BUFFER b;
-		b.SetSparse = FALSE;
-		::DeviceIoControl(m_file_handle, FSCTL_SET_SPARSE, &b, sizeof(b)
-			, 0, 0, &temp, 0);
-#endif
-	}
-
 	size_type file::get_size(error_code& ec) const
 	{
 #ifdef TORRENT_WINDOWS

src/storage.cpp

@@ -467,17 +467,9 @@ namespace libtorrent
 		return error() ? true : false;
 	}
 
-	void default_storage::finalize_file(int index)
+#ifndef TORRENT_NO_DEPRECATE
-	{
+	void default_storage::finalize_file(int index) {}
-		TORRENT_ASSERT(index >= 0 && index < files().num_files());
+#endif
-		if (index < 0 || index >= files().num_files()) return;
-	
-		error_code ec;
-		boost::intrusive_ptr<file> f = open_file(files().begin() + index, file::read_write, ec);
-		if (ec || !f) return;
-
-		f->finalize();
-	}
 
 	bool default_storage::has_any_file()
 	{
@@ -1458,23 +1450,10 @@ ret:
 		m_storage->m_disk_pool = &m_io_thread;
 	}
 
-	void piece_manager::finalize_file(int index)
-	{ m_storage->finalize_file(index); }
-
 	piece_manager::~piece_manager()
 	{
 	}
 
-	void piece_manager::async_finalize_file(int file)
-	{
-		disk_io_job j;
-		j.storage = this;
-		j.action = disk_io_job::finalize_file;
-		j.piece = file;
-		boost::function<void(int, disk_io_job const&)> empty;
-		m_io_thread.add_job(j, empty);
-	}
-
 	void piece_manager::async_save_resume_data(
 		boost::function<void(int, disk_io_job const&)> const& handler)
 	{

src/torrent.cpp

@@ -3239,14 +3239,6 @@ namespace libtorrent
 			{
 				if (!m_torrent_file->files().pad_file_at(file_index))
 				{
-					// don't finalize files if we discover that they exist
-					// in whole (i.e. while checking). In that case, just assume
-					// they were finalized when they completed.
-					// The main purpose of finalizing files is to clear the sparse
-					// flag on windows, which only needs to be done once
-					if (m_owning_storage.get() && m_state == torrent_status::downloading)
-						m_storage->async_finalize_file(file_index);
-
 					if (m_ses.m_alerts.should_post<file_completed_alert>())
 					{
 						// this file just completed, post alert

test/test_storage.cpp

@@ -764,6 +764,10 @@ void test_check_files(std::string const& test_path
 	io.join();
 }
 
+#ifdef TORRENT_NO_DEPRECATE
+#define storage_mode_compact storage_mode_sparse
+#endif
+
 void run_test(std::string const& test_path, bool unbuffered)
 {
 	std::cerr << "\n=== " << test_path << " ===\n" << std::endl;
@@ -903,6 +907,7 @@ void test_fastresume(std::string const& test_path)
 				break;
 			}
 		}
+		// TODO: 3 don't use this deprecated function
 		resume = h.write_resume_data();
 		ses.remove_torrent(h, session::delete_files);
 	}
@@ -990,6 +995,7 @@ void test_rename_file_in_fastresume(std::string const& test_path)
 		std::cout << "stop loop" << std::endl;
 		torrent_status s = h.status();
 		TEST_CHECK(s.state == torrent_status::seeding);
+		// TODO: 3 don't use this deprecated function
 		resume = h.write_resume_data();
 		ses.remove_torrent(h);
 	}
@@ -1022,6 +1028,7 @@ void test_rename_file_in_fastresume(std::string const& test_path)
 		torrent_status stat = h.status();
 		TEST_CHECK(stat.state == torrent_status::seeding);
 
+		// TODO: 3 don't use this deprecated function
 		resume = h.write_resume_data();
 		ses.remove_torrent(h);
 	}

test/test_web_seed.cpp

@@ -84,7 +84,9 @@ void test_transfer(boost::intrusive_ptr<torrent_info> torrent_file
 	p.flags &= ~add_torrent_params::flag_auto_managed;
 	p.ti = torrent_file;
 	p.save_path = "tmp2_web_seed";
+#ifndef TORRENT_NO_DEPRECATE
 	p.storage_mode = storage_mode_compact;
+#endif
 	torrent_handle th = ses.add_torrent(p, ec);
 
 	std::vector<announce_entry> empty;