forked from premiere/premiere-libtorrent
Merge pull request #867 from glassez/storage11
Move storage "file-by-file" (#632)
This commit is contained in:
commit
7ed980659f
|
@ -1,5 +1,6 @@
|
|||
1.1.1 release
|
||||
|
||||
* move files one-by-one when moving storage for a torrent
|
||||
* fix bug in enum_net() for BSD and Mac
|
||||
* fix bug in python binding of announce_entry
|
||||
* fixed bug related to flag_merge_resume_http_seeds flag in add_torrent_params
|
||||
|
|
|
@ -135,6 +135,8 @@ namespace libtorrent
|
|||
, std::string const& new_path, error_code& ec);
|
||||
TORRENT_EXTRA_EXPORT void copy_file(std::string const& f
|
||||
, std::string const& newf, error_code& ec);
|
||||
TORRENT_EXTRA_EXPORT void move_file(std::string const& f
|
||||
, std::string const& newf, error_code& ec);
|
||||
|
||||
// file is expected to exist, link will be created to point to it. If hard
|
||||
// links are not supported by the filesystem or OS, the file will be copied.
|
||||
|
|
|
@ -530,6 +530,11 @@ namespace libtorrent
|
|||
// to file at ``index``.
|
||||
int file_flags(int index) const;
|
||||
|
||||
// returns true if the file at the specified index has been renamed to
|
||||
// have an absolute path, i.e. is not anchored in the save path of the
|
||||
// torrent.
|
||||
bool file_absolute_path(int index) const;
|
||||
|
||||
// returns the index of the file at the given offset in the torrent
|
||||
int file_index_at_offset(boost::int64_t offset) const;
|
||||
|
||||
|
|
42
src/file.cpp
42
src/file.cpp
|
@ -684,6 +684,44 @@ namespace libtorrent
|
|||
#endif // TORRENT_WINDOWS
|
||||
}
|
||||
|
||||
void move_file(std::string const& inf, std::string const& newf, error_code& ec)
|
||||
{
|
||||
ec.clear();
|
||||
|
||||
file_status s;
|
||||
stat_file(inf, &s, ec);
|
||||
if (ec) return;
|
||||
|
||||
if (has_parent_path(newf))
|
||||
{
|
||||
create_directories(parent_path(newf), ec);
|
||||
if (ec) return;
|
||||
}
|
||||
|
||||
rename(inf, newf, ec);
|
||||
|
||||
// on OSX, the error when trying to rename a file across different
|
||||
// volumes is EXDEV, which will make it fall back to copying.
|
||||
|
||||
if (ec)
|
||||
{
|
||||
if (ec != boost::system::errc::no_such_file_or_directory
|
||||
&& ec != boost::system::errc::invalid_argument
|
||||
&& ec != boost::system::errc::permission_denied)
|
||||
{
|
||||
ec.clear();
|
||||
copy_file(inf, newf, ec);
|
||||
|
||||
if (!ec)
|
||||
{
|
||||
// ignore errors when removing
|
||||
error_code ignore;
|
||||
remove(inf, ignore);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::string split_path(std::string const& f)
|
||||
{
|
||||
if (f.empty()) return f;
|
||||
|
@ -879,7 +917,7 @@ namespace libtorrent
|
|||
++len;
|
||||
}
|
||||
return std::string(first, len);
|
||||
|
||||
|
||||
}
|
||||
return std::string(sep + 1);
|
||||
}
|
||||
|
@ -1017,7 +1055,7 @@ namespace libtorrent
|
|||
ret.resize(write_cur - &ret[0]);
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
boost::int64_t file_size(std::string const& f)
|
||||
{
|
||||
|
|
|
@ -816,6 +816,12 @@ namespace libtorrent
|
|||
| (fe.symlink_attribute ? flag_symlink : 0);
|
||||
}
|
||||
|
||||
bool file_storage::file_absolute_path(int index) const
|
||||
{
|
||||
internal_file_entry const& fe = m_files[index];
|
||||
return fe.path_index == -2;
|
||||
}
|
||||
|
||||
#ifndef TORRENT_NO_DEPRECATE
|
||||
void file_storage::set_file_base(int index, boost::int64_t off)
|
||||
{
|
||||
|
|
212
src/storage.cpp
212
src/storage.cpp
|
@ -795,8 +795,8 @@ namespace libtorrent
|
|||
typedef std::set<std::string>::iterator iter_t;
|
||||
for (int i = 0; i < files().num_files(); ++i)
|
||||
{
|
||||
std::string fp = files().file_path(i);
|
||||
bool complete = is_complete(fp);
|
||||
std::string const fp = files().file_path(i);
|
||||
bool const complete = files().file_absolute_path(i);
|
||||
std::string p = complete ? fp : combine_path(m_save_path, fp);
|
||||
if (!complete)
|
||||
{
|
||||
|
@ -1124,32 +1124,32 @@ namespace libtorrent
|
|||
return true;
|
||||
}
|
||||
|
||||
int default_storage::move_storage(std::string const& sp, int flags, storage_error& ec)
|
||||
int default_storage::move_storage(std::string const& sp, int const flags
|
||||
, storage_error& ec)
|
||||
{
|
||||
int ret = piece_manager::no_error;
|
||||
std::string save_path = complete(sp);
|
||||
std::string const save_path = complete(sp);
|
||||
|
||||
// check to see if any of the files exist
|
||||
error_code e;
|
||||
file_storage const& f = files();
|
||||
|
||||
file_status s;
|
||||
if (flags == fail_if_exist)
|
||||
{
|
||||
stat_file(save_path, &s, e);
|
||||
if (e != boost::system::errc::no_such_file_or_directory)
|
||||
file_status s;
|
||||
error_code err;
|
||||
stat_file(save_path, &s, err);
|
||||
if (err != boost::system::errc::no_such_file_or_directory)
|
||||
{
|
||||
// the directory exists, check all the files
|
||||
for (int i = 0; i < f.num_files(); ++i)
|
||||
{
|
||||
// files moved out to absolute paths are ignored
|
||||
if (is_complete(f.file_path(i))) continue;
|
||||
if (f.file_absolute_path(i)) continue;
|
||||
|
||||
std::string new_path = f.file_path(i, save_path);
|
||||
stat_file(new_path, &s, e);
|
||||
if (e != boost::system::errc::no_such_file_or_directory)
|
||||
stat_file(f.file_path(i, save_path), &s, err);
|
||||
if (err != boost::system::errc::no_such_file_or_directory)
|
||||
{
|
||||
ec.ec = e;
|
||||
ec.ec = err;
|
||||
ec.file = i;
|
||||
ec.operation = storage_error::stat;
|
||||
return piece_manager::file_exist;
|
||||
|
@ -1158,39 +1158,30 @@ namespace libtorrent
|
|||
}
|
||||
}
|
||||
|
||||
// collect all directories in to_move. This is because we
|
||||
// try to move entire directories by default (instead of
|
||||
// files independently).
|
||||
std::map<std::string, int> to_move;
|
||||
for (int i = 0; i < f.num_files(); ++i)
|
||||
{
|
||||
// files moved out to absolute paths are not moved
|
||||
if (is_complete(f.file_path(i))) continue;
|
||||
|
||||
std::string split = split_path(f.file_path(i));
|
||||
to_move.insert(to_move.begin(), std::make_pair(split, i));
|
||||
}
|
||||
|
||||
e.clear();
|
||||
stat_file(save_path, &s, e);
|
||||
if (e == boost::system::errc::no_such_file_or_directory)
|
||||
{
|
||||
create_directories(save_path, e);
|
||||
if (e)
|
||||
file_status s;
|
||||
error_code err;
|
||||
stat_file(save_path, &s, err);
|
||||
if (err == boost::system::errc::no_such_file_or_directory)
|
||||
{
|
||||
ec.ec = e;
|
||||
err.clear();
|
||||
create_directories(save_path, err);
|
||||
if (err)
|
||||
{
|
||||
ec.ec = err;
|
||||
ec.file = -1;
|
||||
ec.operation = storage_error::mkdir;
|
||||
return piece_manager::fatal_disk_error;
|
||||
}
|
||||
}
|
||||
else if (err)
|
||||
{
|
||||
ec.ec = err;
|
||||
ec.file = -1;
|
||||
ec.operation = storage_error::mkdir;
|
||||
ec.operation = storage_error::stat;
|
||||
return piece_manager::fatal_disk_error;
|
||||
}
|
||||
}
|
||||
else if (e)
|
||||
{
|
||||
ec.ec = e;
|
||||
ec.file = -1;
|
||||
ec.operation = storage_error::mkdir;
|
||||
return piece_manager::fatal_disk_error;
|
||||
}
|
||||
|
||||
m_pool.release(this);
|
||||
|
||||
|
@ -1198,85 +1189,106 @@ namespace libtorrent
|
|||
print_open_files("release files", m_files.name().c_str());
|
||||
#endif
|
||||
|
||||
for (std::map<std::string, int>::const_iterator i = to_move.begin()
|
||||
, end(to_move.end()); i != end; ++i)
|
||||
int i;
|
||||
error_code e;
|
||||
for (i = 0; i < f.num_files(); ++i)
|
||||
{
|
||||
std::string old_path = combine_path(m_save_path, i->first);
|
||||
std::string new_path = combine_path(save_path, i->first);
|
||||
// files moved out to absolute paths are not moved
|
||||
if (f.file_absolute_path(i)) continue;
|
||||
|
||||
e.clear();
|
||||
rename(old_path, new_path, e);
|
||||
std::string const old_path = combine_path(m_save_path, f.file_path(i));
|
||||
std::string const new_path = combine_path(save_path, f.file_path(i));
|
||||
|
||||
if (flags == dont_replace && exists(new_path))
|
||||
{
|
||||
if (ret == piece_manager::no_error) ret = piece_manager::need_full_check;
|
||||
continue;
|
||||
}
|
||||
|
||||
// TODO: ideally, if we end up copying files because of a move across
|
||||
// volumes, the source should not be deleted until they've all been
|
||||
// copied. That would let us rollback with higher confidence.
|
||||
move_file(old_path, new_path, e);
|
||||
// if the source file doesn't exist. That's not a problem
|
||||
// we just ignore that file
|
||||
if (e == boost::system::errc::no_such_file_or_directory)
|
||||
e.clear();
|
||||
|
||||
// on OSX, the error when trying to rename a file across different
|
||||
// volumes is EXDEV, which will make it fall back to copying.
|
||||
|
||||
if (e)
|
||||
{
|
||||
if (flags == dont_replace && e == boost::system::errc::file_exists)
|
||||
{
|
||||
if (ret == piece_manager::no_error) ret = piece_manager::need_full_check;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (e == boost::system::errc::invalid_argument
|
||||
|| e == boost::system::errc::permission_denied)
|
||||
{
|
||||
ec.ec = e;
|
||||
ec.file = i->second;
|
||||
ec.operation = storage_error::rename;
|
||||
break;
|
||||
}
|
||||
|
||||
if (e != boost::system::errc::no_such_file_or_directory)
|
||||
{
|
||||
e.clear();
|
||||
recursive_copy(old_path, new_path, ec.ec);
|
||||
if (ec.ec == boost::system::errc::no_such_file_or_directory)
|
||||
{
|
||||
// it's a bit weird that rename() would not return
|
||||
// ENOENT, but the file still wouldn't exist. But,
|
||||
// in case it does, we're done.
|
||||
ec.ec.clear();
|
||||
break;
|
||||
}
|
||||
if (ec)
|
||||
{
|
||||
ec.file = i->second;
|
||||
ec.operation = storage_error::copy;
|
||||
}
|
||||
else
|
||||
{
|
||||
// ignore errors when removing
|
||||
error_code ignore;
|
||||
remove_all(old_path, ignore);
|
||||
}
|
||||
break;
|
||||
}
|
||||
ec.ec = e;
|
||||
ec.file = i;
|
||||
ec.operation = storage_error::rename;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!ec)
|
||||
if (!e && m_part_file)
|
||||
{
|
||||
if (m_part_file)
|
||||
m_part_file->move_partfile(save_path, e);
|
||||
if (e)
|
||||
{
|
||||
// TODO: if everything moves OK, except for the partfile
|
||||
// we currently won't update the save path, which breaks things.
|
||||
// it would probably make more sense to give up on the partfile
|
||||
m_part_file->move_partfile(save_path, ec.ec);
|
||||
if (ec)
|
||||
ec.ec = e;
|
||||
ec.file = -1;
|
||||
ec.operation = storage_error::partfile_move;
|
||||
}
|
||||
}
|
||||
|
||||
if (e)
|
||||
{
|
||||
// rollback
|
||||
while (--i >= 0)
|
||||
{
|
||||
// files moved out to absolute paths are not moved
|
||||
if (f.file_absolute_path(i)) continue;
|
||||
|
||||
std::string const old_path = combine_path(m_save_path, f.file_path(i));
|
||||
std::string const new_path = combine_path(save_path, f.file_path(i));
|
||||
|
||||
if (!exists(old_path))
|
||||
{
|
||||
ec.file = -1;
|
||||
ec.operation = storage_error::partfile_move;
|
||||
return piece_manager::fatal_disk_error;
|
||||
// ignore errors when rolling back
|
||||
error_code ignore;
|
||||
move_file(new_path, old_path, ignore);
|
||||
}
|
||||
}
|
||||
|
||||
m_save_path = save_path;
|
||||
return piece_manager::fatal_disk_error;
|
||||
}
|
||||
|
||||
std::string const old_save_path = m_save_path;
|
||||
m_save_path = save_path;
|
||||
|
||||
std::set<std::string> subdirs;
|
||||
for (i = 0; i < f.num_files(); ++i)
|
||||
{
|
||||
// files moved out to absolute paths are not moved
|
||||
if (f.file_absolute_path(i)) continue;
|
||||
|
||||
if (has_parent_path(f.file_path(i)))
|
||||
subdirs.insert(parent_path(f.file_path(i)));
|
||||
|
||||
std::string const old_path = combine_path(old_save_path, f.file_path(i));
|
||||
|
||||
// we may still have some files in old old_save_path
|
||||
// eg. if (flags == dont_replace && exists(new_path))
|
||||
// ignore errors when removing
|
||||
error_code ignore;
|
||||
remove(old_path, ignore);
|
||||
}
|
||||
|
||||
for (std::set<std::string>::iterator it(subdirs.begin())
|
||||
, end(subdirs.end()); it != end; ++it)
|
||||
{
|
||||
error_code err;
|
||||
std::string subdir = combine_path(old_save_path, *it);
|
||||
while (subdir != old_save_path && !err)
|
||||
{
|
||||
remove(subdir, err);
|
||||
subdir = parent_path(subdir);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
|
@ -1251,29 +1251,107 @@ TORRENT_TEST(readwritev_zero_size_files)
|
|||
TEST_CHECK(check_pattern(buf, 0));
|
||||
}
|
||||
|
||||
void delete_dirs(std::string path)
|
||||
{
|
||||
error_code ec;
|
||||
remove_all(path, ec);
|
||||
if (ec && ec != boost::system::errc::no_such_file_or_directory)
|
||||
{
|
||||
fprintf(stderr, "remove_all \"%s\": %s\n"
|
||||
, path.c_str(), ec.message().c_str());
|
||||
}
|
||||
TEST_CHECK(!exists(path));
|
||||
}
|
||||
|
||||
TORRENT_TEST(move_storage_into_self)
|
||||
{
|
||||
std::string const save_path = current_working_directory();
|
||||
delete_dirs(combine_path(save_path, "temp_storage"));
|
||||
|
||||
aux::session_settings set;
|
||||
file_storage fs;
|
||||
std::vector<char> buf;
|
||||
file_pool fp;
|
||||
io_service ios;
|
||||
disk_buffer_pool dp(16 * 1024, ios, boost::bind(&nop));
|
||||
boost::shared_ptr<default_storage> s = setup_torrent(fs, fp, buf
|
||||
, current_working_directory()
|
||||
, set);
|
||||
boost::shared_ptr<default_storage> s = setup_torrent(fs, fp, buf, save_path, set);
|
||||
|
||||
file::iovec_t const b = {&buf[0], 4};
|
||||
storage_error se;
|
||||
s->writev(&b, 1, 2, 0, 0, se);
|
||||
|
||||
std::string const test_path = combine_path(save_path, combine_path("temp_storage", "folder1"));
|
||||
s->move_storage(test_path, 0, se);
|
||||
TEST_EQUAL(se.ec, boost::system::errc::success);
|
||||
|
||||
TEST_CHECK(exists(combine_path(test_path, combine_path("temp_storage"
|
||||
, combine_path("folder1", "test2.tmp")))));
|
||||
|
||||
// these directories and files are created up-front because they are empty files
|
||||
TEST_CHECK(exists(combine_path(test_path, combine_path("temp_storage"
|
||||
, combine_path("folder2", "test3.tmp")))));
|
||||
TEST_CHECK(exists(combine_path(test_path, combine_path("temp_storage"
|
||||
, combine_path("_folder3", "test4.tmp")))));
|
||||
}
|
||||
|
||||
TORRENT_TEST(dont_move_intermingled_files)
|
||||
{
|
||||
std::string const save_path = combine_path(current_working_directory(), "save_path_1");
|
||||
delete_dirs(combine_path(save_path, "temp_storage"));
|
||||
|
||||
std::string test_path = combine_path(current_working_directory(), "save_path_2");
|
||||
delete_dirs(combine_path(test_path, "temp_storage"));
|
||||
|
||||
aux::session_settings set;
|
||||
file_storage fs;
|
||||
std::vector<char> buf;
|
||||
file_pool fp;
|
||||
io_service ios;
|
||||
disk_buffer_pool dp(16 * 1024, ios, boost::bind(&nop));
|
||||
boost::shared_ptr<default_storage> s = setup_torrent(fs, fp, buf, save_path, set);
|
||||
|
||||
file::iovec_t b = {&buf[0], 4};
|
||||
storage_error se;
|
||||
s->writev(&b, 1, 2, 0, 0, se);
|
||||
|
||||
s->move_storage(combine_path("temp_storage", "folder1"), 0, se);
|
||||
error_code ec;
|
||||
create_directory(combine_path(save_path, combine_path("temp_storage"
|
||||
, combine_path("_folder3", "alien_folder1"))), ec);
|
||||
TEST_EQUAL(ec, boost::system::errc::success);
|
||||
file f;
|
||||
f.open(combine_path(save_path, combine_path("temp_storage", "alien1.tmp"))
|
||||
, file::write_only, ec);
|
||||
f.close();
|
||||
TEST_EQUAL(ec, boost::system::errc::success);
|
||||
f.open(combine_path(save_path, combine_path("temp_storage"
|
||||
, combine_path("folder1", "alien2.tmp"))), file::write_only, ec);
|
||||
f.close();
|
||||
TEST_EQUAL(ec, boost::system::errc::success);
|
||||
|
||||
printf("move error: %s\n", se.ec.message().c_str());
|
||||
#ifdef _WIN32
|
||||
TEST_EQUAL(se.ec, boost::system::errc::permission_denied);
|
||||
#else
|
||||
TEST_EQUAL(se.ec, boost::system::errc::invalid_argument);
|
||||
#endif
|
||||
s->move_storage(test_path, 0, se);
|
||||
TEST_EQUAL(se.ec, boost::system::errc::success);
|
||||
|
||||
// torrent files moved to new place
|
||||
TEST_CHECK(exists(combine_path(test_path, combine_path("temp_storage"
|
||||
, combine_path("folder1", "test2.tmp")))));
|
||||
// these directories and files are created up-front because they are empty files
|
||||
TEST_CHECK(exists(combine_path(test_path, combine_path("temp_storage"
|
||||
, combine_path("folder2", "test3.tmp")))));
|
||||
TEST_CHECK(exists(combine_path(test_path, combine_path("temp_storage"
|
||||
, combine_path("_folder3", "test4.tmp")))));
|
||||
|
||||
// intermingled files and directories are still in old place
|
||||
TEST_CHECK(exists(combine_path(save_path, combine_path("temp_storage"
|
||||
, "alien1.tmp"))));
|
||||
TEST_CHECK(!exists(combine_path(test_path, combine_path("temp_storage"
|
||||
, "alien1.tmp"))));
|
||||
TEST_CHECK(exists(combine_path(save_path, combine_path("temp_storage"
|
||||
, combine_path("folder1", "alien2.tmp")))));
|
||||
TEST_CHECK(!exists(combine_path(test_path, combine_path("temp_storage"
|
||||
, combine_path("folder1", "alien2.tmp")))));
|
||||
TEST_CHECK(exists(combine_path(save_path, combine_path("temp_storage"
|
||||
, combine_path("_folder3", "alien_folder1")))));
|
||||
TEST_CHECK(!exists(combine_path(test_path, combine_path("temp_storage"
|
||||
, combine_path("_folder3", "alien_folder1")))));
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue