premiere-libtorrent/src/storage_utils.cpp

602 lines
18 KiB
C++

/*
Copyright (c) 2003-2016, Arvid Norberg
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the distribution.
* Neither the name of the author nor the names of its
contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
*/
#include "libtorrent/aux_/storage_utils.hpp"
#include "libtorrent/file_storage.hpp"
#include "libtorrent/aux_/alloca.hpp"
#include "libtorrent/aux_/path.hpp" // for count_bufs
#include "libtorrent/part_file.hpp"
#include "libtorrent/session.hpp" // for session::delete_files
#include "libtorrent/stat_cache.hpp"
#include "libtorrent/add_torrent_params.hpp"
#include "libtorrent/torrent_status.hpp"
#include <set>
namespace libtorrent { namespace aux {
int copy_bufs(span<iovec_t const> bufs, int bytes
, span<iovec_t> target)
{
TORRENT_ASSERT(bytes >= 0);
auto dst = target.begin();
int ret = 0;
if (bytes == 0) return ret;
for (iovec_t const& src : bufs)
{
std::size_t const to_copy = std::min(src.size(), std::size_t(bytes));
*dst = src.first(to_copy);
bytes -= int(to_copy);
++ret;
++dst;
if (bytes <= 0) return ret;
}
return ret;
}
typed_span<iovec_t> advance_bufs(typed_span<iovec_t> bufs, int const bytes)
{
TORRENT_ASSERT(bytes >= 0);
std::size_t size = 0;
for (;;)
{
size += bufs.front().size();
if (size >= std::size_t(bytes))
{
bufs.front() = bufs.front().last(size - std::size_t(bytes));
return bufs;
}
bufs = bufs.subspan(1);
}
}
void clear_bufs(span<iovec_t const> bufs)
{
for (auto buf : bufs)
std::fill(buf.begin(), buf.end(), 0);
}
#if TORRENT_USE_ASSERTS
namespace {
int count_bufs(span<iovec_t const> bufs, int bytes)
{
std::size_t size = 0;
int count = 0;
if (bytes == 0) return count;
for (auto b : bufs)
{
++count;
size += b.size();
if (size >= std::size_t(bytes)) return count;
}
return count;
}
}
#endif
// much of what needs to be done when reading and writing is buffer
// management and piece to file mapping. Most of that is the same for reading
// and writing. This function is a template, and the fileop decides what to
// do with the file and the buffers.
int readwritev(file_storage const& files, span<iovec_t const> const bufs
, piece_index_t const piece, const int offset
, storage_error& ec, fileop op)
{
TORRENT_ASSERT(piece >= piece_index_t(0));
TORRENT_ASSERT(piece < files.end_piece());
TORRENT_ASSERT(offset >= 0);
TORRENT_ASSERT(bufs.size() > 0);
const int size = bufs_size(bufs);
TORRENT_ASSERT(size > 0);
TORRENT_ASSERT(static_cast<int>(piece) * static_cast<std::int64_t>(files.piece_length())
+ offset + size <= files.total_size());
// find the file iterator and file offset
std::int64_t const torrent_offset = static_cast<int>(piece) * std::int64_t(files.piece_length()) + offset;
file_index_t file_index = files.file_index_at_offset(torrent_offset);
TORRENT_ASSERT(torrent_offset >= files.file_offset(file_index));
TORRENT_ASSERT(torrent_offset < files.file_offset(file_index) + files.file_size(file_index));
std::int64_t file_offset = torrent_offset - files.file_offset(file_index);
// the number of bytes left before this read or write operation is
// completely satisfied.
int bytes_left = size;
TORRENT_ASSERT(bytes_left >= 0);
// copy the iovec array so we can use it to keep track of our current
// location by updating the head base pointer and size. (see
// advance_bufs())
TORRENT_ALLOCA(current_buf, iovec_t, bufs.size());
copy_bufs(bufs, size, current_buf);
TORRENT_ASSERT(count_bufs(current_buf, size) == int(bufs.size()));
TORRENT_ALLOCA(tmp_buf, iovec_t, bufs.size());
while (bytes_left > 0)
{
// the number of bytes left to read in the current file (specified by
// file_index). This is the minimum of (file_size - file_offset) and
// bytes_left.
int file_bytes_left = bytes_left;
if (file_offset + file_bytes_left > files.file_size(file_index))
file_bytes_left = std::max(static_cast<int>(files.file_size(file_index) - file_offset), 0);
// there are no bytes left in this file, move to the next one
// this loop skips over empty files
while (file_bytes_left == 0)
{
++file_index;
file_offset = 0;
TORRENT_ASSERT(file_index < files.end_file());
// this should not happen. bytes_left should be clamped by the total
// size of the torrent, so we should never run off the end of it
if (file_index >= files.end_file()) return size;
file_bytes_left = bytes_left;
if (file_offset + file_bytes_left > files.file_size(file_index))
file_bytes_left = std::max(static_cast<int>(files.file_size(file_index) - file_offset), 0);
}
// make a copy of the iovec array that _just_ covers the next
// file_bytes_left bytes, i.e. just this one operation
int const tmp_bufs_used = copy_bufs(current_buf, file_bytes_left, tmp_buf);
int const bytes_transferred = op(file_index, file_offset
, tmp_buf.first(tmp_bufs_used), ec);
if (ec) return -1;
// advance our position in the iovec array and the file offset.
current_buf = advance_bufs(current_buf, bytes_transferred);
bytes_left -= bytes_transferred;
file_offset += bytes_transferred;
TORRENT_ASSERT(count_bufs(current_buf, bytes_left) <= int(bufs.size()));
// if the file operation returned 0, we've hit end-of-file. We're done
if (bytes_transferred == 0)
{
if (file_bytes_left > 0 )
{
// fill in this information in case the caller wants to treat
// a short-read as an error
ec.file(file_index);
}
return size - bytes_left;
}
}
return size;
}
std::pair<status_t, std::string> move_storage(file_storage const& f
, std::string const& save_path
, std::string const& destination_save_path
, part_file* pf
, move_flags_t const flags, storage_error& ec)
{
status_t ret = status_t::no_error;
std::string const new_save_path = complete(destination_save_path);
// check to see if any of the files exist
if (flags == move_flags_t::fail_if_exist)
{
file_status s;
error_code err;
stat_file(new_save_path, &s, err);
if (err != boost::system::errc::no_such_file_or_directory)
{
// the directory exists, check all the files
for (auto const i : f.file_range())
{
// files moved out to absolute paths are ignored
if (f.file_absolute_path(i)) continue;
stat_file(f.file_path(i, new_save_path), &s, err);
if (err != boost::system::errc::no_such_file_or_directory)
{
ec.ec = err;
ec.file(i);
ec.operation = operation_t::file_stat;
return { status_t::file_exist, save_path };
}
}
}
}
{
file_status s;
error_code err;
stat_file(new_save_path, &s, err);
if (err == boost::system::errc::no_such_file_or_directory)
{
err.clear();
create_directories(new_save_path, err);
if (err)
{
ec.ec = err;
ec.file(file_index_t(-1));
ec.operation = operation_t::mkdir;
return { status_t::fatal_disk_error, save_path };
}
}
else if (err)
{
ec.ec = err;
ec.file(file_index_t(-1));
ec.operation = operation_t::file_stat;
return { status_t::fatal_disk_error, save_path };
}
}
// indices of all files we ended up copying. These need to be deleted
// later
aux::vector<bool, file_index_t> copied_files(std::size_t(f.num_files()), false);
// track how far we got in case of an error
file_index_t file_index{};
error_code e;
for (auto const i : f.file_range())
{
// files moved out to absolute paths are not moved
if (f.file_absolute_path(i)) continue;
std::string const old_path = combine_path(save_path, f.file_path(i));
std::string const new_path = combine_path(new_save_path, f.file_path(i));
if (flags == move_flags_t::dont_replace && exists(new_path))
{
if (ret == status_t::no_error) ret = status_t::need_full_check;
continue;
}
// TODO: ideally, if we end up copying files because of a move across
// volumes, the source should not be deleted until they've all been
// copied. That would let us rollback with higher confidence.
move_file(old_path, new_path, e);
// if the source file doesn't exist. That's not a problem
// we just ignore that file
if (e == boost::system::errc::no_such_file_or_directory)
e.clear();
else if (e
&& e != boost::system::errc::invalid_argument
&& e != boost::system::errc::permission_denied)
{
// moving the file failed
// on OSX, the error when trying to rename a file across different
// volumes is EXDEV, which will make it fall back to copying.
e.clear();
copy_file(old_path, new_path, e);
if (!e) copied_files[i] = true;
}
if (e)
{
ec.ec = e;
ec.file(i);
ec.operation = operation_t::file_rename;
file_index = i;
break;
}
}
if (!e && pf)
{
pf->move_partfile(new_save_path, e);
if (e)
{
ec.ec = e;
ec.file(torrent_status::error_file_partfile);
ec.operation = operation_t::partfile_move;
}
}
if (e)
{
// rollback
while (--file_index >= file_index_t(0))
{
// files moved out to absolute paths are not moved
if (f.file_absolute_path(file_index)) continue;
// if we ended up copying the file, don't do anything during
// roll-back
if (copied_files[file_index]) continue;
std::string const old_path = combine_path(save_path, f.file_path(file_index));
std::string const new_path = combine_path(new_save_path, f.file_path(file_index));
// ignore errors when rolling back
error_code ignore;
move_file(new_path, old_path, ignore);
}
return { status_t::fatal_disk_error, save_path };
}
// TODO: 2 technically, this is where the transaction of moving the files
// is completed. This is where the new save_path should be committed. If
// there is an error in the code below, that should not prevent the new
// save path to be set. Maybe it would make sense to make the save_path
// an in-out parameter
std::set<std::string> subdirs;
for (auto const i : f.file_range())
{
// files moved out to absolute paths are not moved
if (f.file_absolute_path(i)) continue;
if (has_parent_path(f.file_path(i)))
subdirs.insert(parent_path(f.file_path(i)));
// if we ended up renaming the file instead of moving it, there's no
// need to delete the source.
if (copied_files[i] == false) continue;
std::string const old_path = combine_path(save_path, f.file_path(i));
// we may still have some files in old save_path
// eg. if (flags == dont_replace && exists(new_path))
// ignore errors when removing
error_code ignore;
remove(old_path, ignore);
}
for (std::string const& s : subdirs)
{
error_code err;
std::string subdir = combine_path(save_path, s);
while (!compare_path(subdir, save_path) && !err)
{
remove(subdir, err);
subdir = parent_path(subdir);
}
}
return { ret, new_save_path };
}
namespace {
void delete_one_file(std::string const& p, error_code& ec)
{
remove(p, ec);
if (ec == boost::system::errc::no_such_file_or_directory)
ec.clear();
}
}
void delete_files(file_storage const& fs, std::string const& save_path
, std::string const& part_file_name, remove_flags_t const options, storage_error& ec)
{
if (options == session::delete_files)
{
// delete the files from disk
std::set<std::string> directories;
using iter_t = std::set<std::string>::iterator;
for (auto const i : fs.file_range())
{
std::string const fp = fs.file_path(i);
bool const complete = fs.file_absolute_path(i);
std::string const p = complete ? fp : combine_path(save_path, fp);
if (!complete)
{
std::string bp = parent_path(fp);
std::pair<iter_t, bool> ret;
ret.second = true;
while (ret.second && !bp.empty())
{
ret = directories.insert(combine_path(save_path, bp));
bp = parent_path(bp);
}
}
delete_one_file(p, ec.ec);
if (ec) { ec.file(i); ec.operation = operation_t::file_remove; }
}
// remove the directories. Reverse order to delete
// subdirectories first
for (auto i = directories.rbegin()
, end(directories.rend()); i != end; ++i)
{
error_code error;
delete_one_file(*i, error);
if (error && !ec)
{
ec.file(file_index_t(-1));
ec.ec = error;
ec.operation = operation_t::file_remove;
}
}
}
if (options == session::delete_files
|| options == session::delete_partfile)
{
error_code error;
remove(combine_path(save_path, part_file_name), error);
if (error && error != boost::system::errc::no_such_file_or_directory)
{
ec.file(file_index_t(-1));
ec.ec = error;
ec.operation = operation_t::file_remove;
}
}
}
bool verify_resume_data(add_torrent_params const& rd
, aux::vector<std::string, file_index_t> const& links
, file_storage const& fs
, aux::vector<download_priority_t, file_index_t> const& file_priority
, stat_cache& stat
, std::string const& save_path
, storage_error& ec)
{
#ifdef TORRENT_DISABLE_MUTABLE_TORRENTS
TORRENT_UNUSED(links);
#else
if (!links.empty())
{
TORRENT_ASSERT(int(links.size()) == fs.num_files());
// if this is a mutable torrent, and we need to pick up some files
// from other torrents, do that now. Note that there is an inherent
// race condition here. We checked if the files existed on a different
// thread a while ago. These files may no longer exist or may have been
// moved. If so, we just fail. The user is responsible to not touch
// other torrents until a new mutable torrent has been completely
// added.
for (auto const idx : fs.file_range())
{
std::string const& s = links[idx];
if (s.empty()) continue;
error_code err;
std::string file_path = fs.file_path(idx, save_path);
hard_link(s, file_path, err);
// if the file already exists, that's not an error
// TODO: 2 is this risky? The upper layer will assume we have the
// whole file. Perhaps we should verify that at least the size
// of the file is correct
if (!err || err == boost::system::errc::file_exists)
continue;
ec.ec = err;
ec.file(idx);
ec.operation = operation_t::file_hard_link;
return false;
}
}
#endif // TORRENT_DISABLE_MUTABLE_TORRENTS
bool const seed = rd.have_pieces.all_set()
&& rd.have_pieces.size() >= fs.num_pieces();
// parse have bitmask. Verify that the files we expect to have
// actually do exist
for (piece_index_t i(0); i < piece_index_t(rd.have_pieces.size()); ++i)
{
if (rd.have_pieces.get_bit(i) == false) continue;
std::vector<file_slice> f = fs.map_block(i, 0, 1);
TORRENT_ASSERT(!f.empty());
file_index_t const file_index = f[0].file_index;
// files with priority zero may not have been saved to disk at their
// expected location, but is likely to be in a partfile. Just exempt it
// from checking
if (file_index < file_priority.end_index()
&& file_priority[file_index] == dont_download)
continue;
error_code error;
std::int64_t const size = stat.get_filesize(f[0].file_index
, fs, save_path, error);
if (size < 0)
{
if (error != boost::system::errc::no_such_file_or_directory)
{
ec.ec = error;
ec.file(file_index);
ec.operation = operation_t::file_stat;
return false;
}
else
{
ec.ec = errors::mismatching_file_size;
ec.file(file_index);
ec.operation = operation_t::file_stat;
return false;
}
}
if (seed && size != fs.file_size(file_index))
{
// the resume data indicates we're a seed, but this file has
// the wrong size. Reject the resume data
ec.ec = errors::mismatching_file_size;
ec.file(file_index);
ec.operation = operation_t::check_resume;
return false;
}
// OK, this file existed, good. Now, skip all remaining pieces in
// this file. We're just sanity-checking whether the files exist
// or not.
peer_request const pr = fs.map_file(file_index
, fs.file_size(file_index) + 1, 0);
i = std::max(next(i), pr.piece);
}
return true;
}
bool has_any_file(
file_storage const& fs
, std::string const& save_path
, stat_cache& cache
, storage_error& ec)
{
for (auto const i : fs.file_range())
{
std::int64_t const sz = cache.get_filesize(
i, fs, save_path, ec.ec);
if (sz < 0)
{
if (ec && ec.ec != boost::system::errc::no_such_file_or_directory)
{
ec.file(i);
ec.operation = operation_t::file_stat;
cache.clear();
return false;
}
// some files not existing is expected and not an error
ec.ec.clear();
}
if (sz > 0) return true;
}
return false;
}
}}