premiere
/
premiere-libtorrent
2
2
Fork 1
Code Issues Pull Requests Releases Wiki Activity

memory optimization for checking torrents

This commit is contained in:
Arvid Norberg 2009-05-21 16:15:05 +00:00
parent d74f6db11f
commit cb9c3cb37d
9 changed files with 106 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ChangeLog
View File

@ -1,3 +1,7 @@
* applied temporary memory storage optimization to when checking
a torrent as well
* removed hash_for_slot() from storage_interface. It is now implemented
by using the readv() function from the storage implementation
* improved IPv6 support by announcing twice when necessary
* added feature to set a separate global rate limit for local peers
* added preset settings for low memory environments and seed machines

24
docs/manual.rst
View File

@ -5312,7 +5312,6 @@ The interface looks like this::
virtual bool move_slot(int src_slot, int dst_slot) = 0;
virtual bool swap_slots(int slot1, int slot2) = 0;
virtual bool swap_slots3(int slot1, int slot2, int slot3) = 0;
virtual sha1_hash hash_for_slot(int slot, partial_hash& h, int piece_size) = 0;
virtual bool rename_file(int file, std::string const& new_name) = 0;
virtual bool release_files() = 0;
virtual bool delete_files() = 0;
@ -5489,29 +5488,6 @@ This is only used in compact mode.
Returning ``true`` indicates an error occurred.
hash_for_slot()
---------------
::
sha1_hash hash_for_slot(int slot, partial_hash& h, int piece_size) = 0;
The function should read the remaining bytes of the slot and hash it with the
sha-1 state in ``partion_hash``. The ``partial_hash`` struct looks like this::
struct partial_hash
{
partial_hash();
int offset;
hasher h;
};
``offset`` is the number of bytes in the slot that has already been hashed, and
``h`` is the sha-1 state of that hash. ``piece_size`` is the size of the piece
that is stored in the given slot.
The function should return the hash of the piece stored in the slot.
rename_file()
-------------

1
include/libtorrent/error_code.hpp
View File

@ -74,6 +74,7 @@ namespace libtorrent
invalid_torrent_handle,
invalid_entry_type,
missing_info_hash_in_uri,
file_too_short,
};
}

15
include/libtorrent/storage.hpp
View File

@ -154,9 +154,6 @@ namespace libtorrent
// in slot3 and the data in slot3 in slot1
virtual bool swap_slots3(int slot1, int slot2, int slot3) = 0;
// returns the sha1-hash for the data at the given slot
virtual sha1_hash hash_for_slot(int slot, partial_hash& h, int piece_size) = 0;
// this will close all open files that are opened for
// writing. This is called when a torrent has finished
// downloading.
@ -319,6 +316,12 @@ namespace libtorrent
bool allocate_slots(int num_slots, bool abort_on_disk = false);
// updates the ph.h hasher object with the data at the given slot
// and optionally a 'small hash' as well, the hash for
// the partial slot. Returns the number of bytes read
int hash_for_slot(int slot, partial_hash& h, int piece_size
, int small_piece_size = 0, sha1_hash* small_hash = 0);
int read_impl(
file::iovec_t* bufs
, int piece_index
@ -337,7 +340,8 @@ namespace libtorrent
// -1=error 0=ok >0=skip this many pieces
int check_one_piece(int& have_piece);
int identify_data(
char const* piece_data
sha1_hash const& large_hash
, sha1_hash const& small_hash
, int current_slot);
void switch_to_full_mode();
@ -419,9 +423,6 @@ namespace libtorrent
// storage (osed when remapping files)
storage_constructor_type m_storage_constructor;
// temporary buffer used while checking
disk_buffer_holder m_piece_data;
// this maps a piece hash to piece index. It will be
// build the first time it is used (to save time if it
// isn't needed)

25
src/disk_io_thread.cpp
View File

@ -659,8 +659,14 @@ namespace libtorrent
file::iovec_t b = { buf.get(), buffer_size };
ret = p.storage->read_impl(&b, p.piece, start_block * m_block_size, 1);
l.lock();
TORRENT_ASSERT(ret == buffer_size || p.storage->error());
if (p.storage->error()) { return -1; }
if (ret != buffer_size)
{
// this means the file wasn't big enough for this read
p.storage->get_storage_impl()->set_error(""
, error_code(errors::file_too_short, libtorrent_category));
return -1;
}
++m_cache_stats.reads;
}
@ -692,8 +698,14 @@ namespace libtorrent
l.unlock();
ret = p.storage->read_impl(iov, p.piece, start_block * m_block_size, iov_counter);
l.lock();
TORRENT_ASSERT(ret == buffer_size || p.storage->error());
if (p.storage->error()) { return -1; }
if (ret != buffer_size)
{
// this means the file wasn't big enough for this read
p.storage->get_storage_impl()->set_error(""
, error_code(errors::file_too_short, libtorrent_category));
return -1;
}
++m_cache_stats.reads;
}
@ -1314,6 +1326,15 @@ namespace libtorrent
test_error(j);
break;
}
if (ret != j.storage->m_files.piece_size(j.piece) - j.offset)
{
// this means the file wasn't big enough for this read
j.error = error_code(errors::file_too_short, libtorrent_category);
j.error_file.clear();
j.str = j.error.message();
ret = -1;
break;
}
++m_cache_stats.blocks_read;
}
read_holder.release();

1
src/error_code.cpp
View File

@ -70,6 +70,7 @@ namespace libtorrent
"invalid torrent handle used",
"invalid type requested from entry",
"missing info-hash from URI",
"file too short",
};
if (ev < 0 || ev >= sizeof(msgs)/sizeof(msgs[0]))
return "Unknown error";

130
src/storage.cpp
View File

@ -432,7 +432,6 @@ namespace libtorrent
bool swap_slots3(int slot1, int slot2, int slot3);
bool verify_resume_data(lazy_entry const& rd, std::string& error);
bool write_resume_data(entry& rd) const;
sha1_hash hash_for_slot(int slot, partial_hash& ph, int piece_size);
// this identifies a read or write operation
// so that storage::readwrite() knows what to
@ -476,14 +475,16 @@ namespace libtorrent
bool m_allocate_files;
};
sha1_hash storage::hash_for_slot(int slot, partial_hash& ph, int piece_size)
int piece_manager::hash_for_slot(int slot, partial_hash& ph, int piece_size
, int small_piece_size, sha1_hash* small_hash)
{
TORRENT_ASSERT(!error());
int num_read = 0;
int slot_size = piece_size - ph.offset;
if (slot_size > 0)
{
int block_size = 16 * 1024;
if (disk_pool()) block_size = disk_pool()->block_size();
if (m_storage->disk_pool()) block_size = m_storage->disk_pool()->block_size();
int size = slot_size;
int num_blocks = (size + block_size - 1) / block_size;
@ -492,40 +493,68 @@ namespace libtorrent
// and then hash it. When optimizing for memory usage, we read
// one block at a time and hash it. This ends up only using a
// single buffer
if (settings().optimize_hashing_for_speed)
if (m_storage->settings().optimize_hashing_for_speed)
{
file::iovec_t* bufs = TORRENT_ALLOCA(file::iovec_t, num_blocks);
for (int i = 0; i < num_blocks; ++i)
{
bufs[i].iov_base = disk_pool()->allocate_buffer("hash temp");
bufs[i].iov_base = m_storage->disk_pool()->allocate_buffer("hash temp");
bufs[i].iov_len = (std::min)(block_size, size);
size -= bufs[i].iov_len;
}
readv(bufs, slot, ph.offset, num_blocks);
num_read = m_storage->readv(bufs, slot, ph.offset, num_blocks);
for (int i = 0; i < num_blocks; ++i)
{
ph.h.update((char const*)bufs[i].iov_base, bufs[i].iov_len);
disk_pool()->free_buffer((char*)bufs[i].iov_base);
if (small_hash && small_piece_size < block_size)
{
ph.h.update((char const*)bufs[i].iov_base, small_piece_size);
*small_hash = hasher(ph.h).final();
small_hash = 0; // avoid this case again
ph.h.update((char const*)bufs[i].iov_base + small_piece_size
, bufs[i].iov_len - small_piece_size);
}
else
{
ph.h.update((char const*)bufs[i].iov_base, bufs[i].iov_len);
small_piece_size -= bufs[i].iov_len;
}
m_storage->disk_pool()->free_buffer((char*)bufs[i].iov_base);
}
}
else
{
file::iovec_t buf;
disk_buffer_holder holder(*disk_pool(), disk_pool()->allocate_buffer("hash temp"));
disk_buffer_holder holder(*m_storage->disk_pool()
, m_storage->disk_pool()->allocate_buffer("hash temp"));
buf.iov_base = holder.get();
for (int i = 0; i < num_blocks; ++i)
{
buf.iov_len = (std::min)(block_size, size);
readv(&buf, slot, ph.offset, 1);
ph.h.update((char const*)buf.iov_base, buf.iov_len);
int ret = m_storage->readv(&buf, slot, ph.offset, 1);
if (ret > 0) num_read += ret;
if (small_hash && small_piece_size < block_size)
{
ph.h.update((char const*)buf.iov_base, small_piece_size);
*small_hash = hasher(ph.h).final();
small_hash = 0; // avoid this case again
ph.h.update((char const*)buf.iov_base + small_piece_size
, buf.iov_len - small_piece_size);
}
else
{
ph.h.update((char const*)buf.iov_base, buf.iov_len);
small_piece_size -= buf.iov_len;
}
ph.offset += buf.iov_len;
size -= buf.iov_len;
}
}
if (error()) return sha1_hash(0);
if (error()) return 0;
}
return ph.h.final();
return num_read;
}
bool storage::initialize(bool allocate_files)
@ -1333,16 +1362,8 @@ ret:
}
if (file_bytes_left != bytes_transferred)
{
// the file was not big enough
#ifdef TORRENT_WINDOWS
ec = error_code(ERROR_HANDLE_EOF, get_system_category());
#else
ec = error_code(EIO, get_posix_category());
#endif
set_error(m_save_path / file_iter->path, ec);
return bytes_transferred;
}
advance_bufs(current_buf, bytes_transferred);
TORRENT_ASSERT(count_bufs(current_buf, bytes_left - file_bytes_left) <= num_bufs);
}
@ -1440,7 +1461,6 @@ ret:
, m_scratch_buffer2(io, 0)
, m_scratch_piece(-1)
, m_storage_constructor(sc)
, m_piece_data(io, 0)
, m_io_thread(io)
, m_torrent(torrent)
{
@ -1631,7 +1651,9 @@ ret:
int slot = slot_for(piece);
TORRENT_ASSERT(slot != has_no_slot);
return m_storage->hash_for_slot(slot, ph, m_files.piece_size(piece));
hash_for_slot(slot, ph, m_files.piece_size(piece));
if (m_storage->error()) return sha1_hash(0);
return ph.h.final();
}
int piece_manager::move_storage_impl(fs::path const& save_path)
@ -1804,31 +1826,11 @@ ret:
}
int piece_manager::identify_data(
char const* piece_data
sha1_hash const& large_hash
, sha1_hash const& small_hash
, int current_slot)
{
// INVARIANT_CHECK;
const int piece_size = static_cast<int>(m_files.piece_length());
const int last_piece_size = static_cast<int>(m_files.piece_size(
m_files.num_pieces() - 1));
// calculate a small digest, with the same
// size as the last piece. And a large digest
// which has the same size as a normal piece
hasher small_digest;
small_digest.update(piece_data, last_piece_size);
hasher large_digest(small_digest);
TORRENT_ASSERT(piece_size - last_piece_size >= 0);
if (piece_size - last_piece_size > 0)
{
large_digest.update(
piece_data + last_piece_size
, piece_size - last_piece_size);
}
sha1_hash large_hash = large_digest.final();
sha1_hash small_hash = small_digest.final();
typedef std::multimap<sha1_hash, int>::const_iterator map_iter;
map_iter begin1;
map_iter end1;
@ -2359,7 +2361,6 @@ ret:
TORRENT_ASSERT(m_current_slot == m_files.num_pieces());
// clear the memory we've been using
m_piece_data.reset();
std::multimap<sha1_hash, int>().swap(m_hash_to_piece);
if (m_storage_mode != storage_mode_compact)
@ -2433,18 +2434,24 @@ ret:
m_hash_to_piece.insert(std::make_pair(m_info->hash_for_piece(i), i));
}
if (!m_piece_data)
{
int blocks_per_piece = (std::max)(m_files.piece_length() / m_io_thread.block_size(), 1);
m_piece_data.reset(m_io_thread.allocate_buffers(blocks_per_piece, "check piece")
, blocks_per_piece);
}
partial_hash ph;
int num_read = 0;
int piece_size = m_files.piece_size(m_current_slot);
int num_read = m_storage->read(m_piece_data.get()
, m_current_slot, 0, piece_size);
int small_piece_size = m_files.piece_size(m_files.num_pieces() - 1);
bool read_short = true;
sha1_hash small_hash;
if (piece_size == small_piece_size)
{
num_read = hash_for_slot(m_current_slot, ph, piece_size, 0, 0);
}
else
{
num_read = hash_for_slot(m_current_slot, ph, piece_size
, small_piece_size, &small_hash);
}
read_short = num_read != piece_size;
if (num_read < 0)
if (read_short)
{
if (m_storage->error()
#ifdef TORRENT_WINDOWS
@ -2454,17 +2461,14 @@ ret:
&& m_storage->error() != error_code(ENOENT, get_posix_category()))
#endif
{
m_piece_data.reset();
return -1;
}
// if the file is incomplete, skip the rest of it
return skip_file();
}
// if the file is incomplete, skip the rest of it
if (num_read != piece_size)
return skip_file();
int piece_index = identify_data(m_piece_data.get(), m_current_slot);
sha1_hash large_hash = ph.h.final();
int piece_index = identify_data(large_hash, small_hash, m_current_slot);
if (piece_index >= 0) have_piece = piece_index;

2
test/setup_transfer.cpp
View File

@ -299,7 +299,7 @@ setup_transfer(session* ses1, session* ses2, session* ses3
}
char ih_hex[41];
to_hex((char const*)&t->info_hash()[0], 20, ih_hex);
std::cerr << "generated torrent: " << ih_hex << std::endl;
std::cerr << "generated torrent: " << ih_hex << " ./tmp1" << suffix << "/temporary" << std::endl;
}
else
{

2
test/test_swarm.cpp
View File

@ -96,7 +96,7 @@ void test_swarm(bool super_seeding = false, bool strict = false, bool seed_mode
p.seed_mode = seed_mode;
// test using piece sizes smaller than 16kB
boost::tie(tor1, tor2, tor3) = setup_transfer(&ses1, &ses2, &ses3, true
, false, true, "_swarm", 8 * 1024, 0, super_seeding, &p);
, false, true, "_swarm", 32 * 1024, 0, super_seeding, &p);
if (time_critical)
{