added largest_contiguous cache flush algorithm
This commit is contained in:
parent
1392b14fec
commit
c1d9198dc3
|
@ -1,3 +1,5 @@
|
|||
* added another cache flush algorithm to write the largest
|
||||
contiguous blocks instead of the least recently used
|
||||
* introduced a mechanism to be lighter on the disk when checking torrents
|
||||
* applied temporary memory storage optimization to when checking
|
||||
a torrent as well
|
||||
|
|
|
@ -3443,6 +3443,11 @@ session_settings
|
|||
bool optimize_hashing_for_speed;
|
||||
|
||||
int file_checks_delay_per_block;
|
||||
|
||||
enum disk_cache_algo_t
|
||||
{ lru, largest_contiguous };
|
||||
|
||||
disk_cache_algo_t disk_cache_algorithm;
|
||||
};
|
||||
|
||||
``user_agent`` this is the client identification to the tracker.
|
||||
|
@ -3826,6 +3831,15 @@ data is read from the disk while checking. This may be useful for
|
|||
background tasks that doesn't matter if they take a bit longer, as long
|
||||
as they leave disk I/O time for other processes.
|
||||
|
||||
``disk_cache_algorithm`` tells the disk I/O thread which cache flush
|
||||
algorithm to use. The default (and original) algorithm is LRU. This
|
||||
flushes the entire piece, in the write cache, that was least recently
|
||||
written to. This is specified by the ``session_settings::lru`` enum
|
||||
value. ``session_settings::largest_contiguous`` will flush the largest
|
||||
sequences of contiguous blocks from the write cache, regarless of the
|
||||
piece's last use time.
|
||||
|
||||
|
||||
pe_settings
|
||||
===========
|
||||
|
||||
|
|
|
@ -765,6 +765,7 @@ int main(int argc, char* argv[])
|
|||
settings.auto_upload_slots_rate_based = true;
|
||||
settings.announce_to_all_trackers = true;
|
||||
settings.optimize_hashing_for_speed = false;
|
||||
settings.disk_cache_algorithm = session_settings::largest_contiguous;
|
||||
|
||||
int refresh_delay = 1;
|
||||
|
||||
|
|
|
@ -266,8 +266,6 @@ namespace libtorrent
|
|||
std::ofstream m_disk_access_log;
|
||||
#endif
|
||||
|
||||
private:
|
||||
|
||||
struct cached_piece_entry
|
||||
{
|
||||
int piece;
|
||||
|
@ -284,6 +282,8 @@ namespace libtorrent
|
|||
typedef boost::recursive_mutex mutex_t;
|
||||
typedef std::list<cached_piece_entry> cache_t;
|
||||
|
||||
private:
|
||||
|
||||
bool test_error(disk_io_job& j);
|
||||
void post_callback(boost::function<void(int, disk_io_job const&)> const& handler
|
||||
, disk_io_job const& j, int ret);
|
||||
|
@ -296,20 +296,22 @@ namespace libtorrent
|
|||
, disk_io_job const& j, mutex_t::scoped_lock& l);
|
||||
|
||||
// write cache operations
|
||||
void flush_oldest_piece(mutex_t::scoped_lock& l);
|
||||
void flush_cache_blocks(mutex_t::scoped_lock& l, int blocks);
|
||||
void flush_expired_pieces();
|
||||
void flush_and_remove(cache_t::iterator i, mutex_t::scoped_lock& l);
|
||||
void flush(cache_t::iterator i, mutex_t::scoped_lock& l);
|
||||
int flush_contiguous_blocks(disk_io_thread::cache_t::iterator e
|
||||
, mutex_t::scoped_lock& l);
|
||||
void flush_range(cache_t::iterator i, int start, int end, mutex_t::scoped_lock& l);
|
||||
void cache_block(disk_io_job& j, mutex_t::scoped_lock& l);
|
||||
|
||||
// read cache operations
|
||||
bool clear_oldest_read_piece(cache_t::iterator ignore
|
||||
int clear_oldest_read_piece(cache_t::iterator ignore
|
||||
, mutex_t::scoped_lock& l);
|
||||
int read_into_piece(cached_piece_entry& p, int start_block
|
||||
, int options, mutex_t::scoped_lock& l);
|
||||
int cache_read_block(disk_io_job const& j, mutex_t::scoped_lock& l);
|
||||
int cache_read_piece(disk_io_job const& j, mutex_t::scoped_lock& l);
|
||||
void free_piece(cached_piece_entry& p, mutex_t::scoped_lock& l);
|
||||
int free_piece(cached_piece_entry& p, mutex_t::scoped_lock& l);
|
||||
bool make_room(int num_blocks
|
||||
, cache_t::iterator ignore
|
||||
, bool flush_write_cache
|
||||
|
|
|
@ -167,6 +167,7 @@ namespace libtorrent
|
|||
, send_socket_buffer_size(0)
|
||||
, optimize_hashing_for_speed(true)
|
||||
, file_checks_delay_per_block(0)
|
||||
, disk_cache_algorithm(lru)
|
||||
{}
|
||||
|
||||
// this is the user agent that will be sent to the tracker
|
||||
|
@ -570,6 +571,11 @@ namespace libtorrent
|
|||
// the default of 10 ms/16kiB will limit
|
||||
// the checking rate to 1.6 MiB per second
|
||||
int file_checks_delay_per_block;
|
||||
|
||||
enum disk_cache_algo_t
|
||||
{ lru, largest_contiguous };
|
||||
|
||||
disk_cache_algo_t disk_cache_algorithm;
|
||||
};
|
||||
|
||||
#ifndef TORRENT_DISABLE_DHT
|
||||
|
|
|
@ -442,23 +442,28 @@ namespace libtorrent
|
|||
}
|
||||
}
|
||||
|
||||
void disk_io_thread::free_piece(cached_piece_entry& p, mutex_t::scoped_lock& l)
|
||||
// returns the number of blocks that were freed
|
||||
int disk_io_thread::free_piece(cached_piece_entry& p, mutex_t::scoped_lock& l)
|
||||
{
|
||||
int piece_size = p.storage->info()->piece_size(p.piece);
|
||||
int blocks_in_piece = (piece_size + m_block_size - 1) / m_block_size;
|
||||
int ret = 0;
|
||||
|
||||
for (int i = 0; i < blocks_in_piece; ++i)
|
||||
{
|
||||
if (p.blocks[i] == 0) continue;
|
||||
free_buffer(p.blocks[i]);
|
||||
++ret;
|
||||
p.blocks[i] = 0;
|
||||
--p.num_blocks;
|
||||
--m_cache_stats.cache_size;
|
||||
--m_cache_stats.read_cache_size;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool disk_io_thread::clear_oldest_read_piece(
|
||||
// returns the number of blocks that were freed
|
||||
int disk_io_thread::clear_oldest_read_piece(
|
||||
cache_t::iterator ignore
|
||||
, mutex_t::scoped_lock& l)
|
||||
{
|
||||
|
@ -471,38 +476,108 @@ namespace libtorrent
|
|||
if (i != m_read_pieces.end() && i != ignore)
|
||||
{
|
||||
// don't replace an entry that is less than one second old
|
||||
if (time_now() - i->last_use < seconds(1)) return false;
|
||||
free_piece(*i, l);
|
||||
if (time_now() - i->last_use < seconds(1)) return 0;
|
||||
int blocks = free_piece(*i, l);
|
||||
m_read_pieces.erase(i);
|
||||
return true;
|
||||
return blocks;
|
||||
}
|
||||
return false;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void disk_io_thread::flush_oldest_piece(mutex_t::scoped_lock& l)
|
||||
int contiguous_blocks(disk_io_thread::cached_piece_entry const& b)
|
||||
{
|
||||
int ret = 0;
|
||||
int current = 0;
|
||||
int blocks_in_piece = (b.storage->info()->piece_size(b.piece) + 16 * 1024 - 1) / (16 * 1024);
|
||||
for (int i = 0; i < blocks_in_piece; ++i)
|
||||
{
|
||||
if (b.blocks[i]) ++current;
|
||||
else
|
||||
{
|
||||
if (current > ret) ret = current;
|
||||
current = 0;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
int disk_io_thread::flush_contiguous_blocks(disk_io_thread::cache_t::iterator e
|
||||
, mutex_t::scoped_lock& l)
|
||||
{
|
||||
// first find the largest range of contiguous blocks
|
||||
int len = 0;
|
||||
int current = 0;
|
||||
int pos = 0;
|
||||
int start = 0;
|
||||
int blocks_in_piece = (e->storage->info()->piece_size(e->piece)
|
||||
+ m_block_size - 1) / m_block_size;
|
||||
for (int i = 0; i < blocks_in_piece; ++i)
|
||||
{
|
||||
if (e->blocks[i]) ++current;
|
||||
else
|
||||
{
|
||||
if (current > len)
|
||||
{
|
||||
len = current;
|
||||
pos = start;
|
||||
}
|
||||
current = 0;
|
||||
start = i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
flush_range(e, pos, pos + len, l);
|
||||
if (e->num_blocks == 0) m_pieces.erase(e);
|
||||
return len;
|
||||
}
|
||||
|
||||
// flushes 'blocks' blocks from the cache
|
||||
void disk_io_thread::flush_cache_blocks(mutex_t::scoped_lock& l
|
||||
, int blocks)
|
||||
{
|
||||
INVARIANT_CHECK;
|
||||
// first look if there are any read cache entries that can
|
||||
// be cleared
|
||||
if (clear_oldest_read_piece(m_read_pieces.end(), l)) return;
|
||||
int ret = 0;
|
||||
do {
|
||||
ret = clear_oldest_read_piece(m_read_pieces.end(), l);
|
||||
blocks -= ret;
|
||||
} while (ret > 0 && blocks > 0);
|
||||
|
||||
cache_t::iterator i = std::min_element(
|
||||
m_pieces.begin(), m_pieces.end()
|
||||
, bind(&cached_piece_entry::last_use, _1)
|
||||
< bind(&cached_piece_entry::last_use, _2));
|
||||
if (i == m_pieces.end()) return;
|
||||
flush_and_remove(i, l);
|
||||
if (m_settings.disk_cache_algorithm == session_settings::lru)
|
||||
{
|
||||
while (blocks > 0)
|
||||
{
|
||||
cache_t::iterator i = std::min_element(
|
||||
m_pieces.begin(), m_pieces.end()
|
||||
, bind(&cached_piece_entry::last_use, _1)
|
||||
< bind(&cached_piece_entry::last_use, _2));
|
||||
if (i == m_pieces.end()) return;
|
||||
flush_and_remove(i, l);
|
||||
}
|
||||
}
|
||||
else if (m_settings.disk_cache_algorithm == session_settings::largest_contiguous)
|
||||
{
|
||||
while (blocks > 0)
|
||||
{
|
||||
cache_t::iterator i = std::max_element(
|
||||
m_pieces.begin(), m_pieces.end()
|
||||
, bind(&contiguous_blocks, _1)
|
||||
< bind(&contiguous_blocks, _2));
|
||||
if (i == m_pieces.end()) return;
|
||||
blocks -= flush_contiguous_blocks(i, l);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void disk_io_thread::flush_and_remove(disk_io_thread::cache_t::iterator e
|
||||
, mutex_t::scoped_lock& l)
|
||||
{
|
||||
flush(e, l);
|
||||
flush_range(e, 0, INT_MAX, l);
|
||||
m_pieces.erase(e);
|
||||
}
|
||||
|
||||
void disk_io_thread::flush(disk_io_thread::cache_t::iterator e
|
||||
, mutex_t::scoped_lock& l)
|
||||
void disk_io_thread::flush_range(disk_io_thread::cache_t::iterator e
|
||||
, int start, int end, mutex_t::scoped_lock& l)
|
||||
{
|
||||
INVARIANT_CHECK;
|
||||
// TODO: copy *e and unlink it before unlocking
|
||||
|
@ -523,9 +598,10 @@ namespace libtorrent
|
|||
if (m_settings.coalesce_writes) buf.reset(new (std::nothrow) char[piece_size]);
|
||||
else iov = TORRENT_ALLOCA(file::iovec_t, blocks_in_piece);
|
||||
|
||||
for (int i = 0; i <= blocks_in_piece; ++i)
|
||||
end = (std::min)(end, blocks_in_piece);
|
||||
for (int i = start; i <= end; ++i)
|
||||
{
|
||||
if (i == blocks_in_piece || p.blocks[i] == 0)
|
||||
if (i == end || p.blocks[i] == 0)
|
||||
{
|
||||
if (buffer_size == 0) continue;
|
||||
|
||||
|
@ -572,7 +648,7 @@ namespace libtorrent
|
|||
--m_cache_stats.cache_size;
|
||||
}
|
||||
|
||||
for (int i = 0; i < blocks_in_piece; ++i)
|
||||
for (int i = start; i < end; ++i)
|
||||
{
|
||||
if (p.blocks[i] == 0) continue;
|
||||
free_buffer(p.blocks[i]);
|
||||
|
@ -582,7 +658,7 @@ namespace libtorrent
|
|||
TORRENT_ASSERT(buffer_size == 0);
|
||||
// std::cerr << " flushing p: " << p.piece << " cached_blocks: " << m_cache_stats.cache_size << std::endl;
|
||||
#ifdef TORRENT_DEBUG
|
||||
for (int i = 0; i < blocks_in_piece; ++i)
|
||||
for (int i = start; i < end; ++i)
|
||||
TORRENT_ASSERT(p.blocks[i] == 0);
|
||||
#endif
|
||||
}
|
||||
|
@ -1134,7 +1210,7 @@ namespace libtorrent
|
|||
// flush all disk caches
|
||||
for (cache_t::iterator i = m_pieces.begin()
|
||||
, end(m_pieces.end()); i != end; ++i)
|
||||
flush(i, l);
|
||||
flush_range(i, 0, INT_MAX, l);
|
||||
for (cache_t::iterator i = m_read_pieces.begin()
|
||||
, end(m_read_pieces.end()); i != end; ++i)
|
||||
free_piece(*i, l);
|
||||
|
@ -1365,6 +1441,10 @@ namespace libtorrent
|
|||
#endif
|
||||
mutex_t::scoped_lock l(m_piece_mutex);
|
||||
INVARIANT_CHECK;
|
||||
|
||||
if (in_use() >= m_settings.cache_size)
|
||||
flush_cache_blocks(l, in_use() - m_settings.cache_size + 1);
|
||||
|
||||
cache_t::iterator p
|
||||
= find_cached_piece(m_pieces, j, l);
|
||||
int block = j.offset / m_block_size;
|
||||
|
@ -1394,8 +1474,10 @@ namespace libtorrent
|
|||
// in the cache, we should not
|
||||
// free it at the end
|
||||
holder.release();
|
||||
if (in_use() >= m_settings.cache_size)
|
||||
flush_oldest_piece(l);
|
||||
|
||||
if (in_use() > m_settings.cache_size)
|
||||
flush_cache_blocks(l, in_use() - m_settings.cache_size);
|
||||
|
||||
break;
|
||||
}
|
||||
case disk_io_job::hash:
|
||||
|
@ -1459,7 +1541,7 @@ namespace libtorrent
|
|||
{
|
||||
if (i->storage == j.storage)
|
||||
{
|
||||
flush(i, l);
|
||||
flush_range(i, 0, INT_MAX, l);
|
||||
i = m_pieces.erase(i);
|
||||
}
|
||||
else
|
||||
|
|
|
@ -640,6 +640,7 @@ namespace aux {
|
|||
|| m_settings.cache_expiry != s.cache_expiry
|
||||
|| m_settings.optimize_hashing_for_speed != s.optimize_hashing_for_speed
|
||||
|| m_settings.file_checks_delay_per_block != s.file_checks_delay_per_block
|
||||
|| m_settings.disk_cache_algorithm != s.disk_cache_algorithm
|
||||
#ifndef TORRENT_DISABLE_MLOCK
|
||||
|| m_settings.lock_disk_cache != s.lock_disk_cache
|
||||
#endif
|
||||
|
|
Loading…
Reference in New Issue