added largest_contiguous cache flush algorithm
This commit is contained in:
parent
1392b14fec
commit
c1d9198dc3
|
@ -1,3 +1,5 @@
|
||||||
|
* added another cache flush algorithm to write the largest
|
||||||
|
contiguous blocks instead of the least recently used
|
||||||
* introduced a mechanism to be lighter on the disk when checking torrents
|
* introduced a mechanism to be lighter on the disk when checking torrents
|
||||||
* applied temporary memory storage optimization to when checking
|
* applied temporary memory storage optimization to when checking
|
||||||
a torrent as well
|
a torrent as well
|
||||||
|
|
|
@ -3443,6 +3443,11 @@ session_settings
|
||||||
bool optimize_hashing_for_speed;
|
bool optimize_hashing_for_speed;
|
||||||
|
|
||||||
int file_checks_delay_per_block;
|
int file_checks_delay_per_block;
|
||||||
|
|
||||||
|
enum disk_cache_algo_t
|
||||||
|
{ lru, largest_contiguous };
|
||||||
|
|
||||||
|
disk_cache_algo_t disk_cache_algorithm;
|
||||||
};
|
};
|
||||||
|
|
||||||
``user_agent`` this is the client identification to the tracker.
|
``user_agent`` this is the client identification to the tracker.
|
||||||
|
@ -3826,6 +3831,15 @@ data is read from the disk while checking. This may be useful for
|
||||||
background tasks that doesn't matter if they take a bit longer, as long
|
background tasks that doesn't matter if they take a bit longer, as long
|
||||||
as they leave disk I/O time for other processes.
|
as they leave disk I/O time for other processes.
|
||||||
|
|
||||||
|
``disk_cache_algorithm`` tells the disk I/O thread which cache flush
|
||||||
|
algorithm to use. The default (and original) algorithm is LRU. This
|
||||||
|
flushes the entire piece, in the write cache, that was least recently
|
||||||
|
written to. This is specified by the ``session_settings::lru`` enum
|
||||||
|
value. ``session_settings::largest_contiguous`` will flush the largest
|
||||||
|
sequences of contiguous blocks from the write cache, regarless of the
|
||||||
|
piece's last use time.
|
||||||
|
|
||||||
|
|
||||||
pe_settings
|
pe_settings
|
||||||
===========
|
===========
|
||||||
|
|
||||||
|
|
|
@ -765,6 +765,7 @@ int main(int argc, char* argv[])
|
||||||
settings.auto_upload_slots_rate_based = true;
|
settings.auto_upload_slots_rate_based = true;
|
||||||
settings.announce_to_all_trackers = true;
|
settings.announce_to_all_trackers = true;
|
||||||
settings.optimize_hashing_for_speed = false;
|
settings.optimize_hashing_for_speed = false;
|
||||||
|
settings.disk_cache_algorithm = session_settings::largest_contiguous;
|
||||||
|
|
||||||
int refresh_delay = 1;
|
int refresh_delay = 1;
|
||||||
|
|
||||||
|
|
|
@ -266,8 +266,6 @@ namespace libtorrent
|
||||||
std::ofstream m_disk_access_log;
|
std::ofstream m_disk_access_log;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
private:
|
|
||||||
|
|
||||||
struct cached_piece_entry
|
struct cached_piece_entry
|
||||||
{
|
{
|
||||||
int piece;
|
int piece;
|
||||||
|
@ -284,6 +282,8 @@ namespace libtorrent
|
||||||
typedef boost::recursive_mutex mutex_t;
|
typedef boost::recursive_mutex mutex_t;
|
||||||
typedef std::list<cached_piece_entry> cache_t;
|
typedef std::list<cached_piece_entry> cache_t;
|
||||||
|
|
||||||
|
private:
|
||||||
|
|
||||||
bool test_error(disk_io_job& j);
|
bool test_error(disk_io_job& j);
|
||||||
void post_callback(boost::function<void(int, disk_io_job const&)> const& handler
|
void post_callback(boost::function<void(int, disk_io_job const&)> const& handler
|
||||||
, disk_io_job const& j, int ret);
|
, disk_io_job const& j, int ret);
|
||||||
|
@ -296,20 +296,22 @@ namespace libtorrent
|
||||||
, disk_io_job const& j, mutex_t::scoped_lock& l);
|
, disk_io_job const& j, mutex_t::scoped_lock& l);
|
||||||
|
|
||||||
// write cache operations
|
// write cache operations
|
||||||
void flush_oldest_piece(mutex_t::scoped_lock& l);
|
void flush_cache_blocks(mutex_t::scoped_lock& l, int blocks);
|
||||||
void flush_expired_pieces();
|
void flush_expired_pieces();
|
||||||
void flush_and_remove(cache_t::iterator i, mutex_t::scoped_lock& l);
|
void flush_and_remove(cache_t::iterator i, mutex_t::scoped_lock& l);
|
||||||
void flush(cache_t::iterator i, mutex_t::scoped_lock& l);
|
int flush_contiguous_blocks(disk_io_thread::cache_t::iterator e
|
||||||
|
, mutex_t::scoped_lock& l);
|
||||||
|
void flush_range(cache_t::iterator i, int start, int end, mutex_t::scoped_lock& l);
|
||||||
void cache_block(disk_io_job& j, mutex_t::scoped_lock& l);
|
void cache_block(disk_io_job& j, mutex_t::scoped_lock& l);
|
||||||
|
|
||||||
// read cache operations
|
// read cache operations
|
||||||
bool clear_oldest_read_piece(cache_t::iterator ignore
|
int clear_oldest_read_piece(cache_t::iterator ignore
|
||||||
, mutex_t::scoped_lock& l);
|
, mutex_t::scoped_lock& l);
|
||||||
int read_into_piece(cached_piece_entry& p, int start_block
|
int read_into_piece(cached_piece_entry& p, int start_block
|
||||||
, int options, mutex_t::scoped_lock& l);
|
, int options, mutex_t::scoped_lock& l);
|
||||||
int cache_read_block(disk_io_job const& j, mutex_t::scoped_lock& l);
|
int cache_read_block(disk_io_job const& j, mutex_t::scoped_lock& l);
|
||||||
int cache_read_piece(disk_io_job const& j, mutex_t::scoped_lock& l);
|
int cache_read_piece(disk_io_job const& j, mutex_t::scoped_lock& l);
|
||||||
void free_piece(cached_piece_entry& p, mutex_t::scoped_lock& l);
|
int free_piece(cached_piece_entry& p, mutex_t::scoped_lock& l);
|
||||||
bool make_room(int num_blocks
|
bool make_room(int num_blocks
|
||||||
, cache_t::iterator ignore
|
, cache_t::iterator ignore
|
||||||
, bool flush_write_cache
|
, bool flush_write_cache
|
||||||
|
|
|
@ -167,6 +167,7 @@ namespace libtorrent
|
||||||
, send_socket_buffer_size(0)
|
, send_socket_buffer_size(0)
|
||||||
, optimize_hashing_for_speed(true)
|
, optimize_hashing_for_speed(true)
|
||||||
, file_checks_delay_per_block(0)
|
, file_checks_delay_per_block(0)
|
||||||
|
, disk_cache_algorithm(lru)
|
||||||
{}
|
{}
|
||||||
|
|
||||||
// this is the user agent that will be sent to the tracker
|
// this is the user agent that will be sent to the tracker
|
||||||
|
@ -570,6 +571,11 @@ namespace libtorrent
|
||||||
// the default of 10 ms/16kiB will limit
|
// the default of 10 ms/16kiB will limit
|
||||||
// the checking rate to 1.6 MiB per second
|
// the checking rate to 1.6 MiB per second
|
||||||
int file_checks_delay_per_block;
|
int file_checks_delay_per_block;
|
||||||
|
|
||||||
|
enum disk_cache_algo_t
|
||||||
|
{ lru, largest_contiguous };
|
||||||
|
|
||||||
|
disk_cache_algo_t disk_cache_algorithm;
|
||||||
};
|
};
|
||||||
|
|
||||||
#ifndef TORRENT_DISABLE_DHT
|
#ifndef TORRENT_DISABLE_DHT
|
||||||
|
|
|
@ -442,23 +442,28 @@ namespace libtorrent
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void disk_io_thread::free_piece(cached_piece_entry& p, mutex_t::scoped_lock& l)
|
// returns the number of blocks that were freed
|
||||||
|
int disk_io_thread::free_piece(cached_piece_entry& p, mutex_t::scoped_lock& l)
|
||||||
{
|
{
|
||||||
int piece_size = p.storage->info()->piece_size(p.piece);
|
int piece_size = p.storage->info()->piece_size(p.piece);
|
||||||
int blocks_in_piece = (piece_size + m_block_size - 1) / m_block_size;
|
int blocks_in_piece = (piece_size + m_block_size - 1) / m_block_size;
|
||||||
|
int ret = 0;
|
||||||
|
|
||||||
for (int i = 0; i < blocks_in_piece; ++i)
|
for (int i = 0; i < blocks_in_piece; ++i)
|
||||||
{
|
{
|
||||||
if (p.blocks[i] == 0) continue;
|
if (p.blocks[i] == 0) continue;
|
||||||
free_buffer(p.blocks[i]);
|
free_buffer(p.blocks[i]);
|
||||||
|
++ret;
|
||||||
p.blocks[i] = 0;
|
p.blocks[i] = 0;
|
||||||
--p.num_blocks;
|
--p.num_blocks;
|
||||||
--m_cache_stats.cache_size;
|
--m_cache_stats.cache_size;
|
||||||
--m_cache_stats.read_cache_size;
|
--m_cache_stats.read_cache_size;
|
||||||
}
|
}
|
||||||
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool disk_io_thread::clear_oldest_read_piece(
|
// returns the number of blocks that were freed
|
||||||
|
int disk_io_thread::clear_oldest_read_piece(
|
||||||
cache_t::iterator ignore
|
cache_t::iterator ignore
|
||||||
, mutex_t::scoped_lock& l)
|
, mutex_t::scoped_lock& l)
|
||||||
{
|
{
|
||||||
|
@ -471,38 +476,108 @@ namespace libtorrent
|
||||||
if (i != m_read_pieces.end() && i != ignore)
|
if (i != m_read_pieces.end() && i != ignore)
|
||||||
{
|
{
|
||||||
// don't replace an entry that is less than one second old
|
// don't replace an entry that is less than one second old
|
||||||
if (time_now() - i->last_use < seconds(1)) return false;
|
if (time_now() - i->last_use < seconds(1)) return 0;
|
||||||
free_piece(*i, l);
|
int blocks = free_piece(*i, l);
|
||||||
m_read_pieces.erase(i);
|
m_read_pieces.erase(i);
|
||||||
return true;
|
return blocks;
|
||||||
}
|
}
|
||||||
return false;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
void disk_io_thread::flush_oldest_piece(mutex_t::scoped_lock& l)
|
int contiguous_blocks(disk_io_thread::cached_piece_entry const& b)
|
||||||
|
{
|
||||||
|
int ret = 0;
|
||||||
|
int current = 0;
|
||||||
|
int blocks_in_piece = (b.storage->info()->piece_size(b.piece) + 16 * 1024 - 1) / (16 * 1024);
|
||||||
|
for (int i = 0; i < blocks_in_piece; ++i)
|
||||||
|
{
|
||||||
|
if (b.blocks[i]) ++current;
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (current > ret) ret = current;
|
||||||
|
current = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
int disk_io_thread::flush_contiguous_blocks(disk_io_thread::cache_t::iterator e
|
||||||
|
, mutex_t::scoped_lock& l)
|
||||||
|
{
|
||||||
|
// first find the largest range of contiguous blocks
|
||||||
|
int len = 0;
|
||||||
|
int current = 0;
|
||||||
|
int pos = 0;
|
||||||
|
int start = 0;
|
||||||
|
int blocks_in_piece = (e->storage->info()->piece_size(e->piece)
|
||||||
|
+ m_block_size - 1) / m_block_size;
|
||||||
|
for (int i = 0; i < blocks_in_piece; ++i)
|
||||||
|
{
|
||||||
|
if (e->blocks[i]) ++current;
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (current > len)
|
||||||
|
{
|
||||||
|
len = current;
|
||||||
|
pos = start;
|
||||||
|
}
|
||||||
|
current = 0;
|
||||||
|
start = i + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
flush_range(e, pos, pos + len, l);
|
||||||
|
if (e->num_blocks == 0) m_pieces.erase(e);
|
||||||
|
return len;
|
||||||
|
}
|
||||||
|
|
||||||
|
// flushes 'blocks' blocks from the cache
|
||||||
|
void disk_io_thread::flush_cache_blocks(mutex_t::scoped_lock& l
|
||||||
|
, int blocks)
|
||||||
{
|
{
|
||||||
INVARIANT_CHECK;
|
|
||||||
// first look if there are any read cache entries that can
|
// first look if there are any read cache entries that can
|
||||||
// be cleared
|
// be cleared
|
||||||
if (clear_oldest_read_piece(m_read_pieces.end(), l)) return;
|
int ret = 0;
|
||||||
|
do {
|
||||||
|
ret = clear_oldest_read_piece(m_read_pieces.end(), l);
|
||||||
|
blocks -= ret;
|
||||||
|
} while (ret > 0 && blocks > 0);
|
||||||
|
|
||||||
cache_t::iterator i = std::min_element(
|
if (m_settings.disk_cache_algorithm == session_settings::lru)
|
||||||
m_pieces.begin(), m_pieces.end()
|
{
|
||||||
, bind(&cached_piece_entry::last_use, _1)
|
while (blocks > 0)
|
||||||
< bind(&cached_piece_entry::last_use, _2));
|
{
|
||||||
if (i == m_pieces.end()) return;
|
cache_t::iterator i = std::min_element(
|
||||||
flush_and_remove(i, l);
|
m_pieces.begin(), m_pieces.end()
|
||||||
|
, bind(&cached_piece_entry::last_use, _1)
|
||||||
|
< bind(&cached_piece_entry::last_use, _2));
|
||||||
|
if (i == m_pieces.end()) return;
|
||||||
|
flush_and_remove(i, l);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (m_settings.disk_cache_algorithm == session_settings::largest_contiguous)
|
||||||
|
{
|
||||||
|
while (blocks > 0)
|
||||||
|
{
|
||||||
|
cache_t::iterator i = std::max_element(
|
||||||
|
m_pieces.begin(), m_pieces.end()
|
||||||
|
, bind(&contiguous_blocks, _1)
|
||||||
|
< bind(&contiguous_blocks, _2));
|
||||||
|
if (i == m_pieces.end()) return;
|
||||||
|
blocks -= flush_contiguous_blocks(i, l);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void disk_io_thread::flush_and_remove(disk_io_thread::cache_t::iterator e
|
void disk_io_thread::flush_and_remove(disk_io_thread::cache_t::iterator e
|
||||||
, mutex_t::scoped_lock& l)
|
, mutex_t::scoped_lock& l)
|
||||||
{
|
{
|
||||||
flush(e, l);
|
flush_range(e, 0, INT_MAX, l);
|
||||||
m_pieces.erase(e);
|
m_pieces.erase(e);
|
||||||
}
|
}
|
||||||
|
|
||||||
void disk_io_thread::flush(disk_io_thread::cache_t::iterator e
|
void disk_io_thread::flush_range(disk_io_thread::cache_t::iterator e
|
||||||
, mutex_t::scoped_lock& l)
|
, int start, int end, mutex_t::scoped_lock& l)
|
||||||
{
|
{
|
||||||
INVARIANT_CHECK;
|
INVARIANT_CHECK;
|
||||||
// TODO: copy *e and unlink it before unlocking
|
// TODO: copy *e and unlink it before unlocking
|
||||||
|
@ -523,9 +598,10 @@ namespace libtorrent
|
||||||
if (m_settings.coalesce_writes) buf.reset(new (std::nothrow) char[piece_size]);
|
if (m_settings.coalesce_writes) buf.reset(new (std::nothrow) char[piece_size]);
|
||||||
else iov = TORRENT_ALLOCA(file::iovec_t, blocks_in_piece);
|
else iov = TORRENT_ALLOCA(file::iovec_t, blocks_in_piece);
|
||||||
|
|
||||||
for (int i = 0; i <= blocks_in_piece; ++i)
|
end = (std::min)(end, blocks_in_piece);
|
||||||
|
for (int i = start; i <= end; ++i)
|
||||||
{
|
{
|
||||||
if (i == blocks_in_piece || p.blocks[i] == 0)
|
if (i == end || p.blocks[i] == 0)
|
||||||
{
|
{
|
||||||
if (buffer_size == 0) continue;
|
if (buffer_size == 0) continue;
|
||||||
|
|
||||||
|
@ -572,7 +648,7 @@ namespace libtorrent
|
||||||
--m_cache_stats.cache_size;
|
--m_cache_stats.cache_size;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < blocks_in_piece; ++i)
|
for (int i = start; i < end; ++i)
|
||||||
{
|
{
|
||||||
if (p.blocks[i] == 0) continue;
|
if (p.blocks[i] == 0) continue;
|
||||||
free_buffer(p.blocks[i]);
|
free_buffer(p.blocks[i]);
|
||||||
|
@ -582,7 +658,7 @@ namespace libtorrent
|
||||||
TORRENT_ASSERT(buffer_size == 0);
|
TORRENT_ASSERT(buffer_size == 0);
|
||||||
// std::cerr << " flushing p: " << p.piece << " cached_blocks: " << m_cache_stats.cache_size << std::endl;
|
// std::cerr << " flushing p: " << p.piece << " cached_blocks: " << m_cache_stats.cache_size << std::endl;
|
||||||
#ifdef TORRENT_DEBUG
|
#ifdef TORRENT_DEBUG
|
||||||
for (int i = 0; i < blocks_in_piece; ++i)
|
for (int i = start; i < end; ++i)
|
||||||
TORRENT_ASSERT(p.blocks[i] == 0);
|
TORRENT_ASSERT(p.blocks[i] == 0);
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
@ -1134,7 +1210,7 @@ namespace libtorrent
|
||||||
// flush all disk caches
|
// flush all disk caches
|
||||||
for (cache_t::iterator i = m_pieces.begin()
|
for (cache_t::iterator i = m_pieces.begin()
|
||||||
, end(m_pieces.end()); i != end; ++i)
|
, end(m_pieces.end()); i != end; ++i)
|
||||||
flush(i, l);
|
flush_range(i, 0, INT_MAX, l);
|
||||||
for (cache_t::iterator i = m_read_pieces.begin()
|
for (cache_t::iterator i = m_read_pieces.begin()
|
||||||
, end(m_read_pieces.end()); i != end; ++i)
|
, end(m_read_pieces.end()); i != end; ++i)
|
||||||
free_piece(*i, l);
|
free_piece(*i, l);
|
||||||
|
@ -1365,6 +1441,10 @@ namespace libtorrent
|
||||||
#endif
|
#endif
|
||||||
mutex_t::scoped_lock l(m_piece_mutex);
|
mutex_t::scoped_lock l(m_piece_mutex);
|
||||||
INVARIANT_CHECK;
|
INVARIANT_CHECK;
|
||||||
|
|
||||||
|
if (in_use() >= m_settings.cache_size)
|
||||||
|
flush_cache_blocks(l, in_use() - m_settings.cache_size + 1);
|
||||||
|
|
||||||
cache_t::iterator p
|
cache_t::iterator p
|
||||||
= find_cached_piece(m_pieces, j, l);
|
= find_cached_piece(m_pieces, j, l);
|
||||||
int block = j.offset / m_block_size;
|
int block = j.offset / m_block_size;
|
||||||
|
@ -1394,8 +1474,10 @@ namespace libtorrent
|
||||||
// in the cache, we should not
|
// in the cache, we should not
|
||||||
// free it at the end
|
// free it at the end
|
||||||
holder.release();
|
holder.release();
|
||||||
if (in_use() >= m_settings.cache_size)
|
|
||||||
flush_oldest_piece(l);
|
if (in_use() > m_settings.cache_size)
|
||||||
|
flush_cache_blocks(l, in_use() - m_settings.cache_size);
|
||||||
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case disk_io_job::hash:
|
case disk_io_job::hash:
|
||||||
|
@ -1459,7 +1541,7 @@ namespace libtorrent
|
||||||
{
|
{
|
||||||
if (i->storage == j.storage)
|
if (i->storage == j.storage)
|
||||||
{
|
{
|
||||||
flush(i, l);
|
flush_range(i, 0, INT_MAX, l);
|
||||||
i = m_pieces.erase(i);
|
i = m_pieces.erase(i);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
|
|
@ -640,6 +640,7 @@ namespace aux {
|
||||||
|| m_settings.cache_expiry != s.cache_expiry
|
|| m_settings.cache_expiry != s.cache_expiry
|
||||||
|| m_settings.optimize_hashing_for_speed != s.optimize_hashing_for_speed
|
|| m_settings.optimize_hashing_for_speed != s.optimize_hashing_for_speed
|
||||||
|| m_settings.file_checks_delay_per_block != s.file_checks_delay_per_block
|
|| m_settings.file_checks_delay_per_block != s.file_checks_delay_per_block
|
||||||
|
|| m_settings.disk_cache_algorithm != s.disk_cache_algorithm
|
||||||
#ifndef TORRENT_DISABLE_MLOCK
|
#ifndef TORRENT_DISABLE_MLOCK
|
||||||
|| m_settings.lock_disk_cache != s.lock_disk_cache
|
|| m_settings.lock_disk_cache != s.lock_disk_cache
|
||||||
#endif
|
#endif
|
||||||
|
|
Loading…
Reference in New Issue