From 8ca76f9d595ee507a123c6d3536a682b0560685f Mon Sep 17 00:00:00 2001 From: arvidn Date: Sat, 5 Mar 2016 11:02:41 -0500 Subject: [PATCH 1/4] fix support for auto disk cache size and disk buffer pool allocator behavior --- examples/client_test.cpp | 4 +-- simulation/test_swarm.cpp | 2 +- simulation/test_transfer.cpp | 50 ++++++++++++++++++++++++++++++++++++ src/disk_buffer_pool.cpp | 15 ++++++++--- src/disk_io_thread.cpp | 14 +++++----- src/session_stats.cpp | 4 +-- 6 files changed, 73 insertions(+), 16 deletions(-) diff --git a/examples/client_test.cpp b/examples/client_test.cpp index e9e719023..403f473ba 100644 --- a/examples/client_test.cpp +++ b/examples/client_test.cpp @@ -1451,8 +1451,8 @@ int main(int argc, char* argv[]) #endif // TORRENT_USE_I2P case 'C': settings.set_int(settings_pack::cache_size, atoi(arg)); - settings.set_bool(settings_pack::use_read_cache, atoi(arg) > 0); - settings.set_int(settings_pack::cache_buffer_chunk_size, atoi(arg) / 100); + settings.set_bool(settings_pack::use_read_cache, atoi(arg) != 0); + settings.set_int(settings_pack::cache_buffer_chunk_size, 0); break; case 'A': settings.set_int(settings_pack::allowed_fast_set_size, atoi(arg)); break; case 'R': settings.set_int(settings_pack::read_cache_line_size, atoi(arg)); break; diff --git a/simulation/test_swarm.cpp b/simulation/test_swarm.cpp index d8ca4fb1e..e8d2d9bab 100644 --- a/simulation/test_swarm.cpp +++ b/simulation/test_swarm.cpp @@ -97,7 +97,7 @@ TORRENT_TEST(session_stats) // on alert , [=](lt::alert const* a, lt::session& ses) { - lt::session_stats_alert const* ss = lt::alert_cast(a); + auto const* ss = lt::alert_cast(a); if (!ss) return; // there's one downloading torrent diff --git a/simulation/test_transfer.cpp b/simulation/test_transfer.cpp index e3d519aee..a783e4b8a 100644 --- a/simulation/test_transfer.cpp +++ b/simulation/test_transfer.cpp @@ -35,6 +35,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "create_torrent.hpp" #include "settings.hpp" #include "libtorrent/session.hpp" +#include "libtorrent/session_stats.hpp" #include "libtorrent/deadline_timer.hpp" #include "libtorrent/settings_pack.hpp" #include "libtorrent/ip_filter.hpp" @@ -194,6 +195,39 @@ void filter_ips(lt::session& ses) ses.set_ip_filter(filter); } +void set_cache_size(lt::session& ses, int val) +{ + using namespace libtorrent; + settings_pack pack; + pack.set_int(settings_pack::cache_size, val); + ses.apply_settings(pack); +} + +int get_cache_size(lt::session& ses) +{ + using namespace libtorrent; + std::vector stats = session_stats_metrics(); + int const read_cache_idx = find_metric_idx("disk.read_cache_blocks"); + int const write_cache_idx = find_metric_idx("disk.write_cache_blocks"); + TEST_CHECK(read_cache_idx >= 0); + TEST_CHECK(write_cache_idx >= 0); + ses.set_alert_notify([](){}); + ses.post_session_stats(); + std::vector alerts; + ses.pop_alerts(&alerts); + int cache_size = -1; + for (auto const a : alerts) + { + if (auto const* st = alert_cast(a)) + { + cache_size = st->values[read_cache_idx]; + cache_size += st->values[write_cache_idx]; + break; + } + } + return cache_size; +} + void set_proxy(lt::session& ses, int proxy_type, int flags = 0, bool proxy_peer_connections = true) { // apply the proxy settings to session 0 @@ -391,3 +425,19 @@ TORRENT_TEST(no_proxy_utp_banned) ); } +TORRENT_TEST(auto_disk_cache_size) +{ + using namespace libtorrent; + run_test( + [](lt::session& ses0, lt::session& ses1) { set_cache_size(ses0, -1); }, + [](lt::session& ses, lt::alert const* alert) {}, + [](std::shared_ptr ses[2]) { + TEST_EQUAL(is_seed(*ses[0]), true); + + int const cache_size = get_cache_size(*ses[0]); + printf("cache size: %d\n", cache_size); + // this assumes the test torrent is at least 4 blocks + TEST_CHECK(cache_size > 4); + } + ); +} diff --git a/src/disk_buffer_pool.cpp b/src/disk_buffer_pool.cpp index 4e879dc9f..642c55477 100644 --- a/src/disk_buffer_pool.cpp +++ b/src/disk_buffer_pool.cpp @@ -371,11 +371,14 @@ namespace libtorrent #else if (m_using_pool_allocator) { - ret = static_cast(m_pool.malloc()); - int effective_block_size = m_cache_buffer_chunk_size + int const effective_block_size + = m_in_use >= m_max_use + ? 20 // use small increments once we've exceeded the cache size + : m_cache_buffer_chunk_size ? m_cache_buffer_chunk_size : (std::max)(m_max_use / 10, 1); m_pool.set_next_size(effective_block_size); + ret = static_cast(m_pool.malloc()); } else { @@ -453,7 +456,9 @@ namespace libtorrent m_cache_buffer_chunk_size = sett.get_int(settings_pack::cache_buffer_chunk_size); m_lock_disk_cache = sett.get_bool(settings_pack::lock_disk_cache); #ifndef TORRENT_DISABLE_POOL_ALLOCATOR - m_want_pool_allocator = sett.get_bool(settings_pack::use_disk_cache_pool); + // if the chunk size is set to 1, there's no point in creating a pool + m_want_pool_allocator = sett.get_bool(settings_pack::use_disk_cache_pool) + && (m_cache_buffer_chunk_size != 1); // if there are no allocated blocks, it's OK to switch allocator if (m_in_use == 0) m_using_pool_allocator = m_want_pool_allocator; @@ -476,7 +481,7 @@ namespace libtorrent int const cache_size = sett.get_int(settings_pack::cache_size); if (cache_size < 0) { - boost::uint64_t phys_ram = total_physical_ram(); + boost::uint64_t const phys_ram = total_physical_ram(); if (phys_ram == 0) m_max_use = 1024; else m_max_use = phys_ram / 8 / m_block_size; @@ -500,6 +505,8 @@ namespace libtorrent m_exceeded_max_size = true; m_trigger_cache_trim(); } + if (m_cache_buffer_chunk_size > m_max_use) + m_cache_buffer_chunk_size = m_max_use; } #if TORRENT_USE_ASSERTS diff --git a/src/disk_io_thread.cpp b/src/disk_io_thread.cpp index e9db92eaf..f45d83d48 100644 --- a/src/disk_io_thread.cpp +++ b/src/disk_io_thread.cpp @@ -1188,7 +1188,7 @@ namespace libtorrent int disk_io_thread::do_read(disk_io_job* j, jobqueue_t& completed_jobs) { - if (!m_settings.get_bool(settings_pack::use_read_cache) + if (m_settings.get_bool(settings_pack::use_read_cache) == false || m_settings.get_int(settings_pack::cache_size) == 0) { // we're not using a cache. This is the simple path @@ -1462,7 +1462,7 @@ namespace libtorrent // should we put this write job in the cache? // if we don't use the cache we shouldn't. if (m_settings.get_bool(settings_pack::use_write_cache) - && m_settings.get_int(settings_pack::cache_size) > 0) + && m_settings.get_int(settings_pack::cache_size) != 0) { mutex::scoped_lock l(m_cache_mutex); @@ -1578,7 +1578,7 @@ namespace libtorrent TORRENT_ASSERT(j->action == disk_io_job::read); if (m_settings.get_bool(settings_pack::use_read_cache) - && m_settings.get_int(settings_pack::cache_size) > 0) + && m_settings.get_int(settings_pack::cache_size) != 0) { int ret = m_disk_cache.try_read(j); if (ret >= 0) @@ -1691,7 +1691,7 @@ namespace libtorrent TORRENT_ASSERT(m_disk_cache.is_disk_buffer(j->buffer.disk_block)); l_.unlock(); #endif - if (m_settings.get_int(settings_pack::cache_size) > 0 + if (m_settings.get_int(settings_pack::cache_size) != 0 && m_settings.get_bool(settings_pack::use_write_cache)) { TORRENT_ASSERT((r.start % m_disk_cache.block_size()) == 0); @@ -2327,7 +2327,7 @@ namespace libtorrent } } - if (pe == NULL && !m_settings.get_bool(settings_pack::use_read_cache)) + if (pe == NULL && m_settings.get_bool(settings_pack::use_read_cache) == false) { l.unlock(); // if there's no piece in the cache, and the read cache is disabled @@ -2635,7 +2635,7 @@ namespace libtorrent { INVARIANT_CHECK; TORRENT_ASSERT(j->buffer.disk_block == 0); - + if (m_settings.get_int(settings_pack::cache_size) == 0 || m_settings.get_bool(settings_pack::use_read_cache) == false) return 0; @@ -3463,7 +3463,7 @@ namespace libtorrent if (j->action == disk_io_job::read && m_settings.get_bool(settings_pack::use_read_cache) - && m_settings.get_int(settings_pack::cache_size) > 0) + && m_settings.get_int(settings_pack::cache_size) != 0) { int state = prep_read_job_impl(j, false); switch (state) diff --git a/src/session_stats.cpp b/src/session_stats.cpp index 8bcf801a2..9d7e31643 100644 --- a/src/session_stats.cpp +++ b/src/session_stats.cpp @@ -355,10 +355,10 @@ namespace libtorrent // the total number of blocks run through SHA-1 hashing METRIC(disk, num_blocks_hashed) - + // the number of blocks read from the disk cache METRIC(disk, num_blocks_cache_hits) - + // the number of disk I/O operation for reads and writes. One disk // operation may transfer more then one block. METRIC(disk, num_write_ops) From a02c1d2164a6a1d1c57750a190d3ab556bad2ee4 Mon Sep 17 00:00:00 2001 From: arvidn Date: Sat, 5 Mar 2016 17:16:46 -0500 Subject: [PATCH 2/4] record cache settings in the job instead of checking it twice for more robust behavior --- include/libtorrent/disk_io_job.hpp | 6 +++++- simulation/test_transfer.cpp | 17 +++++++++++++++++ src/disk_io_thread.cpp | 21 +++++++++++---------- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/include/libtorrent/disk_io_job.hpp b/include/libtorrent/disk_io_job.hpp index fdb693f2c..ded630b6f 100644 --- a/include/libtorrent/disk_io_job.hpp +++ b/include/libtorrent/disk_io_job.hpp @@ -136,7 +136,11 @@ namespace libtorrent in_progress = 0x20, // turns into file::coalesce_buffers in the file operation - coalesce_buffers = 0x40 + coalesce_buffers = 0x40, + + // the disk cache was enabled when this job was issued, it should use + // the disk cache once it's handled by a disk thread + use_disk_cache = 0x80 }; // for write jobs, returns true if its block diff --git a/simulation/test_transfer.cpp b/simulation/test_transfer.cpp index a783e4b8a..ddae108ba 100644 --- a/simulation/test_transfer.cpp +++ b/simulation/test_transfer.cpp @@ -441,3 +441,20 @@ TORRENT_TEST(auto_disk_cache_size) } ); } + +TORRENT_TEST(disable_disk_cache) +{ + using namespace libtorrent; + run_test( + [](lt::session& ses0, lt::session& ses1) { set_cache_size(ses0, 0); }, + [](lt::session& ses, lt::alert const* alert) {}, + [](std::shared_ptr ses[2]) { + TEST_EQUAL(is_seed(*ses[0]), true); + + int const cache_size = get_cache_size(*ses[0]); + printf("cache size: %d\n", cache_size); + TEST_EQUAL(cache_size, 0); + } + ); +} + diff --git a/src/disk_io_thread.cpp b/src/disk_io_thread.cpp index f45d83d48..57ac499c0 100644 --- a/src/disk_io_thread.cpp +++ b/src/disk_io_thread.cpp @@ -1188,16 +1188,11 @@ namespace libtorrent int disk_io_thread::do_read(disk_io_job* j, jobqueue_t& completed_jobs) { - if (m_settings.get_bool(settings_pack::use_read_cache) == false - || m_settings.get_int(settings_pack::cache_size) == 0) + if ((j->flags & disk_io_job::use_disk_cache) == 0) { // we're not using a cache. This is the simple path // just read straight from the file int ret = do_uncached_read(j); - - mutex::scoped_lock l(m_cache_mutex); - cached_piece_entry* pe = m_disk_cache.find_piece(j); - if (pe) maybe_issue_queued_read_jobs(pe, completed_jobs); return ret; } @@ -1461,8 +1456,7 @@ namespace libtorrent // should we put this write job in the cache? // if we don't use the cache we shouldn't. - if (m_settings.get_bool(settings_pack::use_write_cache) - && m_settings.get_int(settings_pack::cache_size) != 0) + if (j->flags & disk_io_job::use_disk_cache) { mutex::scoped_lock l(m_cache_mutex); @@ -1615,7 +1609,7 @@ namespace libtorrent j->error.operation = storage_error::read; return 0; } - + j->flags |= disk_io_job::use_disk_cache; if (pe->outstanding_read) { TORRENT_PIECE_ASSERT(j->piece == pe->piece, pe); @@ -1695,6 +1689,7 @@ namespace libtorrent && m_settings.get_bool(settings_pack::use_write_cache)) { TORRENT_ASSERT((r.start % m_disk_cache.block_size()) == 0); + j->flags |= disk_io_job::use_disk_cache; if (storage->is_blocked(j)) { @@ -1784,6 +1779,12 @@ namespace libtorrent } l.unlock(); + if (m_settings.get_bool(settings_pack::use_read_cache) + && m_settings.get_int(settings_pack::cache_size) != 0) + { + j->flags |= disk_io_job::use_disk_cache; + } + add_job(j); } @@ -2284,7 +2285,7 @@ namespace libtorrent { INVARIANT_CHECK; - if (m_settings.get_int(settings_pack::cache_size) == 0) + if ((j->flags & disk_io_job::use_disk_cache) == 0) return do_uncached_hash(j); int const piece_size = j->storage->files()->piece_size(j->piece); From adf56344e65479ba386f8be9afb30bfc9cf44b0e Mon Sep 17 00:00:00 2001 From: arvidn Date: Sat, 5 Mar 2016 23:16:33 -0500 Subject: [PATCH 3/4] add interactive option to toggle use of disk cache to client_test --- examples/client_test.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/client_test.cpp b/examples/client_test.cpp index 403f473ba..cfff11bb1 100644 --- a/examples/client_test.cpp +++ b/examples/client_test.cpp @@ -606,6 +606,7 @@ std::string bind_to_interface = ""; int poll_interval = 5; int max_connections_per_torrent = 50; bool seed_mode = false; +int cache_size = 1024; bool share_mode = false; bool disable_storage = false; @@ -1296,6 +1297,7 @@ int main(int argc, char* argv[]) namespace lt = libtorrent; settings_pack settings; + settings.set_int(settings_pack::cache_size, cache_size); settings.set_int(settings_pack::active_loaded_limit, 20); settings.set_int(settings_pack::choking_algorithm, settings_pack::rate_based_choker); @@ -1450,8 +1452,8 @@ int main(int argc, char* argv[]) } #endif // TORRENT_USE_I2P case 'C': - settings.set_int(settings_pack::cache_size, atoi(arg)); - settings.set_bool(settings_pack::use_read_cache, atoi(arg) != 0); + cache_size = atoi(arg); + settings.set_int(settings_pack::cache_size, cache_size); settings.set_int(settings_pack::cache_buffer_chunk_size, 0); break; case 'A': settings.set_int(settings_pack::allowed_fast_set_size, atoi(arg)); break; @@ -1923,6 +1925,13 @@ int main(int argc, char* argv[]) if (c == '5') print_peer_rate = !print_peer_rate; if (c == '6') print_fails = !print_fails; if (c == '7') print_send_bufs = !print_send_bufs; + if (c == 'C') + { + cache_size = (cache_size == 0) ? -1 : 0; + settings_pack p; + p.set_int(settings_pack::cache_size, cache_size); + ses.apply_settings(p); + } if (c == 'h') { clear_screen(); @@ -1933,7 +1942,7 @@ int main(int argc, char* argv[]) "[q] quit client [m] add magnet link\n" "\n" "TORRENT ACTIONS\n" - "[p] pause/unpause selected torrent\n" + "[p] pause/unpause selected torrent [C] toggle disk cache\n" "[s] toggle sequential download [j] force recheck\n" "[space] toggle session pause [c] clear error\n" "[v] scrape [D] delete torrent and data\n" From b40430f17583c8a835afcd202092cd6120979acc Mon Sep 17 00:00:00 2001 From: arvidn Date: Sun, 6 Mar 2016 01:42:46 -0500 Subject: [PATCH 4/4] change the cache size calculation for auto cache size (-1) to be smaller, especially for machines with large amounts of RAM --- src/disk_buffer_pool.cpp | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/disk_buffer_pool.cpp b/src/disk_buffer_pool.cpp index 642c55477..79486d75e 100644 --- a/src/disk_buffer_pool.cpp +++ b/src/disk_buffer_pool.cpp @@ -481,9 +481,35 @@ namespace libtorrent int const cache_size = sett.get_int(settings_pack::cache_size); if (cache_size < 0) { - boost::uint64_t const phys_ram = total_physical_ram(); + boost::uint64_t phys_ram = total_physical_ram(); if (phys_ram == 0) m_max_use = 1024; - else m_max_use = phys_ram / 8 / m_block_size; + else + { + // this is the logic to calculate the automatic disk cache size + // based on the amount of physical RAM. + // The more physical RAM, the smaller portion of it is allocated + // for the cache. + + // we take a 30th of everything exceeding 4 GiB + // a 20th of everything exceeding 1 GiB + // and a 10th of everything below a GiB + + boost::int64_t const gb = 1024 * 1024 * 1024; + + boost::int64_t result = 0; + if (phys_ram > 4 * gb) + { + result += (phys_ram - 4 * gb) / 30; + phys_ram = 4 * gb; + } + if (phys_ram > 1 * gb) + { + result += (phys_ram - 1 * gb) / 20; + phys_ram = 1 * gb; + } + result += phys_ram / 10; + m_max_use = result / m_block_size; + } if (sizeof(void*) == 4) {