premiere
/
premiere-libtorrent
2
2
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #523 from arvidn/disk-cache-1.1 

fix support for auto disk cache size
This commit is contained in:
Arvid Norberg 2016-03-06 16:16:17 -05:00
parent 5febe36106 b40430f175
commit 834d04efdc
7 changed files with 141 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
examples/client_test.cpp
View File

@ -606,6 +606,7 @@ std::string bind_to_interface = "";
int poll_interval = 5;
int max_connections_per_torrent = 50;
bool seed_mode = false;
int cache_size = 1024;
bool share_mode = false;
bool disable_storage = false;
@ -1296,6 +1297,7 @@ int main(int argc, char* argv[])
namespace lt = libtorrent;
settings_pack settings;
settings.set_int(settings_pack::cache_size, cache_size);
settings.set_int(settings_pack::active_loaded_limit, 20);
settings.set_int(settings_pack::choking_algorithm, settings_pack::rate_based_choker);
@ -1450,9 +1452,9 @@ int main(int argc, char* argv[])
}
#endif // TORRENT_USE_I2P
case 'C':
settings.set_int(settings_pack::cache_size, atoi(arg));
settings.set_bool(settings_pack::use_read_cache, atoi(arg) > 0);
settings.set_int(settings_pack::cache_buffer_chunk_size, atoi(arg) / 100);
cache_size = atoi(arg);
settings.set_int(settings_pack::cache_size, cache_size);
settings.set_int(settings_pack::cache_buffer_chunk_size, 0);
break;
case 'A': settings.set_int(settings_pack::allowed_fast_set_size, atoi(arg)); break;
case 'R': settings.set_int(settings_pack::read_cache_line_size, atoi(arg)); break;
@ -1923,6 +1925,13 @@ int main(int argc, char* argv[])
if (c == '5') print_peer_rate = !print_peer_rate;
if (c == '6') print_fails = !print_fails;
if (c == '7') print_send_bufs = !print_send_bufs;
if (c == 'C')
{
cache_size = (cache_size == 0) ? -1 : 0;
settings_pack p;
p.set_int(settings_pack::cache_size, cache_size);
ses.apply_settings(p);
}
if (c == 'h')
{
clear_screen();
@ -1933,7 +1942,7 @@ int main(int argc, char* argv[])
"[q] quit client [m] add magnet link\n"
"\n"
"TORRENT ACTIONS\n"
"[p] pause/unpause selected torrent\n"
"[p] pause/unpause selected torrent [C] toggle disk cache\n"
"[s] toggle sequential download [j] force recheck\n"
"[space] toggle session pause [c] clear error\n"
"[v] scrape [D] delete torrent and data\n"

6
include/libtorrent/disk_io_job.hpp
View File

@ -136,7 +136,11 @@ namespace libtorrent
in_progress = 0x20,
// turns into file::coalesce_buffers in the file operation
coalesce_buffers = 0x40
coalesce_buffers = 0x40,
// the disk cache was enabled when this job was issued, it should use
// the disk cache once it's handled by a disk thread
use_disk_cache = 0x80
};
// for write jobs, returns true if its block

2
simulation/test_swarm.cpp
View File

@ -97,7 +97,7 @@ TORRENT_TEST(session_stats)
// on alert
, [=](lt::alert const* a, lt::session& ses)
{
lt::session_stats_alert const* ss = lt::alert_cast<session_stats_alert>(a);
auto const* ss = lt::alert_cast<session_stats_alert>(a);
if (!ss) return;
// there's one downloading torrent

67
simulation/test_transfer.cpp
View File

@ -35,6 +35,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include "create_torrent.hpp"
#include "settings.hpp"
#include "libtorrent/session.hpp"
#include "libtorrent/session_stats.hpp"
#include "libtorrent/deadline_timer.hpp"
#include "libtorrent/settings_pack.hpp"
#include "libtorrent/ip_filter.hpp"
@ -194,6 +195,39 @@ void filter_ips(lt::session& ses)
ses.set_ip_filter(filter);
}
void set_cache_size(lt::session& ses, int val)
{
using namespace libtorrent;
settings_pack pack;
pack.set_int(settings_pack::cache_size, val);
ses.apply_settings(pack);
}
int get_cache_size(lt::session& ses)
{
using namespace libtorrent;
std::vector<stats_metric> stats = session_stats_metrics();
int const read_cache_idx = find_metric_idx("disk.read_cache_blocks");
int const write_cache_idx = find_metric_idx("disk.write_cache_blocks");
TEST_CHECK(read_cache_idx >= 0);
TEST_CHECK(write_cache_idx >= 0);
ses.set_alert_notify([](){});
ses.post_session_stats();
std::vector<alert*> alerts;
ses.pop_alerts(&alerts);
int cache_size = -1;
for (auto const a : alerts)
{
if (auto const* st = alert_cast<session_stats_alert>(a))
{
cache_size = st->values[read_cache_idx];
cache_size += st->values[write_cache_idx];
break;
}
}
return cache_size;
}
void set_proxy(lt::session& ses, int proxy_type, int flags = 0, bool proxy_peer_connections = true)
{
// apply the proxy settings to session 0
@ -391,3 +425,36 @@ TORRENT_TEST(no_proxy_utp_banned)
);
}
TORRENT_TEST(auto_disk_cache_size)
{
using namespace libtorrent;
run_test(
[](lt::session& ses0, lt::session& ses1) { set_cache_size(ses0, -1); },
[](lt::session& ses, lt::alert const* alert) {},
[](std::shared_ptr<lt::session> ses[2]) {
TEST_EQUAL(is_seed(*ses[0]), true);
int const cache_size = get_cache_size(*ses[0]);
printf("cache size: %d\n", cache_size);
// this assumes the test torrent is at least 4 blocks
TEST_CHECK(cache_size > 4);
}
);
}
TORRENT_TEST(disable_disk_cache)
{
using namespace libtorrent;
run_test(
[](lt::session& ses0, lt::session& ses1) { set_cache_size(ses0, 0); },
[](lt::session& ses, lt::alert const* alert) {},
[](std::shared_ptr<lt::session> ses[2]) {
TEST_EQUAL(is_seed(*ses[0]), true);
int const cache_size = get_cache_size(*ses[0]);
printf("cache size: %d\n", cache_size);
TEST_EQUAL(cache_size, 0);
}
);
}

41
src/disk_buffer_pool.cpp
View File

@ -371,11 +371,14 @@ namespace libtorrent
#else
if (m_using_pool_allocator)
{
ret = static_cast<char*>(m_pool.malloc());
int effective_block_size = m_cache_buffer_chunk_size
int const effective_block_size
= m_in_use >= m_max_use
? 20 // use small increments once we've exceeded the cache size
: m_cache_buffer_chunk_size
? m_cache_buffer_chunk_size
: (std::max)(m_max_use / 10, 1);
m_pool.set_next_size(effective_block_size);
ret = static_cast<char*>(m_pool.malloc());
}
else
{
@ -453,7 +456,9 @@ namespace libtorrent
m_cache_buffer_chunk_size = sett.get_int(settings_pack::cache_buffer_chunk_size);
m_lock_disk_cache = sett.get_bool(settings_pack::lock_disk_cache);
#ifndef TORRENT_DISABLE_POOL_ALLOCATOR
m_want_pool_allocator = sett.get_bool(settings_pack::use_disk_cache_pool);
// if the chunk size is set to 1, there's no point in creating a pool
m_want_pool_allocator = sett.get_bool(settings_pack::use_disk_cache_pool)
&& (m_cache_buffer_chunk_size != 1);
// if there are no allocated blocks, it's OK to switch allocator
if (m_in_use == 0)
m_using_pool_allocator = m_want_pool_allocator;
@ -478,7 +483,33 @@ namespace libtorrent
{
boost::uint64_t phys_ram = total_physical_ram();
if (phys_ram == 0) m_max_use = 1024;
else m_max_use = phys_ram / 8 / m_block_size;
else
{
// this is the logic to calculate the automatic disk cache size
// based on the amount of physical RAM.
// The more physical RAM, the smaller portion of it is allocated
// for the cache.
// we take a 30th of everything exceeding 4 GiB
// a 20th of everything exceeding 1 GiB
// and a 10th of everything below a GiB
boost::int64_t const gb = 1024 * 1024 * 1024;
boost::int64_t result = 0;
if (phys_ram > 4 * gb)
{
result += (phys_ram - 4 * gb) / 30;
phys_ram = 4 * gb;
}
if (phys_ram > 1 * gb)
{
result += (phys_ram - 1 * gb) / 20;
phys_ram = 1 * gb;
}
result += phys_ram / 10;
m_max_use = result / m_block_size;
}
if (sizeof(void*) == 4)
{
@ -500,6 +531,8 @@ namespace libtorrent
m_exceeded_max_size = true;
m_trigger_cache_trim();
}
if (m_cache_buffer_chunk_size > m_max_use)
m_cache_buffer_chunk_size = m_max_use;
}
#if TORRENT_USE_ASSERTS

31
src/disk_io_thread.cpp
View File

@ -1188,16 +1188,11 @@ namespace libtorrent
int disk_io_thread::do_read(disk_io_job* j, jobqueue_t& completed_jobs)
{
if (!m_settings.get_bool(settings_pack::use_read_cache)
|| m_settings.get_int(settings_pack::cache_size) == 0)
if ((j->flags & disk_io_job::use_disk_cache) == 0)
{
// we're not using a cache. This is the simple path
// just read straight from the file
int ret = do_uncached_read(j);
mutex::scoped_lock l(m_cache_mutex);
cached_piece_entry* pe = m_disk_cache.find_piece(j);
if (pe) maybe_issue_queued_read_jobs(pe, completed_jobs);
return ret;
}
@ -1461,8 +1456,7 @@ namespace libtorrent
// should we put this write job in the cache?
// if we don't use the cache we shouldn't.
if (m_settings.get_bool(settings_pack::use_write_cache)
&& m_settings.get_int(settings_pack::cache_size) > 0)
if (j->flags & disk_io_job::use_disk_cache)
{
mutex::scoped_lock l(m_cache_mutex);
@ -1578,7 +1572,7 @@ namespace libtorrent
TORRENT_ASSERT(j->action == disk_io_job::read);
if (m_settings.get_bool(settings_pack::use_read_cache)
&& m_settings.get_int(settings_pack::cache_size) > 0)
&& m_settings.get_int(settings_pack::cache_size) != 0)
{
int ret = m_disk_cache.try_read(j);
if (ret >= 0)
@ -1615,7 +1609,7 @@ namespace libtorrent
j->error.operation = storage_error::read;
return 0;
}
j->flags |= disk_io_job::use_disk_cache;
if (pe->outstanding_read)
{
TORRENT_PIECE_ASSERT(j->piece == pe->piece, pe);
@ -1691,10 +1685,11 @@ namespace libtorrent
TORRENT_ASSERT(m_disk_cache.is_disk_buffer(j->buffer.disk_block));
l_.unlock();
#endif
if (m_settings.get_int(settings_pack::cache_size) > 0
if (m_settings.get_int(settings_pack::cache_size) != 0
&& m_settings.get_bool(settings_pack::use_write_cache))
{
TORRENT_ASSERT((r.start % m_disk_cache.block_size()) == 0);
j->flags |= disk_io_job::use_disk_cache;
if (storage->is_blocked(j))
{
@ -1784,6 +1779,12 @@ namespace libtorrent
}
l.unlock();
if (m_settings.get_bool(settings_pack::use_read_cache)
&& m_settings.get_int(settings_pack::cache_size) != 0)
{
j->flags |= disk_io_job::use_disk_cache;
}
add_job(j);
}
@ -2284,7 +2285,7 @@ namespace libtorrent
{
INVARIANT_CHECK;
if (m_settings.get_int(settings_pack::cache_size) == 0)
if ((j->flags & disk_io_job::use_disk_cache) == 0)
return do_uncached_hash(j);
int const piece_size = j->storage->files()->piece_size(j->piece);
@ -2327,7 +2328,7 @@ namespace libtorrent
}
}
if (pe == NULL && !m_settings.get_bool(settings_pack::use_read_cache))
if (pe == NULL && m_settings.get_bool(settings_pack::use_read_cache) == false)
{
l.unlock();
// if there's no piece in the cache, and the read cache is disabled
@ -2635,7 +2636,7 @@ namespace libtorrent
{
INVARIANT_CHECK;
TORRENT_ASSERT(j->buffer.disk_block == 0);
if (m_settings.get_int(settings_pack::cache_size) == 0
|| m_settings.get_bool(settings_pack::use_read_cache) == false)
return 0;
@ -3463,7 +3464,7 @@ namespace libtorrent
if (j->action == disk_io_job::read
&& m_settings.get_bool(settings_pack::use_read_cache)
&& m_settings.get_int(settings_pack::cache_size) > 0)
&& m_settings.get_int(settings_pack::cache_size) != 0)
{
int state = prep_read_job_impl(j, false);
switch (state)

4
src/session_stats.cpp
View File

@ -355,10 +355,10 @@ namespace libtorrent
// the total number of blocks run through SHA-1 hashing
METRIC(disk, num_blocks_hashed)
// the number of blocks read from the disk cache
METRIC(disk, num_blocks_cache_hits)
// the number of disk I/O operation for reads and writes. One disk
// operation may transfer more then one block.
METRIC(disk, num_write_ops)