removed mmap_cache feature

This commit is contained in:
arvidn 2016-12-25 16:07:31 -08:00 committed by Arvid Norberg
parent 6be7bb9eb7
commit 5ed0086b51
11 changed files with 82 additions and 270 deletions

View File

@ -1,3 +1,4 @@
* removed mmap_cache feature
* strengthened type safety in handling of piece and file indices
* deprecate identify_client() and fingerprint type
* make sequence number for mutable DHT items backed by std::int64_t

View File

@ -448,7 +448,7 @@ namespace libtorrent
#ifndef TORRENT_NO_DEPRECATE
void get_stats(cache_status* ret) const;
#endif
void set_settings(aux::session_settings const& sett, error_code& ec);
void set_settings(aux::session_settings const& sett);
enum reason_t { ref_hashing = 0, ref_reading = 1, ref_flushing = 2 };
bool inc_block_refcount(cached_piece_entry* pe, int block, int reason);

View File

@ -94,7 +94,7 @@ namespace libtorrent
}
int num_to_evict(int num_needed = 0);
void set_settings(aux::session_settings const& sett, error_code& ec);
void set_settings(aux::session_settings const& sett);
protected:
@ -142,18 +142,6 @@ namespace libtorrent
int m_cache_buffer_chunk_size;
#if TORRENT_HAVE_MMAP && !defined TORRENT_NO_DEPRECATE
// the file descriptor of the cache mmap file
int m_cache_fd;
// the pointer to the block of virtual address space
// making up the mmapped cache space
char* m_cache_pool;
// list of block indices that are not in use. block_index
// times 0x4000 + m_cache_pool is the address where the
// corresponding memory lives
std::vector<int> m_free_list;
#endif
#ifndef TORRENT_DISABLE_POOL_ALLOCATOR
// if this is true, all buffers are allocated
// from m_pool. If this is false, all buffers

View File

@ -287,7 +287,7 @@ namespace libtorrent
, int block_size = 16 * 1024);
~disk_io_thread();
void set_settings(settings_pack const* sett, alert_manager& alerts);
void set_settings(settings_pack const* sett);
void abort(bool wait);

View File

@ -1542,7 +1542,7 @@ void block_cache::get_stats(cache_status* ret) const
}
#endif
void block_cache::set_settings(aux::session_settings const& sett, error_code& ec)
void block_cache::set_settings(aux::session_settings const& sett)
{
// the ghost size is the number of pieces to keep track of
// after they are evicted. Since cache_size is blocks, the
@ -1553,7 +1553,7 @@ void block_cache::set_settings(aux::session_settings const& sett, error_code& ec
/ (std::max)(sett.get_int(settings_pack::read_cache_line_size), 4) / 2);
m_max_volatile_blocks = sett.get_int(settings_pack::cache_size_volatile);
disk_buffer_pool::set_settings(sett, ec);
disk_buffer_pool::set_settings(sett);
}
#if TORRENT_USE_INVARIANT_CHECKS

View File

@ -287,9 +287,7 @@ namespace libtorrent
sett.set_int(settings_pack::cache_size, 0);
sett.set_int(settings_pack::aio_threads, 1);
// TODO: this should probably be optional
alert_manager dummy2(0, 0);
disk_thread.set_settings(&sett, dummy2);
disk_thread.set_settings(&sett);
int const piece_read_ahead = std::max(1, 15 * 1024 * 1024 / t.piece_length());

View File

@ -41,10 +41,6 @@ POSSIBILITY OF SUCH DAMAGE.
#include "libtorrent/aux_/disable_warnings_push.hpp"
#if TORRENT_HAVE_MMAP
#include <sys/mman.h>
#endif
#ifdef TORRENT_BSD
#include <sys/sysctl.h>
#endif
@ -81,10 +77,6 @@ namespace libtorrent
, m_exceeded_max_size(false)
, m_ios(ios)
, m_cache_buffer_chunk_size(0)
#if TORRENT_HAVE_MMAP && !defined TORRENT_NO_DEPRECATE
, m_cache_fd(-1)
, m_cache_pool(nullptr)
#endif
#ifndef TORRENT_DISABLE_POOL_ALLOCATOR
, m_using_pool_allocator(false)
, m_want_pool_allocator(false)
@ -104,19 +96,6 @@ namespace libtorrent
m_magic = 0;
#endif
#if TORRENT_HAVE_MMAP && !defined TORRENT_NO_DEPRECATE
if (m_cache_pool)
{
munmap(m_cache_pool, std::uint64_t(m_max_use) * 0x4000);
m_cache_pool = nullptr;
// attempt to make MacOS not flush this to disk, making close()
// block for a long time
int const best_effort = ftruncate(m_cache_fd, 0);
TORRENT_UNUSED(best_effort);
close(m_cache_fd);
m_cache_fd = -1;
}
#endif
}
int disk_buffer_pool::num_to_evict(int const num_needed)
@ -162,14 +141,6 @@ namespace libtorrent
TORRENT_ASSERT(l.owns_lock());
TORRENT_UNUSED(l);
#if TORRENT_HAVE_MMAP && !defined TORRENT_NO_DEPRECATE
if (m_cache_pool)
{
return buffer >= m_cache_pool && buffer < m_cache_pool
+ std::uint64_t(m_max_use) * 0x4000;
}
#endif
#if TORRENT_USE_INVARIANT_CHECKS
return m_buffers_in_use.count(buffer) == 1;
#elif defined TORRENT_DEBUG_BUFFERS
@ -268,24 +239,6 @@ namespace libtorrent
TORRENT_UNUSED(l);
char* ret;
#if TORRENT_HAVE_MMAP && !defined TORRENT_NO_DEPRECATE
if (m_cache_pool)
{
if (int(m_free_list.size()) <= (m_max_use - m_low_watermark)
/ 2 && !m_exceeded_max_size)
{
m_exceeded_max_size = true;
m_trigger_cache_trim();
}
if (m_free_list.empty()) return nullptr;
std::int64_t const slot_index = m_free_list.back();
m_free_list.pop_back();
ret = m_cache_pool + (slot_index * 0x4000);
TORRENT_ASSERT(is_disk_buffer(ret, l));
}
else
#endif
{
#if defined TORRENT_DISABLE_POOL_ALLOCATOR
ret = page_aligned_allocator::malloc(m_block_size);
@ -313,7 +266,6 @@ namespace libtorrent
m_trigger_cache_trim();
return nullptr;
}
}
++m_in_use;
@ -366,11 +318,8 @@ namespace libtorrent
check_buffer_level(l);
}
void disk_buffer_pool::set_settings(aux::session_settings const& sett
, error_code& ec)
void disk_buffer_pool::set_settings(aux::session_settings const& sett)
{
TORRENT_UNUSED(ec);
std::unique_lock<std::mutex> l(m_pool_mutex);
// 0 cache_buffer_chunk_size means 'automatic' (i.e.
@ -385,23 +334,6 @@ namespace libtorrent
m_using_pool_allocator = m_want_pool_allocator;
#endif
#if TORRENT_HAVE_MMAP && !defined TORRENT_NO_DEPRECATE
// if we've already allocated an mmap, we can't change
// anything unless there are no allocations in use
if (m_cache_pool && m_in_use > 0) return;
#endif
// only allow changing size if we're not using mmapped
// cache, or if we're just about to turn it off
if (
#if TORRENT_HAVE_MMAP && !defined TORRENT_NO_DEPRECATE
m_cache_pool == nullptr ||
sett.get_str(settings_pack::mmap_cache).empty()
#else
true
#endif
)
{
int const cache_size = sett.get_int(settings_pack::cache_size);
if (cache_size < 0)
{
@ -464,79 +396,10 @@ namespace libtorrent
}
if (m_cache_buffer_chunk_size > m_max_use)
m_cache_buffer_chunk_size = m_max_use;
}
#if TORRENT_USE_ASSERTS
m_settings_set = true;
#endif
#if TORRENT_HAVE_MMAP && !defined TORRENT_NO_DEPRECATE
// #error support resizing the map
if (m_cache_pool && sett.get_str(settings_pack::mmap_cache).empty())
{
TORRENT_ASSERT(m_in_use == 0);
munmap(m_cache_pool, std::uint64_t(m_max_use) * 0x4000);
m_cache_pool = nullptr;
// attempt to make MacOS not flush this to disk, making close()
// block for a long time
int const best_effort = ftruncate(m_cache_fd, 0);
TORRENT_UNUSED(best_effort);
close(m_cache_fd);
m_cache_fd = -1;
m_free_list.clear();
m_free_list.shrink_to_fit();
}
else if (m_cache_pool == nullptr && !sett.get_str(settings_pack::mmap_cache).empty())
{
// O_TRUNC here is because we don't actually care about what's
// in the file now, there's no need to ever read that into RAM
#ifndef O_EXLOCK
#define O_EXLOCK 0
#endif
m_cache_fd = open(sett.get_str(settings_pack::mmap_cache).c_str(), O_RDWR | O_CREAT | O_EXLOCK | O_TRUNC, 0700);
if (m_cache_fd < 0)
{
ec.assign(errno, boost::system::system_category());
}
else
{
#ifndef MAP_NOCACHE
#define MAP_NOCACHE 0
#endif
if (ftruncate(m_cache_fd, std::uint64_t(m_max_use) * 0x4000) < 0)
{
ec.assign(errno, boost::system::system_category());
m_cache_pool = nullptr;
close(m_cache_fd);
m_cache_fd = -1;
return;
}
m_cache_pool = static_cast<char*>(mmap(nullptr, std::uint64_t(m_max_use) * 0x4000, PROT_READ | PROT_WRITE
, MAP_SHARED | MAP_NOCACHE, m_cache_fd, 0));
if (intptr_t(m_cache_pool) == -1)
{
ec.assign(errno, boost::system::system_category());
m_cache_pool = nullptr;
// attempt to make MacOS not flush this to disk, making close()
// block for a long time
int const best_effort2 = ftruncate(m_cache_fd, 0);
TORRENT_UNUSED(best_effort2);
close(m_cache_fd);
m_cache_fd = -1;
return;
}
else
{
TORRENT_ASSERT((std::size_t(m_cache_pool) & 0xfff) == 0);
m_free_list.reserve(m_max_use);
for (int i = 0; i < m_max_use; ++i)
m_free_list.push_back(i);
}
}
}
#endif // TORRENT_HAVE_MMAP
}
void disk_buffer_pool::remove_buffer_in_use(char* buf)
@ -557,29 +420,6 @@ namespace libtorrent
TORRENT_ASSERT(l.owns_lock());
TORRENT_UNUSED(l);
#if TORRENT_HAVE_MMAP && !defined TORRENT_NO_DEPRECATE
if (m_cache_pool)
{
TORRENT_ASSERT(buf >= m_cache_pool);
TORRENT_ASSERT(buf < m_cache_pool + std::uint64_t(m_max_use) * 0x4000);
int slot_index = int((buf - m_cache_pool) / 0x4000);
m_free_list.push_back(slot_index);
#if defined MADV_FREE
// tell the virtual memory system that we don't actually care
// about the data in these pages anymore. If this block was
// swapped out to the SSD, it (hopefully) means it won't have
// to be read back in once we start writing our new data to it
madvise(buf, 0x4000, MADV_FREE);
#elif defined MADV_DONTNEED && defined TORRENT_LINUX
// rumor has it that MADV_DONTNEED is in fact destructive
// on linux (i.e. it won't flush it to disk or re-read from disk)
// http://kerneltrap.org/mailarchive/linux-kernel/2007/5/1/84410
madvise(buf, 0x4000, MADV_DONTNEED);
#endif
}
else
#endif
{
#if defined TORRENT_DISABLE_POOL_ALLOCATOR
page_aligned_allocator::free(buf);
@ -590,7 +430,6 @@ namespace libtorrent
else
page_aligned_allocator::free(buf);
#endif // TORRENT_DISABLE_POOL_ALLOCATOR
}
--m_in_use;

View File

@ -199,9 +199,7 @@ namespace libtorrent
, m_ios(ios)
{
ADD_OUTSTANDING_ASYNC("disk_io_thread::work");
error_code ec;
m_disk_cache.set_settings(m_settings, ec);
TORRENT_ASSERT(!ec);
m_disk_cache.set_settings(m_settings);
// deduct some margin for epoll/kqueue, log files,
// futexes, shared objects etc.
@ -262,21 +260,12 @@ namespace libtorrent
}
}
void disk_io_thread::set_settings(settings_pack const* pack, alert_manager& alerts)
void disk_io_thread::set_settings(settings_pack const* pack)
{
TORRENT_ASSERT(m_magic == 0x1337);
std::unique_lock<std::mutex> l(m_cache_mutex);
apply_pack(pack, m_settings);
error_code ec;
m_disk_cache.set_settings(m_settings, ec);
#ifndef TORRENT_NO_DEPRECATE
if (ec && alerts.should_post<mmap_cache_alert>())
{
alerts.emplace_alert<mmap_cache_alert>(ec);
}
#else
TORRENT_UNUSED(alerts);
#endif
m_disk_cache.set_settings(m_settings);
int const num_threads = m_settings.get_int(settings_pack::aio_threads);
// add one hasher thread for every three generic threads

View File

@ -1358,7 +1358,7 @@ namespace aux {
#endif
apply_pack(&pack, m_settings, this);
m_disk_thread.set_settings(&pack, m_alerts);
m_disk_thread.set_settings(&pack);
if (init && !reopen_listen_port)
{

View File

@ -99,8 +99,7 @@ static void nop() {}
std::shared_ptr<storage_interface> pm \
= std::make_shared<test_storage_impl>(); \
pm->set_files(&fs); \
error_code ec; \
bc.set_settings(sett, ec); \
bc.set_settings(sett); \
pm->m_settings = &sett; \
disk_io_job rj; \
disk_io_job wj; \

View File

@ -465,9 +465,7 @@ void test_check_files(std::string const& test_path
disk_io_thread io(ios, cnt);
settings_pack sett;
sett.set_int(settings_pack::aio_threads, 1);
// TODO: this should probably be optional
alert_manager dummy2(0, 0);
io.set_settings(&sett, dummy2);
io.set_settings(&sett);
disk_buffer_pool dp(16 * 1024, ios, std::bind(&nop));
storage_params p;