From d1ea80c2c337625e5aad2f0e1c7ef389a45af6a4 Mon Sep 17 00:00:00 2001 From: arvidn Date: Sun, 8 Jan 2017 20:29:24 -0500 Subject: [PATCH] remove disk-access-log build configuration --- ChangeLog | 1 + Jamfile | 5 +- configure.ac | 23 ------- docs/building.rst | 4 -- docs/tuning.rst | 27 -------- include/libtorrent/file.hpp | 7 -- include/libtorrent/storage.hpp | 5 -- src/file.cpp | 38 +---------- src/storage.cpp | 84 ----------------------- tools/Makefile.am | 1 - tools/parse_disk_log.py | 121 --------------------------------- 11 files changed, 3 insertions(+), 313 deletions(-) delete mode 100755 tools/parse_disk_log.py diff --git a/ChangeLog b/ChangeLog index 8b1e6c198..8f698fbbc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ + * removed disk-access-log build configuration * removed mmap_cache feature * strengthened type safety in handling of piece and file indices * deprecate identify_client() and fingerprint type diff --git a/Jamfile b/Jamfile index 4891e8a27..13508babe 100644 --- a/Jamfile +++ b/Jamfile @@ -426,9 +426,6 @@ feature invariant-checks : off on full : composite propagated link-incompatible feature.compose on : TORRENT_USE_INVARIANT_CHECKS=1 ; feature.compose full : TORRENT_USE_INVARIANT_CHECKS=1 TORRENT_EXPENSIVE_INVARIANT_CHECKS ; -feature disk-stats : off on : composite propagated link-incompatible ; -feature.compose on : TORRENT_DISK_STATS ; - feature utp-log : off on : composite propagated link-incompatible ; feature.compose on : TORRENT_UTP_LOG_ENABLE ; @@ -494,7 +491,7 @@ variant test_release : release on ; variant test_debug : debug - : on on + : on debug full shared on on multi on diff --git a/configure.ac b/configure.ac index 99d416391..c729d1447 100644 --- a/configure.ac +++ b/configure.ac @@ -244,15 +244,6 @@ AC_ARG_ENABLE( [[ARG_ENABLE_DEPRECATED=yes]] ) -AC_ARG_ENABLE( - [disk-stats], - [AS_HELP_STRING( - [--enable-disk-stats], - [enable disk activity logging feature [default=no]])], - [[ARG_ENABLE_DISK_STATS=$enableval]], - [[ARG_ENABLE_DISK_STATS=no]] -) - AC_ARG_ENABLE( [examples], [AS_HELP_STRING( @@ -361,19 +352,6 @@ AS_CASE(["$ARG_ENABLE_LOGGING"], AC_MSG_ERROR([Unknown option "$ARG_ENABLE_LOGGING". Use either "yes" or "no"])] ) -AC_MSG_CHECKING([whether disk activity logging should be enabled]) -AS_CASE(["$ARG_ENABLE_DISK_STATS"], - ["yes"|"on"], [ - AC_MSG_RESULT([yes]) - AC_DEFINE([TORRENT_DISK_STATS],[1],[Define to create a log of all disk activities.]) - ], - ["no"|"off"], [ - AC_MSG_RESULT([no]) - ], - [AC_MSG_RESULT([$ARG_ENABLE_DISK_STATS]) - AC_MSG_ERROR([Unknown option "$ARG_ENABLE_DISK_STATS". Use either "yes" or "no".])] -) - AS_ECHO AS_ECHO "Checking features to be enabled:" @@ -608,7 +586,6 @@ Build options: debug build: ${ARG_ENABLE_DEBUG:-no} invariant checks: ${ARG_ENABLE_INVARIANT:-no} logging support: ${ARG_ENABLE_LOGGING:-no} - disk statistics: ${ARG_ENABLE_DISK_STATS:-no} Features: encryption support: ${ARG_ENABLE_ENCRYPTION:-yes} diff --git a/docs/building.rst b/docs/building.rst index dc20f5936..e93446efc 100644 --- a/docs/building.rst +++ b/docs/building.rst @@ -534,10 +534,6 @@ defines you can use to control the build. | | checks in the storage, including logging of | | | piece sorting. | +----------------------------------------+-------------------------------------------------+ -| ``TORRENT_DISK_STATS`` | This will create a log of all disk activity | -| | which later can parsed and graphed using | -| | ``parse_disk_log.py``. | -+----------------------------------------+-------------------------------------------------+ | ``UNICODE`` | If building on windows this will make sure the | | | UTF-8 strings in pathnames are converted into | | | UTF-16 before they are passed to the file | diff --git a/docs/tuning.rst b/docs/tuning.rst index 38af97441..b9f97664d 100644 --- a/docs/tuning.rst +++ b/docs/tuning.rst @@ -380,33 +380,6 @@ preprocessor symbols when building. There are also a number of scripts that parses the log files and generates graphs (requires gnuplot and python). -disk metrics ------------- - -To enable disk I/O instrumentation, define ``TORRENT_DISK_STATS`` when building. When built -with this configuration libtorrent will create three log files, measuring various aspects of -the disk I/O. The following table is an overview of these files and what they measure. - -+--------------------------+--------------------------------------------------------------+ -| filename | description | -+==========================+==============================================================+ -| ``file_access.log`` | This is a low level log of read and write operations, with | -| | timestamps and file offsets. The file offsets are byte | -| | offsets in the torrent (not in any particular file, in the | -| | case of a multi-file torrent). This can be used as an | -| | estimate of the physical drive location. The purpose of | -| | this log is to identify the amount of seeking the drive has | -| | to do. | -| | | -+--------------------------+--------------------------------------------------------------+ - -file_access.log -''''''''''''''' - -The disk access log is a binary file that can be parsed and converted to human -readable by the script ``tools/parse_access_log.py``. This tool produces a -graphical representation of the disk access and requires ``gnuplot``. - understanding the disk threads ============================== diff --git a/include/libtorrent/file.hpp b/include/libtorrent/file.hpp index a0f92e99d..b46786241 100644 --- a/include/libtorrent/file.hpp +++ b/include/libtorrent/file.hpp @@ -305,16 +305,9 @@ namespace libtorrent handle_type native_handle() const { return m_file_handle; } -#ifdef TORRENT_DISK_STATS - std::uint32_t file_id() const { return m_file_id; } -#endif - private: handle_type m_file_handle; -#ifdef TORRENT_DISK_STATS - std::uint32_t m_file_id; -#endif int m_open_mode; #if defined TORRENT_WINDOWS diff --git a/include/libtorrent/storage.hpp b/include/libtorrent/storage.hpp index 608ee2faf..31a413d24 100644 --- a/include/libtorrent/storage.hpp +++ b/include/libtorrent/storage.hpp @@ -523,11 +523,6 @@ namespace libtorrent // file_storage, otherwise returns the original file_storage object. file_storage const& files() const { return m_mapped_files ? *m_mapped_files : m_files; } -#ifdef TORRENT_DISK_STATS - static bool disk_write_access_log(); - static void disk_write_access_log(bool enable); -#endif - private: void delete_one_file(std::string const& p, error_code& ec); diff --git a/src/file.cpp b/src/file.cpp index c896b93e4..558b4fbd4 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -77,10 +77,6 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/aux_/escape_string.hpp" #include "libtorrent/assert.hpp" -#ifdef TORRENT_DISK_STATS -#include "libtorrent/io.hpp" -#endif - #include "libtorrent/aux_/disable_warnings_push.hpp" #include @@ -1313,19 +1309,12 @@ namespace libtorrent file::file() : m_file_handle(INVALID_HANDLE_VALUE) , m_open_mode(0) - { -#ifdef TORRENT_DISK_STATS - m_file_id = 0; -#endif - } + {} file::file(std::string const& path, int mode, error_code& ec) : m_file_handle(INVALID_HANDLE_VALUE) , m_open_mode(0) { -#ifdef TORRENT_DISK_STATS - m_file_id = 0; -#endif // the return value is not important, since the // error code contains the same information open(path, mode, ec); @@ -1336,30 +1325,10 @@ namespace libtorrent close(); } -#ifdef TORRENT_DISK_STATS - namespace - { - std::uint32_t silly_hash(std::string const& str) - { - std::uint32_t ret = 1; - for (auto const ch : str) - { - if (ch == 0) continue; - ret *= std::uint32_t(ch); - } - return ret; - } - } -#endif - bool file::open(std::string const& path, int mode, error_code& ec) { close(); -#ifdef TORRENT_DISK_STATS - m_file_id = silly_hash(path); -#endif - #ifdef TORRENT_WINDOWS struct open_mode_t @@ -1576,7 +1545,6 @@ typedef struct _FILE_ALLOCATED_RANGE_BUFFER { } else if (ret == FALSE) { -// int error = GetLastError(); return true; } @@ -1591,10 +1559,6 @@ typedef struct _FILE_ALLOCATED_RANGE_BUFFER { void file::close() { -#ifdef TORRENT_DISK_STATS - m_file_id = 0; -#endif - if (!is_open()) return; #ifdef TORRENT_WINDOWS diff --git a/src/storage.cpp b/src/storage.cpp index 604f2cef5..a449fbcf9 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -167,49 +167,6 @@ namespace libtorrent } #endif -#ifdef TORRENT_DISK_STATS - static std::atomic event_id; - static std::mutex disk_access_mutex; - - // this is opened and closed by the disk_io_thread class - FILE* g_access_log = nullptr; - - enum access_log_flags_t - { - op_read = 0, - op_write = 1, - op_start = 0, - op_end = 2 - }; - - void write_access_log(std::uint64_t offset, std::uint32_t fileid, int flags, time_point timestamp) - { - if (g_access_log == nullptr) return; - - // the event format in the log is: - // uint64_t timestamp (microseconds) - // uint64_t file offset - // uint32_t file-id - // uint8_t event (0: start read, 1: start write, 2: complete read, 4: complete write) - char event[29]; - char* ptr = event; - detail::write_uint64(timestamp.time_since_epoch().count(), ptr); - detail::write_uint64(offset, ptr); - detail::write_uint64(static_cast(event_id++), ptr); - detail::write_uint32(fileid, ptr); - detail::write_uint8(std::uint8_t(flags), ptr); - - std::unique_lock l(disk_access_mutex); - int const ret = int(fwrite(event, 1, sizeof(event), g_access_log)); - l.unlock(); - if (ret != sizeof(event)) - { - std::fprintf(stderr, "ERROR writing to disk access log: (%d) %s\n" - , errno, strerror(errno)); - } - } -#endif - } // anonymous namespace struct write_fileop final : fileop @@ -266,10 +223,6 @@ namespace libtorrent #endif file_offset; -#ifdef TORRENT_DISK_STATS - write_access_log(adjusted_offset, handle->file_id(), op_start | op_write, clock_type::now()); -#endif - error_code e; int const ret = int(handle->writev(adjusted_offset , bufs, e, m_flags)); @@ -280,10 +233,6 @@ namespace libtorrent // we either get an error or 0 or more bytes read TORRENT_ASSERT(e || ret >= 0); - -#ifdef TORRENT_DISK_STATS - write_access_log(adjusted_offset + ret , handle->file_id(), op_end | op_write, clock_type::now()); -#endif TORRENT_ASSERT(ret <= bufs_size(bufs)); if (e) @@ -351,10 +300,6 @@ namespace libtorrent #endif file_offset; -#ifdef TORRENT_DISK_STATS - write_access_log(adjusted_offset, handle->file_id(), op_start | op_read, clock_type::now()); -#endif - error_code e; int const ret = int(handle->readv(adjusted_offset , bufs, e, m_flags)); @@ -365,10 +310,6 @@ namespace libtorrent // we either get an error or 0 or more bytes read TORRENT_ASSERT(e || ret >= 0); - -#ifdef TORRENT_DISK_STATS - write_access_log(adjusted_offset + ret , handle->file_id(), op_end | op_read, clock_type::now()); -#endif TORRENT_ASSERT(ret <= bufs_size(bufs)); if (e) @@ -1315,31 +1256,6 @@ namespace libtorrent return false; } -#ifdef TORRENT_DISK_STATS - bool default_storage::disk_write_access_log() { - return g_access_log != nullptr; - } - - void default_storage::disk_write_access_log(bool enable) { - if (enable) - { - if (g_access_log == nullptr) - { - g_access_log = fopen("file_access.log", "a+"); - } - } - else - { - if (g_access_log != nullptr) - { - FILE* f = g_access_log; - g_access_log = nullptr; - fclose(f); - } - } - } -#endif - storage_interface* default_storage_constructor(storage_params const& params) { return new default_storage(params); diff --git a/tools/Makefile.am b/tools/Makefile.am index 074aaa8c1..23f67cfb7 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -13,7 +13,6 @@ EXTRA_DIST = Jamfile \ parse_dht_rtt.py \ parse_dht_stats.py \ parse_disk_buffer_log.py\ - parse_disk_log.py \ parse_memory_log.py \ parse_peer_log.py \ parse_sample.py \ diff --git a/tools/parse_disk_log.py b/tools/parse_disk_log.py deleted file mode 100755 index b122c5752..000000000 --- a/tools/parse_disk_log.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -# Copyright Arvid Norberg 2008. Use, modification and distribution is -# subject to the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -import os, sys, time - -lines = open(sys.argv[1], 'rb').readlines() - -if len(sys.argv) < 2: - print "usage: parse_disk_log.py logfile [seconds]" - sys.exit(1) - -keys = ['write', 'read', 'read-cache-hit', 'hash', 'move', 'release', 'idle', \ - 'delete', 'check_fastresume', 'check_files', 'clear-cache', \ - 'abort_thread', 'abort_torrent', 'save_resume_data', 'rename_file', \ - 'flushing', 'update_settings', 'sorting_job', \ - 'check_cache_hit'] -throughput_keys = ['write', 'read', 'read-cache-hit'] - -# logfile format: -# -# example: -# 34523 idle -# 34722 write - -if len(sys.argv) > 2: - quantization = long(sys.argv[2]) * 1000000 -else: - quantization = 1000000 - -out = open('disk_io.dat', 'wb') -out2 = open('disk_throughput.dat', 'wb') -state = 'idle' -time = -1 -start_time = -1 -i = 0 -state_timer = {} -throughput = {} -for k in keys: state_timer[k] = 0 -for k in throughput_keys: throughput[k] = 0 - -for l in lines: - l = l.strip().split() - if len(l) < 2: - print l - continue -# try: - new_time = long(l[0]) - if time == -1: - time = new_time - i = new_time - start_time = new_time - while new_time > i + quantization: - i += quantization - state_timer[state] += i - time - time = i - for k in keys: print >>out, (state_timer[k] / float(quantization) * 100.), - print >>out - print >>out2, time - start_time, - for k in throughput_keys: - print >>out2, throughput[k] * 1000 / float(quantization), - print '-- %s %d' % (k, throughput[k]) - print >>out2 - for k in keys: state_timer[k] = 0 - for k in throughput_keys: throughput[k] = 0 - state_timer[state] += new_time - time - time = new_time - state = l[1] - if state in throughput_keys: - throughput[state] += long(l[2]) -# except: -# print l - -i += quantization -state_timer[state] += i - time -time = i -for k in keys: print >>out, (state_timer[k] / float(quantization) * 100.), -print >>out -print >>out2, time - start_time, -for k in throughput_keys: - print >>out2, throughput[k] * 1000 / float(quantization), - print '-- %s %d' % (k, throughput[k]) -print >>out2 -for k in keys: state_timer[k] = 0 -for k in throughput_keys: throughput[k] = 0 -out.close() -out2.close() - -out = open('disk_io.gnuplot', 'wb') -print >>out, "set term png size 1200,700" - -print >>out, 'set output "disk_throughput.png"' -print >>out, 'set title "disk throughput per %f second(s)"' % (quantization / float(1000000)) -print >>out, 'set ylabel "throughput (kB/s)"' -print >>out, 'plot', -i = 0 -for k in throughput_keys: - print >>out, ' "disk_throughput.dat" using 1:%d title "%s" with lines,' % (i + 2, throughput_keys[i]), - i = i + 1 -print >>out, 'x=0' - -print >>out, 'set output "disk_io.png"' -print >>out, 'set ylabel "utilization (%)"' -print >>out, 'set xrange [0:*]' -print >>out, 'set title "disk io utilization per %f second(s)"' % (quantization / float(1000000)) -print >>out, "set key box" -print >>out, "set style data histogram" -print >>out, "set style histogram rowstacked" -print >>out, "set style fill solid" -print >>out, 'plot', -i = 0 -for k in keys: - if k != 'idle': - print >>out, ' "disk_io.dat" using %d title "%s",' % (i + 1, keys[i]), - i = i + 1 -print >>out, 'x=0' -out.close() - -os.system('gnuplot disk_io.gnuplot'); -