From 596d98cac4f4872b941f2bb8116cbbdcba71c73c Mon Sep 17 00:00:00 2001 From: Andrew Resch Date: Tue, 12 Jun 2018 02:43:13 -0700 Subject: [PATCH] Clean-up python code (#3075) lint and enforce python code formatting --- .travis.yml | 16 +- bindings/python/client.py | 46 +- bindings/python/make_torrent.py | 8 +- bindings/python/setup.py | 41 +- bindings/python/simple_client.py | 2 +- bindings/python/test.py | 198 ++-- docs/gen_reference_doc.py | 1837 ++++++++++++++++-------------- docs/gen_settings_doc.py | 153 +-- docs/gen_stats_doc.py | 150 +-- docs/gen_todo.py | 165 +-- docs/join_rst.py | 31 +- examples/run_benchmarks.py | 656 ++++++----- test/http.py | 126 +- test/socks.py | 74 +- test/web_server.py | 335 +++--- tools/clean.py | 141 +-- tools/dht_flood.py | 43 +- tools/gen_fwd.py | 121 +- tools/gprof2dot.py | 366 +++--- tools/parse_bandwidth_log.py | 34 +- tools/parse_buffer_log.py | 122 +- tools/parse_dht_log.py | 348 +++--- tools/parse_dht_rtt.py | 25 +- tools/parse_dht_stats.py | 63 +- tools/parse_disk_buffer_log.py | 99 +- tools/parse_lookup_log.py | 176 +-- tools/parse_memory_log.py | 164 +-- tools/parse_peer_log.py | 96 +- tools/parse_sample.py | 202 ++-- tools/parse_session_stats.py | 990 ++++++++-------- tools/parse_utp_log.py | 357 +++--- tools/run_benchmark.py | 139 ++- tools/set_version.py | 80 +- tools/update_copyright.py | 47 +- 34 files changed, 4016 insertions(+), 3435 deletions(-) diff --git a/.travis.yml b/.travis.yml index 890b4b3ef..4073f92d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ matrix: fast_finish: true include: - if: repo = arvidn/libtorrent - env: variant=release sonar_scan=1 toolset=gcc + env: variant=release sonar_scan=1 toolset=gcc pylint=1 - env: variant=test_debug lint=1 tests=1 toolset=gcc-sanitizer - env: variant=test_debug sim=1 crypto=openssl toolset=gcc-sanitizer - env: variant=test_release coverage=1 tests=1 toolset=gcc-coverage python=1 @@ -46,6 +46,7 @@ addons: - python2.7-dev - g++-5 - [cmake3, ninja-build] + - python3-pip before_install: @@ -79,7 +80,6 @@ before_install: - ulimit -a install: - - touch ~/user-config.jam - 'if [[ $toolset == "gcc" ]]; then g++-5 --version; @@ -130,7 +130,12 @@ install: - 'echo "using python : 2.7 ;" >> ~/user-config.jam' - if [ "$docs" == "1" ]; then rst2html.py --version; fi - 'if [ "$lint" == "1" ]; then curl "https://raw.githubusercontent.com/google/styleguide/71ec7f1e524969c19ce33cfc72e8e023f2b98ee2/cpplint/cpplint.py" >~/cpplint.py; fi' - + - 'if [ "$pylint" == "1" ]; then + sudo pip install flake8; + flake8 --version; + sudo pip3 install flake8; + python3 -m flake8 --version; + fi' - 'if [ $sonar_scan == "1" ]; then wget https://sonarsource.bintray.com/Distribution/sonar-scanner-cli/sonar-scanner-2.6.1.zip; wget https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip; @@ -183,7 +188,10 @@ script: - 'if [ "$lint" == "1" ]; then python ~/cpplint.py --extensions=cpp --headers=hpp --filter=-,+runtime/explicit,+whitespace/end_of_line --linelength=90 test/*.{cpp,hpp} src/*.cpp include/libtorrent/*.hpp include/libtorrent/kademlia/*.hpp src/kademlia/*.cpp include/libtorrent/aux_/*.hpp include/libtorrent/extensions/*.hpp simulation/*.{cpp,hpp} tools/*.{cpp,hpp} examples/*.{cpp,hpp}; fi' - + - 'if [ "$pylint" == "1" ]; then + flake8 --max-line-length=120; + python3 -m flake8 --max-line-length=120; + fi' - 'if [ "$sonar_scan" == "1" ]; then build-wrapper-linux-x86-64 --out-dir bw-output bjam -a -j3 optimization=off crypto=$crypto deprecated-functions=off $toolset variant=$variant -l300 && sonar-scanner -D sonar.login=$SONAR_TOKEN; diff --git a/bindings/python/client.py b/bindings/python/client.py index 4613bb9c6..26f6d36f8 100755 --- a/bindings/python/client.py +++ b/bindings/python/client.py @@ -3,7 +3,7 @@ # Copyright Daniel Wallin 2006. Use, modification and distribution is # subject to the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -from __future__ import print_function + import sys import atexit @@ -153,6 +153,7 @@ def print_download_queue(console, download_queue): write_line(console, out) + def add_torrent(ses, filename, options): atp = lt.add_torrent_params() if filename.startswith('magnet:'): @@ -160,8 +161,8 @@ def add_torrent(ses, filename, options): else: atp.ti = lt.torrent_info(filename) try: - at.resume_data = open(os.path.join(options.save_path, info.name() + '.fastresume'), 'rb').read() - except: + atp.resume_data = open(os.path.join(options.save_path, atp.info.name() + '.fastresume'), 'rb').read() + except BaseException: pass atp.save_path = options.save_path @@ -171,6 +172,7 @@ def add_torrent(ses, filename, options): | lt.torrent_flags.duplicate_is_error ses.async_add_torrent(atp) + def main(): from optparse import OptionParser @@ -225,12 +227,13 @@ def main(): if options.max_download_rate <= 0: options.max_download_rate = -1 - settings = { 'user_agent': 'python_client/' + lt.__version__, + settings = { + 'user_agent': 'python_client/' + lt.__version__, 'listen_interfaces': '%s:%d' % (options.listen_interface, options.port), 'download_rate_limit': int(options.max_download_rate), 'upload_rate_limit': int(options.max_upload_rate), 'alert_mask': lt.alert.category_t.all_categories, - 'outgoing_interfaces' : options.outgoing_interface + 'outgoing_interfaces': options.outgoing_interface, } if options.proxy_host != '': @@ -258,16 +261,16 @@ def main(): out = '' - for h,t in torrents.items(): + for h, t in torrents.items(): out += 'name: %-40s\n' % t.name[:40] if t.state != lt.torrent_status.seeding: - state_str = ['queued', 'checking', 'downloading metadata', \ - 'downloading', 'finished', 'seeding', \ + state_str = ['queued', 'checking', 'downloading metadata', + 'downloading', 'finished', 'seeding', 'allocating', 'checking fastresume'] out += state_str[t.state] + ' ' - out += '%5.4f%% ' % (t.progress*100) + out += '%5.4f%% ' % (t.progress * 100) out += progress_bar(t.progress, 49) out += '\n' @@ -300,7 +303,7 @@ def main(): out += progress_bar(p / float(f.size), 20) out += ' ' + f.path + '\n' write_line(console, out) - except: + except BaseException: pass write_line(console, 76 * '-' + '\n') @@ -312,7 +315,7 @@ def main(): alerts_log.append(a.message()) # add new torrents to our list of torrent_status - if type(a) == lt.add_torrent_alert: + if isinstance(a, lt.add_torrent_alert): h = a.handle h.set_max_connections(60) h.set_max_uploads(-1) @@ -320,7 +323,7 @@ def main(): # update our torrent_status array for torrents that have # changed some of their state - if type(a) == lt.state_update_alert: + if isinstance(a, lt.state_update_alert): for s in a.status: torrents[s.handle] = s @@ -333,19 +336,23 @@ def main(): c = console.sleep_and_input(0.5) ses.post_torrent_updates() - if not c: continue + if not c: + continue if c == 'r': - for h in torrents.keys(): h.force_reannounce() + for h in torrents: + h.force_reannounce() elif c == 'q': alive = False elif c == 'p': - for h in torrents.keys(): h.pause() + for h in torrents: + h.pause() elif c == 'u': - for h in torrents.keys(): h.resume() + for h in torrents: + h.resume() ses.pause() - for h,t in torrents.items(): + for h, t in torrents.items(): if not h.is_valid() or not t.has_metadata: continue h.save_resume_data() @@ -353,7 +360,7 @@ def main(): while len(torrents) > 0: alerts = ses.pop_alerts() for a in alerts: - if type(a) == lt.save_resume_data_alert: + if isinstance(a, lt.save_resume_data_alert): print(a) data = lt.write_resume_data_buf(a.params) h = a.handle @@ -361,11 +368,12 @@ def main(): open(os.path.join(options.save_path, torrents[h].name + '.fastresume'), 'wb').write(data) del torrents[h] - if type(a) == lt.save_resume_data_failed_alert: + if isinstance(a, lt.save_resume_data_failed_alert): h = a.handle if h in torrents: print('failed to save resume data for ', torrents[h].name) del torrents[h] time.sleep(0.5) + main() diff --git a/bindings/python/make_torrent.py b/bindings/python/make_torrent.py index e1cb6e99e..a5c9137c3 100755 --- a/bindings/python/make_torrent.py +++ b/bindings/python/make_torrent.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function + import sys import os @@ -22,8 +22,8 @@ parent_input = os.path.split(input)[0] # if we have a single file, use it because os.walk does not work on a single files if os.path.isfile(input): - size = os.path.getsize(input) - fs.add_file(input, size) + size = os.path.getsize(input) + fs.add_file(input, size) for root, dirs, files in os.walk(input): # skip directories starting with . @@ -39,7 +39,7 @@ for root, dirs, files in os.walk(input): if f == 'Thumbs.db': continue - fname = os.path.join(root[len(parent_input)+1:], f) + fname = os.path.join(root[len(parent_input) + 1:], f) size = os.path.getsize(os.path.join(parent_input, fname)) print('%10d kiB %s' % (size / 1024, fname)) fs.add_file(fname, size) diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 5a0816800..aadf4cd46 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function + from distutils.core import setup, Extension from distutils.sysconfig import get_config_vars @@ -55,13 +55,13 @@ def target_specific(): try: with open('compile_flags') as _file: extra_cmd = _file.read() -except: +except BaseException: extra_cmd = None try: with open('link_flags') as _file: ldflags = _file.read() -except: +except BaseException: ldflags = None # this is to pull out compiler arguments from the CXX flags set up by the @@ -75,7 +75,7 @@ try: while len(cmd) > 0 and not cmd[0].startswith('-'): cmd = cmd[1:] extra_cmd += ' '.join(cmd) -except: +except BaseException: pass ext = None @@ -85,7 +85,7 @@ if '--bjam' in sys.argv: del sys.argv[sys.argv.index('--bjam')] if '--help' not in sys.argv \ - and '--help-commands' not in sys.argv: + and '--help-commands' not in sys.argv: toolset = '' file_ext = '.so' @@ -129,14 +129,22 @@ if '--bjam' in sys.argv: print('build failed') sys.exit(1) - try: os.mkdir('build') - except: pass - try: shutil.rmtree('build/lib') - except: pass - try: os.mkdir('build/lib') - except: pass - try: os.mkdir('libtorrent') - except: pass + try: + os.mkdir('build') + except BaseException: + pass + try: + shutil.rmtree('build/lib') + except BaseException: + pass + try: + os.mkdir('build/lib') + except BaseException: + pass + try: + os.mkdir('libtorrent') + except BaseException: + pass shutil.copyfile('libtorrent' + file_ext, 'build/lib/libtorrent' + file_ext) @@ -145,13 +153,12 @@ if '--bjam' in sys.argv: else: # Remove '-Wstrict-prototypes' compiler option, which isn't valid for C++. cfg_vars = get_config_vars() - for key, value in cfg_vars.items(): + for key, value in list(cfg_vars.items()): if isinstance(value, str): cfg_vars[key] = value.replace('-Wstrict-prototypes', '') - source_list = os.listdir(os.path.join(os.path.dirname(__file__), "src")) - source_list = [os.path.abspath(os.path.join(os.path.dirname(__file__), - "src", s)) for s in source_list if s.endswith(".cpp")] + src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "src")) + source_list = [os.path.join(src_dir, s) for s in os.listdir(src_dir) if s.endswith(".cpp")] if extra_cmd: flags = flags_parser() diff --git a/bindings/python/simple_client.py b/bindings/python/simple_client.py index 9b0854d19..fb6fdea48 100755 --- a/bindings/python/simple_client.py +++ b/bindings/python/simple_client.py @@ -8,7 +8,7 @@ import libtorrent as lt import time import sys -ses = lt.session({'listen_interfaces':'0.0.0.0:6881'}) +ses = lt.session({'listen_interfaces': '0.0.0.0:6881'}) info = lt.torrent_info(sys.argv[1]) h = ses.add_torrent({'ti': info, 'save_path': '.'}) diff --git a/bindings/python/test.py b/bindings/python/test.py index 2ce13d55b..1487f48e1 100644 --- a/bindings/python/test.py +++ b/bindings/python/test.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 -from __future__ import print_function import libtorrent as lt @@ -13,7 +12,6 @@ import shutil import binascii import subprocess as sub import sys -import inspect import pickle import threading @@ -29,6 +27,7 @@ settings = { 'enable_dht': False, 'enable_lsd': False, 'enable_natpmp': False, 'enable_upnp': False, 'listen_interfaces': '0.0.0.0:0', 'file_pool_size': 1} + class test_create_torrent(unittest.TestCase): def test_from_torrent_info(self): @@ -47,10 +46,10 @@ class test_create_torrent(unittest.TestCase): class test_session_stats(unittest.TestCase): def test_unique(self): - l = lt.session_stats_metrics() - self.assertTrue(len(l) > 40) + metrics = lt.session_stats_metrics() + self.assertTrue(len(metrics) > 40) idx = set() - for m in l: + for m in metrics: self.assertTrue(m.value_index not in idx) idx.add(m.value_index) @@ -118,7 +117,6 @@ class test_torrent_handle(unittest.TestCase): self.assertEqual(len(torrents), 1) self.assertEqual(torrents[self.h], 'bar') - def test_replace_trackers(self): self.setup() trackers = [] @@ -141,20 +139,18 @@ class test_torrent_handle(unittest.TestCase): tracker.fail_limit = 1 trackers = [tracker] self.h.replace_trackers(trackers) - tracker_list = [tracker for tracker in self.h.trackers()] # wait a bit until the endpoints list gets populated - while len(tracker_list[0]['endpoints']) == 0: + while len(self.h.trackers()[0]['endpoints']) == 0: time.sleep(0.1) - tracker_list = [tracker for tracker in self.h.trackers()] - pickled_trackers = pickle.dumps(tracker_list) + pickled_trackers = pickle.dumps(self.h.trackers()) unpickled_trackers = pickle.loads(pickled_trackers) self.assertEqual(unpickled_trackers[0]['url'], 'udp://tracker1.com') self.assertEqual(unpickled_trackers[0]['endpoints'][0]['last_error']['value'], 0) def test_file_status(self): self.setup() - l = self.h.file_status() - print(l) + status = self.h.file_status() + print(status) def test_piece_deadlines(self): self.setup() @@ -165,11 +161,10 @@ class test_torrent_handle(unittest.TestCase): # time, wait for next full second to prevent second increment time.sleep(1 - datetime.datetime.now().microsecond / 1000000.0) - sessionStart = datetime.datetime.now().replace(microsecond=0) self.setup() st = self.h.status() for attr in dir(st): - print('%s: %s' % (attr, getattr(st, attr))) + print('%s: %s' % (attr, getattr(st, attr))) # last upload and download times are at session start time self.assertEqual(st.last_upload, None) self.assertEqual(st.last_download, None) @@ -177,7 +172,7 @@ class test_torrent_handle(unittest.TestCase): def test_serialize_trackers(self): """Test to ensure the dict contains only python built-in types""" self.setup() - self.h.add_tracker({'url':'udp://tracker1.com'}) + self.h.add_tracker({'url': 'udp://tracker1.com'}) tr = self.h.trackers()[0] # wait a bit until the endpoints list gets populated while len(tr['endpoints']) == 0: @@ -215,7 +210,7 @@ class test_torrent_handle(unittest.TestCase): ses = lt.session(settings) h = ses.add_torrent(tp) for attr in dir(tp): - print('%s: %s' % (attr, getattr(tp, attr))) + print('%s: %s' % (attr, getattr(tp, attr))) h.connect_peer(('3.3.3.3', 3)) @@ -246,33 +241,34 @@ class test_torrent_handle(unittest.TestCase): def test_torrent_parameter(self): self.ses = lt.session(settings) - self.ti = lt.torrent_info('url_seed_multi.torrent'); + self.ti = lt.torrent_info('url_seed_multi.torrent') self.h = self.ses.add_torrent({ 'ti': self.ti, 'save_path': os.getcwd(), 'trackers': ['http://test.com/announce'], 'dht_nodes': [('1.2.3.4', 6881), ('4.3.2.1', 6881)], - 'file_priorities': [1,1], + 'file_priorities': [1, 1], 'http_seeds': ['http://test.com/file3'], 'url_seeds': ['http://test.com/announce-url'], 'peers': [('5.6.7.8', 6881)], 'banned_peers': [('8.7.6.5', 6881)], - 'renamed_files': { 0: 'test.txt', 2: 'test.txt' } - }) + 'renamed_files': {0: 'test.txt', 2: 'test.txt'} + }) self.st = self.h.status() self.assertEqual(self.st.save_path, os.getcwd()) - trackers = self.h.trackers(); + trackers = self.h.trackers() self.assertEqual(len(trackers), 1) self.assertEqual(trackers[0].get('url'), 'http://test.com/announce') self.assertEqual(trackers[0].get('tier'), 0) - self.assertEqual(self.h.get_file_priorities(), [1,1]) - self.assertEqual(self.h.http_seeds(),['http://test.com/file3']) + self.assertEqual(self.h.get_file_priorities(), [1, 1]) + self.assertEqual(self.h.http_seeds(), ['http://test.com/file3']) # url_seeds was already set, test that it did not got overwritten self.assertEqual(self.h.url_seeds(), - ['http://test.com/announce-url/', 'http://test.com/file/']) - self.assertEqual(self.h.get_piece_priorities(),[4]) - self.assertEqual(self.ti.merkle_tree(),[]) - self.assertEqual(self.st.verified_pieces,[]) + ['http://test.com/announce-url/', 'http://test.com/file/']) + self.assertEqual(self.h.get_piece_priorities(), [4]) + self.assertEqual(self.ti.merkle_tree(), []) + self.assertEqual(self.st.verified_pieces, []) + class test_torrent_info(unittest.TestCase): @@ -328,13 +324,13 @@ class test_torrent_info(unittest.TestCase): os.path.join('temp', 'foo')) idx += 1 - def test_announce_entry(self): ae = lt.announce_entry('test') - self.assertEquals(ae.url, 'test') - self.assertEquals(ae.tier, 0) - self.assertEquals(ae.verified, False) - self.assertEquals(ae.source, 0) + self.assertEqual(ae.url, 'test') + self.assertEqual(ae.tier, 0) + self.assertEqual(ae.verified, False) + self.assertEqual(ae.source, 0) + class test_alerts(unittest.TestCase): @@ -350,7 +346,7 @@ class test_alerts(unittest.TestCase): alerts = ses.pop_alerts() for a in alerts: if a.what() == 'add_torrent_alert': - self.assertEquals(a.torrent_name, 'temp') + self.assertEqual(a.torrent_name, 'temp') print(a.message()) for field_name in dir(a): if field_name.startswith('__'): @@ -429,10 +425,11 @@ class test_bencoder(unittest.TestCase): class test_sha1hash(unittest.TestCase): def test_sha1hash(self): - h = 'a0'*20 + h = 'a0' * 20 s = lt.sha1_hash(binascii.unhexlify(h)) self.assertEqual(h, str(s)) + class test_magnet_link(unittest.TestCase): def test_parse_magnet_uri(self): @@ -453,70 +450,72 @@ class test_magnet_link(unittest.TestCase): h = ses.add_torrent(p) self.assertEqual(str(h.info_hash()), '178882f042c0c33426a6d81e0333ece346e68a68') + class test_peer_class(unittest.TestCase): - def test_peer_class_ids(self): - s = lt.session(settings) + def test_peer_class_ids(self): + s = lt.session(settings) - print('global_peer_class_id:', lt.session.global_peer_class_id) - print('tcp_peer_class_id:', lt.session.tcp_peer_class_id) - print('local_peer_class_id:', lt.session.local_peer_class_id) + print('global_peer_class_id:', lt.session.global_peer_class_id) + print('tcp_peer_class_id:', lt.session.tcp_peer_class_id) + print('local_peer_class_id:', lt.session.local_peer_class_id) - print('global: ', s.get_peer_class(s.global_peer_class_id)) - print('tcp: ', s.get_peer_class(s.local_peer_class_id)) - print('local: ', s.get_peer_class(s.local_peer_class_id)) + print('global: ', s.get_peer_class(s.global_peer_class_id)) + print('tcp: ', s.get_peer_class(s.local_peer_class_id)) + print('local: ', s.get_peer_class(s.local_peer_class_id)) - def test_peer_class(self): - s = lt.session(settings) + def test_peer_class(self): + s = lt.session(settings) - c = s.create_peer_class('test class') - print('new class: ', s.get_peer_class(c)) + c = s.create_peer_class('test class') + print('new class: ', s.get_peer_class(c)) - nfo = s.get_peer_class(c) - self.assertEqual(nfo['download_limit'], 0) - self.assertEqual(nfo['upload_limit'], 0) - self.assertEqual(nfo['ignore_unchoke_slots'], False) - self.assertEqual(nfo['connection_limit_factor'], 100) - self.assertEqual(nfo['download_priority'], 1) - self.assertEqual(nfo['upload_priority'], 1) - self.assertEqual(nfo['label'], 'test class') + nfo = s.get_peer_class(c) + self.assertEqual(nfo['download_limit'], 0) + self.assertEqual(nfo['upload_limit'], 0) + self.assertEqual(nfo['ignore_unchoke_slots'], False) + self.assertEqual(nfo['connection_limit_factor'], 100) + self.assertEqual(nfo['download_priority'], 1) + self.assertEqual(nfo['upload_priority'], 1) + self.assertEqual(nfo['label'], 'test class') - nfo['download_limit'] = 1337 - nfo['upload_limit'] = 1338 - nfo['ignore_unchoke_slots'] = True - nfo['connection_limit_factor'] = 42 - nfo['download_priority'] = 2 - nfo['upload_priority'] = 3 + nfo['download_limit'] = 1337 + nfo['upload_limit'] = 1338 + nfo['ignore_unchoke_slots'] = True + nfo['connection_limit_factor'] = 42 + nfo['download_priority'] = 2 + nfo['upload_priority'] = 3 - s.set_peer_class(c, nfo) + s.set_peer_class(c, nfo) - nfo2 = s.get_peer_class(c) - self.assertEqual(nfo, nfo2) + nfo2 = s.get_peer_class(c) + self.assertEqual(nfo, nfo2) - def test_peer_class_filter(self): - filt = lt.peer_class_type_filter() - filt.add(lt.peer_class_type_filter.tcp_socket, lt.session.global_peer_class_id); - filt.remove(lt.peer_class_type_filter.utp_socket, lt.session.local_peer_class_id); + def test_peer_class_filter(self): + filt = lt.peer_class_type_filter() + filt.add(lt.peer_class_type_filter.tcp_socket, lt.session.global_peer_class_id) + filt.remove(lt.peer_class_type_filter.utp_socket, lt.session.local_peer_class_id) - filt.disallow(lt.peer_class_type_filter.tcp_socket, lt.session.global_peer_class_id); - filt.allow(lt.peer_class_type_filter.utp_socket, lt.session.local_peer_class_id); + filt.disallow(lt.peer_class_type_filter.tcp_socket, lt.session.global_peer_class_id) + filt.allow(lt.peer_class_type_filter.utp_socket, lt.session.local_peer_class_id) + + def test_peer_class_ip_filter(self): + s = lt.session(settings) + s.set_peer_class_type_filter(lt.peer_class_type_filter()) + s.set_peer_class_filter(lt.ip_filter()) - def test_peer_class_ip_filter(self): - s = lt.session(settings) - s.set_peer_class_type_filter(lt.peer_class_type_filter()) - s.set_peer_class_filter(lt.ip_filter()) class test_session(unittest.TestCase): def test_add_torrent(self): s = lt.session(settings) - h = s.add_torrent({'ti': lt.torrent_info('base.torrent'), - 'save_path': '.', - 'dht_nodes': [('1.2.3.4', 6881), ('4.3.2.1', 6881)], - 'http_seeds': ['http://test.com/seed'], - 'peers': [('5.6.7.8', 6881)], - 'banned_peers': [('8.7.6.5', 6881)], - 'file_priorities': [1,1,1,2,0]}) + s.add_torrent({'ti': lt.torrent_info('base.torrent'), + 'save_path': '.', + 'dht_nodes': [('1.2.3.4', 6881), ('4.3.2.1', 6881)], + 'http_seeds': ['http://test.com/seed'], + 'peers': [('5.6.7.8', 6881)], + 'banned_peers': [('8.7.6.5', 6881)], + 'file_priorities': [1, 1, 1, 2, 0]}) def test_apply_settings(self): @@ -565,10 +564,9 @@ class test_session(unittest.TestCase): self.assertTrue(isinstance(a.active_requests, list)) self.assertTrue(isinstance(a.routing_table, list)) - def test_unknown_settings(self): try: - s = lt.session({'unexpected-key-name': 42}) + lt.session({'unexpected-key-name': 42}) self.assertFalse('should have thrown an exception') except KeyError as e: print(e) @@ -606,52 +604,52 @@ class test_example_client(unittest.TestCase): my_stdin = slave_fd process = sub.Popen( - [sys.executable,"client.py","url_seed_multi.torrent"], + [sys.executable, "client.py", "url_seed_multi.torrent"], stdin=my_stdin, stdout=sub.PIPE, stderr=sub.PIPE) # python2 has no Popen.wait() timeout time.sleep(5) returncode = process.poll() - if returncode == None: + if returncode is None: # this is an expected use-case process.kill() err = process.stderr.read().decode("utf-8") self.assertEqual('', err, 'process throw errors: \n' + err) # check error code if process did unexpected end - if returncode != None: + if returncode is not None: # in case of error return: output stdout if nothing was on stderr if returncode != 0: print("stdout:\n" + process.stdout.read().decode("utf-8")) self.assertEqual(returncode, 0, "returncode: " + str(returncode) + "\n" - + "stderr: empty\n" - + "some configuration does not output errors like missing module members," - + "try to call it manually to get the error message\n") + + "stderr: empty\n" + + "some configuration does not output errors like missing module members," + + "try to call it manually to get the error message\n") def test_execute_simple_client(self): process = sub.Popen( - [sys.executable,"simple_client.py","url_seed_multi.torrent"], + [sys.executable, "simple_client.py", "url_seed_multi.torrent"], stdout=sub.PIPE, stderr=sub.PIPE) # python2 has no Popen.wait() timeout time.sleep(5) returncode = process.poll() - if returncode == None: + if returncode is None: # this is an expected use-case process.kill() err = process.stderr.read().decode("utf-8") self.assertEqual('', err, 'process throw errors: \n' + err) # check error code if process did unexpected end - if returncode != None: + if returncode is not None: # in case of error return: output stdout if nothing was on stderr if returncode != 0: print("stdout:\n" + process.stdout.read().decode("utf-8")) self.assertEqual(returncode, 0, "returncode: " + str(returncode) + "\n" - + "stderr: empty\n" - + "some configuration does not output errors like missing module members," - + "try to call it manually to get the error message\n") + + "stderr: empty\n" + + "some configuration does not output errors like missing module members," + + "try to call it manually to get the error message\n") def test_execute_make_torrent(self): process = sub.Popen( - [sys.executable,"make_torrent.py","url_seed_multi.torrent", - "http://test.com/test"], stdout=sub.PIPE, stderr=sub.PIPE) + [sys.executable, "make_torrent.py", "url_seed_multi.torrent", + "http://test.com/test"], stdout=sub.PIPE, stderr=sub.PIPE) returncode = process.wait() # python2 has no Popen.wait() timeout err = process.stderr.read().decode("utf-8") @@ -660,15 +658,16 @@ class test_example_client(unittest.TestCase): if returncode != 0: print("stdout:\n" + process.stdout.read().decode("utf-8")) self.assertEqual(returncode, 0, "returncode: " + str(returncode) + "\n" - + "stderr: empty\n" - + "some configuration does not output errors like missing module members," - + "try to call it manually to get the error message\n") + + "stderr: empty\n" + + "some configuration does not output errors like missing module members," + + "try to call it manually to get the error message\n") def test_default_settings(self): default = lt.default_settings() print(default) + class test_operation_t(unittest.TestCase): def test_enum(self): @@ -678,6 +677,7 @@ class test_operation_t(unittest.TestCase): self.assertEqual(lt.operation_name(lt.operation_t.partfile_write), "partfile_write") self.assertEqual(lt.operation_name(lt.operation_t.hostname_lookup), "hostname_lookup") + if __name__ == '__main__': print(lt.__version__) shutil.copy(os.path.join('..', '..', 'test', 'test_torrents', diff --git a/docs/gen_reference_doc.py b/docs/gen_reference_doc.py index db83ea8f7..84665c0b8 100644 --- a/docs/gen_reference_doc.py +++ b/docs/gen_reference_doc.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import print_function import glob import os @@ -9,18 +10,18 @@ dump = '--dump' in sys.argv internal = '--internal' in sys.argv plain_output = '--plain-output' in sys.argv if plain_output: - plain_file = open('plain_text_out.txt', 'w+') + plain_file = open('plain_text_out.txt', 'w+') in_code = None paths = ['include/libtorrent/*.hpp', 'include/libtorrent/kademlia/*.hpp', 'include/libtorrent/extensions/*.hpp'] if internal: - paths.append('include/libtorrent/aux_/*.hpp') + paths.append('include/libtorrent/aux_/*.hpp') files = [] for p in paths: - files.extend(glob.glob(os.path.join('..', p))) + files.extend(glob.glob(os.path.join('..', p))) functions = [] classes = [] @@ -35,690 +36,784 @@ symbols = {} # some files that need pre-processing to turn symbols into # links into the reference documentation preprocess_rst = \ -{ - 'manual.rst':'manual-ref.rst', - 'upgrade_to_1.2.rst':'upgrade_to_1.2-ref.rst', - 'settings.rst':'settings-ref.rst' -} + { + 'manual.rst': 'manual-ref.rst', + 'upgrade_to_1.2.rst': 'upgrade_to_1.2-ref.rst', + 'settings.rst': 'settings-ref.rst' + } # some pre-defined sections from the main manual symbols = \ -{ - "queuing_": "manual-ref.html#queuing", - "fast-resume_": "manual-ref.html#fast-resume", - "storage-allocation_": "manual-ref.html#storage-allocation", - "alerts_": "manual-ref.html#alerts", - "upnp-and-nat-pmp_": "manual-ref.html#upnp-and-nat-pmp", - "http-seeding_": "manual-ref.html#http-seeding", - "metadata-from-peers_": "manual-ref.html#metadata-from-peers", - "magnet-links_": "manual-ref.html#magnet-links", - "ssl-torrents_": "manual-ref.html#ssl-torrents", - "dynamic-loading-of-torrent-files_": "manual-ref.html#dynamic-loading-of-torrent-files", - "session-statistics_": "manual-ref.html#session-statistics", - "peer-classes_": "manual-ref.html#peer-classes" -} + { + "queuing_": "manual-ref.html#queuing", + "fast-resume_": "manual-ref.html#fast-resume", + "storage-allocation_": "manual-ref.html#storage-allocation", + "alerts_": "manual-ref.html#alerts", + "upnp-and-nat-pmp_": "manual-ref.html#upnp-and-nat-pmp", + "http-seeding_": "manual-ref.html#http-seeding", + "metadata-from-peers_": "manual-ref.html#metadata-from-peers", + "magnet-links_": "manual-ref.html#magnet-links", + "ssl-torrents_": "manual-ref.html#ssl-torrents", + "dynamic-loading-of-torrent-files_": "manual-ref.html#dynamic-loading-of-torrent-files", + "session-statistics_": "manual-ref.html#session-statistics", + "peer-classes_": "manual-ref.html#peer-classes" + } static_links = \ -{ - ".. _`BEP 3`: https://bittorrent.org/beps/bep_0003.html", - ".. _`BEP 17`: https://bittorrent.org/beps/bep_0017.html", - ".. _`BEP 19`: https://bittorrent.org/beps/bep_0019.html" -} + { + ".. _`BEP 3`: https://bittorrent.org/beps/bep_0003.html", + ".. _`BEP 17`: https://bittorrent.org/beps/bep_0017.html", + ".. _`BEP 19`: https://bittorrent.org/beps/bep_0019.html" + } anon_index = 0 category_mapping = { - 'ed25519.hpp': 'ed25519', - 'session.hpp': 'Core', - 'add_torrent_params.hpp': 'Core', - 'session_status.hpp': 'Core', - 'error_code.hpp': 'Error Codes', - 'storage.hpp': 'Custom Storage', - 'storage_defs.hpp': 'Storage', - 'file_storage.hpp': 'Storage', - 'file_pool.hpp': 'Custom Storage', - 'extensions.hpp': 'Plugins', - 'ut_metadata.hpp': 'Plugins', - 'ut_pex.hpp': 'Plugins', - 'ut_trackers.hpp': 'Plugins', - 'smart_ban.hpp': 'Plugins', - 'create_torrent.hpp': 'Create Torrents', - 'alert.hpp': 'Alerts', - 'alert_types.hpp': 'Alerts', - 'bencode.hpp': 'Bencoding', - 'lazy_entry.hpp': 'Bencoding', - 'bdecode.hpp': 'Bdecoding', - 'entry.hpp': 'Bencoding', - 'time.hpp': 'Time', - 'escape_string.hpp': 'Utility', - 'enum_net.hpp': 'Network', - 'broadcast_socket.hpp': 'Network', - 'socket.hpp': 'Network', - 'socket_io.hpp': 'Network', - 'bitfield.hpp': 'Utility', - 'sha1_hash.hpp': 'Utility', - 'hasher.hpp': 'Utility', - 'hasher512.hpp': 'Utility', - 'identify_client.hpp': 'Utility', - 'ip_filter.hpp': 'Filter', - 'session_settings.hpp': 'Settings', - 'settings_pack.hpp': 'Settings', - 'operations.hpp': 'Alerts', - 'disk_buffer_holder.hpp': 'Custom Storage', - 'alert_dispatcher.hpp': 'Alerts', + 'ed25519.hpp': 'ed25519', + 'session.hpp': 'Core', + 'add_torrent_params.hpp': 'Core', + 'session_status.hpp': 'Core', + 'error_code.hpp': 'Error Codes', + 'storage.hpp': 'Custom Storage', + 'storage_defs.hpp': 'Storage', + 'file_storage.hpp': 'Storage', + 'file_pool.hpp': 'Custom Storage', + 'extensions.hpp': 'Plugins', + 'ut_metadata.hpp': 'Plugins', + 'ut_pex.hpp': 'Plugins', + 'ut_trackers.hpp': 'Plugins', + 'smart_ban.hpp': 'Plugins', + 'create_torrent.hpp': 'Create Torrents', + 'alert.hpp': 'Alerts', + 'alert_types.hpp': 'Alerts', + 'bencode.hpp': 'Bencoding', + 'lazy_entry.hpp': 'Bencoding', + 'bdecode.hpp': 'Bdecoding', + 'entry.hpp': 'Bencoding', + 'time.hpp': 'Time', + 'escape_string.hpp': 'Utility', + 'enum_net.hpp': 'Network', + 'broadcast_socket.hpp': 'Network', + 'socket.hpp': 'Network', + 'socket_io.hpp': 'Network', + 'bitfield.hpp': 'Utility', + 'sha1_hash.hpp': 'Utility', + 'hasher.hpp': 'Utility', + 'hasher512.hpp': 'Utility', + 'identify_client.hpp': 'Utility', + 'ip_filter.hpp': 'Filter', + 'session_settings.hpp': 'Settings', + 'settings_pack.hpp': 'Settings', + 'operations.hpp': 'Alerts', + 'disk_buffer_holder.hpp': 'Custom Storage', + 'alert_dispatcher.hpp': 'Alerts', } category_fun_mapping = { - 'min_memory_usage()': 'Settings', - 'high_performance_seed()': 'Settings', - 'cache_status': 'Core', + 'min_memory_usage()': 'Settings', + 'high_performance_seed()': 'Settings', + 'cache_status': 'Core', } + def categorize_symbol(name, filename): - f = os.path.split(filename)[1] + f = os.path.split(filename)[1] - if name.endswith('_category()') \ - or name.endswith('_error_code') \ - or name.endswith('error_code_enum'): - return 'Error Codes' + if name.endswith('_category()') \ + or name.endswith('_error_code') \ + or name.endswith('error_code_enum'): + return 'Error Codes' - if name in category_fun_mapping: - return category_fun_mapping[name] + if name in category_fun_mapping: + return category_fun_mapping[name] - if f in category_mapping: - return category_mapping[f] + if f in category_mapping: + return category_mapping[f] + + return 'Core' - return 'Core' def suppress_warning(filename, name): - f = os.path.split(filename)[1] - if f != 'alert_types.hpp': return False + f = os.path.split(filename)[1] + if f != 'alert_types.hpp': + return False -# if name.endswith('_alert') or name == 'message()': - return True + # if name.endswith('_alert') or name == 'message()': + return True + + # return False -# return False def first_item(itr): - for i in itr: - return i - return None + for i in itr: + return i + return None + def is_visible(desc): - if desc.strip().startswith('hidden'): return False - if internal: return True - if desc.strip().startswith('internal'): return False - return True + if desc.strip().startswith('hidden'): + return False + if internal: + return True + if desc.strip().startswith('internal'): + return False + return True + def highlight_signature(s): - name = s.split('(', 1) - name2 = name[0].split(' ') - if len(name2[-1]) == 0: return s + name = s.split('(', 1) + name2 = name[0].split(' ') + if len(name2[-1]) == 0: + return s - # make the name of the function bold - name2[-1] = '**' + name2[-1] + '** ' + # make the name of the function bold + name2[-1] = '**' + name2[-1] + '** ' - # if there is a return value, make sure we preserve pointer types - if len(name2) > 1: - name2[0] = name2[0].replace('*', '\\*') - name[0] = ' '.join(name2) + # if there is a return value, make sure we preserve pointer types + if len(name2) > 1: + name2[0] = name2[0].replace('*', '\\*') + name[0] = ' '.join(name2) - # we have to escape asterisks, since this is rendered into - # a parsed literal in rst - name[1] = name[1].replace('*', '\\*') + # we have to escape asterisks, since this is rendered into + # a parsed literal in rst + name[1] = name[1].replace('*', '\\*') - # we also have to escape colons - name[1] = name[1].replace(':', '\\:') + # we also have to escape colons + name[1] = name[1].replace(':', '\\:') - # escape trailing underscores - name[1] = name[1].replace('_', '\\_') + # escape trailing underscores + name[1] = name[1].replace('_', '\\_') + + # comments in signatures are italic + name[1] = name[1].replace('/\\*', '*/\\*') + name[1] = name[1].replace('\\*/', '\\*/*') + return '('.join(name) - # comments in signatures are italic - name[1] = name[1].replace('/\\*', '*/\\*') - name[1] = name[1].replace('\\*/', '\\*/*') - return '('.join(name) def html_sanitize(s): - ret = '' - for i in s: - if i == '<': ret += '<' - elif i == '>': ret += '>' - elif i == '&': ret += '&' - else: ret += i - return ret + ret = '' + for i in s: + if i == '<': + ret += '<' + elif i == '>': + ret += '>' + elif i == '&': + ret += '&' + else: + ret += i + return ret + def looks_like_namespace(line): - line = line.strip() - if line.startswith('namespace'): return True - return False + line = line.strip() + if line.startswith('namespace'): + return True + return False + def looks_like_blank(line): - line = line.split('//')[0] - line = line.replace('{', '') - line = line.replace('}', '') - line = line.replace('[', '') - line = line.replace(']', '') - line = line.replace(';', '') - line = line.strip() - return len(line) == 0 + line = line.split('//')[0] + line = line.replace('{', '') + line = line.replace('}', '') + line = line.replace('[', '') + line = line.replace(']', '') + line = line.replace(';', '') + line = line.strip() + return len(line) == 0 + def looks_like_variable(line): - line = line.split('//')[0] - line = line.strip() - if not ' ' in line and not '\t' in line: return False - if line.startswith('friend '): return False - if line.startswith('enum '): return False - if line.startswith(','): return False - if line.startswith(':'): return False - if line.startswith('typedef'): return False - if line.startswith('using'): return False - if ' = ' in line: return True - if line.endswith(';'): return True - return False + line = line.split('//')[0] + line = line.strip() + if ' ' not in line and '\t' not in line: + return False + if line.startswith('friend '): + return False + if line.startswith('enum '): + return False + if line.startswith(','): + return False + if line.startswith(':'): + return False + if line.startswith('typedef'): + return False + if line.startswith('using'): + return False + if ' = ' in line: + return True + if line.endswith(';'): + return True + return False + def looks_like_forward_decl(line): - line = line.split('//')[0] - line = line.strip() - if not line.endswith(';'): return False - if '{' in line: return False - if '}' in line: return False - if line.startswith('friend '): return True - if line.startswith('struct '): return True - if line.startswith('class '): return True - return False + line = line.split('//')[0] + line = line.strip() + if not line.endswith(';'): + return False + if '{' in line: + return False + if '}' in line: + return False + if line.startswith('friend '): + return True + if line.startswith('struct '): + return True + if line.startswith('class '): + return True + return False + def looks_like_function(line): - if line.startswith('friend'): return False - if '::' in line.split('(')[0].split(' ')[-1]: return False - if line.startswith(','): return False - if line.startswith(':'): return False - return '(' in line; + if line.startswith('friend'): + return False + if '::' in line.split('(')[0].split(' ')[-1]: + return False + if line.startswith(','): + return False + if line.startswith(':'): + return False + return '(' in line + def parse_function(lno, lines, filename): - current_fun = {} - start_paren = 0 - end_paren = 0 - signature = '' + start_paren = 0 + end_paren = 0 + signature = '' - while lno < len(lines): - l = lines[lno].strip() - lno += 1 - if l.startswith('//'): continue + while lno < len(lines): + line = lines[lno].strip() + lno += 1 + if line.startswith('//'): + continue - start_paren += l.count('(') - end_paren += l.count(')') + start_paren += line.count('(') + end_paren += line.count(')') - sig_line = l.replace('TORRENT_EXPORT ', '').replace('TORRENT_EXTRA_EXPORT','').strip() - if signature != '': sig_line = '\n ' + sig_line - signature += sig_line - if verbose: print 'fun %s' % l + sig_line = line.replace('TORRENT_EXPORT ', '').replace('TORRENT_EXTRA_EXPORT', '').strip() + if signature != '': + sig_line = '\n ' + sig_line + signature += sig_line + if verbose: + print('fun %s' % line) - if start_paren > 0 and start_paren == end_paren: - if signature[-1] != ';': - # we also need to consume the function body - start_paren = 0 - end_paren = 0 - for i in range(len(signature)): - if signature[i] == '(': start_paren += 1 - elif signature[i] == ')': end_paren += 1 + if start_paren > 0 and start_paren == end_paren: + if signature[-1] != ';': + # we also need to consume the function body + start_paren = 0 + end_paren = 0 + for i in range(len(signature)): + if signature[i] == '(': + start_paren += 1 + elif signature[i] == ')': + end_paren += 1 - if start_paren > 0 and start_paren == end_paren: - for k in range(i, len(signature)): - if signature[k] == ':' or signature[k] == '{': - signature = signature[0:k].strip() - break - break + if start_paren > 0 and start_paren == end_paren: + for k in range(i, len(signature)): + if signature[k] == ':' or signature[k] == '{': + signature = signature[0:k].strip() + break + break + + lno = consume_block(lno - 1, lines) + signature += ';' + ret = [{'file': filename[11:], 'signatures': set([signature]), 'names': set( + [signature.split('(')[0].split(' ')[-1].strip() + '()'])}, lno] + if first_item(ret[0]['names']) == '()': + return [None, lno] + return ret + if len(signature) > 0: + print('\x1b[31mFAILED TO PARSE FUNCTION\x1b[0m %s\nline: %d\nfile: %s' % (signature, lno, filename)) + return [None, lno] - lno = consume_block(lno - 1, lines) - signature += ';' - ret = [{ 'file': filename[11:], 'signatures': set([ signature ]), 'names': set([ signature.split('(')[0].split(' ')[-1].strip() + '()'])}, lno] - if first_item(ret[0]['names']) == '()': return [None, lno] - return ret - if len(signature) > 0: - print '\x1b[31mFAILED TO PARSE FUNCTION\x1b[0m %s\nline: %d\nfile: %s' % (signature, lno, filename) - return [None, lno] def parse_class(lno, lines, filename): - start_brace = 0 - end_brace = 0 + start_brace = 0 + end_brace = 0 - name = '' - funs = [] - fields = [] - enums = [] - state = 'public' - context = '' - class_type = 'struct' - blanks = 0 - decl = '' + name = '' + funs = [] + fields = [] + enums = [] + state = 'public' + context = '' + class_type = 'struct' + blanks = 0 + decl = '' - while lno < len(lines): - l = lines[lno].strip() - decl += lines[lno].replace('TORRENT_EXPORT ', '').replace('TORRENT_EXTRA_EXPORT', '').split('{')[0].strip() - if '{' in l: break - if verbose: print 'class %s' % l - lno += 1 + while lno < len(lines): + line = lines[lno].strip() + decl += lines[lno].replace('TORRENT_EXPORT ', '').replace('TORRENT_EXTRA_EXPORT', '').split('{')[0].strip() + if '{' in line: + break + if verbose: + print('class %s' % line) + lno += 1 - if decl.startswith('class'): - state = 'private' - class_type = 'class' + if decl.startswith('class'): + state = 'private' + class_type = 'class' - name = decl.split(':')[0].replace('class ', '').replace('struct ', '').replace('final', '').strip() + name = decl.split(':')[0].replace('class ', '').replace('struct ', '').replace('final', '').strip() + while lno < len(lines): + line = lines[lno].strip() + lno += 1 - while lno < len(lines): - l = lines[lno].strip() - lno += 1 + if line == '': + blanks += 1 + context = '' + continue - if l == '': - blanks += 1 - context = '' - continue + if line.startswith('/*'): + lno = consume_comment(lno - 1, lines) + continue - if l.startswith('/*'): - lno = consume_comment(lno - 1, lines) - continue + if line.startswith('#'): + lno = consume_ifdef(lno - 1, lines, True) + continue - if l.startswith('#'): - lno = consume_ifdef(lno - 1, lines, True) - continue + if 'TORRENT_DEFINE_ALERT' in line: + if verbose: + print('xx %s' % line) + blanks += 1 + continue + if 'TORRENT_DEPRECATED' in line: + if verbose: + print('xx %s' % line) + blanks += 1 + continue - if 'TORRENT_DEFINE_ALERT' in l: - if verbose: print 'xx %s' % l - blanks += 1 - continue - if 'TORRENT_DEPRECATED' in l: - if verbose: print 'xx %s' % l - blanks += 1 - continue + if line.startswith('//'): + if verbose: + print('desc %s' % line) - if l.startswith('//'): - if verbose: print 'desc %s' % l + # plain output prints just descriptions and filters out c++ code. + # it's used to run spell checker over + if plain_output: + s = line.split('//')[1] + # if the first character is a space, strip it + if len(s) > 0 and s[0] == ' ': + s = s[1:] + global in_code + if in_code is not None and not s.startswith(in_code) and len(s) > 1: + in_code = None - # plain output prints just descriptions and filters out c++ code. - # it's used to run spell checker over - if plain_output: - line = l.split('//')[1] - # if the first character is a space, strip it - if len(line) > 0 and line[0] == ' ': line = line[1:] - global in_code - if in_code != None and not line.startswith(in_code) and len(line) > 1: - in_code = None + if s.strip().startswith('.. code::'): + in_code = s.split('.. code::')[0] + '\t' - if line.strip().startswith('.. code::'): - in_code = line.split('.. code::')[0] + '\t' + # strip out C++ code from the plain text output since it's meant for + # running spell checking over + if not s.strip().startswith('.. ') and in_code is None: + plain_file.write(s + '\n') + line = line[2:] + if len(line) and line[0] == ' ': + line = line[1:] + context += line + '\n' + continue - # strip out C++ code from the plain text output since it's meant for - # running spell checking over - if not line.strip().startswith('.. ') and in_code == None: - plain_file.write(line + '\n') - l = l[2:] - if len(l) and l[0] == ' ': l = l[1:] - context += l + '\n' - continue + start_brace += line.count('{') + end_brace += line.count('}') - start_brace += l.count('{') - end_brace += l.count('}') + if line == 'private:': + state = 'private' + elif line == 'protected:': + state = 'protected' + elif line == 'public:': + state = 'public' - if l == 'private:': state = 'private' - elif l == 'protected:': state = 'protected' - elif l == 'public:': state = 'public' + if start_brace > 0 and start_brace == end_brace: + return [{'file': filename[11:], 'enums': enums, 'fields':fields, + 'type': class_type, 'name': name, 'decl': decl, 'fun': funs}, lno] - if start_brace > 0 and start_brace == end_brace: - return [{ 'file': filename[11:], 'enums': enums, 'fields':fields, 'type': class_type, 'name': name, 'decl': decl, 'fun': funs}, lno] + if state != 'public' and not internal: + if verbose: + print('private %s' % line) + blanks += 1 + continue - if state != 'public' and not internal: - if verbose: print 'private %s' % l - blanks += 1 - continue + if start_brace - end_brace > 1: + if verbose: + print('scope %s' % line) + blanks += 1 + continue - if start_brace - end_brace > 1: - if verbose: print 'scope %s' % l - blanks += 1 - continue; + if looks_like_function(line): + current_fun, lno = parse_function(lno - 1, lines, filename) + if current_fun is not None and is_visible(context): + if context == '' and blanks == 0 and len(funs): + funs[-1]['signatures'].update(current_fun['signatures']) + funs[-1]['names'].update(current_fun['names']) + else: + current_fun['desc'] = context + if context == '' and not suppress_warning(filename, first_item(current_fun['names'])): + print('WARNING: member function "%s" is not documented: \x1b[34m%s:%d\x1b[0m' + % (name + '::' + first_item(current_fun['names']), filename, lno)) + funs.append(current_fun) + context = '' + blanks = 0 + continue - if looks_like_function(l): - current_fun, lno = parse_function(lno - 1, lines, filename) - if current_fun != None and is_visible(context): - if context == '' and blanks == 0 and len(funs): - funs[-1]['signatures'].update(current_fun['signatures']) - funs[-1]['names'].update(current_fun['names']) - else: - current_fun['desc'] = context - if context == '' and not suppress_warning(filename, first_item(current_fun['names'])): - print 'WARNING: member function "%s" is not documented: \x1b[34m%s:%d\x1b[0m' \ - % (name + '::' + first_item(current_fun['names']), filename, lno) - funs.append(current_fun) - context = '' - blanks = 0 - continue + if looks_like_variable(line): + if verbose: + print('var %s' % line) + if not is_visible(context): + continue + line = line.split('//')[0].strip() + # the name may look like this: + # std::uint8_t fails : 7; + # int scrape_downloaded = -1; + # static constexpr peer_flags_t interesting{0x1}; + n = line.split('=')[0].split('{')[0].strip().split(' : ')[0].split(' ')[-1].split(':')[0].split(';')[0] + if context == '' and blanks == 0 and len(fields): + fields[-1]['names'].append(n) + fields[-1]['signatures'].append(line) + else: + if context == '' and not suppress_warning(filename, n): + print('WARNING: field "%s" is not documented: \x1b[34m%s:%d\x1b[0m' + % (name + '::' + n, filename, lno)) + fields.append({'signatures': [line], 'names': [n], 'desc': context}) + context = '' + blanks = 0 + continue - if looks_like_variable(l): - if verbose: print 'var %s' % l - if not is_visible(context): - continue - l = l.split('//')[0].strip() - # the name may look like this: - # std::uint8_t fails : 7; - # int scrape_downloaded = -1; - # static constexpr peer_flags_t interesting{0x1}; - n = l.split('=')[0].split('{')[0].strip().split(' : ')[0].split(' ')[-1].split(':')[0].split(';')[0] - if context == '' and blanks == 0 and len(fields): - fields[-1]['names'].append(n) - fields[-1]['signatures'].append(l) - else: - if context == '' and not suppress_warning(filename, n): - print 'WARNING: field "%s" is not documented: \x1b[34m%s:%d\x1b[0m' \ - % (name + '::' + n, filename, lno) - fields.append({'signatures': [l], 'names': [n], 'desc': context}) - context = '' - blanks = 0 - continue + if line.startswith('enum '): + if verbose: + print('enum %s' % line) + if not is_visible(context): + consume_block(lno - 1, lines) + else: + enum, lno = parse_enum(lno - 1, lines, filename) + if enum is not None: + enum['desc'] = context + if context == '' and not suppress_warning(filename, enum['name']): + print('WARNING: enum "%s" is not documented: \x1b[34m%s:%d\x1b[0m' + % (name + '::' + enum['name'], filename, lno)) + enums.append(enum) + context = '' + continue - if l.startswith('enum '): - if verbose: print 'enum %s' % l - if not is_visible(context): - consume_block(lno - 1, lines) - else: - enum, lno = parse_enum(lno - 1, lines, filename) - if enum != None: - enum['desc'] = context - if context == '' and not suppress_warning(filename, enum['name']): - print 'WARNING: enum "%s" is not documented: \x1b[34m%s:%d\x1b[0m' \ - % (name + '::' + enum['name'], filename, lno) - enums.append(enum) - context = '' - continue + context = '' - context = '' + if verbose: + if looks_like_forward_decl(line) \ + or looks_like_blank(line) \ + or looks_like_namespace(line): + print('-- %s' % line) + else: + print('?? %s' % line) - if verbose: - if looks_like_forward_decl(l) \ - or looks_like_blank(l) \ - or looks_like_namespace(l): - print '-- %s' % l - else: - print '?? %s' % l + if len(name) > 0: + print('\x1b[31mFAILED TO PARSE CLASS\x1b[0m %s\nfile: %s:%d' % (name, filename, lno)) + return [None, lno] - if len(name) > 0: - print '\x1b[31mFAILED TO PARSE CLASS\x1b[0m %s\nfile: %s:%d' % (name, filename, lno) - return [None, lno] def parse_enum(lno, lines, filename): - start_brace = 0 - end_brace = 0 - global anon_index + start_brace = 0 + end_brace = 0 + global anon_index - l = lines[lno].strip() - name = l.replace('enum ', '').split('{')[0].strip() - if len(name) == 0: - if not internal: - print 'WARNING: anonymous enum at: \x1b[34m%s:%d\x1b[0m' % (filename, lno) - lno = consume_block(lno - 1, lines) - return [None, lno] - name = 'anonymous_enum_%d' % anon_index - anon_index += 1 + line = lines[lno].strip() + name = line.replace('enum ', '').split('{')[0].strip() + if len(name) == 0: + if not internal: + print('WARNING: anonymous enum at: \x1b[34m%s:%d\x1b[0m' % (filename, lno)) + lno = consume_block(lno - 1, lines) + return [None, lno] + name = 'anonymous_enum_%d' % anon_index + anon_index += 1 - values = [] - context = '' - if not '{' in l: - if verbose: print 'enum %s' % lines[lno] - lno += 1 + values = [] + context = '' + if '{' not in line: + if verbose: + print('enum %s' % lines[lno]) + lno += 1 - val = 0 - while lno < len(lines): - l = lines[lno].strip() - lno += 1 + val = 0 + while lno < len(lines): + line = lines[lno].strip() + lno += 1 - if l.startswith('//'): - if verbose: print 'desc %s' % l - l = l[2:] - if len(l) and l[0] == ' ': l = l[1:] - context += l + '\n' - continue + if line.startswith('//'): + if verbose: + print('desc %s' % line) + line = line[2:] + if len(line) and line[0] == ' ': + line = line[1:] + context += line + '\n' + continue - if l.startswith('#'): - lno = consume_ifdef(lno - 1, lines) - continue + if line.startswith('#'): + lno = consume_ifdef(lno - 1, lines) + continue - start_brace += l.count('{') - end_brace += l.count('}') + start_brace += line.count('{') + end_brace += line.count('}') - if '{' in l: - l = l.split('{')[1] - l = l.split('}')[0] + if '{' in line: + line = line.split('{')[1] + line = line.split('}')[0] - if len(l): - if verbose: print 'enumv %s' % lines[lno-1] - for v in l.split(','): - v = v.strip(); - if v.startswith('//'): break - if v == '': continue - valstr = '' - try: - if '=' in v: val = int(v.split('=')[1].strip(), 0) - valstr = str(val) - except: pass + if len(line): + if verbose: + print('enumv %s' % lines[lno - 1]) + for v in line.split(','): + v = v.strip() + if v.startswith('//'): + break + if v == '': + continue + valstr = '' + try: + if '=' in v: + val = int(v.split('=')[1].strip(), 0) + valstr = str(val) + except BaseException: + pass - if '=' in v: v = v.split('=')[0].strip() - if is_visible(context): - values.append({'name': v.strip(), 'desc': context, 'val': valstr}) - if verbose: print 'enumv %s' % valstr - context = '' - val += 1 - else: - if verbose: print '?? %s' % lines[lno-1] + if '=' in v: + v = v.split('=')[0].strip() + if is_visible(context): + values.append({'name': v.strip(), 'desc': context, 'val': valstr}) + if verbose: + print('enumv %s' % valstr) + context = '' + val += 1 + else: + if verbose: + print('?? %s' % lines[lno - 1]) - if start_brace > 0 and start_brace == end_brace: - return [{'file': filename[11:], 'name': name, 'values': values}, lno] + if start_brace > 0 and start_brace == end_brace: + return [{'file': filename[11:], 'name': name, 'values': values}, lno] + + if len(name) > 0: + print('\x1b[31mFAILED TO PARSE ENUM\x1b[0m %s\nline: %d\nfile: %s' % (name, lno, filename)) + return [None, lno] - if len(name) > 0: - print '\x1b[31mFAILED TO PARSE ENUM\x1b[0m %s\nline: %d\nfile: %s' % (name, lno, filename) - return [None, lno] def consume_block(lno, lines): - start_brace = 0 - end_brace = 0 + start_brace = 0 + end_brace = 0 - while lno < len(lines): - l = lines[lno].strip() - if verbose: print 'xx %s' % l - lno += 1 + while lno < len(lines): + line = lines[lno].strip() + if verbose: + print('xx %s' % line) + lno += 1 - start_brace += l.count('{') - end_brace += l.count('}') + start_brace += line.count('{') + end_brace += line.count('}') + + if start_brace > 0 and start_brace == end_brace: + break + return lno - if start_brace > 0 and start_brace == end_brace: - break - return lno def consume_comment(lno, lines): - while lno < len(lines): - l = lines[lno].strip() - if verbose: print 'xx %s' % l - lno += 1 - if '*/' in l: break + while lno < len(lines): + line = lines[lno].strip() + if verbose: + print('xx %s' % line) + lno += 1 + if '*/' in line: + break - return lno + return lno -def trim_define(l): - return l.replace('#ifndef', '').replace('#ifdef', '') \ - .replace('#if', '').replace('defined', '') \ - .replace('TORRENT_USE_IPV6', '').replace('TORRENT_ABI_VERSION == 1', '') \ - .replace('||', '').replace('&&', '').replace('(', '').replace(')','') \ - .replace('!', '').replace('\\', '').strip() -def consume_ifdef(lno, lines, warn_on_ifdefs = False): - l = lines[lno].strip() - lno += 1 +def trim_define(line): + return line.replace('#ifndef', '').replace('#ifdef', '') \ + .replace('#if', '').replace('defined', '') \ + .replace('TORRENT_USE_IPV6', '').replace('TORRENT_ABI_VERSION == 1', '') \ + .replace('||', '').replace('&&', '').replace('(', '').replace(')', '') \ + .replace('!', '').replace('\\', '').strip() - start_if = 1 - end_if = 0 - if verbose: print 'prep %s' % l +def consume_ifdef(lno, lines, warn_on_ifdefs=False): + line = lines[lno].strip() + lno += 1 - if warn_on_ifdefs and l.strip().startswith('#if'): - while l.endswith('\\'): - lno += 1 - l += lines[lno].strip() - if verbose: print 'prep %s' % lines[lno].trim() - define = trim_define(l) - if 'TORRENT_' in define and not 'TORRENT_ABI_VERSION' in define: - print '\x1b[31mWARNING: possible ABI breakage in public struct! "%s" \x1b[34m %s:%d\x1b[0m' % \ - (define, filename, lno) - # we've already warned once, no need to do it twice - warn_on_ifdefs = False - elif define != '': - print '\x1b[33msensitive define in public struct: "%s"\x1b[34m %s:%d\x1b[0m' % (define, filename, lno) + start_if = 1 + end_if = 0 - if (l.startswith('#if') and ( - ' TORRENT_USE_ASSERTS' in l or - ' TORRENT_USE_INVARIANT_CHECKS' in l or - ' TORRENT_ASIO_DEBUGGING' in l) or - l == '#if TORRENT_ABI_VERSION == 1' - ): - while lno < len(lines): - l = lines[lno].strip() - lno += 1 - if verbose: print 'prep %s' % l - if l.startswith('#endif'): end_if += 1 - if l.startswith('#if'): start_if += 1 - if l == '#else' and start_if - end_if == 1: break - if start_if - end_if == 0: break - return lno - else: - while l.endswith('\\') and lno < len(lines): - l = lines[lno].strip() - lno += 1 - if verbose: print 'prep %s' % l + if verbose: + print('prep %s' % line) + + if warn_on_ifdefs and line.strip().startswith('#if'): + while line.endswith('\\'): + lno += 1 + line += lines[lno].strip() + if verbose: + print('prep %s' % lines[lno].trim()) + define = trim_define(line) + if 'TORRENT_' in define and 'TORRENT_ABI_VERSION' not in define: + print('\x1b[31mWARNING: possible ABI breakage in public struct! "%s" \x1b[34m %s:%d\x1b[0m' % + (define, filename, lno)) + # we've already warned once, no need to do it twice + warn_on_ifdefs = False + elif define != '': + print('\x1b[33msensitive define in public struct: "%s"\x1b[34m %s:%d\x1b[0m' % (define, filename, lno)) + + if (line.startswith('#if') and ( + ' TORRENT_USE_ASSERTS' in line or + ' TORRENT_USE_INVARIANT_CHECKS' in line or + ' TORRENT_ASIO_DEBUGGING' in line) or + line == '#if TORRENT_ABI_VERSION == 1'): + while lno < len(lines): + line = lines[lno].strip() + lno += 1 + if verbose: + print('prep %s' % line) + if line.startswith('#endif'): + end_if += 1 + if line.startswith('#if'): + start_if += 1 + if line == '#else' and start_if - end_if == 1: + break + if start_if - end_if == 0: + break + return lno + else: + while line.endswith('\\') and lno < len(lines): + line = lines[lno].strip() + lno += 1 + if verbose: + print('prep %s' % line) + + return lno - return lno for filename in files: - h = open(filename) - lines = h.read().split('\n') + h = open(filename) + lines = h.read().split('\n') - if verbose: print '\n=== %s ===\n' % filename + if verbose: + print('\n=== %s ===\n' % filename) - blanks = 0 - lno = 0 - while lno < len(lines): - l = lines[lno].strip() - lno += 1 + blanks = 0 + lno = 0 + while lno < len(lines): + line = lines[lno].strip() + lno += 1 - if l == '': - blanks += 1 - context = '' - continue + if line == '': + blanks += 1 + context = '' + continue - if l.startswith('//') and l[2:].strip() == 'OVERVIEW': - # this is a section overview - current_overview = '' - while lno < len(lines): - l = lines[lno].strip() - lno += 1 - if not l.startswith('//'): - # end of overview - overviews[filename[11:]] = current_overview - current_overview = '' - break - l = l[2:] - if l.startswith(' '): l = l[1:] - current_overview += l + '\n' + if line.startswith('//') and line[2:].strip() == 'OVERVIEW': + # this is a section overview + current_overview = '' + while lno < len(lines): + line = lines[lno].strip() + lno += 1 + if not line.startswith('//'): + # end of overview + overviews[filename[11:]] = current_overview + current_overview = '' + break + line = line[2:] + if line.startswith(' '): + line = line[1:] + current_overview += line + '\n' - if l.startswith('//'): - if verbose: print 'desc %s' % l - l = l[2:] - if len(l) and l[0] == ' ': l = l[1:] - context += l + '\n' - continue + if line.startswith('//'): + if verbose: + print('desc %s' % line) + line = line[2:] + if len(line) and line[0] == ' ': + line = line[1:] + context += line + '\n' + continue - if l.startswith('/*'): - lno = consume_comment(lno - 1, lines) - continue + if line.startswith('/*'): + lno = consume_comment(lno - 1, lines) + continue - if l.startswith('#'): - lno = consume_ifdef(lno - 1, lines) - continue + if line.startswith('#'): + lno = consume_ifdef(lno - 1, lines) + continue - if (l == 'namespace detail' or \ - l == 'namespace impl' or \ - l == 'namespace aux') \ - and not internal: - lno = consume_block(lno, lines) - continue + if (line == 'namespace detail' or + line == 'namespace impl' or + line == 'namespace aux') \ + and not internal: + lno = consume_block(lno, lines) + continue - if 'TORRENT_DEPRECATED' in l: - if ('class ' in l or 'struct ' in l) and not ';' in l: - lno = consume_block(lno - 1, lines) - context = '' - blanks += 1 - if verbose: print 'xx %s' % l - continue + if 'TORRENT_DEPRECATED' in line: + if ('class ' in line or 'struct ' in line) and ';' not in line: + lno = consume_block(lno - 1, lines) + context = '' + blanks += 1 + if verbose: + print('xx %s' % line) + continue - if 'TORRENT_EXPORT ' in l or l.startswith('inline ') or l.startswith('template') or internal: - if l.startswith('class ') or l.startswith('struct '): - if not l.endswith(';'): - current_class, lno = parse_class(lno -1, lines, filename) - if current_class != None and is_visible(context): - current_class['desc'] = context - if context == '': - print 'WARNING: class "%s" is not documented: \x1b[34m%s:%d\x1b[0m' \ - % (current_class['name'], filename, lno) - classes.append(current_class) - context = '' - blanks += 1 - continue + if 'TORRENT_EXPORT ' in line or line.startswith('inline ') or line.startswith('template') or internal: + if line.startswith('class ') or line.startswith('struct '): + if not line.endswith(';'): + current_class, lno = parse_class(lno - 1, lines, filename) + if current_class is not None and is_visible(context): + current_class['desc'] = context + if context == '': + print('WARNING: class "%s" is not documented: \x1b[34m%s:%d\x1b[0m' + % (current_class['name'], filename, lno)) + classes.append(current_class) + context = '' + blanks += 1 + continue - if looks_like_function(l): - current_fun, lno = parse_function(lno - 1, lines, filename) - if current_fun != None and is_visible(context): - if context == '' and blanks == 0 and len(functions): - functions[-1]['signatures'].update(current_fun['signatures']) - functions[-1]['names'].update(current_fun['names']) - else: - current_fun['desc'] = context - if context == '': - print 'WARNING: function "%s" is not documented: \x1b[34m%s:%d\x1b[0m' \ - % (first_item(current_fun['names']), filename, lno) - functions.append(current_fun) - context = '' - blanks = 0 - continue + if looks_like_function(line): + current_fun, lno = parse_function(lno - 1, lines, filename) + if current_fun is not None and is_visible(context): + if context == '' and blanks == 0 and len(functions): + functions[-1]['signatures'].update(current_fun['signatures']) + functions[-1]['names'].update(current_fun['names']) + else: + current_fun['desc'] = context + if context == '': + print('WARNING: function "%s" is not documented: \x1b[34m%s:%d\x1b[0m' + % (first_item(current_fun['names']), filename, lno)) + functions.append(current_fun) + context = '' + blanks = 0 + continue - if ('class ' in l or 'struct ' in l) and not ';' in l: - lno = consume_block(lno - 1, lines) - context = '' - blanks += 1 - continue + if ('class ' in line or 'struct ' in line) and ';' not in line: + lno = consume_block(lno - 1, lines) + context = '' + blanks += 1 + continue - if l.startswith('enum '): - if not is_visible(context): - consume_block(lno - 1, lines) - else: - current_enum, lno = parse_enum(lno - 1, lines, filename) - if current_enum != None and is_visible(context): - current_enum['desc'] = context - if context == '': - print 'WARNING: enum "%s" is not documented: \x1b[34m%s:%d\x1b[0m' \ - % (current_enum['name'], filename, lno) - enums.append(current_enum) - context = '' - blanks += 1 - continue + if line.startswith('enum '): + if not is_visible(context): + consume_block(lno - 1, lines) + else: + current_enum, lno = parse_enum(lno - 1, lines, filename) + if current_enum is not None and is_visible(context): + current_enum['desc'] = context + if context == '': + print('WARNING: enum "%s" is not documented: \x1b[34m%s:%d\x1b[0m' + % (current_enum['name'], filename, lno)) + enums.append(current_enum) + context = '' + blanks += 1 + continue - blanks += 1 - if verbose: - if looks_like_forward_decl(l) \ - or looks_like_blank(l) \ - or looks_like_namespace(l): - print '-- %s' % l - else: - print '?? %s' % l + blanks += 1 + if verbose: + if looks_like_forward_decl(line) \ + or looks_like_blank(line) \ + or looks_like_namespace(line): + print('-- %s' % line) + else: + print('?? %s' % line) - context = '' - h.close() + context = '' + h.close() # ==================================================================== # @@ -729,266 +824,291 @@ for filename in files: if dump: - if verbose: print '\n===============================\n' + if verbose: + print('\n===============================\n') - for c in classes: - print '\x1b[4m%s\x1b[0m %s\n{' % (c['type'], c['name']) - for f in c['fun']: - for s in f['signatures']: - print ' %s' % s.replace('\n', '\n ') + for c in classes: + print('\x1b[4m%s\x1b[0m %s\n{' % (c['type'], c['name'])) + for f in c['fun']: + for s in f['signatures']: + print(' %s' % s.replace('\n', '\n ')) - if len(c['fun']) > 0 and len(c['fields']) > 0: print '' + if len(c['fun']) > 0 and len(c['fields']) > 0: + print('') - for f in c['fields']: - for s in f['signatures']: - print ' %s' % s + for f in c['fields']: + for s in f['signatures']: + print(' %s' % s) - if len(c['fields']) > 0 and len(c['enums']) > 0: print '' + if len(c['fields']) > 0 and len(c['enums']) > 0: + print('') - for e in c['enums']: - print ' \x1b[4menum\x1b[0m %s\n {' % e['name'] - for v in e['values']: - print ' %s' % v['name'] - print ' };' - print '};\n' + for e in c['enums']: + print(' \x1b[4menum\x1b[0m %s\n {' % e['name']) + for v in e['values']: + print(' %s' % v['name']) + print(' };') + print('};\n') - for f in functions: - print '%s' % f['signature'] + for f in functions: + print('%s' % f['signature']) - for e in enums: - print '\x1b[4menum\x1b[0m %s\n{' % e['name'] - for v in e['values']: - print ' %s' % v['name'] - print '};' + for e in enums: + print('\x1b[4menum\x1b[0m %s\n{' % e['name']) + for v in e['values']: + print(' %s' % v['name']) + print('};') categories = {} for c in classes: - cat = categorize_symbol(c['name'], c['file']) - if not cat in categories: - categories[cat] = { 'classes': [], 'functions': [], 'enums': [], 'filename': 'reference-%s.rst' % cat.replace(' ', '_')} + cat = categorize_symbol(c['name'], c['file']) + if cat not in categories: + categories[cat] = {'classes': [], 'functions': [], 'enums': [], + 'filename': 'reference-%s.rst' % cat.replace(' ', '_')} - if c['file'] in overviews: - categories[cat]['overview'] = overviews[c['file']] + if c['file'] in overviews: + categories[cat]['overview'] = overviews[c['file']] - filename = categories[cat]['filename'].replace('.rst', '.html') + '#' - categories[cat]['classes'].append(c) - symbols[c['name']] = filename + c['name'] - for f in c['fun']: - for n in f['names']: - symbols[n] = filename + n - symbols[c['name'] + '::' + n] = filename + n + filename = categories[cat]['filename'].replace('.rst', '.html') + '#' + categories[cat]['classes'].append(c) + symbols[c['name']] = filename + c['name'] + for f in c['fun']: + for n in f['names']: + symbols[n] = filename + n + symbols[c['name'] + '::' + n] = filename + n - for f in c['fields']: - for n in f['names']: - symbols[c['name'] + '::' + n] = filename + n + for f in c['fields']: + for n in f['names']: + symbols[c['name'] + '::' + n] = filename + n - for e in c['enums']: - symbols[e['name']] = filename + e['name'] - symbols[c['name'] + '::' + e['name']] = filename + e['name'] - for v in e['values']: -# symbols[v['name']] = filename + v['name'] - symbols[e['name'] + '::' + v['name']] = filename + v['name'] - symbols[c['name'] + '::' + v['name']] = filename + v['name'] + for e in c['enums']: + symbols[e['name']] = filename + e['name'] + symbols[c['name'] + '::' + e['name']] = filename + e['name'] + for v in e['values']: + # symbols[v['name']] = filename + v['name'] + symbols[e['name'] + '::' + v['name']] = filename + v['name'] + symbols[c['name'] + '::' + v['name']] = filename + v['name'] for f in functions: - cat = categorize_symbol(first_item(f['names']), f['file']) - if not cat in categories: - categories[cat] = { 'classes': [], 'functions': [], 'enums': [], 'filename': 'reference-%s.rst' % cat.replace(' ', '_')} + cat = categorize_symbol(first_item(f['names']), f['file']) + if cat not in categories: + categories[cat] = {'classes': [], 'functions': [], 'enums': [], + 'filename': 'reference-%s.rst' % cat.replace(' ', '_')} - if f['file'] in overviews: - categories[cat]['overview'] = overviews[f['file']] + if f['file'] in overviews: + categories[cat]['overview'] = overviews[f['file']] - for n in f['names']: - symbols[n] = categories[cat]['filename'].replace('.rst', '.html') + '#' + n - categories[cat]['functions'].append(f) + for n in f['names']: + symbols[n] = categories[cat]['filename'].replace('.rst', '.html') + '#' + n + categories[cat]['functions'].append(f) for e in enums: - cat = categorize_symbol(e['name'], e['file']) - if not cat in categories: - categories[cat] = { 'classes': [], 'functions': [], 'enums': [], 'filename': 'reference-%s.rst' % cat.replace(' ', '_')} - categories[cat]['enums'].append(e) - filename = categories[cat]['filename'].replace('.rst', '.html') + '#' - symbols[e['name']] = filename + e['name'] - for v in e['values']: - symbols[e['name'] + '::' + v['name']] = filename + v['name'] + cat = categorize_symbol(e['name'], e['file']) + if cat not in categories: + categories[cat] = {'classes': [], 'functions': [], 'enums': [], + 'filename': 'reference-%s.rst' % cat.replace(' ', '_')} + categories[cat]['enums'].append(e) + filename = categories[cat]['filename'].replace('.rst', '.html') + '#' + symbols[e['name']] = filename + e['name'] + for v in e['values']: + symbols[e['name'] + '::' + v['name']] = filename + v['name'] + def print_declared_in(out, o): - out.write('Declared in "%s"\n\n' % print_link(o['file'], '../include/%s' % o['file'])) - print >>out, dump_link_targets() + out.write('Declared in "%s"\n\n' % print_link(o['file'], '../include/%s' % o['file'])) + print(dump_link_targets(), file=out) # returns RST marked up string + + def linkify_symbols(string): - lines = string.split('\n') - ret = [] - in_literal = False - lno = 0 - for l in lines: - lno += 1 - # don't touch headlines, i.e. lines whose - # next line entirely contains one of =, - or . - if (lno < len(lines)-1): next_line = lines[lno] - else: next_line = '' + lines = string.split('\n') + ret = [] + in_literal = False + lno = 0 + for line in lines: + lno += 1 + # don't touch headlines, i.e. lines whose + # next line entirely contains one of =, - or . + if (lno < len(lines) - 1): + next_line = lines[lno] + else: + next_line = '' - if len(next_line) > 0 and lines[lno].replace('=',''). \ - replace('-','').replace('.', '') == '': - ret.append(l) - continue + if len(next_line) > 0 and lines[lno].replace('=', ''). \ + replace('-', '').replace('.', '') == '': + ret.append(line) + continue - if l.startswith('|'): - ret.append(l) - continue - if in_literal and not l.startswith('\t') and not l == '': -# print ' end literal: "%s"' % l - in_literal = False - if in_literal: -# print ' literal: "%s"' % l - ret.append(l) - continue - if l.strip() == '.. parsed-literal::' or \ - l.strip().startswith('.. code::') or \ - (not l.strip().startswith('..') and l.endswith('::')): -# print ' start literal: "%s"' % l - in_literal = True - words = l.split(' ') + if line.startswith('|'): + ret.append(line) + continue + if in_literal and not line.startswith('\t') and not line == '': + # print(' end literal: "%s"' % line) + in_literal = False + if in_literal: + # print(' literal: "%s"' % line) + ret.append(line) + continue + if line.strip() == '.. parsed-literal::' or \ + line.strip().startswith('.. code::') or \ + (not line.strip().startswith('..') and line.endswith('::')): + # print(' start literal: "%s"' % line) + in_literal = True + words = line.split(' ') - for i in range(len(words)): - # it's important to preserve leading - # tabs, since that's relevant for - # rst markup + for i in range(len(words)): + # it's important to preserve leading + # tabs, since that's relevant for + # rst markup - leading = '' - w = words[i] + leading = '' + w = words[i] - if len(w) == 0: continue + if len(w) == 0: + continue - while len(w) > 0 and \ - w[0] in ['\t', ' ', '(', '[', '{']: - leading += w[0] - w = w[1:] + while len(w) > 0 and \ + w[0] in ['\t', ' ', '(', '[', '{']: + leading += w[0] + w = w[1:] - # preserve commas and dots at the end - w = w.strip() - trailing = '' + # preserve commas and dots at the end + w = w.strip() + trailing = '' - if len(w) == 0: continue + if len(w) == 0: + continue - while len(w) > 1 and w[-1] in ['.', ',', ')'] and w[-2:] != '()': - trailing = w[-1] + trailing - w = w[:-1] + while len(w) > 1 and w[-1] in ['.', ',', ')'] and w[-2:] != '()': + trailing = w[-1] + trailing + w = w[:-1] - link_name = w; + link_name = w -# print w + # print(w) - if len(w) == 0: continue + if len(w) == 0: + continue - if link_name[-1] == '_': link_name = link_name[:-1] + if link_name[-1] == '_': + link_name = link_name[:-1] + + if w in symbols: + link_name = link_name.replace('-', ' ') + # print(' found %s -> %s' % (w, link_name)) + words[i] = leading + print_link(link_name, symbols[w]) + trailing + ret.append(' '.join(words)) + return '\n'.join(ret) - if w in symbols: - link_name = link_name.replace('-', ' ') -# print ' found %s -> %s' % (w, link_name) - words[i] = leading + print_link(link_name, symbols[w]) + trailing - ret.append(' '.join(words)) - return '\n'.join(ret) link_targets = [] + def print_link(name, target): - global link_targets - link_targets.append(target) - return "`%s`__" % name + global link_targets + link_targets.append(target) + return "`%s`__" % name -def dump_link_targets(indent = ''): - global link_targets - ret = '\n' - for l in link_targets: - ret += '%s__ %s\n' % (indent, l) - link_targets = [] - return ret -def heading(string, c, indent = ''): - string = string.strip() - return '\n' + indent + string + '\n' + indent + (c * len(string)) + '\n' +def dump_link_targets(indent=''): + global link_targets + ret = '\n' + for l in link_targets: + ret += '%s__ %s\n' % (indent, l) + link_targets = [] + return ret + + +def heading(string, c, indent=''): + string = string.strip() + return '\n' + indent + string + '\n' + indent + (c * len(string)) + '\n' + def render_enums(out, enums, print_declared_reference, header_level): - for e in enums: - print >>out, '.. raw:: html\n' - print >>out, '\t' % e['name'] - print >>out, '' - print >>out, heading('enum %s' % e['name'], header_level) + for e in enums: + print('.. raw:: html\n', file=out) + print('\t' % e['name'], file=out) + print('', file=out) + print(heading('enum %s' % e['name'], header_level), file=out) - print_declared_in(out, e) + print_declared_in(out, e) - width = [len('name'), len('value'), len('description')] + width = [len('name'), len('value'), len('description')] - for i in range(len(e['values'])): - e['values'][i]['desc'] = linkify_symbols(e['values'][i]['desc']) + for i in range(len(e['values'])): + e['values'][i]['desc'] = linkify_symbols(e['values'][i]['desc']) - for v in e['values']: - width[0] = max(width[0], len(v['name'])) - width[1] = max(width[1], len(v['val'])) - for d in v['desc'].split('\n'): - width[2] = max(width[2], len(d)) + for v in e['values']: + width[0] = max(width[0], len(v['name'])) + width[1] = max(width[1], len(v['val'])) + for d in v['desc'].split('\n'): + width[2] = max(width[2], len(d)) - print >>out, '+-' + ('-' * width[0]) + '-+-' + ('-' * width[1]) + '-+-' + ('-' * width[2]) + '-+' - print >>out, '| ' + 'name'.ljust(width[0]) + ' | ' + 'value'.ljust(width[1]) + ' | ' + 'description'.ljust(width[2]) + ' |' - print >>out, '+=' + ('=' * width[0]) + '=+=' + ('=' * width[1]) + '=+=' + ('=' * width[2]) + '=+' - for v in e['values']: - d = v['desc'].split('\n') - if len(d) == 0: d = [''] - print >>out, '| ' + v['name'].ljust(width[0]) + ' | ' + v['val'].ljust(width[1]) + ' | ' + d[0].ljust(width[2]) + ' |' - for s in d[1:]: - print >>out, '| ' + (' ' * width[0]) + ' | ' + (' ' * width[1]) + ' | ' + s.ljust(width[2]) + ' |' - print >>out, '+-' + ('-' * width[0]) + '-+-' + ('-' * width[1]) + '-+-' + ('-' * width[2]) + '-+' - print >>out, '' + print('+-' + ('-' * width[0]) + '-+-' + ('-' * width[1]) + '-+-' + ('-' * width[2]) + '-+', file=out) + print(('| ' + 'name'.ljust(width[0]) + ' | ' + 'value'.ljust(width[1]) + ' | ' + 'description'.ljust(width[2]) + ' |'), file=out) + print('+=' + ('=' * width[0]) + '=+=' + ('=' * width[1]) + '=+=' + ('=' * width[2]) + '=+', file=out) + for v in e['values']: + d = v['desc'].split('\n') + if len(d) == 0: + d = [''] + print(('| ' + v['name'].ljust(width[0]) + ' | ' + v['val'].ljust(width[1]) + ' | ' + + d[0].ljust(width[2]) + ' |'), file=out) + for s in d[1:]: + print('| ' + (' ' * width[0]) + ' | ' + (' ' * width[1]) + ' | ' + s.ljust(width[2]) + ' |', file=out) + print('+-' + ('-' * width[0]) + '-+-' + ('-' * width[1]) + '-+-' + ('-' * width[2]) + '-+', file=out) + print('', file=out) + + print(dump_link_targets(), file=out) - print >>out, dump_link_targets() sections = \ -{ - 'Core': 0, - 'Session': 0, - 'Settings': 0, + { + 'Core': 0, + 'Session': 0, + 'Settings': 0, - 'Bencoding': 1, - 'Bdecoding': 1, - 'Filter': 1, - 'Error Codes': 1, - 'Create Torrents': 1, + 'Bencoding': 1, + 'Bdecoding': 1, + 'Filter': 1, + 'Error Codes': 1, + 'Create Torrents': 1, - 'ed25519': 2, - 'Utility': 2, - 'Storage': 2, - 'Custom Storage': 2, - 'Plugins': 2, + 'ed25519': 2, + 'Utility': 2, + 'Storage': 2, + 'Custom Storage': 2, + 'Plugins': 2, + + 'Alerts': 3 + } - 'Alerts': 3 -} def print_toc(out, categories, s): - for cat in categories: - if (s != 2 and cat not in sections) or \ - (cat in sections and sections[cat] != s): continue + for cat in categories: + if (s != 2 and cat not in sections) or \ + (cat in sections and sections[cat] != s): + continue - print >>out, '\t.. rubric:: %s\n' % cat + print('\t.. rubric:: %s\n' % cat, file=out) - if 'overview' in categories[cat]: - print >>out, '\t| overview__' + if 'overview' in categories[cat]: + print('\t| overview__', file=out) - category_filename = categories[cat]['filename'].replace('.rst', '.html') - for c in categories[cat]['classes']: - print >>out, '\t| ' + print_link(c['name'], symbols[c['name']]) - for f in categories[cat]['functions']: - for n in f['names']: - print >>out, '\t| ' + print_link(n, symbols[n]) - for e in categories[cat]['enums']: - print >>out, '\t| ' + print_link(e['name'], symbols[e['name']]) - print >>out, '' + for c in categories[cat]['classes']: + print('\t| ' + print_link(c['name'], symbols[c['name']]), file=out) + for f in categories[cat]['functions']: + for n in f['names']: + print('\t| ' + print_link(n, symbols[n]), file=out) + for e in categories[cat]['enums']: + print('\t| ' + print_link(e['name'], symbols[e['name']]), file=out) + print('', file=out) - if 'overview' in categories[cat]: - print >>out, '\t__ %s#overview' % categories[cat]['filename'].replace('.rst', '.html') - print >>out, dump_link_targets('\t') + if 'overview' in categories[cat]: + print('\t__ %s#overview' % categories[cat]['filename'].replace('.rst', '.html'), file=out) + print(dump_link_targets('\t'), file=out) out = open('reference.rst', 'w+') @@ -1002,19 +1122,19 @@ out.write('`single-page version`__\n\n__ single-page-ref.html\n\n') for i in range(4): - out.write('.. container:: main-toc\n\n') - print_toc(out, categories, i) + out.write('.. container:: main-toc\n\n') + print_toc(out, categories, i) out.close() for cat in categories: - out = open(categories[cat]['filename'], 'w+') + out = open(categories[cat]['filename'], 'w+') - classes = categories[cat]['classes'] - functions = categories[cat]['functions'] - enums = categories[cat]['enums'] + classes = categories[cat]['classes'] + functions = categories[cat]['functions'] + enums = categories[cat]['enums'] - out.write(''' + out.write(''' :Author: Arvid Norberg, arvid@libtorrent.org :Version: 1.2.0 @@ -1030,132 +1150,133 @@ __ reference.html ''' % heading(cat, '=')) - if 'overview' in categories[cat]: - out.write('%s\n' % linkify_symbols(categories[cat]['overview'])) + if 'overview' in categories[cat]: + out.write('%s\n' % linkify_symbols(categories[cat]['overview'])) - for c in classes: + for c in classes: - print >>out, '.. raw:: html\n' - print >>out, '\t' % c['name'] - print >>out, '' + print('.. raw:: html\n', file=out) + print('\t' % c['name'], file=out) + print('', file=out) - out.write('%s\n' % heading(c['name'], '-')) - print_declared_in(out, c) - c['desc'] = linkify_symbols(c['desc']) - out.write('%s\n' % c['desc']) - print >>out, dump_link_targets() + out.write('%s\n' % heading(c['name'], '-')) + print_declared_in(out, c) + c['desc'] = linkify_symbols(c['desc']) + out.write('%s\n' % c['desc']) + print(dump_link_targets(), file=out) - print >>out,'\n.. parsed-literal::\n\t' + print('\n.. parsed-literal::\n\t', file=out) - block = '\n%s\n{\n' % c['decl'] - for f in c['fun']: - for s in f['signatures']: - block += ' %s\n' % highlight_signature(s.replace('\n', '\n ')) + block = '\n%s\n{\n' % c['decl'] + for f in c['fun']: + for s in f['signatures']: + block += ' %s\n' % highlight_signature(s.replace('\n', '\n ')) - if len(c['fun']) > 0 and len(c['enums']) > 0: block += '\n' + if len(c['fun']) > 0 and len(c['enums']) > 0: + block += '\n' - first = True - for e in c['enums']: - if not first: - block += '\n' - first = False - block += ' enum %s\n {\n' % e['name'] - for v in e['values']: - block += ' %s,\n' % v['name'] - block += ' };\n' + first = True + for e in c['enums']: + if not first: + block += '\n' + first = False + block += ' enum %s\n {\n' % e['name'] + for v in e['values']: + block += ' %s,\n' % v['name'] + block += ' };\n' - if len(c['fun']) + len(c['enums']) > 0 and len(c['fields']): block += '\n' + if len(c['fun']) + len(c['enums']) > 0 and len(c['fields']): + block += '\n' - for f in c['fields']: - for s in f['signatures']: - block += ' %s\n' % s + for f in c['fields']: + for s in f['signatures']: + block += ' %s\n' % s - block += '};' + block += '};' - print >>out, block.replace('\n', '\n\t') + '\n' + print(block.replace('\n', '\n\t') + '\n', file=out) - for f in c['fun']: - if f['desc'] == '': continue - title = '' - print >>out, '.. raw:: html\n' - for n in f['names']: - print >>out, '\t' % n - print >>out, '' - for n in f['names']: - title += '%s ' % n - print >>out, heading(title.strip(), '.') + for f in c['fun']: + if f['desc'] == '': + continue + title = '' + print('.. raw:: html\n', file=out) + for n in f['names']: + print('\t' % n, file=out) + print('', file=out) + for n in f['names']: + title += '%s ' % n + print(heading(title.strip(), '.'), file=out) - block = '.. parsed-literal::\n\n' + block = '.. parsed-literal::\n\n' - for s in f['signatures']: - block += highlight_signature(s.replace('\n', '\n ')) + '\n' - print >>out, '%s\n' % block.replace('\n', '\n\t') - f['desc'] = linkify_symbols(f['desc']) - print >>out, '%s' % f['desc'] + for s in f['signatures']: + block += highlight_signature(s.replace('\n', '\n ')) + '\n' + print('%s\n' % block.replace('\n', '\n\t'), file=out) + f['desc'] = linkify_symbols(f['desc']) + print('%s' % f['desc'], file=out) - print >>out, dump_link_targets() + print(dump_link_targets(), file=out) - render_enums(out, c['enums'], False, '.') + render_enums(out, c['enums'], False, '.') - for f in c['fields']: - if f['desc'] == '': continue + for f in c['fields']: + if f['desc'] == '': + continue - print >>out, '.. raw:: html\n' - for n in f['names']: - print >>out, '\t' % n - print >>out, '' + print('.. raw:: html\n', file=out) + for n in f['names']: + print('\t' % n, file=out) + print('', file=out) - for n in f['names']: - print >>out, '%s ' % n, - print >>out, '' - f['desc'] = linkify_symbols(f['desc']) - print >>out, '\t%s' % f['desc'].replace('\n', '\n\t') + for n in f['names']: + print('%s ' % n, end=' ', file=out) + print('', file=out) + f['desc'] = linkify_symbols(f['desc']) + print('\t%s' % f['desc'].replace('\n', '\n\t'), file=out) - print >>out, dump_link_targets() + print(dump_link_targets(), file=out) + for f in functions: + h = '' + print('.. raw:: html\n', file=out) + for n in f['names']: + print('\t' % n, file=out) + print('', file=out) + for n in f['names']: + h += '%s ' % n + print(heading(h, '-'), file=out) + print_declared_in(out, f) - for f in functions: - h = '' - print >>out, '.. raw:: html\n' - for n in f['names']: - print >>out, '\t' % n - print >>out, '' - for n in f['names']: - h += '%s ' % n - print >>out, heading(h, '-') - print_declared_in(out, f) + block = '.. parsed-literal::\n\n' + for s in f['signatures']: + block += highlight_signature(s) + '\n' - block = '.. parsed-literal::\n\n' - for s in f['signatures']: - block += highlight_signature(s) + '\n' + print('%s\n' % block.replace('\n', '\n\t'), file=out) + print(linkify_symbols(f['desc']), file=out) - print >>out, '%s\n' % block.replace('\n', '\n\t') - print >>out, linkify_symbols(f['desc']) + print(dump_link_targets(), file=out) - print >>out, dump_link_targets() + render_enums(out, enums, True, '-') - render_enums(out, enums, True, '-') + print(dump_link_targets(), file=out) - print >>out, dump_link_targets() + for i in static_links: + print(i, file=out) - for i in static_links: - print >>out, i + out.close() - out.close() +# for s in symbols: +# print(s) -#for s in symbols: -# print s - -for i,o in preprocess_rst.items(): - f = open(i, 'r') - out = open(o, 'w+') - print 'processing %s -> %s' % (i, o) - l = linkify_symbols(f.read()) - print >>out, l, - - print >>out, dump_link_targets() - - out.close() - f.close() +for i, o in list(preprocess_rst.items()): + f = open(i, 'r') + out = open(o, 'w+') + print('processing %s -> %s' % (i, o)) + link = linkify_symbols(f.read()) + print(link, end=' ', file=out) + print(dump_link_targets(), file=out) + out.close() + f.close() diff --git a/docs/gen_settings_doc.py b/docs/gen_settings_doc.py index 5329cf0aa..5f5090af4 100755 --- a/docs/gen_settings_doc.py +++ b/docs/gen_settings_doc.py @@ -1,98 +1,123 @@ #!/usr/bin/env python +from __future__ import print_function f = open('../include/libtorrent/settings_pack.hpp') out = open('settings.rst', 'w+') + def print_field(str, width): - return '%s%s' % (str, ' ' * (width - len(str))) + return '%s%s' % (str, ' ' * (width - len(str))) + def render_section(names, description, type, default_values): - max_name_len = max(len(max(names, key=len)), len('name')) - max_type_len = max(len(type), len('type')) - max_val_len = max(len(max(default_values, key=len)), len('default')) + max_name_len = max(len(max(names, key=len)), len('name')) + max_type_len = max(len(type), len('type')) + max_val_len = max(len(max(default_values, key=len)), len('default')) - # add link targets for the rest of the manual to reference - for n in names: - print >>out, '.. _%s:\n' % n + # add link targets for the rest of the manual to reference + for n in names: + print('.. _%s:\n' % n, file=out) - if len(names) > 0: - print >>out, '.. raw:: html\n' - for n in names: - print >>out, '\t' % n - print >>out, '' + if len(names) > 0: + print('.. raw:: html\n', file=out) + for n in names: + print('\t' % n, file=out) + print('', file=out) - separator = '+-' + ('-' * max_name_len) + '-+-' + ('-' * max_type_len) + '-+-' + ('-' * max_val_len) + '-+' + separator = '+-' + ('-' * max_name_len) + '-+-' + ('-' * max_type_len) + '-+-' + ('-' * max_val_len) + '-+' + + # build a table for the settings, their type and default value + print(separator, file=out) + print( + '| %s | %s | %s |' % + (print_field( + 'name', max_name_len), print_field( + 'type', max_type_len), print_field( + 'default', max_val_len)), file=out) + print(separator.replace('-', '='), file=out) + for i in range(len(names)): + print( + '| %s | %s | %s |' % + (print_field( + names[i], max_name_len), print_field( + type, max_type_len), print_field( + default_values[i], max_val_len)), file=out) + print(separator, file=out) + print(file=out) + print(description, file=out) - # build a table for the settings, their type and default value - print >>out, separator - print >>out, '| %s | %s | %s |' % (print_field('name', max_name_len), print_field('type', max_type_len), print_field('default', max_val_len)) - print >>out, separator.replace('-', '=') - for i in range(len(names)): - print >>out, '| %s | %s | %s |' % (print_field(names[i], max_name_len), print_field(type, max_type_len), print_field(default_values[i], max_val_len)) - print >>out, separator - print >>out - print >>out, description mode = '' # parse out default values for settings f2 = open('../src/settings_pack.cpp') def_map = {} -for l in f2: - l = l.strip() - if not l.startswith('SET(') \ - and not l.startswith('SET_NOPREV(') \ - and not l.startswith('DEPRECATED_SET('): continue +for line in f2: + line = line.strip() + if not line.startswith('SET(') \ + and not line.startswith('SET_NOPREV(') \ + and not line.startswith('DEPRECATED_SET('): + continue - l = l.split('(')[1].split(',') - def_map[l[0]] = l[1].strip() - print '%s = %s' % (l[0], l[1].strip()) + line = line.split('(')[1].split(',') + def_map[line[0]] = line[1].strip() + print('%s = %s' % (line[0], line[1].strip())) description = '' names = [] -for l in f: - if 'enum string_types' in l: mode = 'string' - if 'enum bool_types' in l: mode = 'bool' - if 'enum int_types' in l: mode = 'int' - if '#if TORRENT_ABI_VERSION == 1' in l: mode += 'skip' - if '#endif' in l: mode = mode[0:-4] +for line in f: + if 'enum string_types' in line: + mode = 'string' + if 'enum bool_types' in line: + mode = 'bool' + if 'enum int_types' in line: + mode = 'int' + if '#if TORRENT_ABI_VERSION == 1' in line: + mode += 'skip' + if '#endif' in line: + mode = mode[0:-4] - if mode == '': continue - if mode[-4:] == 'skip': continue + if mode == '': + continue + if mode[-4:] == 'skip': + continue - l = l.lstrip() + line = line.lstrip() - if l == '' and len(names) > 0: - if description == '': - for n in names: - print 'WARNING: no description for "%s"' % n - else: - default_values = [] - for n in names: - default_values.append(def_map[n]) - render_section(names, description, mode, default_values) - description = '' - names = [] + if line == '' and len(names) > 0: + if description == '': + for n in names: + print('WARNING: no description for "%s"' % n) + else: + default_values = [] + for n in names: + default_values.append(def_map[n]) + render_section(names, description, mode, default_values) + description = '' + names = [] - if l.startswith('};'): - mode = '' - continue + if line.startswith('};'): + mode = '' + continue - if l.startswith('//'): - if l[2] == ' ': description += l[3:] - else: description += l[2:] - continue + if line.startswith('//'): + if line[2] == ' ': + description += line[3:] + else: + description += line[2:] + continue - l = l.strip() - if l.endswith(','): - l = l[:-1] # strip trailing comma - if '=' in l: l = l.split('=')[0].strip() - if l.endswith('_internal'): continue + line = line.strip() + if line.endswith(','): + line = line[:-1] # strip trailing comma + if '=' in line: + line = line.split('=')[0].strip() + if line.endswith('_internal'): + continue - names.append(l) + names.append(line) out.close() f.close() - diff --git a/docs/gen_stats_doc.py b/docs/gen_stats_doc.py index 5bba99812..dcd6649ed 100755 --- a/docs/gen_stats_doc.py +++ b/docs/gen_stats_doc.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import print_function counter_types = {} @@ -6,33 +7,40 @@ f = open('../include/libtorrent/performance_counters.hpp') counter_type = '' -for l in f: +for line in f: - # ignore anything after // - if '//' in l: l = l.split('//')[0] + # ignore anything after // + if '//' in line: + line = line.split('//')[0] - l = l.strip() + line = line.strip() - if l.startswith('#'): continue - if l == '': continue + if line.startswith('#'): + continue + if line == '': + continue - if 'enum stats_counter_t' in l: - counter_type = 'counter' - continue + if 'enum stats_counter_t' in line: + counter_type = 'counter' + continue - if 'enum stats_gauge_t' in l: - counter_type = 'gauge' - continue + if 'enum stats_gauge_t' in line: + counter_type = 'gauge' + continue - if '{' in l or '}' in l or 'struct' in l or 'namespace' in l: continue - if counter_type == '': continue - if not l.endswith(','): continue + if '{' in line or '}' in line or 'struct' in line or 'namespace' in line: + continue + if counter_type == '': + continue + if not line.endswith(','): + continue - # strip off trailing comma - l = l[:-1] - if '=' in l: l = l[:l.index('=')].strip() + # strip off trailing comma + line = line[:-1] + if '=' in line: + line = line[:line.index('=')].strip() - counter_types[l] = counter_type + counter_types[line] = counter_type f.close() @@ -40,39 +48,42 @@ f = open('../src/session_stats.cpp') out = open('stats_counters.rst', 'w+') + def print_field(str, width): - return '%s%s' % (str, ' ' * (width - len(str))) + return '%s%s' % (str, ' ' * (width - len(str))) + def render_section(names, description, types): - max_name_len = max(len(max(names, key=len)), len('name')) - max_type_len = max(len(max(types, key=len)), len('type')) + max_name_len = max(len(max(names, key=len)), len('name')) + max_type_len = max(len(max(types, key=len)), len('type')) - if description == '': - for n in names: - print 'WARNING: no description for "%s"' % n + if description == '': + for n in names: + print('WARNING: no description for "%s"' % n) - # add link targets for the rest of the manual to reference - for n in names: - print >>out, '.. _%s:\n' % n + # add link targets for the rest of the manual to reference + for n in names: + print('.. _%s:\n' % n, file=out) - if len(names) > 0: - print >>out, '.. raw:: html\n' - for n in names: - print >>out, '\t' % n - print >>out, '' + if len(names) > 0: + print('.. raw:: html\n', file=out) + for n in names: + print('\t' % n, file=out) + print('', file=out) - separator = '+-' + ('-' * max_name_len) + '-+-' + ('-' * max_type_len) + '-+' + separator = '+-' + ('-' * max_name_len) + '-+-' + ('-' * max_type_len) + '-+' + + # build a table for the settings, their type and default value + print(separator, file=out) + print('| %s | %s |' % (print_field('name', max_name_len), print_field('type', max_type_len)), file=out) + print(separator.replace('-', '='), file=out) + for i in range(len(names)): + print('| %s | %s |' % (print_field(names[i], max_name_len), print_field(types[i], max_type_len)), file=out) + print(separator, file=out) + print(file=out) + print(description, file=out) + print('', file=out) - # build a table for the settings, their type and default value - print >>out, separator - print >>out, '| %s | %s |' % (print_field('name', max_name_len), print_field('type', max_type_len)) - print >>out, separator.replace('-', '=') - for i in range(len(names)): - print >>out, '| %s | %s |' % (print_field(names[i], max_name_len), print_field(types[i], max_type_len)) - print >>out, separator - print >>out - print >>out, description - print >>out, '' mode = '' @@ -80,42 +91,43 @@ description = '' names = [] types = [] -for l in f: - description_line = l.lstrip().startswith('//') +for line in f: + description_line = line.lstrip().startswith('//') - l = l.strip() + line = line.strip() - if mode == 'ignore': - if '#endif' in l: mode = '' - continue + if mode == 'ignore': + if '#endif' in line: + mode = '' + continue - if 'TORRENT_ABI_VERSION == 1' in l: - mode = 'ignore' - continue + if 'TORRENT_ABI_VERSION == 1' in line: + mode = 'ignore' + continue - if description_line == True: - if len(names) > 0: - render_section(names, description, types) - description = '' - names = [] - types = [] + if description_line: + if len(names) > 0: + render_section(names, description, types) + description = '' + names = [] + types = [] - description += '\n' + l[3:] + description += '\n' + line[3:] - if '#define' in l: continue + if '#define' in line: + continue - if 'METRIC(' in l: - args = l.split('(')[1].split(')')[0].split(',') + if 'METRIC(' in line: + args = line.split('(')[1].split(')')[0].split(',') - # args: category, name, type + # args: category, name, type - args[1] = args[1].strip() - names.append(args[0].strip() + '.' + args[1].strip()) - types.append(counter_types[args[1]]) + args[1] = args[1].strip() + names.append(args[0].strip() + '.' + args[1].strip()) + types.append(counter_types[args[1]]) if len(names) > 0: - render_section(names, description, types) + render_section(names, description, types) out.close() f.close() - diff --git a/docs/gen_todo.py b/docs/gen_todo.py index c36fb5b27..773d7baae 100755 --- a/docs/gen_todo.py +++ b/docs/gen_todo.py @@ -2,15 +2,23 @@ import glob import os +import sys -paths = ['test/*.cpp', 'src/*.cpp', 'src/kademlia/*.cpp', 'include/libtorrent/*.hpp', 'include/libtorrent/kademlia/*.hpp', 'include/libtorrent/aux_/*.hpp', 'include/libtorrent/extensions/*.hpp'] +paths = [ + 'test/*.cpp', + 'src/*.cpp', + 'src/kademlia/*.cpp', + 'include/libtorrent/*.hpp', + 'include/libtorrent/kademlia/*.hpp', + 'include/libtorrent/aux_/*.hpp', + 'include/libtorrent/extensions/*.hpp'] os.system('(cd .. ; ctags %s 2>/dev/null)' % ' '.join(paths)) files = [] for p in paths: - files.extend(glob.glob(os.path.join('..', p))) + files.extend(glob.glob(os.path.join('..', p))) items = [] @@ -20,79 +28,91 @@ context = [] priority_count = [0, 0, 0, 0, 0] + def html_sanitize(s): - ret = '' - for i in s: - if i == '<': ret += '<' - elif i == '>': ret += '>' - elif i == '&': ret += '&' - else: ret += i - return ret + ret = '' + for i in s: + if i == '<': + ret += '<' + elif i == '>': + ret += '>' + elif i == '&': + ret += '&' + else: + ret += i + return ret + for f in files: - h = open(f) + h = open(f) - state = '' - line_no = 0 - context_lines = 0 + state = '' + line_no = 0 + context_lines = 0 - for l in h: - line_no += 1 - line = l.strip() - if 'TODO:' in line and line.startswith('//'): - line = line.split('TODO:')[1].strip() - state = 'todo' - items.append({}) - items[-1]['location'] = '%s:%d' % (f, line_no) - items[-1]['priority'] = 0 - if line[0] in '0123456789': - items[-1]['priority'] = int(line[0]) - if int(line[0]) > 5: - print 'priority too high: ' + line - sys.exit(1) + for l in h: + line_no += 1 + line = l.strip() + if 'TODO:' in line and line.startswith('//'): + line = line.split('TODO:')[1].strip() + state = 'todo' + items.append({}) + items[-1]['location'] = '%s:%d' % (f, line_no) + items[-1]['priority'] = 0 + if line[0] in '0123456789': + items[-1]['priority'] = int(line[0]) + if int(line[0]) > 5: + print('priority too high: ' + line) + sys.exit(1) - line = line[1:].strip() - items[-1]['todo'] = line - prio = items[-1]['priority'] - if prio >= 0 and prio <= 4: priority_count[prio] += 1 - continue + line = line[1:].strip() + items[-1]['todo'] = line + prio = items[-1]['priority'] + if prio >= 0 and prio <= 4: + priority_count[prio] += 1 + continue - if state == '': - context.append(html_sanitize(l)) - if len(context) > 20: context.pop(0) - continue + if state == '': + context.append(html_sanitize(l)) + if len(context) > 20: + context.pop(0) + continue - if state == 'todo': - if line.strip().startswith('//'): - items[-1]['todo'] += '\n' - items[-1]['todo'] += line[2:].strip() - else: - state = 'context' - items[-1]['context'] = ''.join(context) + '
' + html_sanitize(l) + '
'; - context_lines = 1 + if state == 'todo': + if line.strip().startswith('//'): + items[-1]['todo'] += '\n' + items[-1]['todo'] += line[2:].strip() + else: + state = 'context' + items[-1]['context'] = ''.join(context) + \ + '
' + html_sanitize(l) + '
' + context_lines = 1 - context.append(html_sanitize(l)) - if len(context) > 20: context.pop(0) - continue + context.append(html_sanitize(l)) + if len(context) > 20: + context.pop(0) + continue - if state == 'context': - items[-1]['context'] += html_sanitize(l) - context_lines += 1 + if state == 'context': + items[-1]['context'] += html_sanitize(l) + context_lines += 1 - context.append(html_sanitize(l)) - if len(context) > 20: context.pop(0) - if context_lines > 30: state = '' + context.append(html_sanitize(l)) + if len(context) > 20: + context.pop(0) + if context_lines > 30: + state = '' - h.close() + h.close() -items.sort(key = lambda x: x['priority'], reverse = True) +items.sort(key=lambda x: x['priority'], reverse=True) -#for i in items: -# print '\n\n', i['todo'], '\n' -# print i['location'], '\n' -# print 'prio: ', i['priority'], '\n' -# if 'context' in i: -# print i['context'], '\n' +# for i in items: +# print('\n\n', i['todo'], '\n') +# print(i['location'], '\n') +# print('prio: ', i['priority'], '\n') +# if 'context' in i: +# print(i['context'], '\n') out = open('todo.html', 'w+') out.write(''' @@ -123,21 +143,24 @@ out.write(''' %d relevant %d feasible %d notes -''' % \ - (priority_count[4], priority_count[3], priority_count[2], priority_count[1], priority_count[0])) +
''' # noqa + % (priority_count[4], priority_count[3], priority_count[2], priority_count[1], priority_count[0])) -prio_colors = [ '#ccc', '#ccf', '#cfc', '#fcc', '#f44'] +prio_colors = ['#ccc', '#ccf', '#cfc', '#fcc', '#f44'] index = 0 for i in items: - if not 'context' in i: i['context'] = '' - out.write('' \ - % (prio_colors[i['priority']], i['priority'], index, i['location'], i['todo'].replace('\n', ' '))) + if 'context' not in i: + i['context'] = '' + out.write(('' + '') + % (prio_colors[i['priority']], i['priority'], index, i['location'], i['todo'].replace('\n', ' '))) - out.write('' \ - % (index, i['todo'], i['location'], i['context'])) - index += 1 + out.write( + ('') % + (index, i['todo'], i['location'], i['context'])) + index += 1 out.write('
relevance %d%s%s
relevance %d%s%s
') out.close() - diff --git a/docs/join_rst.py b/docs/join_rst.py index 649057c62..f41007449 100644 --- a/docs/join_rst.py +++ b/docs/join_rst.py @@ -7,19 +7,20 @@ sys.stdout.write(open(sys.argv[1], 'r').read()) sys.stderr.write('joining %s\n' % sys.argv[1]) for name in sys.argv[2:]: - sys.stdout.write('\n') - sys.stderr.write('joining %s\n' % name) - f = open(name, 'r') - for l in f: - # strip out the table of contents from subsequent files - if '.. contents::' in l: - in_directive = True - continue - if ':Author:' in l: continue - if ':Version:' in l: continue - - if l[0] in ' \t' and in_directive: - continue - in_directive = False - sys.stdout.write(l) + sys.stdout.write('\n') + sys.stderr.write('joining %s\n' % name) + f = open(name, 'r') + for l in f: + # strip out the table of contents from subsequent files + if '.. contents::' in l: + in_directive = True + continue + if ':Author:' in l: + continue + if ':Version:' in l: + continue + if l[0] in ' \t' and in_directive: + continue + in_directive = False + sys.stdout.write(l) diff --git a/examples/run_benchmarks.py b/examples/run_benchmarks.py index 8b8141b8e..0151fddc7 100755 --- a/examples/run_benchmarks.py +++ b/examples/run_benchmarks.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import print_function import sys import os @@ -62,378 +63,465 @@ default_cache = cache_sizes[-1] # disk cache is not a significant part of the test, # since download rates will be extremely high while downloading # into RAM -test_duration = 200 # 700 - +test_duration = 200 # 700 # make sure the environment is properly set up try: - if os.name == 'posix': - resource.setrlimit(resource.RLIMIT_NOFILE, (4000, 5000)) -except: - if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < 4000: - print 'please set ulimit -n to at least 4000' - sys.exit(1) + if os.name == 'posix': + resource.setrlimit(resource.RLIMIT_NOFILE, (4000, 5000)) +except BaseException: + if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < 4000: + print('please set ulimit -n to at least 4000') + sys.exit(1) + def build_stage_dirs(): - ret = [] - for i in builds[2:3]: - ret.append('stage_%s' % i) - return ret + ret = [] + for i in builds[2:3]: + ret.append('stage_%s' % i) + return ret + # make sure we have all the binaries available binaries = ['client_test', 'connection_tester', 'fragmentation_test'] for b in build_stage_dirs(): - for i in binaries: - p = os.path.join(b, i) - if not os.path.exists(p): - print 'make sure "%s" is available in ./%s' % (i, b) - sys.exit(1) + for i in binaries: + p = os.path.join(b, i) + if not os.path.exists(p): + print('make sure "%s" is available in ./%s' % (i, b)) + sys.exit(1) for i in filesystem: - if not os.path.exists(i): - print ('the path "%s" does not exist. This is directory/mountpoint is ' + - 'used as the download directory and is the filesystem that will be benchmarked ' + - 'and need to exist.') % i - sys.exit(1) + if not os.path.exists(i): + print(('the path "%s" does not exist. This is directory/mountpoint is ' + + 'used as the download directory and is the filesystem that will be benchmarked ' + + 'and need to exist.') % i) + sys.exit(1) # make sure we have a test torrent if not os.path.exists('test.torrent'): - print 'generating test torrent' - # generate a 100 GB torrent, to make sure it won't all fit in physical RAM - os.system('./stage_aio/connection_tester gen-torrent 10000 test.torrent') + print('generating test torrent') + # generate a 100 GB torrent, to make sure it won't all fit in physical RAM + os.system('./stage_aio/connection_tester gen-torrent 10000 test.torrent') if not os.path.exists('test2.torrent'): - print 'generating test torrent 2' - # generate a 6 GB torrent, to make sure it will fit in physical RAM - os.system('./stage_aio/connection_tester gen-torrent 6000 test2.torrent') + print('generating test torrent 2') + # generate a 6 GB torrent, to make sure it will fit in physical RAM + os.system('./stage_aio/connection_tester gen-torrent 6000 test2.torrent') # use a new port for each test to make sure they keep working # this port is incremented for each test run port = 10000 + random.randint(0, 40000) + def clear_caches(): - if 'linux' in sys.platform: - os.system('sync') - open('/proc/sys/vm/drop_caches', 'w').write('3') - elif 'darwin' in sys.platform: - os.system('purge') + if 'linux' in sys.platform: + os.system('sync') + open('/proc/sys/vm/drop_caches', 'w').write('3') + elif 'darwin' in sys.platform: + os.system('purge') + def build_commandline(config, port): - num_peers = config['num-peers'] - torrent_path = config['torrent'] + num_peers = config['num-peers'] + torrent_path = config['torrent'] - if config['build'] == 'utorrent': - try: os.mkdir('utorrent_session') - except: pass - cfg = open('utorrent_session/settings.dat', 'w+') + if config['build'] == 'utorrent': + try: + os.mkdir('utorrent_session') + except BaseException: + pass + cfg = open('utorrent_session/settings.dat', 'w+') - cfg.write('d') - cfg.write('20:ul_slots_per_torrenti%de' % num_peers) - cfg.write('17:conns_per_torrenti%de' % num_peers) - cfg.write('14:conns_globallyi%de' % num_peers) - cfg.write('9:bind_porti%de' % port) - cfg.write('19:dir_active_download%d:%s' % (len(config['save-path']), config['save-path'])) - cfg.write('19:diskio.sparse_filesi1e') - cfg.write('14:cache.overridei1e') - cfg.write('19:cache.override_sizei%de' % int(config['cache-size'] * 16 / 1024)) - cfg.write('17:dir_autoload_flagi1e') - cfg.write('12:dir_autoload8:autoload') - cfg.write('11:logger_maski4294967295e') - cfg.write('1:vi0e') - cfg.write('12:webui.enablei1e') - cfg.write('19:webui.enable_listeni1e') - cfg.write('14:webui.hashword20:' + hashlib.sha1('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaadmin').digest()) - cfg.write('10:webui.porti8080e') - cfg.write('10:webui.salt32:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') - cfg.write('14:webui.username5:admin') - cfg.write('e') - cfg.close() - try: os.mkdir('utorrent_session/autoload') - except: pass - try: shutil.copy(torrent_path, 'utorrent_session/autoload/') - except: pass - return './utorrent-server-v3_0/utserver -logfile session_stats/alerts_log.txt -settingspath utorrent_session' + cfg.write('d') + cfg.write('20:ul_slots_per_torrenti%de' % num_peers) + cfg.write('17:conns_per_torrenti%de' % num_peers) + cfg.write('14:conns_globallyi%de' % num_peers) + cfg.write('9:bind_porti%de' % port) + cfg.write('19:dir_active_download%d:%s' % (len(config['save-path']), config['save-path'])) + cfg.write('19:diskio.sparse_filesi1e') + cfg.write('14:cache.overridei1e') + cfg.write('19:cache.override_sizei%de' % int(config['cache-size'] * 16 / 1024)) + cfg.write('17:dir_autoload_flagi1e') + cfg.write('12:dir_autoload8:autoload') + cfg.write('11:logger_maski4294967295e') + cfg.write('1:vi0e') + cfg.write('12:webui.enablei1e') + cfg.write('19:webui.enable_listeni1e') + cfg.write('14:webui.hashword20:' + hashlib.sha1('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaadmin').digest()) + cfg.write('10:webui.porti8080e') + cfg.write('10:webui.salt32:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + cfg.write('14:webui.username5:admin') + cfg.write('e') + cfg.close() + try: + os.mkdir('utorrent_session/autoload') + except BaseException: + pass + try: + shutil.copy(torrent_path, 'utorrent_session/autoload/') + except BaseException: + pass + return './utorrent-server-v3_0/utserver -logfile session_stats/alerts_log.txt -settingspath utorrent_session' - if config['build'] == 'rtorrent': - if os.path.exists('rtorrent_session'): - add_command = '' - else: - try: os.mkdir('rtorrent_session') - except: pass - # it seems rtorrent may delete the original torrent when it's being added - try: shutil.copy(torrent_path, 'rtorrent_session/') - except: pass - add_command = '-O load_start_verbose=rtorrent_session/%s ' % torrent_path + if config['build'] == 'rtorrent': + if os.path.exists('rtorrent_session'): + add_command = '' + else: + try: + os.mkdir('rtorrent_session') + except BaseException: + pass + # it seems rtorrent may delete the original torrent when it's being added + try: + shutil.copy(torrent_path, 'rtorrent_session/') + except BaseException: + pass + add_command = '-O load_start_verbose=rtorrent_session/%s ' % torrent_path - return 'rtorrent -d %s -n -p %d-%d -O max_peers=%d -O max_uploads=%d %s -s rtorrent_session -O max_memory_usage=128000000000' \ - % (config['save-path'], port, port, num_peers, num_peers, add_command) + return ('rtorrent -d %s -n -p %d-%d -O max_peers=%d -O max_uploads=%d %s -s ' + 'rtorrent_session -O max_memory_usage=128000000000') % ( + config['save-path'], port, port, num_peers, num_peers, add_command) + + disable_disk = '' + if config['disable-disk']: + disable_disk = '-0' + return ('./stage_%s/client_test -k -N -H -M -B %d -l %d -S %d -T %d -c %d -C %d -s "%s" -p %d -E %d %s ' + '-f session_stats/alerts_log.txt %s') % ( + config['build'], test_duration, num_peers, num_peers, num_peers, num_peers, config['cache-size'], + config['save-path'], port, config['hash-threads'], disable_disk, torrent_path) - disable_disk = '' - if config['disable-disk']: disable_disk = '-0' - return './stage_%s/client_test -k -N -H -M -B %d -l %d -S %d -T %d -c %d -C %d -s "%s" -p %d -E %d %s -f session_stats/alerts_log.txt %s' \ - % (config['build'], test_duration, num_peers, num_peers, num_peers, num_peers, config['cache-size'], config['save-path'], port, \ - config['hash-threads'], disable_disk, torrent_path) def delete_files(files): - for i in files: - try: os.remove(i) - except: - try: shutil.rmtree(i) - except: - try: - if os.path.exists(i): print 'failed to delete %s' % i - except: pass + for i in files: + try: + os.remove(i) + except BaseException: + try: + shutil.rmtree(i) + except BaseException: + try: + if os.path.exists(i): + print('failed to delete %s' % i) + except BaseException: + pass # typically the schedulers available are 'noop', 'deadline' and 'cfq' -def build_test_config(fs=default_fs, num_peers=default_peers, cache_size=default_cache, \ - test='upload', build='aio', profile='', hash_threads=1, torrent='test.torrent', \ - disable_disk = False): - config = {'test': test, 'save-path': os.path.join('./', fs), 'num-peers': num_peers, \ - 'cache-size': cache_size, 'build': build, 'profile':profile, \ - 'hash-threads': hash_threads, 'torrent': torrent, 'disable-disk': disable_disk } - return config + + +def build_test_config(fs=default_fs, num_peers=default_peers, cache_size=default_cache, + test='upload', build='aio', profile='', hash_threads=1, torrent='test.torrent', + disable_disk=False): + config = {'test': test, 'save-path': os.path.join('./', fs), 'num-peers': num_peers, + 'cache-size': cache_size, 'build': build, 'profile': profile, + 'hash-threads': hash_threads, 'torrent': torrent, 'disable-disk': disable_disk} + return config + def prefix_len(text, prefix): - for i in xrange(1, len(prefix)): - if (not text.startswith(prefix[0:i])): return i-1 - return len(prefix) + for i in range(1, len(prefix)): + if (not text.startswith(prefix[0:i])): + return i - 1 + return len(prefix) + def device_name(path): - mount = subprocess.Popen('mount', stdout=subprocess.PIPE) + mount = subprocess.Popen('mount', stdout=subprocess.PIPE) - max_match_len = 0 - match_device = '' - path = os.path.abspath(path) + max_match_len = 0 + match_device = '' + path = os.path.abspath(path) - for mp in mount.stdout.readlines(): - c = mp.split(' ') - device = c[0] - mountpoint = c[2] - prefix = prefix_len(path, mountpoint) - if prefix > max_match_len: - max_match_len = prefix - match_device = device + for mp in mount.stdout.readlines(): + c = mp.split(' ') + device = c[0] + mountpoint = c[2] + prefix = prefix_len(path, mountpoint) + if prefix > max_match_len: + max_match_len = prefix + match_device = device + + device = match_device + device = device.split('/')[-1][0:3] + print('device for path: %s -> %s' % (path, device)) + return device - device = match_device - device = device.split('/')[-1][0:3] - print 'device for path: %s -> %s' % (path, device) - return device def build_target_folder(config): - test = 'seed' - if config['test'] == 'upload': test = 'download' - elif config['test'] == 'dual': test = 'dual' + test = 'seed' + if config['test'] == 'upload': + test = 'download' + elif config['test'] == 'dual': + test = 'dual' - if 'linux' in sys.platform: - io_scheduler = open('/sys/block/%s/queue/scheduler' % device_name(config['save-path'])).read().split('[')[1].split(']')[0] - else: - io_scheduler = sys.platform + if 'linux' in sys.platform: + io_scheduler = open('/sys/block/%s/queue/scheduler' % + device_name(config['save-path'])).read().split('[')[1].split(']')[0] + else: + io_scheduler = sys.platform - no_disk = '' - if config['disable-disk']: no_disk = '_no-disk' + no_disk = '' + if config['disable-disk']: + no_disk = '_no-disk' + + return 'results_%s_%s_%d_%d_%s_%s_h%d%s' % (config['build'], + test, + config['num-peers'], + config['cache-size'], + os.path.split( + config['save-path'])[1], + io_scheduler, + config['hash-threads'], + no_disk) - return 'results_%s_%s_%d_%d_%s_%s_h%d%s' % (config['build'], test, config['num-peers'], \ - config['cache-size'], os.path.split(config['save-path'])[1], io_scheduler, \ - config['hash-threads'], no_disk) def find_library(name): - paths = ['/usr/lib64/', '/usr/local/lib64/', '/usr/lib/', '/usr/local/lib/'] + paths = ['/usr/lib64/', '/usr/local/lib64/', '/usr/lib/', '/usr/local/lib/'] + + for p in paths: + try: + if os.path.exists(p + name): + return p + name + except BaseException: + pass + return name - for p in paths: - try: - if os.path.exists(p + name): return p + name - except: pass - return name def find_binary(names): - paths = ['/usr/bin/', '/usr/local/bin/'] - for n in names: - for p in paths: - try: - if os.path.exists(p + n): return p + n - except: pass - return names[0] + paths = ['/usr/bin/', '/usr/local/bin/'] + for n in names: + for p in paths: + try: + if os.path.exists(p + n): + return p + n + except BaseException: + pass + return names[0] + def run_test(config): - target_folder = build_target_folder(config) - if os.path.exists(target_folder): - print 'results already exists, skipping test (%s)' % target_folder - return + target_folder = build_target_folder(config) + if os.path.exists(target_folder): + print('results already exists, skipping test (%s)' % target_folder) + return - print '\n\n*********************************' - print '* RUNNING TEST *' - print '*********************************\n\n' - print '%s %s' % (config['build'], config['test']) + print('\n\n*********************************') + print('* RUNNING TEST *') + print('*********************************\n\n') + print('%s %s' % (config['build'], config['test'])) - # make sure any previous test file is removed - # don't clean up unless we're running a download-test, so that we leave the test file - # complete for a seed test. - delete_files(['utorrent_session/settings.dat', 'utorrent_session/settings.dat.old', 'asserts.log']) - if config['test'] == 'upload' or config['test'] == 'dual': - print 'deleting files' - delete_files([os.path.join(config['save-path'], 'stress_test_file'), '.ses_state', os.path.join(config['save-path'], '.resume'), 'utorrent_session', '.dht_state', 'session_stats', 'rtorrent_session']) + # make sure any previous test file is removed + # don't clean up unless we're running a download-test, so that we leave the test file + # complete for a seed test. + delete_files(['utorrent_session/settings.dat', 'utorrent_session/settings.dat.old', 'asserts.log']) + if config['test'] == 'upload' or config['test'] == 'dual': + print('deleting files') + delete_files([os.path.join(config['save-path'], + 'stress_test_file'), + '.ses_state', + os.path.join(config['save-path'], + '.resume'), + 'utorrent_session', + '.dht_state', + 'session_stats', + 'rtorrent_session']) - try: os.mkdir('session_stats') - except: pass + try: + os.mkdir('session_stats') + except BaseException: + pass - # save off the command line for reference - global port - cmdline = build_commandline(config, port) - binary = cmdline.split(' ')[0] - environment = None - if config['profile'] == 'tcmalloc': environment = {'LD_PRELOAD':find_library('libprofiler.so.0'), 'CPUPROFILE': 'session_stats/cpu_profile.prof'} - if config['profile'] == 'memory': environment = {'LD_PRELOAD':find_library('libprofiler.so.0'), 'HEAPPROFILE': 'session_stats/heap_profile.prof'} - if config['profile'] == 'perf': cmdline = 'perf timechart record --call-graph --output=session_stats/perf_profile.prof ' + cmdline - f = open('session_stats/cmdline.txt', 'w+') - f.write(cmdline) - f.close() + # save off the command line for reference + global port + cmdline = build_commandline(config, port) + binary = cmdline.split(' ')[0] + environment = None + if config['profile'] == 'tcmalloc': + environment = {'LD_PRELOAD': find_library('libprofiler.so.0'), 'CPUPROFILE': 'session_stats/cpu_profile.prof'} + if config['profile'] == 'memory': + environment = {'LD_PRELOAD': find_library('libprofiler.so.0'), 'HEAPPROFILE': 'session_stats/heap_profile.prof'} + if config['profile'] == 'perf': + cmdline = 'perf timechart record --call-graph --output=session_stats/perf_profile.prof ' + cmdline + f = open('session_stats/cmdline.txt', 'w+') + f.write(cmdline) + f.close() - f = open('session_stats/config.txt', 'w+') - print >>f, config - f.close() + f = open('session_stats/config.txt', 'w+') + print(config, file=f) + f.close() - print 'clearing disk cache' - clear_caches() - print 'OK' - client_output = open('session_stats/client.output', 'w+') - client_error = open('session_stats/client.error', 'w+') - print 'launching: %s' % cmdline - client = subprocess.Popen(shlex.split(cmdline), stdout=client_output, stdin=subprocess.PIPE, stderr=client_error, env=environment) - print 'OK' - # enable disk stats printing - if config['build'] != 'rtorrent' and config['build'] != 'utorrent': - print >>client.stdin, 'x', - time.sleep(4) - cmdline = './stage_aio/connection_tester %s %d 127.0.0.1 %d %s' % (config['test'], config['num-peers'], port, config['torrent']) - print 'launching: %s' % cmdline - tester_output = open('session_stats/tester.output', 'w+') - tester = subprocess.Popen(shlex.split(cmdline), stdout=tester_output) - print 'OK' + print('clearing disk cache') + clear_caches() + print('OK') + client_output = open('session_stats/client.output', 'w+') + client_error = open('session_stats/client.error', 'w+') + print('launching: %s' % cmdline) + client = subprocess.Popen( + shlex.split(cmdline), + stdout=client_output, + stdin=subprocess.PIPE, + stderr=client_error, + env=environment) + print('OK') + # enable disk stats printing + if config['build'] != 'rtorrent' and config['build'] != 'utorrent': + print('x', end=' ', file=client.stdin) + time.sleep(4) + cmdline = './stage_aio/connection_tester %s %d 127.0.0.1 %d %s' % ( + config['test'], config['num-peers'], port, config['torrent']) + print('launching: %s' % cmdline) + tester_output = open('session_stats/tester.output', 'w+') + tester = subprocess.Popen(shlex.split(cmdline), stdout=tester_output) + print('OK') - time.sleep(2) + time.sleep(2) - print '\n' - i = 0 - while True: - time.sleep(1) - tester.poll() - if tester.returncode != None: - print 'tester terminated' - break - client.poll() - if client.returncode != None: - print 'client terminated' - break - print '\r%d / %d' % (i, test_duration), - sys.stdout.flush() - i += 1 - if config['test'] != 'upload' and config['test'] != 'dual' and i >= test_duration: break - print '\n' + print('\n') + i = 0 + while True: + time.sleep(1) + tester.poll() + if tester.returncode is not None: + print('tester terminated') + break + client.poll() + if client.returncode is not None: + print('client terminated') + break + print('\r%d / %d' % (i, test_duration), end=' ') + sys.stdout.flush() + i += 1 + if config['test'] != 'upload' and config['test'] != 'dual' and i >= test_duration: + break + print('\n') - if client.returncode == None: - try: - print 'killing client' - client.send_signal(signal.SIGINT) - except: - pass + if client.returncode is None: + try: + print('killing client') + client.send_signal(signal.SIGINT) + except BaseException: + pass - time.sleep(10) - client.wait() - tester.wait() - tester_output.close() - client_output.close() - terminate = False - if tester.returncode != 0: - print 'tester returned %d' % tester.returncode - terminate = True - if client.returncode != 0: - print 'client returned %d' % client.returncode - terminate = True + time.sleep(10) + client.wait() + tester.wait() + tester_output.close() + client_output.close() + terminate = False + if tester.returncode != 0: + print('tester returned %d' % tester.returncode) + terminate = True + if client.returncode != 0: + print('client returned %d' % client.returncode) + terminate = True - try: shutil.copy('asserts.log', 'session_stats/') - except: pass + try: + shutil.copy('asserts.log', 'session_stats/') + except BaseException: + pass - try: shutil.move('libtorrent_logs0', 'session_stats/') - except: pass - try: shutil.move('libtorrent_logs%s' % port, 'session_stats/') - except: pass + try: + shutil.move('libtorrent_logs0', 'session_stats/') + except BaseException: + pass + try: + shutil.move('libtorrent_logs%s' % port, 'session_stats/') + except BaseException: + pass - # run fragmentation test - print 'analyzing fragmentation' - os.system('./stage_aio/fragmentation_test test.torrent %s' % (config['save-path'])) - try: shutil.copy('fragmentation.log', 'session_stats/') - except: pass + # run fragmentation test + print('analyzing fragmentation') + os.system('./stage_aio/fragmentation_test test.torrent %s' % (config['save-path'])) + try: + shutil.copy('fragmentation.log', 'session_stats/') + except BaseException: + pass - shutil.copy('fragmentation.gnuplot', 'session_stats/') - try: shutil.copy('file_access.log', 'session_stats/') - except: pass + shutil.copy('fragmentation.gnuplot', 'session_stats/') + try: + shutil.copy('file_access.log', 'session_stats/') + except BaseException: + pass - os.system('filefrag %s >session_stats/filefrag.out' % config['save-path']) - os.system('filefrag -v %s >session_stats/filefrag_verbose.out' % config['save-path']) + os.system('filefrag %s >session_stats/filefrag.out' % config['save-path']) + os.system('filefrag -v %s >session_stats/filefrag_verbose.out' % config['save-path']) - os.chdir('session_stats') + os.chdir('session_stats') - # parse session stats - print 'parsing session log' - os.system('python ../../parse_session_stats.py *.0000.log') - os.system('../stage_aio/parse_access_log file_access.log %s' % (os.path.join('..', config['save-path'], 'stress_test_file'))) + # parse session stats + print('parsing session log') + os.system('python ../../parse_session_stats.py *.0000.log') + os.system('../stage_aio/parse_access_log file_access.log %s' % + (os.path.join('..', config['save-path'], 'stress_test_file'))) - os.chdir('..') + os.chdir('..') - if config['profile'] == 'tcmalloc': - print 'analyzing CPU profile [%s]' % binary - os.system('%s --pdf %s session_stats/cpu_profile.prof >session_stats/cpu_profile.pdf' % (find_binary(['google-pprof', 'pprof']), binary)) - if config['profile'] == 'memory': - for i in xrange(1, 300): - profile = 'session_stats/heap_profile.prof.%04d.heap' % i - try: os.stat(profile) - except: break - print 'analyzing heap profile [%s] %d' % (binary, i) - os.system('%s --pdf %s %s >session_stats/heap_profile_%d.pdf' % (find_binary(['google-pprof', 'pprof']), binary, profile, i)) - if config['profile'] == 'perf': - print 'analyzing CPU profile [%s]' % binary - os.system('perf timechart --input=session_stats/perf_profile.prof --output=session_stats/profile_timechart.svg') - os.system('perf report --input=session_stats/perf_profile.prof --threads --show-nr-samples --vmlinux vmlinuz-2.6.38-8-generic.bzip >session_stats/profile.txt') + if config['profile'] == 'tcmalloc': + print('analyzing CPU profile [%s]' % binary) + os.system('%s --pdf %s session_stats/cpu_profile.prof >session_stats/cpu_profile.pdf' % + (find_binary(['google-pprof', 'pprof']), binary)) + if config['profile'] == 'memory': + for i in range(1, 300): + profile = 'session_stats/heap_profile.prof.%04d.heap' % i + try: + os.stat(profile) + except BaseException: + break + print('analyzing heap profile [%s] %d' % (binary, i)) + os.system('%s --pdf %s %s >session_stats/heap_profile_%d.pdf' % + (find_binary(['google-pprof', 'pprof']), binary, profile, i)) + if config['profile'] == 'perf': + print('analyzing CPU profile [%s]' % binary) + os.system('perf timechart --input=session_stats/perf_profile.prof --output=session_stats/profile_timechart.svg') + os.system(('perf report --input=session_stats/perf_profile.prof --threads --show-nr-samples ' + '--vmlinux vmlinuz-2.6.38-8-generic.bzip >session_stats/profile.txt')) - # move the results into its final place - print 'saving results' - os.rename('session_stats', build_target_folder(config)) + # move the results into its final place + print('saving results') + os.rename('session_stats', build_target_folder(config)) - port += 1 + port += 1 + + if terminate: + sys.exit(1) - if terminate: sys.exit(1) for h in range(0, 7): - config = build_test_config(num_peers=30, build='aio', test='upload', torrent='test.torrent', hash_threads=h, disable_disk=True) - run_test(config) + config = build_test_config( + num_peers=30, + build='aio', + test='upload', + torrent='test.torrent', + hash_threads=h, + disable_disk=True) + run_test(config) sys.exit(0) for b in ['aio', 'syncio']: - for test in ['dual', 'upload', 'download']: - config = build_test_config(build=b, test=test) - run_test(config) + for test in ['dual', 'upload', 'download']: + config = build_test_config(build=b, test=test) + run_test(config) sys.exit(0) for b in builds: - for test in ['upload', 'download']: - config = build_test_config(build=b, test=test) - run_test(config) + for test in ['upload', 'download']: + config = build_test_config(build=b, test=test) + run_test(config) for p in peers: - for test in ['upload', 'download']: - config = build_test_config(num_peers=p, test=test) - run_test(config) + for test in ['upload', 'download']: + config = build_test_config(num_peers=p, test=test) + run_test(config) for c in cache_sizes: - for test in ['upload', 'download']: - config = build_test_config(cache_size=c, test=test) - run_test(config) + for test in ['upload', 'download']: + config = build_test_config(cache_size=c, test=test) + run_test(config) for fs in filesystem: - for test in ['upload', 'download']: - config = build_test_config(fs=fs, test=test) - run_test(config) - + for test in ['upload', 'download']: + config = build_test_config(fs=fs, test=test) + run_test(config) diff --git a/test/http.py b/test/http.py index 0e3efe025..26c197e0f 100755 --- a/test/http.py +++ b/test/http.py @@ -1,28 +1,28 @@ # -*- coding: cp1252 -*- # # -#Copyright (c) <2009> +# Copyright (c) <2009> # -#Permission is hereby granted, free of charge, to any person -#obtaining a copy of this software and associated documentation -#files (the "Software"), to deal in the Software without -#restriction, including without limitation the rights to use, -#copy, modify, merge, publish, distribute, sublicense, and/or sell -#copies of the Software, and to permit persons to whom the -#Software is furnished to do so, subject to the following -#conditions: +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: # -#The above copyright notice and this permission notice shall be -#included in all copies or substantial portions of the Software. +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. # -#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -#EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -#OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -#NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -#HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -#WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -#FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -#OTHER DEALINGS IN THE SOFTWARE. +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. """\ Copyright (c) <2009> @@ -81,16 +81,28 @@ Qual a diferença entre um proxy Elite, Anónimo e Transparente? """ -import socket, thread, select, sys, base64, time, errno +import socket +import select +import sys +import base64 +import time +import errno + +# Python 3 renamed thread module to _thread +try: + import _thread as thread +except BaseException: + import thread __version__ = '0.1.0 Draft 1' BUFLEN = 8192 -VERSION = 'Python Proxy/'+__version__ +VERSION = 'Python Proxy/' + __version__ HTTPVER = 'HTTP/1.1' username = None password = None + class ConnectionHandler: def __init__(self, connection, address, timeout): self.client = connection @@ -99,26 +111,26 @@ class ConnectionHandler: self.method, self.path, self.protocol = self.get_base_header() global username global password - if username != None: + if username is not None: auth = base64.b64encode(username + ':' + password) if not 'Proxy-Authorization: Basic ' + auth in self.client_buffer: - print 'PROXY - failed authentication: %s' % self.client_buffer - self.client.send(HTTPVER+' 401 Authentication Failed\n'+ - 'Proxy-agent: %s\n\n'%VERSION) + print('PROXY - failed authentication: %s' % self.client_buffer) + self.client.send(HTTPVER + ' 401 Authentication Failed\n' + + 'Proxy-agent: %s\n\n' % VERSION) self.client.close() return try: if self.method == 'CONNECT': - self.method_CONNECT() + self.method_CONNECT() elif self.method in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE'): self.method_others() - except: + except BaseException: try: - self.client.send(HTTPVER+' 502 Connection failed\n'+ - 'Proxy-agent: %s\n\n'%VERSION) - except Exception, e: - print 'PROXY - ', e + self.client.send(HTTPVER + ' 502 Connection failed\n' + + 'Proxy-agent: %s\n\n' % VERSION) + except Exception as e: + print('PROXY - ', e) self.client.close() return @@ -127,10 +139,10 @@ class ConnectionHandler: def get_base_header(self): retries = 0 - while 1: + while True: try: self.client_buffer += self.client.recv(BUFLEN) - except socket.error, e: + except socket.error as e: err = e.args[0] if (err == errno.EAGAIN or err == errno.EWOULDBLOCK) and retries < 20: time.sleep(0.5) @@ -138,18 +150,18 @@ class ConnectionHandler: continue raise e end = self.client_buffer.find('\r\n\r\n') - if end!=-1: + if end != -1: break line_end = self.client_buffer.find('\n') - print 'PROXY - %s' % self.client_buffer[:line_end]#debug - data = (self.client_buffer[:line_end+1]).split() - self.client_buffer = self.client_buffer[line_end+1:] + print('PROXY - %s' % self.client_buffer[:line_end]) # debug + data = (self.client_buffer[:line_end + 1]).split() + self.client_buffer = self.client_buffer[line_end + 1:] return data def method_CONNECT(self): self._connect_target(self.path) - self.client.send(HTTPVER+' 200 Connection established\n'+ - 'Proxy-agent: %s\n\n'%VERSION) + self.client.send(HTTPVER + ' 200 Connection established\n' + + 'Proxy-agent: %s\n\n' % VERSION) self.client_buffer = '' self._read_write() @@ -159,15 +171,15 @@ class ConnectionHandler: host = self.path[:i] path = self.path[i:] self._connect_target(host) - self.target.send('%s %s %s\n' % (self.method, path, self.protocol)+ + self.target.send('%s %s %s\n' % (self.method, path, self.protocol) + self.client_buffer) self.client_buffer = '' self._read_write() def _connect_target(self, host): i = host.find(':') - if i!=-1: - port = int(host[i+1:]) + if i != -1: + port = int(host[i + 1:]) host = host[:i] else: port = 80 @@ -176,10 +188,10 @@ class ConnectionHandler: self.target.connect(address) def _read_write(self): - time_out_max = self.timeout/3 + time_out_max = self.timeout / 3 socs = [self.client, self.target] count = 0 - while 1: + while True: count += 1 (recv, _, error) = select.select(socs, [], socs, 3) if error: @@ -197,37 +209,39 @@ class ConnectionHandler: if count == time_out_max: break + def start_server(host='localhost', port=8080, IPv6=False, timeout=100, - handler=ConnectionHandler): - if IPv6==True: - soc_type=socket.AF_INET6 + handler=ConnectionHandler): + if IPv6: + soc_type = socket.AF_INET6 else: - soc_type=socket.AF_INET + soc_type = socket.AF_INET soc = socket.socket(soc_type) soc.settimeout(120) - print "PROXY - Serving on %s:%d."%(host, port)#debug + print("PROXY - Serving on %s:%d." % (host, port)) # debug soc.bind((host, port)) soc.listen(0) - while 1: - thread.start_new_thread(handler, soc.accept()+(timeout,)) + while True: + thread.start_new_thread(handler, soc.accept() + (timeout,)) + if __name__ == '__main__': listen_port = 8080 i = 1 while i < len(sys.argv): if sys.argv[i] == '--port': - listen_port = int(sys.argv[i+1]) + listen_port = int(sys.argv[i + 1]) i += 1 elif sys.argv[i] == '--username': - username = sys.argv[i+1] + username = sys.argv[i + 1] i += 1 elif sys.argv[i] == '--password': - password = sys.argv[i+1] + password = sys.argv[i + 1] i += 1 else: - if sys.argv[i] != '--help': print('PROXY - unknown option "%s"' % sys.argv[i]) + if sys.argv[i] != '--help': + print(('PROXY - unknown option "%s"' % sys.argv[i])) print('usage: http.py [--port ]') sys.exit(1) i += 1 start_server(port=listen_port) - diff --git a/test/socks.py b/test/socks.py index 2ce316d4d..461fd0fd2 100755 --- a/test/socks.py +++ b/test/socks.py @@ -1,19 +1,27 @@ #!/usr/bin/env python """Minimal non-feature complete socks proxy""" +from __future__ import print_function -import random import socket -from SocketServer import StreamRequestHandler, ThreadingTCPServer from struct import pack, unpack import threading import sys +# Python 3 renamed SocketServer to socketserver +try: + from socketserver import StreamRequestHandler, ThreadingTCPServer +except BaseException: + from SocketServer import StreamRequestHandler, ThreadingTCPServer + + def debug(s): - print >>sys.stderr, 'socks.py: ', s + print('socks.py: ', s, file=sys.stderr) + def error(s): - print >>sys.stderr, 'socks.py, ERROR: ', s + print('socks.py, ERROR: ', s, file=sys.stderr) + class MyTCPServer(ThreadingTCPServer): allow_reuse_address = True @@ -21,6 +29,7 @@ class MyTCPServer(ThreadingTCPServer): def handle_timeout(self): raise Exception('timeout') + CLOSE = object() VERSION = '\x05' @@ -37,15 +46,19 @@ password = None username = None allow_v4 = False + def send(dest, msg): if msg == CLOSE: - try: dest.shutdown(socket.SHUT_WR) - except: pass + try: + dest.shutdown(socket.SHUT_WR) + except BaseException: + pass dest.close() return 0 else: return dest.sendall(msg) + def recv(source, buffer): data = source.recv(buffer) if data == '': @@ -53,6 +66,7 @@ def recv(source, buffer): else: return data + def forward(source, dest, name): while True: data = recv(source, 4000) @@ -63,11 +77,13 @@ def forward(source, dest, name): # debug('Forwarding (%d) %r' % (len(data), data)) send(dest, data) + def spawn_forwarder(source, dest, name): t = threading.Thread(target=forward, args=(source, dest, name)) t.daemon = True t.start() + class SocksHandler(StreamRequestHandler): """Highly feature incomplete SOCKS 5 implementation""" @@ -109,7 +125,7 @@ class SocksHandler(StreamRequestHandler): c = self.read(1) outbound_sock = socket.socket(socket.AF_INET) - out_address = socket.getaddrinfo(dest_address,dest_port)[0][4] + out_address = socket.getaddrinfo(dest_address, dest_port)[0][4] debug("Creating forwarder connection to %s:%d" % (out_address[0], out_address[1])) outbound_sock.connect(out_address) @@ -130,7 +146,7 @@ class SocksHandler(StreamRequestHandler): global password global username - if password == None and NOAUTH in method_list: + if password is None and NOAUTH in method_list: self.send_no_auth_method() debug('Authenticated (no-auth)') elif USERPASS in method_list: @@ -173,7 +189,7 @@ class SocksHandler(StreamRequestHandler): dest_address = '.'.join(map(str, unpack('>4B', raw_dest_address))) elif address_type == IPV6: raw_dest_address = self.read(16) - dest_address = ":".join(map(lambda x: hex(x)[2:],unpack('>8H',raw_dest_address))) + dest_address = ":".join([hex(x)[2:] for x in unpack('>8H', raw_dest_address)]) elif address_type == DOMAIN_NAME: dns_length = ord(self.read(1)) dns_name = self.read(dns_length) @@ -190,21 +206,21 @@ class SocksHandler(StreamRequestHandler): else: outbound_sock = socket.socket(socket.AF_INET) try: - out_address = socket.getaddrinfo(dest_address,dest_port)[0][4] - except Exception, e: - print e + out_address = socket.getaddrinfo(dest_address, dest_port)[0][4] + except Exception as e: + print(e) return if cmd == UDP_ASSOCIATE: - debug("no UDP support yet, closing") - return; + debug("no UDP support yet, closing") + return debug("Creating forwarder connection to %s:%d" % (out_address[0], out_address[1])) try: outbound_sock.connect(out_address) - except Exception, e: - print e + except Exception as e: + print(e) return if address_type == IPV6: @@ -215,22 +231,25 @@ class SocksHandler(StreamRequestHandler): spawn_forwarder(outbound_sock, self.request, 'destination') try: forward(self.request, outbound_sock, 'client') - except Exception,e: - print e + except Exception as e: + print(e) - def send_reply_v4(self, (bind_addr, bind_port)): + def send_reply_v4(self, xxx_todo_changeme): + (bind_addr, bind_port) = xxx_todo_changeme self.wfile.write('\0\x5a\0\0\0\0\0\0') self.wfile.flush() - def send_reply(self, (bind_addr, bind_port)): + def send_reply(self, xxx_todo_changeme1): + (bind_addr, bind_port) = xxx_todo_changeme1 bind_tuple = tuple(map(int, bind_addr.split('.'))) full_address = bind_tuple + (bind_port,) debug('Setting up forwarding port %r' % (full_address,)) msg = pack('>cccc4BH', VERSION, SUCCESS, '\x00', IPV4, *full_address) self.wfile.write(msg) - def send_reply6(self, (bind_addr, bind_port, unused1, unused2)): - bind_tuple = tuple(map(lambda x: int(x,16), bind_addr.split(':'))) + def send_reply6(self, xxx_todo_changeme2): + (bind_addr, bind_port, unused1, unused2) = xxx_todo_changeme2 + bind_tuple = tuple([int(x, 16) for x in bind_addr.split(':')]) full_address = bind_tuple + (bind_port,) debug('Setting up forwarding port %r' % (full_address,)) msg = pack('>cccc8HH', VERSION, SUCCESS, '\x00', IPV6, *full_address) @@ -252,24 +271,26 @@ class SocksHandler(StreamRequestHandler): self.wfile.write('\x01\x00') self.wfile.flush() + if __name__ == '__main__': listen_port = 8002 i = 1 while i < len(sys.argv): if sys.argv[i] == '--username': - username = sys.argv[i+1] + username = sys.argv[i + 1] i += 1 elif sys.argv[i] == '--password': - password = sys.argv[i+1] + password = sys.argv[i + 1] i += 1 elif sys.argv[i] == '--port': - listen_port = int(sys.argv[i+1]) + listen_port = int(sys.argv[i + 1]) i += 1 elif sys.argv[i] == '--allow-v4': allow_v4 = True else: - if sys.argv[i] != '--help': debug('unknown option "%s"' % sys.argv[i]) + if sys.argv[i] != '--help': + debug('unknown option "%s"' % sys.argv[i]) print('usage: socks.py [--username --password ] [--port ]') sys.exit(1) i += 1 @@ -279,4 +300,3 @@ if __name__ == '__main__': server.timeout = 190 while True: server.handle_request() - diff --git a/test/web_server.py b/test/web_server.py index b0c716200..abbe327c9 100644 --- a/test/web_server.py +++ b/test/web_server.py @@ -1,194 +1,205 @@ #!/usr/bin/env python -import BaseHTTPServer -import SimpleHTTPServer import sys import os import ssl import gzip import base64 +# Python 3 has moved {Simple,Base}HTTPServer to http module +try: + # Remove '.' from sys.path or we try to import the http.py module + # which is not what we want. + sys.path = sys.path[1:] + from http.server import HTTPServer, BaseHTTPRequestHandler +except ImportError: + from SimpleHTTPServer import SimpleHTTPRequestHandler as BaseHTTPRequestHandler + from BaseHTTPServer import HTTPServer + chunked_encoding = False keepalive = True try: - fin = open('test_file', 'rb') - f = gzip.open('test_file.gz', 'wb') - f.writelines(fin) - f.close() - fin.close() -except: - pass + fin = open('test_file', 'rb') + f = gzip.open('test_file.gz', 'wb') + f.writelines(fin) + f.close() + fin.close() +except BaseException: + pass -class http_server_with_timeout(BaseHTTPServer.HTTPServer): - allow_reuse_address = True - timeout = 190 - def handle_timeout(self): - raise Exception('timeout') +class http_server_with_timeout(HTTPServer): + allow_reuse_address = True + timeout = 190 -class http_handler(SimpleHTTPServer.SimpleHTTPRequestHandler): + def handle_timeout(self): + raise Exception('timeout') - def do_GET(s): - print 'INCOMING-REQUEST: ', s.requestline - print s.headers +class http_handler(BaseHTTPRequestHandler): - global chunked_encoding - global keepalive + def do_GET(s): - # if the request contains the hostname and port. strip it - if s.path.startswith('http://') or s.path.startswith('https://'): - s.path = s.path[8:] - s.path = s.path[s.path.find('/'):] + print('INCOMING-REQUEST: ', s.requestline) + print(s.headers) - file_path = os.path.normpath(s.path) - print file_path - print s.path + global chunked_encoding + global keepalive - if s.path == '/password_protected': - passed = False - if 'Authorization' in s.headers: - auth = s.headers['Authorization'] - passed = auth == 'Basic %s' % base64.b64encode('testuser:testpass') + # if the request contains the hostname and port. strip it + if s.path.startswith('http://') or s.path.startswith('https://'): + s.path = s.path[8:] + s.path = s.path[s.path.find('/'):] - if not passed: - s.send_response(401) - s.send_header("Connection", "close") - s.end_headers() - return + file_path = os.path.normpath(s.path) + print(file_path) + print(s.path) - s.path = '/test_file' - file_path = os.path.normpath('/test_file') + if s.path == '/password_protected': + passed = False + if 'Authorization' in s.headers: + auth = s.headers['Authorization'] + passed = auth == 'Basic %s' % base64.b64encode('testuser:testpass') - if s.path == '/redirect': - s.send_response(301) - s.send_header("Location", "/test_file") - s.send_header("Connection", "close") - s.end_headers() - elif s.path == '/infinite_redirect': - s.send_response(301) - s.send_header("Location", "/infinite_redirect") - s.send_header("Connection", "close") - s.end_headers() - elif s.path == '/relative/redirect': - s.send_response(301) - s.send_header("Location", "../test_file") - s.send_header("Connection", "close") - s.end_headers() - elif s.path.startswith('/announce'): - s.send_response(200) - response = 'd8:intervali1800e8:completei1e10:incompletei1e' + \ - '12:min intervali' + min_interval + 'e' + \ - '5:peers12:AAAABBCCCCDD' + \ - '6:peers618:EEEEEEEEEEEEEEEEFF' + \ - 'e' - s.send_header("Content-Length", "%d" % len(response)) - s.send_header("Connection", "close") - s.end_headers() - s.wfile.write(response) - elif os.path.split(s.path)[1].startswith('seed?'): - query = s.path[6:] - args_raw = query.split('&') - args = {} - for a in args_raw: - kvp = a.split('=') - args[kvp[0]] = kvp[1] - piece = int(args['piece']) - ranges = args['ranges'].split('-') + if not passed: + s.send_response(401) + s.send_header("Connection", "close") + s.end_headers() + return - filename = '' - try: - filename = os.path.normpath(s.path[1:s.path.find('seed?') + 4]) - print 'filename = %s' % filename - f = open(filename, 'rb') - f.seek(piece * 32 * 1024 + int(ranges[0])) - data = f.read(int(ranges[1]) - int(ranges[0]) + 1) - f.close() + s.path = '/test_file' + file_path = os.path.normpath('/test_file') - s.send_response(200) - print 'sending %d bytes' % len(data) - s.send_header("Content-Length", "%d" % len(data)) - s.end_headers() - s.wfile.write(data); - except Exception, e: - print 'FILE ERROR: ', filename, e - s.send_response(404) - s.send_header("Content-Length", "0") - s.end_headers() - else: - filename = '' - try: - filename = os.path.normpath(file_path[1:]) - # serve file by invoking default handler - f = open(filename, 'rb') - size = int(os.stat(filename).st_size) - start_range = 0 - end_range = size - if 'Range' in s.headers: - s.send_response(206) - st, e = s.headers['range'][6:].split('-', 1) - sl = len(st) - el = len(e) - if sl > 0: - start_range = int(st) - if el > 0: - end_range = int(e) + 1 - elif el > 0: - ei = int(e) - if ei < size: - start_range = size - ei - s.send_header('Content-Range', 'bytes ' + str(start_range) \ - + '-' + str(end_range - 1) + '/' + str(size)) - else: - s.send_response(200) - s.send_header('Accept-Ranges', 'bytes') - if chunked_encoding: - s.send_header('Transfer-Encoding', 'chunked') - s.send_header('Content-Length', end_range - start_range) - if filename.endswith('.gz'): - s.send_header('Content-Encoding', 'gzip') - if not keepalive: - s.send_header("Connection", "close") - try: - s.request.shutdown(); - except Exception, e: - print 'Failed to shutdown read-channel of socket: ', e + if s.path == '/redirect': + s.send_response(301) + s.send_header("Location", "/test_file") + s.send_header("Connection", "close") + s.end_headers() + elif s.path == '/infinite_redirect': + s.send_response(301) + s.send_header("Location", "/infinite_redirect") + s.send_header("Connection", "close") + s.end_headers() + elif s.path == '/relative/redirect': + s.send_response(301) + s.send_header("Location", "../test_file") + s.send_header("Connection", "close") + s.end_headers() + elif s.path.startswith('/announce'): + s.send_response(200) + response = 'd8:intervali1800e8:completei1e10:incompletei1e' + \ + '12:min intervali' + min_interval + 'e' + \ + '5:peers12:AAAABBCCCCDD' + \ + '6:peers618:EEEEEEEEEEEEEEEEFF' + \ + 'e' + s.send_header("Content-Length", "%d" % len(response)) + s.send_header("Connection", "close") + s.end_headers() + s.wfile.write(response) + elif os.path.split(s.path)[1].startswith('seed?'): + query = s.path[6:] + args_raw = query.split('&') + args = {} + for a in args_raw: + kvp = a.split('=') + args[kvp[0]] = kvp[1] + piece = int(args['piece']) + ranges = args['ranges'].split('-') - s.end_headers() + filename = '' + try: + filename = os.path.normpath(s.path[1:s.path.find('seed?') + 4]) + print('filename = %s' % filename) + f = open(filename, 'rb') + f.seek(piece * 32 * 1024 + int(ranges[0])) + data = f.read(int(ranges[1]) - int(ranges[0]) + 1) + f.close() + + s.send_response(200) + print('sending %d bytes' % len(data)) + s.send_header("Content-Length", "%d" % len(data)) + s.end_headers() + s.wfile.write(data) + except Exception as e: + print('FILE ERROR: ', filename, e) + s.send_response(404) + s.send_header("Content-Length", "0") + s.end_headers() + else: + filename = '' + try: + filename = os.path.normpath(file_path[1:]) + # serve file by invoking default handler + f = open(filename, 'rb') + size = int(os.stat(filename).st_size) + start_range = 0 + end_range = size + if 'Range' in s.headers: + s.send_response(206) + st, e = s.headers['range'][6:].split('-', 1) + sl = len(st) + el = len(e) + if sl > 0: + start_range = int(st) + if el > 0: + end_range = int(e) + 1 + elif el > 0: + ei = int(e) + if ei < size: + start_range = size - ei + s.send_header('Content-Range', 'bytes ' + str(start_range) + + '-' + str(end_range - 1) + '/' + str(size)) + else: + s.send_response(200) + s.send_header('Accept-Ranges', 'bytes') + if chunked_encoding: + s.send_header('Transfer-Encoding', 'chunked') + s.send_header('Content-Length', end_range - start_range) + if filename.endswith('.gz'): + s.send_header('Content-Encoding', 'gzip') + if not keepalive: + s.send_header("Connection", "close") + try: + s.request.shutdown() + except Exception as e: + print('Failed to shutdown read-channel of socket: ', e) + + s.end_headers() + + f.seek(start_range) + length = end_range - start_range + while length > 0: + to_send = min(length, 0x900) + if chunked_encoding: + s.wfile.write('%x\r\n' % to_send) + data = f.read(to_send) + print('read %d bytes' % to_send) + s.wfile.write(data) + if chunked_encoding: + s.wfile.write('\r\n') + length -= to_send + print('sent %d bytes (%d bytes left)' % (len(data), length)) + if chunked_encoding: + s.wfile.write('0\r\n\r\n') + except Exception as e: + print('FILE ERROR: ', filename, e) + s.send_response(404) + s.send_header("Content-Length", "0") + s.end_headers() - f.seek(start_range) - length = end_range - start_range - while length > 0: - to_send = min(length, 0x900) - if chunked_encoding: - s.wfile.write('%x\r\n' % to_send) - data = f.read(to_send) - print 'read %d bytes' % to_send - s.wfile.write(data) - if chunked_encoding: - s.wfile.write('\r\n') - length -= to_send - print 'sent %d bytes (%d bytes left)' % (len(data), length) - if chunked_encoding: - s.wfile.write('0\r\n\r\n') - except Exception, e: - print 'FILE ERROR: ', filename, e - s.send_response(404) - s.send_header("Content-Length", "0") - s.end_headers() if __name__ == '__main__': - port = int(sys.argv[1]) - chunked_encoding = sys.argv[2] != '0' - use_ssl = sys.argv[3] != '0' - keepalive = sys.argv[4] != '0' - min_interval = sys.argv[5] + port = int(sys.argv[1]) + chunked_encoding = sys.argv[2] != '0' + use_ssl = sys.argv[3] != '0' + keepalive = sys.argv[4] != '0' + min_interval = sys.argv[5] - http_handler.protocol_version = 'HTTP/1.1' - httpd = http_server_with_timeout(('127.0.0.1', port), http_handler) - if use_ssl: - httpd.socket = ssl.wrap_socket(httpd.socket, certfile='../ssl/server.pem', server_side=True) + http_handler.protocol_version = 'HTTP/1.1' + httpd = http_server_with_timeout(('127.0.0.1', port), http_handler) + if use_ssl: + httpd.socket = ssl.wrap_socket(httpd.socket, certfile='../ssl/server.pem', server_side=True) - while True: - httpd.handle_request() + while True: + httpd.handle_request() diff --git a/tools/clean.py b/tools/clean.py index bd1f624b4..a6f081ad3 100755 --- a/tools/clean.py +++ b/tools/clean.py @@ -4,78 +4,79 @@ import os import shutil import glob + def clean(): - to_delete = [ - 'session_stats', - 'libtorrent_logs*', - 'round_trip_ms.log', - 'dht.log', - 'upnp.log', - 'natpmp.log', - 'bin', - 'build-aux', - '.deps', - 'test_tmp_*', - 'bjam_build.*.xml' - '*.exe', - '*.pdb', - '*.pyd', - 'dist', - 'build', - '.libs', - '*.cpp.orig', - '*.cpp.rej', - '*.hpp.orig', - '*.hpp.rej', - '*.hpp.gcov', - '*.cpp.gcov', - 'Makefile.in', - 'Makefile', - 'lib*.a', - 'Jamfile.rej', - 'Jamfile.orig', - '*.o', - '*.lo', - 'autom4te.cache', - 'configure', - 'config.report', - 'config.log', - '.lib', - ] + to_delete = [ + 'session_stats', + 'libtorrent_logs*', + 'round_trip_ms.log', + 'dht.log', + 'upnp.log', + 'natpmp.log', + 'bin', + 'build-aux', + '.deps', + 'test_tmp_*', + 'bjam_build.*.xml' + '*.exe', + '*.pdb', + '*.pyd', + 'dist', + 'build', + '.libs', + '*.cpp.orig', + '*.cpp.rej', + '*.hpp.orig', + '*.hpp.rej', + '*.hpp.gcov', + '*.cpp.gcov', + 'Makefile.in', + 'Makefile', + 'lib*.a', + 'Jamfile.rej', + 'Jamfile.orig', + '*.o', + '*.lo', + 'autom4te.cache', + 'configure', + 'config.report', + 'config.log', + '.lib', + ] - directories = [ - 'examples', - 'test', - '.', - 'tools', - 'src', - 'simulation', - os.path.join('src', 'kademlia'), - os.path.join('include', 'libtorrent'), - os.path.join('include', os.path.join('libtorrent', '_aux')), - os.path.join('include', os.path.join('libtorrent', 'kademlia')), - os.path.join('bindings', 'python'), - os.path.join('bindings', os.path.join('python', 'src')), - os.path.join('bindings', 'c'), - os.path.join('bindings', os.path.join('c', 'src')), - os.path.join('simulation', 'libsimulator') - ] + directories = [ + 'examples', + 'test', + '.', + 'tools', + 'src', + 'simulation', + os.path.join('src', 'kademlia'), + os.path.join('include', 'libtorrent'), + os.path.join('include', os.path.join('libtorrent', '_aux')), + os.path.join('include', os.path.join('libtorrent', 'kademlia')), + os.path.join('bindings', 'python'), + os.path.join('bindings', os.path.join('python', 'src')), + os.path.join('bindings', 'c'), + os.path.join('bindings', os.path.join('c', 'src')), + os.path.join('simulation', 'libsimulator') + ] - for d in directories: - for f in to_delete: - path = os.path.join(d, f) - entries = glob.glob(path) - for p in entries: - try: - shutil.rmtree(p) - print p - except Exception, e: - try: - os.remove(p) - print p - except Exception, e: - print p, e + for d in directories: + for f in to_delete: + path = os.path.join(d, f) + entries = glob.glob(path) + for p in entries: + try: + shutil.rmtree(p) + print(p) + except Exception as e: + try: + os.remove(p) + print(p) + except Exception as e: + print(p, e) -if __name__ == "__main__": - clean() +if __name__ == "__main__": + clean() diff --git a/tools/dht_flood.py b/tools/dht_flood.py index 9e3336cd3..dd4a222b2 100755 --- a/tools/dht_flood.py +++ b/tools/dht_flood.py @@ -8,30 +8,36 @@ import random port = int(sys.argv[1]) # from BitTorrent 4.3.0 -def encode_bencached(x,r): + + +def encode_bencached(x, r): r.append(x.bencoded) + def encode_int(x, r): r.extend(('i', str(x), 'e')) + def encode_string(x, r): r.extend((str(len(x)), ':', x)) + def encode_list(x, r): r.append('l') for i in x: encode_func[type(i)](i, r) r.append('e') -def encode_dict(x,r): + +def encode_dict(x, r): r.append('d') - ilist = x.items() - ilist.sort() + ilist = sorted(x.items()) for k, v in ilist: r.extend((str(len(k)), ':', k)) encode_func[type(v)](v, r) r.append('e') + encode_func = {} encode_func[IntType] = encode_int encode_func[LongType] = encode_int @@ -40,29 +46,32 @@ encode_func[ListType] = encode_list encode_func[TupleType] = encode_list encode_func[DictType] = encode_dict + def bencode(x): r = [] encode_func[type(x)](x, r) return ''.join(r) + def send_dht_message(msg): - s.sendto(bencode(msg), 0, ('127.0.0.1', port)) + s.sendto(bencode(msg), 0, ('127.0.0.1', port)) + def random_key(): - ret = '' - for i in range(0, 20): - ret += chr(random.randint(0, 255)) - return ret + ret = '' + for i in range(0, 20): + ret += chr(random.randint(0, 255)) + return ret + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -node_id = '1' * 20; +node_id = '1' * 20 query = 'get_peers' -print 'test random info-hashes' -for i in xrange(1, 30000): - send_dht_message({'a': {'id': node_id, 'info_hash': random_key()}, 'q': query, 'y': 'q', 't': '%d' % i}) - -print 'test random peer-ids' -for i in xrange(1, 30000): - send_dht_message({'a': {'id': random_key(), 'info_hash': random_key()}, 'q': query, 'y': 'q', 't': '%d' % i}) +print('test random info-hashes') +for i in range(1, 30000): + send_dht_message({'a': {'id': node_id, 'info_hash': random_key()}, 'q': query, 'y': 'q', 't': '%d' % i}) +print('test random peer-ids') +for i in range(1, 30000): + send_dht_message({'a': {'id': random_key(), 'info_hash': random_key()}, 'q': query, 'y': 'q', 't': '%d' % i}) diff --git a/tools/gen_fwd.py b/tools/gen_fwd.py index d6ec6a093..9813801ae 100644 --- a/tools/gen_fwd.py +++ b/tools/gen_fwd.py @@ -1,9 +1,8 @@ #!/usr/bin/env python import os -import sys -file_header ='''/* +file_header = '''/* Copyright (c) 2017, Arvid Norberg All rights reserved. @@ -52,79 +51,83 @@ namespace lt = libtorrent; #endif // TORRENT_FWD_HPP ''' -classes = os.popen('git grep "\(TORRENT_EXPORT\|TORRENT_DEPRECATED_EXPORT\|^TORRENT_[A-Z0-9]\+_NAMESPACE\)"').read().split('\n') +classes = os.popen( + r'git grep "\(TORRENT_EXPORT\|TORRENT_DEPRECATED_EXPORT\|^TORRENT_[A-Z0-9]\+_NAMESPACE\)"').read().split('\n') + def print_classes(out, classes, keyword): - current_file = '' - ret = '' - dht_ret = '' + current_file = '' - # [(file, decl), ...] - classes = [(l.split(':')[0].strip(), ':'.join(l.split(':')[1:]).strip()) for l in classes] + # [(file, decl), ...] + classes = [(l.split(':')[0].strip(), ':'.join(l.split(':')[1:]).strip()) for l in classes] - # we only care about header files - # ignore the forward header itself, that's the one we're generating - # also ignore any header in the aux_ directory, those are private - classes = [l for l in classes if l[0].endswith('.hpp') and not l[0].endswith('/fwd.hpp') and '/aux_/' not in l[0]] + # we only care about header files + # ignore the forward header itself, that's the one we're generating + # also ignore any header in the aux_ directory, those are private + classes = [l for l in classes if l[0].endswith('.hpp') and not l[0].endswith('/fwd.hpp') and '/aux_/' not in l[0]] - namespaces = ['TORRENT_VERSION_NAMESPACE_2', 'TORRENT_IPV6_NAMESPACE', - 'TORRENT_VERSION_NAMESPACE_2_END', 'TORRENT_IPV6_NAMESPACE_END'] + namespaces = ['TORRENT_VERSION_NAMESPACE_2', 'TORRENT_IPV6_NAMESPACE', + 'TORRENT_VERSION_NAMESPACE_2_END', 'TORRENT_IPV6_NAMESPACE_END'] - # only include classes with the right kind of export - classes = [l for l in classes if l[1] in namespaces or (l[1].split(' ')[0] in ['class', 'struct'] and l[1].split(' ')[1] == keyword)] + # only include classes with the right kind of export + classes = [ + l for l in classes if l[1] in namespaces or ( + l[1].split(' ')[0] in [ + 'class', + 'struct'] and l[1].split(' ')[1] == keyword)] - # collapse empty namespaces - classes2 = [] - skip = 0 - for i in xrange(len(classes)): - if skip > 0: - skip -= 1 - continue - if classes[i][1] in namespaces \ - and len(classes) > i+1 \ - and classes[i+1][1] == ('%s_END' % classes[i][1]): - skip = 1 - else: - classes2.append(classes[i]) + # collapse empty namespaces + classes2 = [] + skip = 0 + for i in range(len(classes)): + if skip > 0: + skip -= 1 + continue + if classes[i][1] in namespaces \ + and len(classes) > i + 1 \ + and classes[i + 1][1] == ('%s_END' % classes[i][1]): + skip = 1 + else: + classes2.append(classes[i]) - classes = classes2 + classes = classes2 - idx = -1 - for line in classes: - idx += 1 - this_file = line[0] - decl = line[1].split(' ') + idx = -1 + for line in classes: + idx += 1 + this_file = line[0] + decl = line[1].split(' ') - content = '' - if this_file != current_file: - out.write('\n// ' + this_file + '\n') - current_file = this_file; - if len(decl) > 2 and decl[0] in ['struct', 'class']: - decl = decl[0] + ' ' + decl[2] - if not decl.endswith(';'): decl += ';' - content = decl + '\n' - else: - content = line[1] + '\n' + content = '' + if this_file != current_file: + out.write('\n// ' + this_file + '\n') + current_file = this_file + if len(decl) > 2 and decl[0] in ['struct', 'class']: + decl = decl[0] + ' ' + decl[2] + if not decl.endswith(';'): + decl += ';' + content = decl + '\n' + else: + content = line[1] + '\n' + + if 'kademlia' in this_file: + out.write('namespace dht {\n') + out.write(content) + out.write('}\n') + else: + out.write(content) - if 'kademlia' in this_file: - out.write('namespace dht {\n') - out.write(content) - out.write('}\n') - else: - out.write(content) os.remove('include/libtorrent/fwd.hpp') with open('include/libtorrent/fwd.hpp', 'w+') as f: - f.write(file_header) + f.write(file_header) - print_classes(f, classes, 'TORRENT_EXPORT'); + print_classes(f, classes, 'TORRENT_EXPORT') - f.write('\n#if TORRENT_ABI_VERSION == 1\n') + f.write('\n#if TORRENT_ABI_VERSION == 1\n') - print_classes(f, classes, 'TORRENT_DEPRECATED_EXPORT'); - - f.write('\n#endif // TORRENT_ABI_VERSION') - - f.write(file_footer) + print_classes(f, classes, 'TORRENT_DEPRECATED_EXPORT') + f.write('\n#endif // TORRENT_ABI_VERSION') + f.write(file_footer) diff --git a/tools/gprof2dot.py b/tools/gprof2dot.py index 64950124b..6b74cc781 100755 --- a/tools/gprof2dot.py +++ b/tools/gprof2dot.py @@ -35,53 +35,57 @@ import locale # Python 2.x/3.x compatibility if sys.version_info[0] >= 3: PYTHON_3 = True + def compat_iteritems(x): return x.items() # No iteritems() in Python 3 + def compat_itervalues(x): return x.values() # No itervalues() in Python 3 + def compat_keys(x): return list(x.keys()) # keys() is a generator in Python 3 basestring = str # No class basestring in Python 3 - unichr = chr # No unichr in Python 3 - xrange = range # No xrange in Python 3 + unichr = chr # No unichr in Python 3 + xrange = range # No xrange in Python 3 else: PYTHON_3 = False + def compat_iteritems(x): return x.iteritems() + def compat_itervalues(x): return x.itervalues() + def compat_keys(x): return x.keys() -try: - # Debugging helper module - import debug -except ImportError: - pass - - MULTIPLICATION_SIGN = unichr(0xd7) def times(x): return "%u%s" % (x, MULTIPLICATION_SIGN) + def percentage(p): - return "%.02f%%" % (p*100.0,) + return "%.02f%%" % (p * 100.0,) + def add(a, b): return a + b + def equal(a, b): if a == b: return a else: return None + def fail(a, b): assert False tol = 2 ** -23 + def ratio(numerator, denominator): try: - ratio = float(numerator)/float(denominator) + ratio = float(numerator) / float(denominator) except ZeroDivisionError: # 0/0 is undefined, but 1.0 yields more useful results return 1.0 @@ -110,7 +114,7 @@ class UndefinedEvent(Exception): class Event(object): """Describe a kind of event, and its basic operations.""" - def __init__(self, name, null, aggregator, formatter = str): + def __init__(self, name, null, aggregator, formatter=str): self.name = name self._null = null self._aggregator = aggregator @@ -224,7 +228,7 @@ class Function(Object): self.calls[call.callee_id] = call def get_call(self, callee_id): - if not callee_id in self.calls: + if callee_id not in self.calls: call = Call(callee_id) call[SAMPLES] = 0 call[SAMPLES2] = 0 @@ -306,7 +310,9 @@ class Profile(Object): for callee_id in compat_keys(function.calls): assert function.calls[callee_id].callee_id == callee_id if callee_id not in self.functions: - sys.stderr.write('warning: call to undefined function %s from function %s\n' % (str(callee_id), function.name)) + sys.stderr.write( + 'warning: call to undefined function %s from function %s\n' % + (str(callee_id), function.name)) del function.calls[callee_id] def find_cycles(self): @@ -368,7 +374,6 @@ class Profile(Object): pathFunctions[n] = f self.functions = pathFunctions - def getFunctionId(self, funcName): for f in self.functions: if self.functions[f].name == funcName: @@ -496,7 +501,7 @@ class Profile(Object): assert outevent not in call assert call.ratio is not None callee = self.functions[call.callee_id] - subtotal = call.ratio *self._integrate_function(callee, outevent, inevent) + subtotal = call.ratio * self._integrate_function(callee, outevent, inevent) call[outevent] = subtotal return subtotal @@ -535,9 +540,10 @@ class Profile(Object): partials = {} self._rank_cycle_function(cycle, callee, 0, ranks) self._call_ratios_cycle(cycle, callee, ranks, call_ratios, set()) - partial = self._integrate_cycle_function(cycle, callee, call_ratio, partials, ranks, call_ratios, outevent, inevent) + partial = self._integrate_cycle_function( + cycle, callee, call_ratio, partials, ranks, call_ratios, outevent, inevent) assert partial == max(partials.values()) - assert not total or abs(1.0 - partial/(call_ratio*total)) <= 0.001 + assert not total or abs(1.0 - partial / (call_ratio * total)) <= 0.001 return cycle[outevent] @@ -561,20 +567,30 @@ class Profile(Object): call_ratios[callee] = call_ratios.get(callee, 0.0) + call.ratio self._call_ratios_cycle(cycle, callee, ranks, call_ratios, visited) - def _integrate_cycle_function(self, cycle, function, partial_ratio, partials, ranks, call_ratios, outevent, inevent): + def _integrate_cycle_function( + self, + cycle, + function, + partial_ratio, + partials, + ranks, + call_ratios, + outevent, + inevent): if function not in partials: - partial = partial_ratio*function[inevent] + partial = partial_ratio * function[inevent] for call in compat_itervalues(function.calls): if call.callee_id != function.id: callee = self.functions[call.callee_id] if callee.cycle is not cycle: assert outevent in call - partial += partial_ratio*call[outevent] + partial += partial_ratio * call[outevent] else: if ranks[callee] > ranks[function]: - callee_partial = self._integrate_cycle_function(cycle, callee, partial_ratio, partials, ranks, call_ratios, outevent, inevent) + callee_partial = self._integrate_cycle_function( + cycle, callee, partial_ratio, partials, ranks, call_ratios, outevent, inevent) call_ratio = ratio(call.ratio, call_ratios[callee]) - call_partial = call_ratio*callee_partial + call_partial = call_ratio * callee_partial try: call[outevent] += call_partial except UndefinedEvent: @@ -670,7 +686,7 @@ class Profile(Object): class Struct: """Masquerade a dictionary with a structure-like behavior.""" - def __init__(self, attrs = None): + def __init__(self, attrs=None): if attrs is None: attrs = {} self.__dict__['_attrs'] = attrs @@ -761,7 +777,7 @@ XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF = range(4) class XmlToken: - def __init__(self, type, name_or_data, attrs = None, line = None, column = None): + def __init__(self, type, name_or_data, attrs=None, line=None, column=None): assert type in (XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF) self.type = type self.name_or_data = name_or_data @@ -784,7 +800,7 @@ class XmlToken: class XmlTokenizer: """Expat based XML tokenizer.""" - def __init__(self, fp, skip_ws = True): + def __init__(self, fp, skip_ws=True): self.fp = fp self.tokens = [] self.index = 0 @@ -795,8 +811,8 @@ class XmlTokenizer: self.character_data = '' self.parser = xml.parsers.expat.ParserCreate() - self.parser.StartElementHandler = self.handle_element_start - self.parser.EndElementHandler = self.handle_element_end + self.parser.StartElementHandler = self.handle_element_start + self.parser.EndElementHandler = self.handle_element_end self.parser.CharacterDataHandler = self.handle_character_data def handle_element_start(self, name, attributes): @@ -825,7 +841,7 @@ class XmlTokenizer: self.character_data = '' def next(self): - size = 16*1024 + size = 16 * 1024 while self.index >= len(self.tokens) and not self.final: self.tokens = [] self.index = 0 @@ -834,7 +850,7 @@ class XmlTokenizer: try: self.parser.Parse(data, self.final) except xml.parsers.expat.ExpatError as e: - #if e.code == xml.parsers.expat.errors.XML_ERROR_NO_ELEMENTS: + # if e.code == xml.parsers.expat.errors.XML_ERROR_NO_ELEMENTS: if e.code == 3: pass else: @@ -858,7 +874,8 @@ class XmlTokenMismatch(Exception): self.found = found def __str__(self): - return '%u:%u: %s expected, %s found' % (self.found.line, self.found.column, str(self.expected), str(self.found)) + return '%u:%u: %s expected, %s found' % ( + self.found.line, self.found.column, str(self.expected), str(self.found)) class XmlParser(Parser): @@ -898,7 +915,7 @@ class XmlParser(Parser): raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token) self.consume() - def character_data(self, strip = True): + def character_data(self, strip=True): data = '' while self.token.type == XML_CHARACTER_DATA: data += self.token.name_or_data @@ -1095,7 +1112,7 @@ class GprofParser(Parser): # process call graph entries entry_lines = [] - while line != '\014': # form feed + while line != '\014': # form feed if line and not line.isspace(): if self._cg_sep_re.match(line): self.parse_cg_entry(entry_lines) @@ -1227,7 +1244,7 @@ class AXEParser(Parser): '^-----+ ' ) - _cg_footer_re = re.compile('^Index\s+Function\s*$') + _cg_footer_re = re.compile(r'^Index\s+Function\s*$') _cg_primary_re = re.compile( r'^\[(?P\d+)\]?' + @@ -1378,7 +1395,8 @@ class AXEParser(Parser): line = self.readline() def parse(self): - sys.stderr.write('warning: for axe format, edge weights are unreliable estimates derived from\nfunction total times.\n') + sys.stderr.write( + 'warning: for axe format, edge weights are unreliable estimates derived from\nfunction total times.\n') self.parse_cg() self.fp.close() @@ -1440,7 +1458,7 @@ class AXEParser(Parser): for call in compat_itervalues(function.calls): if call.ratio is not None: callee = profile.functions[call.callee_id] - call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]; + call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO] return profile @@ -1452,7 +1470,7 @@ class CallgrindParser(LineParser): - http://valgrind.org/docs/manual/cl-format.html """ - _call_re = re.compile('^calls=\s*(\d+)\s+((\d+|\+\d+|-\d+|\*)\s+)+$') + _call_re = re.compile(r'^calls=\s*(\d+)\s+((\d+|\+\d+|-\d+|\*)\s+)+$') def __init__(self, infile): LineParser.__init__(self, infile) @@ -1541,7 +1559,7 @@ class CallgrindParser(LineParser): if key == 'positions': self.num_positions = len(items) self.cost_positions = items - self.last_positions = [0]*self.num_positions + self.last_positions = [0] * self.num_positions return True def parse_cost_summary(self): @@ -1560,9 +1578,9 @@ class CallgrindParser(LineParser): __subpos_re = r'(0x[0-9a-fA-F]+|\d+|\+\d+|-\d+|\*)' _cost_re = re.compile(r'^' + - __subpos_re + r'( +' + __subpos_re + r')*' + - r'( +\d+)*' + - '$') + __subpos_re + r'( +' + __subpos_re + r')*' + + r'( +\d+)*' + + '$') def parse_cost_line(self, calls=None): line = self.lookahead().rstrip() @@ -1584,9 +1602,9 @@ class CallgrindParser(LineParser): values = line.split() assert len(values) <= self.num_positions + self.num_events - positions = values[0 : self.num_positions] - events = values[self.num_positions : ] - events += ['0']*(self.num_events - len(events)) + positions = values[0: self.num_positions] + events = values[self.num_positions:] + events += ['0'] * (self.num_events - len(events)) for i in range(self.num_positions): position = positions[i] @@ -1631,14 +1649,13 @@ class CallgrindParser(LineParser): _, values = line.split('=', 1) values = values.strip().split() calls = int(values[0]) - call_position = values[1:] self.consume() self.parse_cost_line(calls) return True - _position_re = re.compile('^(?P[cj]?(?:ob|fl|fi|fe|fn))=\s*(?:\((?P\d+)\))?(?:\s*(?P.+))?') + _position_re = re.compile(r'^(?P[cj]?(?:ob|fl|fi|fe|fn))=\s*(?:\((?P\d+)\))?(?:\s*(?P.+))?') _position_table_map = { 'ob': 'ob', @@ -1730,7 +1747,7 @@ class CallgrindParser(LineParser): def make_function(self, module, filename, name): # FIXME: module and filename are not being tracked reliably - #id = '|'.join((module, filename, name)) + # id = '|'.join((module, filename, name)) id = name try: function = self.profile.functions[id] @@ -1802,7 +1819,7 @@ class PerfParser(LineParser): for call in compat_itervalues(function.calls): if call.ratio is not None: callee = profile.functions[call.callee_id] - call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]; + call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO] else: assert False @@ -1931,8 +1948,6 @@ class OprofileParser(LineParser): profile = Profile() - reverse_call_samples = {} - # populate the profile profile[SAMPLES] = 0 for _callers, _function, _callees in compat_itervalues(self.entries): @@ -2016,7 +2031,7 @@ class OprofileParser(LineParser): if entry.symbol.startswith('"') and entry.symbol.endswith('"'): entry.symbol = entry.symbol[1:-1] entry.id = ':'.join((entry.application, entry.image, source, entry.symbol)) - entry.self = fields.get('self', None) != None + entry.self = fields.get('self', None) is not None if entry.self: entry.id += ':self' if entry.symbol: @@ -2036,7 +2051,7 @@ class OprofileParser(LineParser): def match_separator(self): line = self.lookahead() - return line == '-'*len(line) + return line == '-' * len(line) def match_primary(self): line = self.lookahead() @@ -2066,8 +2081,10 @@ class HProfParser(LineParser): # read lookahead self.readline() - while not self.lookahead().startswith('------'): self.consume() - while not self.lookahead().startswith('TRACE '): self.consume() + while not self.lookahead().startswith('------'): + self.consume() + while not self.lookahead().startswith('TRACE '): + self.consume() self.parse_traces() @@ -2084,12 +2101,13 @@ class HProfParser(LineParser): # build up callgraph for id, trace in compat_iteritems(self.traces): - if not id in self.samples: continue + if id not in self.samples: + continue mtime = self.samples[id][0] last = None for func, file, line in trace: - if not func in functions: + if func not in functions: function = Function(func, func) function[SAMPLES] = 0 profile.add_function(function) @@ -2120,17 +2138,16 @@ class HProfParser(LineParser): self.parse_trace() def parse_trace(self): - l = self.consume() - mo = self.trace_id_re.match(l) + consume = self.consume() + mo = self.trace_id_re.match(consume) tid = mo.group(1) - last = None trace = [] while self.lookahead().startswith('\t'): - l = self.consume() - match = self.trace_re.search(l) + consume = self.consume() + match = self.trace_re.search(consume) if not match: - #sys.stderr.write('Invalid line: %s\n' % l) + # sys.stderr.write('Invalid line: %s\n' % consume) break else: function_name, file, line = match.groups() @@ -2280,13 +2297,13 @@ class XPerfParser(Parser): import csv reader = csv.reader( self.stream, - delimiter = ',', - quotechar = None, - escapechar = None, - doublequote = False, - skipinitialspace = True, - lineterminator = '\r\n', - quoting = csv.QUOTE_NONE) + delimiter=',', + quotechar=None, + escapechar=None, + doublequote=False, + skipinitialspace=True, + lineterminator='\r\n', + quoting=csv.QUOTE_NONE) header = True for row in reader: if header: @@ -2548,7 +2565,6 @@ class AQtimeParser(XmlParser): return table def parse_data(self): - rows = [] attrs = self.element_start('DATA') table_id = int(attrs['TABLE_ID']) table_name, field_types, field_names = self.tables[table_id] @@ -2560,7 +2576,7 @@ class AQtimeParser(XmlParser): return table def parse_row(self, field_types): - row = [None]*len(field_types) + row = [None] * len(field_types) children = [] self.element_start('ROW') while self.token.type == XML_ELEMENT_START: @@ -2622,16 +2638,16 @@ class AQtimeParser(XmlParser): function = Function(self.build_id(fields), self.build_name(fields)) function[TIME] = fields['Time'] function[TOTAL_TIME] = fields['Time with Children'] - #function[TIME_RATIO] = fields['% Time']/100.0 - #function[TOTAL_TIME_RATIO] = fields['% with Children']/100.0 + # function[TIME_RATIO] = fields['% Time']/100.0 + # function[TOTAL_TIME_RATIO] = fields['% with Children']/100.0 return function def build_call(self, fields): call = Call(self.build_id(fields)) call[TIME] = fields['Time'] call[TOTAL_TIME] = fields['Time with Children'] - #call[TIME_RATIO] = fields['% Time']/100.0 - #call[TOTAL_TIME_RATIO] = fields['% with Children']/100.0 + # call[TIME_RATIO] = fields['% Time']/100.0 + # call[TOTAL_TIME_RATIO] = fields['% with Children']/100.0 return call def build_id(self, fields): @@ -2694,7 +2710,7 @@ class PstatsParser: call = Call(callee.id) if isinstance(value, tuple): for i in xrange(0, len(value), 4): - nc, cc, tt, ct = value[i:i+4] + nc, cc, tt, ct = value[i:i + 4] if CALLS in call: call[CALLS] += cc else: @@ -2707,11 +2723,11 @@ class PstatsParser: else: call[CALLS] = value - call[TOTAL_TIME] = ratio(value, nc)*ct + call[TOTAL_TIME] = ratio(value, nc) * ct caller.add_call(call) - #self.stats.print_stats() - #self.stats.print_callees() + # self.stats.print_stats() + # self.stats.print_callees() # Compute derived events self.profile.validate() @@ -2724,18 +2740,18 @@ class PstatsParser: class Theme: def __init__(self, - bgcolor = (0.0, 0.0, 1.0), - mincolor = (0.0, 0.0, 0.0), - maxcolor = (0.0, 0.0, 1.0), - fontname = "Arial", - fontcolor = "white", - nodestyle = "filled", - minfontsize = 10.0, - maxfontsize = 10.0, - minpenwidth = 0.5, - maxpenwidth = 4.0, - gamma = 2.2, - skew = 1.0): + bgcolor=(0.0, 0.0, 1.0), + mincolor=(0.0, 0.0, 0.0), + maxcolor=(0.0, 0.0, 1.0), + fontname="Arial", + fontcolor="white", + nodestyle="filled", + minfontsize=10.0, + maxfontsize=10.0, + minpenwidth=0.5, + maxpenwidth=4.0, + gamma=2.2, + skew=1.0): self.bgcolor = bgcolor self.mincolor = mincolor self.maxcolor = maxcolor @@ -2783,7 +2799,7 @@ class Theme: return self.fontsize(weight) def edge_penwidth(self, weight): - return max(weight*self.maxpenwidth, self.minpenwidth) + return max(weight * self.maxpenwidth, self.minpenwidth) def edge_arrowsize(self, weight): return 0.5 * math.sqrt(self.edge_penwidth(weight)) @@ -2800,14 +2816,14 @@ class Theme: if self.skew < 0: raise ValueError("Skew must be greater than 0") elif self.skew == 1.0: - h = hmin + weight*(hmax - hmin) - s = smin + weight*(smax - smin) - l = lmin + weight*(lmax - lmin) + h = hmin + weight * (hmax - hmin) + s = smin + weight * (smax - smin) + l = lmin + weight * (lmax - lmin) # noqa else: base = self.skew - h = hmin + ((hmax-hmin)*(-1.0 + (base ** weight)) / (base - 1.0)) - s = smin + ((smax-smin)*(-1.0 + (base ** weight)) / (base - 1.0)) - l = lmin + ((lmax-lmin)*(-1.0 + (base ** weight)) / (base - 1.0)) + h = hmin + ((hmax - hmin) * (-1.0 + (base ** weight)) / (base - 1.0)) + s = smin + ((smax - smin) * (-1.0 + (base ** weight)) / (base - 1.0)) + l = lmin + ((lmax - lmin) * (-1.0 + (base ** weight)) / (base - 1.0)) # noqa return self.hsl_to_rgb(h, s, l) @@ -2820,16 +2836,16 @@ class Theme: h = h % 1.0 s = min(max(s, 0.0), 1.0) - l = min(max(l, 0.0), 1.0) + l = min(max(l, 0.0), 1.0) # noqa - if l <= 0.5: - m2 = l*(s + 1.0) + if l <= 0.5: # noqa + m2 = l * (s + 1.0) else: - m2 = l + s - l*s - m1 = l*2.0 - m2 - r = self._hue_to_rgb(m1, m2, h + 1.0/3.0) + m2 = l + s - l * s + m1 = l * 2.0 - m2 + r = self._hue_to_rgb(m1, m2, h + 1.0 / 3.0) g = self._hue_to_rgb(m1, m2, h) - b = self._hue_to_rgb(m1, m2, h - 1.0/3.0) + b = self._hue_to_rgb(m1, m2, h - 1.0 / 3.0) # Apply gamma correction r **= self.gamma @@ -2843,50 +2859,50 @@ class Theme: h += 1.0 elif h > 1.0: h -= 1.0 - if h*6 < 1.0: - return m1 + (m2 - m1)*h*6.0 - elif h*2 < 1.0: + if h * 6 < 1.0: + return m1 + (m2 - m1) * h * 6.0 + elif h * 2 < 1.0: return m2 - elif h*3 < 2.0: - return m1 + (m2 - m1)*(2.0/3.0 - h)*6.0 + elif h * 3 < 2.0: + return m1 + (m2 - m1) * (2.0 / 3.0 - h) * 6.0 else: return m1 TEMPERATURE_COLORMAP = Theme( - mincolor = (2.0/3.0, 0.80, 0.25), # dark blue - maxcolor = (0.0, 1.0, 0.5), # satured red - gamma = 1.0 + mincolor=(2.0 / 3.0, 0.80, 0.25), # dark blue + maxcolor=(0.0, 1.0, 0.5), # satured red + gamma=1.0 ) PINK_COLORMAP = Theme( - mincolor = (0.0, 1.0, 0.90), # pink - maxcolor = (0.0, 1.0, 0.5), # satured red + mincolor=(0.0, 1.0, 0.90), # pink + maxcolor=(0.0, 1.0, 0.5), # satured red ) GRAY_COLORMAP = Theme( - mincolor = (0.0, 0.0, 0.85), # light gray - maxcolor = (0.0, 0.0, 0.0), # black + mincolor=(0.0, 0.0, 0.85), # light gray + maxcolor=(0.0, 0.0, 0.0), # black ) BW_COLORMAP = Theme( - minfontsize = 8.0, - maxfontsize = 24.0, - mincolor = (0.0, 0.0, 0.0), # black - maxcolor = (0.0, 0.0, 0.0), # black - minpenwidth = 0.1, - maxpenwidth = 8.0, + minfontsize=8.0, + maxfontsize=24.0, + mincolor=(0.0, 0.0, 0.0), # black + maxcolor=(0.0, 0.0, 0.0), # black + minpenwidth=0.1, + maxpenwidth=8.0, ) PRINT_COLORMAP = Theme( - minfontsize = 18.0, - maxfontsize = 30.0, - fontcolor = "black", - nodestyle = "solid", - mincolor = (0.0, 0.0, 0.0), # black - maxcolor = (0.0, 0.0, 0.0), # black - minpenwidth = 0.1, - maxpenwidth = 8.0, + minfontsize=18.0, + maxfontsize=30.0, + fontcolor="black", + nodestyle="solid", + mincolor=(0.0, 0.0, 0.0), # black + maxcolor=(0.0, 0.0, 0.0), # black + minpenwidth=0.1, + maxpenwidth=8.0, ) @@ -2908,16 +2924,16 @@ class DotWriter: """Split the function name on multiple lines.""" if len(name) > 32: - ratio = 2.0/3.0 - height = max(int(len(name)/(1.0 - ratio) + 0.5), 1) - width = max(len(name)/height, 32) + ratio = 2.0 / 3.0 + height = max(int(len(name) / (1.0 - ratio) + 0.5), 1) + width = max(len(name) / height, 32) # TODO: break lines in symbols name = textwrap.fill(name, width, break_long_words=False) # Take away spaces name = name.replace(", ", ",") name = name.replace("> >", ">>") - name = name.replace("> >", ">>") # catch consecutive + name = name.replace("> >", ">>") # catch consecutive return name @@ -2964,11 +2980,11 @@ class DotWriter: label = '\n'.join(labels) self.node(function.id, - label = label, - color = self.color(theme.node_bgcolor(weight)), - fontcolor = self.color(theme.node_fgcolor(weight)), - fontsize = "%.2f" % theme.node_fontsize(weight), - ) + label=label, + color=self.color(theme.node_bgcolor(weight)), + fontcolor=self.color(theme.node_fgcolor(weight)), + fontsize="%.2f" % theme.node_fontsize(weight), + ) for call in compat_itervalues(function.calls): callee = profile.functions[call.callee_id] @@ -2989,14 +3005,14 @@ class DotWriter: label = '\n'.join(labels) self.edge(function.id, call.callee_id, - label = label, - color = self.color(theme.edge_color(weight)), - fontcolor = self.color(theme.edge_color(weight)), - fontsize = "%.2f" % theme.edge_fontsize(weight), - penwidth = "%.2f" % theme.edge_penwidth(weight), - labeldistance = "%.2f" % theme.edge_penwidth(weight), - arrowsize = "%.2f" % theme.edge_arrowsize(weight), - ) + label=label, + color=self.color(theme.edge_color(weight)), + fontcolor=self.color(theme.edge_color(weight)), + fontsize="%.2f" % theme.edge_fontsize(weight), + penwidth="%.2f" % theme.edge_penwidth(weight), + labeldistance="%.2f" % theme.edge_penwidth(weight), + arrowsize="%.2f" % theme.edge_arrowsize(weight), + ) self.end_graph() @@ -3061,7 +3077,7 @@ class DotWriter: return 0 if f >= 1.0: return 255 - return int(255.0*f + 0.5) + return int(255.0 * f + 0.5) return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)]) @@ -3082,11 +3098,11 @@ class Main: """Main program.""" themes = { - "color": TEMPERATURE_COLORMAP, - "pink": PINK_COLORMAP, - "gray": GRAY_COLORMAP, - "bw": BW_COLORMAP, - "print": PRINT_COLORMAP, + "color": TEMPERATURE_COLORMAP, + "pink": PINK_COLORMAP, + "gray": GRAY_COLORMAP, + "bw": BW_COLORMAP, + "print": PRINT_COLORMAP, } formats = { @@ -3115,8 +3131,7 @@ class Main: global totalMethod - formatNames = list(self.formats.keys()) - formatNames.sort() + formatNames = sorted(self.formats.keys()) optparser = optparse.OptionParser( usage="\n\t%prog [options] [file] ...") @@ -3139,19 +3154,27 @@ class Main: help="profile format: %s [default: %%default]" % self.naturalJoin(formatNames)) optparser.add_option( '--total', - type="choice", choices=('callratios', 'callstacks'), - dest="totalMethod", default=totalMethod, - help="preferred method of calculating total time: callratios or callstacks (currently affects only perf format) [default: %default]") + type="choice", + choices=( + 'callratios', + 'callstacks'), + dest="totalMethod", + default=totalMethod, + help=("preferred method of calculating total time: callratios or callstacks" + " (currently affects only perf format) [default: %default]")) optparser.add_option( '-c', '--colormap', type="choice", choices=('color', 'pink', 'gray', 'bw', 'print'), dest="theme", default="color", help="color map: color, pink, gray, bw, or print [default: %default]") optparser.add_option( - '-s', '--strip', + '-s', + '--strip', action="store_true", - dest="strip", default=False, - help="strip function parameters, template parameters, and const modifiers from demangled C++ function names") + dest="strip", + default=False, + help=("strip function parameters, template parameters, and const modifiers from" + " demangled C++ function names")) optparser.add_option( '-w', '--wrap', action="store_true", @@ -3176,8 +3199,11 @@ class Main: # add a new option to control skew of the colorization curve optparser.add_option( '--skew', - type="float", dest="theme_skew", default=1.0, - help="skew the colorization curve. Values < 1.0 give more variety to lower percentages. Values > 1.0 give less variety to lower percentages") + type="float", + dest="theme_skew", + default=1.0, + help=("skew the colorization curve. Values < 1.0 give more variety to lower" + " percentages. Values > 1.0 give less variety to lower percentages")) (self.options, self.args) = optparser.parse_args(sys.argv[1:]) if len(self.args) > 1 and self.options.format != 'pstats': @@ -3234,18 +3260,24 @@ class Main: dot.show_function_events.append(SAMPLES) profile = self.profile - profile.prune(self.options.node_thres/100.0, self.options.edge_thres/100.0) + profile.prune(self.options.node_thres / 100.0, self.options.edge_thres / 100.0) if self.options.root: rootId = profile.getFunctionId(self.options.root) if not rootId: - sys.stderr.write('root node ' + self.options.root + ' not found (might already be pruned : try -e0 -n0 flags)\n') + sys.stderr.write( + 'root node ' + + self.options.root + + ' not found (might already be pruned : try -e0 -n0 flags)\n') sys.exit(1) profile.prune_root(rootId) if self.options.leaf: leafId = profile.getFunctionId(self.options.leaf) if not leafId: - sys.stderr.write('leaf node ' + self.options.leaf + ' not found (maybe already pruned : try -e0 -n0 flags)\n') + sys.stderr.write( + 'leaf node ' + + self.options.leaf + + ' not found (maybe already pruned : try -e0 -n0 flags)\n') sys.exit(1) profile.prune_leaf(leafId) diff --git a/tools/parse_bandwidth_log.py b/tools/parse_bandwidth_log.py index 2e144dea0..cf5108bda 100755 --- a/tools/parse_bandwidth_log.py +++ b/tools/parse_bandwidth_log.py @@ -1,24 +1,26 @@ #!/usr/bin/env python -import os, sys, time +from __future__ import print_function + +import os +import sys keys = [['upload rate', 'x1y1', 6], ['history entries', 'x1y2', 10], ['queue', 'x1y2', 4]] out = open('bandwidth.gnuplot', 'wb') -print >>out, "set term png size 1200,700" -print >>out, 'set output "bandwidth_manager.png"' -print >>out, 'set xrange [0:*]' -print >>out, 'set xlabel "time (ms)"' -print >>out, 'set ylabel "Rate (B/s)"' -print >>out, 'set ytics 10000' -print >>out, 'set y2label "number"' -print >>out, 'set y2range [0:*]' -#print >>out, "set style data lines" -print >>out, "set key box" -print >>out, 'plot', +print("set term png size 1200,700", file=out) +print('set output "bandwidth_manager.png"', file=out) +print('set xrange [0:*]', file=out) +print('set xlabel "time (ms)"', file=out) +print('set ylabel "Rate (B/s)"', file=out) +print('set ytics 10000', file=out) +print('set y2label "number"', file=out) +print('set y2range [0:*]', file=out) +# print("set style data lines", file=out) +print("set key box", file=out) +print('plot', end=' ', file=out) for k, a, c in keys: - print >>out, ' "%s" using 1:%d title "%s" axes %s with steps,' % (sys.argv[1], c, k, a), -print >>out, 'x=0' + print(' "%s" using 1:%d title "%s" axes %s with steps,' % (sys.argv[1], c, k, a), end=' ', file=out) +print('x=0', file=out) out.close() -os.system('gnuplot bandwidth.gnuplot'); - +os.system('gnuplot bandwidth.gnuplot') diff --git a/tools/parse_buffer_log.py b/tools/parse_buffer_log.py index 9d3257d38..fc9aeb119 100755 --- a/tools/parse_buffer_log.py +++ b/tools/parse_buffer_log.py @@ -3,14 +3,18 @@ # subject to the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -import os, sys, time +from __future__ import print_function + +import os +import sys lines = open(sys.argv[1], 'rb').readlines() -#keys = ['send_buffer_utilization'] +# keys = ['send_buffer_utilization'] keys = ['send_buffer_size', 'used_send_buffer', 'protocol_buffer'] -#keys = ['send_buffer_alloc', 'send_buffer', 'allocate_buffer_alloc', 'allocate_buffer', 'protocol_buffer'] -#keys = ['send_buffer_alloc', 'send_buffer', 'allocate_buffer_alloc', 'allocate_buffer', 'protocol_buffer', 'append_send_buffer'] +# keys = ['send_buffer_alloc', 'send_buffer', 'allocate_buffer_alloc', 'allocate_buffer', 'protocol_buffer'] +# keys = ['send_buffer_alloc', 'send_buffer', 'allocate_buffer_alloc', 'allocate_buffer', 'protocol_buffer', +# 'append_send_buffer'] average = ['send_buffer_utilization', 'send_buffer_size', 'used_send_buffer'] average_interval = 120000 @@ -18,7 +22,7 @@ render = 'lines' time_limit = -1 if len(sys.argv) > 2: - time_limit = long(sys.argv[2]) + time_limit = int(sys.argv[2]) # logfile format: @@ -27,64 +31,68 @@ if len(sys.argv) > 2: # 16434 allocate_buffer: 17 for k in keys: - last_sample = 0 - average_accumulator = 0 - average_samples = 0 - peak = 0 + last_sample = 0 + average_accumulator = 0 + average_samples = 0 + peak = 0 - out = open(k + '.dat', 'wb') - eval_average = False - if k in average: - eval_average = True - peak_out = open(k + '_peak.dat', 'wb') + out = open(k + '.dat', 'wb') + eval_average = False + if k in average: + eval_average = True + peak_out = open(k + '_peak.dat', 'wb') - for l in lines: - l = l.split(' ') - if len(l) != 3: - print l - continue - try: - if l[1] == k + ':': - if time_limit != -1 and long(l[0]) > time_limit: break - time = l[0] - value = l[2] - if eval_average: - while long(time) > last_sample + average_interval: - last_sample = last_sample + average_interval - if average_samples < 1: average_samples = 1 - print >>out, '%d %f' % (last_sample, average_accumulator / average_samples) - print >>peak_out, '%d %f' % (last_sample, peak) - average_accumulator = 0 - average_samples = 0 - peak = 0 - average_accumulator = average_accumulator + float(value) - average_samples = average_samples + 1 - if float(value) > peak: peak = float(value) - else: - print >>out, time + ' ' + value, - except: - print l + for line in lines: + line = line.split(' ') + if len(line) != 3: + print(line) + continue + try: + if line[1] == k + ':': + if time_limit != -1 and int(line[0]) > time_limit: + break + time = line[0] + value = line[2] + if eval_average: + while int(time) > last_sample + average_interval: + last_sample = last_sample + average_interval + if average_samples < 1: + average_samples = 1 + print('%d %f' % (last_sample, average_accumulator / average_samples), file=out) + print('%d %f' % (last_sample, peak), file=peak_out) + average_accumulator = 0 + average_samples = 0 + peak = 0 + average_accumulator = average_accumulator + float(value) + average_samples = average_samples + 1 + if float(value) > peak: + peak = float(value) + else: + print(time + ' ' + value, end=' ', file=out) + except BaseException: + print(line) - out.close() - peak_out.close() + out.close() + peak_out.close() out = open('send_buffer.gnuplot', 'wb') -print >>out, "set term png size 1200,700" -print >>out, 'set output "send_buffer.png"' -print >>out, 'set xrange [0:*]' -print >>out, 'set xlabel "time (ms)"' -print >>out, 'set ylabel "bytes (B)"' -print >>out, "set style data lines" -print >>out, "set key box" -print >>out, 'plot', +print("set term png size 1200,700", file=out) +print('set output "send_buffer.png"', file=out) +print('set xrange [0:*]', file=out) +print('set xlabel "time (ms)"', file=out) +print('set ylabel "bytes (B)"', file=out) +print("set style data lines", file=out) +print("set key box", file=out) +print('plot', end=' ', file=out) for k in keys: - if k in average: - print >>out, ' "%s.dat" using 1:2 title "%s %d seconds average" with %s,' % (k, k, average_interval / 1000., render), - print >>out, ' "%s_peak.dat" using 1:2 title "%s %d seconds peak" with %s,' % (k, k, average_interval / 1000., render), - else: - print >>out, ' "%s.dat" using 1:2 title "%s" with %s,' % (k, k, render), -print >>out, 'x=0' + if k in average: + print(' "%s.dat" using 1:2 title "%s %d seconds average" with %s,' % + (k, k, average_interval / 1000., render), end=' ', file=out) + print(' "%s_peak.dat" using 1:2 title "%s %d seconds peak" with %s,' % + (k, k, average_interval / 1000., render), end=' ', file=out) + else: + print(' "%s.dat" using 1:2 title "%s" with %s,' % (k, k, render), end=' ', file=out) +print('x=0', file=out) out.close() os.system('gnuplot send_buffer.gnuplot') - diff --git a/tools/parse_dht_log.py b/tools/parse_dht_log.py index 83717373d..88b398164 100755 --- a/tools/parse_dht_log.py +++ b/tools/parse_dht_log.py @@ -1,8 +1,8 @@ #!/usr/bin/env python +from __future__ import print_function + import sys import os -import time -import calendar import pprint pp = pprint.PrettyPrinter(indent=4) @@ -13,11 +13,11 @@ f = open(sys.argv[1]) announce_histogram = {} -#TODO: make this histogram into a CDF +# TODO: make this histogram into a CDF node_uptime_histogram = {} -counter = 0; +counter = 0 # maps search_id to a list of events. Each event is a dict containing: # t: timestamp @@ -32,10 +32,12 @@ outstanding_searches = {} # list of completed searches searches = [] + def convert_timestamp(t): - parts = t.split('.') - hms = parts[0].split(':') - return (int(hms[0]) * 3600 + int(hms[1]) * 60 + int(hms[2])) * 1000 + int(parts[1]) + parts = t.split('.') + hms = parts[0].split(':') + return (int(hms[0]) * 3600 + int(hms[1]) * 60 + int(hms[2])) * 1000 + int(parts[1]) + last_incoming = '' @@ -46,111 +48,117 @@ client_version_histogram = {} client_histogram = {} for line in f: - counter += 1 -# if counter % 1000 == 0: -# print '\r%d' % counter, - try: - l = line.split(' ') - if 'starting DHT tracker with node id:' in line: - our_node_id = l[l.index('id:') + 1].strip() + counter += 1 + # if counter % 1000 == 0: + # print '\r%d' % counter, + try: + ls = line.split(' ') + if 'starting DHT tracker with node id:' in line: + our_node_id = ls[ls.index('id:') + 1].strip() - try: - if len(l) > 4 and l[2] == '<==' and l[1] == '[dht_tracker]': - ip = l[3].split(':')[0] - if ip not in unique_ips: - unique_ips.add(ip) - json_blob = line.split(l[3])[1] - version = json_blob.split("'v': '")[1].split("'")[0] - if len(version) == 4: - v = '%s-%d' % (version[0:2], (ord(version[2]) << 8) + ord(version[3])) - elif len(version) == 8: - v = '%c%c-%d' % (chr(int(version[0:2], 16)), chr(int(version[2:4], 16)), int(version[4:8], 16)) - else: - v = 'unknown' + try: + if len(ls) > 4 and ls[2] == '<==' and ls[1] == '[dht_tracker]': + ip = ls[3].split(':')[0] + if ip not in unique_ips: + unique_ips.add(ip) + json_blob = line.split(ls[3])[1] + version = json_blob.split("'v': '")[1].split("'")[0] + if len(version) == 4: + v = '%s-%d' % (version[0:2], (ord(version[2]) << 8) + ord(version[3])) + elif len(version) == 8: + v = '%c%c-%d' % (chr(int(version[0:2], 16)), chr(int(version[2:4], 16)), int(version[4:8], 16)) + else: + v = 'unknown' - if not v in client_version_histogram: - client_version_histogram[v] = 1 - else: - client_version_histogram[v] += 1 + if v not in client_version_histogram: + client_version_histogram[v] = 1 + else: + client_version_histogram[v] += 1 - if not v[0:2] in client_histogram: - client_histogram[v[0:2]] = 1 - else: - client_histogram[v[0:2]] += 1 - except: pass + if not v[0:2] in client_histogram: + client_histogram[v[0:2]] = 1 + else: + client_histogram[v[0:2]] += 1 + except BaseException: + pass - if 'announce-distance:' in line: - idx = l.index('announce-distance:') + if 'announce-distance:' in line: + idx = ls.index('announce-distance:') - d = int(l[idx+1].strip()) - if not d in announce_histogram: announce_histogram[d] = 0 - announce_histogram[d] += 1 - if 'NODE FAILED' in line: - idx = l.index('fails:') - if int(l[idx+1].strip()) != 1: continue; - idx = l.index('up-time:') - d = int(l[idx+1].strip()) - # quantize - d = d - (d % up_time_quanta) - if not d in node_uptime_histogram: node_uptime_histogram[d] = 0 - node_uptime_histogram[d] += 1 + d = int(ls[idx + 1].strip()) + if d not in announce_histogram: + announce_histogram[d] = 0 + announce_histogram[d] += 1 + if 'NODE FAILED' in line: + idx = ls.index('fails:') + if int(ls[idx + 1].strip()) != 1: + continue + idx = ls.index('up-time:') + d = int(ls[idx + 1].strip()) + # quantize + d = d - (d % up_time_quanta) + if d not in node_uptime_histogram: + node_uptime_histogram[d] = 0 + node_uptime_histogram[d] += 1 - search_id = l[2] - ts = l[0] - event = l[3] + search_id = ls[2] + ts = ls[0] + event = ls[3] - if event == 'RESPONSE': - outstanding = int(l[l.index('invoke-count:')+1]) - nid = l[l.index('id:')+1] - addr = l[l.index('addr:')+1] - last_response = addr - outstanding_searches[search_id].append({ 't': ts, 'd': distance, - 'o': outstanding + 1, 'a':addr, 'e': event,'i':nid, 's':source}) - elif event == 'NEW': - nid = l[l.index('target:')+1] - outstanding_searches[search_id] = [{ 't': ts, 'd': 0, 'o': 0, \ - 'e': event, 'abstime': ts, 'i': nid}] - last_response = '' - elif event == 'INVOKE' or event == 'ADD' or event == '1ST_TIMEOUT' or \ - event == 'TIMEOUT' or event == 'PEERS': - if not search_id in outstanding_searches: - print 'orphaned event: %s' % line - else: - outstanding = int(l[l.index('invoke-count:')+1]) - distance = int(l[l.index('distance:')+1]) - nid = l[l.index('id:')+1] - addr = l[l.index('addr:')+1] - source = '' - if event == 'ADD': - if last_response == '': continue - source = last_response + if event == 'RESPONSE': + outstanding = int(ls[ls.index('invoke-count:') + 1]) + distance = int(ls[ls.index('distance:') + 1]) + nid = ls[ls.index('id:') + 1] + addr = ls[ls.index('addr:') + 1] + last_response = addr + outstanding_searches[search_id].append({'t': ts, 'd': distance, + 'o': outstanding + 1, 'a': addr, 'e': event, 'i': nid}) + elif event == 'NEW': + nid = ls[ls.index('target:') + 1] + outstanding_searches[search_id] = [{'t': ts, 'd': 0, 'o': 0, + 'e': event, 'abstime': ts, 'i': nid}] + last_response = '' + elif event == 'INVOKE' or event == 'ADD' or event == '1ST_TIMEOUT' or \ + event == 'TIMEOUT' or event == 'PEERS': + if search_id not in outstanding_searches: + print('orphaned event: %s' % line) + else: + outstanding = int(ls[ls.index('invoke-count:') + 1]) + distance = int(ls[ls.index('distance:') + 1]) + nid = ls[ls.index('id:') + 1] + addr = ls[ls.index('addr:') + 1] + source = '' + if event == 'ADD': + if last_response == '': + continue + source = last_response - outstanding_searches[search_id].append({ 't': ts, 'd': distance, - 'o': outstanding + 1, 'a':addr, 'e': event,'i':nid, 's':source}) - elif event == 'ABORTED': - outstanding_searches[search_id].append({ 't': ts, 'e': event}) - elif event == 'COMPLETED': - distance = int(l[l.index('distance:')+1]) - lookup_type = l[l.index('type:')+1].strip() - outstanding_searches[search_id].append({ 't': ts, 'd': distance, - 'o': 0, 'e': event,'i':''}) + outstanding_searches[search_id].append( + {'t': ts, 'd': distance, 'o': outstanding + 1, 'a': addr, 'e': event, 'i': nid, 's': source}) + elif event == 'ABORTED': + outstanding_searches[search_id].append({'t': ts, 'e': event}) + elif event == 'COMPLETED': + distance = int(ls[ls.index('distance:') + 1]) + lookup_type = ls[ls.index('type:') + 1].strip() + outstanding_searches[search_id].append({'t': ts, 'd': distance, + 'o': 0, 'e': event, 'i': ''}) - outstanding_searches[search_id][0]['type'] = lookup_type + outstanding_searches[search_id][0]['type'] = lookup_type - s = outstanding_searches[search_id] + s = outstanding_searches[search_id] - try: - start_time = convert_timestamp(s[0]['t']) - for i in range(len(s)): - s[i]['t'] = convert_timestamp(s[i]['t']) - start_time - except: - pass - searches.append(s) - del outstanding_searches[search_id] + try: + start_time = convert_timestamp(s[0]['t']) + for i in range(len(s)): + s[i]['t'] = convert_timestamp(s[i]['t']) - start_time + except BaseException: + pass + searches.append(s) + del outstanding_searches[search_id] - except Exception, e: - print e - print line.split(' ') + except Exception as e: + print(e) + print(line.split(' ')) lookup_times_min = [] lookup_times_max = [] @@ -159,26 +167,29 @@ lookup_times_max = [] # to target boundaries lookup_distance = [] for i in range(0, 15): - lookup_distance.append([]) + lookup_distance.append([]) for s in searches: - for i in s: - if not 'last_dist' in i: - i['last_dist'] = -1 - cur_dist = 160 - i['d'] - last_dist = i['last_dist'] - if cur_dist > last_dist: - for j in range(last_dist + 1, cur_dist + 1): - if j >= len(lookup_distance): break - lookup_distance[j].append(i['t']) - i['last_dist'] = cur_dist - if i['e'] != 'PEERS': continue - lookup_times_min.append(i['t']) - break - for i in reversed(s): - if i['e'] != 'PEERS': continue - lookup_times_max.append(i['t']) - break + for i in s: + if 'last_dist' not in i: + i['last_dist'] = -1 + cur_dist = 160 - i['d'] + last_dist = i['last_dist'] + if cur_dist > last_dist: + for j in range(last_dist + 1, cur_dist + 1): + if j >= len(lookup_distance): + break + lookup_distance[j].append(i['t']) + i['last_dist'] = cur_dist + if i['e'] != 'PEERS': + continue + lookup_times_min.append(i['t']) + break + for i in reversed(s): + if i['e'] != 'PEERS': + continue + lookup_times_max.append(i['t']) + break lookup_times_min.sort() @@ -186,74 +197,74 @@ lookup_times_max.sort() out = open('dht_lookup_times_cdf.txt', 'w+') counter = 0 for i in range(len(lookup_times_min)): - counter += 1 - print >>out, '%d\t%d\t%f' % (lookup_times_min[i], lookup_times_max[i], counter / float(len(lookup_times_min))) + counter += 1 + print('%d\t%d\t%f' % (lookup_times_min[i], lookup_times_max[i], counter / float(len(lookup_times_min))), file=out) out.close() for i in lookup_distance: - i.sort() + i.sort() dist = 0 for i in lookup_distance: - out = open('dht_lookup_distance_%d.txt' % dist, 'w+') - dist += 1 - counter = 0 - for j in i: - counter += 1 - print >>out, '%d\t%f' % (j, counter / float(len(i))) - out.close() + out = open('dht_lookup_distance_%d.txt' % dist, 'w+') + dist += 1 + counter = 0 + for j in i: + counter += 1 + print('%d\t%f' % (j, counter / float(len(i))), file=out) + out.close() out = open('dht_lookups.txt', 'w+') for s in searches: - for i in s: - if i['e'] == 'INVOKE': - print >>out, ' ->', i['t'], 160 - i['d'], i['i'], i['a'] - elif i['e'] == '1ST_TIMEOUT': - print >>out, ' x ', i['t'], 160 - i['d'], i['i'], i['a'] - elif i['e'] == 'TIMEOUT': - print >>out, ' X ', i['t'], 160 - i['d'], i['i'], i['a'] - elif i['e'] == 'ADD': - print >>out, ' + ', i['t'], 160 - i['d'], i['i'], i['a'], i['s'] - elif i['e'] == 'RESPONSE': - print >>out, ' <-', i['t'], 160 - i['d'], i['i'], i['a'] - elif i['e'] == 'PEERS': - print >>out, ' <-', i['t'], 160 - i['d'], i['i'], i['a'] - elif i['e'] == 'ABORTED': - print >>out, 'abort' - elif i['e'] == 'COMPLETED': - print >>out, '***', i['t'], 160 - i['d'], '\n' - elif i['e'] == 'NEW': - print >>out, '===', i['abstime'], i['type'], '===' - print >>out, '<> ', 0, our_node_id, i['i'] + for i in s: + if i['e'] == 'INVOKE': + print(' ->', i['t'], 160 - i['d'], i['i'], i['a'], file=out) + elif i['e'] == '1ST_TIMEOUT': + print(' x ', i['t'], 160 - i['d'], i['i'], i['a'], file=out) + elif i['e'] == 'TIMEOUT': + print(' X ', i['t'], 160 - i['d'], i['i'], i['a'], file=out) + elif i['e'] == 'ADD': + print(' + ', i['t'], 160 - i['d'], i['i'], i['a'], i['s'], file=out) + elif i['e'] == 'RESPONSE': + print(' <-', i['t'], 160 - i['d'], i['i'], i['a'], file=out) + elif i['e'] == 'PEERS': + print(' <-', i['t'], 160 - i['d'], i['i'], i['a'], file=out) + elif i['e'] == 'ABORTED': + print('abort', file=out) + elif i['e'] == 'COMPLETED': + print('***', i['t'], 160 - i['d'], '\n', file=out) + elif i['e'] == 'NEW': + print('===', i['abstime'], i['type'], '===', file=out) + print('<> ', 0, our_node_id, i['i'], file=out) out.close() out = open('dht_announce_distribution.dat', 'w+') -print 'announce distribution items: %d' % len(announce_histogram) -for k,v in announce_histogram.items(): - print >>out, '%d %d' % (k, v) - print '%d %d' % (k, v) +print('announce distribution items: %d' % len(announce_histogram)) +for k, v in list(announce_histogram.items()): + print('%d %d' % (k, v), file=out) + print('%d %d' % (k, v)) out.close() out = open('dht_node_uptime_cdf.txt', 'w+') s = 0 total_uptime_nodes = 0 -for k,v in node_uptime_histogram.items(): - total_uptime_nodes += v +for k, v in list(node_uptime_histogram.items()): + total_uptime_nodes += v -for k,v in sorted(node_uptime_histogram.items()): - s += v - print >>out, '%f %f' % (k / float(60), s / float(total_uptime_nodes)) - print '%f %f' % (k / float(60), s / float(total_uptime_nodes)) +for k, v in sorted(node_uptime_histogram.items()): + s += v + print('%f %f' % (k / float(60), s / float(total_uptime_nodes)), file=out) + print('%f %f' % (k / float(60), s / float(total_uptime_nodes))) out.close() -print 'clients by version' -client_version_histogram = sorted(client_version_histogram.items(), key=lambda x: x[1], reverse=True) +print('clients by version') +client_version_histogram = sorted(list(client_version_histogram.items()), key=lambda x: x[1], reverse=True) pp.pprint(client_version_histogram) -print 'clients' -client_histogram = sorted(client_histogram.items(), key=lambda x: x[1], reverse=True) +print('clients') +client_histogram = sorted(list(client_histogram.items()), key=lambda x: x[1], reverse=True) pp.pprint(client_histogram) out = open('dht.gnuplot', 'w+') @@ -265,7 +276,7 @@ set ylabel "portion of lookups" set xlabel "time from start of lookup (ms)" set grid plot "dht_lookup_times_cdf.txt" using 1:3 with lines title "time to first result", \ - "dht_lookup_times_cdf.txt" using 2:3 with lines title "time to last result" + "dht_lookup_times_cdf.txt" using 2:3 with lines title "time to last result" set terminal postscript set output "dht_lookup_times_cdf.ps" @@ -318,12 +329,11 @@ plot ''') dist = 0 for i in lookup_distance: - if dist > 0: out.write(', ') - out.write('"dht_lookup_distance_%d.txt" using 1:2 title "%d" with lines' % (dist, dist)) - dist += 1 + if dist > 0: + out.write(', ') + out.write('"dht_lookup_distance_%d.txt" using 1:2 title "%d" with lines' % (dist, dist)) + dist += 1 out.close() -os.system('gnuplot dht.gnuplot'); - - +os.system('gnuplot dht.gnuplot') diff --git a/tools/parse_dht_rtt.py b/tools/parse_dht_rtt.py index 0548f1152..cb75e181f 100755 --- a/tools/parse_dht_rtt.py +++ b/tools/parse_dht_rtt.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import print_function import sys import os @@ -11,21 +12,22 @@ distribution = {} num_messages = 0 for i in range(0, max_rtt, quantize): - distribution[i] = 0 + distribution[i] = 0 for line in f: - time = int(line.split('\t')[1]) - if (time < 0 or time > max_rtt - quantize): continue - num_messages += 1 - time /= quantize - time *= quantize - distribution[time] += 1 + time = int(line.split('\t')[1]) + if (time < 0 or time > max_rtt - quantize): + continue + num_messages += 1 + time /= quantize + time *= quantize + distribution[time] += 1 f = open('round_trip_distribution.log', 'w+') -for k, v in distribution.items(): - print >>f, '%f %d' % ((k + (quantize / 2)) / 1000.0, v) -f.close(); +for k, v in list(distribution.items()): + print('%f %d' % ((k + (quantize / 2)) / 1000.0, v), file=f) +f.close() f = open('round_trip_distribution.gnuplot', 'w+') @@ -48,5 +50,4 @@ replot ''' % (float(quantize) / 1000.0)) f.close() -os.system('gnuplot round_trip_distribution.gnuplot'); - +os.system('gnuplot round_trip_distribution.gnuplot') diff --git a/tools/parse_dht_stats.py b/tools/parse_dht_stats.py index 15edd0863..74f8e49e7 100755 --- a/tools/parse_dht_stats.py +++ b/tools/parse_dht_stats.py @@ -4,50 +4,59 @@ import os gnuplot_scripts = [] + def gen_stats_gnuplot(name, y, lines): - global gnuplot_scripts + global gnuplot_scripts - stat = open(sys.argv[1]) - line = stat.readline() - while not 'minute:' in line: - line = stat.readline() + stat = open(sys.argv[1]) + line = stat.readline() + while 'minute:' not in line: + line = stat.readline() - names = line.strip().split(':') - counter = 1 - for i in names: - print '%d: %s' % (counter, i) - counter += 1 + names = line.strip().split(':') + counter = 1 + for i in names: + print('%d: %s' % (counter, i)) + counter += 1 - out = open('%s.gnuplot' % name, 'w+') - out.write(''' + out = open('%s.gnuplot' % name, 'w+') + out.write(''' set term png size 1200,700 small set output "%s.png" set title "%s" set ylabel "%s" set xlabel "time (minutes)" plot ''' % (name, name.strip('_'), y)) - first = True - for i in lines: - if not first: - out.write(', \\\n') - first = False - out.write('"%s" using 1:%d title "%s" with lines' % (sys.argv[1], names.index(i)+1, i)) - out.write('\n') + first = True + for i in lines: + if not first: + out.write(', \\\n') + first = False + out.write('"%s" using 1:%d title "%s" with lines' % (sys.argv[1], names.index(i) + 1, i)) + out.write('\n') - out.write('''set terminal postscript + out.write('''set terminal postscript set output "%s.ps" replot ''' % (name)) - out.close() - gnuplot_scripts += [name] + out.close() + gnuplot_scripts += [name] -gen_stats_gnuplot('dht_routing_table_size', 'nodes', ['active nodes','passive nodes', 'confirmed nodes']) + +gen_stats_gnuplot('dht_routing_table_size', 'nodes', ['active nodes', 'passive nodes', 'confirmed nodes']) gen_stats_gnuplot('dht_tracker_table_size', '', ['num torrents', 'num peers']) gen_stats_gnuplot('dht_announces', 'messages per minute', ['announces per min', 'failed announces per min']) -gen_stats_gnuplot('dht_clients', 'messages per minute', ['total msgs per min', 'az msgs per min', 'ut msgs per min', 'lt msgs per min', 'mp msgs per min', 'gr msgs per min']) -gen_stats_gnuplot('dht_rate', 'bytes per second', ['bytes in per sec', 'bytes out per sec']) -gen_stats_gnuplot('dht_errors', 'messages per minute', ['error replies sent', 'error queries recvd']) +gen_stats_gnuplot('dht_clients', + 'messages per minute', + ['total msgs per min', + 'az msgs per min', + 'ut msgs per min', + 'lt msgs per min', + 'mp msgs per min', + 'gr msgs per min']) +gen_stats_gnuplot('dht_rate', 'bytes per second', ['bytes in per sec', 'bytes out per sec']) +gen_stats_gnuplot('dht_errors', 'messages per minute', ['error replies sent', 'error queries recvd']) for i in gnuplot_scripts: - os.system('gnuplot %s.gnuplot' % i); + os.system('gnuplot %s.gnuplot' % i) diff --git a/tools/parse_disk_buffer_log.py b/tools/parse_disk_buffer_log.py index e4640348f..df387127b 100755 --- a/tools/parse_disk_buffer_log.py +++ b/tools/parse_disk_buffer_log.py @@ -1,6 +1,8 @@ #!/usr/bin/env python +from __future__ import print_function -import os, sys, time +import os +import sys lines = open(sys.argv[1], 'rb').readlines() @@ -10,11 +12,11 @@ lines = open(sys.argv[1], 'rb').readlines() # 16434 read cache: 17 key_order = ['receive buffer', 'send buffer', 'released send buffer', 'posted send buffer', - 'received send buffer', 'dispatched send buffer', 'queued send buffer', - 'write cache', 'read cache', 'hash temp'] + 'received send buffer', 'dispatched send buffer', 'queued send buffer', + 'write cache', 'read cache', 'hash temp'] colors = ['30f030', '001070', '101080', '2040a0', - '4070d0', '80a0f0', 'f03030', - '80f080', 'f08080', '4040ff'] + '4070d0', '80a0f0', 'f03030', + '80f080', 'f08080', '4040ff'] keys = [] fields = {} @@ -26,67 +28,70 @@ field_num_samples = {} field_timestamp = {} for c in key_order: - keys.append(c) - fields[c] = 0 - maximum[c] = 0 - field_sum[c] = 0 - field_num_samples[c] = 0 - field_timestamp[c] = 0 + keys.append(c) + fields[c] = 0 + maximum[c] = 0 + field_sum[c] = 0 + field_num_samples[c] = 0 + field_timestamp[c] = 0 last_t = 0 for l in lines: - try: - t = int(l[0:l.find(' ')]) - c = l[l.find(' ')+1:l.find(':')] - n = int(l[l.find(':')+1:-1]) - except: - print l - continue + try: + t = int(l[0:l.find(' ')]) + c = l[l.find(' ') + 1:l.find(':')] + n = int(l[l.find(':') + 1:-1]) + except BaseException: + print(l) + continue - if last_t != t: - print >>out, '%d\t' % last_t, - for i in keys: - print >>out, '%d\t' % maximum[i], - print >>out, '\n', + if last_t != t: + print('%d\t' % last_t, end=' ', file=out) + for i in keys: + print('%d\t' % maximum[i], end=' ', file=out) + print('\n', end=' ', file=out) - if not c in keys: continue + if c not in keys: + continue - field_sum[c] += fields[c] * float(t - field_timestamp[c]) - field_timestamp[c] = t + field_sum[c] += fields[c] * float(t - field_timestamp[c]) + field_timestamp[c] = t - fields[c] = n + fields[c] = n - if n > maximum[c]: maximum[c] = n + if n > maximum[c]: + maximum[c] = n - if last_t != t: - last_t = t - maximum = fields + if last_t != t: + last_t = t + maximum = fields for i in keys: - print '%s: avg: %f' % (i, field_sum[i] / last_t) -print + print('%s: avg: %f' % (i, field_sum[i] / last_t)) +print() out.close() out = open('disk_buffer.gnuplot', 'wb') -print >>out, "set term png size 1200,700" -print >>out, 'set output "disk_buffer.png"' -print >>out, 'set xrange [0:*]' -print >>out, 'set xlabel "time (ms)"' -print >>out, 'set ylabel "buffers"' -print >>out, "set style data lines" -print >>out, "set key box" -print >>out, 'plot', +print("set term png size 1200,700", file=out) +print('set output "disk_buffer.png"', file=out) +print('set xrange [0:*]', file=out) +print('set xlabel "time (ms)"', file=out) +print('set ylabel "buffers"', file=out) +print("set style data lines", file=out) +print("set key box", file=out) +print('plot', end=' ', file=out) count = 1 + len(keys) keys.reverse() comma = '' for k in keys: - expr = "$%d" % count - for i in xrange(2, count): expr += "+$%d" % i - count -= 1 - print >>out, ' %s"disk_buffer_log.dat" using 1:(%s) title "%s" with filledcurves x1 lt rgb "#%s"' % (comma, expr, k, colors[count-1]), - comma = ',' + expr = "$%d" % count + for i in range(2, count): + expr += "+$%d" % i + count -= 1 + print(' %s"disk_buffer_log.dat" using 1:(%s) title "%s" with filledcurves x1 lt rgb "#%s"' % + (comma, expr, k, colors[count - 1]), end=' ', file=out) + comma = ',' out.close() os.system('gnuplot disk_buffer.gnuplot') - diff --git a/tools/parse_lookup_log.py b/tools/parse_lookup_log.py index ed44c928d..c751c8512 100755 --- a/tools/parse_lookup_log.py +++ b/tools/parse_lookup_log.py @@ -1,119 +1,129 @@ #!/usr/bin/env python # this is meant to parse the dht_lookups.log generated by parse_dht_log.py +from __future__ import print_function import os nodes = {} + def get_origin(n): - if n in nodes: - return list(nodes[n]['conns']) - else: - return ['0.0.0.0'] + if n in nodes: + return list(nodes[n]['conns']) + else: + return ['0.0.0.0'] + def calculate_pos(nid, dist): - nid = int(nid[0:7], 16) + nid = int(nid[0:7], 16) - x = 0 - y = 0 - for i in range(0, 28, 2): - x |= (nid & (1 << i)) >> (i / 2) - y |= (nid & (2 << i)) >> (i / 2 + 1) + x = 0 + y = 0 + for i in range(0, 28, 2): + x |= (nid & (1 << i)) >> (i / 2) + y |= (nid & (2 << i)) >> (i / 2 + 1) -# print '%d -> %d %d' % (dist, x, y) +# print '%d -> %d %d' % (dist, x, y) + + return (x / 3, y / 3) - return (x / 3, y / 3) def plot_nodes(nodes, frame): - try: os.mkdir('dht_frames') - except: pass + try: + os.mkdir('dht_frames') + except BaseException: + pass - out = open('dht_frames/plot-%02d.dot' % frame, 'w+') - edges = set() - print >>out, 'graph swarm {' -# print >>out, '"tl" [shape=point pos="0,0!"];' -# print >>out, '"tr" [shape=point pos="1638,0!"];' -# print >>out, '"ll" [shape=point pos="1638,1638!"];' -# print >>out, '"tr" [shape=point pos="0,1638!"];' - for dst, n in nodes.items(): - shape = 'point' - if 's' in n: shape = n['s'] + out = open('dht_frames/plot-%02d.dot' % frame, 'w+') + edges = set() + print('graph swarm {', file=out) +# print >>out, '"tl" [shape=point pos="0,0!"];' +# print >>out, '"tr" [shape=point pos="1638,0!"];' +# print >>out, '"ll" [shape=point pos="1638,1638!"];' +# print >>out, '"tr" [shape=point pos="0,1638!"];' + for dst, n in list(nodes.items()): + shape = 'point' + if 's' in n: + shape = n['s'] - print >>out, '"%s" [shape=%s fillcolor="%s" label="" pos="%d,%d!"];' % (dst, shape, n['c'], n['p'][0], n['p'][1]) - for e in n['conns']: - if (e, dst) in edges: continue + print('"%s" [shape=%s fillcolor="%s" label="" pos="%d,%d!"];' % + (dst, shape, n['c'], n['p'][0], n['p'][1]), file=out) + for e in n['conns']: + if (e, dst) in edges: + continue - # only add an edge once to the .dot file - edges.add((e, dst)) - edges.add((dst, e)) + # only add an edge once to the .dot file + edges.add((e, dst)) + edges.add((dst, e)) - style = 'solid' - col = 'gray' - if nodes[dst]['c'] != 'white' and nodes[e]['c'] != 'white': - style = 'solid' - col = 'black' - print >>out, '"%s" -- "%s" [style="%s" color="%s"];' % (e, dst, style, col) + style = 'solid' + col = 'gray' + if nodes[dst]['c'] != 'white' and nodes[e]['c'] != 'white': + style = 'solid' + col = 'black' + print('"%s" -- "%s" [style="%s" color="%s"];' % (e, dst, style, col), file=out) - print >>out, '}' - out.close() - os.system('neato -n dht_frames/plot-%02d.dot -Tpng -o dht_frames/frame-%02d.png' % (frame, frame)) + print('}', file=out) + out.close() + os.system('neato -n dht_frames/plot-%02d.dot -Tpng -o dht_frames/frame-%02d.png' % (frame, frame)) frame = 0 next_render_time = 100 f = open('dht_lookups.txt') -for l in f: - if l.startswith('***'): break +for line in f: + if line.startswith('***'): + break - kind = l[0:3].strip() - l = l[3:].strip().split(' ') + kind = line[0:3].strip() + line = line[3:].strip().split(' ') - if kind == '===': continue + if kind == '===': + continue - t = int(l[0]) - if t > next_render_time: - plot_nodes(nodes, frame) - frame += 1 - next_render_time += 100 -# sys.exit(0) + t = int(line[0]) + if t > next_render_time: + plot_nodes(nodes, frame) + frame += 1 + next_render_time += 100 + # sys.exit(0) - if kind == '<>': - p = calculate_pos(l[1], 0) - dst = '0.0.0.0' - if not dst in nodes: - nodes[dst] = { 'conns': set(), 'p': p, 'c': 'blue', 's': 'circle'} + if kind == '<>': + p = calculate_pos(line[1], 0) + dst = '0.0.0.0' + if dst not in nodes: + nodes[dst] = {'conns': set(), 'p': p, 'c': 'blue', 's': 'circle'} - p = calculate_pos(l[2], 25) - dst = '255.255.255.255' - if not dst in nodes: - nodes[dst] = { 'conns': set(), 'p': p, 'c': 'yellow', 's': 'circle'} - elif kind == '->': - dst = l[3] + p = calculate_pos(line[2], 25) + dst = '255.255.255.255' + if dst not in nodes: + nodes[dst] = {'conns': set(), 'p': p, 'c': 'yellow', 's': 'circle'} + elif kind == '->': + dst = line[3] - if not dst in nodes: - src = get_origin(dst) - p = calculate_pos(l[2], int(l[1])) - nodes[dst] = { 'conns': set(src), 'p': p, 'c': 'grey'} - nodes[dst]['c'] = 'grey' + if dst not in nodes: + src = get_origin(dst) + p = calculate_pos(line[2], int(line[1])) + nodes[dst] = {'conns': set(src), 'p': p, 'c': 'grey'} + nodes[dst]['c'] = 'grey' - elif kind == '+': - dst = l[3] - src = l[4] - p = calculate_pos(l[2], int(l[1])) - if not dst in nodes: - nodes[dst] = { 'conns': set(), 'p': p, 'c': 'white'} - nodes[dst]['conns'].add(src) + elif kind == '+': + dst = line[3] + src = line[4] + p = calculate_pos(line[2], int(line[1])) + if dst not in nodes: + nodes[dst] = {'conns': set(), 'p': p, 'c': 'white'} + nodes[dst]['conns'].add(src) - elif kind == '<-': - dst = l[3] - nodes[dst]['c'] = 'green' - elif kind == 'x': - dst = l[3] - nodes[dst]['c'] = 'orange' - elif kind == 'X': - dst = l[3] - nodes[dst]['c'] = 'red' + elif kind == '<-': + dst = line[3] + nodes[dst]['c'] = 'green' + elif kind == 'x': + dst = line[3] + nodes[dst]['c'] = 'orange' + elif kind == 'X': + dst = line[3] + nodes[dst]['c'] = 'red' f.close() - diff --git a/tools/parse_memory_log.py b/tools/parse_memory_log.py index 17749d49d..22439c6be 100755 --- a/tools/parse_memory_log.py +++ b/tools/parse_memory_log.py @@ -1,5 +1,8 @@ #!/usr/bin/env python -import os, sys, time +from __future__ import print_function + +import os +import sys # usage: memory.log memory_index.log @@ -13,58 +16,60 @@ index = open(sys.argv[2], 'rb').readlines() allocation_points_to_print = 30 + def print_allocation_point(ap): - print 'space_time: %d kBms' % (ap['spacetime'] / 1024) - print 'allocations: %d' % ap['allocations'] - print 'peak: %d kB' % (ap['peak'] / 1024) - print 'stack: ' - counter = 0 - for e in ap['stack']: - print '#%d %s' % (counter, e) - counter += 1 + print('space_time: %d kBms' % (ap['spacetime'] / 1024)) + print('allocations: %d' % ap['allocations']) + print('peak: %d kB' % (ap['peak'] / 1024)) + print('stack: ') + counter = 0 + for e in ap['stack']: + print('#%d %s' % (counter, e)) + counter += 1 + allocation_points = [] -for l in index: - l = l.split('#') - l.pop(0) - ap = { 'allocations': 0, 'peak': 0, 'spacetime': 0, 'allocation_point': len(allocation_points), 'stack': l} - allocation_points.append(ap); +for line in index: + line = line.split('#') + line.pop(0) + ap = {'allocations': 0, 'peak': 0, 'spacetime': 0, 'allocation_point': len(allocation_points), 'stack': line} + allocation_points.append(ap) -for l in lines: - l = l.lstrip('#').rstrip('\n').split(' ') - if len(l) != 8: - print l - continue - try: - ap = int(l[0]) - allocation_points[ap]['allocations'] += 1 - allocation_points[ap]['peak'] = int(l[7]) - allocation_points[ap]['spacetime'] = int(l[6]) - except Exception, e: - print type(e), e, l +for line in lines: + line = line.lstrip('#').rstrip('\n').split(' ') + if len(line) != 8: + print(line) + continue + try: + ap = int(line[0]) + allocation_points[ap]['allocations'] += 1 + allocation_points[ap]['peak'] = int(line[7]) + allocation_points[ap]['spacetime'] = int(line[6]) + except Exception as e: + print(type(e), e, line) -print '=== space time ===' +print('=== space time ===') hot_ap = [] -allocation_points.sort(key = lambda x:x['spacetime'], reverse=True); +allocation_points.sort(key=lambda x: x['spacetime'], reverse=True) counter = 0 for ap in allocation_points[0:allocation_points_to_print]: - print '== %d ==' % counter - counter += 1 - print_allocation_point(ap) - hot_ap.append(ap['allocation_point']); + print('== %d ==' % counter) + counter += 1 + print_allocation_point(ap) + hot_ap.append(ap['allocation_point']) -print '=== allocations ===' +print('=== allocations ===') -allocation_points.sort(key = lambda x:x['allocations'], reverse=True); +allocation_points.sort(key=lambda x: x['allocations'], reverse=True) for ap in allocation_points[0:allocation_points_to_print]: - print_allocation_point(ap) + print_allocation_point(ap) -print '=== peak ===' +print('=== peak ===') -allocation_points.sort(key = lambda x:x['peak'], reverse=True); +allocation_points.sort(key=lambda x: x['peak'], reverse=True) for ap in allocation_points[0:allocation_points_to_print]: - print_allocation_point(ap) + print_allocation_point(ap) # generate graph lines = open(sys.argv[1], 'rb').readlines() @@ -74,53 +79,54 @@ cur_line = [0] * allocation_points_to_print prev_line = [0] * allocation_points_to_print last_time = 0 -for l in lines: - l = l.lstrip('#').rstrip('\n').split(' ') - if len(l) != 8: - print l - continue - try: - time = int(l[1]) - if time != last_time: - print >>out, last_time, '\t', - for i in range(allocation_points_to_print): - if cur_line[i] == -1: - print >>out, prev_line[i], '\t', - else: - print >>out, cur_line[i], '\t', - prev_line[i] = cur_line[i] - print >>out - cur_line = [-1] * allocation_points_to_print - last_time = time +for line in lines: + line = line.lstrip('#').rstrip('\n').split(' ') + if len(line) != 8: + print(line) + continue + try: + time = int(line[1]) + if time != last_time: + print(last_time, '\t', end=' ', file=out) + for i in range(allocation_points_to_print): + if cur_line[i] == -1: + print(prev_line[i], '\t', end=' ', file=out) + else: + print(cur_line[i], '\t', end=' ', file=out) + prev_line[i] = cur_line[i] + print(file=out) + cur_line = [-1] * allocation_points_to_print + last_time = time - size = int(l[5]) - ap = int(l[0]) - if ap in hot_ap: - index = hot_ap.index(ap) - cur_line[index] = max(cur_line[index], size) + size = int(line[5]) + ap = int(line[0]) + if ap in hot_ap: + index = hot_ap.index(ap) + cur_line[index] = max(cur_line[index], size) - except Exception, e: - print type(e), e, l + except Exception as e: + print(type(e), e, line) out.close() out = open('memory.gnuplot', 'wb') -print >>out, "set term png size 1200,700" -print >>out, 'set output "memory.png"' -print >>out, 'set xrange [0:*]' -print >>out, 'set xlabel "time (ms)"' -print >>out, 'set ylabel "bytes (B)"' -print >>out, "set style data lines" -print >>out, "set key box" -print >>out, 'plot', +print("set term png size 1200,700", file=out) +print('set output "memory.png"', file=out) +print('set xrange [0:*]', file=out) +print('set xlabel "time (ms)"', file=out) +print('set ylabel "bytes (B)"', file=out) +print("set style data lines", file=out) +print("set key box", file=out) +print('plot', end=' ', file=out) for k in range(allocation_points_to_print): - print >>out, ' "memory.dat" using 1:(', - for i in range(k, allocation_points_to_print): - if i == k: print >>out, '$%d' % (i + 2), - else: print >>out, '+$%d' % (i + 2), - print >>out, ') title "%d" with filledcurves x1, \\' % k -print >>out, 'x=0' + print(' "memory.dat" using 1:(', end=' ', file=out) + for i in range(k, allocation_points_to_print): + if i == k: + print('$%d' % (i + 2), end=' ', file=out) + else: + print('+$%d' % (i + 2), end=' ', file=out) + print(') title "%d" with filledcurves x1, \\' % k, file=out) +print('x=0', file=out) out.close() -os.system('gnuplot memory.gnuplot'); - +os.system('gnuplot memory.gnuplot') diff --git a/tools/parse_peer_log.py b/tools/parse_peer_log.py index 45dc1a6e6..5436a7d0d 100755 --- a/tools/parse_peer_log.py +++ b/tools/parse_peer_log.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +from __future__ import print_function + import glob import os import sys @@ -9,64 +11,64 @@ import sys log_files = [] for p in glob.iglob(os.path.join(sys.argv[1], '*.log')): - name = os.path.split(p)[1] - if name == 'main_session.log': continue - print name - f = open(p, 'r') - out_file = p + '.dat' - log_files.append(out_file) - out = open(out_file, 'w+') + name = os.path.split(p)[1] + if name == 'main_session.log': + continue + print(name) + f = open(p, 'r') + out_file = p + '.dat' + log_files.append(out_file) + out = open(out_file, 'w+') - uploaded_blocks = 0; - downloaded_blocks = 0; + uploaded_blocks = 0 + downloaded_blocks = 0 - for l in f: - t = l.split(': ')[0].split('.')[0] - log_line = False - if ' ==> PIECE' in l: - uploaded_blocks+= 1 - log_line = True + for l in f: + t = l.split(': ')[0].split('.')[0] + log_line = False + if ' ==> PIECE' in l: + uploaded_blocks += 1 + log_line = True - if ' <== PIECE' in l: - downloaded_blocks+= 1 - log_line = True + if ' <== PIECE' in l: + downloaded_blocks += 1 + log_line = True - if log_line: - print >>out, '%s\t%d\t%d' % (t, uploaded_blocks, downloaded_blocks) + if log_line: + print('%s\t%d\t%d' % (t, uploaded_blocks, downloaded_blocks), file=out) - out.close() - f.close() + out.close() + f.close() out = open('peers.gnuplot', 'wb') -print >>out, "set term png size 1200,700" -print >>out, 'set xrange [0:*]' -print >>out, 'set xlabel "time"' -print >>out, 'set ylabel "blocks"' -print >>out, 'set key box' -print >>out, 'set xdata time' -print >>out, 'set timefmt "%H:%M:%S"' -print >>out, 'set title "uploaded blocks"' -print >>out, 'set output "peers_upload.png"' -print >>out, 'plot', +print("set term png size 1200,700", file=out) +print('set xrange [0:*]', file=out) +print('set xlabel "time"', file=out) +print('set ylabel "blocks"', file=out) +print('set key box', file=out) +print('set xdata time', file=out) +print('set timefmt "%H:%M:%S"', file=out) +print('set title "uploaded blocks"', file=out) +print('set output "peers_upload.png"', file=out) +print('plot', end=' ', file=out) first = True for n in log_files: - if not first: - print >>out, ',', - first = False - print >>out, ' "%s" using 1:2 title "%s" with steps' % (n, os.path.split(n)[1].split('.log')[0]), -print >>out, '' + if not first: + print(',', end=' ', file=out) + first = False + print(' "%s" using 1:2 title "%s" with steps' % (n, os.path.split(n)[1].split('.log')[0]), end=' ', file=out) +print('', file=out) -print >>out, 'set title "downloaded blocks"' -print >>out, 'set output "peers_download.png"' -print >>out, 'plot', +print('set title "downloaded blocks"', file=out) +print('set output "peers_download.png"', file=out) +print('plot', end=' ', file=out) first = True for n in log_files: - if not first: - print >>out, ',', - first = False - print >>out, ' "%s" using 1:3 title "%s" with steps' % (n, os.path.split(n)[1].split('.log')[0]), -print >>out, '' + if not first: + print(',', end=' ', file=out) + first = False + print(' "%s" using 1:3 title "%s" with steps' % (n, os.path.split(n)[1].split('.log')[0]), end=' ', file=out) +print('', file=out) out.close() -os.system('gnuplot peers.gnuplot'); - +os.system('gnuplot peers.gnuplot') diff --git a/tools/parse_sample.py b/tools/parse_sample.py index 3961914f0..bb8c03b4d 100755 --- a/tools/parse_sample.py +++ b/tools/parse_sample.py @@ -7,103 +7,155 @@ import sys # relevant depths and to filter out low sample counts f = open(sys.argv[1]) -def parse_line(l): - indentation = 0 - while indentation < len(l) and l[indentation] == ' ': - indentation += 1 - if indentation == 0: - return (0, 0, '') +def parse_line(line): + indentation = 0 + while indentation < len(line) and line[indentation] == ' ': + indentation += 1 + if indentation == 0: + return (0, 0, '') - l = l.strip().split(' ') - samples = int(l[0]) - fun = ' '.join(l[1:]) + line = line.strip().split(' ') + samples = int(line[0]) + fun = ' '.join(line[1:]) + + return (indentation, samples, fun) - return (indentation, samples, fun) fold = -1 try: - sample_limit = int(sys.argv[2]) -except: - sample_limit = 5 + sample_limit = int(sys.argv[2]) +except BaseException: + sample_limit = 5 fun_samples = {} for l in f: - if 'Sort by top of stack' in l: break + if 'Sort by top of stack' in l: + break - indentation, samples, fun = parse_line(l) - if samples < sample_limit: continue - if fold != -1 and indentation > fold: continue - fold = -1 + indentation, samples, fun = parse_line(l) + if samples < sample_limit: + continue + if fold != -1 and indentation > fold: + continue + fold = -1 - if '__gnu_cxx::__normal_iterator<' in fun: - fold = indentation - 1 - continue + if '__gnu_cxx::__normal_iterator<' in fun: + fold = indentation - 1 + continue - if 'boost::_bi::bind_t' in fun: continue - if 'boost::_bi::list' in fun: continue - if 'boost::_mfi::mf' in fun: continue - if 'boost::_bi::storage' in fun: continue + if 'boost::_bi::bind_t' in fun: + continue + if 'boost::_bi::list' in fun: + continue + if 'boost::_mfi::mf' in fun: + continue + if 'boost::_bi::storage' in fun: + continue # should only add leaves - if fun in fun_samples: fun_samples[fun] += samples - else: fun_samples[fun] = samples + if fun in fun_samples: + fun_samples[fun] += samples + else: + fun_samples[fun] = samples - output = '%s%-4d %s' % (' ' * (indentation/2), samples, fun) - if len(output) > 200: output = output[0:200] - print output + output = '%s%-4d %s' % (' ' * (indentation / 2), samples, fun) + if len(output) > 200: + output = output[0:200] + print(output) - if 'invariant_checker_impl' in fun: fold = indentation - if 'free_multiple_buffers' in fun: fold = indentation - if 'libtorrent::condition::wait' in fun: fold = indentation - if 'allocate_buffer' in fun: fold = indentation - if '::find_POD' in fun: fold = indentation - if 'SHA1_Update' in fun: fold = indentation - if 'boost::detail::function::basic_vtable' in fun: fold = indentation - if 'operator new' in fun: fold = indentation - if 'malloc' == fun: fold = indentation - if 'free' == fun: fold = indentation - if 'std::_Rb_tree' in fun: fold = indentation - if 'pthread_cond_wait' in fun: fold = indentation - if 'mp_exptmod' == fun: fold = indentation - if '::check_invariant()' in fun: fold = indentation - if 'libtorrent::condition::wait' in fun: fold = indentation - if '_sigtramp' in fun: fold = indentation - if 'time_now_hires' in fun: fold = indentation - if 'libtorrent::sleep' in fun: fold = indentation - if 'puts' == fun: fold = indentation - if 'boost::asio::basic_stream_socket' in fun: fold = indentation - if 'recvmsg' == fun: fold = indentation - if 'sendmsg' == fun: fold = indentation - if 'semaphore_signal_trap' == fun: fold = indentation - if 'boost::detail::atomic_count::operator' in fun: fold = indentation - if 'pthread_mutex_lock' == fun: fold = indentation - if 'pthread_mutex_unlock' == fun: fold = indentation - if '>::~vector()' == fun: fold = indentation - if 'szone_free_definite_size' == fun: fold = indentation - if 'snprintf' == fun: fold = indentation - if 'usleep' == fun: fold = indentation - if 'pthread_mutex_lock' == fun: fold = indentation - if 'pthread_mutex_unlock' == fun: fold = indentation - if 'std::string::append' in fun: fold = indentation - if 'getipnodebyname' == fun: fold = indentation - if '__gnu_debug::_Safe_iterator::~vector()' == fun: + fold = indentation + if 'szone_free_definite_size' == fun: + fold = indentation + if 'snprintf' == fun: + fold = indentation + if 'usleep' == fun: + fold = indentation + if 'pthread_mutex_lock' == fun: + fold = indentation + if 'pthread_mutex_unlock' == fun: + fold = indentation + if 'std::string::append' in fun: + fold = indentation + if 'getipnodebyname' == fun: + fold = indentation + if '__gnu_debug::_Safe_iterator/dev/null' % script) - if ret != 0 and ret != 256: - print 'system: %d\n' % ret - raise Exception("abort") + ret = os.system('gnuplot "%s" 2>/dev/null' % script) + if ret != 0 and ret != 256: + print('system: %d\n' % ret) + raise Exception("abort") + + sys.stdout.write('.') + sys.stdout.flush() - sys.stdout.write('.') - sys.stdout.flush() def to_title(key): - return key.replace('_', ' ').replace('.', ' - ') + return key.replace('_', ' ').replace('.', ' - ') + def gen_report(name, unit, lines, short_unit, generation, log_file, options): - filename = os.path.join(output_dir, '%s_%04d.png' % (name, generation)) - thumb = os.path.join(output_dir, '%s_%04d_thumb.png' % (name, generation)) + filename = os.path.join(output_dir, '%s_%04d.png' % (name, generation)) + thumb = os.path.join(output_dir, '%s_%04d_thumb.png' % (name, generation)) - # don't re-render a graph unless the logfile has changed - try: - dst1 = os.stat(filename) - dst2 = os.stat(thumb) - src = os.stat(log_file) + # don't re-render a graph unless the logfile has changed + try: + dst1 = os.stat(filename) + dst2 = os.stat(thumb) + src = os.stat(log_file) - if dst1.st_mtime > src.st_mtime and dst2.st_mtime > src.st_mtime: - sys.stdout.write('.') - return None + if dst1.st_mtime > src.st_mtime and dst2.st_mtime > src.st_mtime: + sys.stdout.write('.') + return None - except: pass - - script = os.path.join(output_dir, '%s_%04d.gnuplot' % (name, generation)) - out = open(script, 'wb') - print >>out, "set term png size 1200,700" - print >>out, 'set output "%s"' % filename - if not 'allow-negative' in options: - print >>out, 'set yrange [0:*]' - print >>out, "set tics nomirror" - print >>out, "set key box" - print >>out, "set key left top" + except BaseException: + pass - colors = graph_colors - if options['type'] == line_graph: - colors = line_colors + script = os.path.join(output_dir, '%s_%04d.gnuplot' % (name, generation)) + out = open(script, 'wb') + print("set term png size 1200,700", file=out) + print('set output "%s"' % filename, file=out) + if 'allow-negative' not in options: + print('set yrange [0:*]', file=out) + print("set tics nomirror", file=out) + print("set key box", file=out) + print("set key left top", file=out) - try: - if options['colors'] == 'gradient16': - colors = gradient16_colors - elif options['colors'] == 'gradient6': - colors = gradient6_colors - if options['colors'] == 'gradient18': - colors = gradient18_colors - except: pass + colors = graph_colors + if options['type'] == line_graph: + colors = line_colors - if options['type'] == histogram: - binwidth = options['binwidth'] - numbins = int(options['numbins']) + try: + if options['colors'] == 'gradient16': + colors = gradient16_colors + elif options['colors'] == 'gradient6': + colors = gradient6_colors + if options['colors'] == 'gradient18': + colors = gradient18_colors + except BaseException: + pass - print >>out, 'binwidth=%f' % binwidth - print >>out, 'set boxwidth binwidth' - print >>out, 'bin(x,width)=width*floor(x/width) + binwidth/2' - print >>out, 'set xrange [0:%f]' % (binwidth * numbins) - print >>out, 'set xlabel "%s"' % unit - print >>out, 'set ylabel "number"' + if options['type'] == histogram: + binwidth = options['binwidth'] + numbins = int(options['numbins']) - k = lines[0] - try: - column = keys.index(k) + 2 - except: - print '"%s" not found' % k - return - print >>out, 'plot "%s" using (bin($%d,binwidth)):(1.0) smooth freq with boxes' % (log_file, column) - print >>out, '' - print >>out, '' - print >>out, '' + print('binwidth=%f' % binwidth, file=out) + print('set boxwidth binwidth', file=out) + print('bin(x,width)=width*floor(x/width) + binwidth/2', file=out) + print('set xrange [0:%f]' % (binwidth * numbins), file=out) + print('set xlabel "%s"' % unit, file=out) + print('set ylabel "number"', file=out) - elif options['type'] == stacked: - print >>out, 'set xrange [0:*]' - print >>out, 'set ylabel "%s"' % unit - print >>out, 'set xlabel "time (s)"' - print >>out, 'set format y "%%.1s%%c%s";' % short_unit - print >>out, 'set style fill solid 1.0 noborder' - print >>out, 'plot', - column = 2 - first = True - graph = '' - plot_expression = '' - color = 0 - for k in lines: - try: - column = keys.index(k) + 2 - except: - print '"%s" not found' % k - continue; - if not first: - plot_expression = ', ' + plot_expression - graph += '+' - axis = 'x1y1' - graph += '$%d' % column - plot_expression = ' "%s" using 1:(%s) title "%s" axes %s with filledcurves x1 lc rgb "%s"' % (log_file, graph, to_title(k), axis, colors[color % len(colors)]) + plot_expression - first = False - color += 1 - print >>out, plot_expression - elif options['type'] == diff: - print >>out, 'set xrange [0:*]' - print >>out, 'set ylabel "%s"' % unit - print >>out, 'set xlabel "time (s)"' - print >>out, 'set format y "%%.1s%%c%s";' % short_unit - column = 2 - first = True - graph = '' - title = '' - for k in lines: - try: - column = keys.index(k) + 2 - except: - print '"%s" not found' % k - continue; - if not first: - graph += '-' - title += ' - ' - graph += '$%d' % column - title += to_title(k) - first = False - print >>out, 'plot "%s" using 1:(%s) title "%s" with step' % (log_file, graph, title) - else: - print >>out, 'set xrange [0:*]' - print >>out, 'set ylabel "%s"' % unit - print >>out, 'set xlabel "time (s)"' - print >>out, 'set format y "%%.1s%%c%s";' % short_unit - print >>out, 'plot', - column = 2 - first = True - color = 0 - for k in lines: - try: - column = keys.index(k) + 2 - except: - print '"%s" not found' % k - continue; - if not first: print >>out, ', ', - axis = 'x1y1' - print >>out, ' "%s" using 1:%d title "%s" axes %s with steps lc rgb "%s"' % (log_file, column, to_title(k), axis, colors[color % len(colors)]), - first = False - color += 1 - print >>out, '' + k = lines[0] + try: + column = keys.index(k) + 2 + except BaseException: + print('"%s" not found' % k) + return + print('plot "%s" using (bin($%d,binwidth)):(1.0) smooth freq with boxes' % (log_file, column), file=out) + print('', file=out) + print('', file=out) + print('', file=out) + + elif options['type'] == stacked: + print('set xrange [0:*]', file=out) + print('set ylabel "%s"' % unit, file=out) + print('set xlabel "time (s)"', file=out) + print('set format y "%%.1s%%c%s";' % short_unit, file=out) + print('set style fill solid 1.0 noborder', file=out) + print('plot', end=' ', file=out) + column = 2 + first = True + graph = '' + plot_expression = '' + color = 0 + for k in lines: + try: + column = keys.index(k) + 2 + except BaseException: + print('"%s" not found' % k) + continue + if not first: + plot_expression = ', ' + plot_expression + graph += '+' + axis = 'x1y1' + graph += '$%d' % column + plot_expression = ' "%s" using 1:(%s) title "%s" axes %s with filledcurves x1 lc rgb "%s"' % ( + log_file, graph, to_title(k), axis, colors[color % len(colors)]) + plot_expression + first = False + color += 1 + print(plot_expression, file=out) + elif options['type'] == diff: + print('set xrange [0:*]', file=out) + print('set ylabel "%s"' % unit, file=out) + print('set xlabel "time (s)"', file=out) + print('set format y "%%.1s%%c%s";' % short_unit, file=out) + column = 2 + first = True + graph = '' + title = '' + for k in lines: + try: + column = keys.index(k) + 2 + except BaseException: + print('"%s" not found' % k) + continue + if not first: + graph += '-' + title += ' - ' + graph += '$%d' % column + title += to_title(k) + first = False + print('plot "%s" using 1:(%s) title "%s" with step' % (log_file, graph, title), file=out) + else: + print('set xrange [0:*]', file=out) + print('set ylabel "%s"' % unit, file=out) + print('set xlabel "time (s)"', file=out) + print('set format y "%%.1s%%c%s";' % short_unit, file=out) + print('plot', end=' ', file=out) + column = 2 + first = True + color = 0 + for k in lines: + try: + column = keys.index(k) + 2 + except BaseException: + print('"%s" not found' % k) + continue + if not first: + print(', ', end=' ', file=out) + axis = 'x1y1' + print(' "%s" using 1:%d title "%s" axes %s with steps lc rgb "%s"' % + (log_file, column, to_title(k), axis, colors[color % len(colors)]), end=' ', file=out) + first = False + color += 1 + print('', file=out) + + print('set term png size 150,100', file=out) + print('set output "%s"' % thumb, file=out) + print('set key off', file=out) + print('unset tics', file=out) + print('set format x ""', file=out) + print('set format y ""', file=out) + print('set xlabel ""', file=out) + print('set ylabel ""', file=out) + print('set y2label ""', file=out) + print('set rmargin 0', file=out) + print('set lmargin 0', file=out) + print('set tmargin 0', file=out) + print('set bmargin 0', file=out) + print("replot", file=out) + out.close() + return script - print >>out, 'set term png size 150,100' - print >>out, 'set output "%s"' % thumb - print >>out, 'set key off' - print >>out, 'unset tics' - print >>out, 'set format x ""' - print >>out, 'set format y ""' - print >>out, 'set xlabel ""' - print >>out, 'set ylabel ""' - print >>out, 'set y2label ""' - print >>out, 'set rmargin 0' - print >>out, 'set lmargin 0' - print >>out, 'set tmargin 0' - print >>out, 'set bmargin 0' - print >>out, "replot" - out.close() - return script def gen_html(reports, generations): - file = open(os.path.join(output_dir, 'index.html'), 'w+') + file = open(os.path.join(output_dir, 'index.html'), 'w+') - css = '''img { margin: 0} - #head { display: block } - #graphs { white-space:nowrap; } - h1 { line-height: 1; display: inline } - h2 { line-height: 1; display: inline; font-size: 1em; font-weight: normal};''' + css = '''img { margin: 0} +#head { display: block } +#graphs { white-space:nowrap; } +h1 { line-height: 1; display: inline } +h2 { line-height: 1; display: inline; font-size: 1em; font-weight: normal};''' - print >>file, '' % css + print('' % css, file=file) - for i in reports: - print >>file, '