premiere
/
premiere-libtorrent
2
2
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Clean-up python code (#3075) 

lint and enforce python code formatting
This commit is contained in:
Andrew Resch 2018-06-12 02:43:13 -07:00 committed by Arvid Norberg
parent a2ea79de4e
commit 596d98cac4
34 changed files with 4016 additions and 3435 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.travis.yml
View File

@ -5,7 +5,7 @@ matrix:
fast_finish: true fast_finish: true
include: include:
- if: repo = arvidn/libtorrent - if: repo = arvidn/libtorrent
env: variant=release sonar_scan=1 toolset=gcc env: variant=release sonar_scan=1 toolset=gcc pylint=1
- env: variant=test_debug lint=1 tests=1 toolset=gcc-sanitizer - env: variant=test_debug lint=1 tests=1 toolset=gcc-sanitizer
- env: variant=test_debug sim=1 crypto=openssl toolset=gcc-sanitizer - env: variant=test_debug sim=1 crypto=openssl toolset=gcc-sanitizer
- env: variant=test_release coverage=1 tests=1 toolset=gcc-coverage python=1 - env: variant=test_release coverage=1 tests=1 toolset=gcc-coverage python=1
@ -46,6 +46,7 @@ addons:
- python2.7-dev - python2.7-dev
- g++-5 - g++-5
- [cmake3, ninja-build] - [cmake3, ninja-build]
- python3-pip
before_install: before_install:
@ -79,7 +80,6 @@ before_install:
- ulimit -a - ulimit -a
install: install:
- touch ~/user-config.jam - touch ~/user-config.jam
- 'if [[ $toolset == "gcc" ]]; then - 'if [[ $toolset == "gcc" ]]; then
g++-5 --version; g++-5 --version;
@ -130,7 +130,12 @@ install:
- 'echo "using python : 2.7 ;" >> ~/user-config.jam' - 'echo "using python : 2.7 ;" >> ~/user-config.jam'
- if [ "$docs" == "1" ]; then rst2html.py --version; fi - if [ "$docs" == "1" ]; then rst2html.py --version; fi
- 'if [ "$lint" == "1" ]; then curl "https://raw.githubusercontent.com/google/styleguide/71ec7f1e524969c19ce33cfc72e8e023f2b98ee2/cpplint/cpplint.py" >~/cpplint.py; fi' - 'if [ "$lint" == "1" ]; then curl "https://raw.githubusercontent.com/google/styleguide/71ec7f1e524969c19ce33cfc72e8e023f2b98ee2/cpplint/cpplint.py" >~/cpplint.py; fi'
- 'if [ "$pylint" == "1" ]; then
sudo pip install flake8;
flake8 --version;
sudo pip3 install flake8;
python3 -m flake8 --version;
fi'
- 'if [ $sonar_scan == "1" ]; then - 'if [ $sonar_scan == "1" ]; then
wget https://sonarsource.bintray.com/Distribution/sonar-scanner-cli/sonar-scanner-2.6.1.zip; wget https://sonarsource.bintray.com/Distribution/sonar-scanner-cli/sonar-scanner-2.6.1.zip;
wget https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip; wget https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip;
@ -183,7 +188,10 @@ script:
- 'if [ "$lint" == "1" ]; then - 'if [ "$lint" == "1" ]; then
python ~/cpplint.py --extensions=cpp --headers=hpp --filter=-,+runtime/explicit,+whitespace/end_of_line --linelength=90 test/*.{cpp,hpp} src/*.cpp include/libtorrent/*.hpp include/libtorrent/kademlia/*.hpp src/kademlia/*.cpp include/libtorrent/aux_/*.hpp include/libtorrent/extensions/*.hpp simulation/*.{cpp,hpp} tools/*.{cpp,hpp} examples/*.{cpp,hpp}; python ~/cpplint.py --extensions=cpp --headers=hpp --filter=-,+runtime/explicit,+whitespace/end_of_line --linelength=90 test/*.{cpp,hpp} src/*.cpp include/libtorrent/*.hpp include/libtorrent/kademlia/*.hpp src/kademlia/*.cpp include/libtorrent/aux_/*.hpp include/libtorrent/extensions/*.hpp simulation/*.{cpp,hpp} tools/*.{cpp,hpp} examples/*.{cpp,hpp};
fi' fi'
- 'if [ "$pylint" == "1" ]; then
flake8 --max-line-length=120;
python3 -m flake8 --max-line-length=120;
fi'
- 'if [ "$sonar_scan" == "1" ]; then - 'if [ "$sonar_scan" == "1" ]; then
build-wrapper-linux-x86-64 --out-dir bw-output bjam -a -j3 optimization=off crypto=$crypto deprecated-functions=off $toolset variant=$variant -l300 && build-wrapper-linux-x86-64 --out-dir bw-output bjam -a -j3 optimization=off crypto=$crypto deprecated-functions=off $toolset variant=$variant -l300 &&
sonar-scanner -D sonar.login=$SONAR_TOKEN; sonar-scanner -D sonar.login=$SONAR_TOKEN;

46
bindings/python/client.py
View File

@ -3,7 +3,7 @@
# Copyright Daniel Wallin 2006. Use, modification and distribution is # Copyright Daniel Wallin 2006. Use, modification and distribution is
# subject to the Boost Software License, Version 1.0. (See accompanying # subject to the Boost Software License, Version 1.0. (See accompanying
# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
from __future__ import print_function
import sys import sys
import atexit import atexit
@ -153,6 +153,7 @@ def print_download_queue(console, download_queue):
write_line(console, out) write_line(console, out)
def add_torrent(ses, filename, options): def add_torrent(ses, filename, options):
atp = lt.add_torrent_params() atp = lt.add_torrent_params()
if filename.startswith('magnet:'): if filename.startswith('magnet:'):
@ -160,8 +161,8 @@ def add_torrent(ses, filename, options):
else: else:
atp.ti = lt.torrent_info(filename) atp.ti = lt.torrent_info(filename)
try: try:
at.resume_data = open(os.path.join(options.save_path, info.name() + '.fastresume'), 'rb').read() atp.resume_data = open(os.path.join(options.save_path, atp.info.name() + '.fastresume'), 'rb').read()
except: except BaseException:
pass pass
atp.save_path = options.save_path atp.save_path = options.save_path
@ -171,6 +172,7 @@ def add_torrent(ses, filename, options):
| lt.torrent_flags.duplicate_is_error | lt.torrent_flags.duplicate_is_error
ses.async_add_torrent(atp) ses.async_add_torrent(atp)
def main(): def main():
from optparse import OptionParser from optparse import OptionParser
@ -225,12 +227,13 @@ def main():
if options.max_download_rate <= 0: if options.max_download_rate <= 0:
options.max_download_rate = -1 options.max_download_rate = -1
settings = { 'user_agent': 'python_client/' + lt.__version__, settings = {
'user_agent': 'python_client/' + lt.__version__,
'listen_interfaces': '%s:%d' % (options.listen_interface, options.port), 'listen_interfaces': '%s:%d' % (options.listen_interface, options.port),
'download_rate_limit': int(options.max_download_rate), 'download_rate_limit': int(options.max_download_rate),
'upload_rate_limit': int(options.max_upload_rate), 'upload_rate_limit': int(options.max_upload_rate),
'alert_mask': lt.alert.category_t.all_categories, 'alert_mask': lt.alert.category_t.all_categories,
'outgoing_interfaces' : options.outgoing_interface 'outgoing_interfaces': options.outgoing_interface,
} }
if options.proxy_host != '': if options.proxy_host != '':
@ -258,16 +261,16 @@ def main():
out = '' out = ''
for h,t in torrents.items(): for h, t in torrents.items():
out += 'name: %-40s\n' % t.name[:40] out += 'name: %-40s\n' % t.name[:40]
if t.state != lt.torrent_status.seeding: if t.state != lt.torrent_status.seeding:
state_str = ['queued', 'checking', 'downloading metadata', \ state_str = ['queued', 'checking', 'downloading metadata',
'downloading', 'finished', 'seeding', \ 'downloading', 'finished', 'seeding',
'allocating', 'checking fastresume'] 'allocating', 'checking fastresume']
out += state_str[t.state] + ' ' out += state_str[t.state] + ' '
out += '%5.4f%% ' % (t.progress*100) out += '%5.4f%% ' % (t.progress * 100)
out += progress_bar(t.progress, 49) out += progress_bar(t.progress, 49)
out += '\n' out += '\n'
@ -300,7 +303,7 @@ def main():
out += progress_bar(p / float(f.size), 20) out += progress_bar(p / float(f.size), 20)
out += ' ' + f.path + '\n' out += ' ' + f.path + '\n'
write_line(console, out) write_line(console, out)
except: except BaseException:
pass pass
write_line(console, 76 * '-' + '\n') write_line(console, 76 * '-' + '\n')
@ -312,7 +315,7 @@ def main():
alerts_log.append(a.message()) alerts_log.append(a.message())
# add new torrents to our list of torrent_status # add new torrents to our list of torrent_status
if type(a) == lt.add_torrent_alert: if isinstance(a, lt.add_torrent_alert):
h = a.handle h = a.handle
h.set_max_connections(60) h.set_max_connections(60)
h.set_max_uploads(-1) h.set_max_uploads(-1)
@ -320,7 +323,7 @@ def main():
# update our torrent_status array for torrents that have # update our torrent_status array for torrents that have
# changed some of their state # changed some of their state
if type(a) == lt.state_update_alert: if isinstance(a, lt.state_update_alert):
for s in a.status: for s in a.status:
torrents[s.handle] = s torrents[s.handle] = s
@ -333,19 +336,23 @@ def main():
c = console.sleep_and_input(0.5) c = console.sleep_and_input(0.5)
ses.post_torrent_updates() ses.post_torrent_updates()
if not c: continue if not c:
continue
if c == 'r': if c == 'r':
for h in torrents.keys(): h.force_reannounce() for h in torrents:
h.force_reannounce()
elif c == 'q': elif c == 'q':
alive = False alive = False
elif c == 'p': elif c == 'p':
for h in torrents.keys(): h.pause() for h in torrents:
h.pause()
elif c == 'u': elif c == 'u':
for h in torrents.keys(): h.resume() for h in torrents:
h.resume()
ses.pause() ses.pause()
for h,t in torrents.items(): for h, t in torrents.items():
if not h.is_valid() or not t.has_metadata: if not h.is_valid() or not t.has_metadata:
continue continue
h.save_resume_data() h.save_resume_data()
@ -353,7 +360,7 @@ def main():
while len(torrents) > 0: while len(torrents) > 0:
alerts = ses.pop_alerts() alerts = ses.pop_alerts()
for a in alerts: for a in alerts:
if type(a) == lt.save_resume_data_alert: if isinstance(a, lt.save_resume_data_alert):
print(a) print(a)
data = lt.write_resume_data_buf(a.params) data = lt.write_resume_data_buf(a.params)
h = a.handle h = a.handle
@ -361,11 +368,12 @@ def main():
open(os.path.join(options.save_path, torrents[h].name + '.fastresume'), 'wb').write(data) open(os.path.join(options.save_path, torrents[h].name + '.fastresume'), 'wb').write(data)
del torrents[h] del torrents[h]
if type(a) == lt.save_resume_data_failed_alert: if isinstance(a, lt.save_resume_data_failed_alert):
h = a.handle h = a.handle
if h in torrents: if h in torrents:
print('failed to save resume data for ', torrents[h].name) print('failed to save resume data for ', torrents[h].name)
del torrents[h] del torrents[h]
time.sleep(0.5) time.sleep(0.5)
main() main()

4
bindings/python/make_torrent.py
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function
import sys import sys
import os import os
@ -39,7 +39,7 @@ for root, dirs, files in os.walk(input):
if f == 'Thumbs.db': if f == 'Thumbs.db':
continue continue
fname = os.path.join(root[len(parent_input)+1:], f) fname = os.path.join(root[len(parent_input) + 1:], f)
size = os.path.getsize(os.path.join(parent_input, fname)) size = os.path.getsize(os.path.join(parent_input, fname))
print('%10d kiB %s' % (size / 1024, fname)) print('%10d kiB %s' % (size / 1024, fname))
fs.add_file(fname, size) fs.add_file(fname, size)

39
bindings/python/setup.py
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function
from distutils.core import setup, Extension from distutils.core import setup, Extension
from distutils.sysconfig import get_config_vars from distutils.sysconfig import get_config_vars
@ -55,13 +55,13 @@ def target_specific():
try: try:
with open('compile_flags') as _file: with open('compile_flags') as _file:
extra_cmd = _file.read() extra_cmd = _file.read()
except: except BaseException:
extra_cmd = None extra_cmd = None
try: try:
with open('link_flags') as _file: with open('link_flags') as _file:
ldflags = _file.read() ldflags = _file.read()
except: except BaseException:
ldflags = None ldflags = None
# this is to pull out compiler arguments from the CXX flags set up by the # this is to pull out compiler arguments from the CXX flags set up by the
@ -75,7 +75,7 @@ try:
while len(cmd) > 0 and not cmd[0].startswith('-'): while len(cmd) > 0 and not cmd[0].startswith('-'):
cmd = cmd[1:] cmd = cmd[1:]
extra_cmd += ' '.join(cmd) extra_cmd += ' '.join(cmd)
except: except BaseException:
pass pass
ext = None ext = None
@ -129,14 +129,22 @@ if '--bjam' in sys.argv:
print('build failed') print('build failed')
sys.exit(1) sys.exit(1)
try: os.mkdir('build') try:
except: pass os.mkdir('build')
try: shutil.rmtree('build/lib') except BaseException:
except: pass pass
try: os.mkdir('build/lib') try:
except: pass shutil.rmtree('build/lib')
try: os.mkdir('libtorrent') except BaseException:
except: pass pass
try:
os.mkdir('build/lib')
except BaseException:
pass
try:
os.mkdir('libtorrent')
except BaseException:
pass
shutil.copyfile('libtorrent' + file_ext, shutil.copyfile('libtorrent' + file_ext,
'build/lib/libtorrent' + file_ext) 'build/lib/libtorrent' + file_ext)
@ -145,13 +153,12 @@ if '--bjam' in sys.argv:
else: else:
# Remove '-Wstrict-prototypes' compiler option, which isn't valid for C++. # Remove '-Wstrict-prototypes' compiler option, which isn't valid for C++.
cfg_vars = get_config_vars() cfg_vars = get_config_vars()
for key, value in cfg_vars.items(): for key, value in list(cfg_vars.items()):
if isinstance(value, str): if isinstance(value, str):
cfg_vars[key] = value.replace('-Wstrict-prototypes', '') cfg_vars[key] = value.replace('-Wstrict-prototypes', '')
source_list = os.listdir(os.path.join(os.path.dirname(__file__), "src")) src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "src"))
source_list = [os.path.abspath(os.path.join(os.path.dirname(__file__), source_list = [os.path.join(src_dir, s) for s in os.listdir(src_dir) if s.endswith(".cpp")]
"src", s)) for s in source_list if s.endswith(".cpp")]
if extra_cmd: if extra_cmd:
flags = flags_parser() flags = flags_parser()

2
bindings/python/simple_client.py
View File

@ -8,7 +8,7 @@ import libtorrent as lt
import time import time
import sys import sys
ses = lt.session({'listen_interfaces':'0.0.0.0:6881'}) ses = lt.session({'listen_interfaces': '0.0.0.0:6881'})
info = lt.torrent_info(sys.argv[1]) info = lt.torrent_info(sys.argv[1])
h = ses.add_torrent({'ti': info, 'save_path': '.'}) h = ses.add_torrent({'ti': info, 'save_path': '.'})

90
bindings/python/test.py
View File

@ -1,7 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
from __future__ import print_function
import libtorrent as lt import libtorrent as lt
@ -13,7 +12,6 @@ import shutil
import binascii import binascii
import subprocess as sub import subprocess as sub
import sys import sys
import inspect
import pickle import pickle
import threading import threading
@ -29,6 +27,7 @@ settings = {
'enable_dht': False, 'enable_lsd': False, 'enable_natpmp': False, 'enable_dht': False, 'enable_lsd': False, 'enable_natpmp': False,
'enable_upnp': False, 'listen_interfaces': '0.0.0.0:0', 'file_pool_size': 1} 'enable_upnp': False, 'listen_interfaces': '0.0.0.0:0', 'file_pool_size': 1}
class test_create_torrent(unittest.TestCase): class test_create_torrent(unittest.TestCase):
def test_from_torrent_info(self): def test_from_torrent_info(self):
@ -47,10 +46,10 @@ class test_create_torrent(unittest.TestCase):
class test_session_stats(unittest.TestCase): class test_session_stats(unittest.TestCase):
def test_unique(self): def test_unique(self):
l = lt.session_stats_metrics() metrics = lt.session_stats_metrics()
self.assertTrue(len(l) > 40) self.assertTrue(len(metrics) > 40)
idx = set() idx = set()
for m in l: for m in metrics:
self.assertTrue(m.value_index not in idx) self.assertTrue(m.value_index not in idx)
idx.add(m.value_index) idx.add(m.value_index)
@ -118,7 +117,6 @@ class test_torrent_handle(unittest.TestCase):
self.assertEqual(len(torrents), 1) self.assertEqual(len(torrents), 1)
self.assertEqual(torrents[self.h], 'bar') self.assertEqual(torrents[self.h], 'bar')
def test_replace_trackers(self): def test_replace_trackers(self):
self.setup() self.setup()
trackers = [] trackers = []
@ -141,20 +139,18 @@ class test_torrent_handle(unittest.TestCase):
tracker.fail_limit = 1 tracker.fail_limit = 1
trackers = [tracker] trackers = [tracker]
self.h.replace_trackers(trackers) self.h.replace_trackers(trackers)
tracker_list = [tracker for tracker in self.h.trackers()]
# wait a bit until the endpoints list gets populated # wait a bit until the endpoints list gets populated
while len(tracker_list[0]['endpoints']) == 0: while len(self.h.trackers()[0]['endpoints']) == 0:
time.sleep(0.1) time.sleep(0.1)
tracker_list = [tracker for tracker in self.h.trackers()] pickled_trackers = pickle.dumps(self.h.trackers())
pickled_trackers = pickle.dumps(tracker_list)
unpickled_trackers = pickle.loads(pickled_trackers) unpickled_trackers = pickle.loads(pickled_trackers)
self.assertEqual(unpickled_trackers[0]['url'], 'udp://tracker1.com') self.assertEqual(unpickled_trackers[0]['url'], 'udp://tracker1.com')
self.assertEqual(unpickled_trackers[0]['endpoints'][0]['last_error']['value'], 0) self.assertEqual(unpickled_trackers[0]['endpoints'][0]['last_error']['value'], 0)
def test_file_status(self): def test_file_status(self):
self.setup() self.setup()
l = self.h.file_status() status = self.h.file_status()
print(l) print(status)
def test_piece_deadlines(self): def test_piece_deadlines(self):
self.setup() self.setup()
@ -165,7 +161,6 @@ class test_torrent_handle(unittest.TestCase):
# time, wait for next full second to prevent second increment # time, wait for next full second to prevent second increment
time.sleep(1 - datetime.datetime.now().microsecond / 1000000.0) time.sleep(1 - datetime.datetime.now().microsecond / 1000000.0)
sessionStart = datetime.datetime.now().replace(microsecond=0)
self.setup() self.setup()
st = self.h.status() st = self.h.status()
for attr in dir(st): for attr in dir(st):
@ -177,7 +172,7 @@ class test_torrent_handle(unittest.TestCase):
def test_serialize_trackers(self): def test_serialize_trackers(self):
"""Test to ensure the dict contains only python built-in types""" """Test to ensure the dict contains only python built-in types"""
self.setup() self.setup()
self.h.add_tracker({'url':'udp://tracker1.com'}) self.h.add_tracker({'url': 'udp://tracker1.com'})
tr = self.h.trackers()[0] tr = self.h.trackers()[0]
# wait a bit until the endpoints list gets populated # wait a bit until the endpoints list gets populated
while len(tr['endpoints']) == 0: while len(tr['endpoints']) == 0:
@ -246,33 +241,34 @@ class test_torrent_handle(unittest.TestCase):
def test_torrent_parameter(self): def test_torrent_parameter(self):
self.ses = lt.session(settings) self.ses = lt.session(settings)
self.ti = lt.torrent_info('url_seed_multi.torrent'); self.ti = lt.torrent_info('url_seed_multi.torrent')
self.h = self.ses.add_torrent({ self.h = self.ses.add_torrent({
'ti': self.ti, 'ti': self.ti,
'save_path': os.getcwd(), 'save_path': os.getcwd(),
'trackers': ['http://test.com/announce'], 'trackers': ['http://test.com/announce'],
'dht_nodes': [('1.2.3.4', 6881), ('4.3.2.1', 6881)], 'dht_nodes': [('1.2.3.4', 6881), ('4.3.2.1', 6881)],
'file_priorities': [1,1], 'file_priorities': [1, 1],
'http_seeds': ['http://test.com/file3'], 'http_seeds': ['http://test.com/file3'],
'url_seeds': ['http://test.com/announce-url'], 'url_seeds': ['http://test.com/announce-url'],
'peers': [('5.6.7.8', 6881)], 'peers': [('5.6.7.8', 6881)],
'banned_peers': [('8.7.6.5', 6881)], 'banned_peers': [('8.7.6.5', 6881)],
'renamed_files': { 0: 'test.txt', 2: 'test.txt' } 'renamed_files': {0: 'test.txt', 2: 'test.txt'}
}) })
self.st = self.h.status() self.st = self.h.status()
self.assertEqual(self.st.save_path, os.getcwd()) self.assertEqual(self.st.save_path, os.getcwd())
trackers = self.h.trackers(); trackers = self.h.trackers()
self.assertEqual(len(trackers), 1) self.assertEqual(len(trackers), 1)
self.assertEqual(trackers[0].get('url'), 'http://test.com/announce') self.assertEqual(trackers[0].get('url'), 'http://test.com/announce')
self.assertEqual(trackers[0].get('tier'), 0) self.assertEqual(trackers[0].get('tier'), 0)
self.assertEqual(self.h.get_file_priorities(), [1,1]) self.assertEqual(self.h.get_file_priorities(), [1, 1])
self.assertEqual(self.h.http_seeds(),['http://test.com/file3']) self.assertEqual(self.h.http_seeds(), ['http://test.com/file3'])
# url_seeds was already set, test that it did not got overwritten # url_seeds was already set, test that it did not got overwritten
self.assertEqual(self.h.url_seeds(), self.assertEqual(self.h.url_seeds(),
['http://test.com/announce-url/', 'http://test.com/file/']) ['http://test.com/announce-url/', 'http://test.com/file/'])
self.assertEqual(self.h.get_piece_priorities(),[4]) self.assertEqual(self.h.get_piece_priorities(), [4])
self.assertEqual(self.ti.merkle_tree(),[]) self.assertEqual(self.ti.merkle_tree(), [])
self.assertEqual(self.st.verified_pieces,[]) self.assertEqual(self.st.verified_pieces, [])
class test_torrent_info(unittest.TestCase): class test_torrent_info(unittest.TestCase):
@ -328,13 +324,13 @@ class test_torrent_info(unittest.TestCase):
os.path.join('temp', 'foo')) os.path.join('temp', 'foo'))
idx += 1 idx += 1
def test_announce_entry(self): def test_announce_entry(self):
ae = lt.announce_entry('test') ae = lt.announce_entry('test')
self.assertEquals(ae.url, 'test') self.assertEqual(ae.url, 'test')
self.assertEquals(ae.tier, 0) self.assertEqual(ae.tier, 0)
self.assertEquals(ae.verified, False) self.assertEqual(ae.verified, False)
self.assertEquals(ae.source, 0) self.assertEqual(ae.source, 0)
class test_alerts(unittest.TestCase): class test_alerts(unittest.TestCase):
@ -350,7 +346,7 @@ class test_alerts(unittest.TestCase):
alerts = ses.pop_alerts() alerts = ses.pop_alerts()
for a in alerts: for a in alerts:
if a.what() == 'add_torrent_alert': if a.what() == 'add_torrent_alert':
self.assertEquals(a.torrent_name, 'temp') self.assertEqual(a.torrent_name, 'temp')
print(a.message()) print(a.message())
for field_name in dir(a): for field_name in dir(a):
if field_name.startswith('__'): if field_name.startswith('__'):
@ -429,10 +425,11 @@ class test_bencoder(unittest.TestCase):
class test_sha1hash(unittest.TestCase): class test_sha1hash(unittest.TestCase):
def test_sha1hash(self): def test_sha1hash(self):
h = 'a0'*20 h = 'a0' * 20
s = lt.sha1_hash(binascii.unhexlify(h)) s = lt.sha1_hash(binascii.unhexlify(h))
self.assertEqual(h, str(s)) self.assertEqual(h, str(s))
class test_magnet_link(unittest.TestCase): class test_magnet_link(unittest.TestCase):
def test_parse_magnet_uri(self): def test_parse_magnet_uri(self):
@ -453,6 +450,7 @@ class test_magnet_link(unittest.TestCase):
h = ses.add_torrent(p) h = ses.add_torrent(p)
self.assertEqual(str(h.info_hash()), '178882f042c0c33426a6d81e0333ece346e68a68') self.assertEqual(str(h.info_hash()), '178882f042c0c33426a6d81e0333ece346e68a68')
class test_peer_class(unittest.TestCase): class test_peer_class(unittest.TestCase):
def test_peer_class_ids(self): def test_peer_class_ids(self):
@ -495,28 +493,29 @@ class test_peer_class(unittest.TestCase):
def test_peer_class_filter(self): def test_peer_class_filter(self):
filt = lt.peer_class_type_filter() filt = lt.peer_class_type_filter()
filt.add(lt.peer_class_type_filter.tcp_socket, lt.session.global_peer_class_id); filt.add(lt.peer_class_type_filter.tcp_socket, lt.session.global_peer_class_id)
filt.remove(lt.peer_class_type_filter.utp_socket, lt.session.local_peer_class_id); filt.remove(lt.peer_class_type_filter.utp_socket, lt.session.local_peer_class_id)
filt.disallow(lt.peer_class_type_filter.tcp_socket, lt.session.global_peer_class_id); filt.disallow(lt.peer_class_type_filter.tcp_socket, lt.session.global_peer_class_id)
filt.allow(lt.peer_class_type_filter.utp_socket, lt.session.local_peer_class_id); filt.allow(lt.peer_class_type_filter.utp_socket, lt.session.local_peer_class_id)
def test_peer_class_ip_filter(self): def test_peer_class_ip_filter(self):
s = lt.session(settings) s = lt.session(settings)
s.set_peer_class_type_filter(lt.peer_class_type_filter()) s.set_peer_class_type_filter(lt.peer_class_type_filter())
s.set_peer_class_filter(lt.ip_filter()) s.set_peer_class_filter(lt.ip_filter())
class test_session(unittest.TestCase): class test_session(unittest.TestCase):
def test_add_torrent(self): def test_add_torrent(self):
s = lt.session(settings) s = lt.session(settings)
h = s.add_torrent({'ti': lt.torrent_info('base.torrent'), s.add_torrent({'ti': lt.torrent_info('base.torrent'),
'save_path': '.', 'save_path': '.',
'dht_nodes': [('1.2.3.4', 6881), ('4.3.2.1', 6881)], 'dht_nodes': [('1.2.3.4', 6881), ('4.3.2.1', 6881)],
'http_seeds': ['http://test.com/seed'], 'http_seeds': ['http://test.com/seed'],
'peers': [('5.6.7.8', 6881)], 'peers': [('5.6.7.8', 6881)],
'banned_peers': [('8.7.6.5', 6881)], 'banned_peers': [('8.7.6.5', 6881)],
'file_priorities': [1,1,1,2,0]}) 'file_priorities': [1, 1, 1, 2, 0]})
def test_apply_settings(self): def test_apply_settings(self):
@ -565,10 +564,9 @@ class test_session(unittest.TestCase):
self.assertTrue(isinstance(a.active_requests, list)) self.assertTrue(isinstance(a.active_requests, list))
self.assertTrue(isinstance(a.routing_table, list)) self.assertTrue(isinstance(a.routing_table, list))
def test_unknown_settings(self): def test_unknown_settings(self):
try: try:
s = lt.session({'unexpected-key-name': 42}) lt.session({'unexpected-key-name': 42})
self.assertFalse('should have thrown an exception') self.assertFalse('should have thrown an exception')
except KeyError as e: except KeyError as e:
print(e) print(e)
@ -606,18 +604,18 @@ class test_example_client(unittest.TestCase):
my_stdin = slave_fd my_stdin = slave_fd
process = sub.Popen( process = sub.Popen(
[sys.executable,"client.py","url_seed_multi.torrent"], [sys.executable, "client.py", "url_seed_multi.torrent"],
stdin=my_stdin, stdout=sub.PIPE, stderr=sub.PIPE) stdin=my_stdin, stdout=sub.PIPE, stderr=sub.PIPE)
# python2 has no Popen.wait() timeout # python2 has no Popen.wait() timeout
time.sleep(5) time.sleep(5)
returncode = process.poll() returncode = process.poll()
if returncode == None: if returncode is None:
# this is an expected use-case # this is an expected use-case
process.kill() process.kill()
err = process.stderr.read().decode("utf-8") err = process.stderr.read().decode("utf-8")
self.assertEqual('', err, 'process throw errors: \n' + err) self.assertEqual('', err, 'process throw errors: \n' + err)
# check error code if process did unexpected end # check error code if process did unexpected end
if returncode != None: if returncode is not None:
# in case of error return: output stdout if nothing was on stderr # in case of error return: output stdout if nothing was on stderr
if returncode != 0: if returncode != 0:
print("stdout:\n" + process.stdout.read().decode("utf-8")) print("stdout:\n" + process.stdout.read().decode("utf-8"))
@ -628,18 +626,18 @@ class test_example_client(unittest.TestCase):
def test_execute_simple_client(self): def test_execute_simple_client(self):
process = sub.Popen( process = sub.Popen(
[sys.executable,"simple_client.py","url_seed_multi.torrent"], [sys.executable, "simple_client.py", "url_seed_multi.torrent"],
stdout=sub.PIPE, stderr=sub.PIPE) stdout=sub.PIPE, stderr=sub.PIPE)
# python2 has no Popen.wait() timeout # python2 has no Popen.wait() timeout
time.sleep(5) time.sleep(5)
returncode = process.poll() returncode = process.poll()
if returncode == None: if returncode is None:
# this is an expected use-case # this is an expected use-case
process.kill() process.kill()
err = process.stderr.read().decode("utf-8") err = process.stderr.read().decode("utf-8")
self.assertEqual('', err, 'process throw errors: \n' + err) self.assertEqual('', err, 'process throw errors: \n' + err)
# check error code if process did unexpected end # check error code if process did unexpected end
if returncode != None: if returncode is not None:
# in case of error return: output stdout if nothing was on stderr # in case of error return: output stdout if nothing was on stderr
if returncode != 0: if returncode != 0:
print("stdout:\n" + process.stdout.read().decode("utf-8")) print("stdout:\n" + process.stdout.read().decode("utf-8"))
@ -650,7 +648,7 @@ class test_example_client(unittest.TestCase):
def test_execute_make_torrent(self): def test_execute_make_torrent(self):
process = sub.Popen( process = sub.Popen(
[sys.executable,"make_torrent.py","url_seed_multi.torrent", [sys.executable, "make_torrent.py", "url_seed_multi.torrent",
"http://test.com/test"], stdout=sub.PIPE, stderr=sub.PIPE) "http://test.com/test"], stdout=sub.PIPE, stderr=sub.PIPE)
returncode = process.wait() returncode = process.wait()
# python2 has no Popen.wait() timeout # python2 has no Popen.wait() timeout
@ -669,6 +667,7 @@ class test_example_client(unittest.TestCase):
default = lt.default_settings() default = lt.default_settings()
print(default) print(default)
class test_operation_t(unittest.TestCase): class test_operation_t(unittest.TestCase):
def test_enum(self): def test_enum(self):
@ -678,6 +677,7 @@ class test_operation_t(unittest.TestCase):
self.assertEqual(lt.operation_name(lt.operation_t.partfile_write), "partfile_write") self.assertEqual(lt.operation_name(lt.operation_t.partfile_write), "partfile_write")
self.assertEqual(lt.operation_name(lt.operation_t.hostname_lookup), "hostname_lookup") self.assertEqual(lt.operation_name(lt.operation_t.hostname_lookup), "hostname_lookup")
if __name__ == '__main__': if __name__ == '__main__':
print(lt.__version__) print(lt.__version__)
shutil.copy(os.path.join('..', '..', 'test', 'test_torrents', shutil.copy(os.path.join('..', '..', 'test', 'test_torrents',

791
docs/gen_reference_doc.py
View File

File diff suppressed because it is too large Load Diff

107
docs/gen_settings_doc.py
View File

@ -1,12 +1,15 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function
f = open('../include/libtorrent/settings_pack.hpp') f = open('../include/libtorrent/settings_pack.hpp')
out = open('settings.rst', 'w+') out = open('settings.rst', 'w+')
def print_field(str, width): def print_field(str, width):
return '%s%s' % (str, ' ' * (width - len(str))) return '%s%s' % (str, ' ' * (width - len(str)))
def render_section(names, description, type, default_values): def render_section(names, description, type, default_values):
max_name_len = max(len(max(names, key=len)), len('name')) max_name_len = max(len(max(names, key=len)), len('name'))
max_type_len = max(len(type), len('type')) max_type_len = max(len(type), len('type'))
@ -14,60 +17,79 @@ def render_section(names, description, type, default_values):
# add link targets for the rest of the manual to reference # add link targets for the rest of the manual to reference
for n in names: for n in names:
print >>out, '.. _%s:\n' % n print('.. _%s:\n' % n, file=out)
if len(names) > 0: if len(names) > 0:
print >>out, '.. raw:: html\n' print('.. raw:: html\n', file=out)
for n in names: for n in names:
print >>out, '\t<a name="%s"></a>' % n print('\t<a name="%s"></a>' % n, file=out)
print >>out, '' print('', file=out)
separator = '+-' + ('-' * max_name_len) + '-+-' + ('-' * max_type_len) + '-+-' + ('-' * max_val_len) + '-+' separator = '+-' + ('-' * max_name_len) + '-+-' + ('-' * max_type_len) + '-+-' + ('-' * max_val_len) + '-+'
# build a table for the settings, their type and default value # build a table for the settings, their type and default value
print >>out, separator print(separator, file=out)
print >>out, '| %s | %s | %s |' % (print_field('name', max_name_len), print_field('type', max_type_len), print_field('default', max_val_len)) print(
print >>out, separator.replace('-', '=') '| %s | %s | %s |' %
(print_field(
'name', max_name_len), print_field(
'type', max_type_len), print_field(
'default', max_val_len)), file=out)
print(separator.replace('-', '='), file=out)
for i in range(len(names)): for i in range(len(names)):
print >>out, '| %s | %s | %s |' % (print_field(names[i], max_name_len), print_field(type, max_type_len), print_field(default_values[i], max_val_len)) print(
print >>out, separator '| %s | %s | %s |' %
print >>out (print_field(
print >>out, description names[i], max_name_len), print_field(
type, max_type_len), print_field(
default_values[i], max_val_len)), file=out)
print(separator, file=out)
print(file=out)
print(description, file=out)
mode = '' mode = ''
# parse out default values for settings # parse out default values for settings
f2 = open('../src/settings_pack.cpp') f2 = open('../src/settings_pack.cpp')
def_map = {} def_map = {}
for l in f2: for line in f2:
l = l.strip() line = line.strip()
if not l.startswith('SET(') \ if not line.startswith('SET(') \
and not l.startswith('SET_NOPREV(') \ and not line.startswith('SET_NOPREV(') \
and not l.startswith('DEPRECATED_SET('): continue and not line.startswith('DEPRECATED_SET('):
continue
l = l.split('(')[1].split(',') line = line.split('(')[1].split(',')
def_map[l[0]] = l[1].strip() def_map[line[0]] = line[1].strip()
print '%s = %s' % (l[0], l[1].strip()) print('%s = %s' % (line[0], line[1].strip()))
description = '' description = ''
names = [] names = []
for l in f: for line in f:
if 'enum string_types' in l: mode = 'string' if 'enum string_types' in line:
if 'enum bool_types' in l: mode = 'bool' mode = 'string'
if 'enum int_types' in l: mode = 'int' if 'enum bool_types' in line:
if '#if TORRENT_ABI_VERSION == 1' in l: mode += 'skip' mode = 'bool'
if '#endif' in l: mode = mode[0:-4] if 'enum int_types' in line:
mode = 'int'
if '#if TORRENT_ABI_VERSION == 1' in line:
mode += 'skip'
if '#endif' in line:
mode = mode[0:-4]
if mode == '': continue if mode == '':
if mode[-4:] == 'skip': continue continue
if mode[-4:] == 'skip':
continue
l = l.lstrip() line = line.lstrip()
if l == '' and len(names) > 0: if line == '' and len(names) > 0:
if description == '': if description == '':
for n in names: for n in names:
print 'WARNING: no description for "%s"' % n print('WARNING: no description for "%s"' % n)
else: else:
default_values = [] default_values = []
for n in names: for n in names:
@ -76,23 +98,26 @@ for l in f:
description = '' description = ''
names = [] names = []
if l.startswith('};'): if line.startswith('};'):
mode = '' mode = ''
continue continue
if l.startswith('//'): if line.startswith('//'):
if l[2] == ' ': description += l[3:] if line[2] == ' ':
else: description += l[2:] description += line[3:]
else:
description += line[2:]
continue continue
l = l.strip() line = line.strip()
if l.endswith(','): if line.endswith(','):
l = l[:-1] # strip trailing comma line = line[:-1] # strip trailing comma
if '=' in l: l = l.split('=')[0].strip() if '=' in line:
if l.endswith('_internal'): continue line = line.split('=')[0].strip()
if line.endswith('_internal'):
continue
names.append(l) names.append(line)
out.close() out.close()
f.close() f.close()

86
docs/gen_stats_doc.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function
counter_types = {} counter_types = {}
@ -6,33 +7,40 @@ f = open('../include/libtorrent/performance_counters.hpp')
counter_type = '' counter_type = ''
for l in f: for line in f:
# ignore anything after // # ignore anything after //
if '//' in l: l = l.split('//')[0] if '//' in line:
line = line.split('//')[0]
l = l.strip() line = line.strip()
if l.startswith('#'): continue if line.startswith('#'):
if l == '': continue continue
if line == '':
continue
if 'enum stats_counter_t' in l: if 'enum stats_counter_t' in line:
counter_type = 'counter' counter_type = 'counter'
continue continue
if 'enum stats_gauge_t' in l: if 'enum stats_gauge_t' in line:
counter_type = 'gauge' counter_type = 'gauge'
continue continue
if '{' in l or '}' in l or 'struct' in l or 'namespace' in l: continue if '{' in line or '}' in line or 'struct' in line or 'namespace' in line:
if counter_type == '': continue continue
if not l.endswith(','): continue if counter_type == '':
continue
if not line.endswith(','):
continue
# strip off trailing comma # strip off trailing comma
l = l[:-1] line = line[:-1]
if '=' in l: l = l[:l.index('=')].strip() if '=' in line:
line = line[:line.index('=')].strip()
counter_types[l] = counter_type counter_types[line] = counter_type
f.close() f.close()
@ -40,39 +48,42 @@ f = open('../src/session_stats.cpp')
out = open('stats_counters.rst', 'w+') out = open('stats_counters.rst', 'w+')
def print_field(str, width): def print_field(str, width):
return '%s%s' % (str, ' ' * (width - len(str))) return '%s%s' % (str, ' ' * (width - len(str)))
def render_section(names, description, types): def render_section(names, description, types):
max_name_len = max(len(max(names, key=len)), len('name')) max_name_len = max(len(max(names, key=len)), len('name'))
max_type_len = max(len(max(types, key=len)), len('type')) max_type_len = max(len(max(types, key=len)), len('type'))
if description == '': if description == '':
for n in names: for n in names:
print 'WARNING: no description for "%s"' % n print('WARNING: no description for "%s"' % n)
# add link targets for the rest of the manual to reference # add link targets for the rest of the manual to reference
for n in names: for n in names:
print >>out, '.. _%s:\n' % n print('.. _%s:\n' % n, file=out)
if len(names) > 0: if len(names) > 0:
print >>out, '.. raw:: html\n' print('.. raw:: html\n', file=out)
for n in names: for n in names:
print >>out, '\t<a name="%s"></a>' % n print('\t<a name="%s"></a>' % n, file=out)
print >>out, '' print('', file=out)
separator = '+-' + ('-' * max_name_len) + '-+-' + ('-' * max_type_len) + '-+' separator = '+-' + ('-' * max_name_len) + '-+-' + ('-' * max_type_len) + '-+'
# build a table for the settings, their type and default value # build a table for the settings, their type and default value
print >>out, separator print(separator, file=out)
print >>out, '| %s | %s |' % (print_field('name', max_name_len), print_field('type', max_type_len)) print('| %s | %s |' % (print_field('name', max_name_len), print_field('type', max_type_len)), file=out)
print >>out, separator.replace('-', '=') print(separator.replace('-', '='), file=out)
for i in range(len(names)): for i in range(len(names)):
print >>out, '| %s | %s |' % (print_field(names[i], max_name_len), print_field(types[i], max_type_len)) print('| %s | %s |' % (print_field(names[i], max_name_len), print_field(types[i], max_type_len)), file=out)
print >>out, separator print(separator, file=out)
print >>out print(file=out)
print >>out, description print(description, file=out)
print >>out, '' print('', file=out)
mode = '' mode = ''
@ -80,32 +91,34 @@ description = ''
names = [] names = []
types = [] types = []
for l in f: for line in f:
description_line = l.lstrip().startswith('//') description_line = line.lstrip().startswith('//')
l = l.strip() line = line.strip()
if mode == 'ignore': if mode == 'ignore':
if '#endif' in l: mode = '' if '#endif' in line:
mode = ''
continue continue
if 'TORRENT_ABI_VERSION == 1' in l: if 'TORRENT_ABI_VERSION == 1' in line:
mode = 'ignore' mode = 'ignore'
continue continue
if description_line == True: if description_line:
if len(names) > 0: if len(names) > 0:
render_section(names, description, types) render_section(names, description, types)
description = '' description = ''
names = [] names = []
types = [] types = []
description += '\n' + l[3:] description += '\n' + line[3:]
if '#define' in l: continue if '#define' in line:
continue
if 'METRIC(' in l: if 'METRIC(' in line:
args = l.split('(')[1].split(')')[0].split(',') args = line.split('(')[1].split(')')[0].split(',')
# args: category, name, type # args: category, name, type
@ -118,4 +131,3 @@ if len(names) > 0:
out.close() out.close()
f.close() f.close()

75
docs/gen_todo.py
View File

@ -2,8 +2,16 @@
import glob import glob
import os import os
import sys
paths = ['test/*.cpp', 'src/*.cpp', 'src/kademlia/*.cpp', 'include/libtorrent/*.hpp', 'include/libtorrent/kademlia/*.hpp', 'include/libtorrent/aux_/*.hpp', 'include/libtorrent/extensions/*.hpp'] paths = [
'test/*.cpp',
'src/*.cpp',
'src/kademlia/*.cpp',
'include/libtorrent/*.hpp',
'include/libtorrent/kademlia/*.hpp',
'include/libtorrent/aux_/*.hpp',
'include/libtorrent/extensions/*.hpp']
os.system('(cd .. ; ctags %s 2>/dev/null)' % ' '.join(paths)) os.system('(cd .. ; ctags %s 2>/dev/null)' % ' '.join(paths))
@ -20,15 +28,21 @@ context = []
priority_count = [0, 0, 0, 0, 0] priority_count = [0, 0, 0, 0, 0]
def html_sanitize(s): def html_sanitize(s):
ret = '' ret = ''
for i in s: for i in s:
if i == '<': ret += '&lt;' if i == '<':
elif i == '>': ret += '&gt;' ret += '&lt;'
elif i == '&': ret += '&amp;' elif i == '>':
else: ret += i ret += '&gt;'
elif i == '&':
ret += '&amp;'
else:
ret += i
return ret return ret
for f in files: for f in files:
h = open(f) h = open(f)
@ -48,18 +62,20 @@ for f in files:
if line[0] in '0123456789': if line[0] in '0123456789':
items[-1]['priority'] = int(line[0]) items[-1]['priority'] = int(line[0])
if int(line[0]) > 5: if int(line[0]) > 5:
print 'priority too high: ' + line print('priority too high: ' + line)
sys.exit(1) sys.exit(1)
line = line[1:].strip() line = line[1:].strip()
items[-1]['todo'] = line items[-1]['todo'] = line
prio = items[-1]['priority'] prio = items[-1]['priority']
if prio >= 0 and prio <= 4: priority_count[prio] += 1 if prio >= 0 and prio <= 4:
priority_count[prio] += 1
continue continue
if state == '': if state == '':
context.append(html_sanitize(l)) context.append(html_sanitize(l))
if len(context) > 20: context.pop(0) if len(context) > 20:
context.pop(0)
continue continue
if state == 'todo': if state == 'todo':
@ -68,11 +84,13 @@ for f in files:
items[-1]['todo'] += line[2:].strip() items[-1]['todo'] += line[2:].strip()
else: else:
state = 'context' state = 'context'
items[-1]['context'] = ''.join(context) + '<div style="background: #ffff00" width="100%">' + html_sanitize(l) + '</div>'; items[-1]['context'] = ''.join(context) + \
'<div style="background: #ffff00" width="100%">' + html_sanitize(l) + '</div>'
context_lines = 1 context_lines = 1
context.append(html_sanitize(l)) context.append(html_sanitize(l))
if len(context) > 20: context.pop(0) if len(context) > 20:
context.pop(0)
continue continue
if state == 'context': if state == 'context':
@ -80,19 +98,21 @@ for f in files:
context_lines += 1 context_lines += 1
context.append(html_sanitize(l)) context.append(html_sanitize(l))
if len(context) > 20: context.pop(0) if len(context) > 20:
if context_lines > 30: state = '' context.pop(0)
if context_lines > 30:
state = ''
h.close() h.close()
items.sort(key = lambda x: x['priority'], reverse = True) items.sort(key=lambda x: x['priority'], reverse=True)
#for i in items: # for i in items:
# print '\n\n', i['todo'], '\n' # print('\n\n', i['todo'], '\n')
# print i['location'], '\n' # print(i['location'], '\n')
# print 'prio: ', i['priority'], '\n' # print('prio: ', i['priority'], '\n')
# if 'context' in i: # if 'context' in i:
# print i['context'], '\n' # print(i['context'], '\n')
out = open('todo.html', 'w+') out = open('todo.html', 'w+')
out.write('''<html><head> out.write('''<html><head>
@ -123,21 +143,24 @@ out.write('''<html><head>
<span style="color: #3c3">%d relevant</span> <span style="color: #3c3">%d relevant</span>
<span style="color: #77f">%d feasible</span> <span style="color: #77f">%d feasible</span>
<span style="color: #999">%d notes</span> <span style="color: #999">%d notes</span>
<table width="100%%" border="1" style="border-collapse: collapse;">''' % \ <table width="100%%" border="1" style="border-collapse: collapse;">''' # noqa
(priority_count[4], priority_count[3], priority_count[2], priority_count[1], priority_count[0])) % (priority_count[4], priority_count[3], priority_count[2], priority_count[1], priority_count[0]))
prio_colors = [ '#ccc', '#ccf', '#cfc', '#fcc', '#f44'] prio_colors = ['#ccc', '#ccf', '#cfc', '#fcc', '#f44']
index = 0 index = 0
for i in items: for i in items:
if not 'context' in i: i['context'] = '' if 'context' not in i:
out.write('<tr style="background: %s"><td>relevance&nbsp;%d</td><td><a href="javascript:expand(%d)">%s</a></td><td>%s</td></tr>' \ i['context'] = ''
out.write(('<tr style="background: %s"><td>relevance&nbsp;%d</td>'
'<td><a href="javascript:expand(%d)">%s</a></td><td>%s</td></tr>')
% (prio_colors[i['priority']], i['priority'], index, i['location'], i['todo'].replace('\n', ' '))) % (prio_colors[i['priority']], i['priority'], index, i['location'], i['todo'].replace('\n', ' ')))
out.write('<tr id="%d" style="display: none;" colspan="3"><td colspan="3"><h2>%s</h2><h4>%s</h4><pre style="background: #f6f6f6; border: solid 1px #ddd;">%s</pre></td></tr>' \ out.write(
% (index, i['todo'], i['location'], i['context'])) ('<tr id="%d" style="display: none;" colspan="3"><td colspan="3"><h2>%s</h2><h4>%s</h4>'
'<pre style="background: #f6f6f6; border: solid 1px #ddd;">%s</pre></td></tr>') %
(index, i['todo'], i['location'], i['context']))
index += 1 index += 1
out.write('</table></body></html>') out.write('</table></body></html>')
out.close() out.close()

7
docs/join_rst.py
View File

@ -15,11 +15,12 @@ for name in sys.argv[2:]:
if '.. contents::' in l: if '.. contents::' in l:
in_directive = True in_directive = True
continue continue
if ':Author:' in l: continue if ':Author:' in l:
if ':Version:' in l: continue continue
if ':Version:' in l:
continue
if l[0] in ' \t' and in_directive: if l[0] in ' \t' and in_directive:
continue continue
in_directive = False in_directive = False
sys.stdout.write(l) sys.stdout.write(l)

310
examples/run_benchmarks.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function
import sys import sys
import os import os
@ -65,46 +66,47 @@ default_cache = cache_sizes[-1]
test_duration = 200 # 700 test_duration = 200 # 700
# make sure the environment is properly set up # make sure the environment is properly set up
try: try:
if os.name == 'posix': if os.name == 'posix':
resource.setrlimit(resource.RLIMIT_NOFILE, (4000, 5000)) resource.setrlimit(resource.RLIMIT_NOFILE, (4000, 5000))
except: except BaseException:
if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < 4000: if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < 4000:
print 'please set ulimit -n to at least 4000' print('please set ulimit -n to at least 4000')
sys.exit(1) sys.exit(1)
def build_stage_dirs(): def build_stage_dirs():
ret = [] ret = []
for i in builds[2:3]: for i in builds[2:3]:
ret.append('stage_%s' % i) ret.append('stage_%s' % i)
return ret return ret
# make sure we have all the binaries available # make sure we have all the binaries available
binaries = ['client_test', 'connection_tester', 'fragmentation_test'] binaries = ['client_test', 'connection_tester', 'fragmentation_test']
for b in build_stage_dirs(): for b in build_stage_dirs():
for i in binaries: for i in binaries:
p = os.path.join(b, i) p = os.path.join(b, i)
if not os.path.exists(p): if not os.path.exists(p):
print 'make sure "%s" is available in ./%s' % (i, b) print('make sure "%s" is available in ./%s' % (i, b))
sys.exit(1) sys.exit(1)
for i in filesystem: for i in filesystem:
if not os.path.exists(i): if not os.path.exists(i):
print ('the path "%s" does not exist. This is directory/mountpoint is ' + print(('the path "%s" does not exist. This is directory/mountpoint is ' +
'used as the download directory and is the filesystem that will be benchmarked ' + 'used as the download directory and is the filesystem that will be benchmarked ' +
'and need to exist.') % i 'and need to exist.') % i)
sys.exit(1) sys.exit(1)
# make sure we have a test torrent # make sure we have a test torrent
if not os.path.exists('test.torrent'): if not os.path.exists('test.torrent'):
print 'generating test torrent' print('generating test torrent')
# generate a 100 GB torrent, to make sure it won't all fit in physical RAM # generate a 100 GB torrent, to make sure it won't all fit in physical RAM
os.system('./stage_aio/connection_tester gen-torrent 10000 test.torrent') os.system('./stage_aio/connection_tester gen-torrent 10000 test.torrent')
if not os.path.exists('test2.torrent'): if not os.path.exists('test2.torrent'):
print 'generating test torrent 2' print('generating test torrent 2')
# generate a 6 GB torrent, to make sure it will fit in physical RAM # generate a 6 GB torrent, to make sure it will fit in physical RAM
os.system('./stage_aio/connection_tester gen-torrent 6000 test2.torrent') os.system('./stage_aio/connection_tester gen-torrent 6000 test2.torrent')
@ -112,6 +114,7 @@ if not os.path.exists('test2.torrent'):
# this port is incremented for each test run # this port is incremented for each test run
port = 10000 + random.randint(0, 40000) port = 10000 + random.randint(0, 40000)
def clear_caches(): def clear_caches():
if 'linux' in sys.platform: if 'linux' in sys.platform:
os.system('sync') os.system('sync')
@ -119,14 +122,17 @@ def clear_caches():
elif 'darwin' in sys.platform: elif 'darwin' in sys.platform:
os.system('purge') os.system('purge')
def build_commandline(config, port): def build_commandline(config, port):
num_peers = config['num-peers'] num_peers = config['num-peers']
torrent_path = config['torrent'] torrent_path = config['torrent']
if config['build'] == 'utorrent': if config['build'] == 'utorrent':
try: os.mkdir('utorrent_session') try:
except: pass os.mkdir('utorrent_session')
except BaseException:
pass
cfg = open('utorrent_session/settings.dat', 'w+') cfg = open('utorrent_session/settings.dat', 'w+')
cfg.write('d') cfg.write('d')
@ -150,56 +156,77 @@ def build_commandline(config, port):
cfg.write('14:webui.username5:admin') cfg.write('14:webui.username5:admin')
cfg.write('e') cfg.write('e')
cfg.close() cfg.close()
try: os.mkdir('utorrent_session/autoload') try:
except: pass os.mkdir('utorrent_session/autoload')
try: shutil.copy(torrent_path, 'utorrent_session/autoload/') except BaseException:
except: pass pass
try:
shutil.copy(torrent_path, 'utorrent_session/autoload/')
except BaseException:
pass
return './utorrent-server-v3_0/utserver -logfile session_stats/alerts_log.txt -settingspath utorrent_session' return './utorrent-server-v3_0/utserver -logfile session_stats/alerts_log.txt -settingspath utorrent_session'
if config['build'] == 'rtorrent': if config['build'] == 'rtorrent':
if os.path.exists('rtorrent_session'): if os.path.exists('rtorrent_session'):
add_command = '' add_command = ''
else: else:
try: os.mkdir('rtorrent_session') try:
except: pass os.mkdir('rtorrent_session')
except BaseException:
pass
# it seems rtorrent may delete the original torrent when it's being added # it seems rtorrent may delete the original torrent when it's being added
try: shutil.copy(torrent_path, 'rtorrent_session/') try:
except: pass shutil.copy(torrent_path, 'rtorrent_session/')
except BaseException:
pass
add_command = '-O load_start_verbose=rtorrent_session/%s ' % torrent_path add_command = '-O load_start_verbose=rtorrent_session/%s ' % torrent_path
return 'rtorrent -d %s -n -p %d-%d -O max_peers=%d -O max_uploads=%d %s -s rtorrent_session -O max_memory_usage=128000000000' \ return ('rtorrent -d %s -n -p %d-%d -O max_peers=%d -O max_uploads=%d %s -s '
% (config['save-path'], port, port, num_peers, num_peers, add_command) 'rtorrent_session -O max_memory_usage=128000000000') % (
config['save-path'], port, port, num_peers, num_peers, add_command)
disable_disk = '' disable_disk = ''
if config['disable-disk']: disable_disk = '-0' if config['disable-disk']:
return './stage_%s/client_test -k -N -H -M -B %d -l %d -S %d -T %d -c %d -C %d -s "%s" -p %d -E %d %s -f session_stats/alerts_log.txt %s' \ disable_disk = '-0'
% (config['build'], test_duration, num_peers, num_peers, num_peers, num_peers, config['cache-size'], config['save-path'], port, \ return ('./stage_%s/client_test -k -N -H -M -B %d -l %d -S %d -T %d -c %d -C %d -s "%s" -p %d -E %d %s '
config['hash-threads'], disable_disk, torrent_path) '-f session_stats/alerts_log.txt %s') % (
config['build'], test_duration, num_peers, num_peers, num_peers, num_peers, config['cache-size'],
config['save-path'], port, config['hash-threads'], disable_disk, torrent_path)
def delete_files(files): def delete_files(files):
for i in files: for i in files:
try: os.remove(i)
except:
try: shutil.rmtree(i)
except:
try: try:
if os.path.exists(i): print 'failed to delete %s' % i os.remove(i)
except: pass except BaseException:
try:
shutil.rmtree(i)
except BaseException:
try:
if os.path.exists(i):
print('failed to delete %s' % i)
except BaseException:
pass
# typically the schedulers available are 'noop', 'deadline' and 'cfq' # typically the schedulers available are 'noop', 'deadline' and 'cfq'
def build_test_config(fs=default_fs, num_peers=default_peers, cache_size=default_cache, \
test='upload', build='aio', profile='', hash_threads=1, torrent='test.torrent', \
disable_disk = False): def build_test_config(fs=default_fs, num_peers=default_peers, cache_size=default_cache,
config = {'test': test, 'save-path': os.path.join('./', fs), 'num-peers': num_peers, \ test='upload', build='aio', profile='', hash_threads=1, torrent='test.torrent',
'cache-size': cache_size, 'build': build, 'profile':profile, \ disable_disk=False):
'hash-threads': hash_threads, 'torrent': torrent, 'disable-disk': disable_disk } config = {'test': test, 'save-path': os.path.join('./', fs), 'num-peers': num_peers,
'cache-size': cache_size, 'build': build, 'profile': profile,
'hash-threads': hash_threads, 'torrent': torrent, 'disable-disk': disable_disk}
return config return config
def prefix_len(text, prefix): def prefix_len(text, prefix):
for i in xrange(1, len(prefix)): for i in range(1, len(prefix)):
if (not text.startswith(prefix[0:i])): return i-1 if (not text.startswith(prefix[0:i])):
return i - 1
return len(prefix) return len(prefix)
def device_name(path): def device_name(path):
mount = subprocess.Popen('mount', stdout=subprocess.PIPE) mount = subprocess.Popen('mount', stdout=subprocess.PIPE)
@ -218,126 +245,164 @@ def device_name(path):
device = match_device device = match_device
device = device.split('/')[-1][0:3] device = device.split('/')[-1][0:3]
print 'device for path: %s -> %s' % (path, device) print('device for path: %s -> %s' % (path, device))
return device return device
def build_target_folder(config): def build_target_folder(config):
test = 'seed' test = 'seed'
if config['test'] == 'upload': test = 'download' if config['test'] == 'upload':
elif config['test'] == 'dual': test = 'dual' test = 'download'
elif config['test'] == 'dual':
test = 'dual'
if 'linux' in sys.platform: if 'linux' in sys.platform:
io_scheduler = open('/sys/block/%s/queue/scheduler' % device_name(config['save-path'])).read().split('[')[1].split(']')[0] io_scheduler = open('/sys/block/%s/queue/scheduler' %
device_name(config['save-path'])).read().split('[')[1].split(']')[0]
else: else:
io_scheduler = sys.platform io_scheduler = sys.platform
no_disk = '' no_disk = ''
if config['disable-disk']: no_disk = '_no-disk' if config['disable-disk']:
no_disk = '_no-disk'
return 'results_%s_%s_%d_%d_%s_%s_h%d%s' % (config['build'],
test,
config['num-peers'],
config['cache-size'],
os.path.split(
config['save-path'])[1],
io_scheduler,
config['hash-threads'],
no_disk)
return 'results_%s_%s_%d_%d_%s_%s_h%d%s' % (config['build'], test, config['num-peers'], \
config['cache-size'], os.path.split(config['save-path'])[1], io_scheduler, \
config['hash-threads'], no_disk)
def find_library(name): def find_library(name):
paths = ['/usr/lib64/', '/usr/local/lib64/', '/usr/lib/', '/usr/local/lib/'] paths = ['/usr/lib64/', '/usr/local/lib64/', '/usr/lib/', '/usr/local/lib/']
for p in paths: for p in paths:
try: try:
if os.path.exists(p + name): return p + name if os.path.exists(p + name):
except: pass return p + name
except BaseException:
pass
return name return name
def find_binary(names): def find_binary(names):
paths = ['/usr/bin/', '/usr/local/bin/'] paths = ['/usr/bin/', '/usr/local/bin/']
for n in names: for n in names:
for p in paths: for p in paths:
try: try:
if os.path.exists(p + n): return p + n if os.path.exists(p + n):
except: pass return p + n
except BaseException:
pass
return names[0] return names[0]
def run_test(config): def run_test(config):
target_folder = build_target_folder(config) target_folder = build_target_folder(config)
if os.path.exists(target_folder): if os.path.exists(target_folder):
print 'results already exists, skipping test (%s)' % target_folder print('results already exists, skipping test (%s)' % target_folder)
return return
print '\n\n*********************************' print('\n\n*********************************')
print '* RUNNING TEST *' print('* RUNNING TEST *')
print '*********************************\n\n' print('*********************************\n\n')
print '%s %s' % (config['build'], config['test']) print('%s %s' % (config['build'], config['test']))
# make sure any previous test file is removed # make sure any previous test file is removed
# don't clean up unless we're running a download-test, so that we leave the test file # don't clean up unless we're running a download-test, so that we leave the test file
# complete for a seed test. # complete for a seed test.
delete_files(['utorrent_session/settings.dat', 'utorrent_session/settings.dat.old', 'asserts.log']) delete_files(['utorrent_session/settings.dat', 'utorrent_session/settings.dat.old', 'asserts.log'])
if config['test'] == 'upload' or config['test'] == 'dual': if config['test'] == 'upload' or config['test'] == 'dual':
print 'deleting files' print('deleting files')
delete_files([os.path.join(config['save-path'], 'stress_test_file'), '.ses_state', os.path.join(config['save-path'], '.resume'), 'utorrent_session', '.dht_state', 'session_stats', 'rtorrent_session']) delete_files([os.path.join(config['save-path'],
'stress_test_file'),
'.ses_state',
os.path.join(config['save-path'],
'.resume'),
'utorrent_session',
'.dht_state',
'session_stats',
'rtorrent_session'])
try: os.mkdir('session_stats') try:
except: pass os.mkdir('session_stats')
except BaseException:
pass
# save off the command line for reference # save off the command line for reference
global port global port
cmdline = build_commandline(config, port) cmdline = build_commandline(config, port)
binary = cmdline.split(' ')[0] binary = cmdline.split(' ')[0]
environment = None environment = None
if config['profile'] == 'tcmalloc': environment = {'LD_PRELOAD':find_library('libprofiler.so.0'), 'CPUPROFILE': 'session_stats/cpu_profile.prof'} if config['profile'] == 'tcmalloc':
if config['profile'] == 'memory': environment = {'LD_PRELOAD':find_library('libprofiler.so.0'), 'HEAPPROFILE': 'session_stats/heap_profile.prof'} environment = {'LD_PRELOAD': find_library('libprofiler.so.0'), 'CPUPROFILE': 'session_stats/cpu_profile.prof'}
if config['profile'] == 'perf': cmdline = 'perf timechart record --call-graph --output=session_stats/perf_profile.prof ' + cmdline if config['profile'] == 'memory':
environment = {'LD_PRELOAD': find_library('libprofiler.so.0'), 'HEAPPROFILE': 'session_stats/heap_profile.prof'}
if config['profile'] == 'perf':
cmdline = 'perf timechart record --call-graph --output=session_stats/perf_profile.prof ' + cmdline
f = open('session_stats/cmdline.txt', 'w+') f = open('session_stats/cmdline.txt', 'w+')
f.write(cmdline) f.write(cmdline)
f.close() f.close()
f = open('session_stats/config.txt', 'w+') f = open('session_stats/config.txt', 'w+')
print >>f, config print(config, file=f)
f.close() f.close()
print 'clearing disk cache' print('clearing disk cache')
clear_caches() clear_caches()
print 'OK' print('OK')
client_output = open('session_stats/client.output', 'w+') client_output = open('session_stats/client.output', 'w+')
client_error = open('session_stats/client.error', 'w+') client_error = open('session_stats/client.error', 'w+')
print 'launching: %s' % cmdline print('launching: %s' % cmdline)
client = subprocess.Popen(shlex.split(cmdline), stdout=client_output, stdin=subprocess.PIPE, stderr=client_error, env=environment) client = subprocess.Popen(
print 'OK' shlex.split(cmdline),
stdout=client_output,
stdin=subprocess.PIPE,
stderr=client_error,
env=environment)
print('OK')
# enable disk stats printing # enable disk stats printing
if config['build'] != 'rtorrent' and config['build'] != 'utorrent': if config['build'] != 'rtorrent' and config['build'] != 'utorrent':
print >>client.stdin, 'x', print('x', end=' ', file=client.stdin)
time.sleep(4) time.sleep(4)
cmdline = './stage_aio/connection_tester %s %d 127.0.0.1 %d %s' % (config['test'], config['num-peers'], port, config['torrent']) cmdline = './stage_aio/connection_tester %s %d 127.0.0.1 %d %s' % (
print 'launching: %s' % cmdline config['test'], config['num-peers'], port, config['torrent'])
print('launching: %s' % cmdline)
tester_output = open('session_stats/tester.output', 'w+') tester_output = open('session_stats/tester.output', 'w+')
tester = subprocess.Popen(shlex.split(cmdline), stdout=tester_output) tester = subprocess.Popen(shlex.split(cmdline), stdout=tester_output)
print 'OK' print('OK')
time.sleep(2) time.sleep(2)
print '\n' print('\n')
i = 0 i = 0
while True: while True:
time.sleep(1) time.sleep(1)
tester.poll() tester.poll()
if tester.returncode != None: if tester.returncode is not None:
print 'tester terminated' print('tester terminated')
break break
client.poll() client.poll()
if client.returncode != None: if client.returncode is not None:
print 'client terminated' print('client terminated')
break break
print '\r%d / %d' % (i, test_duration), print('\r%d / %d' % (i, test_duration), end=' ')
sys.stdout.flush() sys.stdout.flush()
i += 1 i += 1
if config['test'] != 'upload' and config['test'] != 'dual' and i >= test_duration: break if config['test'] != 'upload' and config['test'] != 'dual' and i >= test_duration:
print '\n' break
print('\n')
if client.returncode == None: if client.returncode is None:
try: try:
print 'killing client' print('killing client')
client.send_signal(signal.SIGINT) client.send_signal(signal.SIGINT)
except: except BaseException:
pass pass
time.sleep(10) time.sleep(10)
@ -347,29 +412,39 @@ def run_test(config):
client_output.close() client_output.close()
terminate = False terminate = False
if tester.returncode != 0: if tester.returncode != 0:
print 'tester returned %d' % tester.returncode print('tester returned %d' % tester.returncode)
terminate = True terminate = True
if client.returncode != 0: if client.returncode != 0:
print 'client returned %d' % client.returncode print('client returned %d' % client.returncode)
terminate = True terminate = True
try: shutil.copy('asserts.log', 'session_stats/') try:
except: pass shutil.copy('asserts.log', 'session_stats/')
except BaseException:
pass
try: shutil.move('libtorrent_logs0', 'session_stats/') try:
except: pass shutil.move('libtorrent_logs0', 'session_stats/')
try: shutil.move('libtorrent_logs%s' % port, 'session_stats/') except BaseException:
except: pass pass
try:
shutil.move('libtorrent_logs%s' % port, 'session_stats/')
except BaseException:
pass
# run fragmentation test # run fragmentation test
print 'analyzing fragmentation' print('analyzing fragmentation')
os.system('./stage_aio/fragmentation_test test.torrent %s' % (config['save-path'])) os.system('./stage_aio/fragmentation_test test.torrent %s' % (config['save-path']))
try: shutil.copy('fragmentation.log', 'session_stats/') try:
except: pass shutil.copy('fragmentation.log', 'session_stats/')
except BaseException:
pass
shutil.copy('fragmentation.gnuplot', 'session_stats/') shutil.copy('fragmentation.gnuplot', 'session_stats/')
try: shutil.copy('file_access.log', 'session_stats/') try:
except: pass shutil.copy('file_access.log', 'session_stats/')
except BaseException:
pass
os.system('filefrag %s >session_stats/filefrag.out' % config['save-path']) os.system('filefrag %s >session_stats/filefrag.out' % config['save-path'])
os.system('filefrag -v %s >session_stats/filefrag_verbose.out' % config['save-path']) os.system('filefrag -v %s >session_stats/filefrag_verbose.out' % config['save-path'])
@ -377,37 +452,51 @@ def run_test(config):
os.chdir('session_stats') os.chdir('session_stats')
# parse session stats # parse session stats
print 'parsing session log' print('parsing session log')
os.system('python ../../parse_session_stats.py *.0000.log') os.system('python ../../parse_session_stats.py *.0000.log')
os.system('../stage_aio/parse_access_log file_access.log %s' % (os.path.join('..', config['save-path'], 'stress_test_file'))) os.system('../stage_aio/parse_access_log file_access.log %s' %
(os.path.join('..', config['save-path'], 'stress_test_file')))
os.chdir('..') os.chdir('..')
if config['profile'] == 'tcmalloc': if config['profile'] == 'tcmalloc':
print 'analyzing CPU profile [%s]' % binary print('analyzing CPU profile [%s]' % binary)
os.system('%s --pdf %s session_stats/cpu_profile.prof >session_stats/cpu_profile.pdf' % (find_binary(['google-pprof', 'pprof']), binary)) os.system('%s --pdf %s session_stats/cpu_profile.prof >session_stats/cpu_profile.pdf' %
(find_binary(['google-pprof', 'pprof']), binary))
if config['profile'] == 'memory': if config['profile'] == 'memory':
for i in xrange(1, 300): for i in range(1, 300):
profile = 'session_stats/heap_profile.prof.%04d.heap' % i profile = 'session_stats/heap_profile.prof.%04d.heap' % i
try: os.stat(profile) try:
except: break os.stat(profile)
print 'analyzing heap profile [%s] %d' % (binary, i) except BaseException:
os.system('%s --pdf %s %s >session_stats/heap_profile_%d.pdf' % (find_binary(['google-pprof', 'pprof']), binary, profile, i)) break
print('analyzing heap profile [%s] %d' % (binary, i))
os.system('%s --pdf %s %s >session_stats/heap_profile_%d.pdf' %
(find_binary(['google-pprof', 'pprof']), binary, profile, i))
if config['profile'] == 'perf': if config['profile'] == 'perf':
print 'analyzing CPU profile [%s]' % binary print('analyzing CPU profile [%s]' % binary)
os.system('perf timechart --input=session_stats/perf_profile.prof --output=session_stats/profile_timechart.svg') os.system('perf timechart --input=session_stats/perf_profile.prof --output=session_stats/profile_timechart.svg')
os.system('perf report --input=session_stats/perf_profile.prof --threads --show-nr-samples --vmlinux vmlinuz-2.6.38-8-generic.bzip >session_stats/profile.txt') os.system(('perf report --input=session_stats/perf_profile.prof --threads --show-nr-samples '
'--vmlinux vmlinuz-2.6.38-8-generic.bzip >session_stats/profile.txt'))
# move the results into its final place # move the results into its final place
print 'saving results' print('saving results')
os.rename('session_stats', build_target_folder(config)) os.rename('session_stats', build_target_folder(config))
port += 1 port += 1
if terminate: sys.exit(1) if terminate:
sys.exit(1)
for h in range(0, 7): for h in range(0, 7):
config = build_test_config(num_peers=30, build='aio', test='upload', torrent='test.torrent', hash_threads=h, disable_disk=True) config = build_test_config(
num_peers=30,
build='aio',
test='upload',
torrent='test.torrent',
hash_threads=h,
disable_disk=True)
run_test(config) run_test(config)
sys.exit(0) sys.exit(0)
@ -436,4 +525,3 @@ for fs in filesystem:
for test in ['upload', 'download']: for test in ['upload', 'download']:
config = build_test_config(fs=fs, test=test) config = build_test_config(fs=fs, test=test)
run_test(config) run_test(config)

122
test/http.py
View File

@ -1,28 +1,28 @@
# -*- coding: cp1252 -*- # -*- coding: cp1252 -*-
# <PythonProxy.py> # <PythonProxy.py>
# #
#Copyright (c) <2009> <Fábio Domingues - fnds3000 in gmail.com> # Copyright (c) <2009> <Fábio Domingues - fnds3000 in gmail.com>
# #
#Permission is hereby granted, free of charge, to any person # Permission is hereby granted, free of charge, to any person
#obtaining a copy of this software and associated documentation # obtaining a copy of this software and associated documentation
#files (the "Software"), to deal in the Software without # files (the "Software"), to deal in the Software without
#restriction, including without limitation the rights to use, # restriction, including without limitation the rights to use,
#copy, modify, merge, publish, distribute, sublicense, and/or sell # copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the # copies of the Software, and to permit persons to whom the
#Software is furnished to do so, subject to the following # Software is furnished to do so, subject to the following
#conditions: # conditions:
# #
#The above copyright notice and this permission notice shall be # The above copyright notice and this permission notice shall be
#included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
# #
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
#OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
#NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
#HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
#WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#OTHER DEALINGS IN THE SOFTWARE. # OTHER DEALINGS IN THE SOFTWARE.
"""\ """\
Copyright (c) <2009> <Fábio Domingues - fnds3000 in gmail.com> <MIT Licence> Copyright (c) <2009> <Fábio Domingues - fnds3000 in gmail.com> <MIT Licence>
@ -81,16 +81,28 @@ Qual a diferença entre um proxy Elite, Anónimo e Transparente?
""" """
import socket, thread, select, sys, base64, time, errno import socket
import select
import sys
import base64
import time
import errno
# Python 3 renamed thread module to _thread
try:
import _thread as thread
except BaseException:
import thread
__version__ = '0.1.0 Draft 1' __version__ = '0.1.0 Draft 1'
BUFLEN = 8192 BUFLEN = 8192
VERSION = 'Python Proxy/'+__version__ VERSION = 'Python Proxy/' + __version__
HTTPVER = 'HTTP/1.1' HTTPVER = 'HTTP/1.1'
username = None username = None
password = None password = None
class ConnectionHandler: class ConnectionHandler:
def __init__(self, connection, address, timeout): def __init__(self, connection, address, timeout):
self.client = connection self.client = connection
@ -99,12 +111,12 @@ class ConnectionHandler:
self.method, self.path, self.protocol = self.get_base_header() self.method, self.path, self.protocol = self.get_base_header()
global username global username
global password global password
if username != None: if username is not None:
auth = base64.b64encode(username + ':' + password) auth = base64.b64encode(username + ':' + password)
if not 'Proxy-Authorization: Basic ' + auth in self.client_buffer: if not 'Proxy-Authorization: Basic ' + auth in self.client_buffer:
print 'PROXY - failed authentication: %s' % self.client_buffer print('PROXY - failed authentication: %s' % self.client_buffer)
self.client.send(HTTPVER+' 401 Authentication Failed\n'+ self.client.send(HTTPVER + ' 401 Authentication Failed\n' +
'Proxy-agent: %s\n\n'%VERSION) 'Proxy-agent: %s\n\n' % VERSION)
self.client.close() self.client.close()
return return
try: try:
@ -113,12 +125,12 @@ class ConnectionHandler:
elif self.method in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', elif self.method in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT',
'DELETE', 'TRACE'): 'DELETE', 'TRACE'):
self.method_others() self.method_others()
except: except BaseException:
try: try:
self.client.send(HTTPVER+' 502 Connection failed\n'+ self.client.send(HTTPVER + ' 502 Connection failed\n' +
'Proxy-agent: %s\n\n'%VERSION) 'Proxy-agent: %s\n\n' % VERSION)
except Exception, e: except Exception as e:
print 'PROXY - ', e print('PROXY - ', e)
self.client.close() self.client.close()
return return
@ -127,10 +139,10 @@ class ConnectionHandler:
def get_base_header(self): def get_base_header(self):
retries = 0 retries = 0
while 1: while True:
try: try:
self.client_buffer += self.client.recv(BUFLEN) self.client_buffer += self.client.recv(BUFLEN)
except socket.error, e: except socket.error as e:
err = e.args[0] err = e.args[0]
if (err == errno.EAGAIN or err == errno.EWOULDBLOCK) and retries < 20: if (err == errno.EAGAIN or err == errno.EWOULDBLOCK) and retries < 20:
time.sleep(0.5) time.sleep(0.5)
@ -138,18 +150,18 @@ class ConnectionHandler:
continue continue
raise e raise e
end = self.client_buffer.find('\r\n\r\n') end = self.client_buffer.find('\r\n\r\n')
if end!=-1: if end != -1:
break break
line_end = self.client_buffer.find('\n') line_end = self.client_buffer.find('\n')
print 'PROXY - %s' % self.client_buffer[:line_end]#debug print('PROXY - %s' % self.client_buffer[:line_end]) # debug
data = (self.client_buffer[:line_end+1]).split() data = (self.client_buffer[:line_end + 1]).split()
self.client_buffer = self.client_buffer[line_end+1:] self.client_buffer = self.client_buffer[line_end + 1:]
return data return data
def method_CONNECT(self): def method_CONNECT(self):
self._connect_target(self.path) self._connect_target(self.path)
self.client.send(HTTPVER+' 200 Connection established\n'+ self.client.send(HTTPVER + ' 200 Connection established\n' +
'Proxy-agent: %s\n\n'%VERSION) 'Proxy-agent: %s\n\n' % VERSION)
self.client_buffer = '' self.client_buffer = ''
self._read_write() self._read_write()
@ -159,15 +171,15 @@ class ConnectionHandler:
host = self.path[:i] host = self.path[:i]
path = self.path[i:] path = self.path[i:]
self._connect_target(host) self._connect_target(host)
self.target.send('%s %s %s\n' % (self.method, path, self.protocol)+ self.target.send('%s %s %s\n' % (self.method, path, self.protocol) +
self.client_buffer) self.client_buffer)
self.client_buffer = '' self.client_buffer = ''
self._read_write() self._read_write()
def _connect_target(self, host): def _connect_target(self, host):
i = host.find(':') i = host.find(':')
if i!=-1: if i != -1:
port = int(host[i+1:]) port = int(host[i + 1:])
host = host[:i] host = host[:i]
else: else:
port = 80 port = 80
@ -176,10 +188,10 @@ class ConnectionHandler:
self.target.connect(address) self.target.connect(address)
def _read_write(self): def _read_write(self):
time_out_max = self.timeout/3 time_out_max = self.timeout / 3
socs = [self.client, self.target] socs = [self.client, self.target]
count = 0 count = 0
while 1: while True:
count += 1 count += 1
(recv, _, error) = select.select(socs, [], socs, 3) (recv, _, error) = select.select(socs, [], socs, 3)
if error: if error:
@ -197,37 +209,39 @@ class ConnectionHandler:
if count == time_out_max: if count == time_out_max:
break break
def start_server(host='localhost', port=8080, IPv6=False, timeout=100, def start_server(host='localhost', port=8080, IPv6=False, timeout=100,
handler=ConnectionHandler): handler=ConnectionHandler):
if IPv6==True: if IPv6:
soc_type=socket.AF_INET6 soc_type = socket.AF_INET6
else: else:
soc_type=socket.AF_INET soc_type = socket.AF_INET
soc = socket.socket(soc_type) soc = socket.socket(soc_type)
soc.settimeout(120) soc.settimeout(120)
print "PROXY - Serving on %s:%d."%(host, port)#debug print("PROXY - Serving on %s:%d." % (host, port)) # debug
soc.bind((host, port)) soc.bind((host, port))
soc.listen(0) soc.listen(0)
while 1: while True:
thread.start_new_thread(handler, soc.accept()+(timeout,)) thread.start_new_thread(handler, soc.accept() + (timeout,))
if __name__ == '__main__': if __name__ == '__main__':
listen_port = 8080 listen_port = 8080
i = 1 i = 1
while i < len(sys.argv): while i < len(sys.argv):
if sys.argv[i] == '--port': if sys.argv[i] == '--port':
listen_port = int(sys.argv[i+1]) listen_port = int(sys.argv[i + 1])
i += 1 i += 1
elif sys.argv[i] == '--username': elif sys.argv[i] == '--username':
username = sys.argv[i+1] username = sys.argv[i + 1]
i += 1 i += 1
elif sys.argv[i] == '--password': elif sys.argv[i] == '--password':
password = sys.argv[i+1] password = sys.argv[i + 1]
i += 1 i += 1
else: else:
if sys.argv[i] != '--help': print('PROXY - unknown option "%s"' % sys.argv[i]) if sys.argv[i] != '--help':
print(('PROXY - unknown option "%s"' % sys.argv[i]))
print('usage: http.py [--port <listen-port>]') print('usage: http.py [--port <listen-port>]')
sys.exit(1) sys.exit(1)
i += 1 i += 1
start_server(port=listen_port) start_server(port=listen_port)

72
test/socks.py
View File

@ -1,19 +1,27 @@
#!/usr/bin/env python #!/usr/bin/env python
"""Minimal non-feature complete socks proxy""" """Minimal non-feature complete socks proxy"""
from __future__ import print_function
import random
import socket import socket
from SocketServer import StreamRequestHandler, ThreadingTCPServer
from struct import pack, unpack from struct import pack, unpack
import threading import threading
import sys import sys
# Python 3 renamed SocketServer to socketserver
try:
from socketserver import StreamRequestHandler, ThreadingTCPServer
except BaseException:
from SocketServer import StreamRequestHandler, ThreadingTCPServer
def debug(s): def debug(s):
print >>sys.stderr, 'socks.py: ', s print('socks.py: ', s, file=sys.stderr)
def error(s): def error(s):
print >>sys.stderr, 'socks.py, ERROR: ', s print('socks.py, ERROR: ', s, file=sys.stderr)
class MyTCPServer(ThreadingTCPServer): class MyTCPServer(ThreadingTCPServer):
allow_reuse_address = True allow_reuse_address = True
@ -21,6 +29,7 @@ class MyTCPServer(ThreadingTCPServer):
def handle_timeout(self): def handle_timeout(self):
raise Exception('timeout') raise Exception('timeout')
CLOSE = object() CLOSE = object()
VERSION = '\x05' VERSION = '\x05'
@ -37,15 +46,19 @@ password = None
username = None username = None
allow_v4 = False allow_v4 = False
def send(dest, msg): def send(dest, msg):
if msg == CLOSE: if msg == CLOSE:
try: dest.shutdown(socket.SHUT_WR) try:
except: pass dest.shutdown(socket.SHUT_WR)
except BaseException:
pass
dest.close() dest.close()
return 0 return 0
else: else:
return dest.sendall(msg) return dest.sendall(msg)
def recv(source, buffer): def recv(source, buffer):
data = source.recv(buffer) data = source.recv(buffer)
if data == '': if data == '':
@ -53,6 +66,7 @@ def recv(source, buffer):
else: else:
return data return data
def forward(source, dest, name): def forward(source, dest, name):
while True: while True:
data = recv(source, 4000) data = recv(source, 4000)
@ -63,11 +77,13 @@ def forward(source, dest, name):
# debug('Forwarding (%d) %r' % (len(data), data)) # debug('Forwarding (%d) %r' % (len(data), data))
send(dest, data) send(dest, data)
def spawn_forwarder(source, dest, name): def spawn_forwarder(source, dest, name):
t = threading.Thread(target=forward, args=(source, dest, name)) t = threading.Thread(target=forward, args=(source, dest, name))
t.daemon = True t.daemon = True
t.start() t.start()
class SocksHandler(StreamRequestHandler): class SocksHandler(StreamRequestHandler):
"""Highly feature incomplete SOCKS 5 implementation""" """Highly feature incomplete SOCKS 5 implementation"""
@ -109,7 +125,7 @@ class SocksHandler(StreamRequestHandler):
c = self.read(1) c = self.read(1)
outbound_sock = socket.socket(socket.AF_INET) outbound_sock = socket.socket(socket.AF_INET)
out_address = socket.getaddrinfo(dest_address,dest_port)[0][4] out_address = socket.getaddrinfo(dest_address, dest_port)[0][4]
debug("Creating forwarder connection to %s:%d" % (out_address[0], out_address[1])) debug("Creating forwarder connection to %s:%d" % (out_address[0], out_address[1]))
outbound_sock.connect(out_address) outbound_sock.connect(out_address)
@ -130,7 +146,7 @@ class SocksHandler(StreamRequestHandler):
global password global password
global username global username
if password == None and NOAUTH in method_list: if password is None and NOAUTH in method_list:
self.send_no_auth_method() self.send_no_auth_method()
debug('Authenticated (no-auth)') debug('Authenticated (no-auth)')
elif USERPASS in method_list: elif USERPASS in method_list:
@ -173,7 +189,7 @@ class SocksHandler(StreamRequestHandler):
dest_address = '.'.join(map(str, unpack('>4B', raw_dest_address))) dest_address = '.'.join(map(str, unpack('>4B', raw_dest_address)))
elif address_type == IPV6: elif address_type == IPV6:
raw_dest_address = self.read(16) raw_dest_address = self.read(16)
dest_address = ":".join(map(lambda x: hex(x)[2:],unpack('>8H',raw_dest_address))) dest_address = ":".join([hex(x)[2:] for x in unpack('>8H', raw_dest_address)])
elif address_type == DOMAIN_NAME: elif address_type == DOMAIN_NAME:
dns_length = ord(self.read(1)) dns_length = ord(self.read(1))
dns_name = self.read(dns_length) dns_name = self.read(dns_length)
@ -190,21 +206,21 @@ class SocksHandler(StreamRequestHandler):
else: else:
outbound_sock = socket.socket(socket.AF_INET) outbound_sock = socket.socket(socket.AF_INET)
try: try:
out_address = socket.getaddrinfo(dest_address,dest_port)[0][4] out_address = socket.getaddrinfo(dest_address, dest_port)[0][4]
except Exception, e: except Exception as e:
print e print(e)
return return
if cmd == UDP_ASSOCIATE: if cmd == UDP_ASSOCIATE:
debug("no UDP support yet, closing") debug("no UDP support yet, closing")
return; return
debug("Creating forwarder connection to %s:%d" % (out_address[0], out_address[1])) debug("Creating forwarder connection to %s:%d" % (out_address[0], out_address[1]))
try: try:
outbound_sock.connect(out_address) outbound_sock.connect(out_address)
except Exception, e: except Exception as e:
print e print(e)
return return
if address_type == IPV6: if address_type == IPV6:
@ -215,22 +231,25 @@ class SocksHandler(StreamRequestHandler):
spawn_forwarder(outbound_sock, self.request, 'destination') spawn_forwarder(outbound_sock, self.request, 'destination')
try: try:
forward(self.request, outbound_sock, 'client') forward(self.request, outbound_sock, 'client')
except Exception,e: except Exception as e:
print e print(e)
def send_reply_v4(self, (bind_addr, bind_port)): def send_reply_v4(self, xxx_todo_changeme):
(bind_addr, bind_port) = xxx_todo_changeme
self.wfile.write('\0\x5a\0\0\0\0\0\0') self.wfile.write('\0\x5a\0\0\0\0\0\0')
self.wfile.flush() self.wfile.flush()
def send_reply(self, (bind_addr, bind_port)): def send_reply(self, xxx_todo_changeme1):
(bind_addr, bind_port) = xxx_todo_changeme1
bind_tuple = tuple(map(int, bind_addr.split('.'))) bind_tuple = tuple(map(int, bind_addr.split('.')))
full_address = bind_tuple + (bind_port,) full_address = bind_tuple + (bind_port,)
debug('Setting up forwarding port %r' % (full_address,)) debug('Setting up forwarding port %r' % (full_address,))
msg = pack('>cccc4BH', VERSION, SUCCESS, '\x00', IPV4, *full_address) msg = pack('>cccc4BH', VERSION, SUCCESS, '\x00', IPV4, *full_address)
self.wfile.write(msg) self.wfile.write(msg)
def send_reply6(self, (bind_addr, bind_port, unused1, unused2)): def send_reply6(self, xxx_todo_changeme2):
bind_tuple = tuple(map(lambda x: int(x,16), bind_addr.split(':'))) (bind_addr, bind_port, unused1, unused2) = xxx_todo_changeme2
bind_tuple = tuple([int(x, 16) for x in bind_addr.split(':')])
full_address = bind_tuple + (bind_port,) full_address = bind_tuple + (bind_port,)
debug('Setting up forwarding port %r' % (full_address,)) debug('Setting up forwarding port %r' % (full_address,))
msg = pack('>cccc8HH', VERSION, SUCCESS, '\x00', IPV6, *full_address) msg = pack('>cccc8HH', VERSION, SUCCESS, '\x00', IPV6, *full_address)
@ -252,24 +271,26 @@ class SocksHandler(StreamRequestHandler):
self.wfile.write('\x01\x00') self.wfile.write('\x01\x00')
self.wfile.flush() self.wfile.flush()
if __name__ == '__main__': if __name__ == '__main__':
listen_port = 8002 listen_port = 8002
i = 1 i = 1
while i < len(sys.argv): while i < len(sys.argv):
if sys.argv[i] == '--username': if sys.argv[i] == '--username':
username = sys.argv[i+1] username = sys.argv[i + 1]
i += 1 i += 1
elif sys.argv[i] == '--password': elif sys.argv[i] == '--password':
password = sys.argv[i+1] password = sys.argv[i + 1]
i += 1 i += 1
elif sys.argv[i] == '--port': elif sys.argv[i] == '--port':
listen_port = int(sys.argv[i+1]) listen_port = int(sys.argv[i + 1])
i += 1 i += 1
elif sys.argv[i] == '--allow-v4': elif sys.argv[i] == '--allow-v4':
allow_v4 = True allow_v4 = True
else: else:
if sys.argv[i] != '--help': debug('unknown option "%s"' % sys.argv[i]) if sys.argv[i] != '--help':
debug('unknown option "%s"' % sys.argv[i])
print('usage: socks.py [--username <user> --password <password>] [--port <listen-port>]') print('usage: socks.py [--username <user> --password <password>] [--port <listen-port>]')
sys.exit(1) sys.exit(1)
i += 1 i += 1
@ -279,4 +300,3 @@ if __name__ == '__main__':
server.timeout = 190 server.timeout = 190
while True: while True:
server.handle_request() server.handle_request()

55
test/web_server.py
View File

@ -1,13 +1,21 @@
#!/usr/bin/env python #!/usr/bin/env python
import BaseHTTPServer
import SimpleHTTPServer
import sys import sys
import os import os
import ssl import ssl
import gzip import gzip
import base64 import base64
# Python 3 has moved {Simple,Base}HTTPServer to http module
try:
# Remove '.' from sys.path or we try to import the http.py module
# which is not what we want.
sys.path = sys.path[1:]
from http.server import HTTPServer, BaseHTTPRequestHandler
except ImportError:
from SimpleHTTPServer import SimpleHTTPRequestHandler as BaseHTTPRequestHandler
from BaseHTTPServer import HTTPServer
chunked_encoding = False chunked_encoding = False
keepalive = True keepalive = True
@ -17,22 +25,24 @@ try:
f.writelines(fin) f.writelines(fin)
f.close() f.close()
fin.close() fin.close()
except: except BaseException:
pass pass
class http_server_with_timeout(BaseHTTPServer.HTTPServer):
class http_server_with_timeout(HTTPServer):
allow_reuse_address = True allow_reuse_address = True
timeout = 190 timeout = 190
def handle_timeout(self): def handle_timeout(self):
raise Exception('timeout') raise Exception('timeout')
class http_handler(SimpleHTTPServer.SimpleHTTPRequestHandler):
class http_handler(BaseHTTPRequestHandler):
def do_GET(s): def do_GET(s):
print 'INCOMING-REQUEST: ', s.requestline print('INCOMING-REQUEST: ', s.requestline)
print s.headers print(s.headers)
global chunked_encoding global chunked_encoding
global keepalive global keepalive
@ -43,8 +53,8 @@ class http_handler(SimpleHTTPServer.SimpleHTTPRequestHandler):
s.path = s.path[s.path.find('/'):] s.path = s.path[s.path.find('/'):]
file_path = os.path.normpath(s.path) file_path = os.path.normpath(s.path)
print file_path print(file_path)
print s.path print(s.path)
if s.path == '/password_protected': if s.path == '/password_protected':
passed = False passed = False
@ -100,19 +110,19 @@ class http_handler(SimpleHTTPServer.SimpleHTTPRequestHandler):
filename = '' filename = ''
try: try:
filename = os.path.normpath(s.path[1:s.path.find('seed?') + 4]) filename = os.path.normpath(s.path[1:s.path.find('seed?') + 4])
print 'filename = %s' % filename print('filename = %s' % filename)
f = open(filename, 'rb') f = open(filename, 'rb')
f.seek(piece * 32 * 1024 + int(ranges[0])) f.seek(piece * 32 * 1024 + int(ranges[0]))
data = f.read(int(ranges[1]) - int(ranges[0]) + 1) data = f.read(int(ranges[1]) - int(ranges[0]) + 1)
f.close() f.close()
s.send_response(200) s.send_response(200)
print 'sending %d bytes' % len(data) print('sending %d bytes' % len(data))
s.send_header("Content-Length", "%d" % len(data)) s.send_header("Content-Length", "%d" % len(data))
s.end_headers() s.end_headers()
s.wfile.write(data); s.wfile.write(data)
except Exception, e: except Exception as e:
print 'FILE ERROR: ', filename, e print('FILE ERROR: ', filename, e)
s.send_response(404) s.send_response(404)
s.send_header("Content-Length", "0") s.send_header("Content-Length", "0")
s.end_headers() s.end_headers()
@ -138,7 +148,7 @@ class http_handler(SimpleHTTPServer.SimpleHTTPRequestHandler):
ei = int(e) ei = int(e)
if ei < size: if ei < size:
start_range = size - ei start_range = size - ei
s.send_header('Content-Range', 'bytes ' + str(start_range) \ s.send_header('Content-Range', 'bytes ' + str(start_range)
+ '-' + str(end_range - 1) + '/' + str(size)) + '-' + str(end_range - 1) + '/' + str(size))
else: else:
s.send_response(200) s.send_response(200)
@ -151,9 +161,9 @@ class http_handler(SimpleHTTPServer.SimpleHTTPRequestHandler):
if not keepalive: if not keepalive:
s.send_header("Connection", "close") s.send_header("Connection", "close")
try: try:
s.request.shutdown(); s.request.shutdown()
except Exception, e: except Exception as e:
print 'Failed to shutdown read-channel of socket: ', e print('Failed to shutdown read-channel of socket: ', e)
s.end_headers() s.end_headers()
@ -164,20 +174,21 @@ class http_handler(SimpleHTTPServer.SimpleHTTPRequestHandler):
if chunked_encoding: if chunked_encoding:
s.wfile.write('%x\r\n' % to_send) s.wfile.write('%x\r\n' % to_send)
data = f.read(to_send) data = f.read(to_send)
print 'read %d bytes' % to_send print('read %d bytes' % to_send)
s.wfile.write(data) s.wfile.write(data)
if chunked_encoding: if chunked_encoding:
s.wfile.write('\r\n') s.wfile.write('\r\n')
length -= to_send length -= to_send
print 'sent %d bytes (%d bytes left)' % (len(data), length) print('sent %d bytes (%d bytes left)' % (len(data), length))
if chunked_encoding: if chunked_encoding:
s.wfile.write('0\r\n\r\n') s.wfile.write('0\r\n\r\n')
except Exception, e: except Exception as e:
print 'FILE ERROR: ', filename, e print('FILE ERROR: ', filename, e)
s.send_response(404) s.send_response(404)
s.send_header("Content-Length", "0") s.send_header("Content-Length", "0")
s.end_headers() s.end_headers()
if __name__ == '__main__': if __name__ == '__main__':
port = int(sys.argv[1]) port = int(sys.argv[1])
chunked_encoding = sys.argv[2] != '0' chunked_encoding = sys.argv[2] != '0'

13
tools/clean.py
View File

@ -4,6 +4,7 @@ import os
import shutil import shutil
import glob import glob
def clean(): def clean():
to_delete = [ to_delete = [
'session_stats', 'session_stats',
@ -68,14 +69,14 @@ def clean():
for p in entries: for p in entries:
try: try:
shutil.rmtree(p) shutil.rmtree(p)
print p print(p)
except Exception, e: except Exception as e:
try: try:
os.remove(p) os.remove(p)
print p print(p)
except Exception, e: except Exception as e:
print p, e print(p, e)
if __name__ == "__main__": if __name__ == "__main__":
clean() clean()

29
tools/dht_flood.py
View File

@ -8,30 +8,36 @@ import random
port = int(sys.argv[1]) port = int(sys.argv[1])
# from BitTorrent 4.3.0 # from BitTorrent 4.3.0
def encode_bencached(x,r):
def encode_bencached(x, r):
r.append(x.bencoded) r.append(x.bencoded)
def encode_int(x, r): def encode_int(x, r):
r.extend(('i', str(x), 'e')) r.extend(('i', str(x), 'e'))
def encode_string(x, r): def encode_string(x, r):
r.extend((str(len(x)), ':', x)) r.extend((str(len(x)), ':', x))
def encode_list(x, r): def encode_list(x, r):
r.append('l') r.append('l')
for i in x: for i in x:
encode_func[type(i)](i, r) encode_func[type(i)](i, r)
r.append('e') r.append('e')
def encode_dict(x,r):
def encode_dict(x, r):
r.append('d') r.append('d')
ilist = x.items() ilist = sorted(x.items())
ilist.sort()
for k, v in ilist: for k, v in ilist:
r.extend((str(len(k)), ':', k)) r.extend((str(len(k)), ':', k))
encode_func[type(v)](v, r) encode_func[type(v)](v, r)
r.append('e') r.append('e')
encode_func = {} encode_func = {}
encode_func[IntType] = encode_int encode_func[IntType] = encode_int
encode_func[LongType] = encode_int encode_func[LongType] = encode_int
@ -40,29 +46,32 @@ encode_func[ListType] = encode_list
encode_func[TupleType] = encode_list encode_func[TupleType] = encode_list
encode_func[DictType] = encode_dict encode_func[DictType] = encode_dict
def bencode(x): def bencode(x):
r = [] r = []
encode_func[type(x)](x, r) encode_func[type(x)](x, r)
return ''.join(r) return ''.join(r)
def send_dht_message(msg): def send_dht_message(msg):
s.sendto(bencode(msg), 0, ('127.0.0.1', port)) s.sendto(bencode(msg), 0, ('127.0.0.1', port))
def random_key(): def random_key():
ret = '' ret = ''
for i in range(0, 20): for i in range(0, 20):
ret += chr(random.randint(0, 255)) ret += chr(random.randint(0, 255))
return ret return ret
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
node_id = '1' * 20; node_id = '1' * 20
query = 'get_peers' query = 'get_peers'
print 'test random info-hashes' print('test random info-hashes')
for i in xrange(1, 30000): for i in range(1, 30000):
send_dht_message({'a': {'id': node_id, 'info_hash': random_key()}, 'q': query, 'y': 'q', 't': '%d' % i}) send_dht_message({'a': {'id': node_id, 'info_hash': random_key()}, 'q': query, 'y': 'q', 't': '%d' % i})
print 'test random peer-ids' print('test random peer-ids')
for i in xrange(1, 30000): for i in range(1, 30000):
send_dht_message({'a': {'id': random_key(), 'info_hash': random_key()}, 'q': query, 'y': 'q', 't': '%d' % i}) send_dht_message({'a': {'id': random_key(), 'info_hash': random_key()}, 'q': query, 'y': 'q', 't': '%d' % i})

33
tools/gen_fwd.py
View File

@ -1,9 +1,8 @@
#!/usr/bin/env python #!/usr/bin/env python
import os import os
import sys
file_header ='''/* file_header = '''/*
Copyright (c) 2017, Arvid Norberg Copyright (c) 2017, Arvid Norberg
All rights reserved. All rights reserved.
@ -52,12 +51,12 @@ namespace lt = libtorrent;
#endif // TORRENT_FWD_HPP #endif // TORRENT_FWD_HPP
''' '''
classes = os.popen('git grep "\(TORRENT_EXPORT\|TORRENT_DEPRECATED_EXPORT\|^TORRENT_[A-Z0-9]\+_NAMESPACE\)"').read().split('\n') classes = os.popen(
r'git grep "\(TORRENT_EXPORT\|TORRENT_DEPRECATED_EXPORT\|^TORRENT_[A-Z0-9]\+_NAMESPACE\)"').read().split('\n')
def print_classes(out, classes, keyword): def print_classes(out, classes, keyword):
current_file = '' current_file = ''
ret = ''
dht_ret = ''
# [(file, decl), ...] # [(file, decl), ...]
classes = [(l.split(':')[0].strip(), ':'.join(l.split(':')[1:]).strip()) for l in classes] classes = [(l.split(':')[0].strip(), ':'.join(l.split(':')[1:]).strip()) for l in classes]
@ -71,18 +70,22 @@ def print_classes(out, classes, keyword):
'TORRENT_VERSION_NAMESPACE_2_END', 'TORRENT_IPV6_NAMESPACE_END'] 'TORRENT_VERSION_NAMESPACE_2_END', 'TORRENT_IPV6_NAMESPACE_END']
# only include classes with the right kind of export # only include classes with the right kind of export
classes = [l for l in classes if l[1] in namespaces or (l[1].split(' ')[0] in ['class', 'struct'] and l[1].split(' ')[1] == keyword)] classes = [
l for l in classes if l[1] in namespaces or (
l[1].split(' ')[0] in [
'class',
'struct'] and l[1].split(' ')[1] == keyword)]
# collapse empty namespaces # collapse empty namespaces
classes2 = [] classes2 = []
skip = 0 skip = 0
for i in xrange(len(classes)): for i in range(len(classes)):
if skip > 0: if skip > 0:
skip -= 1 skip -= 1
continue continue
if classes[i][1] in namespaces \ if classes[i][1] in namespaces \
and len(classes) > i+1 \ and len(classes) > i + 1 \
and classes[i+1][1] == ('%s_END' % classes[i][1]): and classes[i + 1][1] == ('%s_END' % classes[i][1]):
skip = 1 skip = 1
else: else:
classes2.append(classes[i]) classes2.append(classes[i])
@ -98,10 +101,11 @@ def print_classes(out, classes, keyword):
content = '' content = ''
if this_file != current_file: if this_file != current_file:
out.write('\n// ' + this_file + '\n') out.write('\n// ' + this_file + '\n')
current_file = this_file; current_file = this_file
if len(decl) > 2 and decl[0] in ['struct', 'class']: if len(decl) > 2 and decl[0] in ['struct', 'class']:
decl = decl[0] + ' ' + decl[2] decl = decl[0] + ' ' + decl[2]
if not decl.endswith(';'): decl += ';' if not decl.endswith(';'):
decl += ';'
content = decl + '\n' content = decl + '\n'
else: else:
content = line[1] + '\n' content = line[1] + '\n'
@ -113,18 +117,17 @@ def print_classes(out, classes, keyword):
else: else:
out.write(content) out.write(content)
os.remove('include/libtorrent/fwd.hpp') os.remove('include/libtorrent/fwd.hpp')
with open('include/libtorrent/fwd.hpp', 'w+') as f: with open('include/libtorrent/fwd.hpp', 'w+') as f:
f.write(file_header) f.write(file_header)
print_classes(f, classes, 'TORRENT_EXPORT'); print_classes(f, classes, 'TORRENT_EXPORT')
f.write('\n#if TORRENT_ABI_VERSION == 1\n') f.write('\n#if TORRENT_ABI_VERSION == 1\n')
print_classes(f, classes, 'TORRENT_DEPRECATED_EXPORT'); print_classes(f, classes, 'TORRENT_DEPRECATED_EXPORT')
f.write('\n#endif // TORRENT_ABI_VERSION') f.write('\n#endif // TORRENT_ABI_VERSION')
f.write(file_footer) f.write(file_footer)

334
tools/gprof2dot.py
View File

@ -35,53 +35,57 @@ import locale
# Python 2.x/3.x compatibility # Python 2.x/3.x compatibility
if sys.version_info[0] >= 3: if sys.version_info[0] >= 3:
PYTHON_3 = True PYTHON_3 = True
def compat_iteritems(x): return x.items() # No iteritems() in Python 3 def compat_iteritems(x): return x.items() # No iteritems() in Python 3
def compat_itervalues(x): return x.values() # No itervalues() in Python 3 def compat_itervalues(x): return x.values() # No itervalues() in Python 3
def compat_keys(x): return list(x.keys()) # keys() is a generator in Python 3 def compat_keys(x): return list(x.keys()) # keys() is a generator in Python 3
basestring = str # No class basestring in Python 3 basestring = str # No class basestring in Python 3
unichr = chr # No unichr in Python 3 unichr = chr # No unichr in Python 3
xrange = range # No xrange in Python 3 xrange = range # No xrange in Python 3
else: else:
PYTHON_3 = False PYTHON_3 = False
def compat_iteritems(x): return x.iteritems() def compat_iteritems(x): return x.iteritems()
def compat_itervalues(x): return x.itervalues() def compat_itervalues(x): return x.itervalues()
def compat_keys(x): return x.keys() def compat_keys(x): return x.keys()
try:
# Debugging helper module
import debug
except ImportError:
pass
MULTIPLICATION_SIGN = unichr(0xd7) MULTIPLICATION_SIGN = unichr(0xd7)
def times(x): def times(x):
return "%u%s" % (x, MULTIPLICATION_SIGN) return "%u%s" % (x, MULTIPLICATION_SIGN)
def percentage(p): def percentage(p):
return "%.02f%%" % (p*100.0,) return "%.02f%%" % (p * 100.0,)
def add(a, b): def add(a, b):
return a + b return a + b
def equal(a, b): def equal(a, b):
if a == b: if a == b:
return a return a
else: else:
return None return None
def fail(a, b): def fail(a, b):
assert False assert False
tol = 2 ** -23 tol = 2 ** -23
def ratio(numerator, denominator): def ratio(numerator, denominator):
try: try:
ratio = float(numerator)/float(denominator) ratio = float(numerator) / float(denominator)
except ZeroDivisionError: except ZeroDivisionError:
# 0/0 is undefined, but 1.0 yields more useful results # 0/0 is undefined, but 1.0 yields more useful results
return 1.0 return 1.0
@ -110,7 +114,7 @@ class UndefinedEvent(Exception):
class Event(object): class Event(object):
"""Describe a kind of event, and its basic operations.""" """Describe a kind of event, and its basic operations."""
def __init__(self, name, null, aggregator, formatter = str): def __init__(self, name, null, aggregator, formatter=str):
self.name = name self.name = name
self._null = null self._null = null
self._aggregator = aggregator self._aggregator = aggregator
@ -224,7 +228,7 @@ class Function(Object):
self.calls[call.callee_id] = call self.calls[call.callee_id] = call
def get_call(self, callee_id): def get_call(self, callee_id):
if not callee_id in self.calls: if callee_id not in self.calls:
call = Call(callee_id) call = Call(callee_id)
call[SAMPLES] = 0 call[SAMPLES] = 0
call[SAMPLES2] = 0 call[SAMPLES2] = 0
@ -306,7 +310,9 @@ class Profile(Object):
for callee_id in compat_keys(function.calls): for callee_id in compat_keys(function.calls):
assert function.calls[callee_id].callee_id == callee_id assert function.calls[callee_id].callee_id == callee_id
if callee_id not in self.functions: if callee_id not in self.functions:
sys.stderr.write('warning: call to undefined function %s from function %s\n' % (str(callee_id), function.name)) sys.stderr.write(
'warning: call to undefined function %s from function %s\n' %
(str(callee_id), function.name))
del function.calls[callee_id] del function.calls[callee_id]
def find_cycles(self): def find_cycles(self):
@ -368,7 +374,6 @@ class Profile(Object):
pathFunctions[n] = f pathFunctions[n] = f
self.functions = pathFunctions self.functions = pathFunctions
def getFunctionId(self, funcName): def getFunctionId(self, funcName):
for f in self.functions: for f in self.functions:
if self.functions[f].name == funcName: if self.functions[f].name == funcName:
@ -496,7 +501,7 @@ class Profile(Object):
assert outevent not in call assert outevent not in call
assert call.ratio is not None assert call.ratio is not None
callee = self.functions[call.callee_id] callee = self.functions[call.callee_id]
subtotal = call.ratio *self._integrate_function(callee, outevent, inevent) subtotal = call.ratio * self._integrate_function(callee, outevent, inevent)
call[outevent] = subtotal call[outevent] = subtotal
return subtotal return subtotal
@ -535,9 +540,10 @@ class Profile(Object):
partials = {} partials = {}
self._rank_cycle_function(cycle, callee, 0, ranks) self._rank_cycle_function(cycle, callee, 0, ranks)
self._call_ratios_cycle(cycle, callee, ranks, call_ratios, set()) self._call_ratios_cycle(cycle, callee, ranks, call_ratios, set())
partial = self._integrate_cycle_function(cycle, callee, call_ratio, partials, ranks, call_ratios, outevent, inevent) partial = self._integrate_cycle_function(
cycle, callee, call_ratio, partials, ranks, call_ratios, outevent, inevent)
assert partial == max(partials.values()) assert partial == max(partials.values())
assert not total or abs(1.0 - partial/(call_ratio*total)) <= 0.001 assert not total or abs(1.0 - partial / (call_ratio * total)) <= 0.001
return cycle[outevent] return cycle[outevent]
@ -561,20 +567,30 @@ class Profile(Object):
call_ratios[callee] = call_ratios.get(callee, 0.0) + call.ratio call_ratios[callee] = call_ratios.get(callee, 0.0) + call.ratio
self._call_ratios_cycle(cycle, callee, ranks, call_ratios, visited) self._call_ratios_cycle(cycle, callee, ranks, call_ratios, visited)
def _integrate_cycle_function(self, cycle, function, partial_ratio, partials, ranks, call_ratios, outevent, inevent): def _integrate_cycle_function(
self,
cycle,
function,
partial_ratio,
partials,
ranks,
call_ratios,
outevent,
inevent):
if function not in partials: if function not in partials:
partial = partial_ratio*function[inevent] partial = partial_ratio * function[inevent]
for call in compat_itervalues(function.calls): for call in compat_itervalues(function.calls):
if call.callee_id != function.id: if call.callee_id != function.id:
callee = self.functions[call.callee_id] callee = self.functions[call.callee_id]
if callee.cycle is not cycle: if callee.cycle is not cycle:
assert outevent in call assert outevent in call
partial += partial_ratio*call[outevent] partial += partial_ratio * call[outevent]
else: else:
if ranks[callee] > ranks[function]: if ranks[callee] > ranks[function]:
callee_partial = self._integrate_cycle_function(cycle, callee, partial_ratio, partials, ranks, call_ratios, outevent, inevent) callee_partial = self._integrate_cycle_function(
cycle, callee, partial_ratio, partials, ranks, call_ratios, outevent, inevent)
call_ratio = ratio(call.ratio, call_ratios[callee]) call_ratio = ratio(call.ratio, call_ratios[callee])
call_partial = call_ratio*callee_partial call_partial = call_ratio * callee_partial
try: try:
call[outevent] += call_partial call[outevent] += call_partial
except UndefinedEvent: except UndefinedEvent:
@ -670,7 +686,7 @@ class Profile(Object):
class Struct: class Struct:
"""Masquerade a dictionary with a structure-like behavior.""" """Masquerade a dictionary with a structure-like behavior."""
def __init__(self, attrs = None): def __init__(self, attrs=None):
if attrs is None: if attrs is None:
attrs = {} attrs = {}
self.__dict__['_attrs'] = attrs self.__dict__['_attrs'] = attrs
@ -761,7 +777,7 @@ XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF = range(4)
class XmlToken: class XmlToken:
def __init__(self, type, name_or_data, attrs = None, line = None, column = None): def __init__(self, type, name_or_data, attrs=None, line=None, column=None):
assert type in (XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF) assert type in (XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF)
self.type = type self.type = type
self.name_or_data = name_or_data self.name_or_data = name_or_data
@ -784,7 +800,7 @@ class XmlToken:
class XmlTokenizer: class XmlTokenizer:
"""Expat based XML tokenizer.""" """Expat based XML tokenizer."""
def __init__(self, fp, skip_ws = True): def __init__(self, fp, skip_ws=True):
self.fp = fp self.fp = fp
self.tokens = [] self.tokens = []
self.index = 0 self.index = 0
@ -825,7 +841,7 @@ class XmlTokenizer:
self.character_data = '' self.character_data = ''
def next(self): def next(self):
size = 16*1024 size = 16 * 1024
while self.index >= len(self.tokens) and not self.final: while self.index >= len(self.tokens) and not self.final:
self.tokens = [] self.tokens = []
self.index = 0 self.index = 0
@ -834,7 +850,7 @@ class XmlTokenizer:
try: try:
self.parser.Parse(data, self.final) self.parser.Parse(data, self.final)
except xml.parsers.expat.ExpatError as e: except xml.parsers.expat.ExpatError as e:
#if e.code == xml.parsers.expat.errors.XML_ERROR_NO_ELEMENTS: # if e.code == xml.parsers.expat.errors.XML_ERROR_NO_ELEMENTS:
if e.code == 3: if e.code == 3:
pass pass
else: else:
@ -858,7 +874,8 @@ class XmlTokenMismatch(Exception):
self.found = found self.found = found
def __str__(self): def __str__(self):
return '%u:%u: %s expected, %s found' % (self.found.line, self.found.column, str(self.expected), str(self.found)) return '%u:%u: %s expected, %s found' % (
self.found.line, self.found.column, str(self.expected), str(self.found))
class XmlParser(Parser): class XmlParser(Parser):
@ -898,7 +915,7 @@ class XmlParser(Parser):
raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token) raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)
self.consume() self.consume()
def character_data(self, strip = True): def character_data(self, strip=True):
data = '' data = ''
while self.token.type == XML_CHARACTER_DATA: while self.token.type == XML_CHARACTER_DATA:
data += self.token.name_or_data data += self.token.name_or_data
@ -1227,7 +1244,7 @@ class AXEParser(Parser):
'^-----+ ' '^-----+ '
) )
_cg_footer_re = re.compile('^Index\s+Function\s*$') _cg_footer_re = re.compile(r'^Index\s+Function\s*$')
_cg_primary_re = re.compile( _cg_primary_re = re.compile(
r'^\[(?P<index>\d+)\]?' + r'^\[(?P<index>\d+)\]?' +
@ -1378,7 +1395,8 @@ class AXEParser(Parser):
line = self.readline() line = self.readline()
def parse(self): def parse(self):
sys.stderr.write('warning: for axe format, edge weights are unreliable estimates derived from\nfunction total times.\n') sys.stderr.write(
'warning: for axe format, edge weights are unreliable estimates derived from\nfunction total times.\n')
self.parse_cg() self.parse_cg()
self.fp.close() self.fp.close()
@ -1440,7 +1458,7 @@ class AXEParser(Parser):
for call in compat_itervalues(function.calls): for call in compat_itervalues(function.calls):
if call.ratio is not None: if call.ratio is not None:
callee = profile.functions[call.callee_id] callee = profile.functions[call.callee_id]
call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]; call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]
return profile return profile
@ -1452,7 +1470,7 @@ class CallgrindParser(LineParser):
- http://valgrind.org/docs/manual/cl-format.html - http://valgrind.org/docs/manual/cl-format.html
""" """
_call_re = re.compile('^calls=\s*(\d+)\s+((\d+|\+\d+|-\d+|\*)\s+)+$') _call_re = re.compile(r'^calls=\s*(\d+)\s+((\d+|\+\d+|-\d+|\*)\s+)+$')
def __init__(self, infile): def __init__(self, infile):
LineParser.__init__(self, infile) LineParser.__init__(self, infile)
@ -1541,7 +1559,7 @@ class CallgrindParser(LineParser):
if key == 'positions': if key == 'positions':
self.num_positions = len(items) self.num_positions = len(items)
self.cost_positions = items self.cost_positions = items
self.last_positions = [0]*self.num_positions self.last_positions = [0] * self.num_positions
return True return True
def parse_cost_summary(self): def parse_cost_summary(self):
@ -1584,9 +1602,9 @@ class CallgrindParser(LineParser):
values = line.split() values = line.split()
assert len(values) <= self.num_positions + self.num_events assert len(values) <= self.num_positions + self.num_events
positions = values[0 : self.num_positions] positions = values[0: self.num_positions]
events = values[self.num_positions : ] events = values[self.num_positions:]
events += ['0']*(self.num_events - len(events)) events += ['0'] * (self.num_events - len(events))
for i in range(self.num_positions): for i in range(self.num_positions):
position = positions[i] position = positions[i]
@ -1631,14 +1649,13 @@ class CallgrindParser(LineParser):
_, values = line.split('=', 1) _, values = line.split('=', 1)
values = values.strip().split() values = values.strip().split()
calls = int(values[0]) calls = int(values[0])
call_position = values[1:]
self.consume() self.consume()
self.parse_cost_line(calls) self.parse_cost_line(calls)
return True return True
_position_re = re.compile('^(?P<position>[cj]?(?:ob|fl|fi|fe|fn))=\s*(?:\((?P<id>\d+)\))?(?:\s*(?P<name>.+))?') _position_re = re.compile(r'^(?P<position>[cj]?(?:ob|fl|fi|fe|fn))=\s*(?:\((?P<id>\d+)\))?(?:\s*(?P<name>.+))?')
_position_table_map = { _position_table_map = {
'ob': 'ob', 'ob': 'ob',
@ -1730,7 +1747,7 @@ class CallgrindParser(LineParser):
def make_function(self, module, filename, name): def make_function(self, module, filename, name):
# FIXME: module and filename are not being tracked reliably # FIXME: module and filename are not being tracked reliably
#id = '|'.join((module, filename, name)) # id = '|'.join((module, filename, name))
id = name id = name
try: try:
function = self.profile.functions[id] function = self.profile.functions[id]
@ -1802,7 +1819,7 @@ class PerfParser(LineParser):
for call in compat_itervalues(function.calls): for call in compat_itervalues(function.calls):
if call.ratio is not None: if call.ratio is not None:
callee = profile.functions[call.callee_id] callee = profile.functions[call.callee_id]
call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]; call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]
else: else:
assert False assert False
@ -1931,8 +1948,6 @@ class OprofileParser(LineParser):
profile = Profile() profile = Profile()
reverse_call_samples = {}
# populate the profile # populate the profile
profile[SAMPLES] = 0 profile[SAMPLES] = 0
for _callers, _function, _callees in compat_itervalues(self.entries): for _callers, _function, _callees in compat_itervalues(self.entries):
@ -2016,7 +2031,7 @@ class OprofileParser(LineParser):
if entry.symbol.startswith('"') and entry.symbol.endswith('"'): if entry.symbol.startswith('"') and entry.symbol.endswith('"'):
entry.symbol = entry.symbol[1:-1] entry.symbol = entry.symbol[1:-1]
entry.id = ':'.join((entry.application, entry.image, source, entry.symbol)) entry.id = ':'.join((entry.application, entry.image, source, entry.symbol))
entry.self = fields.get('self', None) != None entry.self = fields.get('self', None) is not None
if entry.self: if entry.self:
entry.id += ':self' entry.id += ':self'
if entry.symbol: if entry.symbol:
@ -2036,7 +2051,7 @@ class OprofileParser(LineParser):
def match_separator(self): def match_separator(self):
line = self.lookahead() line = self.lookahead()
return line == '-'*len(line) return line == '-' * len(line)
def match_primary(self): def match_primary(self):
line = self.lookahead() line = self.lookahead()
@ -2066,8 +2081,10 @@ class HProfParser(LineParser):
# read lookahead # read lookahead
self.readline() self.readline()
while not self.lookahead().startswith('------'): self.consume() while not self.lookahead().startswith('------'):
while not self.lookahead().startswith('TRACE '): self.consume() self.consume()
while not self.lookahead().startswith('TRACE '):
self.consume()
self.parse_traces() self.parse_traces()
@ -2084,12 +2101,13 @@ class HProfParser(LineParser):
# build up callgraph # build up callgraph
for id, trace in compat_iteritems(self.traces): for id, trace in compat_iteritems(self.traces):
if not id in self.samples: continue if id not in self.samples:
continue
mtime = self.samples[id][0] mtime = self.samples[id][0]
last = None last = None
for func, file, line in trace: for func, file, line in trace:
if not func in functions: if func not in functions:
function = Function(func, func) function = Function(func, func)
function[SAMPLES] = 0 function[SAMPLES] = 0
profile.add_function(function) profile.add_function(function)
@ -2120,17 +2138,16 @@ class HProfParser(LineParser):
self.parse_trace() self.parse_trace()
def parse_trace(self): def parse_trace(self):
l = self.consume() consume = self.consume()
mo = self.trace_id_re.match(l) mo = self.trace_id_re.match(consume)
tid = mo.group(1) tid = mo.group(1)
last = None
trace = [] trace = []
while self.lookahead().startswith('\t'): while self.lookahead().startswith('\t'):
l = self.consume() consume = self.consume()
match = self.trace_re.search(l) match = self.trace_re.search(consume)
if not match: if not match:
#sys.stderr.write('Invalid line: %s\n' % l) # sys.stderr.write('Invalid line: %s\n' % consume)
break break
else: else:
function_name, file, line = match.groups() function_name, file, line = match.groups()
@ -2280,13 +2297,13 @@ class XPerfParser(Parser):
import csv import csv
reader = csv.reader( reader = csv.reader(
self.stream, self.stream,
delimiter = ',', delimiter=',',
quotechar = None, quotechar=None,
escapechar = None, escapechar=None,
doublequote = False, doublequote=False,
skipinitialspace = True, skipinitialspace=True,
lineterminator = '\r\n', lineterminator='\r\n',
quoting = csv.QUOTE_NONE) quoting=csv.QUOTE_NONE)
header = True header = True
for row in reader: for row in reader:
if header: if header:
@ -2548,7 +2565,6 @@ class AQtimeParser(XmlParser):
return table return table
def parse_data(self): def parse_data(self):
rows = []
attrs = self.element_start('DATA') attrs = self.element_start('DATA')
table_id = int(attrs['TABLE_ID']) table_id = int(attrs['TABLE_ID'])
table_name, field_types, field_names = self.tables[table_id] table_name, field_types, field_names = self.tables[table_id]
@ -2560,7 +2576,7 @@ class AQtimeParser(XmlParser):
return table return table
def parse_row(self, field_types): def parse_row(self, field_types):
row = [None]*len(field_types) row = [None] * len(field_types)
children = [] children = []
self.element_start('ROW') self.element_start('ROW')
while self.token.type == XML_ELEMENT_START: while self.token.type == XML_ELEMENT_START:
@ -2622,16 +2638,16 @@ class AQtimeParser(XmlParser):
function = Function(self.build_id(fields), self.build_name(fields)) function = Function(self.build_id(fields), self.build_name(fields))
function[TIME] = fields['Time'] function[TIME] = fields['Time']
function[TOTAL_TIME] = fields['Time with Children'] function[TOTAL_TIME] = fields['Time with Children']
#function[TIME_RATIO] = fields['% Time']/100.0 # function[TIME_RATIO] = fields['% Time']/100.0
#function[TOTAL_TIME_RATIO] = fields['% with Children']/100.0 # function[TOTAL_TIME_RATIO] = fields['% with Children']/100.0
return function return function
def build_call(self, fields): def build_call(self, fields):
call = Call(self.build_id(fields)) call = Call(self.build_id(fields))
call[TIME] = fields['Time'] call[TIME] = fields['Time']
call[TOTAL_TIME] = fields['Time with Children'] call[TOTAL_TIME] = fields['Time with Children']
#call[TIME_RATIO] = fields['% Time']/100.0 # call[TIME_RATIO] = fields['% Time']/100.0
#call[TOTAL_TIME_RATIO] = fields['% with Children']/100.0 # call[TOTAL_TIME_RATIO] = fields['% with Children']/100.0
return call return call
def build_id(self, fields): def build_id(self, fields):
@ -2694,7 +2710,7 @@ class PstatsParser:
call = Call(callee.id) call = Call(callee.id)
if isinstance(value, tuple): if isinstance(value, tuple):
for i in xrange(0, len(value), 4): for i in xrange(0, len(value), 4):
nc, cc, tt, ct = value[i:i+4] nc, cc, tt, ct = value[i:i + 4]
if CALLS in call: if CALLS in call:
call[CALLS] += cc call[CALLS] += cc
else: else:
@ -2707,11 +2723,11 @@ class PstatsParser:
else: else:
call[CALLS] = value call[CALLS] = value
call[TOTAL_TIME] = ratio(value, nc)*ct call[TOTAL_TIME] = ratio(value, nc) * ct
caller.add_call(call) caller.add_call(call)
#self.stats.print_stats() # self.stats.print_stats()
#self.stats.print_callees() # self.stats.print_callees()
# Compute derived events # Compute derived events
self.profile.validate() self.profile.validate()
@ -2724,18 +2740,18 @@ class PstatsParser:
class Theme: class Theme:
def __init__(self, def __init__(self,
bgcolor = (0.0, 0.0, 1.0), bgcolor=(0.0, 0.0, 1.0),
mincolor = (0.0, 0.0, 0.0), mincolor=(0.0, 0.0, 0.0),
maxcolor = (0.0, 0.0, 1.0), maxcolor=(0.0, 0.0, 1.0),
fontname = "Arial", fontname="Arial",
fontcolor = "white", fontcolor="white",
nodestyle = "filled", nodestyle="filled",
minfontsize = 10.0, minfontsize=10.0,
maxfontsize = 10.0, maxfontsize=10.0,
minpenwidth = 0.5, minpenwidth=0.5,
maxpenwidth = 4.0, maxpenwidth=4.0,
gamma = 2.2, gamma=2.2,
skew = 1.0): skew=1.0):
self.bgcolor = bgcolor self.bgcolor = bgcolor
self.mincolor = mincolor self.mincolor = mincolor
self.maxcolor = maxcolor self.maxcolor = maxcolor
@ -2783,7 +2799,7 @@ class Theme:
return self.fontsize(weight) return self.fontsize(weight)
def edge_penwidth(self, weight): def edge_penwidth(self, weight):
return max(weight*self.maxpenwidth, self.minpenwidth) return max(weight * self.maxpenwidth, self.minpenwidth)
def edge_arrowsize(self, weight): def edge_arrowsize(self, weight):
return 0.5 * math.sqrt(self.edge_penwidth(weight)) return 0.5 * math.sqrt(self.edge_penwidth(weight))
@ -2800,14 +2816,14 @@ class Theme:
if self.skew < 0: if self.skew < 0:
raise ValueError("Skew must be greater than 0") raise ValueError("Skew must be greater than 0")
elif self.skew == 1.0: elif self.skew == 1.0:
h = hmin + weight*(hmax - hmin) h = hmin + weight * (hmax - hmin)
s = smin + weight*(smax - smin) s = smin + weight * (smax - smin)
l = lmin + weight*(lmax - lmin) l = lmin + weight * (lmax - lmin) # noqa
else: else:
base = self.skew base = self.skew
h = hmin + ((hmax-hmin)*(-1.0 + (base ** weight)) / (base - 1.0)) h = hmin + ((hmax - hmin) * (-1.0 + (base ** weight)) / (base - 1.0))
s = smin + ((smax-smin)*(-1.0 + (base ** weight)) / (base - 1.0)) s = smin + ((smax - smin) * (-1.0 + (base ** weight)) / (base - 1.0))
l = lmin + ((lmax-lmin)*(-1.0 + (base ** weight)) / (base - 1.0)) l = lmin + ((lmax - lmin) * (-1.0 + (base ** weight)) / (base - 1.0)) # noqa
return self.hsl_to_rgb(h, s, l) return self.hsl_to_rgb(h, s, l)
@ -2820,16 +2836,16 @@ class Theme:
h = h % 1.0 h = h % 1.0
s = min(max(s, 0.0), 1.0) s = min(max(s, 0.0), 1.0)
l = min(max(l, 0.0), 1.0) l = min(max(l, 0.0), 1.0) # noqa
if l <= 0.5: if l <= 0.5: # noqa
m2 = l*(s + 1.0) m2 = l * (s + 1.0)
else: else:
m2 = l + s - l*s m2 = l + s - l * s
m1 = l*2.0 - m2 m1 = l * 2.0 - m2
r = self._hue_to_rgb(m1, m2, h + 1.0/3.0) r = self._hue_to_rgb(m1, m2, h + 1.0 / 3.0)
g = self._hue_to_rgb(m1, m2, h) g = self._hue_to_rgb(m1, m2, h)
b = self._hue_to_rgb(m1, m2, h - 1.0/3.0) b = self._hue_to_rgb(m1, m2, h - 1.0 / 3.0)
# Apply gamma correction # Apply gamma correction
r **= self.gamma r **= self.gamma
@ -2843,50 +2859,50 @@ class Theme:
h += 1.0 h += 1.0
elif h > 1.0: elif h > 1.0:
h -= 1.0 h -= 1.0
if h*6 < 1.0: if h * 6 < 1.0:
return m1 + (m2 - m1)*h*6.0 return m1 + (m2 - m1) * h * 6.0
elif h*2 < 1.0: elif h * 2 < 1.0:
return m2 return m2
elif h*3 < 2.0: elif h * 3 < 2.0:
return m1 + (m2 - m1)*(2.0/3.0 - h)*6.0 return m1 + (m2 - m1) * (2.0 / 3.0 - h) * 6.0
else: else:
return m1 return m1
TEMPERATURE_COLORMAP = Theme( TEMPERATURE_COLORMAP = Theme(
mincolor = (2.0/3.0, 0.80, 0.25), # dark blue mincolor=(2.0 / 3.0, 0.80, 0.25), # dark blue
maxcolor = (0.0, 1.0, 0.5), # satured red maxcolor=(0.0, 1.0, 0.5), # satured red
gamma = 1.0 gamma=1.0
) )
PINK_COLORMAP = Theme( PINK_COLORMAP = Theme(
mincolor = (0.0, 1.0, 0.90), # pink mincolor=(0.0, 1.0, 0.90), # pink
maxcolor = (0.0, 1.0, 0.5), # satured red maxcolor=(0.0, 1.0, 0.5), # satured red
) )
GRAY_COLORMAP = Theme( GRAY_COLORMAP = Theme(
mincolor = (0.0, 0.0, 0.85), # light gray mincolor=(0.0, 0.0, 0.85), # light gray
maxcolor = (0.0, 0.0, 0.0), # black maxcolor=(0.0, 0.0, 0.0), # black
) )
BW_COLORMAP = Theme( BW_COLORMAP = Theme(
minfontsize = 8.0, minfontsize=8.0,
maxfontsize = 24.0, maxfontsize=24.0,
mincolor = (0.0, 0.0, 0.0), # black mincolor=(0.0, 0.0, 0.0), # black
maxcolor = (0.0, 0.0, 0.0), # black maxcolor=(0.0, 0.0, 0.0), # black
minpenwidth = 0.1, minpenwidth=0.1,
maxpenwidth = 8.0, maxpenwidth=8.0,
) )
PRINT_COLORMAP = Theme( PRINT_COLORMAP = Theme(
minfontsize = 18.0, minfontsize=18.0,
maxfontsize = 30.0, maxfontsize=30.0,
fontcolor = "black", fontcolor="black",
nodestyle = "solid", nodestyle="solid",
mincolor = (0.0, 0.0, 0.0), # black mincolor=(0.0, 0.0, 0.0), # black
maxcolor = (0.0, 0.0, 0.0), # black maxcolor=(0.0, 0.0, 0.0), # black
minpenwidth = 0.1, minpenwidth=0.1,
maxpenwidth = 8.0, maxpenwidth=8.0,
) )
@ -2908,9 +2924,9 @@ class DotWriter:
"""Split the function name on multiple lines.""" """Split the function name on multiple lines."""
if len(name) > 32: if len(name) > 32:
ratio = 2.0/3.0 ratio = 2.0 / 3.0
height = max(int(len(name)/(1.0 - ratio) + 0.5), 1) height = max(int(len(name) / (1.0 - ratio) + 0.5), 1)
width = max(len(name)/height, 32) width = max(len(name) / height, 32)
# TODO: break lines in symbols # TODO: break lines in symbols
name = textwrap.fill(name, width, break_long_words=False) name = textwrap.fill(name, width, break_long_words=False)
@ -2964,10 +2980,10 @@ class DotWriter:
label = '\n'.join(labels) label = '\n'.join(labels)
self.node(function.id, self.node(function.id,
label = label, label=label,
color = self.color(theme.node_bgcolor(weight)), color=self.color(theme.node_bgcolor(weight)),
fontcolor = self.color(theme.node_fgcolor(weight)), fontcolor=self.color(theme.node_fgcolor(weight)),
fontsize = "%.2f" % theme.node_fontsize(weight), fontsize="%.2f" % theme.node_fontsize(weight),
) )
for call in compat_itervalues(function.calls): for call in compat_itervalues(function.calls):
@ -2989,13 +3005,13 @@ class DotWriter:
label = '\n'.join(labels) label = '\n'.join(labels)
self.edge(function.id, call.callee_id, self.edge(function.id, call.callee_id,
label = label, label=label,
color = self.color(theme.edge_color(weight)), color=self.color(theme.edge_color(weight)),
fontcolor = self.color(theme.edge_color(weight)), fontcolor=self.color(theme.edge_color(weight)),
fontsize = "%.2f" % theme.edge_fontsize(weight), fontsize="%.2f" % theme.edge_fontsize(weight),
penwidth = "%.2f" % theme.edge_penwidth(weight), penwidth="%.2f" % theme.edge_penwidth(weight),
labeldistance = "%.2f" % theme.edge_penwidth(weight), labeldistance="%.2f" % theme.edge_penwidth(weight),
arrowsize = "%.2f" % theme.edge_arrowsize(weight), arrowsize="%.2f" % theme.edge_arrowsize(weight),
) )
self.end_graph() self.end_graph()
@ -3061,7 +3077,7 @@ class DotWriter:
return 0 return 0
if f >= 1.0: if f >= 1.0:
return 255 return 255
return int(255.0*f + 0.5) return int(255.0 * f + 0.5)
return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)]) return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)])
@ -3115,8 +3131,7 @@ class Main:
global totalMethod global totalMethod
formatNames = list(self.formats.keys()) formatNames = sorted(self.formats.keys())
formatNames.sort()
optparser = optparse.OptionParser( optparser = optparse.OptionParser(
usage="\n\t%prog [options] [file] ...") usage="\n\t%prog [options] [file] ...")
@ -3139,19 +3154,27 @@ class Main:
help="profile format: %s [default: %%default]" % self.naturalJoin(formatNames)) help="profile format: %s [default: %%default]" % self.naturalJoin(formatNames))
optparser.add_option( optparser.add_option(
'--total', '--total',
type="choice", choices=('callratios', 'callstacks'), type="choice",
dest="totalMethod", default=totalMethod, choices=(
help="preferred method of calculating total time: callratios or callstacks (currently affects only perf format) [default: %default]") 'callratios',
'callstacks'),
dest="totalMethod",
default=totalMethod,
help=("preferred method of calculating total time: callratios or callstacks"
" (currently affects only perf format) [default: %default]"))
optparser.add_option( optparser.add_option(
'-c', '--colormap', '-c', '--colormap',
type="choice", choices=('color', 'pink', 'gray', 'bw', 'print'), type="choice", choices=('color', 'pink', 'gray', 'bw', 'print'),
dest="theme", default="color", dest="theme", default="color",
help="color map: color, pink, gray, bw, or print [default: %default]") help="color map: color, pink, gray, bw, or print [default: %default]")
optparser.add_option( optparser.add_option(
'-s', '--strip', '-s',
'--strip',
action="store_true", action="store_true",
dest="strip", default=False, dest="strip",
help="strip function parameters, template parameters, and const modifiers from demangled C++ function names") default=False,
help=("strip function parameters, template parameters, and const modifiers from"
" demangled C++ function names"))
optparser.add_option( optparser.add_option(
'-w', '--wrap', '-w', '--wrap',
action="store_true", action="store_true",
@ -3176,8 +3199,11 @@ class Main:
# add a new option to control skew of the colorization curve # add a new option to control skew of the colorization curve
optparser.add_option( optparser.add_option(
'--skew', '--skew',
type="float", dest="theme_skew", default=1.0, type="float",
help="skew the colorization curve. Values < 1.0 give more variety to lower percentages. Values > 1.0 give less variety to lower percentages") dest="theme_skew",
default=1.0,
help=("skew the colorization curve. Values < 1.0 give more variety to lower"
" percentages. Values > 1.0 give less variety to lower percentages"))
(self.options, self.args) = optparser.parse_args(sys.argv[1:]) (self.options, self.args) = optparser.parse_args(sys.argv[1:])
if len(self.args) > 1 and self.options.format != 'pstats': if len(self.args) > 1 and self.options.format != 'pstats':
@ -3234,18 +3260,24 @@ class Main:
dot.show_function_events.append(SAMPLES) dot.show_function_events.append(SAMPLES)
profile = self.profile profile = self.profile
profile.prune(self.options.node_thres/100.0, self.options.edge_thres/100.0) profile.prune(self.options.node_thres / 100.0, self.options.edge_thres / 100.0)
if self.options.root: if self.options.root:
rootId = profile.getFunctionId(self.options.root) rootId = profile.getFunctionId(self.options.root)
if not rootId: if not rootId:
sys.stderr.write('root node ' + self.options.root + ' not found (might already be pruned : try -e0 -n0 flags)\n') sys.stderr.write(
'root node ' +
self.options.root +
' not found (might already be pruned : try -e0 -n0 flags)\n')
sys.exit(1) sys.exit(1)
profile.prune_root(rootId) profile.prune_root(rootId)
if self.options.leaf: if self.options.leaf:
leafId = profile.getFunctionId(self.options.leaf) leafId = profile.getFunctionId(self.options.leaf)
if not leafId: if not leafId:
sys.stderr.write('leaf node ' + self.options.leaf + ' not found (maybe already pruned : try -e0 -n0 flags)\n') sys.stderr.write(
'leaf node ' +
self.options.leaf +
' not found (maybe already pruned : try -e0 -n0 flags)\n')
sys.exit(1) sys.exit(1)
profile.prune_leaf(leafId) profile.prune_leaf(leafId)

34
tools/parse_bandwidth_log.py
View File

@ -1,24 +1,26 @@
#!/usr/bin/env python #!/usr/bin/env python
import os, sys, time from __future__ import print_function
import os
import sys
keys = [['upload rate', 'x1y1', 6], ['history entries', 'x1y2', 10], ['queue', 'x1y2', 4]] keys = [['upload rate', 'x1y1', 6], ['history entries', 'x1y2', 10], ['queue', 'x1y2', 4]]
out = open('bandwidth.gnuplot', 'wb') out = open('bandwidth.gnuplot', 'wb')
print >>out, "set term png size 1200,700" print("set term png size 1200,700", file=out)
print >>out, 'set output "bandwidth_manager.png"' print('set output "bandwidth_manager.png"', file=out)
print >>out, 'set xrange [0:*]' print('set xrange [0:*]', file=out)
print >>out, 'set xlabel "time (ms)"' print('set xlabel "time (ms)"', file=out)
print >>out, 'set ylabel "Rate (B/s)"' print('set ylabel "Rate (B/s)"', file=out)
print >>out, 'set ytics 10000' print('set ytics 10000', file=out)
print >>out, 'set y2label "number"' print('set y2label "number"', file=out)
print >>out, 'set y2range [0:*]' print('set y2range [0:*]', file=out)
#print >>out, "set style data lines" # print("set style data lines", file=out)
print >>out, "set key box" print("set key box", file=out)
print >>out, 'plot', print('plot', end=' ', file=out)
for k, a, c in keys: for k, a, c in keys:
print >>out, ' "%s" using 1:%d title "%s" axes %s with steps,' % (sys.argv[1], c, k, a), print(' "%s" using 1:%d title "%s" axes %s with steps,' % (sys.argv[1], c, k, a), end=' ', file=out)
print >>out, 'x=0' print('x=0', file=out)
out.close() out.close()
os.system('gnuplot bandwidth.gnuplot'); os.system('gnuplot bandwidth.gnuplot')

76
tools/parse_buffer_log.py
View File

@ -3,14 +3,18 @@
# subject to the Boost Software License, Version 1.0. (See accompanying # subject to the Boost Software License, Version 1.0. (See accompanying
# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
import os, sys, time from __future__ import print_function
import os
import sys
lines = open(sys.argv[1], 'rb').readlines() lines = open(sys.argv[1], 'rb').readlines()
#keys = ['send_buffer_utilization'] # keys = ['send_buffer_utilization']
keys = ['send_buffer_size', 'used_send_buffer', 'protocol_buffer'] keys = ['send_buffer_size', 'used_send_buffer', 'protocol_buffer']
#keys = ['send_buffer_alloc', 'send_buffer', 'allocate_buffer_alloc', 'allocate_buffer', 'protocol_buffer'] # keys = ['send_buffer_alloc', 'send_buffer', 'allocate_buffer_alloc', 'allocate_buffer', 'protocol_buffer']
#keys = ['send_buffer_alloc', 'send_buffer', 'allocate_buffer_alloc', 'allocate_buffer', 'protocol_buffer', 'append_send_buffer'] # keys = ['send_buffer_alloc', 'send_buffer', 'allocate_buffer_alloc', 'allocate_buffer', 'protocol_buffer',
# 'append_send_buffer']
average = ['send_buffer_utilization', 'send_buffer_size', 'used_send_buffer'] average = ['send_buffer_utilization', 'send_buffer_size', 'used_send_buffer']
average_interval = 120000 average_interval = 120000
@ -18,7 +22,7 @@ render = 'lines'
time_limit = -1 time_limit = -1
if len(sys.argv) > 2: if len(sys.argv) > 2:
time_limit = long(sys.argv[2]) time_limit = int(sys.argv[2])
# logfile format: # logfile format:
@ -38,53 +42,57 @@ for k in keys:
eval_average = True eval_average = True
peak_out = open(k + '_peak.dat', 'wb') peak_out = open(k + '_peak.dat', 'wb')
for l in lines: for line in lines:
l = l.split(' ') line = line.split(' ')
if len(l) != 3: if len(line) != 3:
print l print(line)
continue continue
try: try:
if l[1] == k + ':': if line[1] == k + ':':
if time_limit != -1 and long(l[0]) > time_limit: break if time_limit != -1 and int(line[0]) > time_limit:
time = l[0] break
value = l[2] time = line[0]
value = line[2]
if eval_average: if eval_average:
while long(time) > last_sample + average_interval: while int(time) > last_sample + average_interval:
last_sample = last_sample + average_interval last_sample = last_sample + average_interval
if average_samples < 1: average_samples = 1 if average_samples < 1:
print >>out, '%d %f' % (last_sample, average_accumulator / average_samples) average_samples = 1
print >>peak_out, '%d %f' % (last_sample, peak) print('%d %f' % (last_sample, average_accumulator / average_samples), file=out)
print('%d %f' % (last_sample, peak), file=peak_out)
average_accumulator = 0 average_accumulator = 0
average_samples = 0 average_samples = 0
peak = 0 peak = 0
average_accumulator = average_accumulator + float(value) average_accumulator = average_accumulator + float(value)
average_samples = average_samples + 1 average_samples = average_samples + 1
if float(value) > peak: peak = float(value) if float(value) > peak:
peak = float(value)
else: else:
print >>out, time + ' ' + value, print(time + ' ' + value, end=' ', file=out)
except: except BaseException:
print l print(line)
out.close() out.close()
peak_out.close() peak_out.close()
out = open('send_buffer.gnuplot', 'wb') out = open('send_buffer.gnuplot', 'wb')
print >>out, "set term png size 1200,700" print("set term png size 1200,700", file=out)
print >>out, 'set output "send_buffer.png"' print('set output "send_buffer.png"', file=out)
print >>out, 'set xrange [0:*]' print('set xrange [0:*]', file=out)
print >>out, 'set xlabel "time (ms)"' print('set xlabel "time (ms)"', file=out)
print >>out, 'set ylabel "bytes (B)"' print('set ylabel "bytes (B)"', file=out)
print >>out, "set style data lines" print("set style data lines", file=out)
print >>out, "set key box" print("set key box", file=out)
print >>out, 'plot', print('plot', end=' ', file=out)
for k in keys: for k in keys:
if k in average: if k in average:
print >>out, ' "%s.dat" using 1:2 title "%s %d seconds average" with %s,' % (k, k, average_interval / 1000., render), print(' "%s.dat" using 1:2 title "%s %d seconds average" with %s,' %
print >>out, ' "%s_peak.dat" using 1:2 title "%s %d seconds peak" with %s,' % (k, k, average_interval / 1000., render), (k, k, average_interval / 1000., render), end=' ', file=out)
print(' "%s_peak.dat" using 1:2 title "%s %d seconds peak" with %s,' %
(k, k, average_interval / 1000., render), end=' ', file=out)
else: else:
print >>out, ' "%s.dat" using 1:2 title "%s" with %s,' % (k, k, render), print(' "%s.dat" using 1:2 title "%s" with %s,' % (k, k, render), end=' ', file=out)
print >>out, 'x=0' print('x=0', file=out)
out.close() out.close()
os.system('gnuplot send_buffer.gnuplot') os.system('gnuplot send_buffer.gnuplot')

172
tools/parse_dht_log.py
View File

@ -1,8 +1,8 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function
import sys import sys
import os import os
import time
import calendar
import pprint import pprint
pp = pprint.PrettyPrinter(indent=4) pp = pprint.PrettyPrinter(indent=4)
@ -13,11 +13,11 @@ f = open(sys.argv[1])
announce_histogram = {} announce_histogram = {}
#TODO: make this histogram into a CDF # TODO: make this histogram into a CDF
node_uptime_histogram = {} node_uptime_histogram = {}
counter = 0; counter = 0
# maps search_id to a list of events. Each event is a dict containing: # maps search_id to a list of events. Each event is a dict containing:
# t: timestamp # t: timestamp
@ -32,11 +32,13 @@ outstanding_searches = {}
# list of completed searches # list of completed searches
searches = [] searches = []
def convert_timestamp(t): def convert_timestamp(t):
parts = t.split('.') parts = t.split('.')
hms = parts[0].split(':') hms = parts[0].split(':')
return (int(hms[0]) * 3600 + int(hms[1]) * 60 + int(hms[2])) * 1000 + int(parts[1]) return (int(hms[0]) * 3600 + int(hms[1]) * 60 + int(hms[2])) * 1000 + int(parts[1])
last_incoming = '' last_incoming = ''
our_node_id = '' our_node_id = ''
@ -47,19 +49,19 @@ client_histogram = {}
for line in f: for line in f:
counter += 1 counter += 1
# if counter % 1000 == 0: # if counter % 1000 == 0:
# print '\r%d' % counter, # print '\r%d' % counter,
try: try:
l = line.split(' ') ls = line.split(' ')
if 'starting DHT tracker with node id:' in line: if 'starting DHT tracker with node id:' in line:
our_node_id = l[l.index('id:') + 1].strip() our_node_id = ls[ls.index('id:') + 1].strip()
try: try:
if len(l) > 4 and l[2] == '<==' and l[1] == '[dht_tracker]': if len(ls) > 4 and ls[2] == '<==' and ls[1] == '[dht_tracker]':
ip = l[3].split(':')[0] ip = ls[3].split(':')[0]
if ip not in unique_ips: if ip not in unique_ips:
unique_ips.add(ip) unique_ips.add(ip)
json_blob = line.split(l[3])[1] json_blob = line.split(ls[3])[1]
version = json_blob.split("'v': '")[1].split("'")[0] version = json_blob.split("'v': '")[1].split("'")[0]
if len(version) == 4: if len(version) == 4:
v = '%s-%d' % (version[0:2], (ord(version[2]) << 8) + ord(version[3])) v = '%s-%d' % (version[0:2], (ord(version[2]) << 8) + ord(version[3]))
@ -68,7 +70,7 @@ for line in f:
else: else:
v = 'unknown' v = 'unknown'
if not v in client_version_histogram: if v not in client_version_histogram:
client_version_histogram[v] = 1 client_version_histogram[v] = 1
else: else:
client_version_histogram[v] += 1 client_version_histogram[v] += 1
@ -77,63 +79,69 @@ for line in f:
client_histogram[v[0:2]] = 1 client_histogram[v[0:2]] = 1
else: else:
client_histogram[v[0:2]] += 1 client_histogram[v[0:2]] += 1
except: pass except BaseException:
pass
if 'announce-distance:' in line: if 'announce-distance:' in line:
idx = l.index('announce-distance:') idx = ls.index('announce-distance:')
d = int(l[idx+1].strip()) d = int(ls[idx + 1].strip())
if not d in announce_histogram: announce_histogram[d] = 0 if d not in announce_histogram:
announce_histogram[d] = 0
announce_histogram[d] += 1 announce_histogram[d] += 1
if 'NODE FAILED' in line: if 'NODE FAILED' in line:
idx = l.index('fails:') idx = ls.index('fails:')
if int(l[idx+1].strip()) != 1: continue; if int(ls[idx + 1].strip()) != 1:
idx = l.index('up-time:') continue
d = int(l[idx+1].strip()) idx = ls.index('up-time:')
d = int(ls[idx + 1].strip())
# quantize # quantize
d = d - (d % up_time_quanta) d = d - (d % up_time_quanta)
if not d in node_uptime_histogram: node_uptime_histogram[d] = 0 if d not in node_uptime_histogram:
node_uptime_histogram[d] = 0
node_uptime_histogram[d] += 1 node_uptime_histogram[d] += 1
search_id = l[2] search_id = ls[2]
ts = l[0] ts = ls[0]
event = l[3] event = ls[3]
if event == 'RESPONSE': if event == 'RESPONSE':
outstanding = int(l[l.index('invoke-count:')+1]) outstanding = int(ls[ls.index('invoke-count:') + 1])
nid = l[l.index('id:')+1] distance = int(ls[ls.index('distance:') + 1])
addr = l[l.index('addr:')+1] nid = ls[ls.index('id:') + 1]
addr = ls[ls.index('addr:') + 1]
last_response = addr last_response = addr
outstanding_searches[search_id].append({ 't': ts, 'd': distance, outstanding_searches[search_id].append({'t': ts, 'd': distance,
'o': outstanding + 1, 'a':addr, 'e': event,'i':nid, 's':source}) 'o': outstanding + 1, 'a': addr, 'e': event, 'i': nid})
elif event == 'NEW': elif event == 'NEW':
nid = l[l.index('target:')+1] nid = ls[ls.index('target:') + 1]
outstanding_searches[search_id] = [{ 't': ts, 'd': 0, 'o': 0, \ outstanding_searches[search_id] = [{'t': ts, 'd': 0, 'o': 0,
'e': event, 'abstime': ts, 'i': nid}] 'e': event, 'abstime': ts, 'i': nid}]
last_response = '' last_response = ''
elif event == 'INVOKE' or event == 'ADD' or event == '1ST_TIMEOUT' or \ elif event == 'INVOKE' or event == 'ADD' or event == '1ST_TIMEOUT' or \
event == 'TIMEOUT' or event == 'PEERS': event == 'TIMEOUT' or event == 'PEERS':
if not search_id in outstanding_searches: if search_id not in outstanding_searches:
print 'orphaned event: %s' % line print('orphaned event: %s' % line)
else: else:
outstanding = int(l[l.index('invoke-count:')+1]) outstanding = int(ls[ls.index('invoke-count:') + 1])
distance = int(l[l.index('distance:')+1]) distance = int(ls[ls.index('distance:') + 1])
nid = l[l.index('id:')+1] nid = ls[ls.index('id:') + 1]
addr = l[l.index('addr:')+1] addr = ls[ls.index('addr:') + 1]
source = '' source = ''
if event == 'ADD': if event == 'ADD':
if last_response == '': continue if last_response == '':
continue
source = last_response source = last_response
outstanding_searches[search_id].append({ 't': ts, 'd': distance, outstanding_searches[search_id].append(
'o': outstanding + 1, 'a':addr, 'e': event,'i':nid, 's':source}) {'t': ts, 'd': distance, 'o': outstanding + 1, 'a': addr, 'e': event, 'i': nid, 's': source})
elif event == 'ABORTED': elif event == 'ABORTED':
outstanding_searches[search_id].append({ 't': ts, 'e': event}) outstanding_searches[search_id].append({'t': ts, 'e': event})
elif event == 'COMPLETED': elif event == 'COMPLETED':
distance = int(l[l.index('distance:')+1]) distance = int(ls[ls.index('distance:') + 1])
lookup_type = l[l.index('type:')+1].strip() lookup_type = ls[ls.index('type:') + 1].strip()
outstanding_searches[search_id].append({ 't': ts, 'd': distance, outstanding_searches[search_id].append({'t': ts, 'd': distance,
'o': 0, 'e': event,'i':''}) 'o': 0, 'e': event, 'i': ''})
outstanding_searches[search_id][0]['type'] = lookup_type outstanding_searches[search_id][0]['type'] = lookup_type
@ -143,14 +151,14 @@ for line in f:
start_time = convert_timestamp(s[0]['t']) start_time = convert_timestamp(s[0]['t'])
for i in range(len(s)): for i in range(len(s)):
s[i]['t'] = convert_timestamp(s[i]['t']) - start_time s[i]['t'] = convert_timestamp(s[i]['t']) - start_time
except: except BaseException:
pass pass
searches.append(s) searches.append(s)
del outstanding_searches[search_id] del outstanding_searches[search_id]
except Exception, e: except Exception as e:
print e print(e)
print line.split(' ') print(line.split(' '))
lookup_times_min = [] lookup_times_min = []
lookup_times_max = [] lookup_times_max = []
@ -163,20 +171,23 @@ for i in range(0, 15):
for s in searches: for s in searches:
for i in s: for i in s:
if not 'last_dist' in i: if 'last_dist' not in i:
i['last_dist'] = -1 i['last_dist'] = -1
cur_dist = 160 - i['d'] cur_dist = 160 - i['d']
last_dist = i['last_dist'] last_dist = i['last_dist']
if cur_dist > last_dist: if cur_dist > last_dist:
for j in range(last_dist + 1, cur_dist + 1): for j in range(last_dist + 1, cur_dist + 1):
if j >= len(lookup_distance): break if j >= len(lookup_distance):
break
lookup_distance[j].append(i['t']) lookup_distance[j].append(i['t'])
i['last_dist'] = cur_dist i['last_dist'] = cur_dist
if i['e'] != 'PEERS': continue if i['e'] != 'PEERS':
continue
lookup_times_min.append(i['t']) lookup_times_min.append(i['t'])
break break
for i in reversed(s): for i in reversed(s):
if i['e'] != 'PEERS': continue if i['e'] != 'PEERS':
continue
lookup_times_max.append(i['t']) lookup_times_max.append(i['t'])
break break
@ -187,7 +198,7 @@ out = open('dht_lookup_times_cdf.txt', 'w+')
counter = 0 counter = 0
for i in range(len(lookup_times_min)): for i in range(len(lookup_times_min)):
counter += 1 counter += 1
print >>out, '%d\t%d\t%f' % (lookup_times_min[i], lookup_times_max[i], counter / float(len(lookup_times_min))) print('%d\t%d\t%f' % (lookup_times_min[i], lookup_times_max[i], counter / float(len(lookup_times_min))), file=out)
out.close() out.close()
for i in lookup_distance: for i in lookup_distance:
@ -200,60 +211,60 @@ for i in lookup_distance:
counter = 0 counter = 0
for j in i: for j in i:
counter += 1 counter += 1
print >>out, '%d\t%f' % (j, counter / float(len(i))) print('%d\t%f' % (j, counter / float(len(i))), file=out)
out.close() out.close()
out = open('dht_lookups.txt', 'w+') out = open('dht_lookups.txt', 'w+')
for s in searches: for s in searches:
for i in s: for i in s:
if i['e'] == 'INVOKE': if i['e'] == 'INVOKE':
print >>out, ' ->', i['t'], 160 - i['d'], i['i'], i['a'] print(' ->', i['t'], 160 - i['d'], i['i'], i['a'], file=out)
elif i['e'] == '1ST_TIMEOUT': elif i['e'] == '1ST_TIMEOUT':
print >>out, ' x ', i['t'], 160 - i['d'], i['i'], i['a'] print(' x ', i['t'], 160 - i['d'], i['i'], i['a'], file=out)
elif i['e'] == 'TIMEOUT': elif i['e'] == 'TIMEOUT':
print >>out, ' X ', i['t'], 160 - i['d'], i['i'], i['a'] print(' X ', i['t'], 160 - i['d'], i['i'], i['a'], file=out)
elif i['e'] == 'ADD': elif i['e'] == 'ADD':
print >>out, ' + ', i['t'], 160 - i['d'], i['i'], i['a'], i['s'] print(' + ', i['t'], 160 - i['d'], i['i'], i['a'], i['s'], file=out)
elif i['e'] == 'RESPONSE': elif i['e'] == 'RESPONSE':
print >>out, ' <-', i['t'], 160 - i['d'], i['i'], i['a'] print(' <-', i['t'], 160 - i['d'], i['i'], i['a'], file=out)
elif i['e'] == 'PEERS': elif i['e'] == 'PEERS':
print >>out, ' <-', i['t'], 160 - i['d'], i['i'], i['a'] print(' <-', i['t'], 160 - i['d'], i['i'], i['a'], file=out)
elif i['e'] == 'ABORTED': elif i['e'] == 'ABORTED':
print >>out, 'abort' print('abort', file=out)
elif i['e'] == 'COMPLETED': elif i['e'] == 'COMPLETED':
print >>out, '***', i['t'], 160 - i['d'], '\n' print('***', i['t'], 160 - i['d'], '\n', file=out)
elif i['e'] == 'NEW': elif i['e'] == 'NEW':
print >>out, '===', i['abstime'], i['type'], '===' print('===', i['abstime'], i['type'], '===', file=out)
print >>out, '<> ', 0, our_node_id, i['i'] print('<> ', 0, our_node_id, i['i'], file=out)
out.close() out.close()
out = open('dht_announce_distribution.dat', 'w+') out = open('dht_announce_distribution.dat', 'w+')
print 'announce distribution items: %d' % len(announce_histogram) print('announce distribution items: %d' % len(announce_histogram))
for k,v in announce_histogram.items(): for k, v in list(announce_histogram.items()):
print >>out, '%d %d' % (k, v) print('%d %d' % (k, v), file=out)
print '%d %d' % (k, v) print('%d %d' % (k, v))
out.close() out.close()
out = open('dht_node_uptime_cdf.txt', 'w+') out = open('dht_node_uptime_cdf.txt', 'w+')
s = 0 s = 0
total_uptime_nodes = 0 total_uptime_nodes = 0
for k,v in node_uptime_histogram.items(): for k, v in list(node_uptime_histogram.items()):
total_uptime_nodes += v total_uptime_nodes += v
for k,v in sorted(node_uptime_histogram.items()): for k, v in sorted(node_uptime_histogram.items()):
s += v s += v
print >>out, '%f %f' % (k / float(60), s / float(total_uptime_nodes)) print('%f %f' % (k / float(60), s / float(total_uptime_nodes)), file=out)
print '%f %f' % (k / float(60), s / float(total_uptime_nodes)) print('%f %f' % (k / float(60), s / float(total_uptime_nodes)))
out.close() out.close()
print 'clients by version' print('clients by version')
client_version_histogram = sorted(client_version_histogram.items(), key=lambda x: x[1], reverse=True) client_version_histogram = sorted(list(client_version_histogram.items()), key=lambda x: x[1], reverse=True)
pp.pprint(client_version_histogram) pp.pprint(client_version_histogram)
print 'clients' print('clients')
client_histogram = sorted(client_histogram.items(), key=lambda x: x[1], reverse=True) client_histogram = sorted(list(client_histogram.items()), key=lambda x: x[1], reverse=True)
pp.pprint(client_histogram) pp.pprint(client_histogram)
out = open('dht.gnuplot', 'w+') out = open('dht.gnuplot', 'w+')
@ -318,12 +329,11 @@ plot ''')
dist = 0 dist = 0
for i in lookup_distance: for i in lookup_distance:
if dist > 0: out.write(', ') if dist > 0:
out.write(', ')
out.write('"dht_lookup_distance_%d.txt" using 1:2 title "%d" with lines' % (dist, dist)) out.write('"dht_lookup_distance_%d.txt" using 1:2 title "%d" with lines' % (dist, dist))
dist += 1 dist += 1
out.close() out.close()
os.system('gnuplot dht.gnuplot'); os.system('gnuplot dht.gnuplot')

13
tools/parse_dht_rtt.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function
import sys import sys
import os import os
@ -15,7 +16,8 @@ for i in range(0, max_rtt, quantize):
for line in f: for line in f:
time = int(line.split('\t')[1]) time = int(line.split('\t')[1])
if (time < 0 or time > max_rtt - quantize): continue if (time < 0 or time > max_rtt - quantize):
continue
num_messages += 1 num_messages += 1
time /= quantize time /= quantize
time *= quantize time *= quantize
@ -23,9 +25,9 @@ for line in f:
f = open('round_trip_distribution.log', 'w+') f = open('round_trip_distribution.log', 'w+')
for k, v in distribution.items(): for k, v in list(distribution.items()):
print >>f, '%f %d' % ((k + (quantize / 2)) / 1000.0, v) print('%f %d' % ((k + (quantize / 2)) / 1000.0, v), file=f)
f.close(); f.close()
f = open('round_trip_distribution.gnuplot', 'w+') f = open('round_trip_distribution.gnuplot', 'w+')
@ -48,5 +50,4 @@ replot
''' % (float(quantize) / 1000.0)) ''' % (float(quantize) / 1000.0))
f.close() f.close()
os.system('gnuplot round_trip_distribution.gnuplot'); os.system('gnuplot round_trip_distribution.gnuplot')

21
tools/parse_dht_stats.py
View File

@ -4,19 +4,20 @@ import os
gnuplot_scripts = [] gnuplot_scripts = []
def gen_stats_gnuplot(name, y, lines): def gen_stats_gnuplot(name, y, lines):
global gnuplot_scripts global gnuplot_scripts
stat = open(sys.argv[1]) stat = open(sys.argv[1])
line = stat.readline() line = stat.readline()
while not 'minute:' in line: while 'minute:' not in line:
line = stat.readline() line = stat.readline()
names = line.strip().split(':') names = line.strip().split(':')
counter = 1 counter = 1
for i in names: for i in names:
print '%d: %s' % (counter, i) print('%d: %s' % (counter, i))
counter += 1 counter += 1
out = open('%s.gnuplot' % name, 'w+') out = open('%s.gnuplot' % name, 'w+')
@ -32,7 +33,7 @@ plot ''' % (name, name.strip('_'), y))
if not first: if not first:
out.write(', \\\n') out.write(', \\\n')
first = False first = False
out.write('"%s" using 1:%d title "%s" with lines' % (sys.argv[1], names.index(i)+1, i)) out.write('"%s" using 1:%d title "%s" with lines' % (sys.argv[1], names.index(i) + 1, i))
out.write('\n') out.write('\n')
out.write('''set terminal postscript out.write('''set terminal postscript
@ -42,12 +43,20 @@ replot
out.close() out.close()
gnuplot_scripts += [name] gnuplot_scripts += [name]
gen_stats_gnuplot('dht_routing_table_size', 'nodes', ['active nodes','passive nodes', 'confirmed nodes'])
gen_stats_gnuplot('dht_routing_table_size', 'nodes', ['active nodes', 'passive nodes', 'confirmed nodes'])
gen_stats_gnuplot('dht_tracker_table_size', '', ['num torrents', 'num peers']) gen_stats_gnuplot('dht_tracker_table_size', '', ['num torrents', 'num peers'])
gen_stats_gnuplot('dht_announces', 'messages per minute', ['announces per min', 'failed announces per min']) gen_stats_gnuplot('dht_announces', 'messages per minute', ['announces per min', 'failed announces per min'])
gen_stats_gnuplot('dht_clients', 'messages per minute', ['total msgs per min', 'az msgs per min', 'ut msgs per min', 'lt msgs per min', 'mp msgs per min', 'gr msgs per min']) gen_stats_gnuplot('dht_clients',
'messages per minute',
['total msgs per min',
'az msgs per min',
'ut msgs per min',
'lt msgs per min',
'mp msgs per min',
'gr msgs per min'])
gen_stats_gnuplot('dht_rate', 'bytes per second', ['bytes in per sec', 'bytes out per sec']) gen_stats_gnuplot('dht_rate', 'bytes per second', ['bytes in per sec', 'bytes out per sec'])
gen_stats_gnuplot('dht_errors', 'messages per minute', ['error replies sent', 'error queries recvd']) gen_stats_gnuplot('dht_errors', 'messages per minute', ['error replies sent', 'error queries recvd'])
for i in gnuplot_scripts: for i in gnuplot_scripts:
os.system('gnuplot %s.gnuplot' % i); os.system('gnuplot %s.gnuplot' % i)

51
tools/parse_disk_buffer_log.py
View File

@ -1,6 +1,8 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function
import os, sys, time import os
import sys
lines = open(sys.argv[1], 'rb').readlines() lines = open(sys.argv[1], 'rb').readlines()
@ -37,56 +39,59 @@ last_t = 0
for l in lines: for l in lines:
try: try:
t = int(l[0:l.find(' ')]) t = int(l[0:l.find(' ')])
c = l[l.find(' ')+1:l.find(':')] c = l[l.find(' ') + 1:l.find(':')]
n = int(l[l.find(':')+1:-1]) n = int(l[l.find(':') + 1:-1])
except: except BaseException:
print l print(l)
continue continue
if last_t != t: if last_t != t:
print >>out, '%d\t' % last_t, print('%d\t' % last_t, end=' ', file=out)
for i in keys: for i in keys:
print >>out, '%d\t' % maximum[i], print('%d\t' % maximum[i], end=' ', file=out)
print >>out, '\n', print('\n', end=' ', file=out)
if not c in keys: continue if c not in keys:
continue
field_sum[c] += fields[c] * float(t - field_timestamp[c]) field_sum[c] += fields[c] * float(t - field_timestamp[c])
field_timestamp[c] = t field_timestamp[c] = t
fields[c] = n fields[c] = n
if n > maximum[c]: maximum[c] = n if n > maximum[c]:
maximum[c] = n
if last_t != t: if last_t != t:
last_t = t last_t = t
maximum = fields maximum = fields
for i in keys: for i in keys:
print '%s: avg: %f' % (i, field_sum[i] / last_t) print('%s: avg: %f' % (i, field_sum[i] / last_t))
print print()
out.close() out.close()
out = open('disk_buffer.gnuplot', 'wb') out = open('disk_buffer.gnuplot', 'wb')
print >>out, "set term png size 1200,700" print("set term png size 1200,700", file=out)
print >>out, 'set output "disk_buffer.png"' print('set output "disk_buffer.png"', file=out)
print >>out, 'set xrange [0:*]' print('set xrange [0:*]', file=out)
print >>out, 'set xlabel "time (ms)"' print('set xlabel "time (ms)"', file=out)
print >>out, 'set ylabel "buffers"' print('set ylabel "buffers"', file=out)
print >>out, "set style data lines" print("set style data lines", file=out)
print >>out, "set key box" print("set key box", file=out)
print >>out, 'plot', print('plot', end=' ', file=out)
count = 1 + len(keys) count = 1 + len(keys)
keys.reverse() keys.reverse()
comma = '' comma = ''
for k in keys: for k in keys:
expr = "$%d" % count expr = "$%d" % count
for i in xrange(2, count): expr += "+$%d" % i for i in range(2, count):
expr += "+$%d" % i
count -= 1 count -= 1
print >>out, ' %s"disk_buffer_log.dat" using 1:(%s) title "%s" with filledcurves x1 lt rgb "#%s"' % (comma, expr, k, colors[count-1]), print(' %s"disk_buffer_log.dat" using 1:(%s) title "%s" with filledcurves x1 lt rgb "#%s"' %
(comma, expr, k, colors[count - 1]), end=' ', file=out)
comma = ',' comma = ','
out.close() out.close()
os.system('gnuplot disk_buffer.gnuplot') os.system('gnuplot disk_buffer.gnuplot')

80
tools/parse_lookup_log.py
View File

@ -1,16 +1,19 @@
#!/usr/bin/env python #!/usr/bin/env python
# this is meant to parse the dht_lookups.log generated by parse_dht_log.py # this is meant to parse the dht_lookups.log generated by parse_dht_log.py
from __future__ import print_function
import os import os
nodes = {} nodes = {}
def get_origin(n): def get_origin(n):
if n in nodes: if n in nodes:
return list(nodes[n]['conns']) return list(nodes[n]['conns'])
else: else:
return ['0.0.0.0'] return ['0.0.0.0']
def calculate_pos(nid, dist): def calculate_pos(nid, dist):
nid = int(nid[0:7], 16) nid = int(nid[0:7], 16)
@ -24,25 +27,31 @@ def calculate_pos(nid, dist):
return (x / 3, y / 3) return (x / 3, y / 3)
def plot_nodes(nodes, frame): def plot_nodes(nodes, frame):
try: os.mkdir('dht_frames') try:
except: pass os.mkdir('dht_frames')
except BaseException:
pass
out = open('dht_frames/plot-%02d.dot' % frame, 'w+') out = open('dht_frames/plot-%02d.dot' % frame, 'w+')
edges = set() edges = set()
print >>out, 'graph swarm {' print('graph swarm {', file=out)
# print >>out, '"tl" [shape=point pos="0,0!"];' # print >>out, '"tl" [shape=point pos="0,0!"];'
# print >>out, '"tr" [shape=point pos="1638,0!"];' # print >>out, '"tr" [shape=point pos="1638,0!"];'
# print >>out, '"ll" [shape=point pos="1638,1638!"];' # print >>out, '"ll" [shape=point pos="1638,1638!"];'
# print >>out, '"tr" [shape=point pos="0,1638!"];' # print >>out, '"tr" [shape=point pos="0,1638!"];'
for dst, n in nodes.items(): for dst, n in list(nodes.items()):
shape = 'point' shape = 'point'
if 's' in n: shape = n['s'] if 's' in n:
shape = n['s']
print >>out, '"%s" [shape=%s fillcolor="%s" label="" pos="%d,%d!"];' % (dst, shape, n['c'], n['p'][0], n['p'][1]) print('"%s" [shape=%s fillcolor="%s" label="" pos="%d,%d!"];' %
(dst, shape, n['c'], n['p'][0], n['p'][1]), file=out)
for e in n['conns']: for e in n['conns']:
if (e, dst) in edges: continue if (e, dst) in edges:
continue
# only add an edge once to the .dot file # only add an edge once to the .dot file
edges.add((e, dst)) edges.add((e, dst))
@ -53,9 +62,9 @@ def plot_nodes(nodes, frame):
if nodes[dst]['c'] != 'white' and nodes[e]['c'] != 'white': if nodes[dst]['c'] != 'white' and nodes[e]['c'] != 'white':
style = 'solid' style = 'solid'
col = 'black' col = 'black'
print >>out, '"%s" -- "%s" [style="%s" color="%s"];' % (e, dst, style, col) print('"%s" -- "%s" [style="%s" color="%s"];' % (e, dst, style, col), file=out)
print >>out, '}' print('}', file=out)
out.close() out.close()
os.system('neato -n dht_frames/plot-%02d.dot -Tpng -o dht_frames/frame-%02d.png' % (frame, frame)) os.system('neato -n dht_frames/plot-%02d.dot -Tpng -o dht_frames/frame-%02d.png' % (frame, frame))
@ -63,57 +72,58 @@ def plot_nodes(nodes, frame):
frame = 0 frame = 0
next_render_time = 100 next_render_time = 100
f = open('dht_lookups.txt') f = open('dht_lookups.txt')
for l in f: for line in f:
if l.startswith('***'): break if line.startswith('***'):
break
kind = l[0:3].strip() kind = line[0:3].strip()
l = l[3:].strip().split(' ') line = line[3:].strip().split(' ')
if kind == '===': continue if kind == '===':
continue
t = int(l[0]) t = int(line[0])
if t > next_render_time: if t > next_render_time:
plot_nodes(nodes, frame) plot_nodes(nodes, frame)
frame += 1 frame += 1
next_render_time += 100 next_render_time += 100
# sys.exit(0) # sys.exit(0)
if kind == '<>': if kind == '<>':
p = calculate_pos(l[1], 0) p = calculate_pos(line[1], 0)
dst = '0.0.0.0' dst = '0.0.0.0'
if not dst in nodes: if dst not in nodes:
nodes[dst] = { 'conns': set(), 'p': p, 'c': 'blue', 's': 'circle'} nodes[dst] = {'conns': set(), 'p': p, 'c': 'blue', 's': 'circle'}
p = calculate_pos(l[2], 25) p = calculate_pos(line[2], 25)
dst = '255.255.255.255' dst = '255.255.255.255'
if not dst in nodes: if dst not in nodes:
nodes[dst] = { 'conns': set(), 'p': p, 'c': 'yellow', 's': 'circle'} nodes[dst] = {'conns': set(), 'p': p, 'c': 'yellow', 's': 'circle'}
elif kind == '->': elif kind == '->':
dst = l[3] dst = line[3]
if not dst in nodes: if dst not in nodes:
src = get_origin(dst) src = get_origin(dst)
p = calculate_pos(l[2], int(l[1])) p = calculate_pos(line[2], int(line[1]))
nodes[dst] = { 'conns': set(src), 'p': p, 'c': 'grey'} nodes[dst] = {'conns': set(src), 'p': p, 'c': 'grey'}
nodes[dst]['c'] = 'grey' nodes[dst]['c'] = 'grey'
elif kind == '+': elif kind == '+':
dst = l[3] dst = line[3]
src = l[4] src = line[4]
p = calculate_pos(l[2], int(l[1])) p = calculate_pos(line[2], int(line[1]))
if not dst in nodes: if dst not in nodes:
nodes[dst] = { 'conns': set(), 'p': p, 'c': 'white'} nodes[dst] = {'conns': set(), 'p': p, 'c': 'white'}
nodes[dst]['conns'].add(src) nodes[dst]['conns'].add(src)
elif kind == '<-': elif kind == '<-':
dst = l[3] dst = line[3]
nodes[dst]['c'] = 'green' nodes[dst]['c'] = 'green'
elif kind == 'x': elif kind == 'x':
dst = l[3] dst = line[3]
nodes[dst]['c'] = 'orange' nodes[dst]['c'] = 'orange'
elif kind == 'X': elif kind == 'X':
dst = l[3] dst = line[3]
nodes[dst]['c'] = 'red' nodes[dst]['c'] = 'red'
f.close() f.close()

118
tools/parse_memory_log.py
View File

@ -1,5 +1,8 @@
#!/usr/bin/env python #!/usr/bin/env python
import os, sys, time from __future__ import print_function
import os
import sys
# usage: memory.log memory_index.log # usage: memory.log memory_index.log
@ -13,56 +16,58 @@ index = open(sys.argv[2], 'rb').readlines()
allocation_points_to_print = 30 allocation_points_to_print = 30
def print_allocation_point(ap): def print_allocation_point(ap):
print 'space_time: %d kBms' % (ap['spacetime'] / 1024) print('space_time: %d kBms' % (ap['spacetime'] / 1024))
print 'allocations: %d' % ap['allocations'] print('allocations: %d' % ap['allocations'])
print 'peak: %d kB' % (ap['peak'] / 1024) print('peak: %d kB' % (ap['peak'] / 1024))
print 'stack: ' print('stack: ')
counter = 0 counter = 0
for e in ap['stack']: for e in ap['stack']:
print '#%d %s' % (counter, e) print('#%d %s' % (counter, e))
counter += 1 counter += 1
allocation_points = [] allocation_points = []
for l in index: for line in index:
l = l.split('#') line = line.split('#')
l.pop(0) line.pop(0)
ap = { 'allocations': 0, 'peak': 0, 'spacetime': 0, 'allocation_point': len(allocation_points), 'stack': l} ap = {'allocations': 0, 'peak': 0, 'spacetime': 0, 'allocation_point': len(allocation_points), 'stack': line}
allocation_points.append(ap); allocation_points.append(ap)
for l in lines: for line in lines:
l = l.lstrip('#').rstrip('\n').split(' ') line = line.lstrip('#').rstrip('\n').split(' ')
if len(l) != 8: if len(line) != 8:
print l print(line)
continue continue
try: try:
ap = int(l[0]) ap = int(line[0])
allocation_points[ap]['allocations'] += 1 allocation_points[ap]['allocations'] += 1
allocation_points[ap]['peak'] = int(l[7]) allocation_points[ap]['peak'] = int(line[7])
allocation_points[ap]['spacetime'] = int(l[6]) allocation_points[ap]['spacetime'] = int(line[6])
except Exception, e: except Exception as e:
print type(e), e, l print(type(e), e, line)
print '=== space time ===' print('=== space time ===')
hot_ap = [] hot_ap = []
allocation_points.sort(key = lambda x:x['spacetime'], reverse=True); allocation_points.sort(key=lambda x: x['spacetime'], reverse=True)
counter = 0 counter = 0
for ap in allocation_points[0:allocation_points_to_print]: for ap in allocation_points[0:allocation_points_to_print]:
print '== %d ==' % counter print('== %d ==' % counter)
counter += 1 counter += 1
print_allocation_point(ap) print_allocation_point(ap)
hot_ap.append(ap['allocation_point']); hot_ap.append(ap['allocation_point'])
print '=== allocations ===' print('=== allocations ===')
allocation_points.sort(key = lambda x:x['allocations'], reverse=True); allocation_points.sort(key=lambda x: x['allocations'], reverse=True)
for ap in allocation_points[0:allocation_points_to_print]: for ap in allocation_points[0:allocation_points_to_print]:
print_allocation_point(ap) print_allocation_point(ap)
print '=== peak ===' print('=== peak ===')
allocation_points.sort(key = lambda x:x['peak'], reverse=True); allocation_points.sort(key=lambda x: x['peak'], reverse=True)
for ap in allocation_points[0:allocation_points_to_print]: for ap in allocation_points[0:allocation_points_to_print]:
print_allocation_point(ap) print_allocation_point(ap)
@ -74,53 +79,54 @@ cur_line = [0] * allocation_points_to_print
prev_line = [0] * allocation_points_to_print prev_line = [0] * allocation_points_to_print
last_time = 0 last_time = 0
for l in lines: for line in lines:
l = l.lstrip('#').rstrip('\n').split(' ') line = line.lstrip('#').rstrip('\n').split(' ')
if len(l) != 8: if len(line) != 8:
print l print(line)
continue continue
try: try:
time = int(l[1]) time = int(line[1])
if time != last_time: if time != last_time:
print >>out, last_time, '\t', print(last_time, '\t', end=' ', file=out)
for i in range(allocation_points_to_print): for i in range(allocation_points_to_print):
if cur_line[i] == -1: if cur_line[i] == -1:
print >>out, prev_line[i], '\t', print(prev_line[i], '\t', end=' ', file=out)
else: else:
print >>out, cur_line[i], '\t', print(cur_line[i], '\t', end=' ', file=out)
prev_line[i] = cur_line[i] prev_line[i] = cur_line[i]
print >>out print(file=out)
cur_line = [-1] * allocation_points_to_print cur_line = [-1] * allocation_points_to_print
last_time = time last_time = time
size = int(l[5]) size = int(line[5])
ap = int(l[0]) ap = int(line[0])
if ap in hot_ap: if ap in hot_ap:
index = hot_ap.index(ap) index = hot_ap.index(ap)
cur_line[index] = max(cur_line[index], size) cur_line[index] = max(cur_line[index], size)
except Exception, e: except Exception as e:
print type(e), e, l print(type(e), e, line)
out.close() out.close()
out = open('memory.gnuplot', 'wb') out = open('memory.gnuplot', 'wb')
print >>out, "set term png size 1200,700" print("set term png size 1200,700", file=out)
print >>out, 'set output "memory.png"' print('set output "memory.png"', file=out)
print >>out, 'set xrange [0:*]' print('set xrange [0:*]', file=out)
print >>out, 'set xlabel "time (ms)"' print('set xlabel "time (ms)"', file=out)
print >>out, 'set ylabel "bytes (B)"' print('set ylabel "bytes (B)"', file=out)
print >>out, "set style data lines" print("set style data lines", file=out)
print >>out, "set key box" print("set key box", file=out)
print >>out, 'plot', print('plot', end=' ', file=out)
for k in range(allocation_points_to_print): for k in range(allocation_points_to_print):
print >>out, ' "memory.dat" using 1:(', print(' "memory.dat" using 1:(', end=' ', file=out)
for i in range(k, allocation_points_to_print): for i in range(k, allocation_points_to_print):
if i == k: print >>out, '$%d' % (i + 2), if i == k:
else: print >>out, '+$%d' % (i + 2), print('$%d' % (i + 2), end=' ', file=out)
print >>out, ') title "%d" with filledcurves x1, \\' % k else:
print >>out, 'x=0' print('+$%d' % (i + 2), end=' ', file=out)
print(') title "%d" with filledcurves x1, \\' % k, file=out)
print('x=0', file=out)
out.close() out.close()
os.system('gnuplot memory.gnuplot'); os.system('gnuplot memory.gnuplot')

58
tools/parse_peer_log.py
View File

@ -1,5 +1,7 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function
import glob import glob
import os import os
import sys import sys
@ -10,63 +12,63 @@ log_files = []
for p in glob.iglob(os.path.join(sys.argv[1], '*.log')): for p in glob.iglob(os.path.join(sys.argv[1], '*.log')):
name = os.path.split(p)[1] name = os.path.split(p)[1]
if name == 'main_session.log': continue if name == 'main_session.log':
print name continue
print(name)
f = open(p, 'r') f = open(p, 'r')
out_file = p + '.dat' out_file = p + '.dat'
log_files.append(out_file) log_files.append(out_file)
out = open(out_file, 'w+') out = open(out_file, 'w+')
uploaded_blocks = 0; uploaded_blocks = 0
downloaded_blocks = 0; downloaded_blocks = 0
for l in f: for l in f:
t = l.split(': ')[0].split('.')[0] t = l.split(': ')[0].split('.')[0]
log_line = False log_line = False
if ' ==> PIECE' in l: if ' ==> PIECE' in l:
uploaded_blocks+= 1 uploaded_blocks += 1
log_line = True log_line = True
if ' <== PIECE' in l: if ' <== PIECE' in l:
downloaded_blocks+= 1 downloaded_blocks += 1
log_line = True log_line = True
if log_line: if log_line:
print >>out, '%s\t%d\t%d' % (t, uploaded_blocks, downloaded_blocks) print('%s\t%d\t%d' % (t, uploaded_blocks, downloaded_blocks), file=out)
out.close() out.close()
f.close() f.close()
out = open('peers.gnuplot', 'wb') out = open('peers.gnuplot', 'wb')
print >>out, "set term png size 1200,700" print("set term png size 1200,700", file=out)
print >>out, 'set xrange [0:*]' print('set xrange [0:*]', file=out)
print >>out, 'set xlabel "time"' print('set xlabel "time"', file=out)
print >>out, 'set ylabel "blocks"' print('set ylabel "blocks"', file=out)
print >>out, 'set key box' print('set key box', file=out)
print >>out, 'set xdata time' print('set xdata time', file=out)
print >>out, 'set timefmt "%H:%M:%S"' print('set timefmt "%H:%M:%S"', file=out)
print >>out, 'set title "uploaded blocks"' print('set title "uploaded blocks"', file=out)
print >>out, 'set output "peers_upload.png"' print('set output "peers_upload.png"', file=out)
print >>out, 'plot', print('plot', end=' ', file=out)
first = True first = True
for n in log_files: for n in log_files:
if not first: if not first:
print >>out, ',', print(',', end=' ', file=out)
first = False first = False
print >>out, ' "%s" using 1:2 title "%s" with steps' % (n, os.path.split(n)[1].split('.log')[0]), print(' "%s" using 1:2 title "%s" with steps' % (n, os.path.split(n)[1].split('.log')[0]), end=' ', file=out)
print >>out, '' print('', file=out)
print >>out, 'set title "downloaded blocks"' print('set title "downloaded blocks"', file=out)
print >>out, 'set output "peers_download.png"' print('set output "peers_download.png"', file=out)
print >>out, 'plot', print('plot', end=' ', file=out)
first = True first = True
for n in log_files: for n in log_files:
if not first: if not first:
print >>out, ',', print(',', end=' ', file=out)
first = False first = False
print >>out, ' "%s" using 1:3 title "%s" with steps' % (n, os.path.split(n)[1].split('.log')[0]), print(' "%s" using 1:3 title "%s" with steps' % (n, os.path.split(n)[1].split('.log')[0]), end=' ', file=out)
print >>out, '' print('', file=out)
out.close() out.close()
os.system('gnuplot peers.gnuplot'); os.system('gnuplot peers.gnuplot')

178
tools/parse_sample.py
View File

@ -7,96 +7,149 @@ import sys
# relevant depths and to filter out low sample counts # relevant depths and to filter out low sample counts
f = open(sys.argv[1]) f = open(sys.argv[1])
def parse_line(l):
def parse_line(line):
indentation = 0 indentation = 0
while indentation < len(l) and l[indentation] == ' ': while indentation < len(line) and line[indentation] == ' ':
indentation += 1 indentation += 1
if indentation == 0: if indentation == 0:
return (0, 0, '') return (0, 0, '')
line = line.strip().split(' ')
l = l.strip().split(' ') samples = int(line[0])
samples = int(l[0]) fun = ' '.join(line[1:])
fun = ' '.join(l[1:])
return (indentation, samples, fun) return (indentation, samples, fun)
fold = -1 fold = -1
try: try:
sample_limit = int(sys.argv[2]) sample_limit = int(sys.argv[2])
except: except BaseException:
sample_limit = 5 sample_limit = 5
fun_samples = {} fun_samples = {}
for l in f: for l in f:
if 'Sort by top of stack' in l: break if 'Sort by top of stack' in l:
break
indentation, samples, fun = parse_line(l) indentation, samples, fun = parse_line(l)
if samples < sample_limit: continue if samples < sample_limit:
if fold != -1 and indentation > fold: continue continue
if fold != -1 and indentation > fold:
continue
fold = -1 fold = -1
if '__gnu_cxx::__normal_iterator<' in fun: if '__gnu_cxx::__normal_iterator<' in fun:
fold = indentation - 1 fold = indentation - 1
continue continue
if 'boost::_bi::bind_t' in fun: continue if 'boost::_bi::bind_t' in fun:
if 'boost::_bi::list' in fun: continue continue
if 'boost::_mfi::mf' in fun: continue if 'boost::_bi::list' in fun:
if 'boost::_bi::storage' in fun: continue continue
if 'boost::_mfi::mf' in fun:
continue
if 'boost::_bi::storage' in fun:
continue
# should only add leaves # should only add leaves
if fun in fun_samples: fun_samples[fun] += samples if fun in fun_samples:
else: fun_samples[fun] = samples fun_samples[fun] += samples
else:
fun_samples[fun] = samples
output = '%s%-4d %s' % (' ' * (indentation/2), samples, fun) output = '%s%-4d %s' % (' ' * (indentation / 2), samples, fun)
if len(output) > 200: output = output[0:200] if len(output) > 200:
print output output = output[0:200]
print(output)
if 'invariant_checker_impl' in fun: fold = indentation if 'invariant_checker_impl' in fun:
if 'free_multiple_buffers' in fun: fold = indentation fold = indentation
if 'libtorrent::condition::wait' in fun: fold = indentation if 'free_multiple_buffers' in fun:
if 'allocate_buffer' in fun: fold = indentation fold = indentation
if '::find_POD' in fun: fold = indentation if 'libtorrent::condition::wait' in fun:
if 'SHA1_Update' in fun: fold = indentation fold = indentation
if 'boost::detail::function::basic_vtable' in fun: fold = indentation if 'allocate_buffer' in fun:
if 'operator new' in fun: fold = indentation fold = indentation
if 'malloc' == fun: fold = indentation if '::find_POD' in fun:
if 'free' == fun: fold = indentation fold = indentation
if 'std::_Rb_tree' in fun: fold = indentation if 'SHA1_Update' in fun:
if 'pthread_cond_wait' in fun: fold = indentation fold = indentation
if 'mp_exptmod' == fun: fold = indentation if 'boost::detail::function::basic_vtable' in fun:
if '::check_invariant()' in fun: fold = indentation fold = indentation
if 'libtorrent::condition::wait' in fun: fold = indentation if 'operator new' in fun:
if '_sigtramp' in fun: fold = indentation fold = indentation
if 'time_now_hires' in fun: fold = indentation if 'malloc' == fun:
if 'libtorrent::sleep' in fun: fold = indentation fold = indentation
if 'puts' == fun: fold = indentation if 'free' == fun:
if 'boost::asio::basic_stream_socket' in fun: fold = indentation fold = indentation
if 'recvmsg' == fun: fold = indentation if 'std::_Rb_tree' in fun:
if 'sendmsg' == fun: fold = indentation fold = indentation
if 'semaphore_signal_trap' == fun: fold = indentation if 'pthread_cond_wait' in fun:
if 'boost::detail::atomic_count::operator' in fun: fold = indentation fold = indentation
if 'pthread_mutex_lock' == fun: fold = indentation if 'mp_exptmod' == fun:
if 'pthread_mutex_unlock' == fun: fold = indentation fold = indentation
if '>::~vector()' == fun: fold = indentation if '::check_invariant()' in fun:
if 'szone_free_definite_size' == fun: fold = indentation fold = indentation
if 'snprintf' == fun: fold = indentation if 'libtorrent::condition::wait' in fun:
if 'usleep' == fun: fold = indentation fold = indentation
if 'pthread_mutex_lock' == fun: fold = indentation if '_sigtramp' in fun:
if 'pthread_mutex_unlock' == fun: fold = indentation fold = indentation
if 'std::string::append' in fun: fold = indentation if 'time_now_hires' in fun:
if 'getipnodebyname' == fun: fold = indentation fold = indentation
if '__gnu_debug::_Safe_iterator<std::' in fun: fold = indentation if 'libtorrent::sleep' in fun:
if 'fflush' == fun: fold = indentation fold = indentation
if 'vfprintf' == fun: fold = indentation if 'puts' == fun:
if 'fprintf' == fun: fold = indentation fold = indentation
if 'BN_mod_exp' == fun: fold = indentation if 'boost::asio::basic_stream_socket' in fun:
if 'BN_CTX_free' == fun: fold = indentation fold = indentation
if 'cerror' == fun: fold = indentation if 'recvmsg' == fun:
if '0xffffffff' == fun: fold = indentation fold = indentation
if 'sendmsg' == fun:
fold = indentation
if 'semaphore_signal_trap' == fun:
fold = indentation
if 'boost::detail::atomic_count::operator' in fun:
fold = indentation
if 'pthread_mutex_lock' == fun:
fold = indentation
if 'pthread_mutex_unlock' == fun:
fold = indentation
if '>::~vector()' == fun:
fold = indentation
if 'szone_free_definite_size' == fun:
fold = indentation
if 'snprintf' == fun:
fold = indentation
if 'usleep' == fun:
fold = indentation
if 'pthread_mutex_lock' == fun:
fold = indentation
if 'pthread_mutex_unlock' == fun:
fold = indentation
if 'std::string::append' in fun:
fold = indentation
if 'getipnodebyname' == fun:
fold = indentation
if '__gnu_debug::_Safe_iterator<std::' in fun:
fold = indentation
if 'fflush' == fun:
fold = indentation
if 'vfprintf' == fun:
fold = indentation
if 'fprintf' == fun:
fold = indentation
if 'BN_mod_exp' == fun:
fold = indentation
if 'BN_CTX_free' == fun:
fold = indentation
if 'cerror' == fun:
fold = indentation
if '0xffffffff' == fun:
fold = indentation
list = [] list = []
for k in fun_samples: for k in fun_samples:
@ -105,5 +158,4 @@ for k in fun_samples:
list = sorted(list, reverse=True) list = sorted(list, reverse=True)
for i in list: for i in list:
print '%-4d %s' % (i[0], i[1]) print('%-4d %s' % (i[0], i[1]))

430
tools/parse_session_stats.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function
# Copyright (c) 2016, Arvid Norberg # Copyright (c) 2016, Arvid Norberg
# All rights reserved. # All rights reserved.
@ -31,7 +32,9 @@
# this script can parse and generate reports from the alert log from a # this script can parse and generate reports from the alert log from a
# libtorrent session # libtorrent session
import os, sys, time, os, math import os
import sys
import math
from multiprocessing.pool import ThreadPool from multiprocessing.pool import ThreadPool
thread_pool = ThreadPool(8) thread_pool = ThreadPool(8)
@ -40,19 +43,22 @@ output_dir = 'session_stats_report'
stat = open(sys.argv[1]) stat = open(sys.argv[1])
line = stat.readline() line = stat.readline()
while not 'session stats header:' in line: while 'session stats header:' not in line:
line = stat.readline() line = stat.readline()
keys = line.split('session stats header:')[1].strip().split(', ') keys = line.split('session stats header:')[1].strip().split(', ')
try: os.mkdir(output_dir) try:
except: pass os.mkdir(output_dir)
except BaseException:
pass
data_out = open(os.path.join(output_dir, 'counters.dat'), 'w+') data_out = open(os.path.join(output_dir, 'counters.dat'), 'w+')
idx = 0 idx = 0
for l in stat: for line in stat:
if not 'session stats (' in l: continue if 'session stats (' not in line:
data_out.write(("%d\t" % idx) + l.split(' values): ')[1].strip().replace(', ', '\t') + '\n') continue
data_out.write(("%d\t" % idx) + line.split(' values): ')[1].strip().replace(', ', '\t') + '\n')
idx += 1 idx += 1
data_out.close() data_out.close()
@ -64,7 +70,8 @@ diff = 3
graph_colors = [] graph_colors = []
pattern = [[0,0,1], [0,1,0], [1,0,0], [1,0,1], [0,1,1], [1,1,0]] pattern = [[0, 0, 1], [0, 1, 0], [1, 0, 0], [1, 0, 1], [0, 1, 1], [1, 1, 0]]
def process_color(c, op): def process_color(c, op):
for i in range(3): for i in range(3):
@ -74,12 +81,14 @@ def process_color(c, op):
c[i] = max(0, c[i] - 0x50) c[i] = max(0, c[i] - 0x50)
return c return c
for i in range(0,len(pattern) * 3):
for i in range(0, len(pattern) * 3):
op = i / len(pattern) op = i / len(pattern)
c = list(pattern[i % len(pattern)]) c = list(pattern[i % len(pattern)])
for i in range(3): c[i] *= 0xff for i in range(3):
c[i] *= 0xff
c = process_color(c, op) c = process_color(c, op)
c = '#%02x%02x%02x' % (c[0], c[1], c[2]) c = '#%02x%02x%02x' % (c[0], c[1], c[2])
@ -92,9 +101,9 @@ gradient16_colors = []
for i in range(0, 16): for i in range(0, 16):
f = i / 16. f = i / 16.
pi = 3.1415927 pi = 3.1415927
r = max(int(255 * (math.sin(f*pi)+0.2)), 0) r = max(int(255 * (math.sin(f * pi) + 0.2)), 0)
g = max(int(255 * (math.sin((f-0.5)*pi)+0.2)), 0) g = max(int(255 * (math.sin((f - 0.5) * pi) + 0.2)), 0)
b = max(int(255 * (math.sin((f+0.5)*pi)+0.2)), 0) b = max(int(255 * (math.sin((f + 0.5) * pi) + 0.2)), 0)
c = '#%02x%02x%02x' % (min(r, 255), min(g, 255), min(b, 255)) c = '#%02x%02x%02x' % (min(r, 255), min(g, 255), min(b, 255))
gradient16_colors.append(c) gradient16_colors.append(c)
@ -102,9 +111,9 @@ gradient18_colors = []
for i in range(0, 18): for i in range(0, 18):
f = i / 18. f = i / 18.
pi = 3.1415927 pi = 3.1415927
r = max(int(255 * (math.sin(f*pi)+0.2)), 0) r = max(int(255 * (math.sin(f * pi) + 0.2)), 0)
g = max(int(255 * (math.sin((f-0.5)*pi)+0.2)), 0) g = max(int(255 * (math.sin((f - 0.5) * pi) + 0.2)), 0)
b = max(int(255 * (math.sin((f+0.5)*pi)+0.2)), 0) b = max(int(255 * (math.sin((f + 0.5) * pi) + 0.2)), 0)
c = '#%02x%02x%02x' % (min(r, 255), min(g, 255), min(b, 255)) c = '#%02x%02x%02x' % (min(r, 255), min(g, 255), min(b, 255))
gradient18_colors.append(c) gradient18_colors.append(c)
@ -114,18 +123,21 @@ for i in range(0, 6):
c = '#%02x%02x%02x' % (min(int(255 * (-2 * f + 2)), 255), min(int(255 * (2 * f)), 255), 100) c = '#%02x%02x%02x' % (min(int(255 * (-2 * f + 2)), 255), min(int(255 * (2 * f)), 255), 100)
gradient6_colors.append(c) gradient6_colors.append(c)
def plot_fun(script): def plot_fun(script):
ret = os.system('gnuplot "%s" 2>/dev/null' % script) ret = os.system('gnuplot "%s" 2>/dev/null' % script)
if ret != 0 and ret != 256: if ret != 0 and ret != 256:
print 'system: %d\n' % ret print('system: %d\n' % ret)
raise Exception("abort") raise Exception("abort")
sys.stdout.write('.') sys.stdout.write('.')
sys.stdout.flush() sys.stdout.flush()
def to_title(key): def to_title(key):
return key.replace('_', ' ').replace('.', ' - ') return key.replace('_', ' ').replace('.', ' - ')
def gen_report(name, unit, lines, short_unit, generation, log_file, options): def gen_report(name, unit, lines, short_unit, generation, log_file, options):
filename = os.path.join(output_dir, '%s_%04d.png' % (name, generation)) filename = os.path.join(output_dir, '%s_%04d.png' % (name, generation))
thumb = os.path.join(output_dir, '%s_%04d_thumb.png' % (name, generation)) thumb = os.path.join(output_dir, '%s_%04d_thumb.png' % (name, generation))
@ -140,17 +152,18 @@ def gen_report(name, unit, lines, short_unit, generation, log_file, options):
sys.stdout.write('.') sys.stdout.write('.')
return None return None
except: pass except BaseException:
pass
script = os.path.join(output_dir, '%s_%04d.gnuplot' % (name, generation)) script = os.path.join(output_dir, '%s_%04d.gnuplot' % (name, generation))
out = open(script, 'wb') out = open(script, 'wb')
print >>out, "set term png size 1200,700" print("set term png size 1200,700", file=out)
print >>out, 'set output "%s"' % filename print('set output "%s"' % filename, file=out)
if not 'allow-negative' in options: if 'allow-negative' not in options:
print >>out, 'set yrange [0:*]' print('set yrange [0:*]', file=out)
print >>out, "set tics nomirror" print("set tics nomirror", file=out)
print >>out, "set key box" print("set key box", file=out)
print >>out, "set key left top" print("set key left top", file=out)
colors = graph_colors colors = graph_colors
if options['type'] == line_graph: if options['type'] == line_graph:
@ -163,37 +176,38 @@ def gen_report(name, unit, lines, short_unit, generation, log_file, options):
colors = gradient6_colors colors = gradient6_colors
if options['colors'] == 'gradient18': if options['colors'] == 'gradient18':
colors = gradient18_colors colors = gradient18_colors
except: pass except BaseException:
pass
if options['type'] == histogram: if options['type'] == histogram:
binwidth = options['binwidth'] binwidth = options['binwidth']
numbins = int(options['numbins']) numbins = int(options['numbins'])
print >>out, 'binwidth=%f' % binwidth print('binwidth=%f' % binwidth, file=out)
print >>out, 'set boxwidth binwidth' print('set boxwidth binwidth', file=out)
print >>out, 'bin(x,width)=width*floor(x/width) + binwidth/2' print('bin(x,width)=width*floor(x/width) + binwidth/2', file=out)
print >>out, 'set xrange [0:%f]' % (binwidth * numbins) print('set xrange [0:%f]' % (binwidth * numbins), file=out)
print >>out, 'set xlabel "%s"' % unit print('set xlabel "%s"' % unit, file=out)
print >>out, 'set ylabel "number"' print('set ylabel "number"', file=out)
k = lines[0] k = lines[0]
try: try:
column = keys.index(k) + 2 column = keys.index(k) + 2
except: except BaseException:
print '"%s" not found' % k print('"%s" not found' % k)
return return
print >>out, 'plot "%s" using (bin($%d,binwidth)):(1.0) smooth freq with boxes' % (log_file, column) print('plot "%s" using (bin($%d,binwidth)):(1.0) smooth freq with boxes' % (log_file, column), file=out)
print >>out, '' print('', file=out)
print >>out, '' print('', file=out)
print >>out, '' print('', file=out)
elif options['type'] == stacked: elif options['type'] == stacked:
print >>out, 'set xrange [0:*]' print('set xrange [0:*]', file=out)
print >>out, 'set ylabel "%s"' % unit print('set ylabel "%s"' % unit, file=out)
print >>out, 'set xlabel "time (s)"' print('set xlabel "time (s)"', file=out)
print >>out, 'set format y "%%.1s%%c%s";' % short_unit print('set format y "%%.1s%%c%s";' % short_unit, file=out)
print >>out, 'set style fill solid 1.0 noborder' print('set style fill solid 1.0 noborder', file=out)
print >>out, 'plot', print('plot', end=' ', file=out)
column = 2 column = 2
first = True first = True
graph = '' graph = ''
@ -202,23 +216,24 @@ def gen_report(name, unit, lines, short_unit, generation, log_file, options):
for k in lines: for k in lines:
try: try:
column = keys.index(k) + 2 column = keys.index(k) + 2
except: except BaseException:
print '"%s" not found' % k print('"%s" not found' % k)
continue; continue
if not first: if not first:
plot_expression = ', ' + plot_expression plot_expression = ', ' + plot_expression
graph += '+' graph += '+'
axis = 'x1y1' axis = 'x1y1'
graph += '$%d' % column graph += '$%d' % column
plot_expression = ' "%s" using 1:(%s) title "%s" axes %s with filledcurves x1 lc rgb "%s"' % (log_file, graph, to_title(k), axis, colors[color % len(colors)]) + plot_expression plot_expression = ' "%s" using 1:(%s) title "%s" axes %s with filledcurves x1 lc rgb "%s"' % (
log_file, graph, to_title(k), axis, colors[color % len(colors)]) + plot_expression
first = False first = False
color += 1 color += 1
print >>out, plot_expression print(plot_expression, file=out)
elif options['type'] == diff: elif options['type'] == diff:
print >>out, 'set xrange [0:*]' print('set xrange [0:*]', file=out)
print >>out, 'set ylabel "%s"' % unit print('set ylabel "%s"' % unit, file=out)
print >>out, 'set xlabel "time (s)"' print('set xlabel "time (s)"', file=out)
print >>out, 'set format y "%%.1s%%c%s";' % short_unit print('set format y "%%.1s%%c%s";' % short_unit, file=out)
column = 2 column = 2
first = True first = True
graph = '' graph = ''
@ -226,177 +241,203 @@ def gen_report(name, unit, lines, short_unit, generation, log_file, options):
for k in lines: for k in lines:
try: try:
column = keys.index(k) + 2 column = keys.index(k) + 2
except: except BaseException:
print '"%s" not found' % k print('"%s" not found' % k)
continue; continue
if not first: if not first:
graph += '-' graph += '-'
title += ' - ' title += ' - '
graph += '$%d' % column graph += '$%d' % column
title += to_title(k) title += to_title(k)
first = False first = False
print >>out, 'plot "%s" using 1:(%s) title "%s" with step' % (log_file, graph, title) print('plot "%s" using 1:(%s) title "%s" with step' % (log_file, graph, title), file=out)
else: else:
print >>out, 'set xrange [0:*]' print('set xrange [0:*]', file=out)
print >>out, 'set ylabel "%s"' % unit print('set ylabel "%s"' % unit, file=out)
print >>out, 'set xlabel "time (s)"' print('set xlabel "time (s)"', file=out)
print >>out, 'set format y "%%.1s%%c%s";' % short_unit print('set format y "%%.1s%%c%s";' % short_unit, file=out)
print >>out, 'plot', print('plot', end=' ', file=out)
column = 2 column = 2
first = True first = True
color = 0 color = 0
for k in lines: for k in lines:
try: try:
column = keys.index(k) + 2 column = keys.index(k) + 2
except: except BaseException:
print '"%s" not found' % k print('"%s" not found' % k)
continue; continue
if not first: print >>out, ', ', if not first:
print(', ', end=' ', file=out)
axis = 'x1y1' axis = 'x1y1'
print >>out, ' "%s" using 1:%d title "%s" axes %s with steps lc rgb "%s"' % (log_file, column, to_title(k), axis, colors[color % len(colors)]), print(' "%s" using 1:%d title "%s" axes %s with steps lc rgb "%s"' %
(log_file, column, to_title(k), axis, colors[color % len(colors)]), end=' ', file=out)
first = False first = False
color += 1 color += 1
print >>out, '' print('', file=out)
print >>out, 'set term png size 150,100' print('set term png size 150,100', file=out)
print >>out, 'set output "%s"' % thumb print('set output "%s"' % thumb, file=out)
print >>out, 'set key off' print('set key off', file=out)
print >>out, 'unset tics' print('unset tics', file=out)
print >>out, 'set format x ""' print('set format x ""', file=out)
print >>out, 'set format y ""' print('set format y ""', file=out)
print >>out, 'set xlabel ""' print('set xlabel ""', file=out)
print >>out, 'set ylabel ""' print('set ylabel ""', file=out)
print >>out, 'set y2label ""' print('set y2label ""', file=out)
print >>out, 'set rmargin 0' print('set rmargin 0', file=out)
print >>out, 'set lmargin 0' print('set lmargin 0', file=out)
print >>out, 'set tmargin 0' print('set tmargin 0', file=out)
print >>out, 'set bmargin 0' print('set bmargin 0', file=out)
print >>out, "replot" print("replot", file=out)
out.close() out.close()
return script return script
def gen_html(reports, generations): def gen_html(reports, generations):
file = open(os.path.join(output_dir, 'index.html'), 'w+') file = open(os.path.join(output_dir, 'index.html'), 'w+')
css = '''img { margin: 0} css = '''img { margin: 0}
#head { display: block } #head { display: block }
#graphs { white-space:nowrap; } #graphs { white-space:nowrap; }
h1 { line-height: 1; display: inline } h1 { line-height: 1; display: inline }
h2 { line-height: 1; display: inline; font-size: 1em; font-weight: normal};''' h2 { line-height: 1; display: inline; font-size: 1em; font-weight: normal};'''
print >>file, '<html><head><style type="text/css">%s</style></head><body>' % css print('<html><head><style type="text/css">%s</style></head><body>' % css, file=file)
for i in reports: for i in reports:
print >>file, '<div id="head"><h1>%s </h1><h2>%s</h2><div><div id="graphs">' % (i[0], i[3]) print('<div id="head"><h1>%s </h1><h2>%s</h2><div><div id="graphs">' % (i[0], i[3]), file=file)
for g in generations: for g in generations:
print >>file, '<a href="%s_%04d.png"><img src="%s_%04d_thumb.png"></a>' % (i[0], g, i[0], g) print('<a href="%s_%04d.png"><img src="%s_%04d_thumb.png"></a>' % (i[0], g, i[0], g), file=file)
print >>file, '</div>' print('</div>', file=file)
print >>file, '</body></html>' print('</body></html>', file=file)
file.close() file.close()
reports = [ reports = [
('torrents', 'num', '', 'number of torrents in different torrent states', [ \ ('torrents', 'num', '', 'number of torrents in different torrent states', [
'ses.num_downloading_torrents', \ 'ses.num_downloading_torrents',
'ses.num_seeding_torrents', \ 'ses.num_seeding_torrents',
'ses.num_checking_torrents', \ 'ses.num_checking_torrents',
'ses.num_stopped_torrents', \ 'ses.num_stopped_torrents',
'ses.num_upload_only_torrents', \ 'ses.num_upload_only_torrents',
'ses.num_error_torrents', \ 'ses.num_error_torrents',
'ses.num_queued_seeding_torrents', \ 'ses.num_queued_seeding_torrents',
'ses.num_queued_download_torrents' \ 'ses.num_queued_download_torrents'
], {'type':stacked}), ], {'type': stacked}),
('peers', 'num', '', 'num connected peers', ['peer.num_peers_connected', 'peer.num_peers_half_open'], {'type':stacked}), ('peers', 'num', '', 'num connected peers', [
'peer.num_peers_connected', 'peer.num_peers_half_open'], {'type': stacked}),
('peers_max', 'num', '', 'num connected peers', ['peer.num_peers_connected', 'peer.num_peers_half_open']), ('peers_max', 'num', '', 'num connected peers', ['peer.num_peers_connected', 'peer.num_peers_half_open']),
('peer_churn', 'num', '', 'connecting and disconnecting peers', ['peer.num_peers_half_open', 'peer.connection_attempts']), ('peer_churn', 'num', '', 'connecting and disconnecting peers',
['peer.num_peers_half_open', 'peer.connection_attempts']),
('new_peers', 'num', '', '', ['peer.incoming_connections', 'peer.connection_attempts']), ('new_peers', 'num', '', '', ['peer.incoming_connections', 'peer.connection_attempts']),
('connection_attempts', 'num', '', '', ['peer.connection_attempt_loops', 'peer.connection_attempts']), ('connection_attempts', 'num', '', '', ['peer.connection_attempt_loops', 'peer.connection_attempts']),
('pieces', 'num', '', 'number completed pieces', ['ses.num_total_pieces_added', 'ses.num_piece_passed', 'ses.num_piece_failed']), ('pieces', 'num', '', 'number completed pieces', [
'ses.num_total_pieces_added', 'ses.num_piece_passed', 'ses.num_piece_failed']),
('disk_write_queue', 'Bytes', 'B', 'bytes queued up by peers, to be written to disk', ['disk.queued_write_bytes']), ('disk_write_queue', 'Bytes', 'B', 'bytes queued up by peers, to be written to disk', ['disk.queued_write_bytes']),
('peers_requests', 'num', '', 'incoming piece request rate', [ \ ('peers_requests', 'num', '', 'incoming piece request rate', [
'peer.piece_requests', \ 'peer.piece_requests',
'peer.max_piece_requests', \ 'peer.max_piece_requests',
'peer.invalid_piece_requests', \ 'peer.invalid_piece_requests',
'peer.choked_piece_requests', \ 'peer.choked_piece_requests',
'peer.cancelled_piece_requests' \ 'peer.cancelled_piece_requests'
]), ]),
('peers_upload', 'num', '', 'number of peers by state wrt. uploading', [ \ ('peers_upload', 'num', '', 'number of peers by state wrt. uploading', [
'peer.num_peers_up_disk', \ 'peer.num_peers_up_disk',
'peer.num_peers_up_interested', \ 'peer.num_peers_up_interested',
'peer.num_peers_up_unchoked_all', \ 'peer.num_peers_up_unchoked_all',
'peer.num_peers_up_unchoked_optimistic', \ 'peer.num_peers_up_unchoked_optimistic',
'peer.num_peers_up_unchoked', \ 'peer.num_peers_up_unchoked',
'peer.num_peers_up_requests' \ 'peer.num_peers_up_requests'
]), ]),
('peers_download', 'num', '', 'number of peers by state wrt. downloading', [ \ ('peers_download', 'num', '', 'number of peers by state wrt. downloading', [
'peer.num_peers_down_interested', \ 'peer.num_peers_down_interested',
'peer.num_peers_down_unchoked', \ 'peer.num_peers_down_unchoked',
'peer.num_peers_down_requests', \ 'peer.num_peers_down_requests',
'peer.num_peers_down_disk' \ 'peer.num_peers_down_disk'
]), ]),
('peer_errors', 'num', '', 'number of peers by error that disconnected them', [ \ ('peer_errors', 'num', '', 'number of peers by error that disconnected them', [
'peer.disconnected_peers', \ 'peer.disconnected_peers',
'peer.eof_peers', \ 'peer.eof_peers',
'peer.connreset_peers', \ 'peer.connreset_peers',
'peer.connrefused_peers', \ 'peer.connrefused_peers',
'peer.connaborted_peers', \ 'peer.connaborted_peers',
'peer.perm_peers', \ 'peer.perm_peers',
'peer.buffer_peers', \ 'peer.buffer_peers',
'peer.unreachable_peers', \ 'peer.unreachable_peers',
'peer.broken_pipe_peers', \ 'peer.broken_pipe_peers',
'peer.addrinuse_peers', \ 'peer.addrinuse_peers',
'peer.no_access_peers', \ 'peer.no_access_peers',
'peer.invalid_arg_peers', \ 'peer.invalid_arg_peers',
'peer.aborted_peers' \ 'peer.aborted_peers'
], {'type':stacked}), ], {'type': stacked}),
('peer_errors_incoming', 'num', '', 'number of peers by incoming or outgoing connection', [ \ ('peer_errors_incoming', 'num', '', 'number of peers by incoming or outgoing connection', [
'peer.error_incoming_peers', \ 'peer.error_incoming_peers',
'peer.error_outgoing_peers' \ 'peer.error_outgoing_peers'
]), ]),
('peer_errors_transport', 'num', '', 'number of peers by transport protocol', [ \ ('peer_errors_transport', 'num', '', 'number of peers by transport protocol', [
'peer.error_tcp_peers', \ 'peer.error_tcp_peers',
'peer.error_utp_peers' 'peer.error_utp_peers'
]), ]),
('peer_errors_encryption', 'num', '', 'number of peers by encryption level', [ \ ('peer_errors_encryption', 'num', '', 'number of peers by encryption level', [
'peer.error_encrypted_peers', \ 'peer.error_encrypted_peers',
'peer.error_rc4_peers', \ 'peer.error_rc4_peers',
]), ]),
('incoming requests', 'num', '', 'incoming 16kiB block requests', ['ses.num_incoming_request']), ('incoming requests', 'num', '', 'incoming 16kiB block requests', ['ses.num_incoming_request']),
('waste', 'downloaded bytes', 'B', 'proportion of all downloaded bytes that were wasted', [ \ ('waste', 'downloaded bytes', 'B', 'proportion of all downloaded bytes that were wasted', [
'net.recv_failed_bytes', \ 'net.recv_failed_bytes',
'net.recv_redundant_bytes', \ 'net.recv_redundant_bytes',
'net.recv_ip_overhead_bytes' \ 'net.recv_ip_overhead_bytes'
], {'type':stacked}), ], {'type': stacked}),
('waste by source', 'num wasted bytes', 'B', 'what is causing the waste', [ \ ('waste by source', 'num wasted bytes', 'B', 'what is causing the waste', [
'ses.waste_piece_timed_out', \ 'ses.waste_piece_timed_out',
'ses.waste_piece_cancelled', \ 'ses.waste_piece_cancelled',
'ses.waste_piece_unknown', \ 'ses.waste_piece_unknown',
'ses.waste_piece_seed', \ 'ses.waste_piece_seed',
'ses.waste_piece_end_game', \ 'ses.waste_piece_end_game',
'ses.waste_piece_closing' \ 'ses.waste_piece_closing'
], {'type':stacked}), ], {'type': stacked}),
('disk_time', '% of total disk job time', '%%', 'proportion of time spent by the disk thread', ['disk.disk_read_time', 'disk.disk_write_time', 'disk.disk_hash_time'], {'type': stacked}), ('disk_time', '% of total disk job time', '%%', 'proportion of time spent by the disk thread',
('disk_cache_hits', 'blocks (16kiB)', '', '', ['disk.num_blocks_read', 'disk.num_blocks_cache_hits'], {'type':stacked}), ['disk.disk_read_time', 'disk.disk_write_time', 'disk.disk_hash_time'], {'type': stacked}),
('disk_cache', 'blocks (16kiB)', '', 'disk cache size and usage', ['disk.disk_blocks_in_use', 'disk.read_cache_blocks', 'disk.write_cache_blocks', 'disk.pinned_blocks']), ('disk_cache_hits', 'blocks (16kiB)', '', '', [
('disk_readback', '% of written blocks', '%%', 'portion of written blocks that had to be read back for hash verification', ['disk.num_read_back']), 'disk.num_blocks_read', 'disk.num_blocks_cache_hits'], {'type': stacked}),
('disk_queue', 'number of queued disk jobs', '', 'num disk jobs', ['disk.num_write_jobs', 'disk.num_read_jobs', 'disk.num_jobs', 'disk.queued_disk_jobs', 'disk.blocked_disk_jobs']), ('disk_cache', 'blocks (16kiB)', '', 'disk cache size and usage', [
'disk.disk_blocks_in_use', 'disk.read_cache_blocks', 'disk.write_cache_blocks', 'disk.pinned_blocks']),
('disk_readback',
'% of written blocks',
'%%',
'portion of written blocks that had to be read back for hash verification',
['disk.num_read_back']),
('disk_queue',
'number of queued disk jobs',
'',
'num disk jobs',
['disk.num_write_jobs',
'disk.num_read_jobs',
'disk.num_jobs',
'disk.queued_disk_jobs',
'disk.blocked_disk_jobs']),
('disk fences', 'num', '', 'number of jobs currently blocked by a fence job', ['disk.blocked_disk_jobs']), ('disk fences', 'num', '', 'number of jobs currently blocked by a fence job', ['disk.blocked_disk_jobs']),
# ('fence jobs', 'num', '', 'active fence jobs per type', ['move_storage', 'release_files', 'delete_files', 'check_fastresume', 'save_resume_data', 'rename_file', 'stop_torrent', 'file_priority', 'clear_piece'], {'type':stacked}), # ('fence jobs', 'num', '', 'active fence jobs per type', ['move_storage', 'release_files', 'delete_files',
('disk threads', 'num', '', 'number of disk threads currently writing', ['disk.num_writing_threads', 'disk.num_running_threads']), # 'check_fastresume', 'save_resume_data', 'rename_file', 'stop_torrent', 'file_priority', 'clear_piece'],
# ('mixed mode', 'rate', 'B/s', 'rates by transport protocol', ['TCP up rate','TCP down rate','uTP up rate','uTP down rate','TCP up limit','TCP down limit']), # {'type':stacked}),
('disk threads', 'num', '', 'number of disk threads currently writing',
['disk.num_writing_threads', 'disk.num_running_threads']),
# ('mixed mode', 'rate', 'B/s', 'rates by transport protocol',
# ['TCP up rate','TCP down rate','uTP up rate','uTP down rate','TCP up limit','TCP down limit']),
('connection_type', 'num', '', 'peers by transport protocol', [ \ ('connection_type', 'num', '', 'peers by transport protocol', [ \
'peer.num_tcp_peers', \ 'peer.num_tcp_peers', \
@ -410,9 +451,12 @@ reports = [
'peer.num_ssl_utp_peers' \ 'peer.num_ssl_utp_peers' \
]), ]),
# ('uTP delay', 'buffering delay', 's', 'network delays measured by uTP', ['uTP peak send delay','uTP peak recv delay', 'uTP avg send delay', 'uTP avg recv delay']), # ('uTP delay', 'buffering delay', 's', 'network delays measured by uTP',
# ('uTP send delay histogram', 'buffering delay', 's', 'send delays measured by uTP', ['uTP avg send delay'], {'type': histogram, 'binwidth': 0.05, 'numbins': 100}), # ['uTP peak send delay','uTP peak recv delay', 'uTP avg send delay', 'uTP avg recv delay']),
# ('uTP recv delay histogram', 'buffering delay', 's', 'receive delays measured by uTP', ['uTP avg recv delay'], {'type': histogram, 'binwidth': 0.05, 'numbins': 100}), # ('uTP send delay histogram', 'buffering delay', 's', 'send delays measured by uTP',
# ['uTP avg send delay'], {'type': histogram, 'binwidth': 0.05, 'numbins': 100}),
# ('uTP recv delay histogram', 'buffering delay', 's', 'receive delays measured by uTP',
# ['uTP avg recv delay'], {'type': histogram, 'binwidth': 0.05, 'numbins': 100}),
('uTP stats', 'num', '', 'number of uTP events', [ \ ('uTP stats', 'num', '', 'number of uTP events', [ \
'utp.utp_packet_loss', \ 'utp.utp_packet_loss', \
@ -459,7 +503,7 @@ reports = [
'sock_bufs.socket_send_size18', \ 'sock_bufs.socket_send_size18', \
'sock_bufs.socket_send_size19', \ 'sock_bufs.socket_send_size19', \
'sock_bufs.socket_send_size20' \ 'sock_bufs.socket_send_size20' \
], {'type': stacked, 'colors':'gradient18'}), ], {'type': stacked, 'colors': 'gradient18'}),
('recv_buffer_sizes', 'num', '', '', [ \ ('recv_buffer_sizes', 'num', '', '', [ \
'sock_bufs.socket_recv_size3', \ 'sock_bufs.socket_recv_size3', \
@ -480,7 +524,7 @@ reports = [
'sock_bufs.socket_recv_size18', \ 'sock_bufs.socket_recv_size18', \
'sock_bufs.socket_recv_size19', \ 'sock_bufs.socket_recv_size19', \
'sock_bufs.socket_recv_size20' \ 'sock_bufs.socket_recv_size20' \
], {'type': stacked, 'colors':'gradient18'}), ], {'type': stacked, 'colors': 'gradient18'}),
('ARC', 'num pieces', '', '', [ \ ('ARC', 'num pieces', '', '', [ \
'disk.arc_mru_ghost_size', \ 'disk.arc_mru_ghost_size', \
@ -547,11 +591,19 @@ reports = [
'ses.num_incoming_reject', \ 'ses.num_incoming_reject', \
], {'type': diff}), ], {'type': diff}),
#somewhat uninteresting stats # somewhat uninteresting stats
# ('peer_dl_rates', 'num', '', 'peers split into download rate buckets', ['peers down 0', 'peers down 0-2', 'peers down 2-5', 'peers down 5-10', 'peers down 50-100', 'peers down 100-'], {'type':stacked, 'colors':'gradient6'}), # ('peer_dl_rates', 'num', '', 'peers split into download rate buckets',
# ('peer_dl_rates2', 'num', '', 'peers split into download rate buckets (only downloading peers)', ['peers down 0-2', 'peers down 2-5', 'peers down 5-10', 'peers down 50-100', 'peers down 100-'], {'type':stacked, 'colors':'gradient6'}), # ['peers down 0', 'peers down 0-2', 'peers down 2-5', 'peers down 5-10', 'peers down 50-100', 'peers down 100-'],
# ('peer_ul_rates', 'num', '', 'peers split into upload rate buckets', ['peers up 0', 'peers up 0-2', 'peers up 2-5', 'peers up 5-10', 'peers up 50-100', 'peers up 100-'], {'type':stacked, 'colors':'gradient6'}), # {'type':stacked, 'colors':'gradient6'}),
# ('peer_ul_rates2', 'num', '', 'peers split into upload rate buckets (only uploading peers)', ['peers up 0-2', 'peers up 2-5', 'peers up 5-10', 'peers up 50-100', 'peers up 100-'], {'type':stacked, 'colors':'gradient6'}), # ('peer_dl_rates2', 'num', '', 'peers split into download rate buckets (only downloading peers)',
# ['peers down 0-2', 'peers down 2-5', 'peers down 5-10', 'peers down 50-100', 'peers down 100-'],
# {'type':stacked, 'colors':'gradient6'}),
# ('peer_ul_rates', 'num', '', 'peers split into upload rate buckets',
# ['peers up 0', 'peers up 0-2', 'peers up 2-5', 'peers up 5-10', 'peers up 50-100', 'peers up 100-'],
# {'type':stacked, 'colors':'gradient6'}),
# ('peer_ul_rates2', 'num', '', 'peers split into upload rate buckets (only uploading peers)',
# ['peers up 0-2', 'peers up 2-5', 'peers up 5-10', 'peers up 50-100', 'peers up 100-'],
# {'type':stacked, 'colors':'gradient6'}),
('piece_picker_invocations', 'invocations of piece picker', '', '', [ \ ('piece_picker_invocations', 'invocations of piece picker', '', '', [ \
'picker.reject_piece_picks', \ 'picker.reject_piece_picks', \
@ -562,7 +614,7 @@ reports = [
'picker.snubbed_piece_picks', \ 'picker.snubbed_piece_picks', \
'picker.interesting_piece_picks', \ 'picker.interesting_piece_picks', \
'picker.hash_fail_piece_picks' \ 'picker.hash_fail_piece_picks' \
], {'type':stacked}), ], {'type': stacked}),
('piece_picker_loops', 'loops through piece picker', '', '', [ \ ('piece_picker_loops', 'loops through piece picker', '', '', [ \
'picker.piece_picker_partial_loops', \ 'picker.piece_picker_partial_loops', \
'picker.piece_picker_suggest_loops', \ 'picker.piece_picker_suggest_loops', \
@ -574,24 +626,29 @@ reports = [
'picker.piece_picker_busy_loops' \ 'picker.piece_picker_busy_loops' \
], {'type': stacked}), ], {'type': stacked}),
# ('picker_full_partials_distribution', 'full pieces', '', '', ['num full partial pieces'], {'type': histogram, 'binwidth': 5, 'numbins': 120}), # ('picker_full_partials_distribution', 'full pieces', '', '', ['num full partial pieces'],
# ('picker_partials_distribution', 'partial pieces', '', '', ['num downloading partial pieces'], {'type': histogram, 'binwidth': 5, 'numbins': 120}) # {'type': histogram, 'binwidth': 5, 'numbins': 120}),
# ('picker_partials_distribution', 'partial pieces', '', '', ['num downloading partial pieces'],
# {'type': histogram, 'binwidth': 5, 'numbins': 120})
] ]
print 'generating graphs' print('generating graphs')
g = 0 g = 0
generations = [] generations = []
scripts = [] scripts = []
print '[%s] %04d\r[' % (' ' * len(reports), g), print('[%s] %04d\r[' % (' ' * len(reports), g), end=' ')
for i in reports: for i in reports:
try: options = i[5] try:
except: options = {} options = i[5]
if not 'type' in options: except BaseException:
options = {}
if 'type' not in options:
options['type'] = line_graph options['type'] = line_graph
script = gen_report(i[0], i[1], i[4], i[2], g, os.path.join(output_dir, 'counters.dat'), options) script = gen_report(i[0], i[1], i[4], i[2], g, os.path.join(output_dir, 'counters.dat'), options)
if script != None: scripts.append(script) if script is not None:
scripts.append(script)
generations.append(g) generations.append(g)
g += 1 g += 1
@ -600,6 +657,5 @@ g += 1
thread_pool.map(plot_fun, scripts) thread_pool.map(plot_fun, scripts)
scripts = [] scripts = []
print '\ngenerating html' print('\ngenerating html')
gen_html(reports, generations) gen_html(reports, generations)

201
tools/parse_utp_log.py
View File

@ -1,6 +1,9 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function
import os, sys, time import os
import sys
from functools import reduce
# usage: parse_log.py log-file [socket-index to focus on] # usage: parse_log.py log-file [socket-index to focus on]
@ -9,46 +12,48 @@ socket_filter = None
if len(sys.argv) >= 3: if len(sys.argv) >= 3:
socket_filter = sys.argv[2].strip() socket_filter = sys.argv[2].strip()
if socket_filter == None: if socket_filter is None:
print "scanning for socket with the most packets" print("scanning for socket with the most packets")
file = open(sys.argv[1], 'rb') file = open(sys.argv[1], 'rb')
sockets = {} sockets = {}
for l in file: for l in file:
if not 'our_delay' in l: continue if 'our_delay' not in l:
continue
try: try:
a = l.strip().split(" ") a = l.strip().split(" ")
socket_index = a[1][:-1] socket_index = a[1][:-1]
except: except BaseException:
continue continue
# msvc's runtime library doesn't prefix pointers # msvc's runtime library doesn't prefix pointers
# with '0x' # with '0x'
# if socket_index[:2] != '0x': # if socket_index[:2] != '0x':
# continue # continue
if socket_index in sockets: if socket_index in sockets:
sockets[socket_index] += 1 sockets[socket_index] += 1
else: else:
sockets[socket_index] = 1 sockets[socket_index] = 1
items = sockets.items() items = list(sockets.items())
items.sort(lambda x, y: y[1] - x[1]) items.sort(lambda x, y: y[1] - x[1])
count = 0 count = 0
for i in items: for i in items:
print '%s: %d' % (i[0], i[1]) print('%s: %d' % (i[0], i[1]))
count += 1 count += 1
if count > 5: break if count > 5:
break
file.close() file.close()
socket_filter = items[0][0] socket_filter = items[0][0]
print '\nfocusing on socket %s' % socket_filter print('\nfocusing on socket %s' % socket_filter)
file = open(sys.argv[1], 'rb') file = open(sys.argv[1], 'rb')
out_file = 'utp.out%s' % socket_filter; out_file = 'utp.out%s' % socket_filter
out = open(out_file, 'wb') out = open(out_file, 'wb')
delay_samples = 'points lc rgb "blue"' delay_samples = 'points lc rgb "blue"'
@ -61,27 +66,27 @@ rtt = 'lines lc rgb "light-blue"'
send_buffer = 'lines lc rgb "light-red"' send_buffer = 'lines lc rgb "light-red"'
metrics = { metrics = {
'our_delay':['our delay (ms)', 'x1y2', delay_samples], 'our_delay': ['our delay (ms)', 'x1y2', delay_samples],
'upload_rate':['send rate (B/s)', 'x1y1', 'lines'], 'upload_rate': ['send rate (B/s)', 'x1y1', 'lines'],
'max_window':['cwnd (B)', 'x1y1', cwnd], 'max_window': ['cwnd (B)', 'x1y1', cwnd],
'target_delay':['target delay (ms)', 'x1y2', target_delay], 'target_delay': ['target delay (ms)', 'x1y2', target_delay],
'cur_window':['bytes in-flight (B)', 'x1y1', window_size], 'cur_window': ['bytes in-flight (B)', 'x1y1', window_size],
'cur_window_packets':['number of packets in-flight', 'x1y2', 'steps'], 'cur_window_packets': ['number of packets in-flight', 'x1y2', 'steps'],
'packet_size':['current packet size (B)', 'x1y2', 'steps'], 'packet_size': ['current packet size (B)', 'x1y2', 'steps'],
'rtt':['rtt (ms)', 'x1y2', rtt], 'rtt': ['rtt (ms)', 'x1y2', rtt],
'off_target':['off-target (ms)', 'x1y2', off_target], 'off_target': ['off-target (ms)', 'x1y2', off_target],
'delay_sum':['delay sum (ms)', 'x1y2', 'steps'], 'delay_sum': ['delay sum (ms)', 'x1y2', 'steps'],
'their_delay':['their delay (ms)', 'x1y2', delay_samples], 'their_delay': ['their delay (ms)', 'x1y2', delay_samples],
'get_microseconds':['clock (us)', 'x1y1', 'steps'], 'get_microseconds': ['clock (us)', 'x1y1', 'steps'],
'wnduser':['advertised window size (B)', 'x1y1', 'steps'], 'wnduser': ['advertised window size (B)', 'x1y1', 'steps'],
'ssthres':['slow-start threshold (B)', 'x1y1', 'steps'], 'ssthres': ['slow-start threshold (B)', 'x1y1', 'steps'],
'delay_base':['delay base (us)', 'x1y1', delay_base], 'delay_base': ['delay base (us)', 'x1y1', delay_base],
'their_delay_base':['their delay base (us)', 'x1y1', delay_base], 'their_delay_base': ['their delay base (us)', 'x1y1', delay_base],
'their_actual_delay':['their actual delay (us)', 'x1y1', delay_samples], 'their_actual_delay': ['their actual delay (us)', 'x1y1', delay_samples],
'actual_delay':['actual_delay (us)', 'x1y1', delay_samples], 'actual_delay': ['actual_delay (us)', 'x1y1', delay_samples],
'send_buffer':['send buffer size (B)', 'x1y1', send_buffer], 'send_buffer': ['send buffer size (B)', 'x1y1', send_buffer],
'recv_buffer':['receive buffer size (B)', 'x1y1', 'lines'] 'recv_buffer': ['receive buffer size (B)', 'x1y1', 'lines']
} }
histogram_quantization = 1 histogram_quantization = 1
@ -99,49 +104,51 @@ delay_histogram = {}
packet_size_histogram = {} packet_size_histogram = {}
window_size = {'0': 0, '1': 0} window_size = {'0': 0, '1': 0}
# [35301484] 0x00ec1190: actual_delay:1021583 our_delay:102 their_delay:-1021345 off_target:297 max_window:2687 upload_rate:18942 delay_base:1021481154 delay_sum:-1021242 target_delay:400 acked_bytes:1441 cur_window:2882 scaled_gain:2.432 # [35301484] 0x00ec1190: actual_delay:1021583 our_delay:102 their_delay:-1021345 off_target:297 max_window:2687
# upload_rate:18942 delay_base:1021481154 delay_sum:-1021242 target_delay:400 acked_bytes:1441 cur_window:2882
# scaled_gain:2.432
counter = 0 counter = 0
print "reading log file" print("reading log file")
for l in file: for line in file:
if "UTP_Connect" in l: if "UTP_Connect" in line:
title = l[:-2] title = line[:-2]
if socket_filter != None: if socket_filter is not None:
title += ' socket: %s' % socket_filter title += ' socket: %s' % socket_filter
else: else:
title += ' sum of all sockets' title += ' sum of all sockets'
continue continue
try: try:
a = l.strip().split(" ") a = line.strip().split(" ")
t = a[0][1:-1] t = a[0][1:-1]
socket_index = a[1][:-1] socket_index = a[1][:-1]
except: except BaseException:
continue continue
# if socket_index[:2] != '0x': # if socket_index[:2] != '0x':
# continue # continue
if socket_filter != None and socket_index != socket_filter: if socket_filter is not None and socket_index != socket_filter:
continue continue
counter += 1 counter += 1
if (counter % 300 == 0): if (counter % 300 == 0):
print "\r%d " % counter, print("\r%d " % counter, end=' ')
if "lost." in l: if "lost." in line:
packet_loss = packet_loss + 1 packet_loss = packet_loss + 1
continue continue
if "Packet timeout" in l: if "Packet timeout" in line:
packet_timeout = packet_timeout + 1 packet_timeout = packet_timeout + 1
continue continue
if "sending packet" in l: if "sending packet" in line:
v = l.split('size:')[1].split(' ')[0] v = line.split('size:')[1].split(' ')[0]
packet_size_histogram[v] = 1 + packet_size_histogram.get(v, 0) packet_size_histogram[v] = 1 + packet_size_histogram.get(v, 0)
if "our_delay:" not in l: if "our_delay:" not in line:
continue continue
# used for Logf timestamps # used for Logf timestamps
@ -161,43 +168,44 @@ for l in file:
begin = t begin = t
t = t - begin t = t - begin
# print time. Convert from milliseconds to seconds # print time. Convert from milliseconds to seconds
print >>out, '%f\t' % (float(t)/1000.), print('%f\t' % (float(t) / 1000.), end=' ', file=out)
#if t > 200000: # if t > 200000:
# break # break
fill_columns = not columns fill_columns = not columns
for i in a[2:]: for i in a[2:]:
try: try:
n, v = i.split(':') n, v = i.split(':')
except: except BaseException:
continue continue
v = float(v) v = float(v)
if n == "our_delay": if n == "our_delay":
bucket = int(v / histogram_quantization) bucket = int(v / histogram_quantization)
delay_histogram[bucket] = 1 + delay_histogram.get(bucket, 0) delay_histogram[bucket] = 1 + delay_histogram.get(bucket, 0)
if not n in metrics: continue if n not in metrics:
continue
if fill_columns: if fill_columns:
columns.append(n) columns.append(n)
if n == "max_window": if n == "max_window":
window_size[socket_index] = v window_size[socket_index] = v
print >>out, '%f\t' % int(reduce(lambda a,b: a+b, window_size.values())), print('%f\t' % int(reduce(lambda a, b: a + b, list(window_size.values()))), end=' ', file=out)
else: else:
print >>out, '%f\t' % v, print('%f\t' % v, end=' ', file=out)
print >>out, float(packet_loss * 8000), float(packet_timeout * 8000) print(float(packet_loss * 8000), float(packet_timeout * 8000), file=out)
packet_loss = 0 packet_loss = 0
packet_timeout = 0 packet_timeout = 0
out.close() out.close()
out = open('%s.histogram' % out_file, 'wb') out = open('%s.histogram' % out_file, 'wb')
for d,f in delay_histogram.iteritems(): for d, f in delay_histogram.items():
print >>out, float(d*histogram_quantization) + histogram_quantization / 2, f print(float(d * histogram_quantization) + histogram_quantization / 2, f, file=out)
out.close() out.close()
out = open('%s_packet_size.histogram' % out_file, 'wb') out = open('%s_packet_size.histogram' % out_file, 'wb')
for d,f in packet_size_histogram.iteritems(): for d, f in packet_size_histogram.items():
print >>out, d, f print(d, f, file=out)
out.close() out.close()
plot = [ plot = [
@ -238,7 +246,7 @@ plot = [
'y2': 'Time (ms)' 'y2': 'Time (ms)'
}, },
{ {
'data': ['their_actual_delay','their_delay_base'], 'data': ['their_actual_delay', 'their_delay_base'],
'title': 'their_delay_base', 'title': 'their_delay_base',
'y1': 'Time (us)', 'y1': 'Time (us)',
'y2': '' 'y2': ''
@ -261,59 +269,60 @@ out = open('utp.gnuplot', 'w+')
files = '' files = ''
#print >>out, 'set xtics 0, 20' # print('set xtics 0, 20', file=out)
print >>out, "set term png size 1280,800" print("set term png size 1280,800", file=out)
print >>out, 'set output "%s.delays.png"' % out_file print('set output "%s.delays.png"' % out_file, file=out)
print >>out, 'set xrange [0:200]' print('set xrange [0:200]', file=out)
print >>out, 'set xlabel "delay (ms)"' print('set xlabel "delay (ms)"', file=out)
print >>out, 'set boxwidth 1' print('set boxwidth 1', file=out)
print >>out, 'set ylabel "number of packets"' print('set ylabel "number of packets"', file=out)
print >>out, 'plot "%s.histogram" using 1:2 with boxes fs solid 0.3' % out_file print('plot "%s.histogram" using 1:2 with boxes fs solid 0.3' % out_file, file=out)
files += out_file + '.delays.png ' files += out_file + '.delays.png '
print >>out, 'set output "%s.packet_sizes.png"' % out_file print('set output "%s.packet_sizes.png"' % out_file, file=out)
print >>out, 'set xrange [0:*]' print('set xrange [0:*]', file=out)
print >>out, 'set xlabel "packet size (B)"' print('set xlabel "packet size (B)"', file=out)
print >>out, 'set boxwidth 1' print('set boxwidth 1', file=out)
print >>out, 'set ylabel "number of packets sent"' print('set ylabel "number of packets sent"', file=out)
print >>out, 'set logscale y' print('set logscale y', file=out)
print >>out, 'plot "%s_packet_size.histogram" using 1:2 with boxes fs solid 0.3' % out_file print('plot "%s_packet_size.histogram" using 1:2 with boxes fs solid 0.3' % out_file, file=out)
print >>out, 'set nologscale y' print('set nologscale y', file=out)
files += out_file + '.packet_sizes.png ' files += out_file + '.packet_sizes.png '
print >>out, "set style data steps" print("set style data steps", file=out)
#print >>out, "set yrange [0:*]" # print("set yrange [0:*]", file=out)
print >>out, "set y2range [*:*]" print("set y2range [*:*]", file=out)
#set hidden3d # set hidden3d
#set title "Peer bandwidth distribution" # set title "Peer bandwidth distribution"
#set xlabel "Ratio" # set xlabel "Ratio"
for p in plot: for p in plot:
print >>out, 'set title "%s %s"' % (p['title'], title) print('set title "%s %s"' % (p['title'], title), file=out)
print >>out, 'set xlabel "time (s)"' print('set xlabel "time (s)"', file=out)
print >>out, 'set ylabel "%s"' % p['y1'] print('set ylabel "%s"' % p['y1'], file=out)
print >>out, "set tics nomirror" print("set tics nomirror", file=out)
print >>out, 'set y2tics' print('set y2tics', file=out)
print >>out, 'set y2label "%s"' % p['y2'] print('set y2label "%s"' % p['y2'], file=out)
print >>out, 'set xrange [0:*]' print('set xrange [0:*]', file=out)
print >>out, "set key box" print("set key box", file=out)
print >>out, "set term png size 1280,800" print("set term png size 1280,800", file=out)
print >>out, 'set output "%s-%s.png"' % (out_file, p['title']) print('set output "%s-%s.png"' % (out_file, p['title']), file=out)
files += '%s-%s.png ' % (out_file, p['title']) files += '%s-%s.png ' % (out_file, p['title'])
comma = '' comma = ''
print >>out, "plot", print("plot", end=' ', file=out)
for c in p['data']: for c in p['data']:
if not c in metrics: continue if c not in metrics:
continue
i = columns.index(c) i = columns.index(c)
print >>out, '%s"%s" using ($1/1000):%d title "%s-%s" axes %s with %s' % (comma, out_file, i + 2, metrics[c][0], metrics[c][1], metrics[c][1], metrics[c][2]), print('%s"%s" using ($1/1000):%d title "%s-%s" axes %s with %s' %
(comma, out_file, i + 2, metrics[c][0], metrics[c][1], metrics[c][1], metrics[c][2]), end=' ', file=out)
comma = ', ' comma = ', '
print >>out, '' print('', file=out)
out.close() out.close()
os.system("gnuplot utp.gnuplot") os.system("gnuplot utp.gnuplot")
os.system("open %s" % files) os.system("open %s" % files)

101
tools/run_benchmark.py
View File

@ -14,81 +14,106 @@ if len(sys.argv) > 1:
ret = os.system('cd ../examples && bjam profile statistics=on %s stage_client_test' % toolset) ret = os.system('cd ../examples && bjam profile statistics=on %s stage_client_test' % toolset)
if ret != 0: if ret != 0:
print 'ERROR: build failed: %d' % ret print('ERROR: build failed: %d' % ret)
sys.exit(1) sys.exit(1)
ret = os.system('cd ../examples && bjam release %s stage_connection_tester' % toolset) ret = os.system('cd ../examples && bjam release %s stage_connection_tester' % toolset)
if ret != 0: if ret != 0:
print 'ERROR: build failed: %d' % ret print('ERROR: build failed: %d' % ret)
sys.exit(1) sys.exit(1)
try: os.remove('.ses_state') try:
except Exception, e: print e os.remove('.ses_state')
try: shutil.rmtree('.resume') except Exception as e:
except Exception, e: print e print(e)
try: shutil.rmtree('cpu_benchmark') try:
except Exception, e: print e shutil.rmtree('.resume')
except Exception as e:
print(e)
try:
shutil.rmtree('cpu_benchmark')
except Exception as e:
print(e)
if not os.path.exists('cpu_benchmark.torrent'): if not os.path.exists('cpu_benchmark.torrent'):
ret = os.system('../examples/connection_tester gen-torrent -s 10000 -n 15 -t cpu_benchmark.torrent') ret = os.system('../examples/connection_tester gen-torrent -s 10000 -n 15 -t cpu_benchmark.torrent')
if ret != 0: if ret != 0:
print 'ERROR: connection_tester failed: %d' % ret print('ERROR: connection_tester failed: %d' % ret)
sys.exit(1) sys.exit(1)
try: shutil.rmtree('t') try:
except: pass shutil.rmtree('t')
except BaseException:
pass
def run_test(name, test_cmd, client_arg, num_peers): def run_test(name, test_cmd, client_arg, num_peers):
output_dir = 'logs_%s' % name output_dir = 'logs_%s' % name
try: shutil.rmtree(output_dir) try:
except: pass shutil.rmtree(output_dir)
try: os.mkdir(output_dir) except BaseException:
except: pass pass
try:
os.mkdir(output_dir)
except BaseException:
pass
port = (int(time.time()) % 50000) + 2000 port = (int(time.time()) % 50000) + 2000
try: shutil.rmtree('session_stats') try:
except: pass shutil.rmtree('session_stats')
try: shutil.rmtree('session_stats_report') except BaseException:
except: pass pass
try:
shutil.rmtree('session_stats_report')
except BaseException:
pass
start = time.time(); start = time.time()
client_cmd = '../examples/client_test -p %d cpu_benchmark.torrent -k -z -H -X -q 120 %s -h -c %d -T %d -C %d -f %s/events.log' \ client_cmd = ('../examples/client_test -p %d cpu_benchmark.torrent -k -z -H -X -q 120 %s'
% (port, client_arg, num_peers*2, num_peers*2, cache_size * 16, output_dir) '-h -c %d -T %d -C %d -f %s/events.log').format(
test_cmd = '../examples/connection_tester %s -c %d -d 127.0.0.1 -p %d -t cpu_benchmark.torrent' % (test_cmd, num_peers, port) port, client_arg, num_peers * 2, num_peers * 2, cache_size * 16, output_dir)
test_cmd = '../examples/connection_tester %s -c %d -d 127.0.0.1 -p %d -t cpu_benchmark.torrent' % (
test_cmd, num_peers, port)
client_out = open('%s/client.out' % output_dir, 'w+') client_out = open('%s/client.out' % output_dir, 'w+')
test_out = open('%s/test.out' % output_dir, 'w+') test_out = open('%s/test.out' % output_dir, 'w+')
print client_cmd print(client_cmd)
c = subprocess.Popen(client_cmd.split(' '), stdout=client_out, stderr=client_out, stdin=subprocess.PIPE) c = subprocess.Popen(client_cmd.split(' '), stdout=client_out, stderr=client_out, stdin=subprocess.PIPE)
time.sleep(2) time.sleep(2)
print test_cmd print(test_cmd)
t = subprocess.Popen(test_cmd.split(' '), stdout=test_out, stderr=test_out) t = subprocess.Popen(test_cmd.split(' '), stdout=test_out, stderr=test_out)
t.wait() t.wait()
end = time.time(); end = time.time()
try: c.communicate('q') try:
except: pass c.communicate('q')
except BaseException:
pass
c.wait() c.wait()
client_out.close(); client_out.close()
test_out.close(); test_out.close()
print 'runtime %d seconds' % (end - start) print('runtime %d seconds' % (end - start))
print 'analyzing proile...' print('analyzing proile...')
os.system('gprof ../examples/client_test >%s/gprof.out' % output_dir) os.system('gprof ../examples/client_test >%s/gprof.out' % output_dir)
print 'generating profile graph...' print('generating profile graph...')
os.system('python gprof2dot.py --strip <%s/gprof.out | dot -Tpng -o %s/cpu_profile.png' % (output_dir, output_dir)) os.system('python gprof2dot.py --strip <%s/gprof.out | dot -Tpng -o %s/cpu_profile.png' % (output_dir, output_dir))
os.system('python parse_session_stats.py session_stats/*.log') os.system('python parse_session_stats.py session_stats/*.log')
try: shutil.move('session_stats_report', '%s/session_stats_report' % output_dir) try:
except: pass shutil.move('session_stats_report', '%s/session_stats_report' % output_dir)
try: shutil.move('session_stats', '%s/session_stats' % output_dir) except BaseException:
except: pass pass
try:
shutil.move('session_stats', '%s/session_stats' % output_dir)
except BaseException:
pass
run_test('download', 'upload', '', 50) run_test('download', 'upload', '', 50)
run_test('upload', 'download', '-G', 20) run_test('upload', 'download', '-G', 20)

64
tools/set_version.py
View File

@ -7,45 +7,51 @@ import re
version = (int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])) version = (int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]))
def v(version): def v(version):
ret = () ret = ()
for i in version: for i in version:
if i < 9: ret = ret + (chr(ord('0') + i),) if i < 9:
else: ret = ret + (chr(ord('A') + i - 10),) ret = ret + (chr(ord('0') + i),)
else:
ret = ret + (chr(ord('A') + i - 10),)
return ret return ret
revision = os.popen('git log -1 --format=format:%h').read().strip() revision = os.popen('git log -1 --format=format:%h').read().strip()
def substitute_file(name): def substitute_file(name):
subst = '' subst = ''
f = open(name) f = open(name)
for l in f: for line in f:
if '#define LIBTORRENT_VERSION_MAJOR' in l and name.endswith('.hpp'): if '#define LIBTORRENT_VERSION_MAJOR' in line and name.endswith('.hpp'):
l = '#define LIBTORRENT_VERSION_MAJOR %d\n' % version[0] line = '#define LIBTORRENT_VERSION_MAJOR %d\n' % version[0]
elif '#define LIBTORRENT_VERSION_MINOR' in l and name.endswith('.hpp'): elif '#define LIBTORRENT_VERSION_MINOR' in line and name.endswith('.hpp'):
l = '#define LIBTORRENT_VERSION_MINOR %d\n' % version[1] line = '#define LIBTORRENT_VERSION_MINOR %d\n' % version[1]
elif '#define LIBTORRENT_VERSION_TINY' in l and name.endswith('.hpp'): elif '#define LIBTORRENT_VERSION_TINY' in line and name.endswith('.hpp'):
l = '#define LIBTORRENT_VERSION_TINY %d\n' % version[2] line = '#define LIBTORRENT_VERSION_TINY %d\n' % version[2]
elif '#define LIBTORRENT_VERSION ' in l and name.endswith('.hpp'): elif '#define LIBTORRENT_VERSION ' in line and name.endswith('.hpp'):
l = '#define LIBTORRENT_VERSION "%d.%d.%d.%d"\n' % (version[0], version[1], version[2], version[3]) line = '#define LIBTORRENT_VERSION "%d.%d.%d.%d"\n' % (version[0], version[1], version[2], version[3])
elif '#define LIBTORRENT_REVISION ' in l and name.endswith('.hpp'): elif '#define LIBTORRENT_REVISION ' in line and name.endswith('.hpp'):
l = '#define LIBTORRENT_REVISION "%s"\n' % revision line = '#define LIBTORRENT_REVISION "%s"\n' % revision
elif 'AC_INIT([libtorrent-rasterbar]' in l and name.endswith('.ac'): elif 'AC_INIT([libtorrent-rasterbar]' in line and name.endswith('.ac'):
l = 'AC_INIT([libtorrent-rasterbar],[%d.%d.%d],[arvid@libtorrent.org],\n' % (version[0], version[1], version[2]) line = 'AC_INIT([libtorrent-rasterbar],[%d.%d.%d],[arvid@libtorrent.org],\n' % (
elif 'set (VERSION ' in l and name.endswith('.txt'): version[0], version[1], version[2])
l = 'set (VERSION "%d.%d.%d")\n' % (version[0], version[1], version[2]) elif 'set (VERSION ' in line and name.endswith('.txt'):
elif ':Version: ' in l and (name.endswith('.rst') or name.endswith('.py')): line = 'set (VERSION "%d.%d.%d")\n' % (version[0], version[1], version[2])
l = ':Version: %d.%d.%d\n' % (version[0], version[1], version[2]) elif ':Version: ' in line and (name.endswith('.rst') or name.endswith('.py')):
elif 'VERSION = ' in l and name.endswith('Jamfile'): line = ':Version: %d.%d.%d\n' % (version[0], version[1], version[2])
l = 'VERSION = %d.%d.%d ;\n' % (version[0], version[1], version[2]) elif 'VERSION = ' in line and name.endswith('Jamfile'):
elif 'version=' in l and name.endswith('setup.py'): line = 'VERSION = %d.%d.%d ;\n' % (version[0], version[1], version[2])
l = "\tversion = '%d.%d.%d',\n" % (version[0], version[1], version[2]) elif 'version=' in line and name.endswith('setup.py'):
elif "version = '" in l and name.endswith('setup.py'): line = "\tversion = '%d.%d.%d',\n" % (version[0], version[1], version[2])
l = "\tversion = '%d.%d.%d',\n" % (version[0], version[1], version[2]) elif "version = '" in line and name.endswith('setup.py'):
elif '"-LT' in l and name.endswith('settings_pack.cpp'): line = "\tversion = '%d.%d.%d',\n" % (version[0], version[1], version[2])
l = re.sub('"-LT[0-9A-Za-z]{4}-"', '"-LT%c%c%c%c-"' % v(version), l) elif '"-LT' in line and name.endswith('settings_pack.cpp'):
line = re.sub('"-LT[0-9A-Za-z]{4}-"', '"-LT%c%c%c%c-"' % v(version), line)
subst += l subst += line
f.close() f.close()
open(name, 'w+').write(subst) open(name, 'w+').write(subst)
@ -60,5 +66,3 @@ substitute_file('src/settings_pack.cpp')
for i in glob.glob('docs/*.rst'): for i in glob.glob('docs/*.rst'):
substitute_file(i) substitute_file(i)
substitute_file('Jamfile') substitute_file('Jamfile')

23
tools/update_copyright.py
View File

@ -1,31 +1,31 @@
#!/usr/bin/env python #!/usr/bin/env python
import os
import sys
import glob import glob
import datetime import datetime
this_year = datetime.date.today().year this_year = datetime.date.today().year
print 'current year: %d' % this_year print('current year: %d' % this_year)
def update_file(name): def update_file(name):
subst = '' subst = ''
f = open(name) f = open(name)
for l in f: for line in f:
if 'Copyright (c) ' in l and 'Arvid Norberg' in l: if 'Copyright (c) ' in line and 'Arvid Norberg' in line:
year_idx = l.index('Copyright (c) ') year_idx = line.index('Copyright (c) ')
first_year = int(l[year_idx + 14: year_idx + 18]) first_year = int(line[year_idx + 14: year_idx + 18])
if first_year != this_year: if first_year != this_year:
if l[year_idx + 18] == '-': if line[year_idx + 18] == '-':
l = l[:year_idx + 19] + str(this_year) + l[year_idx + 23:] line = line[:year_idx + 19] + str(this_year) + line[year_idx + 23:]
else: else:
l = l[:year_idx + 18] + '-' + str(this_year) + l[year_idx + 18:] line = line[:year_idx + 18] + '-' + str(this_year) + line[year_idx + 18:]
subst += l subst += line
f.close() f.close()
open(name, 'w+').write(subst) open(name, 'w+').write(subst)
for i in glob.glob('src/*.cpp') + \ for i in glob.glob('src/*.cpp') + \
glob.glob('include/libtorrent/*.hpp') + \ glob.glob('include/libtorrent/*.hpp') + \
glob.glob('include/libtorrent/extensions/*.hpp') + \ glob.glob('include/libtorrent/extensions/*.hpp') + \
@ -33,4 +33,3 @@ for i in glob.glob('src/*.cpp') + \
glob.glob('src/kademlia/*.cpp') + \ glob.glob('src/kademlia/*.cpp') + \
['COPYING', 'LICENSE', 'AUTHORS']: ['COPYING', 'LICENSE', 'AUTHORS']:
update_file(i) update_file(i)