Make tests and tools python3 compatible.

This commit is contained in:
AllSeeingEyeTolledEweSew 2020-05-12 06:04:04 +00:00 committed by Arvid Norberg
parent 0ff6c31af2
commit cbb4f0277e
36 changed files with 708 additions and 388 deletions

View File

@ -219,6 +219,7 @@ script:
fi'
- cd ..
# NB: google's cpplint.py currently does not work with python3...
- 'if [ "$lint" == "1" ]; then
python ~/cpplint.py --extensions=cpp --headers=hpp --filter=-,+runtime/explicit,+whitespace/end_of_line --linelength=90 test/*.{cpp,hpp} src/*.cpp include/libtorrent/*.hpp include/libtorrent/kademlia/*.hpp src/kademlia/*.cpp include/libtorrent/aux_/*.hpp include/libtorrent/extensions/*.hpp simulation/*.{cpp,hpp} tools/*.{cpp,hpp} examples/*.{cpp,hpp};
fi'

View File

@ -43,13 +43,13 @@ install:
- echo %BOOST_ROOT%
- echo %BOOST_BUILD_PATH%
- set PATH=%PATH%;%BOOST_BUILD_PATH%
- ps: '"using msvc : 14.0 ;`nusing gcc : : : <cxxflags>-std=c++11 ;`nusing python : 3.5 : c:\\Python35-x64 : c:\\Python35-x64\\include : c:\\Python35-x64\\libs ;`n" | Set-Content $env:HOMEDRIVE\$env:HOMEPATH\user-config.jam'
- ps: '"using msvc : 14.0 ;`nusing gcc : : : <cxxflags>-std=c++11 ;`nusing python : 3.6 : c:\\Python36-x64 : c:\\Python36-x64\\include : c:\\Python36-x64\\libs ;`n" | Set-Content $env:HOMEDRIVE\$env:HOMEPATH\user-config.jam'
- type %HOMEDRIVE%%HOMEPATH%\user-config.jam
- cd %ROOT_DIRECTORY%
- set PATH=c:\msys64\mingw32\bin;%PATH%
- g++ --version
- set PATH=c:\Python27-x64;%PATH%
- set PYTHON_INTERPRETER=c:\Python27-x64\python.exe
- set PATH=c:\Python36-x64;%PATH%
- set PYTHON_INTERPRETER=c:\Python36-x64\python.exe
- python --version
- echo %ROOT_DIRECTORY%
- cd %BOOST_BUILD_PATH%
@ -106,7 +106,7 @@ test_script:
# we use 64 bit python build
- if defined python (
copy dependencies\*.* .
& c:\Python35-x64\python.exe test.py -b
& c:\Python36-x64\python.exe test.py -b
)
- if defined cmake (

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright Daniel Wallin 2006. Use, modification and distribution is
# subject to the Boost Software License, Version 1.0. (See accompanying

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import sys

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
from distutils.core import setup, Extension

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright Arvid Norberg 2008. Use, modification and distribution is
# subject to the Boost Software License, Version 1.0. (See accompanying
# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

View File

@ -3,7 +3,7 @@
set -e
set -x
python tools/clean.py
python3 tools/clean.py
cd docs
make

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
from __future__ import print_function

View File

@ -1,8 +1,8 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
from __future__ import print_function
import urllib
import urllib.parse
import glob
import os
import sys
@ -1199,8 +1199,8 @@ def print_link(name, target):
def dump_link_targets(indent=''):
global link_targets
ret = '\n'
for l in link_targets:
ret += '%s__ %s\n' % (indent, l)
for link in link_targets:
ret += '%s__ %s\n' % (indent, link)
link_targets = []
return ret
@ -1303,8 +1303,8 @@ def dump_report_issue(h, out):
print(('.. raw:: html\n\n\t<span style="float:right;">[<a style="color:blue;" ' +
'href="http://github.com/arvidn/libtorrent/issues/new?title=docs:{0}&labels=' +
'documentation&body={1}">report issue</a>]</span>\n\n').format(
urllib.quote_plus(h),
urllib.quote_plus('Documentation under heading "' + h + '" could be improved')), file=out)
urllib.parse.quote_plus(h),
urllib.parse.quote_plus('Documentation under heading "' + h + '" could be improved')), file=out)
out = open('reference.rst', 'w+')

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
from __future__ import print_function
f = open('../include/libtorrent/settings_pack.hpp')

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
from __future__ import print_function
counter_types = {}

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import glob
import os
@ -50,9 +50,9 @@ for f in files:
line_no = 0
context_lines = 0
for l in h:
for orig_line in h:
line_no += 1
line = l.strip()
line = orig_line.strip()
if 'TODO:' in line and line.startswith('//'):
line = line.split('TODO:')[1].strip()
state = 'todo'
@ -73,7 +73,7 @@ for f in files:
continue
if state == '':
context.append(html_sanitize(l))
context.append(html_sanitize(orig_line))
if len(context) > 20:
context.pop(0)
continue
@ -85,19 +85,19 @@ for f in files:
else:
state = 'context'
items[-1]['context'] = ''.join(context) + \
'<div style="background: #ffff00" width="100%">' + html_sanitize(l) + '</div>'
'<div style="background: #ffff00" width="100%">' + html_sanitize(orig_line) + '</div>'
context_lines = 1
context.append(html_sanitize(l))
context.append(html_sanitize(orig_line))
if len(context) > 20:
context.pop(0)
continue
if state == 'context':
items[-1]['context'] += html_sanitize(l)
items[-1]['context'] += html_sanitize(orig_line)
context_lines += 1
context.append(html_sanitize(l))
context.append(html_sanitize(orig_line))
if len(context) > 20:
context.pop(0)
if context_lines > 30:

View File

@ -11,17 +11,17 @@ for name in sys.argv[2:]:
sys.stdout.write('\n')
sys.stderr.write('joining %s\n' % name)
f = open(name, 'r')
for l in f:
for line in f:
# strip out the table of contents from subsequent files
if '.. contents::' in l:
if '.. contents::' in line:
in_directive = True
continue
if ':Author:' in l:
if ':Author:' in line:
continue
if ':Version:' in l:
if ':Version:' in line:
continue
if l[0] in ' \t' and in_directive:
if line[0] in ' \t' and in_directive:
continue
in_directive = False
sys.stdout.write(l)
sys.stdout.write(line)

View File

@ -79,14 +79,14 @@ epub: $(TARGETS:=.epub) $(FIGURES:=.png)
all: html pdf
single-page-ref.rst: $(REFERENCE_TARGETS:=.rst)
python join_rst.py $(filter-out reference.rst, $(REFERENCE_TARGETS:=.rst)) >single-page-ref.rst
python3 join_rst.py $(filter-out reference.rst, $(REFERENCE_TARGETS:=.rst)) >single-page-ref.rst
settings.rst hunspell/settings.dic: ../include/libtorrent/settings_pack.hpp hunspell/libtorrent.dic
python gen_settings_doc.py || { rm $@; exit 1; }
python3 gen_settings_doc.py || { rm $@; exit 1; }
cat hunspell/libtorrent.dic >>hunspell/settings.dic
stats_counters.rst: ../src/session_stats.cpp ../include/libtorrent/performance_counters.hpp
python gen_stats_doc.py || { rm $@; exit 1; }
python3 gen_stats_doc.py || { rm $@; exit 1; }
manual.rst: stats_counters.rst
touch manual.rst
@ -98,19 +98,19 @@ manual.rst: stats_counters.rst
#endif
todo.html:gen_todo.py ../src/*.cpp ../include/libtorrent/*.hpp
python gen_todo.py
python3 gen_todo.py
ifneq ($(STAGE),)
cp $@ $(WEB_PATH)/$@
endif
$(REFERENCE_TARGETS:=.rst) plain_text_out.txt:gen_reference_doc.py ../include/libtorrent/*.hpp ../include/libtorrent/kademlia/*.hpp manual.rst tuning.rst settings.rst stats_counters.rst hunspell/settings.dic
python gen_reference_doc.py --plain-output
python3 gen_reference_doc.py --plain-output
spell-check:plain_text_out.txt $(MANUAL_TARGETS:=.html) manual.rst settings.rst
python filter-rst.py manual.rst >manual-plain.txt
python filter-rst.py tuning.rst >tuning-plain.txt
python filter-rst.py settings.rst >settings-plain.txt
python filter-rst.py upgrade_to_1.2.rst >upgrade-1_2-plain.txt
python3 filter-rst.py manual.rst >manual-plain.txt
python3 filter-rst.py tuning.rst >tuning-plain.txt
python3 filter-rst.py settings.rst >settings-plain.txt
python3 filter-rst.py upgrade_to_1.2.rst >upgrade-1_2-plain.txt
hunspell -d hunspell/en_US -p hunspell/libtorrent.dic -l plain_text_out.txt >hunspell-report.txt
hunspell -d hunspell/en_US -p hunspell/libtorrent.dic -l manual-plain.txt >>hunspell-report.txt
hunspell -d hunspell/en_US -p hunspell/libtorrent.dic -l tuning-plain.txt >>hunspell-report.txt

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
from __future__ import print_function

View File

@ -100,17 +100,17 @@ for i in range(101):
for i in range(-10, 200, 20):
for j in range(-1, 1):
for k in range(-1, 1):
for l in range(-1, 1):
for m in range(-1, 1):
messages.append(add_length(struct.pack('>Biiiii', 21, i, j, k, l, m)))
for m in range(-1, 1):
for n in range(-1, 1):
messages.append(add_length(struct.pack('>Biiiii', 21, i, j, k, m, n)))
# hash reject
for i in range(-10, 200, 20):
for j in range(-1, 1):
for k in range(-1, 1):
for l in range(-1, 1):
for m in range(-1, 1):
messages.append(add_length(struct.pack('>Biiiii', 23, i, j, k, l, m)))
for m in range(-1, 1):
for n in range(-1, 1):
messages.append(add_length(struct.pack('>Biiiii', 23, i, j, k, m, n)))
# hash
for i in range(-10, 200, 20):

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import os

View File

@ -1,248 +0,0 @@
# -*- coding: cp1252 -*-
# <PythonProxy.py>
#
# Copyright (c) <2009> <Fábio Domingues - fnds3000 in gmail.com>
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
"""\
Copyright (c) <2009> <Fábio Domingues - fnds3000 in gmail.com> <MIT Licence>
**************************************
*** Python Proxy - A Fast HTTP proxy ***
**************************************
Neste momento este proxy é um Elie Proxy.
Suporta os métodos HTTP:
- OPTIONS;
- GET;
- HEAD;
- POST;
- PUT;
- DELETE;
- TRACE;
- CONENCT.
Suporta:
- Conexões dos cliente em IPv4 ou IPv6;
- Conexões ao alvo em IPv4 e IPv6;
- Conexões todo o tipo de transmissão de dados TCP (CONNECT tunneling),
p.e. ligações SSL, como é o caso do HTTPS.
A fazer:
- Verificar se o input vindo do cliente está correcto;
- Enviar os devidos HTTP erros se não, ou simplesmente quebrar a ligação;
- Criar um gestor de erros;
- Criar ficheiro log de erros;
- Colocar excepções nos sítios onde é previsível a ocorrência de erros,
p.e.sockets e ficheiros;
- Rever tudo e melhorar a estrutura do programar e colocar nomes adequados nas
variáveis e métodos;
- Comentar o programa decentemente;
- Doc Strings.
Funcionalidades futuras:
- Adiconar a funcionalidade de proxy anónimo e transparente;
- Suportar FTP?.
(!) Atenção o que se segue tem efeito em conexões não CONNECT, para estas o
proxy é sempre Elite.
Qual a diferença entre um proxy Elite, Anónimo e Transparente?
- Um proxy elite é totalmente anónimo, o servidor que o recebe não consegue ter
conhecimento da existência do proxy e não recebe o endereço IP do cliente;
- Quando é usado um proxy anónimo o servidor sabe que o cliente está a usar um
proxy mas não sabe o endereço IP do cliente;
É enviado o cabeçalho HTTP "Proxy-agent".
- Um proxy transparente fornece ao servidor o IP do cliente e um informação que
se está a usar um proxy.
São enviados os cabeçalhos HTTP "Proxy-agent" e "HTTP_X_FORWARDED_FOR".
"""
import socket
import select
import sys
import base64
import time
import errno
# Python 3 renamed thread module to _thread
try:
import _thread as thread
except Exception:
import thread
__version__ = '0.1.0 Draft 1'
BUFLEN = 8192
VERSION = 'Python Proxy/' + __version__
HTTPVER = 'HTTP/1.1'
username = None
password = None
class ConnectionHandler:
def __init__(self, connection, address, timeout):
self.client = connection
self.client_buffer = ''
self.timeout = timeout
self.method, self.path, self.protocol = self.get_base_header()
global username
global password
if username is not None:
auth = base64.b64encode(username + ':' + password)
if not 'Proxy-Authorization: Basic ' + auth in self.client_buffer:
print('PROXY - failed authentication: %s' % self.client_buffer)
self.client.send(HTTPVER + ' 401 Authentication Failed\n' +
'Proxy-agent: %s\n\n' % VERSION)
self.client.close()
return
try:
if self.method == 'CONNECT':
self.method_CONNECT()
elif self.method in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT',
'DELETE', 'TRACE'):
self.method_others()
except Exception:
try:
self.client.send(HTTPVER + ' 502 Connection failed\n' +
'Proxy-agent: %s\n\n' % VERSION)
except Exception as e:
print('PROXY - ', e)
self.client.close()
return
self.client.close()
self.target.close()
def get_base_header(self):
retries = 0
while True:
try:
self.client_buffer += self.client.recv(BUFLEN)
except socket.error as e:
err = e.args[0]
if (err == errno.EAGAIN or err == errno.EWOULDBLOCK) and retries < 20:
time.sleep(0.5)
retries += 1
continue
raise e
end = self.client_buffer.find('\r\n\r\n')
if end != -1:
break
line_end = self.client_buffer.find('\n')
print('PROXY - %s' % self.client_buffer[:line_end]) # debug
data = (self.client_buffer[:line_end + 1]).split()
self.client_buffer = self.client_buffer[line_end + 1:]
return data
def method_CONNECT(self):
self._connect_target(self.path)
self.client.send(HTTPVER + ' 200 Connection established\n' +
'Proxy-agent: %s\n\n' % VERSION)
self.client_buffer = ''
self._read_write()
def method_others(self):
self.path = self.path[7:]
i = self.path.find('/')
host = self.path[:i]
path = self.path[i:]
self._connect_target(host)
self.target.send('%s %s %s\n' % (self.method, path, self.protocol) +
self.client_buffer)
self.client_buffer = ''
self._read_write()
def _connect_target(self, host):
i = host.find(':')
if i != -1:
port = int(host[i + 1:])
host = host[:i]
else:
port = 80
(soc_family, _, _, _, address) = socket.getaddrinfo(host, port)[0]
self.target = socket.socket(soc_family)
self.target.connect(address)
def _read_write(self):
time_out_max = self.timeout / 3
socs = [self.client, self.target]
count = 0
while True:
count += 1
(recv, _, error) = select.select(socs, [], socs, 3)
if error:
break
if recv:
for in_ in recv:
data = in_.recv(BUFLEN)
if in_ is self.client:
out = self.target
else:
out = self.client
if data:
out.send(data)
count = 0
if count == time_out_max:
break
def start_server(host='localhost', port=8080, IPv6=False, timeout=10,
handler=ConnectionHandler):
if IPv6:
soc_type = socket.AF_INET6
else:
soc_type = socket.AF_INET
soc = socket.socket(soc_type)
soc.settimeout(120)
print("PROXY - Serving on %s:%d." % (host, port)) # debug
print('python version: %s' % sys.version_info.__str__())
soc.bind((host, port))
soc.listen(0)
while True:
thread.start_new_thread(handler, soc.accept() + (timeout,))
if __name__ == '__main__':
listen_port = 8080
i = 1
while i < len(sys.argv):
if sys.argv[i] == '--port':
listen_port = int(sys.argv[i + 1])
i += 1
elif sys.argv[i] == '--username':
username = sys.argv[i + 1]
i += 1
elif sys.argv[i] == '--password':
password = sys.argv[i + 1]
i += 1
else:
if sys.argv[i] != '--help':
print(('PROXY - unknown option "%s"' % sys.argv[i]))
print('usage: http.py [--port <listen-port>]')
sys.exit(1)
i += 1
start_server(port=listen_port)

550
test/http_proxy.py Executable file
View File

@ -0,0 +1,550 @@
#!/usr/bin/env python3
# The author disclaims copyright to this source code. Please see the
# accompanying UNLICENSE file.
"""A HTTP proxy module suitable for testing.
See http://github.com/AllSeeingEyeTolledEweSew/http_proxy for more information
about this project.
"""
import argparse
import base64
import http.client
import http.server
import select
import socket
import socketserver
import traceback
import urllib.parse
class ChunkError(Exception):
"""Raised while processing chunked encoding, for invalid chunk sizes."""
class _HTTPError(Exception):
"""Raised internally to simplify processing.
Attributes:
code: The HTTP error code (4xx or 5xx) we should return.
message: The "reason"/"message" part we should return in the status
line.
explain: The full body of the error message, usually a traceback.
"""
def __init__(self, code, message=None, explain=None):
super().__init__()
self.code = code
self.message = message
self.explain = explain
def read_to_end_of_chunks(file_like):
"""Reads a chunked-encoded stream from a file-like object.
This will read up to the end of the chunked encoding, including chunk
delimeters, trailers, and the terminal empty line.
The stream will be returned as an iterator of bytes objects. The split
between bytes objects is arbitrary.
Args:
file_like: A file-like object with read() and readline() methods.
Yields:
bytes objects.
Raises:
ChunkError: if an invalid chunk size is encountered.
"""
def inner():
while True:
size_line = file_like.readline()
yield size_line
try:
size = int(size_line, 16)
except ValueError:
raise ChunkError("Invalid chunk size: %r" % size_line)
if size < 0:
raise ChunkError("Invalid chunk size: %d" % size)
if size == 0:
# Allow trailers, if any
while True:
line = file_like.readline()
yield line
if line in (b"\r\n", b"", b"\n"):
return
# Chunk size + crlf
chunk = file_like.read(size + 2)
yield chunk
# Interpret any empty read as a closed connection, and stop
for chunk in inner():
if not chunk:
return
yield chunk
def read_to_limit(file_like, limit, buffer_size):
"""Reads a file-like object up to a number of bytes.
This will read up to the given number of bytes from the given file. The
stream will be returned as an iterator of bytes objects, having size up to
the given buffer_size.
Args:
file_like: A file-like object with a read() method.
limit: The total number of bytes to read.
buffer_size: Read data chunks of this size.
Yields:
bytes objects.
"""
offset = 0
while offset < limit:
amount = min(limit - offset, buffer_size)
buf = file_like.read(amount)
if not buf:
return
yield buf
offset += len(buf)
def read_all(file_like, buffer_size):
"""Reads a file-like object to its end.
This will read an entire file. The stream will be returned as an iterator
of bytes objects, having size up to the given buffer_size.
Args:
file_like: A file-like object with a read() method.
buffer_size: Read data chunks of this size.
Yields:
bytes objects.
"""
while True:
buf = file_like.read(buffer_size)
if not buf:
return
yield buf
class Handler(http.server.BaseHTTPRequestHandler):
"""An HTTP proxy Handler class, for use with http.server classes.
Attributes:
timeout: Timeout value in seconds. Applies to upstream connections,
idle timeouts for CONNECT-method streams, and reading data from
both client and upstream.
basic_auth: If set, proxy will require basic authorization with this
credential.
"""
# BaseHTTPRequestHandler tests this value
protocol_version = "HTTP/1.1"
BUFLEN = 8192
# This is really here to keep pylint happy
close_connection = True
timeout = 30
basic_auth = None
def authorize(self):
"""Returns whether the request is authorized."""
if not self.basic_auth:
return True
header = self.headers.get("Proxy-Authorization", "")
split = header.split(None, 1)
if len(split) != 2:
return False
scheme, credentials = split
if scheme.lower() != "basic":
return False
return credentials == self.basic_auth
def do_auth(self):
"""Fail the request if unauthorized.
Should be called early from the do_* method handler method.
Returns:
False if the request was unauthorized and we sent an error
response, True otherwise.
"""
if self.authorize():
return True
# send_error doesn't let us send headers, so do it by hand
self.log_error("code %d, message %s", 407,
"Proxy authorization required")
self.send_response(407, "Proxy authorization required")
self.send_header("Connection", "close")
self.send_header("Proxy-Authenticate", "Basic")
self.end_headers()
return False
def connect_request(self):
"""Connect to the upstream, for a CONNECT request.
Should be called from the do_CONNECT handler method.
Returns:
A socket connection to the upstream.
Raises:
_HTTPError: If the CONNECT target was invalid, or there was an
error connecting to the upstream.
"""
split = self.path.split(":")
if len(split) != 2:
raise _HTTPError(400, explain="Target must be host:port")
host, port = split
try:
return socket.create_connection((host, port), self.timeout)
except socket.timeout:
raise _HTTPError(504, explain=traceback.format_exc())
except OSError:
raise _HTTPError(502, explain=traceback.format_exc())
def bidirectional_proxy(self, upstream):
"""Forward data between the client and the given upstream.
Should be called from the do_CONNECT method handler.
Runs forever, until either upstream or client close their side of the
connection, or the idle timeout expires.
Args:
upstream: A socket connection to the upstream.
"""
socks = (upstream, self.request)
while True:
(rlist, _, xlist) = select.select(socks, (), socks, self.timeout)
if xlist:
return
if not rlist:
return
for sock in rlist:
data = sock.recv(self.BUFLEN)
if not data:
return
if sock is upstream:
self.request.sendall(data)
else:
upstream.sendall(data)
# pylint:disable=invalid-name
def do_CONNECT(self):
"""Handler for the CONNECT method.
Should be called from the superclass handler logic.
"""
upstream = None
try:
if not self.do_auth():
return
upstream = self.connect_request()
except _HTTPError as err:
self.send_error(err.code, message=err.message, explain=err.explain)
except Exception:
self.log_error("%s", traceback.format_exc())
self.send_error(500, explain=traceback.format_exc())
if upstream is None:
return
self.send_response(200)
self.send_header("Connection", "close")
self.end_headers()
try:
self.bidirectional_proxy(upstream)
except Exception:
self.log_error("%s", traceback.format_exc())
self.close_connection = True
upstream.close()
def proxy_request(self):
"""Forward a normal HTTP request.
Should be called from the do_* handlers for normal HTTP requests (not
CONNECT).
Returns:
A tuple of (HTTPConnection, HTTPResponse).
Raises:
_HTTPError: If the request does not conform to HTTP/1.1
expectations.
"""
url = urllib.parse.urlsplit(self.path)
if url.scheme != "http":
raise _HTTPError(400, message="Target scheme is not http")
# We need to read only the expected amount from the client
# https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.4
if self.headers.get("Transfer-Encoding", "identity") != "identity":
# BaseHTTPHandler never parses chunked encoding itself
message_body = read_to_end_of_chunks(self.rfile)
elif "Content-Length" in self.headers:
try:
length = int(self.headers["Content-Length"])
except ValueError:
raise _HTTPError(411)
message_body = read_to_limit(self.rfile, length, self.BUFLEN)
elif self.command not in ("PATCH", "POST", "PUT"):
# Not expecting a body
message_body = None
else:
raise _HTTPError(411)
try:
upstream = http.client.HTTPConnection(url.netloc,
timeout=self.timeout)
except http.client.InvalidURL as exc:
raise _HTTPError(400,
message=str(exc),
explain=traceback.format_exc())
path = urllib.parse.urlunsplit(("", "", url.path, url.query, ""))
upstream.putrequest(self.command,
path,
skip_host=True,
skip_accept_encoding=True)
connection_tokens = []
filter_headers = set(
("proxy-authorization", "connection", "keep-alive"))
pass_headers = set(("transfer-encoding", "te", "trailer"))
if "Connection" in self.headers:
request_connection_tokens = [
token.strip() for token in self.headers["Connection"].split(",")
]
else:
request_connection_tokens = []
for token in request_connection_tokens:
# Better parsing than base class, I think
if token.lower() == "keep-alive":
self.close_connection = False
filter_headers.add(token.lower())
elif token.lower() == "close":
self.close_connection = True
elif token.lower() in pass_headers:
connection_tokens.append(token)
else:
filter_headers.add(token.lower())
for name, value in self.headers.items():
if name.lower() in filter_headers:
continue
upstream.putheader(name, value)
# No pipelineing to upstream
if "close" not in connection_tokens:
connection_tokens.append("close")
upstream.putheader("Connection", ", ".join(connection_tokens))
try:
# Never use encode_chunked here, as we pass through
# transfer-encoding from the client.
# Calls socket.create_connection, so catch socket exceptions here.
upstream.endheaders(message_body=message_body)
# This parses the upstream response line and headers
return (upstream, upstream.getresponse())
except socket.timeout:
raise _HTTPError(504, explain=traceback.format_exc())
except (OSError, http.client.HTTPException):
upstream.close()
raise _HTTPError(502, explain=traceback.format_exc())
except ChunkError as exc:
upstream.close()
raise _HTTPError(400,
message=str(exc),
explain=traceback.format_exc())
def proxy_response(self, response):
"""Forwards an upstream response back to the client.
Should be called from the do_* handlers for normal HTTP requests (not
CONNECT).
Args:
response: An HTTPResponse from upstream.
"""
# send_response supplies some headers unconditionally
self.log_request(response.code)
self.send_response_only(response.code, response.reason)
connection_tokens = []
filter_headers = set(
("proxy-authorization", "connection", "keep-alive"))
pass_headers = set(("transfer-encoding", "te", "trailer"))
if response.getheader("Connection"):
response_connection_tokens = [
token.strip()
for token in response.getheader("Connection").split(",")
]
else:
response_connection_tokens = []
for token in response_connection_tokens:
if token.lower() == "close":
continue
if token.lower() in pass_headers:
connection_tokens.append(token)
else:
filter_headers.add(token.lower())
# Close the connection if the client requested it
if self.close_connection:
connection_tokens.append("close")
for name, value in response.getheaders():
if name.lower() in filter_headers:
continue
self.send_header(name, value)
if connection_tokens:
self.send_header("Connection", ", ".join(connection_tokens))
self.end_headers()
# HTTPResponse.read() will decode chunks, but we want to pass them
# through. Use this "hack" to pass through the encoding, and just use
# our own reader. Field is undocumented, but public.
response.chunked = False
# https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.4
if response.getheader("Transfer-Encoding", "identity") != "identity":
body = read_to_end_of_chunks(response)
elif response.getheader("Content-Length"):
try:
length = int(response.getheader("Content-Length"))
except ValueError:
body = read_all(response, self.BUFLEN)
else:
body = read_to_limit(response, length, self.BUFLEN)
else:
# May hang if the server wrongly keeps the connection alive
body = read_all(response, self.BUFLEN)
for chunk in body:
self.wfile.write(chunk)
def do_proxy(self):
"""Handles proxying any normal HTTP request (not CONNECT).
This method is a generic implementation of the do_* handlers for normal
HTTP methods.
"""
upstream = None
response = None
try:
if not self.do_auth():
return
upstream, response = self.proxy_request()
except _HTTPError as exc:
self.send_error(exc.code, message=exc.message, explain=exc.explain)
except Exception:
self.log_error("%s", traceback.format_exc())
self.send_error(500, explain=traceback.format_exc())
if not response:
return
try:
self.proxy_response(response)
except Exception:
self.log_error("%s", traceback.format_exc())
self.close_connection = True
upstream.close()
# pylint:disable=invalid-name
def do_GET(self):
"""Handles a proxy GET request."""
self.do_proxy()
# pylint:disable=invalid-name
def do_POST(self):
"""Handles a proxy POST request."""
self.do_proxy()
# pylint:disable=invalid-name
def do_PUT(self):
"""Handles a proxy PUT request."""
self.do_proxy()
# pylint:disable=invalid-name
def do_PATCH(self):
"""Handles a proxy PATCH request."""
self.do_proxy()
# pylint:disable=invalid-name
def do_HEAD(self):
"""Handles a proxy HEAD request."""
self.do_proxy()
# pylint:disable=invalid-name
def do_OPTIONS(self):
"""Handles a proxy OPTIONS request."""
self.do_proxy()
# pylint:disable=invalid-name
def do_DELETE(self):
"""Handles a proxy DELETE request."""
self.do_proxy()
# pylint:disable=invalid-name
def do_TRACE(self):
"""Handles a proxy TRACE request."""
self.do_proxy()
class _ThreadingHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
daemon_threads = True
class Main:
def __init__(self):
self.parser = argparse.ArgumentParser("Simple HTTP proxy")
self.parser.add_argument("--port", type=int, default=8080)
self.parser.add_argument("--basic-auth")
self.parser.add_argument("--timeout", type=int, default=30)
self.parser.add_argument("--bind-host", default="localhost")
self.args = None
self.server = None
self.address = None
def run(self):
"""Command-line entry point for http_proxy."""
self.args = self.parser.parse_args()
self.address = (self.args.bind_host, self.args.port)
if self.args.basic_auth:
Handler.basic_auth = base64.b64encode(
self.args.basic_auth.encode()).decode()
else:
Handler.basic_auth = None
self.server = _ThreadingHTTPServer(self.address, Handler)
self.server.serve_forever()
def shutdown(self):
if self.server is not None:
self.server.shutdown()
if __name__ == "__main__":
Main().run()

View File

@ -604,7 +604,7 @@ std::string get_python()
if (sz == buf.size() - 1) return buf.data();
}
#endif
return "python";
return "python3";
}
}
@ -657,12 +657,12 @@ int start_proxy(int proxy_type)
break;
case settings_pack::http:
type = "http";
cmd = "../http.py";
cmd = "../http_proxy.py";
break;
case settings_pack::http_pw:
type = "http";
auth = " --username testuser --password testpass";
cmd = "../http.py";
auth = " --basic-auth testuser:testpass";
cmd = "../http_proxy.py";
break;
}
std::string python_exe = get_python();

View File

@ -1,18 +1,14 @@
#!/usr/bin/env python
#!/usr/bin/env python3
"""Minimal non-feature complete socks proxy"""
from __future__ import print_function
import socket
from struct import pack, unpack
import threading
import sys
import traceback
# Python 3 renamed SocketServer to socketserver
try:
from socketserver import StreamRequestHandler, ThreadingTCPServer
except Exception:
from SocketServer import StreamRequestHandler, ThreadingTCPServer
from socketserver import StreamRequestHandler, ThreadingTCPServer
def debug(s):
@ -34,15 +30,15 @@ class MyTCPServer(ThreadingTCPServer):
CLOSE = object()
VERSION = '\x05'
NOAUTH = '\x00'
USERPASS = '\x02'
CONNECT = '\x01'
UDP_ASSOCIATE = '\x03'
IPV4 = '\x01'
IPV6 = '\x04'
DOMAIN_NAME = '\x03'
SUCCESS = '\x00'
VERSION = b'\x05'
NOAUTH = b'\x00'
USERPASS = b'\x02'
CONNECT = b'\x01'
UDP_ASSOCIATE = b'\x03'
IPV4 = b'\x01'
IPV6 = b'\x04'
DOMAIN_NAME = b'\x03'
SUCCESS = b'\x00'
password = None
username = None
@ -61,9 +57,9 @@ def send(dest, msg):
return dest.sendall(msg)
def recv(source, buffer):
data = source.recv(buffer)
if data == '':
def recv(source, n):
data = source.recv(n)
if data == b'':
return CLOSE
else:
return data
@ -93,21 +89,29 @@ class SocksHandler(StreamRequestHandler):
self.server.close_request(self.request)
def read(self, n):
data = ''
data = b''
while len(data) < n:
extra = self.rfile.read(n)
if extra == '':
if extra == b'':
raise Exception('Connection closed')
data += extra
return data
def handle(self):
try:
self.inner_handle()
except Exception:
error("Unhandled exception")
traceback.print_exc(file=sys.stdout)
sys.stdout.flush()
def inner_handle(self):
# IMRPOVEMENT: Report who requests are from in logging
# IMPROVEMENT: Timeout on client
debug('Connection - authenticating')
version = self.read(1)
if allow_v4 and version == '\x04':
if allow_v4 and version == b'\x04':
cmd = self.read(1)
if cmd != CONNECT and cmd != UDP_ASSOCIATE:
error('Only supports connect and udp-associate method not (%r) closing' % cmd)
@ -120,9 +124,9 @@ class SocksHandler(StreamRequestHandler):
raw_dest_address = self.read(4)
dest_address = '.'.join(map(str, unpack('>4B', raw_dest_address)))
user_id = ''
user_id = b''
c = self.read(1)
while c != '\0':
while c != b'\0':
user_id += c
c = self.read(1)
@ -137,7 +141,7 @@ class SocksHandler(StreamRequestHandler):
forward(self.request, outbound_sock, 'client')
return
if version != '\x05':
if version != b'\x05':
error('Wrong version number (%r) closing...' % version)
self.close_request()
return
@ -154,7 +158,7 @@ class SocksHandler(StreamRequestHandler):
elif USERPASS in method_list:
self.send_user_pass_auth_method()
auth_version = self.read(1)
if auth_version != '\x01':
if auth_version != b'\x01':
error('Wrong sub-negotiation version number (%r) closing...' % version)
self.close_request()
return
@ -163,7 +167,7 @@ class SocksHandler(StreamRequestHandler):
pwd_len = ord(self.read(1))
pwd = self.read(pwd_len)
if usr_name != username or pwd != password:
if usr_name != username.encode() or pwd != password.encode():
error('Invalid username or password')
self.close_request()
return
@ -175,14 +179,17 @@ class SocksHandler(StreamRequestHandler):
return
# If we were authenticating it would go here
version, cmd, zero, address_type = self.read(4)
if version != '\x05':
version = self.read(1)
cmd = self.read(1)
zero = self.read(1)
address_type = self.read(1)
if version != b'\x05':
error('Wrong version number (%r) closing...' % version)
self.close_request()
elif cmd != CONNECT and cmd != UDP_ASSOCIATE:
error('Only supports connect and udp-associate method not (%r) closing' % cmd)
self.close_request()
elif zero != '\x00':
elif zero != b'\x00':
error('Mangled request. Reserved field (%r) is not null' % zero)
self.close_request()
@ -211,6 +218,7 @@ class SocksHandler(StreamRequestHandler):
out_address = socket.getaddrinfo(dest_address, dest_port)[0][4]
except Exception as e:
error('%s' % e)
traceback.print_exc(file=sys.stdout)
return
if cmd == UDP_ASSOCIATE:
@ -223,6 +231,7 @@ class SocksHandler(StreamRequestHandler):
outbound_sock.connect(out_address)
except Exception as e:
error('%s' % e)
traceback.print_exc(file=sys.stdout)
return
if address_type == IPV6:
@ -235,10 +244,11 @@ class SocksHandler(StreamRequestHandler):
forward(self.request, outbound_sock, 'client')
except Exception as e:
error('%s' % e)
traceback.print_exc(file=sys.stdout)
def send_reply_v4(self, xxx_todo_changeme):
(bind_addr, bind_port) = xxx_todo_changeme
self.wfile.write('\0\x5a\0\0\0\0\0\0')
self.wfile.write(b'\0\x5a\0\0\0\0\0\0')
self.wfile.flush()
def send_reply(self, xxx_todo_changeme1):
@ -246,7 +256,7 @@ class SocksHandler(StreamRequestHandler):
bind_tuple = tuple(map(int, bind_addr.split('.')))
full_address = bind_tuple + (bind_port,)
debug('Setting up forwarding port %r' % (full_address,))
msg = pack('>cccc4BH', VERSION, SUCCESS, '\x00', IPV4, *full_address)
msg = pack('>cccc4BH', VERSION, SUCCESS, b'\x00', IPV4, *full_address)
self.wfile.write(msg)
def send_reply6(self, xxx_todo_changeme2):
@ -254,23 +264,23 @@ class SocksHandler(StreamRequestHandler):
bind_tuple = tuple([int(x, 16) for x in bind_addr.split(':')])
full_address = bind_tuple + (bind_port,)
debug('Setting up forwarding port %r' % (full_address,))
msg = pack('>cccc8HH', VERSION, SUCCESS, '\x00', IPV6, *full_address)
msg = pack('>cccc8HH', VERSION, SUCCESS, b'\x00', IPV6, *full_address)
self.wfile.write(msg)
def send_no_method(self):
self.wfile.write('\x05\xff')
self.wfile.write(b'\x05\xff')
self.close_request()
def send_no_auth_method(self):
self.wfile.write('\x05\x00')
self.wfile.write(b'\x05\x00')
self.wfile.flush()
def send_user_pass_auth_method(self):
self.wfile.write('\x05\x02')
self.wfile.write(b'\x05\x02')
self.wfile.flush()
def send_authenticated(self):
self.wfile.write('\x01\x00')
self.wfile.write(b'\x01\x00')
self.wfile.flush()

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import sys
import os
@ -6,16 +6,9 @@ import ssl
import gzip
import base64
import socket
import traceback
# Python 3 has moved {Simple,Base}HTTPServer to http module
try:
# Remove '.' from sys.path or we try to import the http.py module
# which is not what we want.
sys.path = sys.path[1:]
from http.server import HTTPServer, BaseHTTPRequestHandler
except ImportError:
from SimpleHTTPServer import SimpleHTTPRequestHandler as BaseHTTPRequestHandler
from BaseHTTPServer import HTTPServer
from http.server import HTTPServer, BaseHTTPRequestHandler
chunked_encoding = False
keepalive = True
@ -40,7 +33,15 @@ class http_server_with_timeout(HTTPServer):
class http_handler(BaseHTTPRequestHandler):
def do_GET(s):
def do_GET(self):
try:
self.inner_do_GET()
except Exception:
print('EXCEPTION')
traceback.print_exc(file=sys.stdout)
sys.stdout.flush()
def inner_do_GET(s):
print('INCOMING-REQUEST [from: {}]: {}'.format(s.request.getsockname(), s.requestline))
print(s.headers)
@ -61,7 +62,7 @@ class http_handler(BaseHTTPRequestHandler):
passed = False
if 'Authorization' in s.headers:
auth = s.headers['Authorization']
passed = auth == 'Basic %s' % base64.b64encode('testuser:testpass')
passed = auth == 'Basic %s' % base64.b64encode(b'testuser:testpass').decode()
if not passed:
s.send_response(401)
@ -89,11 +90,11 @@ class http_handler(BaseHTTPRequestHandler):
s.end_headers()
elif s.path.startswith('/announce'):
s.send_response(200)
response = 'd8:intervali1800e8:completei1e10:incompletei1e' + \
'12:min intervali' + min_interval + 'e' + \
'5:peers12:AAAABBCCCCDD' + \
'6:peers618:EEEEEEEEEEEEEEEEFF' + \
'e'
response = b'd8:intervali1800e8:completei1e10:incompletei1e' + \
b'12:min intervali' + min_interval.encode() + b'e' + \
b'5:peers12:AAAABBCCCCDD' + \
b'6:peers618:EEEEEEEEEEEEEEEEFF' + \
b'e'
s.send_header("Content-Length", "%d" % len(response))
s.send_header("Connection", "close")
s.end_headers()
@ -126,6 +127,7 @@ class http_handler(BaseHTTPRequestHandler):
s.wfile.write(data)
except Exception as e:
print('FILE ERROR: ', filename, e)
traceback.print_exc(file=sys.stdout)
sys.stdout.flush()
s.send_response(404)
s.send_header("Content-Length", "0")
@ -174,25 +176,30 @@ class http_handler(BaseHTTPRequestHandler):
while length > 0:
to_send = min(length, 0x900)
if chunked_encoding:
s.wfile.write('%x\r\n' % to_send)
s.wfile.write(b'%x\r\n' % to_send)
data = f.read(to_send)
print('read %d bytes' % to_send)
sys.stdout.flush()
s.wfile.write(data)
if chunked_encoding:
s.wfile.write('\r\n')
s.wfile.write(b'\r\n')
length -= to_send
print('sent %d bytes (%d bytes left)' % (len(data), length))
sys.stdout.flush()
if chunked_encoding:
s.wfile.write('0\r\n\r\n')
s.wfile.write(b'0\r\n\r\n')
except Exception as e:
print('FILE ERROR: ', filename, e)
traceback.print_exc(file=sys.stdout)
sys.stdout.flush()
s.send_response(404)
s.send_header("Content-Length", "0")
s.end_headers()
print("...DONE")
sys.stdout.flush()
s.wfile.flush()
if __name__ == '__main__':
port = int(sys.argv[1])

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
import os

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
import socket

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
import os
@ -60,22 +60,22 @@ def print_classes(out, classes, keyword):
current_file = ''
# [(file, decl), ...]
classes = [(l.split(':')[0].strip(), ':'.join(l.split(':')[1:]).strip()) for l in classes]
classes = [(x.split(':')[0].strip(), ':'.join(x.split(':')[1:]).strip()) for x in classes]
# we only care about header files
# ignore the forward header itself, that's the one we're generating
# also ignore any header in the aux_ directory, those are private
classes = [l for l in classes if l[0].endswith('.hpp') and not l[0].endswith('/fwd.hpp') and '/aux_/' not in l[0]]
classes = [x for x in classes if x[0].endswith('.hpp') and not x[0].endswith('/fwd.hpp') and '/aux_/' not in x[0]]
namespaces = ['TORRENT_VERSION_NAMESPACE_2',
'TORRENT_VERSION_NAMESPACE_2_END']
# only include classes with the right kind of export
classes = [
l for l in classes if l[1] in namespaces or (
l[1].split(' ')[0] in [
x for x in classes if x[1] in namespaces or (
x[1].split(' ')[0] in [
'class',
'struct'] and l[1].split(' ')[1] == keyword)]
'struct'] and x[1].split(' ')[1] == keyword)]
# collapse empty namespaces
classes2 = []

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
from __future__ import print_function

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
from __future__ import print_function

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
import sys

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
# this is meant to parse the dht_lookups.log generated by parse_dht_log.py

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
from __future__ import print_function
@ -24,14 +24,14 @@ for p in glob.iglob(os.path.join(sys.argv[1], '*.log')):
uploaded_blocks = 0
downloaded_blocks = 0
for l in f:
t = l.split(': ')[0].split('.')[0]
for line in f:
t = line.split(': ')[0].split('.')[0]
log_line = False
if ' ==> PIECE' in l:
if ' ==> PIECE' in line:
uploaded_blocks += 1
log_line = True
if ' <== PIECE' in l:
if ' <== PIECE' in line:
downloaded_blocks += 1
log_line = True

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
import sys
@ -32,11 +32,11 @@ except Exception:
fun_samples = {}
for l in f:
if 'Sort by top of stack' in l:
for line in f:
if 'Sort by top of stack' in line:
break
indentation, samples, fun = parse_line(l)
indentation, samples, fun = parse_line(line)
if samples < sample_limit:
continue
if fold != -1 and indentation > fold:

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
from __future__ import print_function

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
from __future__ import print_function
@ -20,12 +20,12 @@ if socket_filter is None:
sockets = {}
for l in file:
if 'our_delay' not in l:
for line in file:
if 'our_delay' not in line:
continue
try:
a = l.strip().split(" ")
a = line.strip().split(" ")
socket_index = a[1][:-1]
except Exception:
continue

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
import os
@ -113,7 +113,7 @@ def run_test(name, test_cmd, client_arg, num_peers):
except Exception:
print('please install gprof2dot and dot:\nsudo pip install gprof2dot\nsudo apt install graphviz')
os.system('python parse_session_stats.py %s/events.log' % output_dir)
os.system('python3 parse_session_stats.py %s/events.log' % output_dir)
try:
shutil.move('session_stats_report', '%s/session_stats_report' % output_dir)

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
import os

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
import glob