fixed bugs in http seed connection and added unit test for it

This commit is contained in:
Arvid Norberg 2010-10-17 16:15:32 +00:00
parent 559c4bdf65
commit 3948ca3179
9 changed files with 193 additions and 60 deletions

View File

@ -52,6 +52,7 @@
* added more detailed instrumentation of the disk I/O thread
* fixed bugs in http seed connection and added unit test for it
* fixed error reporting when fallocate fails
* deprecate support for separate proxies for separate kinds of connections

View File

@ -358,6 +358,9 @@ namespace libtorrent
// or with something incorrect, so that we removed the web seed
// immediately, before we disconnected
if (i == m_web_seeds.end()) return;
#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_ERROR_LOGGING
(*m_ses.m_logger) << time_now_string() << " disconnect_web_seed: " << i->url << "\n";
#endif
TORRENT_ASSERT(i->connection);
i->connection = 0;
}

View File

@ -200,6 +200,7 @@ namespace libtorrent
if (error)
{
m_statistics.received_bytes(0, bytes_transferred);
#ifdef TORRENT_VERBOSE_LOGGING
(*m_logger) << "*** http_seed_connection error: "
<< error.message() << "\n";
@ -220,6 +221,7 @@ namespace libtorrent
TORRENT_ASSERT(!m_requests.empty());
if (m_requests.empty())
{
m_statistics.received_bytes(0, bytes_transferred);
disconnect(errors::http_error, 2);
return;
}
@ -239,6 +241,7 @@ namespace libtorrent
if (error)
{
m_statistics.received_bytes(0, bytes_transferred);
disconnect(errors::http_parse_error, 2);
return;
}
@ -270,6 +273,7 @@ namespace libtorrent
m_ses.m_alerts.post_alert(url_seed_alert(t->get_handle(), url()
, error_msg));
}
m_statistics.received_bytes(0, bytes_transferred);
disconnect(errors::http_error, 1);
return;
}
@ -289,6 +293,7 @@ namespace libtorrent
// this means we got a redirection request
// look for the location header
std::string location = m_parser.header("location");
m_statistics.received_bytes(0, bytes_transferred);
if (location.empty())
{
@ -318,6 +323,7 @@ namespace libtorrent
m_response_left = atol(m_parser.header("content-length").c_str());
if (m_response_left == -1)
{
m_statistics.received_bytes(0, bytes_transferred);
// we should not try this server again.
t->remove_web_seed(this);
disconnect(errors::no_content_length, 2);
@ -350,6 +356,7 @@ namespace libtorrent
(*m_logger) << time_now_string() << ": retrying in " << retry_time << " seconds\n";
#endif
m_statistics.received_bytes(0, bytes_transferred);
// temporarily unavailable, retry later
t->retry_web_seed(this, retry_time);
disconnect(errors::http_error, 1);

View File

@ -714,19 +714,24 @@ namespace aux {
update_connections_limit();
update_unchoke_limit();
#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING
(*m_logger) << time_now_string() << " spawning network thread\n";
#endif
m_thread.reset(new thread(boost::bind(&session_impl::main_thread, this)));
}
void session_impl::start()
{
#if defined TORRENT_LOGGING || defined TORRENT_VERBOSE_LOGGING
(*m_logger) << time_now_string() << " *** session start\n";
#endif
// this is where we should set up all async operations. This
// is called from within the network thread as opposed to the
// constructor which is called from the main thread
error_code ec;
m_timer.expires_from_now(milliseconds(m_settings.tick_interval), ec);
m_timer.async_wait(boost::bind(&session_impl::on_tick, this, _1));
TORRENT_ASSERT(!ec);
m_io_service.post(boost::bind(&session_impl::on_tick, this, ec));
int delay = (std::max)(m_settings.local_service_announce_interval
/ (std::max)(int(m_torrents.size()), 1), 1);
@ -744,8 +749,14 @@ namespace aux {
TORRENT_ASSERT(!ec);
#endif
#if defined TORRENT_LOGGING || defined TORRENT_VERBOSE_LOGGING
(*m_logger) << time_now_string() << " open listen port\n";
#endif
// no reuse_address
open_listen_port(false);
#if defined TORRENT_LOGGING || defined TORRENT_VERBOSE_LOGGING
(*m_logger) << time_now_string() << " done starting session\n";
#endif
}
void session_impl::save_state(entry* eptr, boost::uint32_t flags) const
@ -1392,7 +1403,7 @@ namespace aux {
char msg[200];
snprintf(msg, 200, "failed to bind to interface \"%s\": %s"
, print_endpoint(ep).c_str(), ec.message().c_str());
(*m_logger) << msg << "\n";
(*m_logger) << time_now_string() << " " << msg << "\n";
#endif
ec = error_code();
TORRENT_ASSERT_VAL(!ec, ec);
@ -1417,7 +1428,7 @@ namespace aux {
char msg[200];
snprintf(msg, 200, "cannot bind to interface \"%s\": %s"
, print_endpoint(ep).c_str(), ec.message().c_str());
(*m_logger) << msg << "\n";
(*m_logger) << time_now_string() << msg << "\n";
#endif
return listen_socket_t();
}
@ -1431,7 +1442,7 @@ namespace aux {
char msg[200];
snprintf(msg, 200, "cannot listen on interface \"%s\": %s"
, print_endpoint(ep).c_str(), ec.message().c_str());
(*m_logger) << msg << "\n";
(*m_logger) << time_now_string() << msg << "\n";
#endif
return listen_socket_t();
}
@ -1440,7 +1451,7 @@ namespace aux {
m_alerts.post_alert(listen_succeeded_alert(ep));
#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING
(*m_logger) << "listening on: " << ep
(*m_logger) << time_now_string() << " listening on: " << ep
<< " external port: " << s.external_port << "\n";
#endif
return s;
@ -2032,7 +2043,7 @@ namespace aux {
if (e)
{
#if defined TORRENT_LOGGING
#if defined TORRENT_LOGGING || defined TORRENT_VERBOSE_LOGGING
(*m_logger) << "*** TICK TIMER FAILED " << e.message() << "\n";
#endif
::abort();

View File

@ -3337,6 +3337,11 @@ namespace libtorrent
#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING
(*m_ses.m_logger) << time_now_string() << " failed to parse web seed url: " << ec.message() << "\n";
#endif
if (m_ses.m_alerts.should_post<url_seed_alert>())
{
m_ses.m_alerts.post_alert(
url_seed_alert(get_handle(), web->url, ec));
}
// never try it again
m_web_seeds.erase(web);
return;
@ -3384,7 +3389,8 @@ namespace libtorrent
if (web->endpoint.port() != 0)
{
// TODO: we have already resolved this URL, just connect
connect_web_seed(web, web->endpoint);
return;
}
if (m_ses.m_port_filter.access(port) & port_filter::blocked)
@ -3615,10 +3621,14 @@ namespace libtorrent
m_connections.insert(boost::get_pointer(c));
m_ses.m_connections.insert(c);
TORRENT_ASSERT(!web->connection);
web->connection = c.get();
c->start();
#if defined TORRENT_VERBOSE_LOGGING
(*m_ses.m_logger) << time_now_string() << " web seed connection started " << web->url << "\n";
#endif
m_ses.m_half_open.enqueue(
boost::bind(&peer_connection::on_connect, c, _1)
, boost::bind(&peer_connection::on_timeout, c)

View File

@ -65,17 +65,14 @@ namespace libtorrent
, std::string const& auth
, web_seed_entry::headers_t const& extra_headers)
: peer_connection(ses, t, s, remote, peerinfo)
, m_first_request(true)
, m_ssl(false)
, m_external_auth(auth)
, m_extra_headers(extra_headers)
, m_first_request(true)
, m_ssl(false)
, m_body_start(0)
{
INVARIANT_CHECK;
// we want large blocks as well, so
// we can request more bytes at once
request_large_blocks(true);
// we only want left-over bandwidth
set_priority(1);

View File

@ -82,6 +82,12 @@ namespace libtorrent
// from web seeds
prefer_whole_pieces((1024 * 1024) / tor->torrent_file().piece_length());
// we want large blocks as well, so
// we can request more bytes at once
// this setting will merge adjacent requests
// into single larger ones
request_large_blocks(true);
#ifdef TORRENT_VERBOSE_LOGGING
(*m_logger) << "*** web_peer_connection " << url << "\n";
#endif

View File

@ -578,7 +578,7 @@ void on_accept(error_code const& ec)
}
else
{
fprintf(stderr, "accepting connection\n");
// fprintf(stderr, "accepting connection\n");
accept_done = true;
}
}
@ -652,6 +652,7 @@ void web_server_thread(int* port, bool ssl)
{
if (connection_close)
{
// fprintf(stderr, "closing connection\n");
s.close(ec);
connection_close = false;
}
@ -678,7 +679,7 @@ void web_server_thread(int* port, bool ssl)
fprintf(stderr, "accept failed: %s\n", ec.message().c_str());
return;
}
fprintf(stderr, "accepting incoming connection\n");
// fprintf(stderr, "accepting incoming connection\n");
if (!s.is_open()) continue;
#ifdef TORRENT_USE_OPENSSL
@ -708,6 +709,7 @@ void web_server_thread(int* port, bool ssl)
while (!p.finished())
{
TORRENT_ASSERT(len < sizeof(buf));
size_t received = s.read_some(boost::asio::buffer(&buf[len]
, sizeof(buf) - len), ec);
// fprintf(stderr, "read: %d\n", int(received));
@ -738,6 +740,7 @@ void web_server_thread(int* port, bool ssl)
// the Via: header is an indicator of delegate making the request
if (connection == "close" || !via.empty())
{
// fprintf(stderr, "got connection close\n");
connection_close = true;
}
@ -745,6 +748,7 @@ void web_server_thread(int* port, bool ssl)
if (failed)
{
fprintf(stderr, "connection failed\n");
connection_close = true;
break;
}
@ -792,6 +796,59 @@ void web_server_thread(int* port, bool ssl)
write(s, boost::asio::buffer(&buf[0], buf.size()), boost::asio::transfer_all(), ec);
}
if (path.substr(0, 6) == "/seed?")
{
char const* piece = strstr(path.c_str(), "&piece=");
if (piece == 0)
{
fprintf(stderr, "invalid web seed request: %s\n", path.c_str());
break;
}
boost::uint64_t idx = atoi(piece + 7);
char const* range = strstr(path.c_str(), "&ranges=");
int range_end = 0;
int range_start = 0;
if (range)
{
range_start = atoi(range + 8);
range = strchr(range, '-');
if (range == 0)
{
fprintf(stderr, "invalid web seed request: %s\n", path.c_str());
break;
}
range_end = atoi(range + 1);
}
else
{
range_start = 0;
// assume piece size of 16
range_end = 16-1;
}
int size = range_end - range_start + 1;
boost::uint64_t off = idx * 16 + range_start;
std::vector<char> file_buf;
int res = load_file("./tmp1_web_seed/seed", file_buf);
error_code ec;
if (res == -1 || file_buf.empty())
{
send_response(s, ec, 404, "Not Found", 0, 0);
continue;
}
send_response(s, ec, 200, "OK", 0, size);
// fprintf(stderr, "sending %d bytes of payload [%d, %d)\n"
// , size, int(off), int(off + size));
write(s, boost::asio::buffer(&file_buf[0] + off, size)
, boost::asio::transfer_all(), ec);
memmove(buf, buf + offset, len - offset);
len -= offset;
offset = 0;
continue;
}
// fprintf(stderr, ">> serving file %s\n", path.c_str());
std::vector<char> file_buf;
// remove the / from the path

View File

@ -48,7 +48,7 @@ using namespace libtorrent;
// proxy: 0=none, 1=socks4, 2=socks5, 3=socks5_pw 4=http 5=http_pw
void test_transfer(boost::intrusive_ptr<torrent_info> torrent_file
, int proxy, int port, char const* protocol)
, int proxy, int port, char const* protocol, bool url_seed)
{
using namespace libtorrent;
@ -63,7 +63,8 @@ void test_transfer(boost::intrusive_ptr<torrent_info> torrent_file
char const* test_name[] = {"no", "SOCKS4", "SOCKS5", "SOCKS5 password", "HTTP", "HTTP password"};
fprintf(stderr, "\n\n ==== TESTING %s proxy ==== %s ====\n\n\n", test_name[proxy], protocol);
fprintf(stderr, "\n\n ==== TESTING %s proxy ==== %s ==== %s ===\n\n\n"
, test_name[proxy], protocol, url_seed ? "URL seed" : "HTTP seed");
if (proxy)
{
@ -130,8 +131,8 @@ void test_transfer(boost::intrusive_ptr<torrent_info> torrent_file
test_sleep(500);
}
TEST_CHECK(cs.cache_size == 0);
TEST_CHECK(cs.total_used_buffers == 0);
TEST_EQUAL(cs.cache_size, 0);
TEST_EQUAL(cs.total_used_buffers, 0);
std::cerr << "total_size: " << total_size
<< " rate_sum: " << rate_sum
@ -153,63 +154,100 @@ void test_transfer(boost::intrusive_ptr<torrent_info> torrent_file
remove_all("./tmp2_web_seed", ec);
}
int run_suite(char const* protocol)
void save_file(char const* filename, char const* data, int size)
{
error_code ec;
file out(filename, file::write_only, ec);
TEST_CHECK(!ec);
if (ec)
{
fprintf(stderr, "ERROR opening file '%s': %s\n", filename, ec.message().c_str());
return;
}
file::iovec_t b = { (void*)data, size };
out.writev(0, &b, 1, ec);
TEST_CHECK(!ec);
if (ec)
{
fprintf(stderr, "ERROR writing file '%s': %s\n", filename, ec.message().c_str());
return;
}
}
// test_url_seed determines whether to use url-seed or http-seed
int run_suite(char const* protocol, bool test_url_seed)
{
using namespace libtorrent;
error_code ec;
create_directories("./tmp1_web_seed/test_torrent_dir", ec);
int file_sizes[] =
{ 5, 16 - 5, 16, 17, 10, 30, 30, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
,1,1,1,1,1,1,13,65,34,75,2,3,4,5,23,9,43,4,43,6, 4};
char random_data[300000];
std::srand(10);
for (int i = 0; i != sizeof(file_sizes)/sizeof(file_sizes[0]); ++i)
{
std::generate(random_data, random_data + sizeof(random_data), &std::rand);
char filename[200];
snprintf(filename, sizeof(filename), "./tmp1_web_seed/test_torrent_dir/test%d", i);
error_code ec;
file out(filename, file::write_only, ec);
TEST_CHECK(!ec);
if (ec)
{
fprintf(stderr, "ERROR opening file '%s': %s\n", filename, ec.message().c_str());
return 1;
}
file::iovec_t b = { random_data, file_sizes[i]};
out.writev(0, &b, 1, ec);
TEST_CHECK(!ec);
if (ec)
{
fprintf(stderr, "ERROR writing file '%s': %s\n", filename, ec.message().c_str());
return 1;
}
}
file_storage fs;
add_files(fs, "./tmp1_web_seed/test_torrent_dir");
if (test_url_seed)
{
int file_sizes[] =
{ 5, 16 - 5, 16, 17, 10, 30, 30, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
,1,1,1,1,1,1,13,65,34,75,2,3,4,5,23,9,43,4,43,6, 4};
char random_data[300000];
std::srand(10);
for (int i = 0; i != sizeof(file_sizes)/sizeof(file_sizes[0]); ++i)
{
std::generate(random_data, random_data + sizeof(random_data), &std::rand);
char filename[200];
snprintf(filename, sizeof(filename), "./tmp1_web_seed/test_torrent_dir/test%d", i);
save_file(filename, random_data, file_sizes[i]);
}
add_files(fs, "./tmp1_web_seed/test_torrent_dir");
}
else
{
char random_data[10000];
std::srand(10);
std::generate(random_data, random_data + sizeof(random_data), &std::rand);
save_file("./tmp1_web_seed/seed", random_data, sizeof(random_data));
fs.add_file("seed", sizeof(random_data));
}
int port = start_web_server(strcmp(protocol, "https") == 0);
libtorrent::create_torrent t(fs, 16);
char tmp[512];
snprintf(tmp, sizeof(tmp), "%s://127.0.0.1:%d/tmp1_web_seed", protocol, port);
t.add_url_seed(tmp);
if (test_url_seed)
{
snprintf(tmp, sizeof(tmp), "%s://127.0.0.1:%d/tmp1_web_seed", protocol, port);
t.add_url_seed(tmp);
}
else
{
snprintf(tmp, sizeof(tmp), "http://127.0.0.1:%d/seed", port);
t.add_http_seed(tmp);
}
// calculate the hash for all pieces
set_piece_hashes(t, "./tmp1_web_seed", ec);
if (ec)
{
fprintf(stderr, "error creating hashes for test torrent: %s\n"
, ec.message().c_str());
TEST_CHECK(false);
return 0;
}
std::vector<char> buf;
bencode(std::back_inserter(buf), t.generate());
boost::intrusive_ptr<torrent_info> torrent_file(new torrent_info(&buf[0], buf.size(), ec));
for (int i = 0; i < 6; ++i)
test_transfer(torrent_file, i, port, protocol);
test_transfer(torrent_file, i, port, protocol, test_url_seed);
torrent_file->rename_file(0, "./tmp2_web_seed/test_torrent_dir/renamed_test1");
test_transfer(torrent_file, 0, port, protocol);
if (test_url_seed)
{
torrent_file->rename_file(0, "./tmp2_web_seed/test_torrent_dir/renamed_test1");
test_transfer(torrent_file, 0, port, protocol, test_url_seed);
}
stop_web_server();
remove_all("./tmp1_web_seed", ec);
@ -219,10 +257,13 @@ int run_suite(char const* protocol)
int test_main()
{
int ret = 0;
for (int i = 0; i < 2; ++i)
{
#ifdef TORRENT_USE_OPENSSL
ret += run_suite("https");
run_suite("https", i);
#endif
ret += run_suite("http");
run_suite("http", i);
}
return ret;
}