diff --git a/ChangeLog b/ChangeLog index 3afcf4c43..37c34eede 100644 --- a/ChangeLog +++ b/ChangeLog @@ -52,6 +52,7 @@ * added more detailed instrumentation of the disk I/O thread + * fixed bugs in http seed connection and added unit test for it * fixed error reporting when fallocate fails * deprecate support for separate proxies for separate kinds of connections diff --git a/include/libtorrent/torrent.hpp b/include/libtorrent/torrent.hpp index 5b14f83ba..b9d2cc17e 100644 --- a/include/libtorrent/torrent.hpp +++ b/include/libtorrent/torrent.hpp @@ -358,6 +358,9 @@ namespace libtorrent // or with something incorrect, so that we removed the web seed // immediately, before we disconnected if (i == m_web_seeds.end()) return; +#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_ERROR_LOGGING + (*m_ses.m_logger) << time_now_string() << " disconnect_web_seed: " << i->url << "\n"; +#endif TORRENT_ASSERT(i->connection); i->connection = 0; } diff --git a/src/http_seed_connection.cpp b/src/http_seed_connection.cpp index 2bf1d7fa7..d5bb8337c 100644 --- a/src/http_seed_connection.cpp +++ b/src/http_seed_connection.cpp @@ -200,6 +200,7 @@ namespace libtorrent if (error) { + m_statistics.received_bytes(0, bytes_transferred); #ifdef TORRENT_VERBOSE_LOGGING (*m_logger) << "*** http_seed_connection error: " << error.message() << "\n"; @@ -220,6 +221,7 @@ namespace libtorrent TORRENT_ASSERT(!m_requests.empty()); if (m_requests.empty()) { + m_statistics.received_bytes(0, bytes_transferred); disconnect(errors::http_error, 2); return; } @@ -239,6 +241,7 @@ namespace libtorrent if (error) { + m_statistics.received_bytes(0, bytes_transferred); disconnect(errors::http_parse_error, 2); return; } @@ -270,6 +273,7 @@ namespace libtorrent m_ses.m_alerts.post_alert(url_seed_alert(t->get_handle(), url() , error_msg)); } + m_statistics.received_bytes(0, bytes_transferred); disconnect(errors::http_error, 1); return; } @@ -289,6 +293,7 @@ namespace libtorrent // this means we got a redirection request // look for the location header std::string location = m_parser.header("location"); + m_statistics.received_bytes(0, bytes_transferred); if (location.empty()) { @@ -318,6 +323,7 @@ namespace libtorrent m_response_left = atol(m_parser.header("content-length").c_str()); if (m_response_left == -1) { + m_statistics.received_bytes(0, bytes_transferred); // we should not try this server again. t->remove_web_seed(this); disconnect(errors::no_content_length, 2); @@ -350,6 +356,7 @@ namespace libtorrent (*m_logger) << time_now_string() << ": retrying in " << retry_time << " seconds\n"; #endif + m_statistics.received_bytes(0, bytes_transferred); // temporarily unavailable, retry later t->retry_web_seed(this, retry_time); disconnect(errors::http_error, 1); diff --git a/src/session_impl.cpp b/src/session_impl.cpp index 97da08750..d27f7d0cb 100644 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -714,19 +714,24 @@ namespace aux { update_connections_limit(); update_unchoke_limit(); +#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING + (*m_logger) << time_now_string() << " spawning network thread\n"; +#endif m_thread.reset(new thread(boost::bind(&session_impl::main_thread, this))); } void session_impl::start() { +#if defined TORRENT_LOGGING || defined TORRENT_VERBOSE_LOGGING + (*m_logger) << time_now_string() << " *** session start\n"; +#endif + // this is where we should set up all async operations. This // is called from within the network thread as opposed to the // constructor which is called from the main thread error_code ec; - m_timer.expires_from_now(milliseconds(m_settings.tick_interval), ec); - m_timer.async_wait(boost::bind(&session_impl::on_tick, this, _1)); - TORRENT_ASSERT(!ec); + m_io_service.post(boost::bind(&session_impl::on_tick, this, ec)); int delay = (std::max)(m_settings.local_service_announce_interval / (std::max)(int(m_torrents.size()), 1), 1); @@ -744,8 +749,14 @@ namespace aux { TORRENT_ASSERT(!ec); #endif +#if defined TORRENT_LOGGING || defined TORRENT_VERBOSE_LOGGING + (*m_logger) << time_now_string() << " open listen port\n"; +#endif // no reuse_address open_listen_port(false); +#if defined TORRENT_LOGGING || defined TORRENT_VERBOSE_LOGGING + (*m_logger) << time_now_string() << " done starting session\n"; +#endif } void session_impl::save_state(entry* eptr, boost::uint32_t flags) const @@ -1392,7 +1403,7 @@ namespace aux { char msg[200]; snprintf(msg, 200, "failed to bind to interface \"%s\": %s" , print_endpoint(ep).c_str(), ec.message().c_str()); - (*m_logger) << msg << "\n"; + (*m_logger) << time_now_string() << " " << msg << "\n"; #endif ec = error_code(); TORRENT_ASSERT_VAL(!ec, ec); @@ -1417,7 +1428,7 @@ namespace aux { char msg[200]; snprintf(msg, 200, "cannot bind to interface \"%s\": %s" , print_endpoint(ep).c_str(), ec.message().c_str()); - (*m_logger) << msg << "\n"; + (*m_logger) << time_now_string() << msg << "\n"; #endif return listen_socket_t(); } @@ -1431,7 +1442,7 @@ namespace aux { char msg[200]; snprintf(msg, 200, "cannot listen on interface \"%s\": %s" , print_endpoint(ep).c_str(), ec.message().c_str()); - (*m_logger) << msg << "\n"; + (*m_logger) << time_now_string() << msg << "\n"; #endif return listen_socket_t(); } @@ -1440,7 +1451,7 @@ namespace aux { m_alerts.post_alert(listen_succeeded_alert(ep)); #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING - (*m_logger) << "listening on: " << ep + (*m_logger) << time_now_string() << " listening on: " << ep << " external port: " << s.external_port << "\n"; #endif return s; @@ -2032,7 +2043,7 @@ namespace aux { if (e) { -#if defined TORRENT_LOGGING +#if defined TORRENT_LOGGING || defined TORRENT_VERBOSE_LOGGING (*m_logger) << "*** TICK TIMER FAILED " << e.message() << "\n"; #endif ::abort(); diff --git a/src/torrent.cpp b/src/torrent.cpp index 09270dd98..cfc743888 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -3337,6 +3337,11 @@ namespace libtorrent #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING (*m_ses.m_logger) << time_now_string() << " failed to parse web seed url: " << ec.message() << "\n"; #endif + if (m_ses.m_alerts.should_post()) + { + m_ses.m_alerts.post_alert( + url_seed_alert(get_handle(), web->url, ec)); + } // never try it again m_web_seeds.erase(web); return; @@ -3384,7 +3389,8 @@ namespace libtorrent if (web->endpoint.port() != 0) { - // TODO: we have already resolved this URL, just connect + connect_web_seed(web, web->endpoint); + return; } if (m_ses.m_port_filter.access(port) & port_filter::blocked) @@ -3615,10 +3621,14 @@ namespace libtorrent m_connections.insert(boost::get_pointer(c)); m_ses.m_connections.insert(c); + TORRENT_ASSERT(!web->connection); web->connection = c.get(); c->start(); +#if defined TORRENT_VERBOSE_LOGGING + (*m_ses.m_logger) << time_now_string() << " web seed connection started " << web->url << "\n"; +#endif m_ses.m_half_open.enqueue( boost::bind(&peer_connection::on_connect, c, _1) , boost::bind(&peer_connection::on_timeout, c) diff --git a/src/web_connection_base.cpp b/src/web_connection_base.cpp index f10122131..416fb33c0 100644 --- a/src/web_connection_base.cpp +++ b/src/web_connection_base.cpp @@ -65,17 +65,14 @@ namespace libtorrent , std::string const& auth , web_seed_entry::headers_t const& extra_headers) : peer_connection(ses, t, s, remote, peerinfo) - , m_first_request(true) - , m_ssl(false) , m_external_auth(auth) , m_extra_headers(extra_headers) + , m_first_request(true) + , m_ssl(false) + , m_body_start(0) { INVARIANT_CHECK; - // we want large blocks as well, so - // we can request more bytes at once - request_large_blocks(true); - // we only want left-over bandwidth set_priority(1); diff --git a/src/web_peer_connection.cpp b/src/web_peer_connection.cpp index 2e2f28eb5..304fed85c 100644 --- a/src/web_peer_connection.cpp +++ b/src/web_peer_connection.cpp @@ -82,6 +82,12 @@ namespace libtorrent // from web seeds prefer_whole_pieces((1024 * 1024) / tor->torrent_file().piece_length()); + // we want large blocks as well, so + // we can request more bytes at once + // this setting will merge adjacent requests + // into single larger ones + request_large_blocks(true); + #ifdef TORRENT_VERBOSE_LOGGING (*m_logger) << "*** web_peer_connection " << url << "\n"; #endif diff --git a/test/setup_transfer.cpp b/test/setup_transfer.cpp index d950e597a..ad531dd1f 100644 --- a/test/setup_transfer.cpp +++ b/test/setup_transfer.cpp @@ -578,7 +578,7 @@ void on_accept(error_code const& ec) } else { - fprintf(stderr, "accepting connection\n"); +// fprintf(stderr, "accepting connection\n"); accept_done = true; } } @@ -652,6 +652,7 @@ void web_server_thread(int* port, bool ssl) { if (connection_close) { +// fprintf(stderr, "closing connection\n"); s.close(ec); connection_close = false; } @@ -678,7 +679,7 @@ void web_server_thread(int* port, bool ssl) fprintf(stderr, "accept failed: %s\n", ec.message().c_str()); return; } - fprintf(stderr, "accepting incoming connection\n"); +// fprintf(stderr, "accepting incoming connection\n"); if (!s.is_open()) continue; #ifdef TORRENT_USE_OPENSSL @@ -708,6 +709,7 @@ void web_server_thread(int* port, bool ssl) while (!p.finished()) { + TORRENT_ASSERT(len < sizeof(buf)); size_t received = s.read_some(boost::asio::buffer(&buf[len] , sizeof(buf) - len), ec); // fprintf(stderr, "read: %d\n", int(received)); @@ -738,6 +740,7 @@ void web_server_thread(int* port, bool ssl) // the Via: header is an indicator of delegate making the request if (connection == "close" || !via.empty()) { +// fprintf(stderr, "got connection close\n"); connection_close = true; } @@ -745,6 +748,7 @@ void web_server_thread(int* port, bool ssl) if (failed) { + fprintf(stderr, "connection failed\n"); connection_close = true; break; } @@ -792,6 +796,59 @@ void web_server_thread(int* port, bool ssl) write(s, boost::asio::buffer(&buf[0], buf.size()), boost::asio::transfer_all(), ec); } + if (path.substr(0, 6) == "/seed?") + { + char const* piece = strstr(path.c_str(), "&piece="); + if (piece == 0) + { + fprintf(stderr, "invalid web seed request: %s\n", path.c_str()); + break; + } + boost::uint64_t idx = atoi(piece + 7); + char const* range = strstr(path.c_str(), "&ranges="); + int range_end = 0; + int range_start = 0; + if (range) + { + range_start = atoi(range + 8); + range = strchr(range, '-'); + if (range == 0) + { + fprintf(stderr, "invalid web seed request: %s\n", path.c_str()); + break; + } + range_end = atoi(range + 1); + } + else + { + range_start = 0; + // assume piece size of 16 + range_end = 16-1; + } + + int size = range_end - range_start + 1; + boost::uint64_t off = idx * 16 + range_start; + std::vector file_buf; + int res = load_file("./tmp1_web_seed/seed", file_buf); + + error_code ec; + if (res == -1 || file_buf.empty()) + { + send_response(s, ec, 404, "Not Found", 0, 0); + continue; + } + send_response(s, ec, 200, "OK", 0, size); +// fprintf(stderr, "sending %d bytes of payload [%d, %d)\n" +// , size, int(off), int(off + size)); + write(s, boost::asio::buffer(&file_buf[0] + off, size) + , boost::asio::transfer_all(), ec); + + memmove(buf, buf + offset, len - offset); + len -= offset; + offset = 0; + continue; + } + // fprintf(stderr, ">> serving file %s\n", path.c_str()); std::vector file_buf; // remove the / from the path diff --git a/test/test_web_seed.cpp b/test/test_web_seed.cpp index a560cd15e..871704417 100644 --- a/test/test_web_seed.cpp +++ b/test/test_web_seed.cpp @@ -48,7 +48,7 @@ using namespace libtorrent; // proxy: 0=none, 1=socks4, 2=socks5, 3=socks5_pw 4=http 5=http_pw void test_transfer(boost::intrusive_ptr torrent_file - , int proxy, int port, char const* protocol) + , int proxy, int port, char const* protocol, bool url_seed) { using namespace libtorrent; @@ -63,7 +63,8 @@ void test_transfer(boost::intrusive_ptr torrent_file char const* test_name[] = {"no", "SOCKS4", "SOCKS5", "SOCKS5 password", "HTTP", "HTTP password"}; - fprintf(stderr, "\n\n ==== TESTING %s proxy ==== %s ====\n\n\n", test_name[proxy], protocol); + fprintf(stderr, "\n\n ==== TESTING %s proxy ==== %s ==== %s ===\n\n\n" + , test_name[proxy], protocol, url_seed ? "URL seed" : "HTTP seed"); if (proxy) { @@ -130,8 +131,8 @@ void test_transfer(boost::intrusive_ptr torrent_file test_sleep(500); } - TEST_CHECK(cs.cache_size == 0); - TEST_CHECK(cs.total_used_buffers == 0); + TEST_EQUAL(cs.cache_size, 0); + TEST_EQUAL(cs.total_used_buffers, 0); std::cerr << "total_size: " << total_size << " rate_sum: " << rate_sum @@ -153,63 +154,100 @@ void test_transfer(boost::intrusive_ptr torrent_file remove_all("./tmp2_web_seed", ec); } -int run_suite(char const* protocol) +void save_file(char const* filename, char const* data, int size) +{ + error_code ec; + file out(filename, file::write_only, ec); + TEST_CHECK(!ec); + if (ec) + { + fprintf(stderr, "ERROR opening file '%s': %s\n", filename, ec.message().c_str()); + return; + } + file::iovec_t b = { (void*)data, size }; + out.writev(0, &b, 1, ec); + TEST_CHECK(!ec); + if (ec) + { + fprintf(stderr, "ERROR writing file '%s': %s\n", filename, ec.message().c_str()); + return; + } + +} + +// test_url_seed determines whether to use url-seed or http-seed +int run_suite(char const* protocol, bool test_url_seed) { using namespace libtorrent; error_code ec; create_directories("./tmp1_web_seed/test_torrent_dir", ec); - int file_sizes[] = - { 5, 16 - 5, 16, 17, 10, 30, 30, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 - ,1,1,1,1,1,1,13,65,34,75,2,3,4,5,23,9,43,4,43,6, 4}; - - char random_data[300000]; - std::srand(10); - for (int i = 0; i != sizeof(file_sizes)/sizeof(file_sizes[0]); ++i) - { - std::generate(random_data, random_data + sizeof(random_data), &std::rand); - char filename[200]; - snprintf(filename, sizeof(filename), "./tmp1_web_seed/test_torrent_dir/test%d", i); - error_code ec; - file out(filename, file::write_only, ec); - TEST_CHECK(!ec); - if (ec) - { - fprintf(stderr, "ERROR opening file '%s': %s\n", filename, ec.message().c_str()); - return 1; - } - file::iovec_t b = { random_data, file_sizes[i]}; - out.writev(0, &b, 1, ec); - TEST_CHECK(!ec); - if (ec) - { - fprintf(stderr, "ERROR writing file '%s': %s\n", filename, ec.message().c_str()); - return 1; - } - } - file_storage fs; - add_files(fs, "./tmp1_web_seed/test_torrent_dir"); + if (test_url_seed) + { + int file_sizes[] = + { 5, 16 - 5, 16, 17, 10, 30, 30, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 + ,1,1,1,1,1,1,13,65,34,75,2,3,4,5,23,9,43,4,43,6, 4}; + + char random_data[300000]; + std::srand(10); + for (int i = 0; i != sizeof(file_sizes)/sizeof(file_sizes[0]); ++i) + { + std::generate(random_data, random_data + sizeof(random_data), &std::rand); + char filename[200]; + snprintf(filename, sizeof(filename), "./tmp1_web_seed/test_torrent_dir/test%d", i); + save_file(filename, random_data, file_sizes[i]); + } + + add_files(fs, "./tmp1_web_seed/test_torrent_dir"); + } + else + { + char random_data[10000]; + std::srand(10); + std::generate(random_data, random_data + sizeof(random_data), &std::rand); + save_file("./tmp1_web_seed/seed", random_data, sizeof(random_data)); + fs.add_file("seed", sizeof(random_data)); + } int port = start_web_server(strcmp(protocol, "https") == 0); libtorrent::create_torrent t(fs, 16); char tmp[512]; - snprintf(tmp, sizeof(tmp), "%s://127.0.0.1:%d/tmp1_web_seed", protocol, port); - t.add_url_seed(tmp); - + if (test_url_seed) + { + snprintf(tmp, sizeof(tmp), "%s://127.0.0.1:%d/tmp1_web_seed", protocol, port); + t.add_url_seed(tmp); + } + else + { + snprintf(tmp, sizeof(tmp), "http://127.0.0.1:%d/seed", port); + t.add_http_seed(tmp); + } // calculate the hash for all pieces set_piece_hashes(t, "./tmp1_web_seed", ec); + + if (ec) + { + fprintf(stderr, "error creating hashes for test torrent: %s\n" + , ec.message().c_str()); + TEST_CHECK(false); + return 0; + } + std::vector buf; bencode(std::back_inserter(buf), t.generate()); boost::intrusive_ptr torrent_file(new torrent_info(&buf[0], buf.size(), ec)); for (int i = 0; i < 6; ++i) - test_transfer(torrent_file, i, port, protocol); + test_transfer(torrent_file, i, port, protocol, test_url_seed); - torrent_file->rename_file(0, "./tmp2_web_seed/test_torrent_dir/renamed_test1"); - test_transfer(torrent_file, 0, port, protocol); + if (test_url_seed) + { + torrent_file->rename_file(0, "./tmp2_web_seed/test_torrent_dir/renamed_test1"); + test_transfer(torrent_file, 0, port, protocol, test_url_seed); + } stop_web_server(); remove_all("./tmp1_web_seed", ec); @@ -219,10 +257,13 @@ int run_suite(char const* protocol) int test_main() { int ret = 0; + for (int i = 0; i < 2; ++i) + { #ifdef TORRENT_USE_OPENSSL - ret += run_suite("https"); + run_suite("https", i); #endif - ret += run_suite("http"); + run_suite("http", i); + } return ret; }