premiere
/
premiere-libtorrent
2
2
Fork 1
Code Issues Pull Requests Releases Wiki Activity

support banning web seeds that send corrupt data

This commit is contained in:
Arvid Norberg 2012-03-24 01:29:31 +00:00
parent bad2857cbe
commit 6fcc469aef
10 changed files with 194 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ChangeLog
View File

@ -1,3 +1,4 @@
* support banning web seeds sending corrupt data
* don't let hung outgoing connection attempts block incoming connections
* improve SSL torrent support by using SNI and a single SSL listen socket
* improved peer exchange performance by sharing incoming connections which advertize listen port

8
include/libtorrent/policy.hpp
View File

@ -162,7 +162,7 @@ namespace libtorrent
// 40 1 1 fast_reconnects, trust_points
// 41 1 1 source, pe_support, is_v6_addr
// 42 1 1 on_parole, banned, added_to_dht, supports_utp,
// supports_holepunch
// supports_holepunch, web_seed
// 43 1 1 <padding>
// 44
struct TORRENT_EXTRA_EXPORT peer
@ -311,6 +311,12 @@ namespace libtorrent
// we have been connected via uTP at least once
bool confirmed_supports_utp:1;
bool supports_holepunch:1;
// this is set to one for web seeds. Web seeds
// are not stored in the policy m_peers list,
// and are excempt from connect candidate bookkeeping
// so, any peer with the web_seed bit set, is
// never considered a connect candidate
bool web_seed:1;
#if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS
bool in_use:1;
#endif

36
include/libtorrent/torrent.hpp
View File

@ -370,42 +370,12 @@ namespace libtorrent
m_web_seeds.push_back(web_seed_entry(url, type, auth, extra_headers));
}
void remove_web_seed(std::string const& url, web_seed_entry::type_t type)
{
std::list<web_seed_entry>::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end()
, (boost::bind(&web_seed_entry::url, _1)
== url && boost::bind(&web_seed_entry::type, _1) == type));
if (i != m_web_seeds.end()) remove_web_seed(i);
}
void disconnect_web_seed(peer_connection* p)
{
std::list<web_seed_entry>::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end()
, (boost::bind(&web_seed_entry::connection, _1) == p));
// this happens if the web server responded with a redirect
// or with something incorrect, so that we removed the web seed
// immediately, before we disconnected
if (i == m_web_seeds.end()) return;
TORRENT_ASSERT(i->resolving == false);
#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_ERROR_LOGGING
(*m_ses.m_logger) << time_now_string() << " disconnect_web_seed: " << i->url << "\n";
#endif
TORRENT_ASSERT(i->connection);
i->connection = 0;
}
void remove_web_seed(std::string const& url, web_seed_entry::type_t type);
void disconnect_web_seed(peer_connection* p);
void retry_web_seed(peer_connection* p, int retry = 0);
void remove_web_seed(peer_connection* p)
{
std::list<web_seed_entry>::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end()
, (boost::bind(&web_seed_entry::connection, _1) == p));
TORRENT_ASSERT(i != m_web_seeds.end());
if (i == m_web_seeds.end()) return;
m_web_seeds.erase(i);
}
void remove_web_seed(peer_connection* p);
std::list<web_seed_entry> web_seeds() const
{ return m_web_seeds; }

14
include/libtorrent/torrent_info.hpp
View File

@ -58,6 +58,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include "libtorrent/file_storage.hpp"
#include "libtorrent/copy_ptr.hpp"
#include "libtorrent/socket.hpp"
#include "libtorrent/policy.hpp" // for policy::peer
namespace libtorrent
{
@ -171,12 +172,7 @@ namespace libtorrent
web_seed_entry(std::string const& url_, type_t type_
, std::string const& auth_ = std::string()
, headers_t const& extra_headers_ = headers_t())
: url(url_), type(type_)
, auth(auth_), extra_headers(extra_headers_)
, retry(time_now()), resolving(false), removed(false)
, connection(0)
{}
, headers_t const& extra_headers_ = headers_t());
bool operator==(web_seed_entry const& e) const
{ return url == e.url && type == e.type; }
@ -208,7 +204,11 @@ namespace libtorrent
tcp::endpoint endpoint;
peer_connection* connection;
// this is the peer_info field used for the
// connection, just to count hash failures
// it's also used to hold the peer_connection
// pointer, when the web seed is connected
policy::peer peer_info;
};
#ifndef BOOST_NO_EXCEPTIONS

5
src/peer_connection.cpp
View File

@ -3503,6 +3503,8 @@ namespace libtorrent
m_disconnect_started = true;
#endif
if (m_disconnecting) return;
#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_ERROR_LOGGING
switch (error)
{
@ -3585,7 +3587,6 @@ namespace libtorrent
// we cannot do this in a constructor
TORRENT_ASSERT(m_in_constructor == false);
if (error > 0) m_failed = true;
if (m_disconnecting) return;
boost::intrusive_ptr<peer_connection> me(this);
INVARIANT_CHECK;
@ -5884,7 +5885,7 @@ namespace libtorrent
}
}
#ifdef TORRENT_EXPENSIVE_INVARIANT_CHECKS
if (m_peer_info)
if (m_peer_info && type() == bittorrent_connection)
{
policy::const_iterator i = t->get_policy().begin_peer();
policy::const_iterator end = t->get_policy().end_peer();

21
src/policy.cpp
View File

@ -590,6 +590,7 @@ namespace libtorrent
{
if (p.connection
|| p.banned
|| p.web_seed
|| !p.connectable
|| (p.seed && finished)
|| int(p.failcount) >= m_torrent->settings().max_failcount)
@ -1061,6 +1062,7 @@ namespace libtorrent
if (m_num_connect_candidates < 0) m_num_connect_candidates = 0;
}
if (p->web_seed) return;
if (s) ++m_num_seeds;
else --m_num_seeds;
TORRENT_ASSERT(m_num_seeds >= 0);
@ -1408,15 +1410,18 @@ namespace libtorrent
peer* p = c.peer_info_struct();
TORRENT_ASSERT((std::find_if(
m_peers.begin()
, m_peers.end()
, match_peer_connection(c))
!= m_peers.end()) == (p != 0));
// if we couldn't find the connection in our list, just ignore it.
if (p == 0) return;
// web seeds are special, they're not connected via the peer list
// so they're not kept in m_peers
TORRENT_ASSERT(p->web_seed
|| std::find_if(
m_peers.begin()
, m_peers.end()
, match_peer_connection(c))
!= m_peers.end());
TORRENT_ASSERT(p->connection == &c);
TORRENT_ASSERT(!is_connect_candidate(*p, m_finished));
@ -1616,6 +1621,9 @@ namespace libtorrent
policy::peer* p = static_cast<policy::peer*>(*i);
if (p == 0) continue;
if (p->connection == 0) continue;
// web seeds are special, they're not connected via the peer list
// so they're not kept in m_peers
if (p->connection->type() != peer_connection::bittorrent_connection) continue;
TORRENT_ASSERT(std::find_if(m_peers.begin(), m_peers.end()
, match_peer_connection_or_endpoint(*p->connection)) != m_peers.end());
}
@ -1677,6 +1685,7 @@ namespace libtorrent
, supports_utp(true) // assume peers support utp
, confirmed_supports_utp(false)
, supports_holepunch(false)
, web_seed(false)
#if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS
, in_use(false)
#endif

2
src/session_impl.cpp
View File

@ -6093,7 +6093,7 @@ namespace aux {
++num_optimistic;
TORRENT_ASSERT(!p->is_choked());
}
if (t && p->peer_info_struct())
if (t && p->peer_info_struct() && !p->peer_info_struct()->web_seed)
{
TORRENT_ASSERT(t->get_policy().has_connection(p));
}

72
src/torrent.cpp
View File

@ -4353,6 +4353,22 @@ namespace libtorrent
remove_web_seed(web);
return;
}
if (web->peer_info.banned)
{
#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING
(*m_ses.m_logger) << time_now_string() << "banned web seed: " << web->url << "\n";
#endif
if (m_ses.m_alerts.should_post<url_seed_alert>())
{
m_ses.m_alerts.post_alert(
url_seed_alert(get_handle(), web->url
, error_code(libtorrent::errors::peer_banned, get_libtorrent_category())));
}
// never try it again
remove_web_seed(web);
return;
}
#ifdef TORRENT_USE_OPENSSL
if (protocol != "http" && protocol != "https")
@ -4575,7 +4591,7 @@ namespace libtorrent
}
TORRENT_ASSERT(web->resolving == false);
TORRENT_ASSERT(web->connection == 0);
TORRENT_ASSERT(web->peer_info.connection == 0);
web->endpoint = a;
@ -4661,13 +4677,13 @@ namespace libtorrent
if (web->type == web_seed_entry::url_seed)
{
c = new (std::nothrow) web_peer_connection(
m_ses, shared_from_this(), s, a, web->url, 0, // TODO: pass in web
m_ses, shared_from_this(), s, a, web->url, &web->peer_info, // TODO: pass in web
web->auth, web->extra_headers);
}
else if (web->type == web_seed_entry::http_seed)
{
c = new (std::nothrow) http_seed_connection(
m_ses, shared_from_this(), s, a, web->url, 0, // TODO: pass in web
m_ses, shared_from_this(), s, a, web->url, &web->peer_info, // TODO: pass in web
web->auth, web->extra_headers);
}
if (!c) return;
@ -4692,9 +4708,16 @@ namespace libtorrent
m_connections.insert(boost::get_pointer(c));
m_ses.m_connections.insert(c);
TORRENT_ASSERT(!web->connection);
web->connection = c.get();
TORRENT_ASSERT(!web->peer_info.connection);
web->peer_info.connection = c.get();
#if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS
web->peer_info.in_use = true;
#endif
c->add_stat(size_type(web->peer_info.prev_amount_download) << 10
, size_type(web->peer_info.prev_amount_upload) << 10);
web->peer_info.prev_amount_download = 0;
web->peer_info.prev_amount_upload = 0;
#if defined TORRENT_VERBOSE_LOGGING
(*m_ses.m_logger) << time_now_string() << " web seed connection started " << web->url << "\n";
#endif
@ -7355,7 +7378,7 @@ namespace libtorrent
i != m_web_seeds.end();)
{
std::list<web_seed_entry>::iterator w = i++;
if (w->connection) continue;
if (w->peer_info.connection) continue;
if (w->retry > time_now()) continue;
if (w->resolving) continue;
@ -7808,11 +7831,46 @@ namespace libtorrent
return ret;
}
void torrent::remove_web_seed(std::string const& url, web_seed_entry::type_t type)
{
std::list<web_seed_entry>::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end()
, (boost::bind(&web_seed_entry::url, _1)
== url && boost::bind(&web_seed_entry::type, _1) == type));
if (i != m_web_seeds.end()) remove_web_seed(i);
}
void torrent::disconnect_web_seed(peer_connection* p)
{
std::list<web_seed_entry>::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end()
, (boost::bind(&policy::peer::connection, boost::bind(&web_seed_entry::peer_info, _1)) == p));
// this happens if the web server responded with a redirect
// or with something incorrect, so that we removed the web seed
// immediately, before we disconnected
if (i == m_web_seeds.end()) return;
TORRENT_ASSERT(i->resolving == false);
#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_ERROR_LOGGING
(*m_ses.m_logger) << time_now_string() << " disconnect_web_seed: " << i->url << "\n";
#endif
TORRENT_ASSERT(i->peer_info.connection);
i->peer_info.connection = 0;
}
void torrent::remove_web_seed(peer_connection* p)
{
std::list<web_seed_entry>::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end()
, (boost::bind(&policy::peer::connection, boost::bind(&web_seed_entry::peer_info, _1)) == p));
TORRENT_ASSERT(i != m_web_seeds.end());
if (i == m_web_seeds.end()) return;
m_web_seeds.erase(i);
}
void torrent::retry_web_seed(peer_connection* p, int retry)
{
TORRENT_ASSERT(m_ses.is_network_thread());
std::list<web_seed_entry>::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end()
, (boost::bind(&web_seed_entry::connection, _1) == p));
, (boost::bind(&policy::peer::connection, boost::bind(&web_seed_entry::peer_info, _1)) == p));
TORRENT_ASSERT(i != m_web_seeds.end());
if (i == m_web_seeds.end()) return;

11
src/torrent_info.cpp
View File

@ -498,6 +498,17 @@ namespace libtorrent
url.erase(url.begin());
}
web_seed_entry::web_seed_entry(std::string const& url_, type_t type_
, std::string const& auth_
, headers_t const& extra_headers_)
: url(url_), type(type_)
, auth(auth_), extra_headers(extra_headers_)
, retry(time_now()), resolving(false), removed(false)
, peer_info(0, true, 0)
{
peer_info.web_seed = true;
}
torrent_info::torrent_info(torrent_info const& t, int flags)
: m_merkle_first_leaf(t.m_merkle_first_leaf)
, m_files(t.m_files)

107
test/test_web_seed.cpp
View File

@ -48,7 +48,7 @@ using namespace libtorrent;
// proxy: 0=none, 1=socks4, 2=socks5, 3=socks5_pw 4=http 5=http_pw
void test_transfer(boost::intrusive_ptr<torrent_info> torrent_file
, int proxy, int port, char const* protocol, bool url_seed, bool chunked_encoding)
, int proxy, int port, char const* protocol, bool url_seed, bool chunked_encoding, bool test_ban)
{
using namespace libtorrent;
@ -65,8 +65,8 @@ void test_transfer(boost::intrusive_ptr<torrent_info> torrent_file
char const* test_name[] = {"no", "SOCKS4", "SOCKS5", "SOCKS5 password", "HTTP", "HTTP password"};
fprintf(stderr, "\n\n ==== TESTING === proxy: %s ==== protocol: %s ==== seed: %s === transfer-encoding: %s\n\n\n"
, test_name[proxy], protocol, url_seed ? "URL seed" : "HTTP seed", chunked_encoding ? "chunked": "none");
fprintf(stderr, "\n\n ==== TESTING === proxy: %s ==== protocol: %s ==== seed: %s === transfer-encoding: %s === corruption: %s\n\n\n"
, test_name[proxy], protocol, url_seed ? "URL seed" : "HTTP seed", chunked_encoding ? "chunked": "none", test_ban ? "yes" : "no");
if (proxy)
{
@ -128,7 +128,13 @@ void test_transfer(boost::intrusive_ptr<torrent_info> torrent_file
<< " buffers: " << cs.total_used_buffers
<< std::endl;
*/
print_alerts(ses, " >> ses", false, false, false, 0, true);
print_alerts(ses, " >> ses", test_ban, false, false, 0, true);
if (test_ban && th.url_seeds().empty())
{
// when we don't have any web seeds left, we know we successfully banned it
break;
}
if (s.is_seeding /* && ss.download_rate == 0.f*/)
{
@ -142,6 +148,10 @@ void test_transfer(boost::intrusive_ptr<torrent_info> torrent_file
test_sleep(500);
}
// for test_ban tests, make sure we removed
// the url seed (i.e. banned it)
TEST_CHECK(!test_ban || th.url_seeds().empty());
TEST_EQUAL(cs.cache_size, 0);
TEST_EQUAL(cs.total_used_buffers, 0);
@ -157,12 +167,14 @@ void test_transfer(boost::intrusive_ptr<torrent_info> torrent_file
// TEST_CHECK(fabs(rate_sum - total_size) < total_size * .1f);
// TEST_CHECK(fabs(ses_rate_sum - total_size) < total_size * .1f);
TEST_CHECK(th.status().is_seeding);
// if test_ban is true, we're not supposed to have completed the download
// otherwise, we are supposed to have
TEST_CHECK(th.status().is_seeding == !test_ban);
if (proxy) stop_proxy(8002);
TEST_CHECK(exists(combine_path("tmp2_web_seed", torrent_file->files().file_path(
torrent_file->file_at(0)))));
torrent_file->file_at(0)))) || test_ban);
remove_all("tmp2_web_seed", ec);
}
@ -198,7 +210,7 @@ sha1_hash file_hash(std::string const& name)
}
// test_url_seed determines whether to use url-seed or http-seed
int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding)
int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding, bool test_ban)
{
using namespace libtorrent;
@ -208,12 +220,12 @@ int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding)
file_storage fs;
std::srand(10);
int piece_size = 0x4000;
static const int file_sizes[] =
{ 5, 16 - 5, 16000, 17, 10, 8000, 8000, 1,1,1,1,1,100,1,1,1,1,100,1,1,1,1,1,1
,1,1,1,1,1,1,13,65000,34,75,2,30,400,500,23000,900,43000,400,4300,6, 4};
if (test_url_seed)
{
int file_sizes[] =
{ 5, 16 - 5, 16000, 17, 10, 8000, 8000, 1,1,1,1,1,100,1,1,1,1,100,1,1,1,1,1,1
,1,1,1,1,1,1,13,65000,34,75,2,30,400,500,23000,900,43000,400,4300,6, 4};
char* random_data = (char*)malloc(300000);
for (int i = 0; i != sizeof(file_sizes)/sizeof(file_sizes[0]); ++i)
{
@ -252,6 +264,7 @@ int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding)
// are not requested web seeds
libtorrent::create_torrent t(fs, piece_size, 0x4000, libtorrent::create_torrent::optimize
| libtorrent::create_torrent::calculate_file_hashes);
char tmp[512];
if (test_url_seed)
{
@ -284,29 +297,68 @@ int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding)
return 0;
}
if (test_ban)
{
// corrupt the files now, so that the web seed will be banned
if (test_url_seed)
{
char* random_data = (char*)malloc(300000);
for (int i = 0; i != sizeof(file_sizes)/sizeof(file_sizes[0]); ++i)
{
std::generate(random_data, random_data + 300000, &std::rand);
char filename[200];
snprintf(filename, sizeof(filename), "tmp1_web_seed/test_torrent_dir/test%d", i);
int to_write = file_sizes[i];
file f(filename, file::write_only, ec);
size_type offset = 0;
while (to_write > 0)
{
int s = (std::min)(to_write, 300000);
file::iovec_t b = { random_data, s};
f.writev(offset, &b, 1, ec);
offset += s;
to_write -= s;
}
}
free(random_data);
}
else
{
piece_size = 64 * 1024;
char* random_data = (char*)malloc(64 * 1024 * 25);
std::generate(random_data, random_data + 64 * 1024 * 25, &std::rand);
save_file("tmp1_web_seed/seed", random_data, 64 * 1024 * 25);
free(random_data);
}
}
std::vector<char> buf;
bencode(std::back_inserter(buf), t.generate());
boost::intrusive_ptr<torrent_info> torrent_file(new torrent_info(&buf[0], buf.size(), ec));
// verify that the file hashes are correct
for (int i = 0; i < torrent_file->num_files(); ++i)
// no point in testing the hashes since we know the data is corrupt
if (!test_ban)
{
sha1_hash h1 = torrent_file->file_at(i).filehash;
sha1_hash h2 = file_hash(combine_path("tmp1_web_seed"
, torrent_file->file_at(i).path));
// fprintf(stderr, "%s: %s == %s\n"
// , torrent_file->file_at(i).path.c_str()
// , to_hex(h1.to_string()).c_str(), to_hex(h2.to_string()).c_str());
TEST_EQUAL(h1, h2);
// verify that the file hashes are correct
for (int i = 0; i < torrent_file->num_files(); ++i)
{
sha1_hash h1 = torrent_file->file_at(i).filehash;
sha1_hash h2 = file_hash(combine_path("tmp1_web_seed"
, torrent_file->file_at(i).path));
// fprintf(stderr, "%s: %s == %s\n"
// , torrent_file->file_at(i).path.c_str()
// , to_hex(h1.to_string()).c_str(), to_hex(h2.to_string()).c_str());
TEST_EQUAL(h1, h2);
}
}
for (int i = 0; i < 6; ++i)
test_transfer(torrent_file, i, port, protocol, test_url_seed, chunked_encoding);
test_transfer(torrent_file, i, port, protocol, test_url_seed, chunked_encoding, test_ban);
if (test_url_seed)
{
torrent_file->rename_file(0, "tmp2_web_seed/test_torrent_dir/renamed_test1");
test_transfer(torrent_file, 0, port, protocol, test_url_seed, chunked_encoding);
test_transfer(torrent_file, 0, port, protocol, test_url_seed, chunked_encoding, test_ban);
}
stop_web_server();
@ -317,14 +369,17 @@ int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding)
int test_main()
{
int ret = 0;
for (int i = 0; i < 2; ++i)
for (int url_seed = 0; url_seed < 2; ++url_seed)
{
for (int j = 0; j < 2; ++j)
for (int chunked = 0; chunked < 2; ++chunked)
{
for (int ban = 0; ban < 2; ++ban)
{
#ifdef TORRENT_USE_OPENSSL
run_suite("https", i, j);
run_suite("https", url_seed, chunked, ban);
#endif
run_suite("http", i, j);
run_suite("http", url_seed, chunked, ban);
}
}
}
return ret;