From 69f82c5274a9ca14376db61b1b80cf7672d830e2 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Wed, 27 Jul 2016 00:15:26 -0400 Subject: [PATCH 01/10] add support for entry::preformatted to python binding (#952) --- bindings/python/src/entry.cpp | 22 ++++++++++++++++++++++ bindings/python/test.py | 15 +++++++++++++++ test/Makefile.am | 1 + test/test_torrent_info.cpp | 1 + test/test_torrents/unordered.torrent | 1 + 5 files changed, 40 insertions(+) create mode 100644 test/test_torrents/unordered.torrent diff --git a/bindings/python/src/entry.cpp b/bindings/python/src/entry.cpp index bced76d65..5019920f1 100644 --- a/bindings/python/src/entry.cpp +++ b/bindings/python/src/entry.cpp @@ -45,6 +45,15 @@ struct entry_to_python return convert(e.list()); case entry::dictionary_t: return convert(e.dict()); + case entry::preformatted_t: + { + std::vector const& pre = e.preformatted(); + list l; + for (std::vector::const_iterator i = pre.begin() + , end(pre.end()); i != end; ++i) + l.append(*i); + return tuple(l); + } default: return object(); } @@ -136,6 +145,19 @@ struct entry_from_python { return entry(extract(e)()); } + else if (extract(e).check()) + { + tuple t = extract(e); + + std::size_t const length = extract(t.attr("__len__")()); + std::vector preformatted(length); + for (std::size_t i = 0; i < length; ++i) + { + preformatted[i] = extract(t[i]); + } + + return entry(preformatted); + } return entry(); } diff --git a/bindings/python/test.py b/bindings/python/test.py index 48c5a0f2a..d46e5b0ab 100644 --- a/bindings/python/test.py +++ b/bindings/python/test.py @@ -8,6 +8,20 @@ import os import shutil import binascii + +class test_create_torrent(unittest.TestCase): + + def test_from_torrent_info(self): + ti = lt.torrent_info('unordered.torrent') + ct = lt.create_torrent(ti) + entry = ct.generate() + content = lt.bencode(entry).strip() + with open('unordered.torrent', 'r') as f: + file_content = f.read().strip() + print file_content + print content + self.assertEqual(content, file_content) + class test_torrent_handle(unittest.TestCase): def test_torrent_handle(self): @@ -165,5 +179,6 @@ if __name__ == '__main__': print(lt.__version__) shutil.copy(os.path.join('..', '..', 'test', 'test_torrents', 'url_seed_multi.torrent'), '.') shutil.copy(os.path.join('..', '..', 'test', 'test_torrents', 'base.torrent'), '.') + shutil.copy(os.path.join('..', '..', 'test', 'test_torrents', 'unordered.torrent'), '.') unittest.main() diff --git a/test/Makefile.am b/test/Makefile.am index ac8899f29..cb25fc037 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -94,6 +94,7 @@ EXTRA_DIST = Jamfile \ test_torrents/string.torrent \ test_torrents/symlink1.torrent \ test_torrents/unaligned_pieces.torrent \ + test_torrents/unordered.torrent \ test_torrents/url_list.torrent \ test_torrents/url_list2.torrent \ test_torrents/url_list3.torrent \ diff --git a/test/test_torrent_info.cpp b/test/test_torrent_info.cpp index 51f8ff471..87b092dc6 100644 --- a/test/test_torrent_info.cpp +++ b/test/test_torrent_info.cpp @@ -131,6 +131,7 @@ static test_torrent_t test_torrents[] = { "invalid_name2.torrent" }, { "invalid_name3.torrent" }, { "symlink1.torrent" }, + { "unordered.torrent" }, }; struct test_failing_torrent_t diff --git a/test/test_torrents/unordered.torrent b/test/test_torrents/unordered.torrent new file mode 100644 index 000000000..0d59f70dd --- /dev/null +++ b/test/test_torrents/unordered.torrent @@ -0,0 +1 @@ +d10:created by10:libtorrent13:creation datei1359599503e4:infod4:name4:temp6:lengthi425e12:piece lengthi16384e6:pieces20:‚ž¼Œ&¾ÇJW›}ÜA4u,·¼‘‡ee From f52a37fe86250116e985ddfeb454e21702daf5f5 Mon Sep 17 00:00:00 2001 From: Calum Lind Date: Wed, 27 Jul 2016 23:16:05 +0100 Subject: [PATCH 02/10] [#766] Fix building bindings with python-dbg (#958) * In ax_python_devel grep and remove '[xxxx refs]' if found in python-dbg output so that it will still catch any errors redirected to stdout. * In ax_boost_python change the specific _BOOST_PYTHON_MODULE test for the old version that used a generic _main method test. --- m4/ax_boost_python.m4 | 5 ++--- m4/ax_python_devel.m4 | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/m4/ax_boost_python.m4 b/m4/ax_boost_python.m4 index 385af3ef5..5d21fd31e 100644 --- a/m4/ax_boost_python.m4 +++ b/m4/ax_boost_python.m4 @@ -101,12 +101,11 @@ if test "$ac_cv_boost_python" = "yes"; then fi]) BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` for ax_lib in $ax_python_lib $ax_boost_python_lib `ls $BOOSTLIBDIR/libboost_python*.so* $BOOSTLIBDIR/libboost_python*.dylib* $BOOSTLIBDIR/libboost_python*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_python.*\)\.so.*$;\1;' -e 's;^lib\(boost_python.*\)\.dylib.*$;\1;' -e 's;^lib\(boost_python.*\)\.a.*$;\1;' ` boost_python boost_python3; do - AS_VAR_PUSHDEF([ax_Lib], [ax_cv_lib_$ax_lib''_BOOST_PYTHON_MODULE])dnl + AS_VAR_PUSHDEF([ax_Lib], [ax_cv_lib_$ax_lib''_main])dnl AC_CACHE_CHECK([whether $ax_lib is the correct library], [ax_Lib], [LIBS="-l$ax_lib $ax_boost_python_save_LIBS $PYTHON_LIBS" AC_LINK_IFELSE([AC_LANG_PROGRAM([[ -#include -BOOST_PYTHON_MODULE(test) { throw "Boost::Python test."; }]], [])], + ]], [])], [AS_VAR_SET([ax_Lib], [yes])], [AS_VAR_SET([ax_Lib], [no])])]) AS_VAR_IF([ax_Lib], [yes], [BOOST_PYTHON_LIB=$ax_lib break], []) diff --git a/m4/ax_python_devel.m4 b/m4/ax_python_devel.m4 index f18ed8a01..11f1f51ac 100644 --- a/m4/ax_python_devel.m4 +++ b/m4/ax_python_devel.m4 @@ -136,7 +136,7 @@ variable to configure. See ``configure --help'' for reference. # Check if you have distutils, else fail # AC_MSG_CHECKING([for the distutils Python package]) - ac_distutils_result=`$PYTHON -c "import distutils" 2>&1` + ac_distutils_result=`$PYTHON -c "import distutils" 2>&1 | grep -v '^\[[0-9]\{1,\} refs\]'` if test -z "$ac_distutils_result"; then AC_MSG_RESULT([yes]) else From 6c0f4cc082c5db3619e98df5c26857bca74e7129 Mon Sep 17 00:00:00 2001 From: Calum Lind Date: Thu, 28 Jul 2016 00:19:24 +0100 Subject: [PATCH 03/10] Fix reoccurrence of py bindings -Wstrict-prototype warning (#959) * Recent commit df368c2d158 has allowed the -Wstrict-prototype flag to once again be passed to gcc so this fix removes all occurrences not just from 'OPT'. --- bindings/python/setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 86f43ea83..13b25e84a 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from distutils.core import setup, Extension -from distutils.sysconfig import get_config_var +from distutils.sysconfig import get_config_vars import os import platform import sys @@ -102,8 +102,10 @@ if '--bjam' in sys.argv: else: # Remove the '-Wstrict-prototypes' compiler option, which isn't valid for C++. - os.environ['OPT'] = ' '.join( - flag for flag in get_config_var('OPT').split() if flag != '-Wstrict-prototypes') + cfg_vars = get_config_vars() + for key, value in cfg_vars.items(): + if isinstance(value, str): + cfg_vars[key] = value.replace('-Wstrict-prototypes', '') source_list = os.listdir(os.path.join(os.path.dirname(__file__), "src")) source_list = [os.path.abspath(os.path.join(os.path.dirname(__file__), "src", s)) for s in source_list if s.endswith(".cpp")] From 633d0621f84838a0f469e27b760728fe7bfb1b64 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Thu, 28 Jul 2016 08:34:22 -0400 Subject: [PATCH 04/10] update required boost version and ax_boost_base.m4 (#937) bump boost version requirement to 1.53, update ax_boost_base.m4 --- configure.ac | 2 +- m4/ax_boost_base.m4 | 25 ++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 413da26cb..c085f9def 100644 --- a/configure.ac +++ b/configure.ac @@ -117,7 +117,7 @@ AC_CACHE_CHECK([for __attribute__((visibility("hidden")))], AS_ECHO AS_ECHO "Checking for boost libraries:" -AX_BOOST_BASE([1.47]) +AX_BOOST_BASE([1.53]) AX_BOOST_SYSTEM() AS_IF([test -z "$BOOST_SYSTEM_LIB"], diff --git a/m4/ax_boost_base.m4 b/m4/ax_boost_base.m4 index ac9ee75ec..d35d466de 100644 --- a/m4/ax_boost_base.m4 +++ b/m4/ax_boost_base.m4 @@ -33,7 +33,7 @@ # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 22 +#serial 26 AC_DEFUN([AX_BOOST_BASE], [ @@ -92,8 +92,11 @@ if test "x$want_boost" = "xyes"; then libsubdirs="lib" ax_arch=`uname -m` case $ax_arch in - x86_64|ppc64|s390x|sparc64|aarch64) - libsubdirs="lib64 lib lib64" + x86_64) + libsubdirs="lib64 libx32 lib lib64" + ;; + ppc64|s390x|sparc64|aarch64|ppc64le) + libsubdirs="lib64 lib lib64 ppc64le" ;; esac @@ -103,6 +106,12 @@ if test "x$want_boost" = "xyes"; then AC_REQUIRE([AC_CANONICAL_HOST]) libsubdirs="lib/${host_cpu}-${host_os} $libsubdirs" + case ${host_cpu} in + i?86) + libsubdirs="lib/i386-${host_os} $libsubdirs" + ;; + esac + dnl first we check the system location for boost libraries dnl this location ist chosen if boost libraries are installed with the --layout=system option dnl or if you install boost with RPM @@ -164,6 +173,10 @@ if test "x$want_boost" = "xyes"; then dnl if we found no boost with system layout we search for boost libraries dnl built and installed without the --layout=system option or for a staged(not installed) version if test "x$succeeded" != "xyes"; then + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" + BOOST_CPPFLAGS= + BOOST_LDFLAGS= _version=0 if test "$ac_boost_path" != ""; then if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then @@ -176,6 +189,12 @@ if test "x$want_boost" = "xyes"; then VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` BOOST_CPPFLAGS="-I$ac_boost_path/include/boost-$VERSION_UNDERSCORE" done + dnl if nothing found search for layout used in Windows distributions + if test -z "$BOOST_CPPFLAGS"; then + if test -d "$ac_boost_path/boost" && test -r "$ac_boost_path/boost"; then + BOOST_CPPFLAGS="-I$ac_boost_path" + fi + fi fi else if test "$cross_compiling" != yes; then From 192559f245892d016f3395fd9c5ede710bb52f3f Mon Sep 17 00:00:00 2001 From: arvidn Date: Sun, 31 Jul 2016 03:19:31 -0400 Subject: [PATCH 05/10] fix warnings of unused return values from ftruncate() calls --- src/disk_buffer_pool.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/disk_buffer_pool.cpp b/src/disk_buffer_pool.cpp index 8ebd8c4a3..cda5fb188 100644 --- a/src/disk_buffer_pool.cpp +++ b/src/disk_buffer_pool.cpp @@ -123,7 +123,8 @@ namespace libtorrent m_cache_pool = 0; // attempt to make MacOS not flush this to disk, making close() // block for a long time - ftruncate(m_cache_fd, 0); + int const best_effort = ftruncate(m_cache_fd, 0); + TORRENT_UNUSED(best_effort); close(m_cache_fd); m_cache_fd = -1; } @@ -460,7 +461,8 @@ namespace libtorrent m_cache_pool = 0; // attempt to make MacOS not flush this to disk, making close() // block for a long time - ftruncate(m_cache_fd, 0); + int const best_effort = ftruncate(m_cache_fd, 0); + TORRENT_UNUSED(best_effort); close(m_cache_fd); m_cache_fd = -1; std::vector().swap(m_free_list); @@ -482,7 +484,8 @@ namespace libtorrent #ifndef MAP_NOCACHE #define MAP_NOCACHE 0 #endif - ftruncate(m_cache_fd, boost::uint64_t(m_max_use) * 0x4000); + int const best_effort = ftruncate(m_cache_fd, boost::uint64_t(m_max_use) * 0x4000); + TORRENT_UNUSED(best_effort); m_cache_pool = static_cast(mmap(0, boost::uint64_t(m_max_use) * 0x4000, PROT_READ | PROT_WRITE , MAP_SHARED | MAP_NOCACHE, m_cache_fd, 0)); if (intptr_t(m_cache_pool) == -1) @@ -492,7 +495,8 @@ namespace libtorrent m_cache_pool = 0; // attempt to make MacOS not flush this to disk, making close() // block for a long time - ftruncate(m_cache_fd, 0); + int const best_effort = ftruncate(m_cache_fd, 0); + TORRENT_UNUSED(best_effort); close(m_cache_fd); m_cache_fd = -1; } From 24dd6d094511a9e5f018179019e198d5a6edaf52 Mon Sep 17 00:00:00 2001 From: arvidn Date: Sun, 31 Jul 2016 10:02:54 -0400 Subject: [PATCH 06/10] fix warning --- src/disk_buffer_pool.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/disk_buffer_pool.cpp b/src/disk_buffer_pool.cpp index cda5fb188..f8442418e 100644 --- a/src/disk_buffer_pool.cpp +++ b/src/disk_buffer_pool.cpp @@ -495,8 +495,8 @@ namespace libtorrent m_cache_pool = 0; // attempt to make MacOS not flush this to disk, making close() // block for a long time - int const best_effort = ftruncate(m_cache_fd, 0); - TORRENT_UNUSED(best_effort); + int const best_effort2 = ftruncate(m_cache_fd, 0); + TORRENT_UNUSED(best_effort2); close(m_cache_fd); m_cache_fd = -1; } From 61031f11866511e7bc83e9dab0106545d3468837 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Tue, 2 Aug 2016 00:07:36 -0400 Subject: [PATCH 07/10] hack to allow timeout utp sockets that are about to be closed (#976) --- Jamfile | 4 ++-- src/utp_stream.cpp | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Jamfile b/Jamfile index 726637487..ba51a7be5 100644 --- a/Jamfile +++ b/Jamfile @@ -538,8 +538,8 @@ lib libiconv : : iconv shared /usr/local/lib ; # openssl on linux/bsd/macos etc. lib gcrypt : : gcrypt shared /opt/local/lib ; lib z : : shared z /usr/lib ; -lib crypto : : crypto shared /usr/lib z ; -lib ssl : : ssl shared crypto /opt/local/lib ; +lib crypto : : crypto shared /usr/lib z : : /opt/local/include ; +lib ssl : : ssl shared crypto /opt/local/lib : : /opt/local/include ; lib dl : : shared dl ; # time functions used on linux require librt diff --git a/src/utp_stream.cpp b/src/utp_stream.cpp index f85412ef3..bf6d791bd 100644 --- a/src/utp_stream.cpp +++ b/src/utp_stream.cpp @@ -3572,9 +3572,16 @@ void utp_socket_impl::tick(time_point now) // TIMEOUT! // set cwnd to 1 MSS - m_sm->inc_stats_counter(counters::utp_timeout); - - if (m_outbuf.size()) ++m_num_timeouts; + // the close_reason here is a bit of a hack. When it's set, it indicates + // that the upper layer intends to close the socket. However, it has been + // observed that the SSL shutdown sometimes can hang in a state where + // there's no outstanding data, and it won't receive any more from the + // other end. This catches that case and let the socket time out. + if (m_outbuf.size() || m_close_reason != 0) + { + ++m_num_timeouts; + m_sm->inc_stats_counter(counters::utp_timeout); + } UTP_LOGV("%8p: timeout num-timeouts: %d max-resends: %d confirmed: %d " " acked-seq-num: %d mtu-seq: %d\n" From d4296730d087498b2afa310b8958dc525c222f44 Mon Sep 17 00:00:00 2001 From: Steven Siloti Date: Sun, 7 Aug 2016 12:27:04 -0700 Subject: [PATCH 08/10] fix re-prioritizing a piece while it is downloading (#987) update_piece_state does not expect that the piece's priority has also changed since the last time the piece was updated. Make sure the piece's priority is updated before trying to update the state. --- src/piece_picker.cpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/piece_picker.cpp b/src/piece_picker.cpp index 694553580..ac759699a 100644 --- a/src/piece_picker.cpp +++ b/src/piece_picker.cpp @@ -1834,6 +1834,18 @@ namespace libtorrent p.piece_priority = new_piece_priority; int new_priority = p.priority(this); + if (prev_priority != new_priority && !m_dirty) + { + if (prev_priority == -1) + { + add(index); + } + else + { + update(prev_priority, p.index); + } + } + if (p.downloading()) { std::vector::iterator i = find_dl_piece( @@ -1842,17 +1854,6 @@ namespace libtorrent update_piece_state(i); } - if (prev_priority == new_priority) return ret; - - if (m_dirty) return ret; - if (prev_priority == -1) - { - add(index); - } - else - { - update(prev_priority, p.index); - } return ret; } From 354a866636d1286440d41a33adc95078ea12cd45 Mon Sep 17 00:00:00 2001 From: arvidn Date: Sun, 7 Aug 2016 15:31:48 -0400 Subject: [PATCH 09/10] update changelog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index fcdc10f54..f23781709 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ 1.1.1 release + * fix piece picker bug that could result in division by zero * fix value of current_tracker when all tracker failed * deprecate lt_trackers extension * remove load_asnum_db and load_country_db from python bindings From bc369683dfbcbb520447c26ee3aecb0e262a60af Mon Sep 17 00:00:00 2001 From: Steven Siloti Date: Sun, 7 Aug 2016 13:04:29 -0700 Subject: [PATCH 10/10] handle short reads when exporting a file (#990) handle short reads when exporting a file. Pieces in the part file are not guaranteed to be fully written so short reads need to be handled rather than asserting. --- src/part_file.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/part_file.cpp b/src/part_file.cpp index 9cc9200db..bba8bb1cb 100644 --- a/src/part_file.cpp +++ b/src/part_file.cpp @@ -329,14 +329,14 @@ namespace libtorrent // don't hold the lock during disk I/O l.unlock(); - file::iovec_t const v = { buf.get(), size_t(block_to_copy) }; - int ret = m_file.readv(slot_offset + piece_offset, &v, 1, ec); - TORRENT_ASSERT(ec || ret == block_to_copy); - if (ec || ret != block_to_copy) return; + file::iovec_t v = { buf.get(), size_t(block_to_copy) }; + v.iov_len = m_file.readv(slot_offset + piece_offset, &v, 1, ec); + TORRENT_ASSERT(!ec); + if (ec || v.iov_len == 0) return; - ret = f.writev(file_offset, &v, 1, ec); - TORRENT_ASSERT(ec || ret == block_to_copy); - if (ec || ret != block_to_copy) return; + boost::int64_t ret = f.writev(file_offset, &v, 1, ec); + TORRENT_ASSERT(ec || ret == v.iov_len); + if (ec || ret != v.iov_len) return; // we're done with the disk I/O, grab the lock again to update // the slot map