merged changes from RC_1_0

ChangeLog

@@ -27,6 +27,8 @@
 	* almost completely changed the storage interface (for custom storage)
 	* added support for hashing pieces in multiple threads
 
+1.0.2 release
+
 	* added missing force_proxy to python binding
 	* anonymous_mode defaults to false
 	* make DHT DOS detection more forgiving to bursts
@@ -103,6 +105,9 @@
 	* fix uTP edge case where udp socket buffer fills up
 	* fix nagle implementation in uTP
 
+0.16.18 release
+
+	* fix python3 support
 	* fix bug in lt_donthave extension
 	* expose i2p_alert to python. cleaning up of i2p connection code
 	* fixed overflow and download performance issue when downloading at high rates

bindings/python/src/big_number.cpp

@@ -4,6 +4,19 @@
 
 #include <libtorrent/sha1_hash.hpp>
 #include <boost/python.hpp>
+#include "bytes.hpp"
+
+long get_hash(boost::python::object o)
+{
+    using namespace boost::python;
+    return PyObject_Hash(str(o).ptr());
+}
+
+using namespace libtorrent;
+
+bytes sha1_hash_bytes(const sha1_hash& bn) {
+    return bytes(bn.to_string());
+}
 
 void bind_sha1_hash()
 {
@@ -19,7 +32,8 @@ void bind_sha1_hash()
         .def("clear", &sha1_hash::clear)
         .def("is_all_zeros", &sha1_hash::is_all_zeros)
         .def("to_string", &sha1_hash::to_string)
-//        .def("__getitem__", &sha1_hash::opreator[])
+        .def("__hash__", get_hash)
+        .def("to_bytes", sha1_hash_bytes)
         ;
 
     scope().attr("big_number") = scope().attr("sha1_hash"); 

bindings/python/src/bytes.hpp

@@ -0,0 +1,18 @@
+// Copyright Arvid Norberg 2006-2013. Use, modification and distribution is
+// subject to the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#ifndef BYTES_HPP
+#define BYTES_HPP
+
+#include <string>
+
+struct bytes
+{
+    bytes(std::string const& s): arr(s) {}
+    bytes() {}
+    std::string arr;
+};
+
+#endif
+

bindings/python/src/create_torrent.cpp

@@ -6,20 +6,21 @@
 #include <libtorrent/create_torrent.hpp>
 #include <libtorrent/file_storage.hpp>
 #include "libtorrent/torrent_info.hpp"
+#include "bytes.hpp"
 
 using namespace boost::python;
 using namespace libtorrent;
 
 namespace
 {
-    void set_hash(create_torrent& c, int p, char const* hash)
+    void set_hash(create_torrent& c, int p, bytes const& b)
     {
-        c.set_hash(p, sha1_hash(hash));
+        c.set_hash(p, sha1_hash(b.arr));
     }
 
-    void set_file_hash(create_torrent& c, int f, char const* hash)
+    void set_file_hash(create_torrent& c, int f, bytes const& b)
     {
-        c.set_file_hash(f, sha1_hash(hash));
+        c.set_file_hash(f, sha1_hash(b.arr));
     }
 
     void call_python_object(boost::python::object const& obj, int i)

bindings/python/src/entry.cpp

@@ -4,6 +4,7 @@
 
 #include <boost/python.hpp>
 #include <libtorrent/session.hpp>
+#include "bytes.hpp"
 
 using namespace boost::python;
 using namespace libtorrent;
@@ -27,7 +28,7 @@ struct entry_to_python
         dict result;
 
         for (entry::dictionary_type::const_iterator i(d.begin()), e(d.end()); i != e; ++i)
-            result[i->first] = i->second;
+            result[bytes(i->first)] = i->second;
 
         return result;
     }
@@ -39,7 +40,7 @@ struct entry_to_python
         case entry::int_t:
             return object(e.integer());
         case entry::string_t:
-            return object(e.string());
+            return object(bytes(e.string()));
         case entry::list_t:
             return convert(e.list());
         case entry::dictionary_t:
@@ -86,14 +87,26 @@ struct entry_from_python
             entry result(entry::dictionary_t);
 
             for (std::size_t i = 0; i < length; ++i)
+            {
+                if (extract<bytes>(items[i][0]).check())
                 {
                     result.dict().insert(
                         std::make_pair(
-                        extract<char const*>(items[i][0])()
+                            extract<bytes>(items[i][0])().arr,
-                      , construct0(items[i][1])
+                            construct0(items[i][1])
                         )
                     );
                 }
+                else
+                {
+                    result.dict().insert(
+                        std::make_pair(
+                            extract<char const*>(items[i][0])(),
+                            construct0(items[i][1])
+                        )
+                    );
+                }
+            }
 
             return result;
         }
@@ -111,6 +124,10 @@ struct entry_from_python
 
             return result;
         }
+        else if (extract<bytes>(e).check())
+        {
+            return entry(extract<bytes>(e)().arr);
+        }
         else if (extract<str>(e).check())
         {
              return entry(extract<std::string>(e)());

bindings/python/src/session.cpp

@@ -23,6 +23,7 @@
 #include <libtorrent/extensions/ut_pex.hpp>
 
 #include "gil.hpp"
+#include "bytes.hpp"
 
 using namespace boost::python;
 using namespace libtorrent;
@@ -171,7 +172,7 @@ namespace
             p.ti = extract<boost::shared_ptr<torrent_info> >(params["ti"]);
 
         if (params.has_key("info_hash"))
-            p.info_hash = extract<sha1_hash>(params["info_hash"]);
+            p.info_hash = sha1_hash(bytes(extract<bytes>(params["info_hash"])).arr);
         if (params.has_key("name"))
             p.name = extract<std::string>(params["name"]);
         p.save_path = extract<std::string>(params["save_path"]);

bindings/python/src/torrent_info.cpp

@@ -7,6 +7,7 @@
 #include <libtorrent/torrent_info.hpp>
 #include "libtorrent/session_settings.hpp"
 #include "libtorrent/time.hpp"
+#include "bytes.hpp"
 
 using namespace boost::python;
 using namespace libtorrent;
@@ -73,7 +74,7 @@ namespace
         for (std::vector<sha1_hash>::const_iterator i = mt.begin()
             , end(mt.end()); i != end; ++i)
         {
-            ret.append(i->to_string());
+            ret.append(bytes(i->to_string()));
         }
         return ret;
     }
@@ -82,7 +83,7 @@ namespace
     {
         std::vector<sha1_hash> h;
         for (int i = 0, e = len(hashes); i < e; ++i)
-            h.push_back(sha1_hash(extract<char const*>(hashes[i])));
+            h.push_back(sha1_hash(bytes(extract<bytes>(hashes[i])).arr));
 
         ti.set_merkle_tree(h);
     }

bindings/python/src/utility.cpp

@@ -5,30 +5,84 @@
 #include <libtorrent/identify_client.hpp>
 #include <libtorrent/bencode.hpp>
 #include <boost/python.hpp>
+#include "bytes.hpp"
 
 using namespace boost::python;
 using namespace libtorrent;
 
+struct bytes_to_python
+{
+    static PyObject* convert(bytes const& p)
+    {
+#if PY_MAJOR_VERSION >= 3
+        PyObject *ret = PyBytes_FromStringAndSize(p.arr.c_str(), p.arr.size());
+#else
+        PyObject *ret = PyString_FromStringAndSize(p.arr.c_str(), p.arr.size());
+#endif
+        return ret;
+    }
+};
+
+struct bytes_from_python
+{
+    bytes_from_python()
+    {
+        converter::registry::push_back(
+            &convertible, &construct, type_id<bytes>());
+    }
+
+    static void* convertible(PyObject* x)
+    {
+#if PY_MAJOR_VERSION >= 3
+        return PyBytes_Check(x) ? x : NULL;
+#else
+        return PyString_Check(x) ? x : NULL;
+#endif
+    }
+
+    static void construct(PyObject* x, converter::rvalue_from_python_stage1_data* data)
+    {
+#if PY_MAJOR_VERSION >= 3
+        void* storage = ((converter::rvalue_from_python_storage<bytes>*)data)->storage.bytes;
+        bytes* ret = new (storage) bytes();
+        ret->arr.resize(PyBytes_Size(x));
+        memcpy(&ret->arr[0], PyBytes_AsString(x), ret->arr.size());
+        data->convertible = storage;
+#else
+        void* storage = ((converter::rvalue_from_python_storage<bytes>*)data)->storage.bytes;
+        bytes* ret = new (storage) bytes();
+        ret->arr.resize(PyString_Size(x));
+        memcpy(&ret->arr[0], PyString_AsString(x), ret->arr.size());
+        data->convertible = storage;
+#endif
+    }
+};
+
+
 object client_fingerprint_(peer_id const& id)
 {
     boost::optional<fingerprint> result = client_fingerprint(id);
     return result ? object(*result) : object();
 }
 
-entry bdecode_(std::string const& data)
+entry bdecode_(bytes const& data)
 {
-    return bdecode(data.begin(), data.end());
+    return bdecode(data.arr.begin(), data.arr.end());
 }
 
-std::string bencode_(entry const& e)
+bytes bencode_(entry const& e)
 {
-    std::string result;
+    bytes result;
-    bencode(std::back_inserter(result), e);
+    bencode(std::back_inserter(result.arr), e);
     return result;
 }
 
 void bind_utility()
 {
+    // TODO: it would be nice to install converters for sha1_hash as well
+    to_python_converter<bytes, bytes_to_python>();
+    bytes_from_python();
+
     def("identify_client", &libtorrent::identify_client);
     def("client_fingerprint", &client_fingerprint_);
     def("bdecode", &bdecode_);

docs/manual.rst

@@ -124,7 +124,9 @@ cannot rely on ``error_code::message()`` to generate your strings.
 The numeric values of the errors are part of the API and will stay the same, although
 new error codes may be appended at the end.
 
-Here's a simple example of how to translate error codes::
+Here's a simple example of how to translate error codes:
+
+.. code:: c++
 
 	std::string error_code_to_string(boost::system::error_code const& ec)
 	{

include/libtorrent/sliding_average.hpp

@@ -33,44 +33,51 @@ POSSIBILITY OF SUCH DAMAGE.
 #ifndef TORRENT_SLIDING_AVERAGE_HPP_INCLUDED
 #define TORRENT_SLIDING_AVERAGE_HPP_INCLUDED
 
+#include <boost/cstdint.hpp>
+
 namespace libtorrent
 {
 
-// a sliding average accumulator. Add samples to it and it
+// an exponential moving average accumulator. Add samples to it and it keeps
-// keeps track of a sliding mean value and an average deviation
+// track of a moving mean value and an average deviation
-template <int history_size>
+template <int inverted_gain>
 struct sliding_average
 {
-	sliding_average(): m_mean(-1), m_average_deviation(-1) {}
+	sliding_average(): m_mean(0), m_average_deviation(0), m_num_samples(0) {}
 
 	void add_sample(int s)
 	{
-		TORRENT_ASSERT(s >= 0);
+		// fixed point
-		if (s < 0) s = 0;
+		s *= 64;
-		if (m_mean == -1)
+		int deviation;
-		{
-			m_mean = s;
-			return;
-		}
-		int deviation = abs(m_mean - s);
 
-		m_mean = m_mean - m_mean / history_size + s / history_size;
+		if (m_num_samples > 0)
+			deviation = abs(m_mean - s);
 
-		if (m_average_deviation == -1)
+		if (m_num_samples < inverted_gain)
-		{
+			++m_num_samples;
-			m_average_deviation = deviation;
+
-			return;
+		m_mean += (s - m_mean) / m_num_samples;
+
+		if (m_num_samples > 1) {
+			// the the exact same thing for deviation off the mean except -1 on
+			// the samples, because the number of deviation samples always lags
+			// behind by 1 (you need to actual samples to have a single deviation
+			// sample).
+			m_average_deviation += (deviation - m_average_deviation) / (m_num_samples - 1);
 		}
-		m_average_deviation = m_average_deviation - m_average_deviation
-			/ history_size + deviation / history_size;
 	}
 
-	int mean() const { return m_mean != -1 ? m_mean : 0; }
+	int mean() const { return m_num_samples > 0 ? (m_mean + 32) / 64 : 0; }
-	int avg_deviation() const { return m_average_deviation != -1 ? m_average_deviation : 0; }
+	int avg_deviation() const { return m_num_samples > 1 ? (m_average_deviation + 32) / 64 : 0; }
 
 private:
+	// both of these are fixed point values (* 64)
 	int m_mean;
 	int m_average_deviation;
+	// the number of samples we have received, but no more than inverted_gain
+	// this is the effective inverted_gain
+	int m_num_samples;
 };
 
 struct average_accumulator
@@ -100,7 +107,7 @@ struct average_accumulator
 	}
 
 	int m_num_samples;
-	size_type m_sample_sum;
+	boost::uint64_t m_sample_sum;
 };
 
 }

test/Jamfile

@@ -89,6 +89,7 @@ feature launcher : none valgrind : composite ;
 feature.compose <launcher>valgrind : <testing.launcher>"valgrind --tool=memcheck -v --num-callers=20 --read-var-info=yes --track-origins=yes --error-exitcode=222 --suppressions=valgrind_suppressions.txt" <valgrind>on ;
 
 test-suite libtorrent : 	
+	[ run test_sliding_average.cpp ]
 	[ run test_socket_io.cpp ]
 	[ run test_random.cpp ]
 	[ run test_utf8.cpp ]

test/Makefile.am

@@ -58,7 +58,8 @@ test_programs = \
   test_remap_files           \
   test_gzip                  \
   test_utf8                  \
-  test_socket_io
+  test_socket_io             \
+  test_sliding_average
 
 if ENABLE_TESTS
 check_PROGRAMS = $(test_programs)
@@ -188,6 +189,7 @@ test_remap_files_SOURCES = test_remap_files.cpp
 test_gzip_SOURCES = test_gzip.cpp
 test_utf8_SOURCES = test_utf8.cpp
 test_socket_io_SOURCES = test_socket_io.cpp
+test_sliding_average_SOURCES = test_sliding_average.cpp
 
 LDADD = libtest.la $(top_builddir)/src/libtorrent-rasterbar.la
 

test/test_sliding_average.cpp

@@ -0,0 +1,95 @@
+/*
+
+Copyright (c) 2014, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "test.hpp"
+#include "libtorrent/sliding_average.hpp"
+
+// normal distributed samples. mean=60 stddev=10
+int samples[] =  {
+49, 51, 60, 46, 65, 53, 76, 59, 57, 54, 56, 51, 45, 80, 53, 62,
+69, 67, 66, 56, 56, 61, 52, 61, 61, 62, 59, 53, 48, 68, 47, 47,
+63, 51, 53, 54, 46, 65, 64, 64, 45, 68, 64, 66, 53, 42, 57, 58,
+57, 47, 55, 59, 64, 61, 37, 67, 55, 52, 60, 60, 44, 57, 50, 77,
+56, 54, 49, 68, 66, 64, 47, 60, 46, 47, 81, 74, 65, 62, 44, 75,
+65, 43, 58, 59, 53, 67, 49, 51, 33, 47, 49, 50, 54, 48, 55, 80,
+67, 51, 66, 52, 48, 57, 30, 51, 72, 65, 78, 56, 74, 68, 49, 66,
+63, 57, 61, 62, 64, 62, 61, 52, 67, 64, 59, 61, 69, 60, 54, 69 };
+
+
+int test_main()
+{
+	using namespace libtorrent;
+
+	// make sure we react quickly for the first few samples
+	{
+		sliding_average<10> avg;
+
+		avg.add_sample(-10);
+		avg.add_sample(10);
+
+		TEST_EQUAL(avg.mean(), 0);
+	}
+	{
+		sliding_average<10> avg;
+
+		avg.add_sample(10);
+		avg.add_sample(20);
+
+		TEST_EQUAL(avg.mean(), 15);
+	}
+
+	// make sure we converge
+	{
+		sliding_average<10> avg;
+		avg.add_sample(100);
+		for (int i = 0; i < 20; ++i)
+			avg.add_sample(10);
+		TEST_CHECK(abs(avg.mean() - 10) <= 3);
+	}
+	{
+		sliding_average<10> avg;
+		avg.add_sample(-100);
+		for (int i = 0; i < 20; ++i)
+			avg.add_sample(-10);
+		TEST_CHECK(abs(avg.mean() + 10) <= 3);
+	}
+
+	// test with a more realistic input
+	{
+		sliding_average<10> avg;
+		for (int i = 0; i < sizeof(samples)/sizeof(samples[0]); ++i)
+			avg.add_sample(samples[i]);
+		TEST_CHECK(abs(avg.mean() - 60) <= 3);
+	}
+	return 0;
+}
+