support for sha1 file-hashes
This commit is contained in:
parent
c6f57ce5d7
commit
b4abe6677d
|
@ -1,3 +1,4 @@
|
||||||
|
* supports calculating sha1 file-hashes when creating torrents
|
||||||
* made the send_buffer_watermark performance warning more meaningful
|
* made the send_buffer_watermark performance warning more meaningful
|
||||||
* supports complete_ago extension
|
* supports complete_ago extension
|
||||||
* dropped zlib as a dependency and builds using puff.c instead
|
* dropped zlib as a dependency and builds using puff.c instead
|
||||||
|
|
|
@ -222,6 +222,7 @@ The ``create_torrent`` class has the following synopsis::
|
||||||
, merkle = 2
|
, merkle = 2
|
||||||
, modification_time = 4
|
, modification_time = 4
|
||||||
, symlink = 8
|
, symlink = 8
|
||||||
|
, calculate_file_hashes = 16
|
||||||
};
|
};
|
||||||
create_torrent(file_storage& fs, int piece_size = 0, int pad_size_limit = -1, int flags = optimize);
|
create_torrent(file_storage& fs, int piece_size = 0, int pad_size_limit = -1, int flags = optimize);
|
||||||
create_torrent(torrent_info const& ti);
|
create_torrent(torrent_info const& ti);
|
||||||
|
@ -233,6 +234,7 @@ The ``create_torrent`` class has the following synopsis::
|
||||||
void set_comment(char const* str);
|
void set_comment(char const* str);
|
||||||
void set_creator(char const* str);
|
void set_creator(char const* str);
|
||||||
void set_hash(int index, sha1_hash const& h);
|
void set_hash(int index, sha1_hash const& h);
|
||||||
|
void set_file_hash(int index, sha1_hash const& h);
|
||||||
void add_url_seed(std::string const& url);
|
void add_url_seed(std::string const& url);
|
||||||
void add_node(std::pair<std::string, int> const& node);
|
void add_node(std::pair<std::string, int> const& node);
|
||||||
void add_tracker(std::string const& url, int tier = 0);
|
void add_tracker(std::string const& url, int tier = 0);
|
||||||
|
@ -254,6 +256,7 @@ create_torrent()
|
||||||
, merkle = 2
|
, merkle = 2
|
||||||
, modification_time = 4
|
, modification_time = 4
|
||||||
, symlink = 8
|
, symlink = 8
|
||||||
|
, calculate_file_hashes = 16
|
||||||
};
|
};
|
||||||
create_torrent(file_storage& fs, int piece_size = 0, int pad_size_limit = -1, int flags = optimize);
|
create_torrent(file_storage& fs, int piece_size = 0, int pad_size_limit = -1, int flags = optimize);
|
||||||
create_torrent(torrent_info const& ti);
|
create_torrent(torrent_info const& ti);
|
||||||
|
@ -303,6 +306,12 @@ symlink
|
||||||
of the symlink so that the original directory structure can be reproduced
|
of the symlink so that the original directory structure can be reproduced
|
||||||
on the downloading side.
|
on the downloading side.
|
||||||
|
|
||||||
|
calculate_file_hashes
|
||||||
|
If this is set, the `set_piece_hashes()`_ function will, as it calculates
|
||||||
|
the piece hashes, also calculate the file hashes and add those associated
|
||||||
|
with each file. Note that unless you use the `set_piece_hashes()`_ function,
|
||||||
|
this flag will have no effect.
|
||||||
|
|
||||||
generate()
|
generate()
|
||||||
----------
|
----------
|
||||||
|
|
||||||
|
@ -366,6 +375,17 @@ to set the hash for every piece in the torrent before generating it. If you have
|
||||||
the files on disk, you can use the high level convenience function to do this.
|
the files on disk, you can use the high level convenience function to do this.
|
||||||
See `set_piece_hashes()`_.
|
See `set_piece_hashes()`_.
|
||||||
|
|
||||||
|
set_file_hash()
|
||||||
|
---------------
|
||||||
|
|
||||||
|
::
|
||||||
|
|
||||||
|
void set_file_hash(int index, sha1_hash const& h);
|
||||||
|
|
||||||
|
This sets the sha1 hash for this file. This hash will end up under the key ``sha1``
|
||||||
|
associated with this file (for multi-file torrents) or in the root info dictionary
|
||||||
|
for single-file torrents.
|
||||||
|
|
||||||
add_url_seed()
|
add_url_seed()
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
|
|
|
@ -1653,9 +1653,12 @@ iterators with the type ``file_entry``.
|
||||||
size_type offset;
|
size_type offset;
|
||||||
size_type size;
|
size_type size;
|
||||||
size_type file_base;
|
size_type file_base;
|
||||||
|
std::string symlink_path;
|
||||||
|
boost::shared_ptr<sha1_hash> filehash;
|
||||||
bool pad_file:1;
|
bool pad_file:1;
|
||||||
bool hidden_attribute:1;
|
bool hidden_attribute:1;
|
||||||
bool executable_attribute:1;
|
bool executable_attribute:1;
|
||||||
|
bool symlink_attribute:1;
|
||||||
};
|
};
|
||||||
|
|
||||||
The ``path`` is the full (relative) path of each file. i.e. if it is a multi-file
|
The ``path`` is the full (relative) path of each file. i.e. if it is a multi-file
|
||||||
|
@ -1678,6 +1681,16 @@ They are just there to make sure the next file is aligned to a particular byte o
|
||||||
or piece boundry. These files should typically be hidden from an end user. They are
|
or piece boundry. These files should typically be hidden from an end user. They are
|
||||||
not written to disk.
|
not written to disk.
|
||||||
|
|
||||||
|
``hidden_attribute`` is true if the file was marked as hidden (on windows).
|
||||||
|
|
||||||
|
``executable_attribute`` is true if the file was marked as executable (posix)
|
||||||
|
|
||||||
|
``symlink_attribute`` is true if the file was a symlink. If this is the case
|
||||||
|
the ``symlink_path`` specifies the original location where the data for this file
|
||||||
|
was found.
|
||||||
|
|
||||||
|
``filehash`` is a pointer that is set in case the torrent file included a sha1 hash
|
||||||
|
for this file. This may be use to look up more sources for this file on other networks.
|
||||||
|
|
||||||
num_files() file_at()
|
num_files() file_at()
|
||||||
---------------------
|
---------------------
|
||||||
|
|
|
@ -119,13 +119,15 @@ int main(int argc, char* argv[])
|
||||||
{
|
{
|
||||||
int first = t.map_file(index, 0, 0).piece;
|
int first = t.map_file(index, 0, 0).piece;
|
||||||
int last = t.map_file(index, (std::max)(i->size-1, size_type(0)), 0).piece;
|
int last = t.map_file(index, (std::max)(i->size-1, size_type(0)), 0).piece;
|
||||||
printf(" %11"PRId64" %c%c%c%c [ %4d, %4d ] %s %s%s\n"
|
printf(" %11"PRId64" %c%c%c%c [ %4d, %4d ] %s %s %s%s\n"
|
||||||
, i->size
|
, i->size
|
||||||
, (i->pad_file?'p':'-')
|
, (i->pad_file?'p':'-')
|
||||||
, (i->executable_attribute?'x':'-')
|
, (i->executable_attribute?'x':'-')
|
||||||
, (i->hidden_attribute?'h':'-')
|
, (i->hidden_attribute?'h':'-')
|
||||||
, (i->symlink_attribute?'l':'-')
|
, (i->symlink_attribute?'l':'-')
|
||||||
, first, last, i->path.c_str()
|
, first, last
|
||||||
|
, i->filehash ? to_hex(i->filehash->to_string()).c_str() : ""
|
||||||
|
, i->path.c_str()
|
||||||
, i->symlink_attribute ? "-> ": ""
|
, i->symlink_attribute ? "-> ": ""
|
||||||
, i->symlink_attribute ? i->symlink_path.c_str() : "");
|
, i->symlink_attribute ? i->symlink_path.c_str() : "");
|
||||||
}
|
}
|
||||||
|
|
|
@ -66,6 +66,9 @@ void print_usage()
|
||||||
"OPTIONS:\n"
|
"OPTIONS:\n"
|
||||||
"-m generate a merkle hash tree torrent.\n"
|
"-m generate a merkle hash tree torrent.\n"
|
||||||
" merkle torrents require client support\n"
|
" merkle torrents require client support\n"
|
||||||
|
"-f include sha-1 file hashes in the torrent\n"
|
||||||
|
" this helps supporting mixing sources from\n"
|
||||||
|
" other networks\n"
|
||||||
"-w url adds a web seed to the torrent with\n"
|
"-w url adds a web seed to the torrent with\n"
|
||||||
" the specified url\n"
|
" the specified url\n"
|
||||||
"-t url adds the specified tracker to the\n"
|
"-t url adds the specified tracker to the\n"
|
||||||
|
@ -144,6 +147,9 @@ int main(int argc, char* argv[])
|
||||||
++i;
|
++i;
|
||||||
outfile = argv[i];
|
outfile = argv[i];
|
||||||
break;
|
break;
|
||||||
|
case 'f':
|
||||||
|
flags |= create_torrent::calculate_file_hashes;
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
print_usage();
|
print_usage();
|
||||||
return 1;
|
return 1;
|
||||||
|
|
|
@ -71,6 +71,7 @@ namespace libtorrent
|
||||||
, merkle = 2
|
, merkle = 2
|
||||||
, modification_time = 4
|
, modification_time = 4
|
||||||
, symlinks = 8
|
, symlinks = 8
|
||||||
|
, calculate_file_hashes = 16
|
||||||
};
|
};
|
||||||
|
|
||||||
create_torrent(file_storage& fs, int piece_size = 0
|
create_torrent(file_storage& fs, int piece_size = 0
|
||||||
|
@ -83,6 +84,7 @@ namespace libtorrent
|
||||||
void set_comment(char const* str);
|
void set_comment(char const* str);
|
||||||
void set_creator(char const* str);
|
void set_creator(char const* str);
|
||||||
void set_hash(int index, sha1_hash const& h);
|
void set_hash(int index, sha1_hash const& h);
|
||||||
|
void set_file_hash(int index, sha1_hash const& h);
|
||||||
void add_url_seed(std::string const& url);
|
void add_url_seed(std::string const& url);
|
||||||
void add_node(std::pair<std::string, int> const& node);
|
void add_node(std::pair<std::string, int> const& node);
|
||||||
void add_tracker(std::string const& url, int tier = 0);
|
void add_tracker(std::string const& url, int tier = 0);
|
||||||
|
@ -93,6 +95,8 @@ namespace libtorrent
|
||||||
int piece_size(int i) const { return m_files.piece_size(i); }
|
int piece_size(int i) const { return m_files.piece_size(i); }
|
||||||
bool priv() const { return m_private; }
|
bool priv() const { return m_private; }
|
||||||
|
|
||||||
|
bool should_add_file_hashes() const { return m_calculate_file_hashes; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
file_storage& m_files;
|
file_storage& m_files;
|
||||||
|
@ -109,6 +113,8 @@ namespace libtorrent
|
||||||
|
|
||||||
std::vector<sha1_hash> m_piece_hash;
|
std::vector<sha1_hash> m_piece_hash;
|
||||||
|
|
||||||
|
std::vector<sha1_hash> m_filehashes;
|
||||||
|
|
||||||
// dht nodes to add to the routing table/bootstrap from
|
// dht nodes to add to the routing table/bootstrap from
|
||||||
typedef std::vector<std::pair<std::string, int> > nodes_t;
|
typedef std::vector<std::pair<std::string, int> > nodes_t;
|
||||||
nodes_t m_nodes;
|
nodes_t m_nodes;
|
||||||
|
@ -153,6 +159,11 @@ namespace libtorrent
|
||||||
// the torrent file. The full data of the pointed-to
|
// the torrent file. The full data of the pointed-to
|
||||||
// file is still included
|
// file is still included
|
||||||
bool m_include_symlinks:1;
|
bool m_include_symlinks:1;
|
||||||
|
|
||||||
|
// this is only used by set_piece_hashes(). It will
|
||||||
|
// calculate sha1 hashes for each file and add it
|
||||||
|
// to the file list
|
||||||
|
bool m_calculate_file_hashes:1;
|
||||||
};
|
};
|
||||||
|
|
||||||
namespace detail
|
namespace detail
|
||||||
|
@ -234,6 +245,11 @@ namespace libtorrent
|
||||||
boost::scoped_ptr<storage_interface> st(
|
boost::scoped_ptr<storage_interface> st(
|
||||||
default_storage_constructor(const_cast<file_storage&>(t.files()), 0, p, fp));
|
default_storage_constructor(const_cast<file_storage&>(t.files()), 0, p, fp));
|
||||||
|
|
||||||
|
// if we're calculating file hashes as well, use this hasher
|
||||||
|
hasher filehash;
|
||||||
|
int file_idx = 0;
|
||||||
|
size_type left_in_file = t.files().at(0).size;
|
||||||
|
|
||||||
// calculate the hash for all pieces
|
// calculate the hash for all pieces
|
||||||
int num = t.num_pieces();
|
int num = t.num_pieces();
|
||||||
piece_holder buf(t.piece_length());
|
piece_holder buf(t.piece_length());
|
||||||
|
@ -247,6 +263,29 @@ namespace libtorrent
|
||||||
ec = st->error();
|
ec = st->error();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (t.should_add_file_hashes())
|
||||||
|
{
|
||||||
|
int left_in_piece = t.piece_size(i);
|
||||||
|
// the number of bytes from this file we just read
|
||||||
|
while (left_in_piece > 0)
|
||||||
|
{
|
||||||
|
int to_hash_for_file = (std::min)(size_type(left_in_piece), left_in_file);
|
||||||
|
filehash.update(buf.bytes(), to_hash_for_file);
|
||||||
|
left_in_file -= to_hash_for_file;
|
||||||
|
left_in_piece -= to_hash_for_file;
|
||||||
|
if (left_in_file == 0)
|
||||||
|
{
|
||||||
|
if (!t.files().at(file_idx).pad_file)
|
||||||
|
t.set_file_hash(file_idx, filehash.final());
|
||||||
|
filehash.reset();
|
||||||
|
file_idx++;
|
||||||
|
if (file_idx >= t.files().num_files()) break;
|
||||||
|
left_in_file = t.files().at(file_idx).size;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
hasher h(buf.bytes(), t.piece_size(i));
|
hasher h(buf.bytes(), t.piece_size(i));
|
||||||
t.set_hash(i, h.final());
|
t.set_hash(i, h.final());
|
||||||
f(i);
|
f(i);
|
||||||
|
|
|
@ -36,10 +36,12 @@ POSSIBILITY OF SUCH DAMAGE.
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <ctime>
|
#include <ctime>
|
||||||
|
#include <boost/shared_ptr.hpp>
|
||||||
|
|
||||||
#include "libtorrent/size_type.hpp"
|
#include "libtorrent/size_type.hpp"
|
||||||
#include "libtorrent/assert.hpp"
|
#include "libtorrent/assert.hpp"
|
||||||
#include "libtorrent/peer_request.hpp"
|
#include "libtorrent/peer_request.hpp"
|
||||||
|
#include "libtorrent/peer_id.hpp"
|
||||||
|
|
||||||
namespace libtorrent
|
namespace libtorrent
|
||||||
{
|
{
|
||||||
|
@ -59,11 +61,12 @@ namespace libtorrent
|
||||||
// compressed into a single file, such as a so-called part file.
|
// compressed into a single file, such as a so-called part file.
|
||||||
size_type file_base;
|
size_type file_base;
|
||||||
std::time_t mtime;
|
std::time_t mtime;
|
||||||
|
std::string symlink_path;
|
||||||
|
boost::shared_ptr<sha1_hash> filehash;
|
||||||
bool pad_file:1;
|
bool pad_file:1;
|
||||||
bool hidden_attribute:1;
|
bool hidden_attribute:1;
|
||||||
bool executable_attribute:1;
|
bool executable_attribute:1;
|
||||||
bool symlink_attribute:1;
|
bool symlink_attribute:1;
|
||||||
std::string symlink_path;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct TORRENT_EXPORT file_slice
|
struct TORRENT_EXPORT file_slice
|
||||||
|
|
|
@ -115,6 +115,7 @@ namespace libtorrent
|
||||||
, m_merkle_torrent(flags & merkle)
|
, m_merkle_torrent(flags & merkle)
|
||||||
, m_include_mtime(flags & modification_time)
|
, m_include_mtime(flags & modification_time)
|
||||||
, m_include_symlinks(flags & symlinks)
|
, m_include_symlinks(flags & symlinks)
|
||||||
|
, m_calculate_file_hashes(flags & calculate_file_hashes)
|
||||||
{
|
{
|
||||||
TORRENT_ASSERT(fs.num_files() > 0);
|
TORRENT_ASSERT(fs.num_files() > 0);
|
||||||
|
|
||||||
|
@ -303,6 +304,10 @@ namespace libtorrent
|
||||||
for (char const* e = split.c_str(); e != 0; e = next_path_element(e))
|
for (char const* e = split.c_str(); e != 0; e = next_path_element(e))
|
||||||
sympath_e.list().push_back(entry(e));
|
sympath_e.list().push_back(entry(e));
|
||||||
}
|
}
|
||||||
|
if (!m_filehashes.empty())
|
||||||
|
{
|
||||||
|
info["sha1"] = m_filehashes[0].to_string();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -347,6 +352,11 @@ namespace libtorrent
|
||||||
for (char const* e = split.c_str(); e != 0; e = next_path_element(e))
|
for (char const* e = split.c_str(); e != 0; e = next_path_element(e))
|
||||||
sympath_e.list().push_back(entry(e));
|
sympath_e.list().push_back(entry(e));
|
||||||
}
|
}
|
||||||
|
int file_index = i - m_files.begin();
|
||||||
|
if (!m_filehashes.empty() && m_filehashes[file_index] != sha1_hash())
|
||||||
|
{
|
||||||
|
file_e["sha1"] = m_filehashes[file_index].to_string();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -423,6 +433,14 @@ namespace libtorrent
|
||||||
m_piece_hash[index] = h;
|
m_piece_hash[index] = h;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void create_torrent::set_file_hash(int index, sha1_hash const& h)
|
||||||
|
{
|
||||||
|
TORRENT_ASSERT(index >= 0);
|
||||||
|
TORRENT_ASSERT(index < (int)m_files.num_files());
|
||||||
|
if (m_filehashes.empty()) m_filehashes.resize(m_files.num_files());
|
||||||
|
m_filehashes[index] = h;
|
||||||
|
}
|
||||||
|
|
||||||
void create_torrent::add_node(std::pair<std::string, int> const& node)
|
void create_torrent::add_node(std::pair<std::string, int> const& node)
|
||||||
{
|
{
|
||||||
m_nodes.push_back(node);
|
m_nodes.push_back(node);
|
||||||
|
|
|
@ -291,6 +291,13 @@ namespace libtorrent
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
lazy_entry const* fh = dict.dict_find_string("sha1");
|
||||||
|
if (fh && fh->string_length() == 20)
|
||||||
|
{
|
||||||
|
target.filehash.reset(new sha1_hash);
|
||||||
|
std::memcpy(&(*target.filehash)[0], fh->string_ptr(), 20);
|
||||||
|
}
|
||||||
|
|
||||||
lazy_entry const* s_p = dict.dict_find("symlink path");
|
lazy_entry const* s_p = dict.dict_find("symlink path");
|
||||||
if (s_p != 0 && s_p->type() == lazy_entry::list_t)
|
if (s_p != 0 && s_p->type() == lazy_entry::list_t)
|
||||||
{
|
{
|
||||||
|
@ -339,7 +346,7 @@ namespace libtorrent
|
||||||
int cnt = 0;
|
int cnt = 0;
|
||||||
std::set<std::string, string_less_no_case> files;
|
std::set<std::string, string_less_no_case> files;
|
||||||
|
|
||||||
// as long as we this file already exists
|
// as long as this file already exists
|
||||||
// increase the counter
|
// increase the counter
|
||||||
while (!files.insert(e.path).second)
|
while (!files.insert(e.path).second)
|
||||||
{
|
{
|
||||||
|
@ -793,6 +800,13 @@ namespace libtorrent
|
||||||
e.symlink_path = combine_path(e.symlink_path, path_element);
|
e.symlink_path = combine_path(e.symlink_path, path_element);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
lazy_entry const* fh = info.dict_find_string("sha1");
|
||||||
|
if (fh && fh->string_length() == 20)
|
||||||
|
{
|
||||||
|
e.filehash.reset(new sha1_hash);
|
||||||
|
std::memcpy(&(*e.filehash)[0], fh->string_ptr(), 20);
|
||||||
|
}
|
||||||
|
|
||||||
// bitcomet pad file
|
// bitcomet pad file
|
||||||
if (e.path.find("_____padding_file_") != std::string::npos)
|
if (e.path.find("_____padding_file_") != std::string::npos)
|
||||||
e.pad_file = true;
|
e.pad_file = true;
|
||||||
|
|
Loading…
Reference in New Issue