From c223291fb4462e0b364f847e9cff1aeaa688e92a Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Tue, 18 Jan 2011 03:41:54 +0000 Subject: [PATCH] added support for RSS feeds --- CMakeLists.txt | 1 + ChangeLog | 1 + Jamfile | 1 + bindings/python/Makefile.am | 1 + bindings/python/rss_reader.py | 35 + bindings/python/src/session.cpp | 89 ++- docs/manual.html | 818 ++++++++++++++++------ docs/manual.rst | 253 +++++++ docs/python_binding.html | 6 + examples/Jamfile | 1 + examples/Makefile.am | 7 + examples/rss_reader.cpp | 173 +++++ include/libtorrent/Makefile.am | 1 + include/libtorrent/add_torrent_params.hpp | 2 + include/libtorrent/alert.hpp | 1 + include/libtorrent/alert_types.hpp | 25 +- include/libtorrent/aux_/session_impl.hpp | 20 +- include/libtorrent/rss.hpp | 174 +++++ include/libtorrent/session.hpp | 8 +- include/libtorrent/settings.hpp | 5 +- include/libtorrent/torrent.hpp | 9 + include/libtorrent/torrent_handle.hpp | 1 + src/Makefile.am | 1 + src/alert.cpp | 9 + src/http_connection.cpp | 4 +- src/rss.cpp | 580 +++++++++++++++ src/session.cpp | 19 + src/session_impl.cpp | 124 +++- src/settings.cpp | 14 +- src/torrent.cpp | 95 +++ test/test_natpmp.cpp | 3 +- 31 files changed, 2245 insertions(+), 236 deletions(-) create mode 100644 bindings/python/rss_reader.py create mode 100644 examples/rss_reader.cpp create mode 100644 include/libtorrent/rss.hpp create mode 100644 src/rss.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f4a1ff62e..fe4ffbc22 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,7 @@ set(sources piece_picker policy puff + rss session session_impl settings diff --git a/ChangeLog b/ChangeLog index 3adafc2a5..abd82514f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ + * added support for RSS feeds * fixed up some edge cases in DHT routing table and improved unit test of it * added error category and error codes for HTTP errors * made the DHT implementation slightly more robust against routing table poisoning and node ID spoofing diff --git a/Jamfile b/Jamfile index fad45554e..e194a3a71 100755 --- a/Jamfile +++ b/Jamfile @@ -397,6 +397,7 @@ SOURCES = piece_picker policy puff + rss session session_impl settings diff --git a/bindings/python/Makefile.am b/bindings/python/Makefile.am index 4bc8b9c50..1d525f551 100644 --- a/bindings/python/Makefile.am +++ b/bindings/python/Makefile.am @@ -3,6 +3,7 @@ EXTRA_DIST = \ Jamfile \ setup.py \ client.py \ + rss_reader.py \ simple_client.py \ src/alert.cpp \ src/big_number.cpp \ diff --git a/bindings/python/rss_reader.py b/bindings/python/rss_reader.py new file mode 100644 index 000000000..17a2c59fe --- /dev/null +++ b/bindings/python/rss_reader.py @@ -0,0 +1,35 @@ +#!/usr/bin/python + +import sys +import libtorrent as lt +import time + +if len(sys.argv) != 2: + print('usage: rss_reader.py rss-feed-url') + sys.exit(1) + +ses = lt.session() + +h = ses.add_feed({'url': sys.argv[1], 'auto_download': False}) +f = h.get_feed_status() +spinner = ['|', '/', '-', '\\'] +i = 0 +while f['updating']: + time.sleep(0.1) + i = (i + 1) % 4 + print('\b%s' % spinner[i]), + sys.stdout.flush() + f = h.get_feed_status() + +print('\n\nFEED: %s' % f['url']) +if len(f['error']) > 0: + print('ERROR: %s' % f['error']) + +print(' %s\n %s\n' % (f['title'], f['description'])) +print(' ttl: %d minutes' % f['ttl']) + +for item in f['items']: + print('\n%s\n------------------------------------------------------' % item['title']) + print(' url: %s\n size: %d\n uuid: %s\n description: %s' % (item['url'], item['size'], item['uuid'], item['description'])) + print(' comment: %s\n category: %s' % (item['comment'], item['category'])) + diff --git a/bindings/python/src/session.cpp b/bindings/python/src/session.cpp index 59067ac1b..19aff8099 100644 --- a/bindings/python/src/session.cpp +++ b/bindings/python/src/session.cpp @@ -140,10 +140,8 @@ namespace #endif #endif - torrent_handle add_torrent(session& s, dict params) + void dict_to_add_torrent_params(dict params, add_torrent_params& p) { - add_torrent_params p; - if (params.has_key("ti")) p.ti = new torrent_info(extract(params["ti"])); @@ -191,6 +189,12 @@ namespace p.trackerid = extract(params["trackerid"]); if (params.has_key("url")) p.url = extract(params["url"]); + } + + torrent_handle add_torrent(session& s, dict params) + { + add_torrent_params p; + dict_to_add_torrent_params(params, p); #ifndef BOOST_NO_EXCEPTIONS return s.add_torrent(p); @@ -200,6 +204,76 @@ namespace #endif } + void dict_to_feed_settings(dict params, feed_settings& feed) + { + if (params.has_key("auto_download")) + feed.auto_download = extract(params["auto_download"]); + if (params.has_key("default_ttl")) + feed.default_ttl = extract(params["default_ttl"]); + if (params.has_key("url")) + feed.url = extract(params["url"]); + if (params.has_key("add_args")) + dict_to_add_torrent_params(dict(params["add_args"]), feed.add_args); + } + + feed_handle add_feed(session& s, dict params) + { + feed_settings feed; + dict_to_feed_settings(params, feed); + + return s.add_feed(feed); + } + + dict get_feed_status(feed_handle const& h) + { + feed_status s = h.get_feed_status(); + dict ret; + ret["url"] = s.url; + ret["title"] = s.title; + ret["description"] = s.description; + ret["last_update"] = s.last_update; + ret["next_update"] = s.next_update; + ret["updating"] = s.updating; + ret["error"] = s.error ? s.error.message() : ""; + ret["ttl"] = s.ttl; + + list items; + for (std::vector::iterator i = s.items.begin() + , end(s.items.end()); i != end; ++i) + { + dict item; + item["url"] = i->url; + item["uuid"] = i->uuid; + item["title"] = i->title; + item["description"] = i->description; + item["comment"] = i->comment; + item["category"] = i->category; + item["size"] = i->size; + item["handle"] = i->handle; + item["info_hash"] = i->info_hash.to_string(); + items.append(item); + } + ret["items"] = items; + return ret; + } + + void set_feed_settings(feed_handle& h, dict sett) + { + feed_settings feed; + dict_to_feed_settings(sett, feed); + h.set_settings(feed); + } + + dict get_feed_settings(feed_handle& h) + { + feed_settings s = h.settings(); + dict ret; + ret["url"] = s.url; + ret["auto_download"] = s.auto_download; + ret["default_ttl"] = s.default_ttl; + return ret; + } + void start_natpmp(session& s) { allow_threading_guard guard; @@ -399,6 +473,7 @@ void bind_session() ) #endif #endif + .def("add_feed", &add_feed) .def("remove_torrent", allow_threads(&session::remove_torrent), arg("option") = session::none ) .def("set_local_download_rate_limit", allow_threads(&session::set_local_download_rate_limit)) @@ -484,5 +559,13 @@ void bind_session() .value("save_as_map", session::save_as_map) ; + class_("feed_handle") + .def("update_feed", &feed_handle::update_feed) + .def("get_feed_status", &get_feed_status) + .def("set_settings", &set_feed_settings) + .def("settings", &get_feed_settings) + ; + + register_ptr_to_python >(); } diff --git a/docs/manual.html b/docs/manual.html index 8075fd1c8..d95eb457b 100644 --- a/docs/manual.html +++ b/docs/manual.html @@ -78,236 +78,248 @@
  • is_listening() listen_port() listen_on()
  • set_alert_mask()
  • pop_alert() wait_for_alert() set_alert_queue_size_limit()
    • -
  • add_extension()
    • -
  • set_settings() set_pe_settings()
    • -
  • set_proxy() proxy()
    • -
  • set_i2p_proxy() i2p_proxy()
    • -
  • start_dht() stop_dht() set_dht_settings() dht_state() is_dht_running()
    • -
  • add_dht_node() add_dht_router()
    • -
  • start_lsd() stop_lsd()
    • -
  • start_upnp() stop_upnp()
    • -
  • start_natpmp() stop_natpmp()
    • +
  • add_feed()
    • +
  • remove_feed()
    • +
  • get_feeds()
    • +
  • add_extension()
    • +
  • set_settings() set_pe_settings()
    • +
  • set_proxy() proxy()
    • +
  • set_i2p_proxy() i2p_proxy()
    • +
  • start_dht() stop_dht() set_dht_settings() dht_state() is_dht_running()
    • +
  • add_dht_node() add_dht_router()
    • +
  • start_lsd() stop_lsd()
    • +
  • start_upnp() stop_upnp()
    • +
  • start_natpmp() stop_natpmp()
    • -
  • entry
      -
    • integer() string() list() dict() type()
      • -
    • operator[]
      • -
    • find_key()
      • +
    • entry
      • -
    • torrent_info
        -
      • torrent_info()
        • -
      • add_tracker()
        • -
      • files() orig_files()
        • -
      • remap_files()
        • -
      • rename_file()
        • -
      • begin_files() end_files() rbegin_files() rend_files()
        • -
      • num_files() file_at()
        • -
      • map_block()
        • -
      • map_file()
        • -
      • add_url_seed() add_http_seed()
        • -
      • trackers()
        • -
      • total_size() piece_length() piece_size() num_pieces()
        • -
      • hash_for_piece() hash_for_piece_ptr() info_hash()
        • -
      • name() comment() creation_date() creator()
        • -
      • priv()
        • -
      • nodes()
        • -
      • add_node()
        • -
      • metadata() metadata_size()
        • +
      • torrent_info
        • -
      • torrent_handle
          -
        • set_piece_deadline()
          • -
        • piece_availability()
          • -
        • piece_priority() prioritize_pieces() piece_priorities()
          • -
        • file_priority() prioritize_files() file_priorities()
          • -
        • file_progress()
          • -
        • save_path()
          • -
        • move_storage()
          • -
        • rename_file()
          • -
        • get_storage_impl()
          • -
        • super_seeding()
          • -
        • add_piece()
          • -
        • read_piece()
          • -
        • force_reannounce() force_dht_announce()
          • -
        • scrape_tracker()
          • -
        • connect_peer()
          • -
        • name()
          • -
        • set_ratio()
          • -
        • set_upload_limit() set_download_limit() upload_limit() download_limit()
          • -
        • set_sequential_download()
          • -
        • get_peer_download_limit() get_peer_upload_limit() set_peer_upload_limit() set_peer_download_limit()
          • -
        • pause() resume()
          • -
        • flush_cache()
          • -
        • force_recheck()
          • -
        • clear_error()
          • -
        • set_upload_mode()
          • -
        • set_share_mode()
          • -
        • resolve_countries()
          • -
        • is_seed()
          • -
        • auto_managed()
          • -
        • set_metadata()
          • -
        • set_tracker_login()
          • -
        • trackers() replace_trackers() add_tracker()
          • -
        • add_url_seed() remove_url_seed() url_seeds()
          • -
        • add_http_seed() remove_http_seed() http_seeds()
          • -
        • queue_position() queue_position_up() queue_position_down() queue_position_top() queue_position_bottom()
          • -
        • set_priority()
          • -
        • use_interface()
          • -
        • info_hash()
          • -
        • set_max_uploads() max_uploads()
          • -
        • set_max_connections() max_connections()
          • -
        • save_resume_data()
          • -
        • need_save_resume_data()
          • -
        • status()
          • -
        • get_download_queue()
          • -
        • get_peer_info()
          • -
        • get_torrent_info()
          • -
        • is_valid()
          • +
        • torrent_handle
          • -
        • torrent_status
          • -
        • peer_info
          • -
        • session customization
            -
          • presets
            • -
          • session_settings
            • +
          • torrent_status
            • +
          • peer_info
            • +
          • feed_handle
            • -
          • pe_settings
            • -
          • proxy_settings
            • -
          • ip_filter
              -
            • ip_filter()
              • -
            • add_rule()
              • -
            • access()
              • -
            • export_filter()
              • +
            • feed_item
              • +
            • session customization
              • -
            • big_number
              • -
            • bitfield
              • -
            • hasher
              • -
            • fingerprint
              • -
            • UPnP and NAT-PMP
                -
              • add_mapping()
                • -
              • delete_mapping()
                • -
              • router_model()
                • +
              • pe_settings
                • +
              • proxy_settings
                • +
              • ip_filter
                • -
              • free functions
                  -
                • identify_client()
                  • -
                • client_fingerprint()
                  • -
                • lazy_bdecode()
                  • -
                • bdecode() bencode()
                  • -
                • add_magnet_uri()
                  • -
                • make_magnet_uri()
                  • +
                • big_number
                  • +
                • bitfield
                  • +
                • hasher
                  • +
                • fingerprint
                  • +
                • UPnP and NAT-PMP
                  • -
                • alerts
                    -
                  • read_piece_alert
                    • -
                  • external_ip_alert
                    • -
                  • listen_failed_alert
                    • -
                  • listen_succeeded_alert
                    • -
                  • portmap_error_alert
                    • -
                  • portmap_alert
                    • -
                  • portmap_log_alert
                    • -
                  • file_error_alert
                    • -
                  • file_renamed_alert
                    • -
                  • file_rename_failed_alert
                    • -
                  • tracker_announce_alert
                    • -
                  • tracker_error_alert
                    • -
                  • tracker_reply_alert
                    • -
                  • dht_reply_alert
                    • -
                  • tracker_warning_alert
                    • -
                  • scrape_reply_alert
                    • -
                  • scrape_failed_alert
                    • -
                  • url_seed_alert
                    • -
                  • hash_failed_alert
                    • -
                  • peer_alert
                    • -
                  • peer_connect_alert
                    • -
                  • peer_ban_alert
                    • -
                  • peer_snubbed_alert
                    • -
                  • peer_unsnubbed_alert
                    • -
                  • peer_error_alert
                    • -
                  • peer_connected_alert
                    • -
                  • peer_disconnected_alert
                    • -
                  • invalid_request_alert
                    • -
                  • request_dropped_alert
                    • -
                  • block_timeout_alert
                    • -
                  • block_finished_alert
                    • -
                  • lsd_peer_alert
                    • -
                  • file_completed_alert
                    • -
                  • block_downloading_alert
                    • -
                  • unwanted_block_alert
                    • -
                  • torrent_delete_failed_alert
                    • -
                  • torrent_deleted_alert
                    • -
                  • torrent_finished_alert
                    • -
                  • performance_alert
                    • -
                  • state_changed_alert
                    • -
                  • metadata_failed_alert
                    • -
                  • metadata_received_alert
                    • -
                  • fastresume_rejected_alert
                    • -
                  • peer_blocked_alert
                    • -
                  • storage_moved_alert
                    • -
                  • storage_moved_failed_alert
                    • -
                  • torrent_paused_alert
                    • -
                  • torrent_resumed_alert
                    • -
                  • save_resume_data_alert
                    • -
                  • save_resume_data_failed_alert
                    • -
                  • stats_alert
                    • -
                  • cache_flushed_alert
                    • -
                  • dht_announce_alert
                    • -
                  • dht_get_peers_alert
                    • -
                  • anonymous_mode_alert
                    • +
                  • free functions
                    • -
                  • alert dispatcher
                    • -
                  • exceptions
                      -
                    • libtorrent_exception
                      • +
                    • alerts
                      • -
                    • error_code
                        -
                      • translating error codes
                        • +
                      • alert dispatcher
                        • +
                      • exceptions
                        • -
                      • storage_interface
                          -
                        • initialize()
                          • -
                        • has_any_file()
                          • -
                        • readv() writev()
                          • -
                        • sparse_end()
                          • -
                        • move_storage()
                          • -
                        • verify_resume_data()
                          • -
                        • write_resume_data()
                          • -
                        • move_slot()
                          • -
                        • swap_slots()
                          • -
                        • swap_slots3()
                          • -
                        • rename_file()
                          • -
                        • release_files()
                          • -
                        • delete_files()
                          • -
                        • finalize_file()
                          • +
                        • error_code
                          • -
                        • magnet links
                          • -
                        • queuing
                            -
                          • downloading
                            • -
                          • seeding
                            • +
                          • storage_interface
                            • -
                          • fast resume
                              -
                            • file format
                              • +
                            • magnet links
                              • +
                            • queuing
                              • -
                            • threads
                              • -
                            • storage allocation
                                -
                              • sparse allocation
                                • -
                              • full allocation
                                • -
                              • compact allocation
                                • +
                              • fast resume
                                • -
                              • extensions
                                @@ -422,6 +434,7 @@ class session: public boost::noncopyable save_i2p_proxy = 0x010, save_encryption_settings = 0x020, save_as_map = 0x040, + save_feeds = 0x080, }; void load_state(lazy_entry const& e); @@ -456,6 +469,7 @@ class session: public boost::noncopyable std::vector<torrent_handle> get_torrents() const; void set_settings(session_settings const& settings); + session_settings settings() const; void set_pe_settings(pe_settings const& settings); void set_upload_rate_limit(int bytes_per_second); @@ -507,6 +521,10 @@ class session: public boost::noncopyable size_t set_alert_queue_size_limit( size_t queue_size_limit_); + feed_handle session::add_feed(feed_settings const& feed); + void session::remove_feed(feed_handle h); + void session::get_feeds(std::vector<feed_handle>& f) const; + void add_extension(boost::function< boost::shared_ptr<torrent_plugin>(torrent*)> ext); @@ -598,6 +616,7 @@ enum save_state_flags_t save_i2p_proxy = 0x010, save_encryption_settings = 0x020, save_as_map = 0x040, + save_feeds = 0x080 };
                                @@ -672,6 +691,7 @@ struct add_torrent_params std::vector<boost::uint8_t> const* file_priorities; bool share_mode; std::string trackerid; + std::string url; }; torrent_handle add_torrent(add_torrent_params const& params); @@ -683,9 +703,10 @@ torrent_handle add_torrent(add_torrent_params const& params object with all the parameters.

                                The overload that does not take an error_code throws an exception on error and is not available when building without exception support.

                                -

                                The only mandatory parameter is save_path which is the directory where you +

                                The only mandatory parameters are save_path which is the directory where you want the files to be saved. You also need to specify either the ti (the -torrent file) or info_hash (the info hash of the torrent). If you specify the +torrent file), the info_hash (the info hash of the torrent) or the url +(the URL to where to download the .torrent file from). If you specify the info-hash, the torrent file will be downloaded from peers, which requires them to support the metadata extension. For the metadata extension to work, libtorrent must be built with extensions enabled (TORRENT_DISABLE_EXTENSIONS must not be @@ -695,6 +716,10 @@ the torrent as long as it doesn't have metadata. See tracker_url can be 0, otherwise you might specify a tracker url that tracks this torrent.

                                +

                                If you specify a url, the torrent will be set in downloading_metadata state +until the .torrent file has been downloaded. If there's any error while downloading, +the torrent will be stopped and the torrent error state (torrent_status::error) +will indicate what went wrong. The url may also refer to a magnet link.

                                If the torrent you are trying to add already exists in the session (is either queued for checking, being checked or downloading) add_torrent() will throw libtorrent_exception which derives from std::exception unless duplicate_is_error @@ -871,6 +896,16 @@ struct dht_lookup int timeouts; int responses; int branch_factor; + int nodes_left; + int last_sent; + int first_timeout; +}; + +struct dht_routing_bucket +{ + int num_nodes; + int num_replacements; + int last_active; }; struct utp_status @@ -926,6 +961,7 @@ struct session_status int dht_torrents; size_type dht_global_nodes; std::vector<dht_lookup> active_requests; + std::vector<dht_routing_table> dht_routing_table; int dht_total_allocations; utp_status utp_stats; @@ -976,6 +1012,8 @@ becomes unresponsive.

                                dht_global_nodes is an estimation of the total number of nodes in the DHT network.

                                active_requests is a vector of the currently running DHT lookups.

                                +

                                dht_routing_table contains information about every bucket in the DHT routing +table.

                                dht_total_allocations is the number of nodes allocated dynamically for a particular DHT lookup. This represents roughly the amount of memory used by the DHT.

                                @@ -1155,6 +1193,71 @@ by calling pop_alert.

                                save_resume_data_alert and save_resume_data_failed_alert are always posted, regardelss of the alert mask.

                                +
                                +

                                add_feed()

                                +
                                +
                                +feed_handle session::add_feed(feed_settings const& feed);
+
                                +
                                +

                                This adds an RSS feed to the session. The feed will be refreshed +regularly and optionally add all torrents from the feed, as they +appear. The feed is defined by the feed_settings object:

                                +
                                +struct feed_settings
+{
+        feed_settings();
+
+std::string url;
+        bool auto_download;
+        int default_ttl;
+        add_torrent_params add_args;
+};
+
                                +

                                By default auto_download is true, which means all torrents in +the feed will be downloaded. Set this to false in order to manually +add torrents to the session. You may react to the rss_alert when +a feed has been updated to poll it for the new items in the feed +when adding torrents manually. When torrents are added automatically, +you have to call session::get_torrents() to get the handles to +the new torrents.

                                +

                                Before adding the feed, you must set the url field to the +feed's url. It may point to an RSS or an atom feed.

                                +

                                The default_ttl is the default interval for refreshing a feed. +This may be overridden by the feed itself (by specifying the <ttl> +tag) and defaults to 30 minutes. The field specifies the number of +minutes between refreshes.

                                +

                                If torrents are added automatically, you may want to set the +add_args to appropriate values for download directory etc. +This object is used as a template for adding torrents from feeds, +but some torrent specific fields will be overridden by the +individual torrent being added. For more information on the +add_torrent_params, see add_torrent().

                                +

                                The returned feed_handle is a handle which is used to interact +with the feed, things like forcing a refresh or querying for +information about the items in the feed. For more information, +see feed_handle.

                                +
                                +
                                +

                                remove_feed()

                                +
                                +
                                +void session::remove_feed(feed_handle h);
+
                                +
                                +

                                Removes a feed from being watched by the session. When this +call returns, the feed handle is invalid and won't refer +to any feed.

                                +
                                +
                                +

                                get_feeds()

                                +
                                +
                                +void session::get_feeds(std::vector<feed_handle>& f) const;
+
                                +
                                +

                                Returns a list of all RSS feeds that are being watched by the session.

                                +

                                add_extension()

                                @@ -1285,6 +1388,8 @@ struct dht_settings int max_peers_reply; int search_branching; int max_fail_count; + bool restrict_routing_ips; + bool restrict_search_ips; };

                                max_peers_reply is the maximum number of peers the node will send in @@ -1297,6 +1402,13 @@ before it is removed from the routing table. If there are known working nodes that are ready to replace a failing node, it will be replaced immediately, this limit is only used to clear out nodes that don't have any node that can replace them.

                                +

                                restrict_routing_ips determines if the routing table entries should restrict +entries to one per IP. This defaults to true, which helps mitigate some attacks +on the DHT. It prevents adding multiple nodes with IPs with a very close CIDR +distance.

                                +

                                restrict_search_ips determines if DHT searches should prevent adding nodes +with IPs with very close CIDR distance. This also defaults to true and helps +mitigate certain attacks on the DHT.

                                The dht_settings struct used to contain a service_port member to control which port the DHT would listen on and send messages from. This field is deprecated and ignored. libtorrent always tries to open the UDP socket on the same port @@ -3412,7 +3524,8 @@ struct peer_info optimistic_unchoke = 0x800, snubbed = 0x1000, upload_only = 0x2000, - holepunched = 0x4000, + endgame_mode = 0x4000, + holepunched = 0x8000, rc4_encrypted = 0x100000, plaintext_encrypted = 0x200000 }; @@ -3579,6 +3692,12 @@ or implicitly (by becoming a seed) told us that it will not downloading anything more, regardless of which pieces we have. +endgame_mode +This means the last time this peer picket a piece, +it could not pick as many as it wanted because there +were not enough free ones. i.e. all pieces this peer +has were already requested from other peers. + holepunched This flag is set if the peer was in holepunch mode when the connection succeeded. This typically only @@ -3758,6 +3877,117 @@ floating point operations are diabled, instead use address of the interface it's going out over. This may be useful for multi-homed clients with multiple interfaces to the internet.

                                +
                                +

                                feed_handle

                                +

                                The feed_handle refers to a specific RSS feed which is watched by the session. +The feed_item struct is defined in <libtorrent/rss.hpp>. It has the following +functions:

                                +
                                +struct feed_handle
+{
+        feed_handle();
+        void update_feed();
+        feed_status get_feed_status() const;
+        void set_settings(feed_settings const& s);
+        feed_settings settings() const;
+};
+
                                +
                                +

                                update_feed()

                                +
                                +
                                +void update_feed();
+
                                +
                                +

                                Forces an update/refresh of the feed. Regular updates of the feed is managed +by libtorrent, be careful to not call this too frequently since it may +overload the RSS server.

                                +
                                +
                                +

                                get_feed_status()

                                +
                                +
                                +feed_status get_feed_status() const;
+
                                +
                                +

                                Queries the RSS feed for information, including all the items in the feed. +The feed_status object has the following fields:

                                +
                                +struct feed_status
+{
+        std::string url;
+        std::string title;
+        std::string description;
+        time_t last_update;
+        int next_update;
+        bool updating;
+        std::vector<feed_item> items;
+        error_code error;
+        int ttl;
+};
+
                                +

                                url is the URL of the feed.

                                +

                                title is the name of the feed (as specified by the feed itself). This +may be empty if we have not recevied a response from the RSS server yet, +or if the feed does not specify a title.

                                +

                                description is the feed description (as specified by the feed itself). +This may be empty if we have not received a response from the RSS server +yet, or if the feed does not specify a description.

                                +

                                last_update is the posix time of the last successful response from the feed.

                                +

                                next_update is the number of seconds, from now, when the feed will be +updated again.

                                +

                                updating is true if the feed is currently being updated (i.e. waiting for +DNS resolution, connecting to the server or waiting for the response to the +HTTP request, or receiving the response).

                                +

                                items is a vector of all items that we have received from the feed. See +feed_item for more information.

                                +

                                error is set to the appropriate error code if the feed encountered an +error.

                                +

                                ttl is the current refresh time (in minutes). It's either the configured +default ttl, or the ttl specified by the feed.

                                +
                                +
                                +

                                set_settings() settings()

                                +
                                +
                                +void set_settings(feed_settings const& s);
+feed_settings settings() const;
+
                                +
                                +

                                Sets and gets settings for this feed. For more information on the +available settings, see add_feed().

                                +
                                +
                                +
                                +

                                feed_item

                                +
                                +
                                The feed_item struct is defined in <libtorrent/rss.hpp>.
                                +
                                +struct feed_item
+{
+        feed_item();
+        std::string url;
+        std::string uuid;
+        std::string title;
+        std::string description;
+        std::string comment;
+        std::string category;
+        size_type size;
+        torrent_handle handle;
+        sha1_hash info_hash;
+};
+
                                +
                                +
                                +

                                size is the total size of the content the torrent refers to, or -1 +if no size was specified by the feed.

                                +

                                handle is the handle to the torrent, if the session is already downloading +this torrent.

                                +

                                info_hash is the info-hash of the torrent, or cleared (i.e. all zeroes) if +the feed does not specify the info-hash.

                                +

                                All the strings are self explanatory and may be empty if the feed does not specify +those fields.

                                +

                                session customization

                                You have some control over session configuration through the session_settings object. You @@ -4007,6 +4237,9 @@ struct session_settings int listen_queue_size; bool announce_double_nat; + + int torrent_connect_boost; + bool seeding_outgoing_connections; };

                                version is automatically set to the libtorrent version you're using @@ -4674,6 +4907,17 @@ is called again (or for the first time).

                                if announce_double_nat is true, the &ip= argument in tracker requests (unless otherwise specified) will be set to the intermediate IP address, if the user is double NATed. If ther user is not double NATed, this option has no affect.

                                +

                                torrent_connect_boost is the number of peers to try to connect to immediately +when the first tracker response is received for a torrent. This is a boost to +given to new torrents to accelerate them starting up. The normal connect scheduler +is run once every second, this allows peers to be connected immediately instead +of waiting for the session tick to trigger connections.

                                +

                                seeding_outgoing_connections determines if seeding (and finished) torrents +should attempt to make outgoing connections or not. By default this is true. It +may be set to false in very specific applications where the cost of making +outgoing connections is high, and there are no or small benefits of doing so. +For instance, if no nodes are behind a firewall or a NAT, seeds don't need to +make outgoing connections.

                                @@ -5236,6 +5480,8 @@ failed. To control some initial settings of the torrent, sepcify those in the add_torrent_params, p. See add_torrent().

                                The overload that does not take an error_code throws an exception on error and is not available when building without exception support.

                                +

                                A simpler way to add a magnet link to a session is to pass in the +link through add_torrent_params::url argument to session::add_torrent().

                                For more information about magnet links, see magnet links.

                                @@ -5319,6 +5565,15 @@ approximately once every second, for every active torrent. These alerts contain all statistics counters for the interval since the lasts stats alert. +dht_notification +Alerts on events in the DHT node. For incoming searches or +bootstrapping being done etc. + +rss_notification +Alerts on RSS related events, like feeds being updated, feed error +conditions and successful RSS feed updates. Enabling this categoty +will make you receive rss_alert alerts. + all_categories The full bitmask, representing all available categories. @@ -5356,6 +5611,7 @@ public: ip_block_notification = implementation defined, performance_warning = implementation defined, dht_notification = implementation defined, + stats_notification = implementation defined, all_categories = implementation defined }; @@ -5638,20 +5894,6 @@ struct tracker_reply_alert: tracker_alert

                                The num_peers tells how many peers were returned from the tracker. This is not necessarily all new peers, some of them may already be connected.

                                -
                                -

                                dht_reply_alert

                                -

                                This alert is generated each time the DHT receives peers from a node. num_peers -is the number of peers we received in this packet. Typically these packets are -received from multiple DHT nodes, and so the alerts are typically generated -a few at a time.

                                -
                                -struct dht_reply_alert: tracker_alert
-{
-        // ...
-        int num_peers;
-};
-
                                -

                                tracker_warning_alert

                                This alert is triggered if the tracker reply contains a warning field. Usually this @@ -5812,7 +6054,7 @@ struct peer_disconnected_alert: peer_alert

                                invalid_request_alert

                                This is a debug alert that is generated by an incoming invalid piece request. -Ïp is the address of the peer and the request is the actual incoming +ìp is the address of the peer and the request is the actual incoming request from the peer.

                                 struct invalid_request_alert: peer_alert
@@ -6222,6 +6464,31 @@ struct dht_get_peers_alert: alert
 };
                                +
                                +

                                dht_reply_alert

                                +

                                This alert is generated each time the DHT receives peers from a node. num_peers +is the number of peers we received in this packet. Typically these packets are +received from multiple DHT nodes, and so the alerts are typically generated +a few at a time.

                                +
                                +struct dht_reply_alert: tracker_alert
+{
+        // ...
+        int num_peers;
+};
+
                                +
                                +
                                +

                                dht_bootstrap_alert

                                +

                                This alert is posted when the initial DHT bootstrap is done. There's no any other +relevant information associated with this alert.

                                +
                                +struct dht_bootstrap_alert: alert
+{
+        // ...
+};
+
                                +

                                anonymous_mode_alert

                                This alert is posted when a bittorrent feature is blocked because of the @@ -6245,6 +6512,47 @@ struct anonymous_mode_alert: tracker_alert communication and the tracker will not be contacted. The tracker which this failed for is specified in the str member.

                                +
                                +

                                rss_alert

                                +

                                This alert is posted on RSS feed events such as start of RSS feed updates, +successful completed updates and errors during updates.

                                +

                                This alert is only posted if the rss_notifications category is enabled +in the alert mask.

                                +
                                +struct rss_alert: alert
+{
+        // ..
+        virtual std::string message() const;
+
+        enum state_t
+        {
+                state_updating, state_updated, state_error
+        };
+
+        feed_handle handle;
+        std::string url;
+        int state;
+        error_code error;
+};
+
                                +

                                handle is the handle to the feed which generated this alert.

                                +

                                url is a short cut to access the url of the feed, without +having to call get_settings().

                                +

                                state is one of:

                                +
                                +
                                rss_alert::state_updating
                                +
                                An update of this feed was just initiated, it will either succeed +or fail soon.
                                +
                                rss_alert::state_updated
                                +
                                The feed just completed a successful update, there may be new items +in it. If you're adding torrents manually, you may want to request +the feed status of the feed and look through the items vector.
                                +
                                rss_akert::state_error
                                +
                                An error just occurred. See the error field for information on +what went wrong.
                                +
                                +

                                error is an error code used for when an error occurs on the feed.

                                +

                                alert dispatcher

                                @@ -7089,6 +7397,72 @@ specific port

                                The UPnP errors are declared in the libtorrent::upnp_errors namespace.

                                +

                                HTTP errors are reported in the libtorrent::http_category, with error code enums in +the libtorrent::errors namespace.

                                + ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                codesymbol
                                100cont
                                200ok
                                201created
                                202accepted
                                204no_content
                                300multiple_choices
                                301moved_permanently
                                302moved_temporarily
                                304not_modified
                                400bad_request
                                401unauthorized
                                403forbidden
                                404not_found
                                500internal_server_error
                                501not_implemented
                                502bad_gateway
                                503service_unavailable

                                translating error codes

                                The error_code::message() function will typically return a localized error string, @@ -7115,13 +7489,13 @@ std::string error_code_to_string(boost::system::error_code const& ec) static const char const* swedish[] = { "inget fel", - "en fil i torrenten kolliderar med en fil frÂn en annan torrent", + "en fil i torrenten kolliderar med en fil från en annan torrent", "hash check misslyckades", - "torrent filen ‰r inte en dictionary", - "'info'-nyckeln saknas eller ‰r korrupt i torrentfilen", - "'info'-f‰ltet ‰r inte en dictionary", - "'piece length' f‰ltet saknas eller ‰r korrupt i torrentfilen", - "torrentfilen saknar namnf‰ltet", + "torrent filen är inte en dictionary", + "'info'-nyckeln saknas eller är korrupt i torrentfilen", + "'info'-fältet är inte en dictionary", + "'piece length' fältet saknas eller är korrupt i torrentfilen", + "torrentfilen saknar namnfältet", "ogiltigt namn i torrentfilen (kan vara en attack)", // ... more strings here }; diff --git a/docs/manual.rst b/docs/manual.rst index b1aa4b5a7..c43125f0c 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -126,6 +126,7 @@ The ``session`` class has the following synopsis:: save_i2p_proxy = 0x010, save_encryption_settings = 0x020, save_as_map = 0x040, + save_feeds = 0x080, }; void load_state(lazy_entry const& e); @@ -212,6 +213,10 @@ The ``session`` class has the following synopsis:: size_t set_alert_queue_size_limit( size_t queue_size_limit_); + feed_handle session::add_feed(feed_settings const& feed); + void session::remove_feed(feed_handle h); + void session::get_feeds(std::vector& f) const; + void add_extension(boost::function< boost::shared_ptr(torrent*)> ext); @@ -309,6 +314,7 @@ torrents). These are the possible flags. A flag that's set, means those settings save_i2p_proxy = 0x010, save_encryption_settings = 0x020, save_as_map = 0x040, + save_feeds = 0x080 }; @@ -965,6 +971,78 @@ by calling ``pop_alert``. Default value is 1000. ``save_resume_data_alert`` and ``save_resume_data_failed_alert`` are always posted, regardelss of the alert mask. +add_feed() +---------- + + :: + + feed_handle session::add_feed(feed_settings const& feed); + +This adds an RSS feed to the session. The feed will be refreshed +regularly and optionally add all torrents from the feed, as they +appear. The feed is defined by the ``feed_settings`` object:: + + struct feed_settings + { + feed_settings(); + + std::string url; + bool auto_download; + int default_ttl; + add_torrent_params add_args; + }; + +By default ``auto_download`` is true, which means all torrents in +the feed will be downloaded. Set this to false in order to manually +add torrents to the session. You may react to the rss_alert_ when +a feed has been updated to poll it for the new items in the feed +when adding torrents manually. When torrents are added automatically, +you have to call ``session::get_torrents()`` to get the handles to +the new torrents. + +Before adding the feed, you must set the ``url`` field to the +feed's url. It may point to an RSS or an atom feed. + +The ``default_ttl`` is the default interval for refreshing a feed. +This may be overridden by the feed itself (by specifying the ```` +tag) and defaults to 30 minutes. The field specifies the number of +minutes between refreshes. + +If torrents are added automatically, you may want to set the +``add_args`` to appropriate values for download directory etc. +This object is used as a template for adding torrents from feeds, +but some torrent specific fields will be overridden by the +individual torrent being added. For more information on the +``add_torrent_params``, see `add_torrent()`_. + +The returned feed_handle_ is a handle which is used to interact +with the feed, things like forcing a refresh or querying for +information about the items in the feed. For more information, +see feed_handle_. + + +remove_feed() +------------- + + :: + + void session::remove_feed(feed_handle h); + +Removes a feed from being watched by the session. When this +call returns, the feed handle is invalid and won't refer +to any feed. + + +get_feeds() +----------- + + :: + + void session::get_feeds(std::vector& f) const; + +Returns a list of all RSS feeds that are being watched by the session. + + add_extension() --------------- @@ -3772,6 +3850,127 @@ floating point operations are diabled, instead use ``progress_ppm``. address of the interface it's going out over. This may be useful for multi-homed clients with multiple interfaces to the internet. +feed_handle +=========== + +The ``feed_handle`` refers to a specific RSS feed which is watched by the session. +The ``feed_item`` struct is defined in ````. It has the following +functions:: + + struct feed_handle + { + feed_handle(); + void update_feed(); + feed_status get_feed_status() const; + void set_settings(feed_settings const& s); + feed_settings settings() const; + }; + +update_feed() +------------- + + :: + + void update_feed(); + +Forces an update/refresh of the feed. Regular updates of the feed is managed +by libtorrent, be careful to not call this too frequently since it may +overload the RSS server. + +get_feed_status() +----------------- + + :: + + feed_status get_feed_status() const; + +Queries the RSS feed for information, including all the items in the feed. +The ``feed_status`` object has the following fields:: + + struct feed_status + { + std::string url; + std::string title; + std::string description; + time_t last_update; + int next_update; + bool updating; + std::vector items; + error_code error; + int ttl; + }; + +``url`` is the URL of the feed. + +``title`` is the name of the feed (as specified by the feed itself). This +may be empty if we have not recevied a response from the RSS server yet, +or if the feed does not specify a title. + +``description`` is the feed description (as specified by the feed itself). +This may be empty if we have not received a response from the RSS server +yet, or if the feed does not specify a description. + +``last_update`` is the posix time of the last successful response from the feed. + +``next_update`` is the number of seconds, from now, when the feed will be +updated again. + +``updating`` is true if the feed is currently being updated (i.e. waiting for +DNS resolution, connecting to the server or waiting for the response to the +HTTP request, or receiving the response). + +``items`` is a vector of all items that we have received from the feed. See +feed_item_ for more information. + +``error`` is set to the appropriate error code if the feed encountered an +error. + +``ttl`` is the current refresh time (in minutes). It's either the configured +default ttl, or the ttl specified by the feed. + + +set_settings() settings() +------------------------- + + :: + + void set_settings(feed_settings const& s); + feed_settings settings() const; + +Sets and gets settings for this feed. For more information on the +available settings, see `add_feed()`_. + +feed_item +========= + +The ``feed_item`` struct is defined in ````. + :: + + struct feed_item + { + feed_item(); + std::string url; + std::string uuid; + std::string title; + std::string description; + std::string comment; + std::string category; + size_type size; + torrent_handle handle; + sha1_hash info_hash; + }; + +``size`` is the total size of the content the torrent refers to, or -1 +if no size was specified by the feed. + +``handle`` is the handle to the torrent, if the session is already downloading +this torrent. + +``info_hash`` is the info-hash of the torrent, or cleared (i.e. all zeroes) if +the feed does not specify the info-hash. + +All the strings are self explanatory and may be empty if the feed does not specify +those fields. session customization ===================== @@ -5541,6 +5740,10 @@ is a bitmask with the following bits: | ``dht_notification`` | Alerts on events in the DHT node. For incoming searches or | | | bootstrapping being done etc. | +--------------------------------+---------------------------------------------------------------------+ +| ``rss_notification`` | Alerts on RSS related events, like feeds being updated, feed error | +| | conditions and successful RSS feed updates. Enabling this categoty | +| | will make you receive ``rss_alert`` alerts. | ++--------------------------------+---------------------------------------------------------------------+ | ``all_categories`` | The full bitmask, representing all available categories. | +--------------------------------+---------------------------------------------------------------------+ @@ -6660,6 +6863,56 @@ when in anonymous mode. communication and the tracker will not be contacted. The tracker which this failed for is specified in the ``str`` member. +rss_alert +--------- + +This alert is posted on RSS feed events such as start of RSS feed updates, +successful completed updates and errors during updates. + +This alert is only posted if the ``rss_notifications`` category is enabled +in the alert mask. + +:: + + struct rss_alert: alert + { + // .. + virtual std::string message() const; + + enum state_t + { + state_updating, state_updated, state_error + }; + + feed_handle handle; + std::string url; + int state; + error_code error; + }; + +``handle`` is the handle to the feed which generated this alert. + +``url`` is a short cut to access the url of the feed, without +having to call ``get_settings()``. + +``state`` is one of: + +``rss_alert::state_updating`` + An update of this feed was just initiated, it will either succeed + or fail soon. + +``rss_alert::state_updated`` + The feed just completed a successful update, there may be new items + in it. If you're adding torrents manually, you may want to request + the feed status of the feed and look through the ``items`` vector. + +``rss_akert::state_error`` + An error just occurred. See the ``error`` field for information on + what went wrong. + +``error`` is an error code used for when an error occurs on the feed. + + alert dispatcher ================ diff --git a/docs/python_binding.html b/docs/python_binding.html index bb6e0aa3c..a02c56faa 100644 --- a/docs/python_binding.html +++ b/docs/python_binding.html @@ -141,6 +141,12 @@ a list of entries.

                              create_torrent::add_node() takes two arguments, one string and one integer, instead of a pair. The string is the address and the integer is the port.

                              +

                              session::set_settings() not only accepts a session_settings object, but also +a dictionary with keys matching the names of the members of the session_settings struct. +When calling set_settings, the dictionary does not need to have every settings set, +keys that are not present, are set to their default value.

                              +

                              For backwards compatibility, session::settings() still returns a session_settings +struct. To get a python dictionary of the settings, call session::get_settings.

                              For an example python program, see client.py in the bindings/python directory.

                              A very simple example usage of the module would be something like this:

                              diff --git a/examples/Jamfile b/examples/Jamfile index 2a0ea3b4f..9c5fe6284 100644 --- a/examples/Jamfile +++ b/examples/Jamfile @@ -24,5 +24,6 @@ exe make_torrent : make_torrent.cpp ; exe enum_if : enum_if.cpp ; exe connection_tester : connection_tester.cpp ; exe fragmentation_test : fragmentation_test.cpp ; +exe rss_reader : rss_reader.cpp ; exe upnp_test : upnp_test.cpp ; diff --git a/examples/Makefile.am b/examples/Makefile.am index 0c422e523..0bf59b5af 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -4,6 +4,7 @@ example_programs = \ enum_if \ make_torrent \ simple_client \ + rss_reader \ utp_test if ENABLE_EXAMPLES @@ -28,6 +29,12 @@ simple_client_SOURCES = simple_client.cpp enum_if_SOURCES = enum_if.cpp #enum_if_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la +rss_reader_SOURCES = rss_reader.cpp +#rss_reader_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la + +utp_test_SOURCES = rss_reader.cpp +#utp_test_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la + LDADD = $(top_builddir)/src/libtorrent-rasterbar.la AM_CPPFLAGS = -ftemplate-depth-50 -I$(top_srcdir)/include @DEBUGFLAGS@ diff --git a/examples/rss_reader.cpp b/examples/rss_reader.cpp new file mode 100644 index 000000000..33b36ac96 --- /dev/null +++ b/examples/rss_reader.cpp @@ -0,0 +1,173 @@ + +#include "libtorrent/rss.hpp" +#include "libtorrent/session.hpp" +#include "libtorrent/bencode.hpp" + +using namespace libtorrent; + +void print_feed(feed_status const& f) +{ + printf("FEED: %s\n",f.url.c_str()); + if (f.error) + printf("ERROR: %s\n", f.error.message().c_str()); + + printf(" %s\n %s\n", f.title.c_str(), f.description.c_str()); + printf(" ttl: %d minutes\n", f.ttl); + + for (std::vector::const_iterator i = f.items.begin() + , end(f.items.end()); i != end; ++i) + { + printf("\033[32m%s\033[0m\n------------------------------------------------------\n" + " url: %s\n size: %"PRId64"\n info-hash: %s\n uuid: %s\n description: %s\n" + " comment: %s\n category: %s\n" + , i->title.c_str(), i->url.c_str(), i->size + , i->info_hash.is_all_zeros() ? "" : to_hex(i->info_hash.to_string()).c_str() + , i->uuid.c_str(), i->description.c_str(), i->comment.c_str(), i->category.c_str()); + } +} + +std::string const& progress_bar(int progress, int width) +{ + static std::string bar; + bar.clear(); + bar.reserve(width + 10); + + int progress_chars = (progress * width + 500) / 1000; + std::fill_n(std::back_inserter(bar), progress_chars, '#'); + std::fill_n(std::back_inserter(bar), width - progress_chars, '-'); + return bar; +} + +int save_file(std::string const& filename, std::vector& v) +{ + using namespace libtorrent; + + file f; + error_code ec; + if (!f.open(filename, file::write_only, ec)) return -1; + if (ec) return -1; + file::iovec_t b = {&v[0], v.size()}; + size_type written = f.writev(0, &b, 1, ec); + if (written != v.size()) return -3; + if (ec) return -3; + return 0; +} + +volatile bool quit = false; + +void sig(int num) +{ + quit = true; +} + +int main(int argc, char* argv[]) +{ + if ((argc == 2 && strcmp(argv[1], "--help") == 0) || argc > 2) + { + std::cerr << "usage: rss_reader [rss-url]\n"; + return 0; + } + + session ses; + + session_settings sett; + sett.active_downloads = 2; + sett.active_seeds = 1; + sett.active_limit = 3; + ses.set_settings(sett); + + std::vector in; + if (load_file(".ses_state", in) == 0) + { + lazy_entry e; + error_code ec; + if (lazy_bdecode(&in[0], &in[0] + in.size(), e, ec) == 0) + ses.load_state(e); + } + + feed_handle fh; + if (argc == 2) + { + feed_settings feed; + feed.url = argv[1]; + feed.add_args.save_path = "."; + fh = ses.add_feed(feed); + fh.update_feed(); + } + else + { + std::vector handles; + ses.get_feeds(handles); + if (handles.empty()) + { + printf("usage: rss_reader rss-url\n"); + return 1; + } + fh = handles[0]; + } + feed_status fs = fh.get_feed_status(); + int i = 0; + char spinner[] = {'|', '/', '-', '\\'}; + fprintf(stderr, "fetching feed ... %c", spinner[i]); + while (fs.updating) + { + sleep(100); + i = (i + 1) % 4; + fprintf(stderr, "\b%c", spinner[i]); + fs = fh.get_feed_status(); + } + fprintf(stderr, "\bDONE\n"); + + print_feed(fs); + + signal(SIGTERM, &sig); + signal(SIGINT, &sig); + + while (!quit) + { + std::vector t = ses.get_torrents(); + for (std::vector::iterator i = t.begin() + , end(t.end()); i != end; ++i) + { + torrent_status st = i->status(); + std::string const& progress = progress_bar(st.progress_ppm / 1000, 40); + std::string name = i->name(); + if (name.size() > 70) name.resize(70); + std::string error = st.error; + if (error.size() > 40) error.resize(40); + + static char const* state_str[] = + {"checking (q)", "checking", "dl metadata" + , "downloading", "finished", "seeding", "allocating", "checking (r)"}; + std::string status = st.paused ? "queued" : state_str[st.state]; + + int attribute = 0; + if (st.paused) attribute = 33; + else if (st.state == torrent_status::downloading) attribute = 1; + + printf("\033[%dm%2d %-70s d:%-4d u:%-4d %-40s %4d(%4d) %-12s\033[0m\n" + , attribute, st.queue_position + , name.c_str(), st.download_rate / 1000 + , st.upload_rate / 1000, !error.empty() ? error.c_str() : progress.c_str() + , st.num_peers, st.num_seeds, status.c_str()); + } + + sleep(500); + if (quit) break; + printf("\033[%dA", int(t.size())); + } + + printf("saving session state\n"); + { + entry session_state; + ses.save_state(session_state); + + std::vector out; + bencode(std::back_inserter(out), session_state); + save_file(".ses_state", out); + } + + printf("closing session"); + return 0; +} + diff --git a/include/libtorrent/Makefile.am b/include/libtorrent/Makefile.am index e518e1063..671a296e6 100644 --- a/include/libtorrent/Makefile.am +++ b/include/libtorrent/Makefile.am @@ -78,6 +78,7 @@ nobase_include_HEADERS = \ proxy_base.hpp \ ptime.hpp \ puff.hpp \ + rss.hpp \ session.hpp \ session_settings.hpp \ session_status.hpp \ diff --git a/include/libtorrent/add_torrent_params.hpp b/include/libtorrent/add_torrent_params.hpp index 73f90475c..cc63722c0 100644 --- a/include/libtorrent/add_torrent_params.hpp +++ b/include/libtorrent/add_torrent_params.hpp @@ -86,6 +86,8 @@ namespace libtorrent bool share_mode; std::string trackerid; std::string url; + std::string uuid; + std::string source_feed_url; }; } diff --git a/include/libtorrent/alert.hpp b/include/libtorrent/alert.hpp index 32d9b6a4d..2acfb5295 100644 --- a/include/libtorrent/alert.hpp +++ b/include/libtorrent/alert.hpp @@ -88,6 +88,7 @@ namespace libtorrent { performance_warning = 0x200, dht_notification = 0x400, stats_notification = 0x800, + rss_notification = 0x1000, all_categories = 0xffffffff }; diff --git a/include/libtorrent/alert_types.hpp b/include/libtorrent/alert_types.hpp index 1902fcc72..bca03bab2 100644 --- a/include/libtorrent/alert_types.hpp +++ b/include/libtorrent/alert_types.hpp @@ -41,6 +41,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/identify_client.hpp" #include "libtorrent/address.hpp" #include "libtorrent/stat.hpp" +#include "libtorrent/rss.hpp" // for feed_handle // lines reserved for future includes // the type-ids of the alert types @@ -50,7 +51,6 @@ POSSIBILITY OF SUCH DAMAGE. - namespace libtorrent { struct TORRENT_EXPORT torrent_alert: alert @@ -1194,6 +1194,29 @@ namespace libtorrent virtual std::string message() const; }; + struct TORRENT_EXPORT rss_alert: alert + { + rss_alert(feed_handle h, std::string const& url_, int state_, error_code const& ec) + : handle(h), url(url_), state(state_), error(ec) + {} + + TORRENT_DEFINE_ALERT(rss_alert); + + const static int static_category = alert::rss_notification; + virtual std::string message() const; + + enum state_t + { + state_updating, state_updated, state_error + }; + + feed_handle handle; + std::string url; + int state; + error_code error; + }; + + } diff --git a/include/libtorrent/aux_/session_impl.hpp b/include/libtorrent/aux_/session_impl.hpp index 077fb103e..ec958bfe6 100644 --- a/include/libtorrent/aux_/session_impl.hpp +++ b/include/libtorrent/aux_/session_impl.hpp @@ -84,6 +84,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/address.hpp" #include "libtorrent/utp_socket_manager.hpp" #include "libtorrent/bloom_filter.hpp" +#include "libtorrent/rss.hpp" #ifdef TORRENT_STATS #include @@ -201,7 +202,13 @@ namespace libtorrent } #endif - boost::weak_ptr find_torrent(const sha1_hash& info_hash); + feed_handle add_feed(feed_settings const& feed); + void remove_feed(feed_handle h); + void get_feeds(std::vector* f) const; + + boost::weak_ptr find_torrent(sha1_hash const& info_hash); + boost::weak_ptr find_torrent(std::string const& uuid); + peer_id const& get_peer_id() const { return m_peer_id; } void close_connection(peer_connection const* p, error_code const& ec); @@ -551,6 +558,8 @@ namespace libtorrent tracker_manager m_tracker_manager; torrent_map m_torrents; + std::map > m_uuids; + typedef std::list > check_queue_t; // this has all torrents that wants to be checked in it @@ -726,6 +735,13 @@ namespace libtorrent // to decide which ones to choke/unchoke ptime m_last_choke; + // the time when the next rss feed needs updating + ptime m_next_rss_update; + + // update any rss feeds that need updating and + // recalculate m_next_rss_update + void update_rss_feeds(); + // when outgoing_ports is configured, this is the // port we'll bind the next outgoing socket to int m_next_port; @@ -900,6 +916,8 @@ namespace libtorrent size_type m_total_failed_bytes; size_type m_total_redundant_bytes; + std::vector > m_feeds; + // the main working thread boost::scoped_ptr m_thread; diff --git a/include/libtorrent/rss.hpp b/include/libtorrent/rss.hpp new file mode 100644 index 000000000..5f317c038 --- /dev/null +++ b/include/libtorrent/rss.hpp @@ -0,0 +1,174 @@ +/* + +Copyright (c) 2010, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef TORRENT_RSS_HPP_INCLUDED +#define TORRENT_RSS_HPP_INCLUDED + +#include "libtorrent/torrent_handle.hpp" +#include "libtorrent/add_torrent_params.hpp" +#include "libtorrent/size_type.hpp" + +#include +#include + +namespace libtorrent +{ + namespace aux + { struct session_impl; } + + struct feed_item + { + feed_item(): size(-1) {} + std::string url; + std::string uuid; + std::string title; + std::string description; + std::string comment; + std::string category; + size_type size; + torrent_handle handle; + sha1_hash info_hash; + }; + +#ifndef BOOST_NO_EXCEPTIONS + torrent_handle add_feed_item(session& s, feed_item const& fi + , add_torrent_params const& p); +#endif + + torrent_handle add_feed_item(session& s, feed_item const& fi + , add_torrent_params const& p, error_code& ec); + + // the feed_settings object is all the information + // and configuration for a specific feed. All of + // these settings can be changed by the user + // after adding the feed + struct feed_settings + { + feed_settings() + : auto_download(true) + , default_ttl(30) + {} + + std::string url; + + bool auto_download; + + // in minutes + int default_ttl; + + // used when adding torrents + add_torrent_params add_args; + }; + + struct feed_status + { + std::string url; + std::string title; + std::string description; + time_t last_update; + int next_update; + bool updating; + std::vector items; + error_code error; + int ttl; + }; + + struct feed; + + struct feed_handle + { + feed_handle() {} + void update_feed(); + feed_status get_feed_status() const; + void set_settings(feed_settings const& s); + feed_settings settings() const; + private: + friend struct aux::session_impl; + friend struct feed; + feed_handle(boost::weak_ptr const& p); + boost::weak_ptr m_feed_ptr; + }; + + struct feed_state; + struct http_parser; + + boost::shared_ptr new_feed(aux::session_impl& ses, feed_settings const& sett); + + // this is the internal object holding all state about an + // RSS feed. All user interaction with this object + // goes through the feed_handle, which makes sure all calls + // are posted to the network thread + struct feed : boost::enable_shared_from_this + { + friend void parse_feed(feed_state& f, int token, char const* name, char const* val); + + feed(aux::session_impl& ses, feed_settings const& feed); + + void on_feed(error_code const& ec, http_parser const& parser + , char const* data, int size); + + void update_feed(); + + aux::session_impl& session() const { return m_ses; } + + void set_settings(feed_settings const& s); + void get_settings(feed_settings* s) const; + void get_feed_status(feed_status* ret) const; + + int next_update(time_t now) const; + + void load_state(lazy_entry const& rd); + void save_state(entry& rd) const; + +// private: + + feed_handle my_handle(); + + error_code m_error; + std::vector m_items; + std::string m_title; + std::string m_description; + time_t m_last_attempt; + time_t m_last_update; + // refresh rate of this feed in minutes + int m_ttl; + // true while waiting for the server to respond + bool m_updating; + feed_settings m_settings; + + aux::session_impl& m_ses; + }; + +}; + +#endif + diff --git a/include/libtorrent/session.hpp b/include/libtorrent/session.hpp index 9dbee25e6..799f4f1ec 100644 --- a/include/libtorrent/session.hpp +++ b/include/libtorrent/session.hpp @@ -56,6 +56,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/peer_id.hpp" #include "libtorrent/alert.hpp" // alert::error_notification #include "libtorrent/add_torrent_params.hpp" +#include "libtorrent/rss.hpp" #include "libtorrent/storage.hpp" @@ -160,7 +161,8 @@ namespace libtorrent save_proxy = 0x008, save_i2p_proxy = 0x010, save_encryption_settings = 0x020, - save_as_map = 0x040 + save_as_map = 0x040, + save_feeds = 0x080 #ifndef TORRENT_NO_DEPRECATE , @@ -237,6 +239,10 @@ namespace libtorrent void get_cache_info(sha1_hash const& ih , std::vector& ret) const; + feed_handle add_feed(feed_settings const& feed); + void remove_feed(feed_handle h); + void get_feeds(std::vector& f) const; + #ifndef TORRENT_DISABLE_DHT void start_dht(); void stop_dht(); diff --git a/include/libtorrent/settings.hpp b/include/libtorrent/settings.hpp index 872128650..7383948fd 100644 --- a/include/libtorrent/settings.hpp +++ b/include/libtorrent/settings.hpp @@ -41,8 +41,9 @@ namespace libtorrent struct lazy_entry; struct entry; - enum { std_string = 0, character = 1, short_integer = 2 - , integer = 3, floating_point = 4, boolean = 5}; + enum { std_string, character, integer + , floating_point, boolean, size_integer + , time_integer }; // this is used to map struct entries // to names in a bencoded dictionary to diff --git a/include/libtorrent/torrent.hpp b/include/libtorrent/torrent.hpp index 23dca903a..be0ceb2ea 100644 --- a/include/libtorrent/torrent.hpp +++ b/include/libtorrent/torrent.hpp @@ -699,6 +699,7 @@ namespace libtorrent torrent_info const& torrent_file() const { return *m_torrent_file; } + std::string const& uuid() const { return m_uuid; } std::string const& url() const { return m_url; } std::vector const& trackers() const @@ -979,6 +980,14 @@ namespace libtorrent // the torrent file std::string m_url; + // if this was added from an RSS feed, this is the unique + // identifier in the feed. + std::string m_uuid; + + // if this torrent was added by an RSS feed, this is the + // URL to that feed + std::string m_source_feed_url; + // this is used as temporary storage while downloading // the .torrent file from m_url std::vector m_torrent_file_buf; diff --git a/include/libtorrent/torrent_handle.hpp b/include/libtorrent/torrent_handle.hpp index 50a2f3da4..cd3325228 100644 --- a/include/libtorrent/torrent_handle.hpp +++ b/include/libtorrent/torrent_handle.hpp @@ -420,6 +420,7 @@ namespace libtorrent { friend class invariant_access; friend struct aux::session_impl; + friend struct feed; friend class torrent; torrent_handle() {} diff --git a/src/Makefile.am b/src/Makefile.am index 7616e743f..568217778 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -63,6 +63,7 @@ libtorrent_rasterbar_la_SOURCES = \ packet_buffer.cpp \ policy.cpp \ puff.cpp \ + rss.cpp \ session.cpp \ session_impl.cpp \ settings.cpp \ diff --git a/src/alert.cpp b/src/alert.cpp index c3a4bb193..98838bee2 100644 --- a/src/alert.cpp +++ b/src/alert.cpp @@ -496,5 +496,14 @@ namespace libtorrent { return "DHT bootstrap complete"; } + std::string rss_alert::message() const + { + char msg[600]; + char const* state_msg[] = {"updating", "updated", "error"}; + snprintf(msg, sizeof(msg), "RSS feed %s: %s (%s)" + , url.c_str(), state_msg[state], error.message().c_str()); + return msg; + } + } // namespace libtorrent diff --git a/src/http_connection.cpp b/src/http_connection.cpp index 04b48d78c..d467a049c 100644 --- a/src/http_connection.cpp +++ b/src/http_connection.cpp @@ -116,7 +116,7 @@ void http_connection::get(std::string const& url, time_duration timeout, int pri { // if we're using an http proxy and not an ssl // connection, just do a regular http proxy request - APPEND_FMT1("GET %s HTTP/1.0\r\n", url.c_str()); + APPEND_FMT1("GET %s HTTP/1.1\r\n", url.c_str()); if (ps->type == proxy_settings::http_pw) APPEND_FMT1("Proxy-Authorization: Basic %s\r\n", base64encode( ps->username + ":" + ps->password).c_str()); @@ -125,7 +125,7 @@ void http_connection::get(std::string const& url, time_duration timeout, int pri } else { - APPEND_FMT2("GET %s HTTP/1.0\r\n" + APPEND_FMT2("GET %s HTTP/1.1\r\n" "Host: %s", path.c_str(), hostname.c_str()); if (port != default_port) APPEND_FMT1(":%d\r\n", port); else APPEND_FMT("\r\n"); diff --git a/src/rss.cpp b/src/rss.cpp new file mode 100644 index 000000000..81f868f42 --- /dev/null +++ b/src/rss.cpp @@ -0,0 +1,580 @@ +/* + +Copyright (c) 2010, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "libtorrent/rss.hpp" +#include "libtorrent/xml_parse.hpp" +#include "libtorrent/http_parser.hpp" +#include "libtorrent/http_connection.hpp" +#include "libtorrent/aux_/session_impl.hpp" +#include "libtorrent/session.hpp" +#include "libtorrent/settings.hpp" +#include "libtorrent/alert_types.hpp" // for rss_alert + +#include + +namespace libtorrent { + +struct feed_state +{ + feed_state(feed& r) + : in_item(false) + , type(none) + , ret(r) + {} + + bool in_item; + std::string current_tag; + enum feed_type + { + none, atom, rss2 + } type; + feed_item current_item; + feed& ret; + + bool is_item(char const* tag) const + { + switch (type) + { + case atom: return string_equal_no_case(tag, "entry"); + case rss2: return string_equal_no_case(tag, "item"); + default: return false; + } + } + + bool is_title(char const* tag) const + { + switch (type) + { + case atom: + case rss2: return string_equal_no_case(tag, "title"); + default: return false; + } + } + + bool is_url(char const* tag) const + { + switch (type) + { + case atom: + case rss2: return string_equal_no_case(tag, "link"); + default: return false; + } + } + + bool is_desc(char const* tag) const + { + switch (type) + { + case atom: return string_equal_no_case(tag, "summary"); + case rss2: return string_equal_no_case(tag, "description") + || string_equal_no_case(tag, "media:text"); + default: return false; + } + } + + bool is_uuid(char const* tag) const + { + switch (type) + { + case atom: return string_equal_no_case(tag, "id"); + case rss2: return string_equal_no_case(tag, "guid"); + default: return false; + } + } + + bool is_comment(char const* tag) const + { + switch (type) + { + case atom: return false; + case rss2: return string_equal_no_case(tag, "comments"); + default: return false; + } + } + + bool is_category(char const* tag) const + { + switch (type) + { + case atom: return false; + case rss2: return string_equal_no_case(tag, "category"); + default: return false; + } + } + + bool is_size(char const* tag) const + { + return string_equal_no_case(tag, "size"); + } + + bool is_hash(char const* tag) const + { + return string_equal_no_case(tag, "hash") + || string_equal_no_case(tag, "media:hash"); + } + + bool is_ttl(char const* tag) const + { + return string_equal_no_case(tag, "ttl"); + } +}; + +void parse_feed(feed_state& f, int token, char const* name, char const* val) +{ + switch (token) + { + case xml_parse_error: + f.ret.m_error = errors::parse_failed; + return; + case xml_start_tag: + case xml_empty_tag: + { + f.current_tag = name; + if (f.type == feed_state::none) + { + if (string_equal_no_case(f.current_tag.c_str(), "feed")) + f.type = feed_state::atom; + else if (string_equal_no_case(f.current_tag.c_str(), "rss")) + f.type = feed_state::rss2; + } + if (f.is_item(name)) f.in_item = true; + return; + } + case xml_attribute: + { + if (!f.in_item) return; + if (f.is_url(f.current_tag.c_str()) + && f.type == feed_state::atom) + { + // atom feeds have items like this: + // + if (string_equal_no_case(name, "href")) + f.current_item.url = val; + else if (string_equal_no_case(name, "length")) + f.current_item.size = strtoll(val, 0, 10); + } + else if (f.type == feed_state::rss2 + && string_equal_no_case(f.current_tag.c_str(), "enclosure")) + { + // rss feeds have items like this: + // + if (string_equal_no_case(name, "url")) + f.current_item.url = val; + else if (string_equal_no_case(name, "length")) + f.current_item.size = strtoll(val, 0, 10); + } + else if (f.type == feed_state::rss2 + && string_equal_no_case(f.current_tag.c_str(), "media:content")) + { + // rss feeds sometimes have items like this: + // + if (string_equal_no_case(name, "url")) + f.current_item.url = val; + else if (string_equal_no_case(name, "filesize")) + f.current_item.size = strtoll(val, 0, 10); + } + return; + } + case xml_end_tag: + { + if (f.in_item && f.is_item(name)) + { + f.in_item = false; + if (!f.current_item.title.empty() + && !f.current_item.url.empty()) + f.ret.m_items.push_back(f.current_item); + f.current_item = feed_item(); + } + f.current_tag = ""; + return; + } + case xml_string: + { + if (!f.in_item) + { + if (f.is_title(f.current_tag.c_str())) + f.ret.m_title = name; + else if (f.is_desc(f.current_tag.c_str())) + f.ret.m_description = name; + else if (f.is_ttl(f.current_tag.c_str())) + { + int tmp = atoi(name); + if (tmp > 0) f.ret.m_ttl = tmp; + } + return; + } + if (f.is_title(f.current_tag.c_str())) + f.current_item.title = name; + else if (f.is_desc(f.current_tag.c_str())) + f.current_item.description = name; + else if (f.is_uuid(f.current_tag.c_str())) + f.current_item.uuid = name; + else if (f.is_url(f.current_tag.c_str()) && f.type != feed_state::atom) + f.current_item.url = name; + else if (f.is_comment(f.current_tag.c_str())) + f.current_item.comment = name; + else if (f.is_category(f.current_tag.c_str())) + f.current_item.category = name; + else if (f.is_size(f.current_tag.c_str())) + f.current_item.size = strtoll(name, 0, 10); + else if (f.is_hash(f.current_tag.c_str()) && strlen(name) == 40) + { + if (!from_hex(name, 40, (char*)&f.current_item.info_hash[0])) + { + // hex parsing failed + f.current_item.info_hash.clear(); + } + } + return; + } + case xml_declaration_tag: return; + case xml_comment: return; + } + +} + +torrent_handle add_feed_item(session& s, feed_item const& fi + , add_torrent_params const& tp, error_code& ec) +{ + add_torrent_params p = tp; + p.url = fi.url; + p.uuid = fi.uuid; + // #error figure out how to get the feed url in here +// p.source_feed_url = ???; + p.ti.reset(); + p.info_hash.clear(); + p.name = fi.title.c_str(); + return s.add_torrent(p, ec); +} + +#ifndef BOOST_NO_EXCEPTIONS +torrent_handle add_feed_item(session& s, feed_item const& fi + , add_torrent_params const& tp) +{ + error_code ec; + torrent_handle ret = add_feed_item(s, fi, tp, ec); + if (ec) throw libtorrent_exception(ec); + return ret; +} +#endif + +boost::shared_ptr new_feed(aux::session_impl& ses, feed_settings const& sett) +{ + return boost::shared_ptr(new feed(ses, sett)); +} + +feed::feed(aux::session_impl& ses, feed_settings const& sett) + : m_last_attempt(0) + , m_last_update(0) + , m_ttl(-1) + , m_updating(false) + , m_settings(sett) + , m_ses(ses) +{ +} + +void feed::set_settings(feed_settings const& s) +{ + m_settings = s; +} + +void feed::get_settings(feed_settings* s) const +{ + *s = m_settings; +} + +feed_handle feed::my_handle() +{ + return feed_handle(boost::weak_ptr(shared_from_this())); +} + +void feed::on_feed(error_code const& ec + , http_parser const& parser, char const* data, int size) +{ + TORRENT_ASSERT(m_updating); + m_updating = false; + + if (ec && ec != asio::error::eof) + { + m_error = ec; + if (m_ses.m_alerts.should_post()) + { + m_ses.m_alerts.post_alert(rss_alert(my_handle(), m_settings.url + , rss_alert::state_error, m_error)); + } + return; + } + + if (parser.status_code() != 200) + { + m_error = error_code(parser.status_code(), get_http_category()); + if (m_ses.m_alerts.should_post()) + { + m_ses.m_alerts.post_alert(rss_alert(my_handle(), m_settings.url + , rss_alert::state_error, m_error)); + } + return; + } + + char* buf = const_cast(data); + + feed_state s(*this); + xml_parse(buf, buf + size, boost::bind(&parse_feed, boost::ref(s), _1, _2, _3)); + + for (std::vector::iterator i = m_items.begin() + , end(m_items.end()); i != end; ++i) + { + i->handle = torrent_handle(m_ses.find_torrent(i->uuid.empty() ? i->url : i->uuid)); + + // if we're already downloading this torrent, or if we + // don't have auto-download enabled, just move along to + // the next one + if (i->handle.is_valid() || !m_settings.auto_download) continue; + + // this means we should add this torrent to the session + add_torrent_params p = m_settings.add_args; + p.url = i->url; + p.uuid = i->uuid; + p.source_feed_url = m_settings.url; + p.ti.reset(); + p.info_hash.clear(); + p.name = i->title.c_str(); + error_code ec; + // #error session_impl::add_torrent doesn't support magnet links via url + m_ses.add_torrent(p, ec); + + if (ec) + { +// #error alert! + } + } + + m_last_update = time(0); + + // report that we successfully updated the feed + if (m_ses.m_alerts.should_post()) + { + m_ses.m_alerts.post_alert(rss_alert(my_handle(), m_settings.url + , rss_alert::state_updated, error_code())); + } + + // update m_ses.m_next_rss_update timestamps + // now that we have updated our timestamp + m_ses.update_rss_feeds(); +} + +#define TORRENT_SETTING(t, x) {#x, offsetof(feed_settings,x), t}, + bencode_map_entry feed_settings_map[] = + { + TORRENT_SETTING(std_string, url) + TORRENT_SETTING(boolean, auto_download) + TORRENT_SETTING(integer, default_ttl) + }; +#undef TORRENT_SETTING + +#define TORRENT_SETTING(t, x) {#x, offsetof(feed_item,x), t}, + bencode_map_entry feed_item_map[] = + { + TORRENT_SETTING(std_string, url) + TORRENT_SETTING(std_string, uuid) + TORRENT_SETTING(std_string, title) + TORRENT_SETTING(std_string, description) + TORRENT_SETTING(std_string, comment) + TORRENT_SETTING(std_string, category) + TORRENT_SETTING(size_integer, size) + }; +#undef TORRENT_SETTING + +#define TORRENT_SETTING(t, x) {#x, offsetof(feed,x), t}, + bencode_map_entry feed_map[] = + { + TORRENT_SETTING(std_string, m_title) + TORRENT_SETTING(std_string, m_description) + TORRENT_SETTING(time_integer, m_last_attempt) + TORRENT_SETTING(time_integer, m_last_update) + }; +#undef TORRENT_SETTING + +#define TORRENT_SETTING(t, x) {#x, offsetof(add_torrent_params,x), t}, + bencode_map_entry add_torrent_map[] = + { + TORRENT_SETTING(std_string, save_path) + TORRENT_SETTING(boolean, paused) + TORRENT_SETTING(boolean, auto_managed) + TORRENT_SETTING(boolean, duplicate_is_error) + TORRENT_SETTING(boolean, seed_mode) + TORRENT_SETTING(boolean, override_resume_data) + TORRENT_SETTING(boolean, upload_mode) + TORRENT_SETTING(boolean, share_mode) + TORRENT_SETTING(std_string, trackerid) + }; +#undef TORRENT_SETTING + +void feed::load_state(lazy_entry const& rd) +{ + load_struct(rd, this, feed_map, sizeof(feed_map)/sizeof(feed_map[0])); + lazy_entry const* e = rd.dict_find_list("items"); + if (e) + { + m_items.reserve(e->list_size()); + for (int i = 0; i < e->list_size(); ++i) + { + if (e->list_at(i)->type() != lazy_entry::dict_t) continue; + m_items.push_back(feed_item()); + load_struct(*e->list_at(i), &m_items.back(), feed_item_map + , sizeof(feed_item_map)/sizeof(feed_item_map[0])); + } + } + load_struct(rd, &m_settings, feed_settings_map + , sizeof(feed_settings_map)/sizeof(feed_settings_map[0])); + e = rd.dict_find_dict("add_params"); + if (e) + { + load_struct(*e, &m_settings.add_args, add_torrent_map + , sizeof(add_torrent_map)/sizeof(add_torrent_map[0])); + } +} + +void feed::save_state(entry& rd) const +{ + save_struct(rd, this, feed_map, sizeof(feed_map)/sizeof(feed_map[0])); + entry::list_type& items = rd["items"].list(); + for (std::vector::const_iterator i = m_items.begin() + , end(m_items.end()); i != end; ++i) + { + items.push_back(entry()); + entry& item = items.back(); + save_struct(item, &*i, feed_item_map, sizeof(feed_item_map)/sizeof(feed_item_map[0])); + } + feed_settings sett_def; + save_struct(rd, &m_settings, feed_settings_map + , sizeof(feed_settings_map)/sizeof(feed_settings_map[0]), &sett_def); + entry& add = rd["add_params"]; + add_torrent_params add_def; + save_struct(add, &m_settings.add_args, add_torrent_map + , sizeof(add_torrent_map)/sizeof(add_torrent_map[0]), &add_def); +} + +void feed::update_feed() +{ + if (m_updating) return; + + m_last_attempt = time(0); + + if (m_ses.m_alerts.should_post()) + { + m_ses.m_alerts.post_alert(rss_alert(my_handle(), m_settings.url + , rss_alert::state_updating, error_code())); + } + + boost::shared_ptr feed( + new http_connection(m_ses.m_io_service, m_ses.m_half_open + , boost::bind(&feed::on_feed, shared_from_this() + , _1, _2, _3, _4))); + + m_updating = true; + feed->get(m_settings.url, seconds(30), 0, 0, 5, m_ses.m_settings.user_agent); +} + +void feed::get_feed_status(feed_status* ret) const +{ + ret->items = m_items; + ret->last_update = m_last_update; + ret->updating = m_updating; + ret->url = m_settings.url; + ret->title = m_title; + ret->description = m_description; + ret->error = m_error; + ret->ttl = m_ttl == -1 ? m_settings.default_ttl : m_ttl; + ret->next_update = next_update(time(0)); +} + +int feed::next_update(time_t now) const +{ + int ttl = m_ttl == -1 ? m_settings.default_ttl : m_ttl; + return (m_last_update + ttl * 60) - now; +} + +// defined in session.cpp +void fun_wrap(bool* done, condition* e, mutex* m, boost::function f); + +#define TORRENT_ASYNC_CALL(x) \ + boost::shared_ptr f = m_feed_ptr.lock(); \ + if (!f) return; \ + aux::session_impl& ses = f->session(); \ + ses.m_io_service.post(boost::bind(&feed:: x, f)) + +#define TORRENT_ASYNC_CALL1(x, a1) \ + boost::shared_ptr f = m_feed_ptr.lock(); \ + if (!f) return; \ + aux::session_impl& ses = f->session(); \ + ses.m_io_service.post(boost::bind(&feed:: x, f, a1)) + +#define TORRENT_SYNC_CALL1(x, a1) \ + boost::shared_ptr f = m_feed_ptr.lock(); \ + if (f) { \ + bool done = false; \ + aux::session_impl& ses = f->session(); \ + mutex::scoped_lock l(ses.mut); \ + ses.m_io_service.post(boost::bind(&fun_wrap, &done, &ses.cond, &ses.mut, boost::function(boost::bind(&feed:: x, f, a1)))); \ + f.reset(); \ + do { ses.cond.wait(l); } while(!done); } + +feed_handle::feed_handle(boost::weak_ptr const& p) + : m_feed_ptr(p) {} + +void feed_handle::update_feed() +{ + TORRENT_ASYNC_CALL(update_feed); +} + +feed_status feed_handle::get_feed_status() const +{ + feed_status ret; + TORRENT_SYNC_CALL1(get_feed_status, &ret); + return ret; +} + +void feed_handle::set_settings(feed_settings const& s) +{ + TORRENT_SYNC_CALL1(set_settings, s); +} + +feed_settings feed_handle::settings() const +{ + feed_settings ret; + TORRENT_SYNC_CALL1(get_settings, &ret); + return ret; +} + +}; + diff --git a/src/session.cpp b/src/session.cpp index 5331906fd..851187e61 100644 --- a/src/session.cpp +++ b/src/session.cpp @@ -375,6 +375,25 @@ namespace libtorrent TORRENT_SYNC_CALL1(load_state, &e); } + feed_handle session::add_feed(feed_settings const& feed) + { + // if you have auto-download enabled, you must specify a download directory! + TORRENT_ASSERT(!feed.auto_download || !feed.add_args.save_path.empty()); + TORRENT_SYNC_CALL_RET1(feed_handle, add_feed, feed); + return r; + } + + void session::remove_feed(feed_handle h) + { + TORRENT_ASYNC_CALL1(remove_feed, h); + } + + void session::get_feeds(std::vector& f) const + { + f.clear(); + TORRENT_SYNC_CALL1(get_feeds, &f); + } + #ifndef TORRENT_DISABLE_EXTENSIONS void session::add_extension(boost::function(torrent*, void*)> ext) { diff --git a/src/session_impl.cpp b/src/session_impl.cpp index d6c5d601b..bc4bd6f57 100644 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -500,6 +500,7 @@ namespace aux { , m_last_tick(m_created) , m_last_second_tick(m_created - milliseconds(900)) , m_last_choke(m_created) + , m_next_rss_update(min_time()) #ifndef TORRENT_DISABLE_DHT , m_dht_announce_timer(m_io_service) #endif @@ -914,7 +915,6 @@ namespace aux { { e["dht state"] = m_dht->state(); } - #endif #if TORRENT_USE_I2P @@ -940,6 +940,17 @@ namespace aux { } #endif + if (flags & session::save_feeds) + { + entry::list_type& feeds = e["feeds"].list(); + for (std::vector >::const_iterator i = + m_feeds.begin(), end(m_feeds.end()); i != end; ++i) + { + feeds.push_back(entry()); + (*i)->save_state(feeds.back()); + } + } + } void session_impl::set_proxy(proxy_settings const& s) @@ -1018,6 +1029,19 @@ namespace aux { m_lsd->use_broadcast(true); update_disk_thread_settings(); + + settings = e->dict_find_list("feeds"); + if (settings) + { + m_feeds.reserve(settings->list_size()); + for (int i = 0; i < settings->list_size(); ++i) + { + boost::shared_ptr f(new_feed(*this, feed_settings())); + if (settings->list_at(i)->type() != lazy_entry::dict_t) continue; + f->load_state(*settings->list_at(i)); + m_feeds.push_back(f); + } + } } #ifndef TORRENT_DISABLE_GEO_IP @@ -1151,6 +1175,51 @@ namespace aux { } #endif + feed_handle session_impl::add_feed(feed_settings const& sett) + { + TORRENT_ASSERT(is_network_thread()); + + // look for duplicates. If we already have a feed with this + // URL, return a handle to the existing one + for (std::vector >::const_iterator i + = m_feeds.begin(), end(m_feeds.end()); i != end; ++i) + { + if (sett.url != (*i)->m_settings.url) continue; + return feed_handle(*i); + } + + boost::shared_ptr f(new_feed(*this, sett)); + m_feeds.push_back(f); + update_rss_feeds(); + return feed_handle(f); + } + + void session_impl::remove_feed(feed_handle h) + { + TORRENT_ASSERT(is_network_thread()); + + boost::shared_ptr f = h.m_feed_ptr.lock(); + if (!f) return; + + std::vector >::iterator i + = std::find(m_feeds.begin(), m_feeds.end(), f); + + if (i == m_feeds.end()) return; + + m_feeds.erase(i); + } + + void session_impl::get_feeds(std::vector* ret) const + { + TORRENT_ASSERT(is_network_thread()); + + ret->clear(); + ret->reserve(m_feeds.size()); + for (std::vector >::const_iterator i = m_feeds.begin() + , end(m_feeds.end()); i != end; ++i) + ret->push_back(feed_handle(*i)); + } + void session_impl::pause() { TORRENT_ASSERT(is_network_thread()); @@ -2309,6 +2378,12 @@ namespace aux { } } + // -------------------------------------------------------------- + // RSS feeds + // -------------------------------------------------------------- + if (now > m_next_rss_update) + update_rss_feeds(); + switch (m_settings.mixed_mode_algorithm) { case session_settings::prefer_tcp: @@ -2790,6 +2865,27 @@ namespace aux { // m_peer_pool.release_memory(); } + void session_impl::update_rss_feeds() + { + time_t now_posix = time(0); + ptime min_update = max_time(); + ptime now = time_now(); + for (std::vector >::iterator i + = m_feeds.begin(), end(m_feeds.end()); i != end; ++i) + { + feed& f = **i; + int delta = f.next_update(now_posix); + if (delta <= 0) + { + f.update_feed(); + continue; + } + ptime next_update = now + seconds(delta); + if (next_update < min_update) min_update = next_update; + } + m_next_rss_update = min_update; + } + #ifndef TORRENT_DISABLE_DHT void session_impl::on_dht_announce(error_code const& e) @@ -3404,6 +3500,16 @@ namespace aux { return boost::weak_ptr(); } + boost::weak_ptr session_impl::find_torrent(std::string const& uuid) + { + TORRENT_ASSERT(is_network_thread()); + + std::map >::iterator i + = m_uuids.find(uuid); + if (i != m_uuids.end()) return i->second; + return boost::weak_ptr(); + } + #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING boost::shared_ptr session_impl::create_log(std::string const& name , int instance, bool append) @@ -3469,6 +3575,8 @@ namespace aux { // is the torrent already active? boost::shared_ptr torrent_ptr = find_torrent(*ih).lock(); + if (!torrent_ptr && !params.uuid.empty()) torrent_ptr = find_torrent(params.uuid).lock(); + if (torrent_ptr) { if (!params.duplicate_is_error) @@ -3511,6 +3619,9 @@ namespace aux { #endif m_torrents.insert(std::make_pair(*ih, torrent_ptr)); + if (!params.uuid.empty() || !params.url.empty()) + m_uuids.insert(std::make_pair(params.uuid.empty() + ? params.url : params.uuid, torrent_ptr)); // if this is an auto managed torrent, force a recalculation // of which torrents to have active @@ -3547,9 +3658,10 @@ namespace aux { { TORRENT_ASSERT(*i == t || (*i)->should_check_files()); if (*i == t) done = i; - if (next_check == t || next_check->queue_position() > (*i)->queue_position()) + else if (next_check == t || next_check->queue_position() > (*i)->queue_position()) next_check = *i; } + TORRENT_ASSERT(next_check != t || m_queued_for_checking.size() == 1); // only start a new one if we removed the one that is checking TORRENT_ASSERT(done != m_queued_for_checking.end()); if (done == m_queued_for_checking.end()) return; @@ -3572,6 +3684,14 @@ namespace aux { INVARIANT_CHECK; + // remove from uuid list + if (!tptr->uuid().empty()) + { + std::map >::iterator j + = m_uuids.find(tptr->uuid()); + if (j != m_uuids.end()) m_uuids.erase(j); + } + session_impl::torrent_map::iterator i = m_torrents.find(tptr->torrent_file().info_hash()); diff --git a/src/settings.cpp b/src/settings.cpp index 8811965ed..2cd25575f 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -58,6 +58,8 @@ namespace libtorrent case character: case boolean: case integer: + case size_integer: + case time_integer: case floating_point: { if (key->type() != lazy_entry::int_t) continue; @@ -66,6 +68,8 @@ namespace libtorrent { case character: *((char*)dest) = val; break; case integer: *((int*)dest) = val; break; + case size_integer: *((size_type*)dest) = val; break; + case time_integer: *((time_t*)dest) = val; break; case floating_point: *((float*)dest) = float(val) / 1000.f; break; case boolean: *((bool*)dest) = val; break; } @@ -76,7 +80,7 @@ namespace libtorrent void save_struct(entry& e, void const* s, bencode_map_entry const* m, int num, void const* def) { - e = entry(entry::dictionary_t); + if (e.type() != entry::dictionary_t) e = entry(entry::dictionary_t); for (int i = 0; i < num; ++i) { char const* key = m[i].name; @@ -97,6 +101,12 @@ namespace libtorrent case integer: if (*((int*)src) == *((int*)default_value)) continue; break; + case size_integer: + if (*((size_type*)src) == *((size_type*)default_value)) continue; + break; + case time_integer: + if (*((time_t*)src) == *((time_t*)default_value)) continue; + break; case floating_point: if (*((float*)src) == *((float*)default_value)) continue; break; @@ -113,6 +123,8 @@ namespace libtorrent case std_string: val = *((std::string*)src); break; case character: val = *((char*)src); break; case integer: val = *((int*)src); break; + case size_integer: val = *((size_type*)src); break; + case time_integer: val = *((time_t*)src); break; case floating_point: val = size_type(*((float*)src) * 1000.f); break; case boolean: val = *((bool*)src); break; default: TORRENT_ASSERT(false); diff --git a/src/torrent.cpp b/src/torrent.cpp index 8b6287c8a..0ffc6c48a 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -337,6 +337,8 @@ namespace libtorrent , m_trackerid(p.trackerid) , m_save_path(complete(p.save_path)) , m_url(p.url) + , m_uuid(p.uuid) + , m_source_feed_url(p.source_feed_url) , m_storage_constructor(p.storage) , m_ratio(0.f) , m_available_free_upload(0) @@ -400,6 +402,8 @@ namespace libtorrent , m_graceful_pause_mode(false) , m_need_connect_boost(true) { + if (!m_url.empty() && m_uuid.empty()) m_uuid = m_url; + TORRENT_ASSERT(m_ses.is_network_thread()); #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING (*m_ses.m_logger) << time_now_string() << " creating torrent: " @@ -442,6 +446,10 @@ namespace libtorrent } } +#if 1 + + // NON BOTTLED VERSION. SUPPORTS PROGRESS REPORTING + // since this download is not bottled, this callback will // be called every time we receive another piece of the // .torrent file @@ -516,6 +524,85 @@ namespace libtorrent TORRENT_ASSERT(num_torrents == m_ses.m_torrents.size()); + // if the user added any trackers while downloading the + // .torrent file, serge them into the new tracker list + std::vector new_trackers = m_torrent_file->trackers(); + for (std::vector::iterator i = m_trackers.begin() + , end(m_trackers.end()); i != end; ++i) + { + // if we already have this tracker, ignore it + if (std::find_if(new_trackers.begin(), new_trackers.end() + , boost::bind(&announce_entry::url, _1) == i->url) != new_trackers.end()) + continue; + + // insert the tracker ordered by tier + new_trackers.insert(std::find_if(new_trackers.begin(), new_trackers.end() + , boost::bind(&announce_entry::tier, _1) >= i->tier), *i); + } + m_trackers.swap(new_trackers); + +#ifndef TORRENT_DISABLE_ENCRYPTION + hasher h; + h.update("req2", 4); + h.update((char*)&m_torrent_file->info_hash()[0], 20); + m_obfuscated_hash = h.final(); +#endif + + if (m_ses.m_alerts.should_post()) + { + m_ses.m_alerts.post_alert(metadata_received_alert( + get_handle())); + } + + set_state(torrent_status::downloading); + + m_override_resume_data = true; + init(); + announce_with_tracker(); + } +#else + + void torrent::on_torrent_download(error_code const& ec + , http_parser const& parser, char const* data, int size) + { + if (m_abort) return; + + if (ec && ec != asio::error::eof) + { + set_error(ec, m_url); + pause(); + return; + } + + if (parser.status_code() != 200) + { + // #error there should really be an error code category for HTTP + set_error(errors::http_error, parser.message()); + pause(); + return; + } + + error_code e; + intrusive_ptr tf(new torrent_info(data, size, e)); + if (e) + { + set_error(e, m_url); + pause(); + return; + } + + // update our torrent_info object and move the + // torrent from the old info-hash to the new one + // as we replace the torrent_info object +#ifdef TORRENT_DEBUG + int num_torrents = m_ses.m_torrents.size(); +#endif + m_ses.m_torrents.erase(m_torrent_file->info_hash()); + m_torrent_file = tf; + m_ses.m_torrents.insert(std::make_pair(m_torrent_file->info_hash(), shared_from_this())); + + TORRENT_ASSERT(num_torrents == m_ses.m_torrents.size()); + // TODO: if the user added any trackers while downloading the // .torrent file, they are overwritten. Merge them into the // new tracker list @@ -541,6 +628,8 @@ namespace libtorrent announce_with_tracker(); } +#endif + void torrent::start() { TORRENT_ASSERT(m_ses.is_network_thread()); @@ -4086,6 +4175,9 @@ namespace libtorrent m_last_download = rd.dict_find_int_value("last_download", 0); m_last_upload = rd.dict_find_int_value("last_upload", 0); + m_url = rd.dict_find_string_value("url"); + m_uuid = rd.dict_find_string_value("uuid"); + m_added_time = rd.dict_find_int_value("added_time", m_added_time); m_completed_time = rd.dict_find_int_value("completed_time", m_completed_time); if (m_completed_time != 0 && m_completed_time < m_added_time) @@ -4255,6 +4347,9 @@ namespace libtorrent ret["last_scrape"] = m_last_scrape; ret["last_download"] = m_last_download; ret["last_upload"] = m_last_upload; + + if (!m_url.empty()) ret["url"] = m_url; + if (!m_uuid.empty()) ret["uuid"] = m_uuid; const sha1_hash& info_hash = torrent_file().info_hash(); ret["info-hash"] = std::string((char*)info_hash.begin(), (char*)info_hash.end()); diff --git a/test/test_natpmp.cpp b/test/test_natpmp.cpp index cb0fd8a67..b921f33d9 100644 --- a/test/test_natpmp.cpp +++ b/test/test_natpmp.cpp @@ -8,11 +8,12 @@ using namespace libtorrent; -void callback(int mapping, int port, error_code const& err) +void callback(int mapping, address extip, int port, error_code const& err) { std::cerr << "mapping: " << mapping << ", port: " << port + << ", external-IP: " << print_address(extip) << ", error: \"" << err.message() << "\"\n"; }