[brightcove] Set the 'Referer' header if the url has the 'linkBaseUrl' parameter (fixes #1553)

youtube_dl/extractor/brightcove.py

@@ -10,10 +10,12 @@ from ..utils import (
     find_xpath_attr,
     compat_urlparse,
     compat_str,
+    compat_urllib_request,
 
     ExtractorError,
 )
 
+
 class BrightcoveIE(InfoExtractor):
     _VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*\?(?P<query>.*)'
     _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
@@ -80,6 +82,9 @@ class BrightcoveIE(InfoExtractor):
         videoPlayer = find_xpath_attr(object_doc, './param', 'name', '@videoPlayer')
         if videoPlayer is not None:
             params['@videoPlayer'] = videoPlayer.attrib['value']
+        linkBase = find_xpath_attr(object_doc, './param', 'name', 'linkBaseURL')
+        if linkBase is not None:
+            params['linkBaseURL'] = linkBase.attrib['value']
         data = compat_urllib_parse.urlencode(params)
         return cls._FEDERATED_URL_TEMPLATE % data
 
@@ -107,14 +112,18 @@ class BrightcoveIE(InfoExtractor):
 
         videoPlayer = query.get('@videoPlayer')
         if videoPlayer:
-            return self._get_video_info(videoPlayer[0], query_str)
+            return self._get_video_info(videoPlayer[0], query_str, query)
         else:
             player_key = query['playerKey']
             return self._get_playlist_info(player_key[0])
 
-    def _get_video_info(self, video_id, query):
+    def _get_video_info(self, video_id, query_str, query):
-        request_url = self._FEDERATED_URL_TEMPLATE % query
+        request_url = self._FEDERATED_URL_TEMPLATE % query_str
-        webpage = self._download_webpage(request_url, video_id)
+        req = compat_urllib_request.Request(request_url)
+        linkBase = query.get('linkBaseURL')
+        if linkBase is not None:
+            req.add_header('Referer', linkBase[0])
+        webpage = self._download_webpage(req, video_id)
 
         self.report_extraction(video_id)
         info = self._search_regex(r'var experienceJSON = ({.*?});', webpage, 'json')

youtube_dl/extractor/generic.py

@@ -55,15 +55,17 @@ class GenericIE(InfoExtractor):
             u'skip': u'There is a limit of 200 free downloads / month for the test song',
         },
         # embedded brightcove video
+        # it also tests brightcove videos that need to set the 'Referer' in the
+        # http requests
         {
             u'add_ie': ['Brightcove'],
-            u'url': u'http://www.scientificamerican.com/article.cfm?id=soap-bubble-physics',
+            u'url': u'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
             u'info_dict': {
-                u'id': u'2365799484001',
+                u'id': u'2765128793001',
                 u'ext': u'mp4',
-                u'title': u'Bubble Simulation',
+                u'title': u'Le cours de bourse : l’analyse technique',
-                u'description': u'A visualization from a new computer model of foam behavior.',
+                u'description': u'md5:7e9ad046e968cb2d1114004aba466fd9',
-                u'uploader': u'Scientific American',
+                u'uploader': u'BFM BUSINESS',
             },
             u'params': {
                 u'skip_download': True,