diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..58c855615 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +* text=auto + +Makefile* text whitespace=-tab-in-indent +*.sh text eol=lf diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md index 32c14aa85..5dc6b85f3 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.md +++ b/.github/ISSUE_TEMPLATE/1_broken_site.md @@ -20,16 +20,16 @@ assignees: '' ## Checklist - [ ] I'm reporting a broken site support -- [ ] I've verified that I'm running youtube-dlc version **2020.10.31** +- [ ] I've verified that I'm running yt-dlp version **2021.07.24** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar issues including closed ones @@ -38,13 +38,13 @@ Carefully read and work through this check list in order to prevent the most com ## Verbose log ## Description diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md index fe1aade05..54ce076c9 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.md +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md @@ -20,16 +20,16 @@ assignees: '' ## Checklist - [ ] I'm reporting a new site support request -- [ ] I've verified that I'm running youtube-dlcc version **2020.10.31** +- [ ] I've verified that I'm running yt-dlp version **2021.07.24** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that none of provided URLs violate any copyrights - [ ] I've searched the bugtracker for similar site support requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md index cddb81dda..76adac9b7 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md @@ -20,21 +20,21 @@ assignees: '' ## Checklist - [ ] I'm reporting a site feature request -- [ ] I've verified that I'm running youtube-dlc version **2020.10.31** +- [ ] I've verified that I'm running yt-dlp version **2021.07.24** - [ ] I've searched the bugtracker for similar site feature requests including closed ones ## Description WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md index 920ae8dbc..f13010b94 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.md +++ b/.github/ISSUE_TEMPLATE/4_bug_report.md @@ -20,17 +20,17 @@ assignees: '' ## Checklist - [ ] I'm reporting a broken site support issue -- [ ] I've verified that I'm running youtube-dlc version **2020.10.31** +- [ ] I've verified that I'm running yt-dlp version **2021.07.24** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar bug reports including closed ones @@ -40,13 +40,13 @@ Carefully read and work through this check list in order to prevent the most com ## Verbose log ## Description diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md index 7cc390f58..204e78357 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.md +++ b/.github/ISSUE_TEMPLATE/5_feature_request.md @@ -20,21 +20,21 @@ assignees: '' ## Checklist - [ ] I'm reporting a feature request -- [ ] I've verified that I'm running youtube-dlc version **2020.10.31** +- [ ] I've verified that I'm running yt-dlp version **2021.07.24** - [ ] I've searched the bugtracker for similar feature requests including closed ones ## Description WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/6_question.md b/.github/ISSUE_TEMPLATE/6_question.md index 3c3ae0f3b..dd2857c09 100644 --- a/.github/ISSUE_TEMPLATE/6_question.md +++ b/.github/ISSUE_TEMPLATE/6_question.md @@ -20,10 +20,10 @@ assignees: '' ## Checklist - [ ] I'm asking a question @@ -34,7 +34,7 @@ Carefully read and work through this check list in order to prevent the most com ## Question WRITE QUESTION HERE diff --git a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.md b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.md index 3fe4d6968..6da13a7b5 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.md +++ b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.md @@ -1,7 +1,10 @@ --- name: Broken site support about: Report broken or misfunctioning site -title: '' +title: "[Broken]" +labels: Broken +assignees: '' + --- - [ ] I'm reporting a broken site support -- [ ] I've verified that I'm running youtube-dlc version **%(version)s** +- [ ] I've verified that I'm running yt-dlp version **%(version)s** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar issues including closed ones @@ -35,13 +38,13 @@ Carefully read and work through this check list in order to prevent the most com ## Verbose log ## Description diff --git a/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md index aad8fa054..9e7eb5a20 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md +++ b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md @@ -1,8 +1,10 @@ --- name: Site support request about: Request support for a new site -title: '' -labels: 'site-support-request' +title: "[Site Request]" +labels: Request +assignees: '' + --- - [ ] I'm reporting a new site support request -- [ ] I've verified that I'm running youtube-dlc version **%(version)s** +- [ ] I've verified that I'm running yt-dlp version **%(version)s** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that none of provided URLs violate any copyrights - [ ] I've searched the bugtracker for similar site support requests including closed ones diff --git a/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md index 2fb82f828..d74b6e279 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md +++ b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md @@ -1,7 +1,10 @@ --- name: Site feature request about: Request a new functionality for a site -title: '' +title: "[Site Request]" +labels: Request +assignees: '' + --- - [ ] I'm reporting a site feature request -- [ ] I've verified that I'm running youtube-dlc version **%(version)s** +- [ ] I've verified that I'm running yt-dlp version **%(version)s** - [ ] I've searched the bugtracker for similar site feature requests including closed ones diff --git a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.md b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.md index b7bebf8ab..af1774462 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.md +++ b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.md @@ -2,6 +2,9 @@ name: Bug report about: Report a bug unrelated to any particular site or extractor title: '' +labels: '' +assignees: '' + --- - [ ] I'm reporting a broken site support issue -- [ ] I've verified that I'm running youtube-dlc version **%(version)s** +- [ ] I've verified that I'm running yt-dlp version **%(version)s** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar bug reports including closed ones @@ -37,13 +40,13 @@ Carefully read and work through this check list in order to prevent the most com ## Verbose log ## Description diff --git a/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.md b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.md index 99592f79d..4a0209db1 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.md +++ b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.md @@ -1,8 +1,10 @@ --- name: Feature request about: Request a new functionality unrelated to any particular site or extractor -title: '' -labels: 'request' +title: "[Feature Request]" +labels: Request +assignees: '' + --- - [ ] I'm reporting a feature request -- [ ] I've verified that I'm running youtube-dlc version **%(version)s** +- [ ] I've verified that I'm running yt-dlp version **%(version)s** - [ ] I've searched the bugtracker for similar feature requests including closed ones diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e69b907d8..f711701cb 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -8,7 +8,7 @@ ### Before submitting a *pull request* make sure you have: - [ ] At least skimmed through [adding new extractor tutorial](https://github.com/ytdl-org/youtube-dl#adding-support-for-a-new-site) and [youtube-dl coding conventions](https://github.com/ytdl-org/youtube-dl#youtube-dl-coding-conventions) sections -- [ ] [Searched](https://github.com/ytdl-org/youtube-dl/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests +- [ ] [Searched](https://github.com/yt-dlp/yt-dlp/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests - [ ] Checked the code with [flake8](https://pypi.python.org/pypi/flake8) ### In order to be accepted and merged into youtube-dl each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dd6a95256..a9fa01d54 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,13 +7,13 @@ on: jobs: build_unix: - runs-on: ubuntu-latest outputs: - ytdlc_version: ${{ steps.bump_version.outputs.ytdlc_version }} + ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }} upload_url: ${{ steps.create_release.outputs.upload_url }} - sha2_unix: ${{ steps.sha2_file.outputs.sha2_unix }} + sha256_unix: ${{ steps.sha256_file.outputs.sha256_unix }} + sha512_unix: ${{ steps.sha512_file.outputs.sha512_unix }} steps: - uses: actions/checkout@v2 @@ -25,132 +25,169 @@ jobs: run: sudo apt-get -y install zip pandoc man - name: Bump version id: bump_version - run: python scripts/update-version-workflow.py - - name: Check the output from My action - run: echo "${{ steps.bump_version.outputs.ytdlc_version }}" + run: python devscripts/update-version.py + - name: Print version + run: echo "${{ steps.bump_version.outputs.ytdlp_version }}" - name: Run Make - run: make + run: make all tar - name: Create Release id: create_release uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${{ steps.bump_version.outputs.ytdlc_version }} - release_name: youtube-dlc ${{ steps.bump_version.outputs.ytdlc_version }} + tag_name: ${{ steps.bump_version.outputs.ytdlp_version }} + release_name: yt-dlp ${{ steps.bump_version.outputs.ytdlp_version }} body: | Changelog: PLACEHOLDER draft: false prerelease: false - - name: Upload youtube-dlc Unix binary - id: upload-release-asset + - name: Upload yt-dlp Unix binary + id: upload-release-asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./youtube-dlc - asset_name: youtube-dlc + asset_path: ./yt-dlp + asset_name: yt-dlp asset_content_type: application/octet-stream - - name: Get SHA2-256SUMS for youtube-dlc - id: sha2_file + - name: Upload Source tar + uses: actions/upload-release-asset@v1 env: - SHA2: ${{ hashFiles('youtube-dlc') }} - run: echo "::set-output name=sha2_unix::$SHA2" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./yt-dlp.tar.gz + asset_name: yt-dlp.tar.gz + asset_content_type: application/gzip + - name: Get SHA2-256SUMS for yt-dlp + id: sha256_file + run: echo "::set-output name=sha256_unix::$(sha256sum yt-dlp | awk '{print $1}')" + - name: Get SHA2-512SUMS for yt-dlp + id: sha512_file + run: echo "::set-output name=sha512_unix::$(sha512sum yt-dlp | awk '{print $1}')" - name: Install dependencies for pypi + env: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + if: "env.PYPI_TOKEN != ''" run: | python -m pip install --upgrade pip pip install setuptools wheel twine - - name: Build and publish + - name: Build and publish on pypi env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + if: "env.TWINE_PASSWORD != ''" run: | rm -rf dist/* python setup.py sdist bdist_wheel twine upload dist/* build_windows: - runs-on: windows-latest - needs: build_unix + outputs: + sha256_windows: ${{ steps.sha256_file_win.outputs.sha256_windows }} + sha512_windows: ${{ steps.sha512_file_win.outputs.sha512_windows }} + steps: - uses: actions/checkout@v2 - - name: Set up Python + # 3.8 is used for Win7 support + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: python-version: '3.8' + - name: Upgrade pip and enable wheel support + run: python -m pip install --upgrade pip setuptools wheel - name: Install Requirements - run: pip install pyinstaller + run: pip install pyinstaller mutagen pycryptodome websockets - name: Bump version - run: python scripts/update-version-workflow.py + id: bump_version + run: python devscripts/update-version.py + - name: Print version + run: echo "${{ steps.bump_version.outputs.ytdlp_version }}" - name: Run PyInstaller Script - run: python pyinst.py - - name: Upload youtube-dlc.exe Windows binary + run: python pyinst.py 64 + - name: Upload yt-dlp.exe Windows binary id: upload-release-windows uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ needs.build_unix.outputs.upload_url }} - asset_path: ./dist/youtube-dlc.exe - asset_name: youtube-dlc.exe + asset_path: ./dist/yt-dlp.exe + asset_name: yt-dlp.exe asset_content_type: application/vnd.microsoft.portable-executable - - name: Get SHA2-256SUMS for youtube-dlc.exe - id: sha2_file_win - env: - SHA2_win: ${{ hashFiles('dist/youtube-dlc.exe') }} - run: echo "::set-output name=sha2_windows::$SHA2_win" + - name: Get SHA2-256SUMS for yt-dlp.exe + id: sha256_file_win + run: echo "::set-output name=sha256_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())" + - name: Get SHA2-512SUMS for yt-dlp.exe + id: sha512_file_win + run: echo "::set-output name=sha512_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())" build_windows32: - runs-on: windows-latest - needs: [build_unix, build_windows] + outputs: + sha256_windows32: ${{ steps.sha256_file_win32.outputs.sha256_windows32 }} + sha512_windows32: ${{ steps.sha512_file_win32.outputs.sha512_windows32 }} + steps: - uses: actions/checkout@v2 - - name: Set up Python 3.4.4 32-Bit + # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390 + - name: Set up Python 3.7 32-Bit uses: actions/setup-python@v2 with: - python-version: '3.4.4' + python-version: '3.7' architecture: 'x86' - - name: Install Requirements for 32 Bit - run: pip install pyinstaller==3.5 + - name: Upgrade pip and enable wheel support + run: python -m pip install --upgrade pip setuptools wheel + - name: Install Requirements + run: pip install pyinstaller mutagen pycryptodome websockets - name: Bump version - run: python scripts/update-version-workflow.py + id: bump_version + run: python devscripts/update-version.py + - name: Print version + run: echo "${{ steps.bump_version.outputs.ytdlp_version }}" - name: Run PyInstaller Script for 32 Bit - run: python pyinst32.py - - name: Upload Executable youtube-dlc_x86.exe + run: python pyinst.py 32 + - name: Upload Executable yt-dlp_x86.exe id: upload-release-windows32 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ needs.build_unix.outputs.upload_url }} - asset_path: ./dist/youtube-dlc_x86.exe - asset_name: youtube-dlc_x86.exe + asset_path: ./dist/yt-dlp_x86.exe + asset_name: yt-dlp_x86.exe asset_content_type: application/vnd.microsoft.portable-executable - - name: Get SHA2-256SUMS for youtube-dlc_x86.exe - id: sha2_file_win32 - env: - SHA2_win32: ${{ hashFiles('dist/youtube-dlc_x86.exe') }} - run: echo "::set-output name=sha2_windows32::$SHA2_win32" + - name: Get SHA2-256SUMS for yt-dlp_x86.exe + id: sha256_file_win32 + run: echo "::set-output name=sha256_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())" + - name: Get SHA2-512SUMS for yt-dlp_x86.exe + id: sha512_file_win32 + run: echo "::set-output name=sha512_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA512).Hash.ToLower())" + + finish: + runs-on: ubuntu-latest + needs: [build_unix, build_windows, build_windows32] + + steps: - name: Make SHA2-256SUMS file env: - SHA2_WINDOWS: ${{ needs.build_windows.outputs.sha2_windows }} - SHA2_WINDOWS32: ${{ steps.sha2_file_win32.outputs.sha2_windows32 }} - SHA2_UNIX: ${{ needs.build_unix.outputs.sha2_unix }} - YTDLC_VERSION: ${{ needs.build_unix.outputs.ytdlc_version }} + SHA256_WINDOWS: ${{ needs.build_windows.outputs.sha256_windows }} + SHA256_WINDOWS32: ${{ needs.build_windows32.outputs.sha256_windows32 }} + SHA256_UNIX: ${{ needs.build_unix.outputs.sha256_unix }} + YTDLP_VERSION: ${{ needs.build_unix.outputs.ytdlp_version }} run: | - echo "version:${env:YTDLC_VERSION}" >> SHA2-256SUMS - echo "youtube-dlc.exe:${env:SHA2_WINDOWS}" >> SHA2-256SUMS - echo "youtube-dlc_x86.exe:${env:SHA2_WINDOWS32}" >> SHA2-256SUMS - echo "youtube-dlc:${env:SHA2_UNIX}" >> SHA2-256SUMS - + echo "version:${{ env.YTDLP_VERSION }}" >> SHA2-256SUMS + echo "yt-dlp.exe:${{ env.SHA256_WINDOWS }}" >> SHA2-256SUMS + echo "yt-dlp_x86.exe:${{ env.SHA256_WINDOWS32 }}" >> SHA2-256SUMS + echo "yt-dlp:${{ env.SHA256_UNIX }}" >> SHA2-256SUMS - name: Upload 256SUMS file id: upload-sums uses: actions/upload-release-asset@v1 @@ -161,3 +198,22 @@ jobs: asset_path: ./SHA2-256SUMS asset_name: SHA2-256SUMS asset_content_type: text/plain + - name: Make SHA2-512SUMS file + env: + SHA512_WINDOWS: ${{ needs.build_windows.outputs.sha512_windows }} + SHA512_WINDOWS32: ${{ needs.build_windows32.outputs.sha512_windows32 }} + SHA512_UNIX: ${{ needs.build_unix.outputs.sha512_unix }} + run: | + echo "${{ env.SHA512_WINDOWS }} yt-dlp.exe" >> SHA2-512SUMS + echo "${{ env.SHA512_WINDOWS32 }} yt-dlp_x86.exe" >> SHA2-512SUMS + echo "${{ env.SHA512_UNIX }} yt-dlp" >> SHA2-512SUMS + - name: Upload 512SUMS file + id: upload-512sums + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.build_unix.outputs.upload_url }} + asset_path: ./SHA2-512SUMS + asset_name: SHA2-512SUMS + asset_content_type: text/plain diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml new file mode 100644 index 000000000..4fb65e0c1 --- /dev/null +++ b/.github/workflows/core.yml @@ -0,0 +1,31 @@ +name: Core Tests +on: [push, pull_request] +jobs: + tests: + name: Core Tests + if: "!contains(github.event.head_commit.message, 'ci skip')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-18.04] + # py3.9 is in quick-test + python-version: [3.7, 3.8, 3.10-dev, pypy-3.6, pypy-3.7] + run-tests-ext: [sh] + include: + # atleast one of the tests must be in windows + - os: windows-latest + python-version: 3.6 + run-tests-ext: bat + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install pytest + run: pip install pytest + - name: Run tests + continue-on-error: False + run: ./devscripts/run_tests.${{ matrix.run-tests-ext }} core + # Linter is in quick-test diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml new file mode 100644 index 000000000..dd242fa56 --- /dev/null +++ b/.github/workflows/download.yml @@ -0,0 +1,28 @@ +name: Download Tests +on: [push, pull_request] +jobs: + tests: + name: Download Tests + if: "contains(github.event.head_commit.message, 'ci run dl')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [ubuntu-18.04] + python-version: [3.7, 3.8, 3.9, 3.10-dev, pypy-3.6, pypy-3.7] + run-tests-ext: [sh] + include: + - os: windows-latest + python-version: 3.6 + run-tests-ext: bat + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install pytest + run: pip install pytest + - name: Run tests + continue-on-error: true + run: ./devscripts/run_tests.${{ matrix.run-tests-ext }} download diff --git a/.github/workflows/python-publish.yml.disable b/.github/workflows/python-publish.yml.disable deleted file mode 100644 index 224a00230..000000000 --- a/.github/workflows/python-publish.yml.disable +++ /dev/null @@ -1,33 +0,0 @@ -# This workflows will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -name: Upload Python Package - -on: - push: - branches: - - release - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - rm -rf dist/* - python setup.py sdist bdist_wheel - twine upload dist/* diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml new file mode 100644 index 000000000..7d409dfc4 --- /dev/null +++ b/.github/workflows/quick-test.yml @@ -0,0 +1,31 @@ +name: Quick Test +on: [push, pull_request] +jobs: + tests: + name: Core Test + if: "!contains(github.event.head_commit.message, 'ci skip all')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install test requirements + run: pip install pytest pycryptodome + - name: Run tests + run: ./devscripts/run_tests.sh core + flake8: + name: Linter + if: "!contains(github.event.head_commit.message, 'ci skip all')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install flake8 + run: pip install flake8 + - name: Run flake8 + run: flake8 . diff --git a/.gitignore b/.gitignore index 065a14f49..7ed34448a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,34 +1,10 @@ -*.pyc -*.pyo -*.class -*~ -*.DS_Store -wine-py2exe/ -py2exe.log -*.kate-swp -build/ -dist/ -MANIFEST -README.txt -youtube-dl.1 -youtube-dlc.1 -youtube-dl.bash-completion -youtube-dlc.bash-completion -youtube-dl.fish -youtube-dlc.fish -youtube_dl/extractor/lazy_extractors.py -youtube_dlc/extractor/lazy_extractors.py -youtube-dl -youtube-dlc -youtube-dl.exe -youtube-dlc.exe -youtube-dl.tar.gz -youtube-dlc.tar.gz -youtube-dlc.spec -.coverage -cover/ -updates_key.pem -*.egg-info +# Config +*.conf +*.spec +cookies +cookies.txt + +# Downloaded *.srt *.ttml *.sbv @@ -39,26 +15,90 @@ updates_key.pem *.m4v *.mp3 *.3gp +*.webm *.wav *.ape *.mkv *.swf *.part +*.part-* *.ytdl +*.dump +*.frag +*.frag.urls +*.aria2 *.swp -test/local_parameters.json -.tox -youtube-dl.zsh -youtube-dlc.zsh +*.ogg +*.opus +*.info.json +*.live_chat.json +*.jpg +*.jpeg +*.png +*.webp +*.annotations.xml +*.description -# IntelliJ related files -.idea -*.iml +# Allow config/media files in testdata +!test/testdata/** +# Python +*.pyc +*.pyo +.pytest_cache +wine-py2exe/ +py2exe.log +build/ +dist/ +zip/ tmp/ venv/ +completions/ -# VS Code related files +# Misc +*~ +*.DS_Store +*.kate-swp +MANIFEST +test/local_parameters.json +.coverage +cover/ +secrets/ +updates_key.pem +*.egg-info +.tox +*.class + +# Generated +AUTHORS +README.txt +.mailmap +*.1 +*.bash-completion +*.fish +*.exe +*.tar.gz +*.zsh +*.spec +test/testdata/player-*.js + +# Binary +/youtube-dl +/youtube-dlc +/yt-dlp +yt-dlp.zip +*.exe + +# Text Editor / IDE +.idea +*.iml .vscode +*.sublime-* -cookies.txt +# Lazy extractors +*/extractor/lazy_extractors.py + +# Plugins +ytdlp_plugins/extractor/* +!ytdlp_plugins/extractor/__init__.py +!ytdlp_plugins/extractor/sample.py diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 000000000..052f7bfca --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally build your docs in additional formats such as PDF +formats: + - epub + - pdf + - htmlzip + +# Optionally set the version of Python and requirements required to build your docs +python: + version: 3 + install: + - requirements: docs/requirements.txt diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index fb499845e..000000000 --- a/.travis.yml +++ /dev/null @@ -1,38 +0,0 @@ -language: python -python: - - "2.6" - - "2.7" - - "3.2" - - "3.3" - - "3.4" - - "3.5" - - "3.6" - - "pypy" - - "pypy3" -dist: trusty -env: - - YTDL_TEST_SET=core -jobs: - include: - - python: 3.7 - dist: xenial - env: YTDL_TEST_SET=core - - python: 3.8 - dist: xenial - env: YTDL_TEST_SET=core - - python: 3.8-dev - dist: xenial - env: YTDL_TEST_SET=core - - env: JYTHON=true; YTDL_TEST_SET=core - - name: flake8 - python: 3.8 - dist: xenial - install: pip install flake8 - script: flake8 . - fast_finish: true - allow_failures: - - env: YTDL_TEST_SET=download - - env: JYTHON=true; YTDL_TEST_SET=core -before_install: - - if [ "$JYTHON" == "true" ]; then ./devscripts/install_jython.sh; export PATH="$HOME/jython/bin:$PATH"; fi -script: ./devscripts/run_tests.sh diff --git a/AUTHORS b/AUTHORS deleted file mode 100644 index b507cb8df..000000000 --- a/AUTHORS +++ /dev/null @@ -1,248 +0,0 @@ -Ricardo Garcia Gonzalez -Danny Colligan -Benjamin Johnson -Vasyl' Vavrychuk -Witold Baryluk -Paweł Paprota -Gergely Imreh -Rogério Brito -Philipp Hagemeister -Sören Schulze -Kevin Ngo -Ori Avtalion -shizeeg -Filippo Valsorda -Christian Albrecht -Dave Vasilevsky -Jaime Marquínez Ferrándiz -Jeff Crouse -Osama Khalid -Michael Walter -M. Yasoob Ullah Khalid -Julien Fraichard -Johny Mo Swag -Axel Noack -Albert Kim -Pierre Rudloff -Huarong Huo -Ismael Mejía -Steffan Donal -Andras Elso -Jelle van der Waa -Marcin Cieślak -Anton Larionov -Takuya Tsuchida -Sergey M. -Michael Orlitzky -Chris Gahan -Saimadhav Heblikar -Mike Col -Oleg Prutz -pulpe -Andreas Schmitz -Michael Kaiser -Niklas Laxström -David Triendl -Anthony Weems -David Wagner -Juan C. Olivares -Mattias Harrysson -phaer -Sainyam Kapoor -Nicolas Évrard -Jason Normore -Hoje Lee -Adam Thalhammer -Georg Jähnig -Ralf Haring -Koki Takahashi -Ariset Llerena -Adam Malcontenti-Wilson -Tobias Bell -Naglis Jonaitis -Charles Chen -Hassaan Ali -Dobrosław Żybort -David Fabijan -Sebastian Haas -Alexander Kirk -Erik Johnson -Keith Beckman -Ole Ernst -Aaron McDaniel (mcd1992) -Magnus Kolstad -Hari Padmanaban -Carlos Ramos -5moufl -lenaten -Dennis Scheiba -Damon Timm -winwon -Xavier Beynon -Gabriel Schubiner -xantares -Jan Matějka -Mauroy Sébastien -William Sewell -Dao Hoang Son -Oskar Jauch -Matthew Rayfield -t0mm0 -Tithen-Firion -Zack Fernandes -cryptonaut -Adrian Kretz -Mathias Rav -Petr Kutalek -Will Glynn -Max Reimann -Cédric Luthi -Thijs Vermeir -Joel Leclerc -Christopher Krooss -Ondřej Caletka -Dinesh S -Johan K. Jensen -Yen Chi Hsuan -Enam Mijbah Noor -David Luhmer -Shaya Goldberg -Paul Hartmann -Frans de Jonge -Robin de Rooij -Ryan Schmidt -Leslie P. Polzer -Duncan Keall -Alexander Mamay -Devin J. Pohly -Eduardo Ferro Aldama -Jeff Buchbinder -Amish Bhadeshia -Joram Schrijver -Will W. -Mohammad Teimori Pabandi -Roman Le Négrate -Matthias Küch -Julian Richen -Ping O. -Mister Hat -Peter Ding -jackyzy823 -George Brighton -Remita Amine -Aurélio A. Heckert -Bernhard Minks -sceext -Zach Bruggeman -Tjark Saul -slangangular -Behrouz Abbasi -ngld -nyuszika7h -Shaun Walbridge -Lee Jenkins -Anssi Hannula -Lukáš Lalinský -Qijiang Fan -Rémy Léone -Marco Ferragina -reiv -Muratcan Simsek -Evan Lu -flatgreen -Brian Foley -Vignesh Venkat -Tom Gijselinck -Founder Fang -Andrew Alexeyew -Saso Bezlaj -Erwin de Haan -Jens Wille -Robin Houtevelts -Patrick Griffis -Aidan Rowe -mutantmonkey -Ben Congdon -Kacper Michajłow -José Joaquín Atria -Viťas Strádal -Kagami Hiiragi -Philip Huppert -blahgeek -Kevin Deldycke -inondle -Tomáš Čech -Déstin Reed -Roman Tsiupa -Artur Krysiak -Jakub Adam Wieczorek -Aleksandar Topuzović -Nehal Patel -Rob van Bekkum -Petr Zvoníček -Pratyush Singh -Aleksander Nitecki -Sebastian Blunt -Matěj Cepl -Xie Yanbo -Philip Xu -John Hawkinson -Rich Leeper -Zhong Jianxin -Thor77 -Mattias Wadman -Arjan Verwer -Costy Petrisor -Logan B -Alex Seiler -Vijay Singh -Paul Hartmann -Stephen Chen -Fabian Stahl -Bagira -Odd Stråbø -Philip Herzog -Thomas Christlieb -Marek Rusinowski -Tobias Gruetzmacher -Olivier Bilodeau -Lars Vierbergen -Juanjo Benages -Xiao Di Guan -Thomas Winant -Daniel Twardowski -Jeremie Jarosh -Gerard Rovira -Marvin Ewald -Frédéric Bournival -Timendum -gritstub -Adam Voss -Mike Fährmann -Jan Kundrát -Giuseppe Fabiano -Örn Guðjónsson -Parmjit Virk -Genki Sky -Ľuboš Katrinec -Corey Nicholson -Ashutosh Chaudhary -John Dong -Tatsuyuki Ishi -Daniel Weber -Kay Bouché -Yang Hongbo -Lei Wang -Petr Novák -Leonardo Taccari -Martin Weinelt -Surya Oktafendri -TingPing -Alexandre Macabies -Bastian de Groot -Niklas Haas -András Veres-Szentkirályi -Enes Solak -Nathan Rossi -Thomas van der Berg -Luca Cherubin diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 58ab3a4b8..5faf97b10 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ $ youtube-dl -v [debug] System config: [] [debug] User config: [] -[debug] Command-line args: [u'-v', u'https://www.youtube.com/watch?v=BaW_jenozKcj'] +[debug] Command-line args: [u'-v', u'https://www.youtube.com/watch?v=BaW_jenozKc'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 [debug] youtube-dl version 2015.12.06 [debug] Git HEAD: 135392e @@ -81,16 +81,17 @@ To run the test, simply invoke your favorite test runner, or execute a test file python -m unittest discover python test/test_download.py nosetests + pytest See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases. If you want to create a build of youtube-dl yourself, you'll need -* python +* python3 * make (only GNU make is supported) * pandoc * zip -* nosetests +* pytest ### Adding support for a new site diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 000000000..f0d00068c --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,65 @@ +pukkandan (owner) +shirt-dev (collaborator) +colethedj (collaborator) +Ashish0804 (collaborator) +h-h-h-h +pauldubois98 +nixxo +GreyAlien502 +kyuyeunk +siikamiika +jbruchon +alexmerkel +glenn-slayden +Unrud +wporr +mariuszskon +ohnonot +samiksome +alxnull +FelixFrog +Zocker1999NET +nao20010128nao +kurumigi +bbepis +animelover1984 +Pccode66 +RobinD42 +hseg +DennyDai +codeasashu +teesid +kevinoconnor7 +damianoamatruda +2ShedsJackson +CXwudi +xtkoba +llacb47 +hheimbuerger +B0pol +lkho +fstirlitz +Lamieur +tsukumijima +Hadi0609 +b5eff52 +craftingmod +tpikonen +tripulse +king-millez +alex-gedeon +hhirtz +louie-github +MinePlayersPE +olifre +rhsmachine/zenerdi0de +nihil-admirari +krichbanana +ohmybahgosh +nyuszika7h +blackjack4494 +pyx +TpmKranz +mzbaulhaque +zackmark29 +mbway diff --git a/ChangeLog b/ChangeLog deleted file mode 100644 index 9b52b7bd2..000000000 --- a/ChangeLog +++ /dev/null @@ -1,5294 +0,0 @@ -version 2020.09.20 - -Core -* [extractor/common] Relax interaction count extraction in _json_ld -+ [extractor/common] Extract author as uploader for VideoObject in _json_ld -* [downloader/hls] Fix incorrect end byte in Range HTTP header for - media segments with EXT-X-BYTERANGE (#14748, #24512) -* [extractor/common] Handle ssl.CertificateError in _request_webpage (#26601) -* [downloader/http] Improve timeout detection when reading block of data - (#10935) -* [downloader/http] Retry download when urlopen times out (#10935, #26603) - -Extractors -* [redtube] Extend URL regular expression (#26506) -* [twitch] Refactor -* [twitch:stream] Switch to GraphQL and fix reruns (#26535) -+ [telequebec] Add support for brightcove videos (#25833) -* [pornhub] Extract metadata from JSON-LD (#26614) -* [pornhub] Fix view count extraction (#26621, #26614) - - -version 2020.09.14 - -Core -+ [postprocessor/embedthumbnail] Add support for non jpg/png thumbnails - (#25687, #25717) - -Extractors -* [rtlnl] Extend URL regular expression (#26549, #25821) -* [youtube] Fix empty description extraction (#26575, #26006) -* [srgssr] Extend URL regular expression (#26555, #26556, #26578) -* [googledrive] Use redirect URLs for source format (#18877, #23919, #24689, - #26565) -* [svtplay] Fix id extraction (#26576) -* [redbulltv] Improve support for rebull.com TV localized URLs (#22063) -+ [redbulltv] Add support for new redbull.com TV URLs (#22037, #22063) -* [soundcloud:pagedplaylist] Reduce pagination limit (#26557) - - -version 2020.09.06 - -Core -+ [utils] Recognize wav mimetype (#26463) - -Extractors -* [nrktv:episode] Improve video id extraction (#25594, #26369, #26409) -* [youtube] Fix age gate content detection (#26100, #26152, #26311, #26384) -* [youtube:user] Extend URL regular expression (#26443) -* [xhamster] Improve initials regular expression (#26526, #26353) -* [svtplay] Fix video id extraction (#26425, #26428, #26438) -* [twitch] Rework extractors (#12297, #20414, #20604, #21811, #21812, #22979, - #24263, #25010, #25553, #25606) - * Switch to GraphQL - + Add support for collections - + Add support for clips and collections playlists -* [biqle] Improve video ext extraction -* [xhamster] Fix extraction (#26157, #26254) -* [xhamster] Extend URL regular expression (#25789, #25804, #25927)) - - -version 2020.07.28 - -Extractors -* [youtube] Fix sigfunc name extraction (#26134, #26135, #26136, #26137) -* [youtube] Improve description extraction (#25937, #25980) -* [wistia] Restrict embed regular expression (#25969) -* [youtube] Prevent excess HTTP 301 (#25786) -+ [youtube:playlists] Extend URL regular expression (#25810) -+ [bellmedia] Add support for cp24.com clip URLs (#25764) -* [brightcove] Improve embed detection (#25674) - - -version 2020.06.16.1 - -Extractors -* [youtube] Force old layout (#25682, #25683, #25680, #25686) -* [youtube] Fix categories and improve tags extraction - - -version 2020.06.16 - -Extractors -* [youtube] Fix uploader id and uploader URL extraction -* [youtube] Improve view count extraction -* [youtube] Fix upload date extraction (#25677) -* [youtube] Fix thumbnails extraction (#25676) -* [youtube] Fix playlist and feed extraction (#25675) -+ [facebook] Add support for single-video ID links -+ [youtube] Extract chapters from JSON (#24819) -+ [kaltura] Add support for multiple embeds on a webpage (#25523) - - -version 2020.06.06 - -Extractors -* [tele5] Bypass geo restriction -+ [jwplatform] Add support for bypass geo restriction -* [tele5] Prefer jwplatform over nexx (#25533) -* [twitch:stream] Expect 400 and 410 HTTP errors from API -* [twitch:stream] Fix extraction (#25528) -* [twitch] Fix thumbnails extraction (#25531) -+ [twitch] Pass v5 Accept HTTP header (#25531) -* [brightcove] Fix subtitles extraction (#25540) -+ [malltv] Add support for sk.mall.tv (#25445) -* [periscope] Fix untitled broadcasts (#25482) -* [jwplatform] Improve embeds extraction (#25467) - - -version 2020.05.29 - -Core -* [postprocessor/ffmpeg] Embed series metadata with --add-metadata -* [utils] Fix file permissions in write_json_file (#12471, #25122) - -Extractors -* [ard:beta] Extend URL regular expression (#25405) -+ [youtube] Add support for more invidious instances (#25417) -* [giantbomb] Extend URL regular expression (#25222) -* [ard] Improve URL regular expression (#25134, #25198) -* [redtube] Improve formats extraction and extract m3u8 formats (#25311, - #25321) -* [indavideo] Switch to HTTPS for API request (#25191) -* [redtube] Improve title extraction (#25208) -* [vimeo] Improve format extraction and sorting (#25285) -* [soundcloud] Reduce API playlist page limit (#25274) -+ [youtube] Add support for yewtu.be (#25226) -* [mailru] Fix extraction (#24530, #25239) -* [bellator] Fix mgid extraction (#25195) - - -version 2020.05.08 - -Core -* [downloader/http] Request last data block of exact remaining size -* [downloader/http] Finish downloading once received data length matches - expected -* [extractor/common] Use compat_cookiejar_Cookie for _set_cookie to always - ensure cookie name and value are bytestrings on python 2 (#23256, #24776) -+ [compat] Introduce compat_cookiejar_Cookie -* [utils] Improve cookie files support - + Add support for UTF-8 in cookie files - * Skip malformed cookie file entries instead of crashing (invalid entry - length, invalid expires at) - -Extractors -* [youtube] Improve signature cipher extraction (#25187, #25188) -* [iprima] Improve extraction (#25138) -* [uol] Fix extraction (#22007) -+ [orf] Add support for more radio stations (#24938, #24968) -* [dailymotion] Fix typo -- [puhutv] Remove no longer available HTTP formats (#25124) - - -version 2020.05.03 - -Core -+ [extractor/common] Extract multiple JSON-LD entries -* [options] Clarify doc on --exec command (#19087, #24883) -* [extractor/common] Skip malformed ISM manifest XMLs while extracting - ISM formats (#24667) - -Extractors -* [crunchyroll] Fix and improve extraction (#25096, #25060) -* [youtube] Improve player id extraction -* [youtube] Use redirected video id if any (#25063) -* [yahoo] Fix GYAO Player extraction and relax URL regular expression - (#24178, #24778) -* [tvplay] Fix Viafree extraction (#15189, #24473, #24789) -* [tenplay] Relax URL regular expression (#25001) -+ [prosiebensat1] Extract series metadata -* [prosiebensat1] Improve extraction and remove 7tv.de support (#24948) -- [prosiebensat1] Remove 7tv.de support (#24948) -* [youtube] Fix DRM videos detection (#24736) -* [thisoldhouse] Fix video id extraction (#24548, #24549) -+ [soundcloud] Extract AAC format (#19173, #24708) -* [youtube] Skip broken multifeed videos (#24711) -* [nova:embed] Fix extraction (#24700) -* [motherless] Fix extraction (#24699) -* [twitch:clips] Extend URL regular expression (#24290, #24642) -* [tv4] Fix ISM formats extraction (#24667) -* [tele5] Fix extraction (#24553) -+ [mofosex] Add support for generic embeds (#24633) -+ [youporn] Add support for generic embeds -+ [spankwire] Add support for generic embeds (#24633) -* [spankwire] Fix extraction (#18924, #20648) - - -version 2020.03.24 - -Core -- [utils] Revert support for cookie files with spaces used instead of tabs - -Extractors -* [teachable] Update upskillcourses and gns3 domains -* [generic] Look for teachable embeds before wistia -+ [teachable] Extract chapter metadata (#24421) -+ [bilibili] Add support for player.bilibili.com (#24402) -+ [bilibili] Add support for new URL schema with BV ids (#24439, #24442) -* [limelight] Remove disabled API requests (#24255) -* [soundcloud] Fix download URL extraction (#24394) -+ [cbc:watch] Add support for authentication (#19160) -* [hellporno] Fix extraction (#24399) -* [xtube] Fix formats extraction (#24348) -* [ndr] Fix extraction (#24326) -* [nhk] Update m3u8 URL and use native HLS downloader (#24329) -- [nhk] Remove obsolete rtmp formats (#24329) -* [nhk] Relax URL regular expression (#24329) -- [vimeo] Revert fix showcase password protected video extraction (#24224) - - -version 2020.03.08 - -Core -+ [utils] Add support for cookie files with spaces used instead of tabs - -Extractors -+ [pornhub] Add support for pornhubpremium.com (#24288) -- [youtube] Remove outdated code and unnecessary requests -* [youtube] Improve extraction in 429 HTTP error conditions (#24283) -* [nhk] Update API version (#24270) - - -version 2020.03.06 - -Extractors -* [youtube] Fix age-gated videos support without login (#24248) -* [vimeo] Fix showcase password protected video extraction (#24224) -* [pornhub] Improve title extraction (#24184) -* [peertube] Improve extraction (#23657) -+ [servus] Add support for new URL schema (#23475, #23583, #24142) -* [vimeo] Fix subtitles URLs (#24209) - - -version 2020.03.01 - -Core -* [YoutubeDL] Force redirect URL to unicode on python 2 -- [options] Remove duplicate short option -v for --version (#24162) - -Extractors -* [xhamster] Fix extraction (#24205) -* [franceculture] Fix extraction (#24204) -+ [telecinco] Add support for article opening videos -* [telecinco] Fix extraction (#24195) -* [xtube] Fix metadata extraction (#21073, #22455) -* [youjizz] Fix extraction (#24181) -- Remove no longer needed compat_str around geturl -* [pornhd] Fix extraction (#24128) -+ [teachable] Add support for multiple videos per lecture (#24101) -+ [wistia] Add support for multiple generic embeds (#8347, 11385) -* [imdb] Fix extraction (#23443) -* [tv2dk:bornholm:play] Fix extraction (#24076) - - -version 2020.02.16 - -Core -* [YoutubeDL] Fix playlist entry indexing with --playlist-items (#10591, - #10622) -* [update] Fix updating via symlinks (#23991) -+ [compat] Introduce compat_realpath (#23991) - -Extractors -+ [npr] Add support for streams (#24042) -+ [24video] Add support for porn.24video.net (#23779, #23784) -- [jpopsuki] Remove extractor (#23858) -* [nova] Improve extraction (#23690) -* [nova:embed] Improve (#23690) -* [nova:embed] Fix extraction (#23672) -+ [abc:iview] Add support for 720p (#22907, #22921) -* [nytimes] Improve format sorting (#24010) -+ [toggle] Add support for mewatch.sg (#23895, #23930) -* [thisoldhouse] Fix extraction (#23951) -+ [popcorntimes] Add support for popcorntimes.tv (#23949) -* [sportdeutschland] Update to new API -* [twitch:stream] Lowercase channel id for stream request (#23917) -* [tv5mondeplus] Fix extraction (#23907, #23911) -* [tva] Relax URL regular expression (#23903) -* [vimeo] Fix album extraction (#23864) -* [viewlift] Improve extraction - * Fix extraction (#23851) - + Add support for authentication - + Add support for more domains -* [svt] Fix series extraction (#22297) -* [svt] Fix article extraction (#22897, #22919) -* [soundcloud] Imporve private playlist/set tracks extraction (#3707) - - -version 2020.01.24 - -Extractors -* [youtube] Fix sigfunc name extraction (#23819) -* [stretchinternet] Fix extraction (#4319) -* [voicerepublic] Fix extraction -* [azmedien] Fix extraction (#23783) -* [businessinsider] Fix jwplatform id extraction (#22929, #22954) -+ [24video] Add support for 24video.vip (#23753) -* [ivi:compilation] Fix entries extraction (#23770) -* [ard] Improve extraction (#23761) - * Simplify extraction - + Extract age limit and series - * Bypass geo-restriction -+ [nbc] Add support for nbc multi network URLs (#23049) -* [americastestkitchen] Fix extraction -* [zype] Improve extraction - + Extract subtitles (#21258) - + Support URLs with alternative keys/tokens (#21258) - + Extract more metadata -* [orf:tvthek] Improve geo restricted videos detection (#23741) -* [soundcloud] Restore previews extraction (#23739) - - -version 2020.01.15 - -Extractors -* [yourporn] Fix extraction (#21645, #22255, #23459) -+ [canvas] Add support for new API endpoint (#17680, #18629) -* [ndr:base:embed] Improve thumbnails extraction (#23731) -+ [vodplatform] Add support for embed.kwikmotion.com domain -+ [twitter] Add support for promo_video_website cards (#23711) -* [orf:radio] Clean description and improve extraction -* [orf:fm4] Fix extraction (#23599) -* [safari] Fix kaltura session extraction (#23679, #23670) -* [lego] Fix extraction and extract subtitle (#23687) -* [cloudflarestream] Improve extraction - + Add support for bytehighway.net domain - + Add support for signed URLs - + Extract thumbnail -* [naver] Improve extraction - * Improve geo-restriction handling - + Extract automatic captions - + Extract uploader metadata - + Extract VLive HLS formats - * Improve metadata extraction -- [pandatv] Remove extractor (#23630) -* [dctp] Fix format extraction (#23656) -+ [scrippsnetworks] Add support for www.discovery.com videos -* [discovery] Fix anonymous token extraction (#23650) -* [nrktv:seriebase] Fix extraction (#23625, #23537) -* [wistia] Improve format extraction and extract subtitles (#22590) -* [vice] Improve extraction (#23631) -* [redtube] Detect private videos (#23518) - - -version 2020.01.01 - -Extractors -* [brightcove] Invalidate policy key cache on failing requests -* [pornhub] Improve locked videos detection (#22449, #22780) -+ [pornhub] Add support for m3u8 formats -* [pornhub] Fix extraction (#22749, #23082) -* [brightcove] Update policy key on failing requests -* [spankbang] Improve removed video detection (#23423) -* [spankbang] Fix extraction (#23307, #23423, #23444) -* [soundcloud] Automatically update client id on failing requests -* [prosiebensat1] Improve geo restriction handling (#23571) -* [brightcove] Cache brightcove player policy keys -* [teachable] Fail with error message if no video URL found -* [teachable] Improve locked lessons detection (#23528) -+ [scrippsnetworks] Add support for Scripps Networks sites (#19857, #22981) -* [mitele] Fix extraction (#21354, #23456) -* [soundcloud] Update client id (#23516) -* [mailru] Relax URL regular expressions (#23509) - - -version 2019.12.25 - -Core -* [utils] Improve str_to_int -+ [downloader/hls] Add ability to override AES decryption key URL (#17521) - -Extractors -* [mediaset] Fix parse formats (#23508) -+ [tv2dk:bornholm:play] Add support for play.tv2bornholm.dk (#23291) -+ [slideslive] Add support for url and vimeo service names (#23414) -* [slideslive] Fix extraction (#23413) -* [twitch:clips] Fix extraction (#23375) -+ [soundcloud] Add support for token protected embeds (#18954) -* [vk] Improve extraction - * Fix User Videos extraction (#23356) - * Extract all videos for lists with more than 1000 videos (#23356) - + Add support for video albums (#14327, #14492) -- [kontrtube] Remove extractor -- [videopremium] Remove extractor -- [musicplayon] Remove extractor (#9225) -+ [ufctv] Add support for ufcfightpass.imgdge.com and - ufcfightpass.imggaming.com (#23343) -+ [twitch] Extract m3u8 formats frame rate (#23333) -+ [imggaming] Add support for playlists and extract subtitles -+ [ufcarabia] Add support for UFC Arabia (#23312) -* [ufctv] Fix extraction -* [yahoo] Fix gyao brightcove player id (#23303) -* [vzaar] Override AES decryption key URL (#17521) -+ [vzaar] Add support for AES HLS manifests (#17521, #23299) -* [nrl] Fix extraction -* [teachingchannel] Fix extraction -* [nintendo] Fix extraction and partially add support for Nintendo Direct - videos (#4592) -+ [ooyala] Add better fallback values for domain and streams variables -+ [youtube] Add support youtubekids.com (#23272) -* [tv2] Detect DRM protection -+ [tv2] Add support for katsomo.fi and mtv.fi (#10543) -* [tv2] Fix tv2.no article extraction -* [msn] Improve extraction - + Add support for YouTube and NBCSports embeds - + Add support for articles with multiple videos - * Improve AOL embed support - * Improve format extraction -* [abcotvs] Relax URL regular expression and improve metadata extraction - (#18014) -* [channel9] Reduce response size -* [adobetv] Improve extaction - * Use OnDemandPagedList for list extractors - * Reduce show extraction requests - * Extract original video format and subtitles - + Add support for adobe tv embeds - - -version 2019.11.28 - -Core -+ [utils] Add generic caesar cipher and rot47 -* [utils] Handle rd-suffixed day parts in unified_strdate (#23199) - -Extractors -* [vimeo] Improve extraction - * Fix review extraction - * Fix ondemand extraction - * Make password protected player case as an expected error (#22896) - * Simplify channel based extractors code -- [openload] Remove extractor (#11999) -- [verystream] Remove extractor -- [streamango] Remove extractor (#15406) -* [dailymotion] Improve extraction - * Extract http formats included in m3u8 manifest - * Fix user extraction (#3553, #21415) - + Add suport for User Authentication (#11491) - * Fix password protected videos extraction (#23176) - * Respect age limit option and family filter cookie value (#18437) - * Handle video url playlist query param - * Report allowed countries for geo-restricted videos -* [corus] Improve extraction - + Add support for Series Plus, W Network, YTV, ABC Spark, disneychannel.com - and disneylachaine.ca (#20861) - + Add support for self hosted videos (#22075) - * Detect DRM protection (#14910, #9164) -* [vivo] Fix extraction (#22328, #22279) -+ [bitchute] Extract upload date (#22990, #23193) -* [soundcloud] Update client id (#23214) - - -version 2019.11.22 - -Core -+ [extractor/common] Clean jwplayer description HTML tags -+ [extractor/common] Add data, headers and query to all major extract formats - methods - -Extractors -* [chaturbate] Fix extraction (#23010, #23012) -+ [ntvru] Add support for non relative file URLs (#23140) -* [vk] Fix wall audio thumbnails extraction (#23135) -* [ivi] Fix format extraction (#21991) -- [comcarcoff] Remove extractor -+ [drtv] Add support for new URL schema (#23059) -+ [nexx] Add support for Multi Player JS Setup (#23052) -+ [teamcoco] Add support for new videos (#23054) -* [soundcloud] Check if the soundtrack has downloads left (#23045) -* [facebook] Fix posts video data extraction (#22473) -- [addanime] Remove extractor -- [minhateca] Remove extractor -- [daisuki] Remove extractor -* [seeker] Fix extraction -- [revision3] Remove extractors -* [twitch] Fix video comments URL (#18593, #15828) -* [twitter] Improve extraction - + Add support for generic embeds (#22168) - * Always extract http formats for native videos (#14934) - + Add support for Twitter Broadcasts (#21369) - + Extract more metadata - * Improve VMap format extraction - * Unify extraction code for both twitter statuses and cards -+ [twitch] Add support for Clip embed URLs -* [lnkgo] Fix extraction (#16834) -* [mixcloud] Improve extraction - * Improve metadata extraction (#11721) - * Fix playlist extraction (#22378) - * Fix user mixes extraction (#15197, #17865) -+ [kinja] Add support for Kinja embeds (#5756, #11282, #22237, #22384) -* [onionstudios] Fix extraction -+ [hotstar] Pass Referer header to format requests (#22836) -* [dplay] Minimize response size -+ [patreon] Extract uploader_id and filesize -* [patreon] Minimize response size -* [roosterteeth] Fix login request (#16094, #22689) - - -version 2019.11.05 - -Extractors -+ [scte] Add support for learning.scte.org (#22975) -+ [msn] Add support for Vidible and AOL embeds (#22195, #22227) -* [myspass] Fix video URL extraction and improve metadata extraction (#22448) -* [jamendo] Improve extraction - * Fix album extraction (#18564) - * Improve metadata extraction (#18565, #21379) -* [mediaset] Relax URL guid matching (#18352) -+ [mediaset] Extract unprotected M3U and MPD manifests (#17204) -* [telegraaf] Fix extraction -+ [bellmedia] Add support for marilyn.ca videos (#22193) -* [stv] Fix extraction (#22928) -- [iconosquare] Remove extractor -- [keek] Remove extractor -- [gameone] Remove extractor (#21778) -- [flipagram] Remove extractor -- [bambuser] Remove extractor -* [wistia] Reduce embed extraction false positives -+ [wistia] Add support for inline embeds (#22931) -- [go90] Remove extractor -* [kakao] Remove raw request -+ [kakao] Extract format total bitrate -* [daum] Fix VOD and Clip extracton (#15015) -* [kakao] Improve extraction - + Add support for embed URLs - + Add support for Kakao Legacy vid based embed URLs - * Only extract fields used for extraction - * Strip description and extract tags -* [mixcloud] Fix cloudcast data extraction (#22821) -* [yahoo] Improve extraction - + Add support for live streams (#3597, #3779, #22178) - * Bypass cookie consent page for european domains (#16948, #22576) - + Add generic support for embeds (#20332) -* [tv2] Fix and improve extraction (#22787) -+ [tv2dk] Add support for TV2 DK sites -* [onet] Improve extraction … - + Add support for onet100.vod.pl - + Extract m3u8 formats - * Correct audio only format info -* [fox9] Fix extraction - - -version 2019.10.29 - -Core -* [utils] Actualize major IPv4 address blocks per country - -Extractors -+ [go] Add support for abc.com and freeform.com (#22823, #22864) -+ [mtv] Add support for mtvjapan.com -* [mtv] Fix extraction for mtv.de (#22113) -* [videodetective] Fix extraction -* [internetvideoarchive] Fix extraction -* [nbcnews] Fix extraction (#12569, #12576, #21703, #21923) -- [hark] Remove extractor -- [tutv] Remove extractor -- [learnr] Remove extractor -- [macgamestore] Remove extractor -* [la7] Update Kaltura service URL (#22358) -* [thesun] Fix extraction (#16966) -- [makertv] Remove extractor -+ [tenplay] Add support for 10play.com.au (#21446) -* [soundcloud] Improve extraction - * Improve format extraction (#22123) - + Extract uploader_id and uploader_url (#21916) - + Extract all known thumbnails (#19071, #20659) - * Fix extration for private playlists (#20976) - + Add support for playlist embeds (#20976) - * Skip preview formats (#22806) -* [dplay] Improve extraction - + Add support for dplay.fi, dplay.jp and es.dplay.com (#16969) - * Fix it.dplay.com extraction (#22826) - + Extract creator, tags and thumbnails - * Handle playback API call errors -+ [discoverynetworks] Add support for dplay.co.uk -* [vk] Improve extraction - + Add support for Odnoklassniki embeds - + Extract more videos from user lists (#4470) - + Fix wall post audio extraction (#18332) - * Improve error detection (#22568) -+ [odnoklassniki] Add support for embeds -* [puhutv] Improve extraction - * Fix subtitles extraction - * Transform HLS URLs to HTTP URLs - * Improve metadata extraction -* [ceskatelevize] Skip DRM media -+ [facebook] Extract subtitles (#22777) -* [globo] Handle alternative hash signing method - - -version 2019.10.22 - -Core -* [utils] Improve subtitles_filename (#22753) - -Extractors -* [facebook] Bypass download rate limits (#21018) -+ [contv] Add support for contv.com -- [viewster] Remove extractor -* [xfileshare] Improve extractor (#17032, #17906, #18237, #18239) - * Update the list of domains - + Add support for aa-encoded video data - * Improve jwplayer format extraction - + Add support for Clappr sources -* [mangomolo] Fix video format extraction and add support for player URLs -* [audioboom] Improve metadata extraction -* [twitch] Update VOD URL matching (#22395, #22727) -- [mit] Remove support for video.mit.edu (#22403) -- [servingsys] Remove extractor (#22639) -* [dumpert] Fix extraction (#22428, #22564) -* [atresplayer] Fix extraction (#16277, #16716) - - -version 2019.10.16 - -Core -* [extractor/common] Make _is_valid_url more relaxed - -Extractors -* [vimeo] Improve album videos id extraction (#22599) -+ [globo] Extract subtitles (#22713) -* [bokecc] Improve player params extraction (#22638) -* [nexx] Handle result list (#22666) -* [vimeo] Fix VHX embed extraction -* [nbc] Switch to graphql API (#18581, #22693, #22701) -- [vessel] Remove extractor -- [promptfile] Remove extractor (#6239) -* [kaltura] Fix service URL extraction (#22658) -* [kaltura] Fix embed info strip (#22658) -* [globo] Fix format extraction (#20319) -* [redtube] Improve metadata extraction (#22492, #22615) -* [pornhub:uservideos:upload] Fix extraction (#22619) -+ [telequebec:squat] Add support for squat.telequebec.tv (#18503) -- [wimp] Remove extractor (#22088, #22091) -+ [gfycat] Extend URL regular expression (#22225) -+ [chaturbate] Extend URL regular expression (#22309) -* [peertube] Update instances (#22414) -+ [telequebec] Add support for coucou.telequebec.tv (#22482) -+ [xvideos] Extend URL regular expression (#22471) -- [youtube] Remove support for invidious.enkirton.net (#22543) -+ [openload] Add support for oload.monster (#22592) -* [nrktv:seriebase] Fix extraction (#22596) -+ [youtube] Add support for yt.lelux.fi (#22597) -* [orf:tvthek] Make manifest requests non fatal (#22578) -* [teachable] Skip login when already logged in (#22572) -* [viewlift] Improve extraction (#22545) -* [nonktube] Fix extraction (#22544) - - -version 2019.09.28 - -Core -* [YoutubeDL] Honour all --get-* options with --flat-playlist (#22493) - -Extractors -* [vk] Fix extraction (#22522) -* [heise] Fix kaltura embeds extraction (#22514) -* [ted] Check for resources validity and extract subtitled downloads (#22513) -+ [youtube] Add support for - owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya.b32.i2p (#22292) -+ [nhk] Add support for clips -* [nhk] Fix video extraction (#22249, #22353) -* [byutv] Fix extraction (#22070) -+ [openload] Add support for oload.online (#22304) -+ [youtube] Add support for invidious.drycat.fr (#22451) -* [jwplatfom] Do not match video URLs (#20596, #22148) -* [youtube:playlist] Unescape playlist uploader (#22483) -+ [bilibili] Add support audio albums and songs (#21094) -+ [instagram] Add support for tv URLs -+ [mixcloud] Allow uppercase letters in format URLs (#19280) -* [brightcove] Delegate all supported legacy URLs to new extractor (#11523, - #12842, #13912, #15669, #16303) -* [hotstar] Use native HLS downloader by default -+ [hotstar] Extract more formats (#22323) -* [9now] Fix extraction (#22361) -* [zdf] Bypass geo restriction -+ [tv4] Extract series metadata -* [tv4] Fix extraction (#22443) - - -version 2019.09.12.1 - -Extractors -* [youtube] Remove quality and tbr for itag 43 (#22372) - - -version 2019.09.12 - -Extractors -* [youtube] Quick extraction tempfix (#22367, #22163) - - -version 2019.09.01 - -Core -+ [extractor/generic] Add support for squarespace embeds (#21294, #21802, - #21859) -+ [downloader/external] Respect mtime option for aria2c (#22242) - -Extractors -+ [xhamster:user] Add support for user pages (#16330, #18454) -+ [xhamster] Add support for more domains -+ [verystream] Add support for woof.tube (#22217) -+ [dailymotion] Add support for lequipe.fr (#21328, #22152) -+ [openload] Add support for oload.vip (#22205) -+ [bbccouk] Extend URL regular expression (#19200) -+ [youtube] Add support for invidious.nixnet.xyz and yt.elukerio.org (#22223) -* [safari] Fix authentication (#22161, #22184) -* [usanetwork] Fix extraction (#22105) -+ [einthusan] Add support for einthusan.ca (#22171) -* [youtube] Improve unavailable message extraction (#22117) -+ [piksel] Extract subtitles (#20506) - - -version 2019.08.13 - -Core -* [downloader/fragment] Fix ETA calculation of resumed download (#21992) -* [YoutubeDL] Check annotations availability (#18582) - -Extractors -* [youtube:playlist] Improve flat extraction (#21927) -* [youtube] Fix annotations extraction (#22045) -+ [discovery] Extract series meta field (#21808) -* [youtube] Improve error detection (#16445) -* [vimeo] Fix album extraction (#1933, #15704, #15855, #18967, #21986) -+ [roosterteeth] Add support for watch URLs -* [discovery] Limit video data by show slug (#21980) - - -version 2019.08.02 - -Extractors -+ [tvigle] Add support for HLS and DASH formats (#21967) -* [tvigle] Fix extraction (#21967) -+ [yandexvideo] Add support for DASH formats (#21971) -* [discovery] Use API call for video data extraction (#21808) -+ [mgtv] Extract format_note (#21881) -* [tvn24] Fix metadata extraction (#21833, #21834) -* [dlive] Relax URL regular expression (#21909) -+ [openload] Add support for oload.best (#21913) -* [youtube] Improve metadata extraction for age gate content (#21943) - - -version 2019.07.30 - -Extractors -* [youtube] Fix and improve title and description extraction (#21934) - - -version 2019.07.27 - -Extractors -+ [yahoo:japannews] Add support for yahoo.co.jp (#21698, #21265) -+ [discovery] Add support go.discovery.com URLs -* [youtube:playlist] Relax video regular expression (#21844) -* [generic] Restrict --default-search schemeless URLs detection pattern - (#21842) -* [vrv] Fix CMS signing query extraction (#21809) - - -version 2019.07.16 - -Extractors -+ [asiancrush] Add support for yuyutv.com, midnightpulp.com and cocoro.tv - (#21281, #21290) -* [kaltura] Check source format URL (#21290) -* [ctsnews] Fix YouTube embeds extraction (#21678) -+ [einthusan] Add support for einthusan.com (#21748, #21775) -+ [youtube] Add support for invidious.mastodon.host (#21777) -+ [gfycat] Extend URL regular expression (#21779, #21780) -* [youtube] Restrict is_live extraction (#21782) - - -version 2019.07.14 - -Extractors -* [porn91] Fix extraction (#21312) -+ [yandexmusic] Extract track number and disk number (#21421) -+ [yandexmusic] Add support for multi disk albums (#21420, #21421) -* [lynda] Handle missing subtitles (#20490, #20513) -+ [youtube] Add more invidious instances to URL regular expression (#21694) -* [twitter] Improve uploader id extraction (#21705) -* [spankbang] Fix and improve metadata extraction -* [spankbang] Fix extraction (#21763, #21764) -+ [dlive] Add support for dlive.tv (#18080) -+ [livejournal] Add support for livejournal.com (#21526) -* [roosterteeth] Fix free episode extraction (#16094) -* [dbtv] Fix extraction -* [bellator] Fix extraction -- [rudo] Remove extractor (#18430, #18474) -* [facebook] Fallback to twitter:image meta for thumbnail extraction (#21224) -* [bleacherreport] Fix Bleacher Report CMS extraction -* [espn] Fix fivethirtyeight.com extraction -* [5tv] Relax video URL regular expression and support https URLs -* [youtube] Fix is_live extraction (#21734) -* [youtube] Fix authentication (#11270) - - -version 2019.07.12 - -Core -+ [adobepass] Add support for AT&T U-verse (mso ATT) (#13938, #21016) - -Extractors -+ [mgtv] Pass Referer HTTP header for format URLs (#21726) -+ [beeg] Add support for api/v6 v2 URLs without t argument (#21701) -* [voxmedia:volume] Improvevox embed extraction (#16846) -* [funnyordie] Move extraction to VoxMedia extractor (#16846) -* [gameinformer] Fix extraction (#8895, #15363, #17206) -* [funk] Fix extraction (#17915) -* [packtpub] Relax lesson URL regular expression (#21695) -* [packtpub] Fix extraction (#21268) -* [philharmoniedeparis] Relax URL regular expression (#21672) -* [peertube] Detect embed URLs in generic extraction (#21666) -* [mixer:vod] Relax URL regular expression (#21657, #21658) -+ [lecturio] Add support id based URLs (#21630) -+ [go] Add site info for disneynow (#21613) -* [ted] Restrict info regular expression (#21631) -* [twitch:vod] Actualize m3u8 URL (#21538, #21607) -* [vzaar] Fix videos with empty title (#21606) -* [tvland] Fix extraction (#21384) -* [arte] Clean extractor (#15583, #21614) - - -version 2019.07.02 - -Core -+ [utils] Introduce random_user_agent and use as default User-Agent (#21546) - -Extractors -+ [vevo] Add support for embed.vevo.com URLs (#21565) -+ [openload] Add support for oload.biz (#21574) -* [xiami] Update API base URL (#21575) -* [yourporn] Fix extraction (#21585) -+ [acast] Add support for URLs with episode id (#21444) -+ [dailymotion] Add support for DM.player embeds -* [soundcloud] Update client id - - -version 2019.06.27 - -Extractors -+ [go] Add support for disneynow.com (#21528) -* [mixer:vod] Relax URL regular expression (#21531, #21536) -* [drtv] Relax URL regular expression -* [fusion] Fix extraction (#17775, #21269) -- [nfb] Remove extractor (#21518) -+ [beeg] Add support for api/v6 v2 URLs (#21511) -+ [brightcove:new] Add support for playlists (#21331) -+ [openload] Add support for oload.life (#21495) -* [vimeo:channel,group] Make title extraction non fatal -* [vimeo:likes] Implement extrator in terms of channel extractor (#21493) -+ [pornhub] Add support for more paged video sources -+ [pornhub] Add support for downloading single pages and search pages (#15570) -* [pornhub] Rework extractors (#11922, #16078, #17454, #17936) -+ [youtube] Add another signature function pattern -* [tf1] Fix extraction (#21365, #21372) -* [crunchyroll] Move Accept-Language workaround to video extractor since - it causes playlists not to list any videos -* [crunchyroll:playlist] Fix and relax title extraction (#21291, #21443) - - -version 2019.06.21 - -Core -* [utils] Restrict parse_codecs and add theora as known vcodec (#21381) - -Extractors -* [youtube] Update signature function patterns (#21469, #21476) -* [youtube] Make --write-annotations non fatal (#21452) -+ [sixplay] Add support for rtlmost.hu (#21405) -* [youtube] Hardcode codec metadata for av01 video only formats (#21381) -* [toutv] Update client key (#21370) -+ [biqle] Add support for new embed domain -* [cbs] Improve DRM protected videos detection (#21339) - - -version 2019.06.08 - -Core -* [downloader/common] Improve rate limit (#21301) -* [utils] Improve strip_or_none -* [extractor/common] Strip src attribute for HTML5 entries code (#18485, - #21169) - -Extractors -* [ted] Fix playlist extraction (#20844, #21032) -* [vlive:playlist] Fix video extraction when no playlist is found (#20590) -+ [vlive] Add CH+ support (#16887, #21209) -+ [openload] Add support for oload.website (#21329) -+ [tvnow] Extract HD formats (#21201) -+ [redbulltv] Add support for rrn:content URLs (#21297) -* [youtube] Fix average rating extraction (#21304) -+ [bitchute] Extract HTML5 formats (#21306) -* [cbsnews] Fix extraction (#9659, #15397) -* [vvvvid] Relax URL regular expression (#21299) -+ [prosiebensat1] Add support for new API (#21272) -+ [vrv] Extract adaptive_hls formats (#21243) -* [viki] Switch to HTTPS (#21001) -* [LiveLeak] Check if the original videos exist (#21206, #21208) -* [rtp] Fix extraction (#15099) -* [youtube] Improve DRM protected videos detection (#1774) -+ [srgssrplay] Add support for popupvideoplayer URLs (#21155) -+ [24video] Add support for porno.24video.net (#21194) -+ [24video] Add support for 24video.site (#21193) -- [pornflip] Remove extractor -- [criterion] Remove extractor (#21195) -* [pornhub] Use HTTPS (#21061) -* [bitchute] Fix uploader extraction (#21076) -* [streamcloud] Reduce waiting time to 6 seconds (#21092) -- [novamov] Remove extractors (#21077) -+ [openload] Add support for oload.press (#21135) -* [vivo] Fix extraction (#18906, #19217) - - -version 2019.05.20 - -Core -+ [extractor/common] Move workaround for applying first Set-Cookie header - into a separate _apply_first_set_cookie_header method - -Extractors -* [safari] Fix authentication (#21090) -* [vk] Use _apply_first_set_cookie_header -* [vrt] Fix extraction (#20527) -+ [canvas] Add support for vrtnieuws and sporza site ids and extract - AES HLS formats -+ [vrv] Extract captions (#19238) -* [tele5] Improve video id extraction -* [tele5] Relax URL regular expression (#21020, #21063) -* [svtplay] Update API URL (#21075) -+ [yahoo:gyao] Add X-User-Agent header to dam proxy requests (#21071) - - -version 2019.05.11 - -Core -* [utils] Transliterate "þ" as "th" (#20897) - -Extractors -+ [cloudflarestream] Add support for videodelivery.net (#21049) -+ [byutv] Add support for DVR videos (#20574, #20676) -+ [gfycat] Add support for URLs with tags (#20696, #20731) -+ [openload] Add support for verystream.com (#20701, #20967) -* [youtube] Use sp field value for signature field name (#18841, #18927, - #21028) -+ [yahoo:gyao] Extend URL regular expression (#21008) -* [youtube] Fix channel id extraction (#20982, #21003) -+ [sky] Add support for news.sky.com (#13055) -+ [youtube:entrylistbase] Retry on 5xx HTTP errors (#20965) -+ [francetvinfo] Extend video id extraction (#20619, #20740) -* [4tube] Update token hosts (#20918) -* [hotstar] Move to API v2 (#20931) -* [fox] Fix API error handling under python 2 (#20925) -+ [redbulltv] Extend URL regular expression (#20922) - - -version 2019.04.30 - -Extractors -* [openload] Use real Chrome versions (#20902) -- [youtube] Remove info el for get_video_info request -* [youtube] Improve extraction robustness -- [dramafever] Remove extractor (#20868) -* [adn] Fix subtitle extraction (#12724) -+ [ccc] Extract creator (#20355) -+ [ccc:playlist] Add support for media.ccc.de playlists (#14601, #20355) -+ [sverigesradio] Add support for sverigesradio.se (#18635) -+ [cinemax] Add support for cinemax.com -* [sixplay] Try extracting non-DRM protected manifests (#20849) -+ [youtube] Extract Youtube Music Auto-generated metadata (#20599, #20742) -- [wrzuta] Remove extractor (#20684, #20801) -* [twitch] Prefer source format (#20850) -+ [twitcasting] Add support for private videos (#20843) -* [reddit] Validate thumbnail URL (#20030) -* [yandexmusic] Fix track URL extraction (#20820) - - -version 2019.04.24 - -Extractors -* [youtube] Fix extraction (#20758, #20759, #20761, #20762, #20764, #20766, - #20767, #20769, #20771, #20768, #20770) -* [toutv] Fix extraction and extract series info (#20757) -+ [vrv] Add support for movie listings (#19229) -+ [youtube] Print error when no data is available (#20737) -+ [soundcloud] Add support for new rendition and improve extraction (#20699) -+ [ooyala] Add support for geo verification proxy -+ [nrl] Add support for nrl.com (#15991) -+ [vimeo] Extract live archive source format (#19144) -+ [vimeo] Add support for live streams and improve info extraction (#19144) -+ [ntvcojp] Add support for cu.ntv.co.jp -+ [nhk] Extract RTMPT format -+ [nhk] Add support for audio URLs -+ [udemy] Add another course id extraction pattern (#20491) -+ [openload] Add support for oload.services (#20691) -+ [openload] Add support for openloed.co (#20691, #20693) -* [bravotv] Fix extraction (#19213) - - -version 2019.04.17 - -Extractors -* [openload] Randomize User-Agent (#20688) -+ [openload] Add support for oladblock domains (#20471) -* [adn] Fix subtitle extraction (#12724) -+ [aol] Add support for localized websites -+ [yahoo] Add support GYAO episode URLs -+ [yahoo] Add support for streaming.yahoo.co.jp (#5811, #7098) -+ [yahoo] Add support for gyao.yahoo.co.jp -* [aenetworks] Fix history topic extraction and extract more formats -+ [cbs] Extract smpte and vtt subtitles -+ [streamango] Add support for streamcherry.com (#20592) -+ [yourporn] Add support for sxyprn.com (#20646) -* [mgtv] Fix extraction (#20650) -* [linkedin:learning] Use urljoin for form action URL (#20431) -+ [gdc] Add support for kaltura embeds (#20575) -* [dispeak] Improve mp4 bitrate extraction -* [kaltura] Sanitize embed URLs -* [jwplatfom] Do not match manifest URLs (#20596) -* [aol] Restrict URL regular expression and improve format extraction -+ [tiktok] Add support for new URL schema (#20573) -+ [stv:player] Add support for player.stv.tv (#20586) - - -version 2019.04.07 - -Core -+ [downloader/external] Pass rtmp_conn to ffmpeg - -Extractors -+ [ruutu] Add support for audio podcasts (#20473, #20545) -+ [xvideos] Extract all thumbnails (#20432) -+ [platzi] Add support for platzi.com (#20562) -* [dvtv] Fix extraction (#18514, #19174) -+ [vrv] Add basic support for individual movie links (#19229) -+ [bfi:player] Add support for player.bfi.org.uk (#19235) -* [hbo] Fix extraction and extract subtitles (#14629, #13709) -* [youtube] Extract srv[1-3] subtitle formats (#20566) -* [adultswim] Fix extraction (#18025) -* [teamcoco] Fix extraction and add suport for subdomains (#17099, #20339) -* [adn] Fix subtitle compatibility with ffmpeg -* [adn] Fix extraction and add support for positioning styles (#20549) -* [vk] Use unique video id (#17848) -* [newstube] Fix extraction -* [rtl2] Actualize extraction -+ [adobeconnect] Add support for adobeconnect.com (#20283) -+ [gaia] Add support for authentication (#14605) -+ [mediasite] Add support for dashed ids and named catalogs (#20531) - - -version 2019.04.01 - -Core -* [utils] Improve int_or_none and float_or_none (#20403) -* Check for valid --min-sleep-interval when --max-sleep-interval is specified - (#20435) - -Extractors -+ [weibo] Extend URL regular expression (#20496) -+ [xhamster] Add support for xhamster.one (#20508) -+ [mediasite] Add support for catalogs (#20507) -+ [teamtreehouse] Add support for teamtreehouse.com (#9836) -+ [ina] Add support for audio URLs -* [ina] Improve extraction -* [cwtv] Fix episode number extraction (#20461) -* [npo] Improve DRM detection -+ [pornhub] Add support for DASH formats (#20403) -* [svtplay] Update API endpoint (#20430) - - -version 2019.03.18 - -Core -* [extractor/common] Improve HTML5 entries extraction -+ [utils] Introduce parse_bitrate -* [update] Hide update URLs behind redirect -* [extractor/common] Fix url meta field for unfragmented DASH formats (#20346) - -Extractors -+ [yandexvideo] Add extractor -* [openload] Improve embed detection -+ [corus] Add support for bigbrothercanada.ca (#20357) -+ [orf:radio] Extract series (#20012) -+ [cbc:watch] Add support for gem.cbc.ca (#20251, #20359) -- [anysex] Remove extractor (#19279) -+ [ciscolive] Add support for new URL schema (#20320, #20351) -+ [youtube] Add support for invidiou.sh (#20309) -- [anitube] Remove extractor (#20334) -- [ruleporn] Remove extractor (#15344, #20324) -* [npr] Fix extraction (#10793, #13440) -* [biqle] Fix extraction (#11471, #15313) -* [viddler] Modernize -* [moevideo] Fix extraction -* [primesharetv] Remove extractor -* [hypem] Modernize and extract more metadata (#15320) -* [veoh] Fix extraction -* [escapist] Modernize -- [videomega] Remove extractor (#10108) -+ [beeg] Add support for beeg.porn (#20306) -* [vimeo:review] Improve config url extraction and extract original format - (#20305) -* [fox] Detect geo restriction and authentication errors (#20208) - - -version 2019.03.09 - -Core -* [extractor/common] Use compat_etree_Element -+ [compat] Introduce compat_etree_Element -* [extractor/common] Fallback url to base URL for DASH formats -* [extractor/common] Do not fail on invalid data while parsing F4M manifest - in non fatal mode -* [extractor/common] Return MPD manifest as format's url meta field (#20242) -* [utils] Strip #HttpOnly_ prefix from cookies files (#20219) - -Extractors -* [francetv:site] Relax video id regular expression (#20268) -* [toutv] Detect invalid login error -* [toutv] Fix authentication (#20261) -+ [urplay] Extract timestamp (#20235) -+ [openload] Add support for oload.space (#20246) -* [facebook] Improve uploader extraction (#20250) -* [bbc] Use compat_etree_Element -* [crunchyroll] Use compat_etree_Element -* [npo] Improve ISM extraction -* [rai] Improve extraction (#20253) -* [paramountnetwork] Fix mgid extraction (#20241) -* [libsyn] Improve extraction (#20229) -+ [youtube] Add more invidious instances to URL regular expression (#20228) -* [spankbang] Fix extraction (#20023) -* [espn] Extend URL regular expression (#20013) -* [sixplay] Handle videos with empty assets (#20016) -+ [vimeo] Add support for Vimeo Pro portfolio protected videos (#20070) - - -version 2019.03.01 - -Core -+ [downloader/external] Add support for rate limit and retries for wget -* [downloader/external] Fix infinite retries for curl (#19303) - -Extractors -* [npo] Fix extraction (#20084) -* [francetv:site] Extend video id regex (#20029, #20071) -+ [periscope] Extract width and height (#20015) -* [servus] Fix extraction (#19297) -* [bbccouk] Make subtitles non fatal (#19651) -* [metacafe] Fix family filter bypass (#19287) - - -version 2019.02.18 - -Extractors -* [tvp:website] Fix and improve extraction -+ [tvp] Detect unavailable videos -* [tvp] Fix description extraction and make thumbnail optional -+ [linuxacademy] Add support for linuxacademy.com (#12207) -* [bilibili] Update keys (#19233) -* [udemy] Extend URL regular expressions (#14330, #15883) -* [udemy] Update User-Agent and detect captcha (#14713, #15839, #18126) -* [noovo] Fix extraction (#19230) -* [rai] Relax URL regular expression (#19232) -+ [vshare] Pass Referer to download request (#19205, #19221) -+ [openload] Add support for oload.live (#19222) -* [imgur] Use video id as title fallback (#18590) -+ [twitch] Add new source format detection approach (#19193) -* [tvplayhome] Fix video id extraction (#19190) -* [tvplayhome] Fix episode metadata extraction (#19190) -* [rutube:embed] Fix extraction (#19163) -+ [rutube:embed] Add support private videos (#19163) -+ [soundcloud] Extract more metadata -+ [trunews] Add support for trunews.com (#19153) -+ [linkedin:learning] Extract chapter_number and chapter_id (#19162) - - -version 2019.02.08 - -Core -* [utils] Improve JSON-LD regular expression (#18058) -* [YoutubeDL] Fallback to ie_key of matching extractor while making - download archive id when no explicit ie_key is provided (#19022) - -Extractors -+ [malltv] Add support for mall.tv (#18058, #17856) -+ [spankbang:playlist] Add support for playlists (#19145) -* [spankbang] Extend URL regular expression -* [trutv] Fix extraction (#17336) -* [toutv] Fix authentication (#16398, #18700) -* [pornhub] Fix tags and categories extraction (#13720, #19135) -* [pornhd] Fix formats extraction -+ [pornhd] Extract like count (#19123, #19125) -* [radiocanada] Switch to the new media requests (#19115) -+ [teachable] Add support for courses.workitdaily.com (#18871) -- [vporn] Remove extractor (#16276) -+ [soundcloud:pagedplaylist] Add ie and title to entries (#19022, #19086) -+ [drtuber] Extract duration (#19078) -* [soundcloud] Fix paged playlists extraction, add support for albums and update client id -* [soundcloud] Update client id -* [drtv] Improve preference (#19079) -+ [openload] Add support for openload.pw and oload.pw (#18930) -+ [openload] Add support for oload.info (#19073) -* [crackle] Authorize media detail request (#16931) - - -version 2019.01.30.1 - -Core -* [postprocessor/ffmpeg] Fix avconv processing broken in #19025 (#19067) - - -version 2019.01.30 - -Core -* [postprocessor/ffmpeg] Do not copy Apple TV chapter tracks while embedding - subtitles (#19024, #19042) -* [postprocessor/ffmpeg] Disable "Last message repeated" messages (#19025) - -Extractors -* [yourporn] Fix extraction and extract duration (#18815, #18852, #19061) -* [drtv] Improve extraction (#19039) - + Add support for EncryptedUri videos - + Extract more metadata - * Fix subtitles extraction -+ [fox] Add support for locked videos using cookies (#19060) -* [fox] Fix extraction for free videos (#19060) -+ [zattoo] Add support for tv.salt.ch (#19059) - - -version 2019.01.27 - -Core -+ [extractor/common] Extract season in _json_ld -* [postprocessor/ffmpeg] Fallback to ffmpeg/avconv for audio codec detection - (#681) - -Extractors -* [vice] Fix extraction for locked videos (#16248) -+ [wakanim] Detect DRM protected videos -+ [wakanim] Add support for wakanim.tv (#14374) -* [usatoday] Fix extraction for videos with custom brightcove partner id - (#18990) -* [drtv] Fix extraction (#18989) -* [nhk] Extend URL regular expression (#18968) -* [go] Fix Adobe Pass requests for Disney Now (#18901) -+ [openload] Add support for oload.club (#18969) - - -version 2019.01.24 - -Core -* [YoutubeDL] Fix negation for string operators in format selection (#18961) - - -version 2019.01.23 - -Core -* [utils] Fix urljoin for paths with non-http(s) schemes -* [extractor/common] Improve jwplayer relative URL handling (#18892) -+ [YoutubeDL] Add negation support for string comparisons in format selection - expressions (#18600, #18805) -* [extractor/common] Improve HLS video-only format detection (#18923) - -Extractors -* [crunchyroll] Extend URL regular expression (#18955) -* [pornhub] Bypass scrape detection (#4822, #5930, #7074, #10175, #12722, - #17197, #18338 #18842, #18899) -+ [vrv] Add support for authentication (#14307) -* [videomore:season] Fix extraction -* [videomore] Improve extraction (#18908) -+ [tnaflix] Pass Referer in metadata request (#18925) -* [radiocanada] Relax DRM check (#18608, #18609) -* [vimeo] Fix video password verification for videos protected by - Referer HTTP header -+ [hketv] Add support for hkedcity.net (#18696) -+ [streamango] Add support for fruithosts.net (#18710) -+ [instagram] Add support for tags (#18757) -+ [odnoklassniki] Detect paid videos (#18876) -* [ted] Correct acodec for HTTP formats (#18923) -* [cartoonnetwork] Fix extraction (#15664, #17224) -* [vimeo] Fix extraction for password protected player URLs (#18889) - - -version 2019.01.17 - -Extractors -* [youtube] Extend JS player signature function name regular expressions - (#18890, #18891, #18893) - - -version 2019.01.16 - -Core -+ [test/helper] Add support for maxcount and count collection len checkers -* [downloader/hls] Fix uplynk ad skipping (#18824) -* [postprocessor/ffmpeg] Improve ffmpeg version parsing (#18813) - -Extractors -* [youtube] Skip unsupported adaptive stream type (#18804) -+ [youtube] Extract DASH formats from player response (#18804) -* [funimation] Fix extraction (#14089) -* [skylinewebcams] Fix extraction (#18853) -+ [curiositystream] Add support for non app URLs -+ [bitchute] Check formats (#18833) -* [wistia] Extend URL regular expression (#18823) -+ [playplustv] Add support for playplus.com (#18789) - - -version 2019.01.10 - -Core -* [extractor/common] Use episode name as title in _json_ld -+ [extractor/common] Add support for movies in _json_ld -* [postprocessor/ffmpeg] Embed subtitles with non-standard language codes - (#18765) -+ [utils] Add language codes replaced in 1989 revision of ISO 639 - to ISO639Utils (#18765) - -Extractors -* [youtube] Extract live HLS URL from player response (#18799) -+ [outsidetv] Add support for outsidetv.com (#18774) -* [jwplatform] Use JW Platform Delivery API V2 and add support for more URLs -+ [fox] Add support National Geographic (#17985, #15333, #14698) -+ [playplustv] Add support for playplus.tv (#18789) -* [globo] Set GLBID cookie manually (#17346) -+ [gaia] Add support for gaia.com (#14605) -* [youporn] Fix title and description extraction (#18748) -+ [hungama] Add support for hungama.com (#17402, #18771) -* [dtube] Fix extraction (#18741) -* [tvnow] Fix and rework extractors and prepare for a switch to the new API - (#17245, #18499) -* [carambatv:page] Fix extraction (#18739) - - -version 2019.01.02 - -Extractors -* [discovery] Use geo verification headers (#17838) -+ [packtpub] Add support for subscription.packtpub.com (#18718) -* [yourporn] Fix extraction (#18583) -+ [acast:channel] Add support for play.acast.com (#18587) -+ [extractors] Add missing age limits (#18621) -+ [rmcdecouverte] Add support for live stream -* [rmcdecouverte] Bypass geo restriction -* [rmcdecouverte] Update URL regular expression (#18595, 18697) -* [manyvids] Fix extraction (#18604, #18614) -* [bitchute] Fix extraction (#18567) - - -version 2018.12.31 - -Extractors -+ [bbc] Add support for another embed pattern (#18643) -+ [npo:live] Add support for npostart.nl (#18644) -* [beeg] Fix extraction (#18610, #18626) -* [youtube] Unescape HTML for series (#18641) -+ [youtube] Extract more format metadata -* [youtube] Detect DRM protected videos (#1774) -* [youtube] Relax HTML5 player regular expressions (#18465, #18466) -* [youtube] Extend HTML5 player regular expression (#17516) -+ [liveleak] Add support for another embed type and restore original - format extraction -+ [crackle] Extract ISM and HTTP formats -+ [twitter] Pass Referer with card request (#18579) -* [mediasite] Extend URL regular expression (#18558) -+ [lecturio] Add support for lecturio.de (#18562) -+ [discovery] Add support for Scripps Networks watch domains (#17947) - - -version 2018.12.17 - -Extractors -* [ard:beta] Improve geo restricted videos extraction -* [ard:beta] Fix subtitles extraction -* [ard:beta] Improve extraction robustness -* [ard:beta] Relax URL regular expression (#18441) -* [acast] Add support for embed.acast.com and play.acast.com (#18483) -* [iprima] Relax URL regular expression (#18515, #18540) -* [vrv] Fix initial state extraction (#18553) -* [youtube] Fix mark watched (#18546) -+ [safari] Add support for learning.oreilly.com (#18510) -* [youtube] Fix multifeed extraction (#18531) -* [lecturio] Improve subtitles extraction (#18488) -* [uol] Fix format URL extraction (#18480) -+ [ard:mediathek] Add support for classic.ardmediathek.de (#18473) - - -version 2018.12.09 - -Core -* [YoutubeDL] Keep session cookies in cookie file between runs -* [YoutubeDL] Recognize session cookies with expired set to 0 (#12929) - -Extractors -+ [teachable] Add support for teachable platform sites (#5451, #18150, #18272) -+ [aenetworks] Add support for historyvault.com (#18460) -* [imgur] Improve gallery and album detection and extraction (#9133, #16577, - #17223, #18404) -* [iprima] Relax URL regular expression (#18453) -* [hotstar] Fix video data extraction (#18386) -* [ard:mediathek] Fix title and description extraction (#18349, #18371) -* [xvideos] Switch to HTTPS (#18422, #18427) -+ [lecturio] Add support for lecturio.com (#18405) -+ [nrktv:series] Add support for extra materials -* [nrktv:season,series] Fix extraction (#17159, #17258) -* [nrktv] Relax URL regular expression (#18304, #18387) -* [yourporn] Fix extraction (#18424, #18425) -* [tbs] Fix info extraction (#18403) -+ [gamespot] Add support for review URLs - - -version 2018.12.03 - -Core -* [utils] Fix random_birthday to generate existing dates only (#18284) - -Extractors -+ [tiktok] Add support for tiktok.com (#18108, #18135) -* [pornhub] Use actual URL host for requests (#18359) -* [lynda] Fix authentication (#18158, #18217) -* [gfycat] Update API endpoint (#18333, #18343) -+ [hotstar] Add support for alternative app state layout (#18320) -* [azmedien] Fix extraction (#18334, #18336) -+ [vimeo] Add support for VHX (Vimeo OTT) (#14835) -* [joj] Fix extraction (#18280, #18281) -+ [wistia] Add support for fast.wistia.com (#18287) - - -version 2018.11.23 - -Core -+ [setup.py] Add more relevant classifiers - -Extractors -* [mixcloud] Fallback to hardcoded decryption key (#18016) -* [nbc:news] Fix article extraction (#16194) -* [foxsports] Fix extraction (#17543) -* [loc] Relax regular expression and improve formats extraction -+ [ciscolive] Add support for ciscolive.cisco.com (#17984) -* [nzz] Relax kaltura regex (#18228) -* [sixplay] Fix formats extraction -* [bitchute] Improve title extraction -* [kaltura] Limit requested MediaEntry fields -+ [americastestkitchen] Add support for zype embeds (#18225) -+ [pornhub] Add pornhub.net alias -* [nova:embed] Fix extraction (#18222) - - -version 2018.11.18 - -Extractors -+ [wwe] Extract subtitles -+ [wwe] Add support for playlistst (#14781) -+ [wwe] Add support for wwe.com (#14781, #17450) -* [vk] Detect geo restriction (#17767) -* [openload] Use original host during extraction (#18211) -* [atvat] Fix extraction (#18041) -+ [rte] Add support for new API endpoint (#18206) -* [tnaflixnetwork:embed] Fix extraction (#18205) -* [picarto] Use API and add token support (#16518) -+ [zype] Add support for player.zype.com (#18143) -* [vivo] Fix extraction (#18139) -* [ruutu] Update API endpoint (#18138) - - -version 2018.11.07 - -Extractors -+ [youtube] Add another JS signature function name regex (#18091, #18093, - #18094) -* [facebook] Fix tahoe request (#17171) -* [cliphunter] Fix extraction (#18083) -+ [youtube:playlist] Add support for invidio.us (#18077) -* [zattoo] Arrange API hosts for derived extractors (#18035) -+ [youtube] Add fallback metadata extraction from videoDetails (#18052) - - -version 2018.11.03 - -Core -* [extractor/common] Ensure response handle is not prematurely closed before - it can be read if it matches expected_status (#17195, #17846, #17447) - -Extractors -* [laola1tv:embed] Set correct stream access URL scheme (#16341) -+ [ehftv] Add support for ehftv.com (#15408) -* [azmedien] Adopt to major site redesign (#17745, #17746) -+ [twitcasting] Add support for twitcasting.tv (#17981) -* [orf:tvthek] Fix extraction (#17737, #17956, #18024) -+ [openload] Add support for oload.fun (#18045) -* [njpwworld] Fix authentication (#17427) -+ [linkedin:learning] Add support for linkedin.com/learning (#13545) -* [theplatform] Improve error detection (#13222) -* [cnbc] Simplify extraction (#14280, #17110) -+ [cbnc] Add support for new URL schema (#14193) -* [aparat] Improve extraction and extract more metadata (#17445, #18008) -* [aparat] Fix extraction - - -version 2018.10.29 - -Core -+ [extractor/common] Add validation for JSON-LD URLs - -Extractors -+ [sportbox] Add support for matchtv.ru -* [sportbox] Fix extraction (#17978) -* [screencast] Fix extraction (#14590, #14617, #17990) -+ [openload] Add support for oload.icu -+ [ivi] Add support for ivi.tv -* [crunchyroll] Improve extraction failsafeness (#17991) -* [dailymail] Fix formats extraction (#17976) -* [viewster] Reduce format requests -* [cwtv] Handle API errors (#17905) -+ [rutube] Use geo verification headers (#17897) -+ [brightcove:legacy] Add fallbacks to brightcove:new (#13912) -- [tv3] Remove extractor (#10461, #15339) -* [ted] Fix extraction for HTTP and RTMP formats (#5941, #17572, #17894) -+ [openload] Add support for oload.cc (#17823) -+ [patreon] Extract post_file URL (#17792) -* [patreon] Fix extraction (#14502, #10471) - - -version 2018.10.05 - -Extractors -* [pluralsight] Improve authentication (#17762) -* [dailymotion] Fix extraction (#17699) -* [crunchyroll] Switch to HTTPS for RpcApi (#17749) -+ [philharmoniedeparis] Add support for pad.philharmoniedeparis.fr (#17705) -* [philharmoniedeparis] Fix extraction (#17705) -+ [jamendo] Add support for licensing.jamendo.com (#17724) -+ [openload] Add support for oload.cloud (#17710) -* [pluralsight] Fix subtitles extraction (#17726, #17728) -+ [vimeo] Add another config regular expression (#17690) -* [spike] Fix Paramount Network extraction (#17677) -* [hotstar] Fix extraction (#14694, #14931, #17637) - - -version 2018.09.26 - -Extractors -* [pluralsight] Fix subtitles extraction (#17671) -* [mediaset] Improve embed support (#17668) -+ [youtube] Add support for invidio.us (#17613) -+ [zattoo] Add support for more zattoo platform sites -* [zattoo] Fix extraction (#17175, #17542) - - -version 2018.09.18 - -Core -+ [extractor/common] Introduce channel meta fields - -Extractors -* [adobepass] Don't pollute default headers dict -* [udemy] Don't pollute default headers dict -* [twitch] Don't pollute default headers dict -* [youtube] Don't pollute default query dict (#17593) -* [crunchyroll] Prefer hardsubless formats and formats in locale language -* [vrv] Make format ids deterministic -* [vimeo] Fix ondemand playlist extraction (#14591) -+ [pornhub] Extract upload date (#17574) -+ [porntube] Extract channel meta fields -+ [vimeo] Extract channel meta fields -+ [youtube] Extract channel meta fields (#9676, #12939) -* [porntube] Fix extraction (#17541) -* [asiancrush] Fix extraction (#15630) -+ [twitch:clips] Extend URL regular expression (#17559) -+ [vzaar] Add support for HLS -* [tube8] Fix metadata extraction (#17520) -* [eporner] Extract JSON-LD (#17519) - - -version 2018.09.10 - -Core -+ [utils] Properly recognize AV1 codec (#17506) - -Extractors -+ [iprima] Add support for prima.iprima.cz (#17514) -+ [tele5] Add support for tele5.de (#7805, #7922, #17331, #17414) -* [nbc] Fix extraction of percent encoded URLs (#17374) - - -version 2018.09.08 - -Extractors -* [youtube] Fix extraction (#17457, #17464) -+ [pornhub:uservideos] Add support for new URLs (#17388) -* [iprima] Confirm adult check (#17437) -* [slideslive] Make check for video service name case-insensitive (#17429) -* [radiojavan] Fix extraction (#17151) -* [generic] Skip unsuccessful jwplayer extraction (#16735) - - -version 2018.09.01 - -Core -* [utils] Skip remote IP addresses non matching to source address' IP version - when creating a connection (#13422, #17362) - -Extractors -+ [ard] Add support for one.ard.de (#17397) -* [niconico] Fix extraction on python3 (#17393, #17407) -* [ard] Extract f4m formats -* [crunchyroll] Parse vilos media data (#17343) -+ [ard] Add support for Beta ARD Mediathek -+ [bandcamp] Extract more metadata (#13197) -* [internazionale] Fix extraction of non-available-abroad videos (#17386) - - -version 2018.08.28 - -Extractors -+ [youtube:playlist] Add support for music album playlists (OLAK5uy_ prefix) - (#17361) -* [bitchute] Fix extraction by pass custom User-Agent (#17360) -* [webofstories:playlist] Fix extraction (#16914) -+ [tvplayhome] Add support for new tvplay URLs (#17344) -+ [generic] Allow relative src for videojs embeds (#17324) -+ [xfileshare] Add support for vidto.se (#17317) -+ [vidzi] Add support for vidzi.nu (#17316) -+ [nova:embed] Add support for media.cms.nova.cz (#17282) - - -version 2018.08.22 - -Core -* [utils] Use pure browser header for User-Agent (#17236) - -Extractors -+ [kinopoisk] Add support for kinopoisk.ru (#17283) -+ [yourporn] Add support for yourporn.sexy (#17298) -+ [go] Add support for disneynow.go.com (#16299, #17264) -+ [6play] Add support for play.rtl.hr (#17249) -* [anvato] Fallback to generic API key for access-key-to-API-key lookup - (#16788, #17254) -* [lci] Fix extraction (#17274) -* [bbccouk] Extend id URL regular expression (#17270) -* [cwtv] Fix extraction (#17256) -* [nova] Fix extraction (#17241) -+ [generic] Add support for expressen embeds -* [raywenderlich] Adapt to site redesign (#17225) -+ [redbulltv] Add support redbull.com tv URLs (#17218) -+ [bitchute] Add support for bitchute.com (#14052) -+ [clyp] Add support for token protected media (#17184) -* [imdb] Fix extension extraction (#17167) - - -version 2018.08.04 - -Extractors -* [funk:channel] Improve byChannelAlias extraction (#17142) -* [twitch] Fix authentication (#17024, #17126) -* [twitch:vod] Improve URL regular expression (#17135) -* [watchbox] Fix extraction (#17107) -* [pbs] Fix extraction (#17109) -* [theplatform] Relax URL regular expression (#16181, #17097) -+ [viqeo] Add support for viqeo.tv (#17066) - - -version 2018.07.29 - -Extractors -* [crunchyroll:playlist] Restrict URL regular expression (#17069, #17076) -+ [pornhub] Add support for subtitles (#16924, #17088) -* [ceskatelevize] Use https for API call (#16997, #16999) -* [dailymotion:playlist] Fix extraction (#16894) -* [ted] Improve extraction -* [ted] Fix extraction for videos without nativeDownloads (#16756, #17085) -* [telecinco] Fix extraction (#17080) -* [mitele] Reduce number of requests -* [rai] Return non HTTP relinker URL intact (#17055) -* [vk] Fix extraction for inline only videos (#16923) -* [streamcloud] Fix extraction (#17054) -* [facebook] Fix tahoe player extraction with authentication (#16655) -+ [puhutv] Add support for puhutv.com (#12712, #16010, #16269) - - -version 2018.07.21 - -Core -+ [utils] Introduce url_or_none -* [utils] Allow JSONP without function name (#17028) -+ [extractor/common] Extract DASH and MSS formats from SMIL manifests - -Extractors -+ [bbc] Add support for BBC Radio Play pages (#17022) -* [iwara] Fix download URLs (#17026) -* [vrtnu] Relax title extraction and extract JSON-LD (#17018) -+ [viu] Pass Referer and Origin headers and area id (#16992) -+ [vimeo] Add another config regular expression (#17013) -+ [facebook] Extract view count (#16942) -* [dailymotion] Improve description extraction (#16984) -* [slutload] Fix and improve extraction (#17001) -* [mediaset] Fix extraction (#16977) -+ [theplatform] Add support for theplatform TLD customization (#16977) -* [imgur] Relax URL regular expression (#16987) -* [pornhub] Improve extraction and extract all formats (#12166, #15891, #16262, - #16959) - - -version 2018.07.10 - -Core -* [utils] Share JSON-LD regular expression -* [downloader/dash] Improve error handling (#16927) - -Extractors -+ [nrktv] Add support for new season and serie URL schema -+ [nrktv] Add support for new episode URL schema (#16909) -+ [frontendmasters] Add support for frontendmasters.com (#3661, #16328) -* [funk] Fix extraction (#16918) -* [watchbox] Fix extraction (#16904) -* [dplayit] Sort formats -* [dplayit] Fix extraction (#16901) -* [youtube] Improve login error handling (#13822) - - -version 2018.07.04 - -Core -* [extractor/common] Properly escape % in MPD templates (#16867) -* [extractor/common] Use source URL as Referer for HTML5 entries (16849) -* Prefer ffmpeg over avconv by default (#8622) - -Extractors -* [pluralsight] Switch to graphql (#16889, #16895, #16896, #16899) -* [lynda] Simplify login and improve error capturing (#16891) -+ [go90] Add support for embed URLs (#16873) -* [go90] Detect geo restriction error and pass geo verification headers - (#16874) -* [vlive] Fix live streams extraction (#16871) -* [npo] Fix typo (#16872) -+ [mediaset] Add support for new videos and extract all formats (#16568) -* [dctptv] Restore extraction based on REST API (#16850) -* [svt] Improve extraction and add support for pages (#16802) -* [porncom] Fix extraction (#16808) - - -version 2018.06.25 - -Extractors -* [joj] Relax URL regular expression (#16771) -* [brightcove] Workaround sonyliv DRM protected videos (#16807) -* [motherless] Fix extraction (#16786) -* [itv] Make SOAP request non fatal and extract metadata from webpage (#16780) -- [foxnews:insider] Remove extractor (#15810) -+ [foxnews] Add support for iframe embeds (#15810, #16711) - - -version 2018.06.19 - -Core -+ [extractor/common] Introduce expected_status in _download_* methods - for convenient accept of HTTP requests failed with non 2xx status codes -+ [compat] Introduce compat_integer_types - -Extractors -* [peertube] Improve generic support (#16733) -+ [6play] Use geo verification headers -* [rtbf] Fix extraction for python 3.2 -* [vgtv] Improve HLS formats extraction -+ [vgtv] Add support for www.aftonbladet.se/tv URLs -* [bbccouk] Use expected_status -* [markiza] Expect 500 HTTP status code -* [tvnow] Try all clear manifest URLs (#15361) - - -version 2018.06.18 - -Core -* [downloader/rtmp] Fix downloading in verbose mode (#16736) - -Extractors -+ [markiza] Add support for markiza.sk (#16750) -* [wat] Try all supported adaptive URLs -+ [6play] Add support for rtlplay.be and extract hd usp formats -+ [rtbf] Add support for audio and live streams (#9638, #11923) -+ [rtbf] Extract HLS, DASH and all HTTP formats -+ [rtbf] Extract subtitles -+ [rtbf] Fixup specific HTTP URLs (#16101) -+ [expressen] Add support for expressen.se -* [vidzi] Fix extraction (#16678) -* [pbs] Improve extraction (#16623, #16684) -* [bilibili] Restrict cid regular expression (#16638, #16734) - - -version 2018.06.14 - -Core -* [downloader/http] Fix retry on error when streaming to stdout (#16699) - -Extractors -+ [discoverynetworks] Add support for disco-api videos (#16724) -+ [dailymotion] Add support for password protected videos (#9789) -+ [abc:iview] Add support for livestreams (#12354) -* [abc:iview] Fix extraction (#16704) -+ [crackle] Add support for sonycrackle.com (#16698) -+ [tvnet] Add support for tvnet.gov.vn (#15462) -* [nrk] Update API hosts and try all previously known ones (#16690) -* [wimp] Fix Youtube embeds extraction - - -version 2018.06.11 - -Extractors -* [npo] Extend URL regular expression and add support for npostart.nl (#16682) -+ [inc] Add support for another embed schema (#16666) -* [tv4] Fix format extraction (#16650) -+ [nexx] Add support for free cdn (#16538) -+ [pbs] Add another cove id pattern (#15373) -+ [rbmaradio] Add support for 192k format (#16631) - - -version 2018.06.04 - -Extractors -+ [camtube] Add support for camtube.co -+ [twitter:card] Extract guest token (#16609) -+ [chaturbate] Use geo verification headers -+ [bbc] Add support for bbcthree (#16612) -* [youtube] Move metadata extraction after video availability check -+ [youtube] Extract track and artist -+ [safari] Add support for new URL schema (#16614) -* [adn] Fix extraction - - -version 2018.06.02 - -Core -* [utils] Improve determine_ext - -Extractors -+ [facebook] Add support for tahoe player videos (#15441, #16554) -* [cbc] Improve extraction (#16583, #16593) -* [openload] Improve ext extraction (#16595) -+ [twitter:card] Add support for another endpoint (#16586) -+ [openload] Add support for oload.win and oload.download (#16592) -* [audimedia] Fix extraction (#15309) -+ [francetv] Add support for sport.francetvinfo.fr (#15645) -* [mlb] Improve extraction (#16587) -- [nhl] Remove old extractors -* [rbmaradio] Check formats availability (#16585) - - -version 2018.05.30 - -Core -* [downloader/rtmp] Generalize download messages and report time elapsed - on finish -* [downloader/rtmp] Gracefully handle live streams interrupted by user - -Extractors -* [teamcoco] Fix extraction for full episodes (#16573) -* [spiegel] Fix info extraction (#16538) -+ [apa] Add support for apa.at (#15041, #15672) -+ [bellmedia] Add support for bnnbloomberg.ca (#16560) -+ [9c9media] Extract MPD formats and subtitles -* [cammodels] Use geo verification headers -+ [ufctv] Add support for authentication (#16542) -+ [cammodels] Add support for cammodels.com (#14499) -* [utils] Fix style id extraction for namespaced id attribute in dfxp2srt - (#16551) -* [soundcloud] Detect format extension (#16549) -* [cbc] Fix playlist title extraction (#16502) -+ [tumblr] Detect and report sensitive media (#13829) -+ [tumblr] Add support for authentication (#15133) - - -version 2018.05.26 - -Core -* [utils] Improve parse_age_limit - -Extractors -* [audiomack] Stringify video id (#15310) -* [izlesene] Fix extraction (#16233, #16271, #16407) -+ [indavideo] Add support for generic embeds (#11989) -* [indavideo] Fix extraction (#11221) -* [indavideo] Sign download URLs (#16174) -+ [peertube] Add support for PeerTube based sites (#16301, #16329) -* [imgur] Fix extraction (#16537) -+ [hidive] Add support for authentication (#16534) -+ [nbc] Add support for stream.nbcsports.com (#13911) -+ [viewlift] Add support for hoichoi.tv (#16536) -* [go90] Extract age limit and detect DRM protection(#10127) -* [viewlift] fix extraction for snagfilms.com (#15766) -* [globo] Improve extraction (#4189) - * Add support for authentication - * Simplify URL signing - * Extract DASH and MSS formats -* [leeco] Fix extraction (#16464) -* [teamcoco] Add fallback for format extraction (#16484) -* [teamcoco] Improve URL regular expression (#16484) -* [imdb] Improve extraction (#4085, #14557) - - -version 2018.05.18 - -Extractors -* [vimeo:likes] Relax URL regular expression and fix single page likes - extraction (#16475) -* [pluralsight] Fix clip id extraction (#16460) -+ [mychannels] Add support for mychannels.com (#15334) -- [moniker] Remove extractor (#15336) -* [pbs] Fix embed data extraction (#16474) -+ [mtv] Add support for paramountnetwork.com and bellator.com (#15418) -* [youtube] Fix hd720 format position -* [dailymotion] Remove fragment part from m3u8 URLs (#8915) -* [3sat] Improve extraction (#15350) - * Extract all formats - * Extract more format metadata - * Improve format sorting - * Use hls native downloader - * Detect and bypass geo-restriction -+ [dtube] Add support for d.tube (#15201) -* [options] Fix typo (#16450) -* [youtube] Improve format filesize extraction (#16453) -* [youtube] Make uploader extraction non fatal (#16444) -* [youtube] Fix extraction for embed restricted live streams (#16433) -* [nbc] Improve info extraction (#16440) -* [twitch:clips] Fix extraction (#16429) -* [redditr] Relax URL regular expression (#16426, #16427) -* [mixcloud] Bypass throttling for HTTP formats (#12579, #16424) -+ [nick] Add support for nickjr.de (#13230) -* [teamcoco] Fix extraction (#16374) - - -version 2018.05.09 - -Core -* [YoutubeDL] Ensure ext exists for automatic captions -* Introduce --geo-bypass-ip-block - -Extractors -+ [udemy] Extract asset captions -+ [udemy] Extract stream URLs (#16372) -+ [businessinsider] Add support for businessinsider.com (#16387, #16388, #16389) -+ [cloudflarestream] Add support for cloudflarestream.com (#16375) -* [watchbox] Fix extraction (#16356) -* [discovery] Extract Affiliate/Anonymous Auth Token from cookies (#14954) -+ [itv:btcc] Add support for itv.com/btcc (#16139) -* [tunein] Use live title for live streams (#16347) -* [itv] Improve extraction (#16253) - - -version 2018.05.01 - -Core -* [downloader/fragment] Restart download if .ytdl file is corrupt (#16312) -+ [extractor/common] Extract interaction statistic -+ [utils] Add merge_dicts -+ [extractor/common] Add _download_json_handle - -Extractors -* [kaltura] Improve iframe embeds detection (#16337) -+ [udemy] Extract outputs renditions (#16289, #16291, #16320, #16321, #16334, - #16335) -+ [zattoo] Add support for zattoo.com and mobiltv.quickline.com (#14668, #14676) -* [yandexmusic] Convert release_year to int -* [udemy] Override _download_webpage_handle instead of _download_webpage -* [xiami] Override _download_webpage_handle instead of _download_webpage -* [yandexmusic] Override _download_webpage_handle instead of _download_webpage -* [youtube] Correctly disable polymer on all requests (#16323, #16326) -* [generic] Prefer enclosures over links in RSS feeds (#16189) -+ [redditr] Add support for old.reddit.com URLs (#16274) -* [nrktv] Update API host (#16324) -+ [imdb] Extract all formats (#16249) -+ [vimeo] Extract JSON-LD (#16295) -* [funk:channel] Improve extraction (#16285) - - -version 2018.04.25 - -Core -* [utils] Fix match_str for boolean meta fields -+ [Makefile] Add support for pandoc 2 and disable smart extension (#16251) -* [YoutubeDL] Fix typo in media extension compatibility checker (#16215) - -Extractors -+ [openload] Recognize IPv6 stream URLs (#16136, #16137, #16205, #16246, - #16250) -+ [twitch] Extract is_live according to status (#16259) -* [pornflip] Relax URL regular expression (#16258) -- [etonline] Remove extractor (#16256) -* [breakcom] Fix extraction (#16254) -+ [youtube] Add ability to authenticate with cookies -* [youtube:feed] Implement lazy playlist extraction (#10184) -+ [svt] Add support for TV channel live streams (#15279, #15809) -* [ccma] Fix video extraction (#15931) -* [rentv] Fix extraction (#15227) -+ [nick] Add support for nickjr.nl (#16230) -* [extremetube] Fix metadata extraction -+ [keezmovies] Add support for generic embeds (#16134, #16154) -* [nexx] Extract new azure URLs (#16223) -* [cbssports] Fix extraction (#16217) -* [kaltura] Improve embeds detection (#16201) -* [instagram:user] Fix extraction (#16119) -* [cbs] Skip DRM asset types (#16104) - - -version 2018.04.16 - -Extractors -* [smotri:broadcast] Fix extraction (#16180) -+ [picarto] Add support for picarto.tv (#6205, #12514, #15276, #15551) -* [vine:user] Fix extraction (#15514, #16190) -* [pornhub] Relax URL regular expression (#16165) -* [cbc:watch] Re-acquire device token when expired (#16160) -+ [fxnetworks] Add support for https theplatform URLs (#16125, #16157) -+ [instagram:user] Add request signing (#16119) -+ [twitch] Add support for mobile URLs (#16146) - - -version 2018.04.09 - -Core -* [YoutubeDL] Do not save/restore console title while simulate (#16103) -* [extractor/common] Relax JSON-LD context check (#16006) - -Extractors -+ [generic] Add support for tube8 embeds -+ [generic] Add support for share-videos.se embeds (#16089, #16115) -* [odnoklassniki] Extend URL regular expression (#16081) -* [steam] Bypass mature content check (#16113) -+ [acast] Extract more metadata -* [acast] Fix extraction (#16118) -* [instagram:user] Fix extraction (#16119) -* [drtuber] Fix title extraction (#16107, #16108) -* [liveleak] Extend URL regular expression (#16117) -+ [openload] Add support for oload.xyz -* [openload] Relax stream URL regular expression -* [openload] Fix extraction (#16099) -+ [svtplay:series] Add support for season URLs -+ [svtplay:series] Add support for series (#11130, #16059) - - -version 2018.04.03 - -Extractors -+ [tvnow] Add support for shows (#15837) -* [dramafever] Fix authentication (#16067) -* [afreecatv] Use partial view only when necessary (#14450) -+ [afreecatv] Add support for authentication (#14450) -+ [nationalgeographic] Add support for new URL schema (#16001, #16054) -* [xvideos] Fix thumbnail extraction (#15978, #15979) -* [medialaan] Fix vod id (#16038) -+ [openload] Add support for oload.site (#16039) -* [naver] Fix extraction (#16029) -* [dramafever] Partially switch to API v5 (#16026) -* [abc:iview] Unescape title and series meta fields (#15994) -* [videa] Extend URL regular expression (#16003) - - -version 2018.03.26.1 - -Core -+ [downloader/external] Add elapsed time to progress hook (#10876) -* [downloader/external,fragment] Fix download finalization when writing file - to stdout (#10809, #10876, #15799) - -Extractors -* [vrv] Fix extraction on python2 (#15928) -* [afreecatv] Update referrer (#15947) -+ [24video] Add support for 24video.sexy (#15973) -* [crackle] Bypass geo restriction -* [crackle] Fix extraction (#15969) -+ [lenta] Add support for lenta.ru (#15953) -+ [instagram:user] Add pagination (#15934) -* [youku] Update ccode (#15939) -* [libsyn] Adapt to new page structure - - -version 2018.03.20 - -Core -* [extractor/common] Improve thumbnail extraction for HTML5 entries -* Generalize XML manifest processing code and improve XSPF parsing -+ [extractor/common] Add _download_xml_handle -+ [extractor/common] Add support for relative URIs in _parse_xspf (#15794) - -Extractors -+ [7plus] Extract series metadata (#15862, #15906) -* [9now] Bypass geo restriction (#15920) -* [cbs] Skip unavailable assets (#13490, #13506, #15776) -+ [canalc2] Add support for HTML5 videos (#15916, #15919) -+ [ceskatelevize] Add support for iframe embeds (#15918) -+ [prosiebensat1] Add support for galileo.tv (#15894) -+ [generic] Add support for xfileshare embeds (#15879) -* [bilibili] Switch to v2 playurl API -* [bilibili] Fix and improve extraction (#15048, #15430, #15622, #15863) -* [heise] Improve extraction (#15496, #15784, #15026) -* [instagram] Fix user videos extraction (#15858) - - -version 2018.03.14 - -Extractors -* [soundcloud] Update client id (#15866) -+ [tennistv] Add support for tennistv.com -+ [line] Add support for tv.line.me (#9427) -* [xnxx] Fix extraction (#15817) -* [njpwworld] Fix authentication (#15815) - - -version 2018.03.10 - -Core -* [downloader/hls] Skip uplynk ad fragments (#15748) - -Extractors -* [pornhub] Don't override session cookies (#15697) -+ [raywenderlich] Add support for videos.raywenderlich.com (#15251) -* [funk] Fix extraction and rework extractors (#15792) -* [nexx] Restore reverse engineered approach -+ [heise] Add support for kaltura embeds (#14961, #15728) -+ [tvnow] Extract series metadata (#15774) -* [ruutu] Continue formats extraction on NOT-USED URLs (#15775) -* [vrtnu] Use redirect URL for building video JSON URL (#15767, #15769) -* [vimeo] Modernize login code and improve error messaging -* [archiveorg] Fix extraction (#15770, #15772) -+ [hidive] Add support for hidive.com (#15494) -* [afreecatv] Detect deleted videos -* [afreecatv] Fix extraction (#15755) -* [vice] Fix extraction and rework extractors (#11101, #13019, #13622, #13778) -+ [vidzi] Add support for vidzi.si (#15751) -* [npo] Fix typo - - -version 2018.03.03 - -Core -+ [utils] Add parse_resolution -Revert respect --prefer-insecure while updating - -Extractors -+ [yapfiles] Add support for yapfiles.ru (#15726, #11085) -* [spankbang] Fix formats extraction (#15727) -* [adn] Fix extraction (#15716) -+ [toggle] Extract DASH and ISM formats (#15721) -+ [nickelodeon] Add support for nickelodeon.com.tr (#15706) -* [npo] Validate and filter format URLs (#15709) - - -version 2018.02.26 - -Extractors -* [udemy] Use custom User-Agent (#15571) - - -version 2018.02.25 - -Core -* [postprocessor/embedthumbnail] Skip embedding when there aren't any - thumbnails (#12573) -* [extractor/common] Improve jwplayer subtitles extraction (#15695) - -Extractors -+ [vidlii] Add support for vidlii.com (#14472, #14512, #14779) -+ [streamango] Capture and output error messages -* [streamango] Fix extraction (#14160, #14256) -+ [telequebec] Add support for emissions (#14649, #14655) -+ [telequebec:live] Add support for live streams (#15688) -+ [mailru:music] Add support for mail.ru/music (#15618) -* [aenetworks] Switch to akamai HLS formats (#15612) -* [ytsearch] Fix flat title extraction (#11260, #15681) - - -version 2018.02.22 - -Core -+ [utils] Fixup some common URL typos in sanitize_url (#15649) -* Respect --prefer-insecure while updating (#15497) - -Extractors -* [vidio] Fix HLS URL extraction (#15675) -+ [nexx] Add support for arc.nexx.cloud URLs -* [nexx] Switch to arc API (#15652) -* [redtube] Fix duration extraction (#15659) -+ [sonyliv] Respect referrer (#15648) -+ [brightcove:new] Use referrer for formats' HTTP headers -+ [cbc] Add support for olympics.cbc.ca (#15535) -+ [fusion] Add support for fusion.tv (#15628) -* [npo] Improve quality metadata extraction -* [npo] Relax URL regular expression (#14987, #14994) -+ [npo] Capture and output error message -+ [pornhub] Add support for channels (#15613) -* [youtube] Handle shared URLs with generic extractor (#14303) - - -version 2018.02.11 - -Core -+ [YoutubeDL] Add support for filesize_approx in format selector (#15550) - -Extractors -+ [francetv] Add support for live streams (#13689) -+ [francetv] Add support for zouzous.fr and ludo.fr (#10454, #13087, #13103, - #15012) -* [francetv] Separate main extractor and rework others to delegate to it -* [francetv] Improve manifest URL signing (#15536) -+ [francetv] Sign m3u8 manifest URLs (#15565) -+ [veoh] Add support for embed URLs (#15561) -* [afreecatv] Fix extraction (#15556) -* [periscope] Use accessVideoPublic endpoint (#15554) -* [discovery] Fix auth request (#15542) -+ [6play] Extract subtitles (#15541) -* [newgrounds] Fix metadata extraction (#15531) -+ [nbc] Add support for stream.nbcolympics.com (#10295) -* [dvtv] Fix live streams extraction (#15442) - - -version 2018.02.08 - -Extractors -+ [myvi] Extend URL regular expression -+ [myvi:embed] Add support for myvi.tv embeds (#15521) -+ [prosiebensat1] Extend URL regular expression (#15520) -* [pokemon] Relax URL regular expression and extend title extraction (#15518) -+ [gameinformer] Use geo verification headers -* [la7] Fix extraction (#15501, #15502) -* [gameinformer] Fix brightcove id extraction (#15416) -+ [afreecatv] Pass referrer to video info request (#15507) -+ [telebruxelles] Add support for live streams -* [telebruxelles] Relax URL regular expression -* [telebruxelles] Fix extraction (#15504) -* [extractor/common] Respect secure schemes in _extract_wowza_formats - - -version 2018.02.04 - -Core -* [downloader/http] Randomize HTTP chunk size -+ [downloader/http] Add ability to pass downloader options via info dict -* [downloader/http] Fix 302 infinite loops by not reusing requests -+ Document http_chunk_size - -Extractors -+ [brightcove] Pass embed page URL as referrer (#15486) -+ [youtube] Enforce using chunked HTTP downloading for DASH formats - - -version 2018.02.03 - -Core -+ Introduce --http-chunk-size for chunk-based HTTP downloading -+ Add support for IronPython -* [downloader/ism] Fix Python 3.2 support - -Extractors -* [redbulltv] Fix extraction (#15481) -* [redtube] Fix metadata extraction (#15472) -* [pladform] Respect platform id and extract HLS formats (#15468) -- [rtlnl] Remove progressive formats (#15459) -* [6play] Do no modify asset URLs with a token (#15248) -* [nationalgeographic] Relax URL regular expression -* [dplay] Relax URL regular expression (#15458) -* [cbsinteractive] Fix data extraction (#15451) -+ [amcnetworks] Add support for sundancetv.com (#9260) - - -version 2018.01.27 - -Core -* [extractor/common] Improve _json_ld for articles -* Switch codebase to use compat_b64decode -+ [compat] Add compat_b64decode - -Extractors -+ [seznamzpravy] Add support for seznam.cz and seznamzpravy.cz (#14102, #14616) -* [dplay] Bypass geo restriction -+ [dplay] Add support for disco-api videos (#15396) -* [youtube] Extract precise error messages (#15284) -* [teachertube] Capture and output error message -* [teachertube] Fix and relax thumbnail extraction (#15403) -+ [prosiebensat1] Add another clip id regular expression (#15378) -* [tbs] Update tokenizer url (#15395) -* [mixcloud] Use compat_b64decode (#15394) -- [thesixtyone] Remove extractor (#15341) - - -version 2018.01.21 - -Core -* [extractor/common] Improve jwplayer DASH formats extraction (#9242, #15187) -* [utils] Improve scientific notation handling in js_to_json (#14789) - -Extractors -+ [southparkdk] Add support for southparkstudios.nu -+ [southpark] Add support for collections (#14803) -* [franceinter] Fix upload date extraction (#14996) -+ [rtvs] Add support for rtvs.sk (#9242, #15187) -* [restudy] Fix extraction and extend URL regular expression (#15347) -* [youtube:live] Improve live detection (#15365) -+ [springboardplatform] Add support for springboardplatform.com -* [prosiebensat1] Add another clip id regular expression (#15290) -- [ringtv] Remove extractor (#15345) - - -version 2018.01.18 - -Extractors -* [soundcloud] Update client id (#15306) -- [kamcord] Remove extractor (#15322) -+ [spiegel] Add support for nexx videos (#15285) -* [twitch] Fix authentication and error capture (#14090, #15264) -* [vk] Detect more errors due to copyright complaints (#15259) - - -version 2018.01.14 - -Extractors -* [youtube] Fix live streams extraction (#15202) -* [wdr] Bypass geo restriction -* [wdr] Rework extractors (#14598) -+ [wdr] Add support for wdrmaus.de/elefantenseite (#14598) -+ [gamestar] Add support for gamepro.de (#3384) -* [viafree] Skip rtmp formats (#15232) -+ [pandoratv] Add support for mobile URLs (#12441) -+ [pandoratv] Add support for new URL format (#15131) -+ [ximalaya] Add support for ximalaya.com (#14687) -+ [digg] Add support for digg.com (#15214) -* [limelight] Tolerate empty pc formats (#15150, #15151, #15207) -* [ndr:embed:base] Make separate formats extraction non fatal (#15203) -+ [weibo] Add extractor (#15079) -+ [ok] Add support for live streams -* [canalplus] Fix extraction (#15072) -* [bilibili] Fix extraction (#15188) - - -version 2018.01.07 - -Core -* [utils] Fix youtube-dl under PyPy3 on Windows -* [YoutubeDL] Output python implementation in debug header - -Extractors -+ [jwplatform] Add support for multiple embeds (#15192) -* [mitele] Fix extraction (#15186) -+ [motherless] Add support for groups (#15124) -* [lynda] Relax URL regular expression (#15185) -* [soundcloud] Fallback to avatar picture for thumbnail (#12878) -* [youku] Fix list extraction (#15135) -* [openload] Fix extraction (#15166) -* [lynda] Skip invalid subtitles (#15159) -* [twitch] Pass video id to url_result when extracting playlist (#15139) -* [rtve.es:alacarta] Fix extraction of some new URLs -* [acast] Fix extraction (#15147) - - -version 2017.12.31 - -Core -+ [extractor/common] Add container meta field for formats extracted - in _parse_mpd_formats (#13616) -+ [downloader/hls] Use HTTP headers for key request -* [common] Use AACL as the default fourcc when AudioTag is 255 -* [extractor/common] Fix extraction of DASH formats with the same - representation id (#15111) - -Extractors -+ [slutload] Add support for mobile URLs (#14806) -* [abc:iview] Bypass geo restriction -* [abc:iview] Fix extraction (#14711, #14782, #14838, #14917, #14963, #14985, - #15035, #15057, #15061, #15071, #15095, #15106) -* [openload] Fix extraction (#15118) -- [sandia] Remove extractor -- [collegerama] Remove extractor -+ [mediasite] Add support for sites based on Mediasite Video Platform (#5428, - #11185, #14343) -+ [ufctv] Add support for ufc.tv (#14520) -* [pluralsight] Fix missing first line of subtitles (#11118) -* [openload] Fallback on f-page extraction (#14665, #14879) -* [vimeo] Improve password protected videos extraction (#15114) -* [aws] Fix canonical/signed headers generation on python 2 (#15102) - - -version 2017.12.28 - -Extractors -+ [internazionale] Add support for internazionale.it (#14973) -* [playtvak] Relax video regular expression and make description optional - (#15037) -+ [filmweb] Add support for filmweb.no (#8773, #10368) -+ [23video] Add support for 23video.com -+ [espn] Add support for fivethirtyeight.com (#6864) -+ [umg:de] Add support for universal-music.de (#11582, #11584) -+ [espn] Add support for espnfc and extract more formats (#8053) -* [youku] Update ccode (#14880) -+ [openload] Add support for oload.stream (#15070) -* [youku] Fix list extraction (#15065) - - -version 2017.12.23 - -Core -* [extractor/common] Move X-Forwarded-For setup code into _request_webpage -+ [YoutubeDL] Add support for playlist_uploader and playlist_uploader_id in - output template (#11427, #15018) -+ [extractor/common] Introduce uploader, uploader_id and uploader_url - meta fields for playlists (#11427, #15018) -* [downloader/fragment] Encode filename of fragment being removed (#15020) -+ [utils] Add another date format pattern (#14999) - -Extractors -+ [kaltura] Add another embed pattern for entry_id -+ [7plus] Add support for 7plus.com.au (#15043) -* [animeondemand] Relax login error regular expression -+ [shahid] Add support for show pages (#7401) -+ [youtube] Extract uploader, uploader_id and uploader_url for playlists - (#11427, #15018) -* [afreecatv] Improve format extraction (#15019) -+ [cspan] Add support for audio only pages and catch page errors (#14995) -+ [mailru] Add support for embed URLs (#14904) -* [crunchyroll] Future-proof XML element checks (#15013) -* [cbslocal] Fix timestamp extraction (#14999, #15000) -* [discoverygo] Correct TTML subtitle extension -* [vk] Make view count optional (#14979) -* [disney] Skip Apple FairPlay formats (#14982) -* [voot] Fix format extraction (#14758) - - -version 2017.12.14 - -Core -* [postprocessor/xattr] Clarify NO_SPACE message (#14970) -* [downloader/http] Return actual download result from real_download (#14971) - -Extractors -+ [itv] Extract more subtitles and duration -* [itv] Improve extraction (#14944) -+ [byutv] Add support for geo restricted videos -* [byutv] Fix extraction (#14966, #14967) -+ [bbccouk] Fix extraction for 320k HLS streams -+ [toutv] Add support for special video URLs (#14179) -* [discovery] Fix free videos extraction (#14157, #14954) -* [tvnow] Fix extraction (#7831) -+ [nickelodeon:br] Add support for nickelodeon brazil websites (#14893) -* [nick] Improve extraction (#14876) -* [tbs] Fix extraction (#13658) - - -version 2017.12.10 - -Core -+ [utils] Add sami mimetype to mimetype2ext - -Extractors -* [culturebox] Improve video id extraction (#14947) -* [twitter] Improve extraction (#14197) -+ [udemy] Extract more HLS formats -* [udemy] Improve course id extraction (#14938) -+ [stretchinternet] Add support for portal.stretchinternet.com (#14576) -* [ellentube] Fix extraction (#14407, #14570) -+ [raiplay:playlist] Add support for playlists (#14563) -* [sonyliv] Bypass geo restriction -* [sonyliv] Extract higher quality formats (#14922) -* [fox] Extract subtitles -+ [fox] Add support for Adobe Pass authentication (#14205, #14489) -- [dailymotion:cloud] Remove extractor (#6794) -* [xhamster] Fix thumbnail extraction (#14780) -+ [xhamster] Add support for mobile URLs (#14780) -* [generic] Don't pass video id as mpd id while extracting DASH (#14902) -* [ard] Skip invalid stream URLs (#14906) -* [porncom] Fix metadata extraction (#14911) -* [pluralsight] Detect agreement request (#14913) -* [toutv] Fix login (#14614) - - -version 2017.12.02 - -Core -+ [downloader/fragment] Commit part file after each fragment -+ [extractor/common] Add durations for DASH fragments with bare SegmentURLs -+ [extractor/common] Add support for DASH manifests with SegmentLists with - bare SegmentURLs (#14844) -+ [utils] Add hvc1 codec code to parse_codecs - -Extractors -* [xhamster] Fix extraction (#14884) -* [youku] Update ccode (#14872) -* [mnet] Fix format extraction (#14883) -+ [xiami] Add Referer header to API request -* [mtv] Correct scc extention in extracted subtitles (#13730) -* [vvvvid] Fix extraction for kenc videos (#13406) -+ [br] Add support for BR Mediathek videos (#14560, #14788) -+ [daisuki] Add support for motto.daisuki.com (#14681) -* [odnoklassniki] Fix API metadata request (#14862) -* [itv] Fix HLS formats extraction -+ [pbs] Add another media id regular expression - - -version 2017.11.26 - -Core -* [extractor/common] Use final URL when dumping request (#14769) - -Extractors -* [fczenit] Fix extraction -- [firstpost] Remove extractor -* [freespeech] Fix extraction -* [nexx] Extract more formats -+ [openload] Add support for openload.link (#14763) -* [empflix] Relax URL regular expression -* [empflix] Fix extractrion -* [tnaflix] Don't modify download URLs (#14811) -- [gamersyde] Remove extractor -* [francetv:generationwhat] Fix extraction -+ [massengeschmacktv] Add support for Massengeschmack TV -* [fox9] Fix extraction -* [faz] Fix extraction and add support for Perform Group embeds (#14714) -+ [performgroup] Add support for performgroup.com -+ [jwplatform] Add support for iframes (#14828) -* [culturebox] Fix extraction (#14827) -* [youku] Fix extraction; update ccode (#14815) -* [livestream] Make SMIL extraction non fatal (#14792) -+ [drtuber] Add support for mobile URLs (#14772) -+ [spankbang] Add support for mobile URLs (#14771) -* [instagram] Fix description, timestamp and counters extraction (#14755) - - -version 2017.11.15 - -Core -* [common] Skip Apple FairPlay m3u8 manifests (#14741) -* [YoutubeDL] Fix playlist range optimization for --playlist-items (#14740) - -Extractors -* [vshare] Capture and output error message -* [vshare] Fix extraction (#14473) -* [crunchyroll] Extract old RTMP formats -* [tva] Fix extraction (#14736) -* [gamespot] Lower preference of HTTP formats (#14652) -* [instagram:user] Fix extraction (#14699) -* [ccma] Fix typo (#14730) -- Remove sensitive data from logging in messages -* [instagram:user] Fix extraction (#14699) -+ [gamespot] Add support for article URLs (#14652) -* [gamespot] Skip Brightcove Once HTTP formats (#14652) -* [cartoonnetwork] Update tokenizer_src (#14666) -+ [wsj] Recognize another URL pattern (#14704) -* [pandatv] Update API URL and sign format URLs (#14693) -* [crunchyroll] Use old login method (#11572) - - -version 2017.11.06 - -Core -+ [extractor/common] Add protocol for f4m formats -* [f4m] Prefer baseURL for relative URLs (#14660) -* [extractor/common] Respect URL query in _extract_wowza_formats (14645) - -Extractors -+ [hotstar:playlist] Add support for playlists (#12465) -* [hotstar] Bypass geo restriction (#14672) -- [22tracks] Remove extractor (#11024, #14628) -+ [skysport] Sdd support ooyala videos protected with embed_token (#14641) -* [gamespot] Extract formats referenced with new data fields (#14652) -* [spankbang] Detect unavailable videos (#14644) - - -version 2017.10.29 - -Core -* [extractor/common] Prefix format id for audio only HLS formats -+ [utils] Add support for zero years and months in parse_duration - -Extractors -* [egghead] Fix extraction (#14388) -+ [fxnetworks] Extract series metadata (#14603) -+ [younow] Add support for younow.com (#9255, #9432, #12436) -* [dctptv] Fix extraction (#14599) -* [youtube] Restrict embed regular expression (#14600) -* [vimeo] Restrict iframe embed regular expression (#14600) -* [soundgasm] Improve extraction (#14588) -- [myvideo] Remove extractor (#8557) -+ [nbc] Add support for classic-tv videos (#14575) -+ [vrtnu] Add support for cookies authentication and simplify (#11873) -+ [canvas] Add support for vrt.be/vrtnu (#11873) -* [twitch:clips] Fix title extraction (#14566) -+ [ndtv] Add support for sub-sites (#14534) -* [dramafever] Fix login error message extraction -+ [nick] Add support for more nickelodeon sites (no, dk, se, ch, fr, es, pt, - ro, hu) (#14553) - - -version 2017.10.20 - -Core -* [downloader/fragment] Report warning instead of error on inconsistent - download state -* [downloader/hls] Fix total fragments count when ad fragments exist - -Extractors -* [parliamentliveuk] Fix extraction (#14524) -* [soundcloud] Update client id (#14546) -+ [servus] Add support for servus.com (#14362) -+ [unity] Add support for unity3d.com (#14528) -* [youtube] Replace youtube redirect URLs in description (#14517) -* [pbs] Restrict direct video URL regular expression (#14519) -* [drtv] Respect preference for direct HTTP formats (#14509) -+ [eporner] Add support for embed URLs (#14507) -* [arte] Capture and output error message -* [niconico] Improve uploader metadata extraction robustness (#14135) - - -version 2017.10.15.1 - -Core -* [downloader/hls] Ignore anvato ad fragments (#14496) -* [downloader/fragment] Output ad fragment count - -Extractors -* [scrippsnetworks:watch] Bypass geo restriction -+ [anvato] Add ability to bypass geo restriction -* [redditr] Fix extraction for URLs with query (#14495) - - -version 2017.10.15 - -Core -+ [common] Add support for jwplayer youtube embeds - -Extractors -* [scrippsnetworks:watch] Fix extraction (#14389) -* [anvato] Process master m3u8 manifests -* [youtube] Fix relative URLs in description -* [spike] Bypass geo restriction -+ [howstuffworks] Add support for more domains -* [infoq] Fix http format downloading -+ [rtlnl] Add support for another type of embeds -+ [onionstudios] Add support for bulbs-video embeds -* [udn] Fix extraction -* [shahid] Fix extraction (#14448) -* [kaltura] Ignore Widevine encrypted video (.wvm) (#14471) -* [vh1] Fix extraction (#9613) - - -version 2017.10.12 - -Core -* [YoutubeDL] Improve _default_format_spec (#14461) - -Extractors -* [steam] Fix extraction (#14067) -+ [funk] Add support for funk.net (#14464) -+ [nexx] Add support for shortcuts and relax domain id extraction -+ [voxmedia] Add support for recode.net (#14173) -+ [once] Add support for vmap URLs -+ [generic] Add support for channel9 embeds (#14469) -* [tva] Fix extraction (#14328) -+ [tubitv] Add support for new URL format (#14460) -- [afreecatv:global] Remove extractor -- [youtube:shared] Removed extractor (#14420) -+ [slideslive] Add support for slideslive.com (#2680) -+ [facebook] Support thumbnails (#14416) -* [vvvvid] Fix episode number extraction (#14456) -* [hrti:playlist] Relax URL regular expression -* [wdr] Relax media link regular expression (#14447) -* [hrti] Relax URL regular expression (#14443) -* [fox] Delegate extraction to uplynk:preplay (#14147) -+ [youtube] Add support for hooktube.com (#14437) - - -version 2017.10.07 - -Core -* [YoutubeDL] Ignore duplicates in --playlist-items -* [YoutubeDL] Fix out of range --playlist-items for iterable playlists and - reduce code duplication (#14425) -+ [utils] Use cache in OnDemandPagedList by default -* [postprocessor/ffmpeg] Convert to opus using libopus (#14381) - -Extractors -* [reddit] Sort formats (#14430) -* [lnkgo] Relax URL regular expression (#14423) -* [pornflip] Extend URL regular expression (#14405, #14406) -+ [xtube] Add support for embed URLs (#14417) -+ [xvideos] Add support for embed URLs and improve extraction (#14409) -* [beeg] Fix extraction (#14403) -* [tvn24] Relax URL regular expression (#14395) -* [nbc] Fix extraction (#13651, #13715, #14137, #14198, #14312, #14314, #14378, - #14392, #14414, #14419, #14431) -+ [ketnet] Add support for videos without direct sources (#14377) -* [canvas] Generalize mediazone.vrt.be extractor and rework canvas and een -+ [afreecatv] Add support for adult videos (#14376) - - -version 2017.10.01 - -Core -* [YoutubeDL] Document youtube_include_dash_manifest - -Extractors -+ [tvp] Add support for new URL schema (#14368) -+ [generic] Add support for single format Video.js embeds (#14371) -* [yahoo] Bypass geo restriction for brightcove (#14210) -* [yahoo] Use extracted brightcove account id (#14210) -* [rtve:alacarta] Fix extraction (#14290) -+ [yahoo] Add support for custom brigthcove embeds (#14210) -+ [generic] Add support for Video.js embeds -+ [gfycat] Add support for /gifs/detail URLs (#14322) -* [generic] Fix infinite recursion for twitter:player URLs (#14339) -* [xhamsterembed] Fix extraction (#14308) - - -version 2017.09.24 - -Core -+ [options] Accept lrc as a subtitle conversion target format (#14292) -* [utils] Fix handling raw TTML subtitles (#14191) - -Extractors -* [24video] Fix timestamp extraction and make non fatal (#14295) -+ [24video] Add support for 24video.adult (#14295) -+ [kakao] Add support for tv.kakao.com (#12298, #14007) -+ [twitter] Add support for URLs without user id (#14270) -+ [americastestkitchen] Add support for americastestkitchen.com (#10764, - #13996) -* [generic] Fix support for multiple HTML5 videos on one page (#14080) -* [mixcloud] Fix extraction (#14088, #14132) -+ [lynda] Add support for educourse.ga (#14286) -* [beeg] Fix extraction (#14275) -* [nbcsports:vplayer] Correct theplatform URL (#13873) -* [twitter] Fix duration extraction (#14141) -* [tvplay] Bypass geo restriction -+ [heise] Add support for YouTube embeds (#14109) -+ [popcorntv] Add support for popcorntv.it (#5914, #14211) -* [viki] Update app data (#14181) -* [morningstar] Relax URL regular expression (#14222) -* [openload] Fix extraction (#14225, #14257) -* [noovo] Fix extraction (#14214) -* [dailymotion:playlist] Relax URL regular expression (#14219) -+ [twitch] Add support for go.twitch.tv URLs (#14215) -* [vgtv] Relax URL regular expression (#14223) - - -version 2017.09.15 - -Core -* [downloader/fragment] Restart inconsistent incomplete fragment downloads - (#13731) -* [YoutubeDL] Download raw subtitles files (#12909, #14191) - -Extractors -* [condenast] Fix extraction (#14196, #14207) -+ [orf] Add support for f4m stories -* [tv4] Relax URL regular expression (#14206) -* [animeondemand] Bypass geo restriction -+ [animeondemand] Add support for flash videos (#9944) - - -version 2017.09.11 - -Extractors -* [rutube:playlist] Fix suitable (#14166) - - -version 2017.09.10 - -Core -+ [utils] Introduce bool_or_none -* [YoutubeDL] Ensure dir existence for each requested format (#14116) - -Extractors -* [fox] Fix extraction (#14147) -* [rutube] Use bool_or_none -* [rutube] Rework and generalize playlist extractors (#13565) -+ [rutube:playlist] Add support for playlists (#13534, #13565) -+ [radiocanada] Add fallback for title extraction (#14145) -* [vk] Use dedicated YouTube embeds extraction routine -* [vice] Use dedicated YouTube embeds extraction routine -* [cracked] Use dedicated YouTube embeds extraction routine -* [chilloutzone] Use dedicated YouTube embeds extraction routine -* [abcnews] Use dedicated YouTube embeds extraction routine -* [youtube] Separate methods for embeds extraction -* [redtube] Fix formats extraction (#14122) -* [arte] Relax unavailability check (#14112) -+ [manyvids] Add support for preview videos from manyvids.com (#14053, #14059) -* [vidme:user] Relax URL regular expression (#14054) -* [bpb] Fix extraction (#14043, #14086) -* [soundcloud] Fix download URL with private tracks (#14093) -* [aliexpress:live] Add support for live.aliexpress.com (#13698, #13707) -* [viidea] Capture and output lecture error message (#14099) -* [radiocanada] Skip unsupported platforms (#14100) - - -version 2017.09.02 - -Extractors -* [youtube] Force old layout for each webpage (#14068, #14072, #14074, #14076, - #14077, #14079, #14082, #14083, #14094, #14095, #14096) -* [youtube] Fix upload date extraction (#14065) -+ [charlierose] Add support for episodes (#14062) -+ [bbccouk] Add support for w-prefixed ids (#14056) -* [googledrive] Extend URL regular expression (#9785) -+ [googledrive] Add support for source format (#14046) -* [pornhd] Fix extraction (#14005) - - -version 2017.08.27.1 - -Extractors - -* [youtube] Fix extraction with --youtube-skip-dash-manifest enabled (#14037) - - -version 2017.08.27 - -Core -+ [extractor/common] Extract height and format id for HTML5 videos (#14034) -* [downloader/http] Rework HTTP downloader (#506, #809, #2849, #4240, #6023, - #8625, #9483) - * Simplify code and split into separate routines to facilitate maintaining - * Make retry mechanism work on errors during actual download not only - during connection establishment phase - * Retry on ECONNRESET and ETIMEDOUT during reading data from network - * Retry on content too short - * Show error description on retry - -Extractors -* [generic] Lower preference for extraction from LD-JSON -* [rai] Fix audio formats extraction (#14024) -* [youtube] Fix controversy videos extraction (#14027, #14029) -* [mixcloud] Fix extraction (#14015, #14020) - - -version 2017.08.23 - -Core -+ [extractor/common] Introduce _parse_xml -* [extractor/common] Make HLS and DASH extraction in_parse_html5_media_entries - non fatal (#13970) -* [utils] Fix unescapeHTML for misformed string like "&a"" (#13935) - -Extractors -* [cbc:watch] Bypass geo restriction (#13993) -* [toutv] Relax DRM check (#13994) -+ [googledrive] Add support for subtitles (#13619, #13638) -* [pornhub] Relax uploader regular expression (#13906, #13975) -* [bandcamp:album] Extract track titles (#13962) -+ [bbccouk] Add support for events URLs (#13893) -+ [liveleak] Support multi-video pages (#6542) -+ [liveleak] Support another liveleak embedding pattern (#13336) -* [cda] Fix extraction (#13935) -+ [laola1tv] Add support for tv.ittf.com (#13965) -* [mixcloud] Fix extraction (#13958, #13974, #13980, #14003) - - -version 2017.08.18 - -Core -* [YoutubeDL] Sanitize byte string format URLs (#13951) -+ [extractor/common] Add support for float durations in _parse_mpd_formats - (#13919) - -Extractors -* [arte] Detect unavailable videos (#13945) -* [generic] Convert redirect URLs to unicode strings (#13951) -* [udemy] Fix paid course detection (#13943) -* [pluralsight] Use RPC API for course extraction (#13937) -+ [clippit] Add support for clippituser.tv -+ [qqmusic] Support new URL schemes (#13805) -* [periscope] Renew HLS extraction (#13917) -* [mixcloud] Extract decrypt key - - -version 2017.08.13 - -Core -* [YoutubeDL] Make sure format id is not empty -* [extractor/common] Make _family_friendly_search optional -* [extractor/common] Respect source's type attribute for HTML5 media (#13892) - -Extractors -* [pornhub:playlistbase] Skip videos from drop-down menu (#12819, #13902) -+ [fourtube] Add support pornerbros.com (#6022) -+ [fourtube] Add support porntube.com (#7859, #13901) -+ [fourtube] Add support fux.com -* [limelight] Improve embeds detection (#13895) -+ [reddit] Add support for v.redd.it and reddit.com (#13847) -* [aparat] Extract all formats (#13887) -* [mixcloud] Fix play info decryption (#13885) -+ [generic] Add support for vzaar embeds (#13876) - - -version 2017.08.09 - -Core -* [utils] Skip missing params in cli_bool_option (#13865) - -Extractors -* [xxxymovies] Fix title extraction (#13868) -+ [nick] Add support for nick.com.pl (#13860) -* [mixcloud] Fix play info decryption (#13867) -* [20min] Fix embeds extraction (#13852) -* [dplayit] Fix extraction (#13851) -+ [niconico] Support videos with multiple formats (#13522) -+ [niconico] Support HTML5-only videos (#13806) - - -version 2017.08.06 - -Core -* Use relative paths for DASH fragments (#12990) - -Extractors -* [pluralsight] Fix format selection -- [mpora] Remove extractor (#13826) -+ [voot] Add support for voot.com (#10255, #11644, #11814, #12350, #13218) -* [vlive:channel] Limit number of videos per page to 100 (#13830) -* [podomatic] Extend URL regular expression (#13827) -* [cinchcast] Extend URL regular expression -* [yandexdisk] Relax URL regular expression (#13824) -* [vidme] Extract DASH and HLS formats -- [teamfour] Remove extractor (#13782) -* [pornhd] Fix extraction (#13783) -* [udemy] Fix subtitles extraction (#13812) -* [mlb] Extend URL regular expression (#13740, #13773) -+ [pbs] Add support for new URL schema (#13801) -* [nrktv] Update API host (#13796) - - -version 2017.07.30.1 - -Core -* [downloader/hls] Use redirect URL as manifest base (#13755) -* [options] Correctly hide login info from debug outputs (#13696) - -Extractors -+ [watchbox] Add support for watchbox.de (#13739) -- [clipfish] Remove extractor -+ [youjizz] Fix extraction (#13744) -+ [generic] Add support for another ooyala embed pattern (#13727) -+ [ard] Add support for lives (#13771) -* [soundcloud] Update client id -+ [soundcloud:trackstation] Add support for track stations (#13733) -* [svtplay] Use geo verification proxy for API request -* [svtplay] Update API URL (#13767) -+ [yandexdisk] Add support for yadi.sk (#13755) -+ [megaphone] Add support for megaphone.fm -* [amcnetworks] Make rating optional (#12453) -* [cloudy] Fix extraction (#13737) -+ [nickru] Add support for nickelodeon.ru -* [mtv] Improve thumbnal extraction -* [nick] Automate geo-restriction bypass (#13711) -* [niconico] Improve error reporting (#13696) - - -version 2017.07.23 - -Core -* [YoutubeDL] Improve default format specification (#13704) -* [YoutubeDL] Do not override id, extractor and extractor_key for - url_transparent entities -* [extractor/common] Fix playlist_from_matches - -Extractors -* [itv] Fix production id extraction (#13671, #13703) -* [vidio] Make duration non fatal and fix typo -* [mtv] Skip missing video parts (#13690) -* [sportbox:embed] Fix extraction -+ [npo] Add support for npo3.nl URLs (#13695) -* [dramafever] Remove video id from title (#13699) -+ [egghead:lesson] Add support for lessons (#6635) -* [funnyordie] Extract more metadata (#13677) -* [youku:show] Fix playlist extraction (#13248) -+ [dispeak] Recognize sevt subdomain (#13276) -* [adn] Improve error reporting (#13663) -* [crunchyroll] Relax series and season regular expression (#13659) -+ [spiegel:article] Add support for nexx iframe embeds (#13029) -+ [nexx:embed] Add support for iframe embeds -* [nexx] Improve JS embed extraction -+ [pearvideo] Add support for pearvideo.com (#13031) - - -version 2017.07.15 - -Core -* [YoutubeDL] Don't expand environment variables in meta fields (#13637) - -Extractors -* [spiegeltv] Delegate extraction to nexx extractor (#13159) -+ [nexx] Add support for nexx.cloud (#10807, #13465) -* [generic] Fix rutube embeds extraction (#13641) -* [karrierevideos] Fix title extraction (#13641) -* [youtube] Don't capture YouTube Red ad for creator meta field (#13621) -* [slideshare] Fix extraction (#13617) -+ [5tv] Add another video URL pattern (#13354, #13606) -* [drtv] Make HLS and HDS extraction non fatal -* [ted] Fix subtitles extraction (#13628, #13629) -* [vine] Make sure the title won't be empty -+ [twitter] Support HLS streams in vmap URLs -+ [periscope] Support pscp.tv URLs in embedded frames -* [twitter] Extract mp4 urls via mobile API (#12726) -* [niconico] Fix authentication error handling (#12486) -* [giantbomb] Extract m3u8 formats (#13626) -+ [vlive:playlist] Add support for playlists (#13613) - - -version 2017.07.09 - -Core -+ [extractor/common] Add support for AMP tags in _parse_html5_media_entries -+ [utils] Support attributes with no values in get_elements_by_attribute - -Extractors -+ [dailymail] Add support for embeds -+ [joj] Add support for joj.sk (#13268) -* [abc.net.au:iview] Extract more formats (#13492, #13489) -* [egghead:course] Fix extraction (#6635, #13370) -+ [cjsw] Add support for cjsw.com (#13525) -+ [eagleplatform] Add support for referrer protected videos (#13557) -+ [eagleplatform] Add support for another embed pattern (#13557) -* [veoh] Extend URL regular expression (#13601) -* [npo:live] Fix live stream id extraction (#13568, #13605) -* [googledrive] Fix height extraction (#13603) -+ [dailymotion] Add support for new layout (#13580) -- [yam] Remove extractor -* [xhamster] Extract all formats and fix duration extraction (#13593) -+ [xhamster] Add support for new URL schema (#13593) -* [espn] Extend URL regular expression (#13244, #13549) -* [kaltura] Fix typo in subtitles extraction (#13569) -* [vier] Adapt extraction to redesign (#13575) - - -version 2017.07.02 - -Core -* [extractor/common] Improve _json_ld - -Extractors -+ [thisoldhouse] Add more fallbacks for video id -* [thisoldhouse] Fix video id extraction (#13540, #13541) -* [xfileshare] Extend format regular expression (#13536) -* [ted] Fix extraction (#13535) -+ [tastytrade] Add support for tastytrade.com (#13521) -* [dplayit] Relax video id regular expression (#13524) -+ [generic] Extract more generic metadata (#13527) -+ [bbccouk] Capture and output error message (#13501, #13518) -* [cbsnews] Relax video info regular expression (#13284, #13503) -+ [facebook] Add support for plugin video embeds and multiple embeds (#13493) -* [soundcloud] Switch to https for API requests (#13502) -* [pandatv] Switch to https for API and download URLs -+ [pandatv] Add support for https URLs (#13491) -+ [niconico] Support sp subdomain (#13494) - - -version 2017.06.25 - -Core -+ [adobepass] Add support for DIRECTV NOW (mso ATTOTT) (#13472) -* [YoutubeDL] Skip malformed formats for better extraction robustness - -Extractors -+ [wsj] Add support for barrons.com (#13470) -+ [ign] Add another video id pattern (#13328) -+ [raiplay:live] Add support for live streams (#13414) -+ [redbulltv] Add support for live videos and segments (#13486) -+ [onetpl] Add support for videos embedded via pulsembed (#13482) -* [ooyala] Make more robust -* [ooyala] Skip empty format URLs (#13471, #13476) -* [hgtv.com:show] Fix typo - - -version 2017.06.23 - -Core -* [adobepass] Fix extraction on older python 2.6 - -Extractors -* [youtube] Adapt to new automatic captions rendition (#13467) -* [hgtv.com:show] Relax video config regular expression (#13279, #13461) -* [drtuber] Fix formats extraction (#12058) -* [youporn] Fix upload date extraction -* [youporn] Improve formats extraction -* [youporn] Fix title extraction (#13456) -* [googledrive] Fix formats sorting (#13443) -* [watchindianporn] Fix extraction (#13411, #13415) -+ [vimeo] Add fallback mp4 extension for original format -+ [ruv] Add support for ruv.is (#13396) -* [viu] Fix extraction on older python 2.6 -* [pandora.tv] Fix upload_date extraction (#12846) -+ [asiancrush] Add support for asiancrush.com (#13420) - - -version 2017.06.18 - -Core -* [downloader/common] Use utils.shell_quote for debug command line -* [utils] Use compat_shlex_quote in shell_quote -* [postprocessor/execafterdownload] Encode command line (#13407) -* [compat] Fix compat_shlex_quote on Windows (#5889, #10254) -* [postprocessor/metadatafromtitle] Fix missing optional meta fields processing - in --metadata-from-title (#13408) -* [extractor/common] Fix json dumping with --geo-bypass -+ [extractor/common] Improve jwplayer subtitles extraction -+ [extractor/common] Improve jwplayer formats extraction (#13379) - -Extractors -* [polskieradio] Fix extraction (#13392) -+ [xfileshare] Add support for fastvideo.me (#13385) -* [bilibili] Fix extraction of videos with double quotes in titles (#13387) -* [4tube] Fix extraction (#13381, #13382) -+ [disney] Add support for disneychannel.de (#13383) -* [npo] Improve URL regular expression (#13376) -+ [corus] Add support for showcase.ca -+ [corus] Add support for history.ca (#13359) - - -version 2017.06.12 - -Core -* [utils] Handle compat_HTMLParseError in extract_attributes (#13349) -+ [compat] Introduce compat_HTMLParseError -* [utils] Improve unified_timestamp -* [extractor/generic] Ensure format id is unicode string -* [extractor/common] Return unicode string from _match_id -+ [YoutubeDL] Sanitize more fields (#13313) - -Extractors -+ [xfileshare] Add support for rapidvideo.tv (#13348) -* [xfileshare] Modernize and pass Referer -+ [rutv] Add support for testplayer.vgtrk.com (#13347) -+ [newgrounds] Extract more metadata (#13232) -+ [newgrounds:playlist] Add support for playlists (#10611) -* [newgrounds] Improve formats and uploader extraction (#13346) -* [msn] Fix formats extraction -* [turbo] Ensure format id is string -* [sexu] Ensure height is int -* [jove] Ensure comment count is int -* [golem] Ensure format id is string -* [gfycat] Ensure filesize is int -* [foxgay] Ensure height is int -* [flickr] Ensure format id is string -* [sohu] Fix numeric fields -* [safari] Improve authentication detection (#13319) -* [liveleak] Ensure height is int (#13313) -* [streamango] Make title optional (#13292) -* [rtlnl] Improve URL regular expression (#13295) -* [tvplayer] Fix extraction (#13291) - - -version 2017.06.05 - -Core -* [YoutubeDL] Don't emit ANSI escape codes on Windows (#13270) - -Extractors -+ [bandcamp:weekly] Add support for bandcamp weekly (#12758) -* [pornhub:playlist] Fix extraction (#13281) -- [godtv] Remove extractor (#13175) -* [safari] Fix typo (#13252) -* [youtube] Improve chapters extraction (#13247) -* [1tv] Lower preference for HTTP formats (#13246) -* [francetv] Relax URL regular expression -* [drbonanza] Fix extraction (#13231) -* [packtpub] Fix authentication (#13240) - - -version 2017.05.29 - -Extractors -* [youtube] Fix DASH MPD extraction for videos with non-encrypted format URLs - (#13211) -* [xhamster] Fix uploader and like/dislike count extraction (#13216)) -+ [xhamster] Extract categories (#11728) -+ [abcnews] Add support for embed URLs (#12851) -* [gaskrank] Fix extraction (#12493) -* [medialaan] Fix videos with missing videoUrl (#12774) -* [dvtv] Fix playlist support -+ [dvtv] Add support for DASH and HLS formats (#3063) -+ [beam:vod] Add support for beam.pro/mixer.com VODs (#13032)) -* [cbsinteractive] Relax URL regular expression (#13213) -* [adn] Fix formats extraction -+ [youku] Extract more metadata (#10433) -* [cbsnews] Fix extraction (#13205) - - -version 2017.05.26 - -Core -+ [utils] strip_jsonp() can recognize more patterns -* [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182) - -Extractors -+ [youtube] DASH MPDs with cipher signatures are recognized now (#11381) -+ [bbc] Add support for authentication -* [tudou] Merge into youku extractor (#12214) -* [youku:show] Fix extraction -* [youku] Fix extraction (#13191) -* [udemy] Fix extraction for outputs' format entries without URL (#13192) -* [vimeo] Fix formats' sorting (#13189) -* [cbsnews] Fix extraction for 60 Minutes videos (#12861) - - -version 2017.05.23 - -Core -+ [downloader/external] Pass -loglevel to ffmpeg downloader (#13183) -+ [adobepass] Add support for Bright House Networks (#13149) - -Extractors -+ [streamcz] Add support for subtitles (#13174) -* [youtube] Fix DASH manifest signature decryption (#8944, #13156) -* [toggle] Relax URL regular expression (#13172) -* [toypics] Fix extraction (#13077) -* [njpwworld] Fix extraction (#13162, #13169) -+ [hitbox] Add support for smashcast.tv (#13154) -* [mitele] Update app key regular expression (#13158) - - -version 2017.05.18.1 - -Core -* [jsinterp] Fix typo and cleanup regular expressions (#13134) - - -version 2017.05.18 - -Core -+ [jsinterp] Add support for quoted names and indexers (#13123, #13124, #13125, - #13126, #13128, #13129, #13130, #13131, #13132) -+ [extractor/common] Add support for schemeless URLs in _extract_wowza_formats - (#13088, #13092) -+ [utils] Recognize more audio codecs (#13081) - -Extractors -+ [vier] Extract more metadata (#12539) -* [vier] Improve extraction (#12801) - + Add support for authentication - * Bypass authentication when no credentials provided - * Improve extraction robustness -* [dailymail] Fix sources extraction (#13057) -* [dailymotion] Extend URL regular expression (#13079) - - -version 2017.05.14 - -Core -+ [extractor/common] Respect Width and Height attributes in ISM manifests -+ [postprocessor/metadatafromtitle] Add support regular expression syntax for - --metadata-from-title (#13065) - -Extractors -+ [mediaset] Add support for video.mediaset.it (#12708, #12964) -* [orf:radio] Fix extraction (#11643, #12926) -* [aljazeera] Extend URL regular expression (#13053) -* [imdb] Relax URL regular expression (#13056) -+ [francetv] Add support for mobile.france.tv (#13068) -+ [upskill] Add support for upskillcourses.com (#13043) -* [thescene] Fix extraction (#13061) -* [condenast] Improve embed support -* [liveleak] Fix extraction (#12053) -+ [douyu] Support Douyu shows (#12228) -* [myspace] Improve URL regular expression (#13040) -* [adultswim] Use desktop platform in assets URL (#13041) - - -version 2017.05.09 - -Core -* [YoutubeDL] Force --restrict-filenames when no locale is set on all python - versions (#13027) - -Extractors -* [francetv] Adapt to site redesign (#13034) -+ [packtpub] Add support for authentication (#12622) -* [drtv] Lower preference for SignLanguage formats (#13013, #13016) -+ [cspan] Add support for brightcove live embeds (#13028) -* [vrv] Extract DASH formats and subtitles -* [funimation] Fix authentication (#13021) -* [adultswim] Fix extraction (#8640, #10950, #11042, #12121) - + Add support for Adobe Pass authentication - + Add support for live streams - + Add support for show pages -* [turner] Extract thumbnail, is_live and strip description -+ [nonktube] Add support for nonktube.com (#8647, #13024) -+ [nuevo] Pass headers to _extract_nuevo -* [nbc] Improve extraction (#12364) - - -version 2017.05.07 - -Common -* [extractor/common] Fix typo in _extract_akamai_formats -+ [postprocessor/ffmpeg] Embed chapters into media file with --add-metadata -+ [extractor/common] Introduce chapters meta field - -Extractors -* [youtube] Fix authentication (#12820, #12927, #12973, #12992, #12993, #12995, - #13003) -* [bilibili] Fix video downloading (#13001) -* [rmcdecouverte] Fix extraction (#12937) -* [theplatform] Extract chapters -* [bandcamp] Fix thumbnail extraction (#12980) -* [pornhub] Extend URL regular expression (#12996) -+ [youtube] Extract chapters -+ [nrk] Extract chapters -+ [vice] Add support for ooyala embeds in article pages -+ [vice] Support vice articles (#12968) -* [vice] Fix extraction for non en_us videos (#12967) -* [gdcvault] Fix extraction for some videos (#12733) -* [pbs] Improve multipart video support (#12981) -* [laola1tv] Fix extraction (#12880) -+ [cda] Support birthday verification (#12789) -* [leeco] Fix extraction (#12974) -+ [pbs] Extract chapters -* [amp] Imporove thumbnail and subtitles extraction -* [foxsports] Fix extraction (#12945) -- [coub] Remove comment count extraction (#12941) - - -version 2017.05.01 - -Core -+ [extractor/common] Extract view count from JSON-LD -* [utils] Improve unified_timestamp -+ [utils] Add video/mp2t to mimetype2ext -* [downloader/external] Properly handle live stream downloading cancellation - (#8932) -+ [utils] Add support for unicode whitespace in clean_html on python 2 (#12906) - -Extractors -* [infoq] Make audio format extraction non fatal (#12938) -* [brightcove] Allow whitespace around attribute names in embedded code -+ [zaq1] Add support for zaq1.pl (#12693) -+ [xvideos] Extract duration (#12828) -* [vevo] Fix extraction (#12879) -+ [noovo] Add support for noovo.ca (#12792) -+ [washingtonpost] Add support for embeds (#12699) -* [yandexmusic:playlist] Fix extraction for python 3 (#12888) -* [anvato] Improve extraction (#12913) - * Promote to regular shortcut based extractor - * Add mcp to access key mapping table - * Add support for embeds extraction - * Add support for anvato embeds in generic extractor -* [xtube] Fix extraction for older FLV videos (#12734) -* [tvplayer] Fix extraction (#12908) - - -version 2017.04.28 - -Core -+ [adobepass] Use geo verification headers for all requests -- [downloader/fragment] Remove assert for resume_len when no fragments - downloaded -+ [extractor/common] Add manifest_url for explicit group rendition formats -* [extractor/common] Fix manifest_url for m3u8 formats -- [extractor/common] Don't list master m3u8 playlists in format list (#12832) - -Extractor -* [aenetworks] Fix extraction for shows with single season -+ [go] Add support for Disney, DisneyJunior and DisneyXD show pages -* [youtube] Recognize new locale-based player URLs (#12885) -+ [streamable] Add support for new embedded URL schema (#12844) -* [arte:+7] Relax URL regular expression (#12837) - - -version 2017.04.26 - -Core -* Introduce --keep-fragments for keeping fragments of fragmented download - on disk after download is finished -* [YoutubeDL] Fix output template for missing timestamp (#12796) -* [socks] Handle cases where credentials are required but missing -* [extractor/common] Improve HLS extraction (#12211) - * Extract m3u8 parsing to separate method - * Improve rendition groups extraction - * Build stream name according stream GROUP-ID - * Ignore reference to AUDIO group without URI when stream has no CODECS - * Use float for scaled tbr in _parse_m3u8_formats -* [utils] Add support for TTML styles in dfxp2srt -* [downloader/hls] No need to download keys for fragments that have been - already downloaded -* [downloader/fragment] Improve fragment downloading - * Resume immediately - * Don't concatenate fragments and decrypt them on every resume - * Optimize disk storage usage, don't store intermediate fragments on disk - * Store bookkeeping download state file -+ [extractor/common] Add support for multiple getters in try_get -+ [extractor/common] Add support for video of WebPage context in _json_ld - (#12778) -+ [extractor/common] Relax JWPlayer regular expression and remove - duplicate URLs (#12768) - -Extractors -* [iqiyi] Fix extraction of Yule videos -* [vidio] Improve extraction and sort formats -+ [brightcove] Match only video elements with data-video-id attribute -* [iqiyi] Fix playlist detection (#12504) -- [azubu] Remove extractor (#12813) -* [porn91] Fix extraction (#12814) -* [vidzi] Fix extraction (#12793) -+ [amp] Extract error message (#12795) -+ [xfileshare] Add support for gorillavid.com and daclips.com (#12776) -* [instagram] Fix extraction (#12777) -+ [generic] Support Brightcove videos in ' - manifest_url = self._html_search_regex( - PLAYER_REGEX, webpage, 'manifest_url') + return { + 'id': video_id, + 'display_id': display_id, + 'url': video_url, + 'title': title, + } - partner_id = self._search_regex( - r'/p(?:artner_id)?/(\d+)', manifest_url, 'partner id', - default='1670711') + embed_url = KalturaIE._extract_url(start_page) + if embed_url: + embed_url = smuggle_url(embed_url, {'source_url': url}) + ie_key = 'Kaltura' + else: + PLAYER_REGEX = r'', + start_page, 'xml filename', default=None) + if not xml_name: + info = self._parse_html5_media_entries(url, start_page, video_id)[0] + info.update({ + 'title': remove_start(self._search_regex( + r'>Session Name:\s*<.*?>\s*(.+?)', start_page, + 'title', default=None) or self._og_search_title( + start_page, default=None), 'GDC Vault - '), + 'id': video_id, + 'display_id': display_id, + }) + return info + embed_url = '%s/xml/%s' % (xml_root, xml_name) + ie_key = 'DigitallySpeaking' return { '_type': 'url_transparent', - 'url': 'kaltura:%s:%s' % (partner_id, kaltura_id), - 'ie_key': KalturaIE.ie_key(), 'id': video_id, 'display_id': display_id, - 'title': title, + 'url': embed_url, + 'ie_key': ie_key, } diff --git a/yt_dlp/extractor/gedidigital.py b/yt_dlp/extractor/gedidigital.py new file mode 100644 index 000000000..ef9e03e67 --- /dev/null +++ b/yt_dlp/extractor/gedidigital.py @@ -0,0 +1,210 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + base_url, + determine_ext, + int_or_none, + url_basename, + urljoin, +) + + +class GediDigitalIE(InfoExtractor): + _VALID_URL = r'''(?x)(?P(?:https?:)//video\. + (?: + (?: + (?:espresso\.)?repubblica + |lastampa + |ilsecoloxix + |huffingtonpost + )| + (?: + iltirreno + |messaggeroveneto + |ilpiccolo + |gazzettadimantova + |mattinopadova + |laprovinciapavese + |tribunatreviso + |nuovavenezia + |gazzettadimodena + |lanuovaferrara + |corrierealpi + |lasentinella + )\.gelocal + )\.it(?:/[^/]+){2,4}/(?P\d+))(?:$|[?&].*)''' + _TESTS = [{ + 'url': 'https://video.lastampa.it/politica/il-paradosso-delle-regionali-la-lega-vince-ma-sembra-aver-perso/121559/121683', + 'md5': '84658d7fb9e55a6e57ecc77b73137494', + 'info_dict': { + 'id': '121683', + 'ext': 'mp4', + 'title': 'Il paradosso delle Regionali: ecco perché la Lega vince ma sembra aver perso', + 'description': 'md5:de7f4d6eaaaf36c153b599b10f8ce7ca', + 'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-full-.+?\.jpg$', + 'duration': 125, + }, + }, { + 'url': 'https://video.huffingtonpost.it/embed/politica/cotticelli-non-so-cosa-mi-sia-successo-sto-cercando-di-capire-se-ho-avuto-un-malore/29312/29276?responsive=true&el=video971040871621586700', + 'only_matching': True, + }, { + 'url': 'https://video.espresso.repubblica.it/embed/tutti-i-video/01-ted-villa/14772/14870&width=640&height=360', + 'only_matching': True, + }, { + 'url': 'https://video.repubblica.it/motori/record-della-pista-a-spa-francorchamps-la-pagani-huayra-roadster-bc-stupisce/367415/367963', + 'only_matching': True, + }, { + 'url': 'https://video.ilsecoloxix.it/sport/cassani-e-i-brividi-azzurri-ai-mondiali-di-imola-qui-mi-sono-innamorato-del-ciclismo-da-ragazzino-incredibile-tornarci-da-ct/66184/66267', + 'only_matching': True, + }, { + 'url': 'https://video.iltirreno.gelocal.it/sport/dentro-la-notizia-ferrari-cosa-succede-a-maranello/141059/142723', + 'only_matching': True, + }, { + 'url': 'https://video.messaggeroveneto.gelocal.it/locale/maria-giovanna-elmi-covid-vaccino/138155/139268', + 'only_matching': True, + }, { + 'url': 'https://video.ilpiccolo.gelocal.it/dossier/big-john/dinosauro-big-john-al-via-le-visite-guidate-a-trieste/135226/135751', + 'only_matching': True, + }, { + 'url': 'https://video.gazzettadimantova.gelocal.it/locale/dal-ponte-visconteo-di-valeggio-l-and-8217sos-dei-ristoratori-aprire-anche-a-cena/137310/137818', + 'only_matching': True, + }, { + 'url': 'https://video.mattinopadova.gelocal.it/dossier/coronavirus-in-veneto/covid-a-vo-un-anno-dopo-un-cuore-tricolore-per-non-dimenticare/138402/138964', + 'only_matching': True, + }, { + 'url': 'https://video.laprovinciapavese.gelocal.it/locale/mede-zona-rossa-via-alle-vaccinazioni-per-gli-over-80/137545/138120', + 'only_matching': True, + }, { + 'url': 'https://video.tribunatreviso.gelocal.it/dossier/coronavirus-in-veneto/ecco-le-prima-vaccinazioni-di-massa-nella-marca/134485/135024', + 'only_matching': True, + }, { + 'url': 'https://video.nuovavenezia.gelocal.it/locale/camion-troppo-alto-per-il-ponte-ferroviario-perde-il-carico/135734/136266', + 'only_matching': True, + }, { + 'url': 'https://video.gazzettadimodena.gelocal.it/locale/modena-scoperta-la-proteina-che-predice-il-livello-di-gravita-del-covid/139109/139796', + 'only_matching': True, + }, { + 'url': 'https://video.lanuovaferrara.gelocal.it/locale/due-bombole-di-gpl-aperte-e-abbandonate-i-vigili-bruciano-il-gas/134391/134957', + 'only_matching': True, + }, { + 'url': 'https://video.corrierealpi.gelocal.it/dossier/cortina-2021-i-mondiali-di-sci-alpino/mondiali-di-sci-il-timelapse-sulla-splendida-olympia/133760/134331', + 'only_matching': True, + }, { + 'url': 'https://video.lasentinella.gelocal.it/locale/vestigne-centra-un-auto-e-si-ribalta/138931/139466', + 'only_matching': True, + }, { + 'url': 'https://video.espresso.repubblica.it/tutti-i-video/01-ted-villa/14772', + 'only_matching': True, + }] + + @staticmethod + def _sanitize_urls(urls): + # add protocol if missing + for i, e in enumerate(urls): + if e.startswith('//'): + urls[i] = 'https:%s' % e + # clean iframes urls + for i, e in enumerate(urls): + urls[i] = urljoin(base_url(e), url_basename(e)) + return urls + + @staticmethod + def _extract_urls(webpage): + entries = [ + mobj.group('eurl') + for mobj in re.finditer(r'''(?x) + (?: + data-frame-src=| + %s)\1''' % GediDigitalIE._VALID_URL, webpage)] + return GediDigitalIE._sanitize_urls(entries) + + @staticmethod + def _extract_url(webpage): + urls = GediDigitalIE._extract_urls(webpage) + return urls[0] if urls else None + + @staticmethod + def _clean_formats(formats): + format_urls = set() + clean_formats = [] + for f in formats: + if f['url'] not in format_urls: + if f.get('audio_ext') != 'none' and not f.get('acodec'): + continue + format_urls.add(f['url']) + clean_formats.append(f) + formats[:] = clean_formats + + def _real_extract(self, url): + video_id = self._match_id(url) + url = re.match(self._VALID_URL, url).group('url') + webpage = self._download_webpage(url, video_id) + title = self._html_search_meta( + ['twitter:title', 'og:title'], webpage, fatal=True) + player_data = re.findall( + r"PlayerFactory\.setParam\('(?Pformat|param)',\s*'(?P[^']+)',\s*'(?P[^']+)'\);", + webpage) + + formats = [] + duration = thumb = None + for t, n, v in player_data: + if t == 'format': + if n in ('video-hds-vod-ec', 'video-hls-vod-ec', 'video-viralize', 'video-youtube-pfp'): + continue + elif n.endswith('-vod-ak'): + formats.extend(self._extract_akamai_formats( + v, video_id, {'http': 'media.gedidigital.it'})) + else: + ext = determine_ext(v) + if ext == 'm3u8': + formats.extend(self._extract_m3u8_formats( + v, video_id, 'mp4', 'm3u8_native', m3u8_id=n, fatal=False)) + continue + f = { + 'format_id': n, + 'url': v, + } + if ext == 'mp3': + abr = int_or_none(self._search_regex( + r'-mp3-audio-(\d+)', v, 'abr', default=None)) + f.update({ + 'abr': abr, + 'tbr': abr, + 'acodec': ext, + 'vcodec': 'none' + }) + else: + mobj = re.match(r'^video-rrtv-(\d+)(?:-(\d+))?$', n) + if mobj: + f.update({ + 'height': int(mobj.group(1)), + 'vbr': int_or_none(mobj.group(2)), + }) + if not f.get('vbr'): + f['vbr'] = int_or_none(self._search_regex( + r'-video-rrtv-(\d+)', v, 'abr', default=None)) + formats.append(f) + elif t == 'param': + if n in ['image_full', 'image']: + thumb = v + elif n == 'videoDuration': + duration = int_or_none(v) + + self._clean_formats(formats) + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'description': self._html_search_meta( + ['twitter:description', 'og:description', 'description'], webpage), + 'thumbnail': thumb or self._og_search_thumbnail(webpage), + 'formats': formats, + 'duration': duration, + } diff --git a/youtube_dlc/extractor/generic.py b/yt_dlp/extractor/generic.py similarity index 92% rename from youtube_dlc/extractor/generic.py rename to yt_dlp/extractor/generic.py index db4d3a933..7e0598e58 100644 --- a/youtube_dlc/extractor/generic.py +++ b/yt_dlp/extractor/generic.py @@ -20,19 +20,24 @@ from ..utils import ( ExtractorError, float_or_none, HEADRequest, + int_or_none, is_html, js_to_json, KNOWN_EXTENSIONS, merge_dicts, mimetype2ext, orderedSet, + parse_duration, sanitized_Request, smuggle_url, unescapeHTML, - unified_strdate, + unified_timestamp, unsmuggle_url, UnsupportedError, + url_or_none, + xpath_attr, xpath_text, + xpath_with_ns, ) from .commonprotocols import RtmpIE from .brightcove import ( @@ -48,7 +53,6 @@ from .ooyala import OoyalaIE from .rutv import RUTVIE from .tvc import TVCIE from .sportbox import SportBoxIE -from .smotri import SmotriIE from .myvi import MyviIE from .condenast import CondeNastIE from .udn import UDNEmbedIE @@ -63,7 +67,10 @@ from .tube8 import Tube8IE from .mofosex import MofosexEmbedIE from .spankwire import SpankwireIE from .youporn import YouPornIE -from .vimeo import VimeoIE +from .vimeo import ( + VimeoIE, + VHXEmbedIE, +) from .dailymotion import DailymotionIE from .dailymail import DailyMailIE from .onionstudios import OnionStudiosIE @@ -77,7 +84,6 @@ from .jwplatform import JWPlatformIE from .digiteka import DigitekaIE from .arkena import ArkenaIE from .instagram import InstagramIE -from .liveleak import LiveLeakIE from .threeqsdn import ThreeQSDNIE from .theplatform import ThePlatformIE from .kaltura import KalturaIE @@ -119,7 +125,16 @@ from .viqeo import ViqeoIE from .expressen import ExpressenIE from .zype import ZypeIE from .odnoklassniki import OdnoklassnikiIE +from .vk import VKIE from .kinja import KinjaEmbedIE +from .gedidigital import GediDigitalIE +from .rcs import RCSEmbedsIE +from .bitchute import BitChuteIE +from .rumble import RumbleEmbedIE +from .arcpublishing import ArcPublishingIE +from .medialaan import MedialaanIE +from .simplecast import SimplecastIE +from .wimtv import WimTVIE class GenericIE(InfoExtractor): @@ -198,11 +213,46 @@ class GenericIE(InfoExtractor): { 'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml', 'info_dict': { - 'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624', - 'ext': 'm4v', - 'upload_date': '20150228', - 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624', - } + 'id': 'http://podcastfeeds.nbcnews.com/nbcnews/video/podcast/MSNBC-MADDOW-NETCAST-M4V.xml', + 'title': 'MSNBC Rachel Maddow (video)', + 'description': 're:.*her unique approach to storytelling.*', + }, + 'playlist': [{ + 'info_dict': { + 'ext': 'mov', + 'id': 'pdv_maddow_netcast_mov-12-03-2020-223726', + 'title': 'MSNBC Rachel Maddow (video) - 12-03-2020-223726', + 'description': 're:.*her unique approach to storytelling.*', + 'upload_date': '20201204', + }, + }], + }, + # RSS feed with item with description and thumbnails + { + 'url': 'https://anchor.fm/s/dd00e14/podcast/rss', + 'info_dict': { + 'id': 'https://anchor.fm/s/dd00e14/podcast/rss', + 'title': 're:.*100% Hydrogen.*', + 'description': 're:.*In this episode.*', + }, + 'playlist': [{ + 'info_dict': { + 'ext': 'm4a', + 'id': 'c1c879525ce2cb640b344507e682c36d', + 'title': 're:Hydrogen!', + 'description': 're:.*In this episode we are going.*', + 'timestamp': 1567977776, + 'upload_date': '20190908', + 'duration': 459, + 'thumbnail': r're:^https?://.*\.jpg$', + 'episode_number': 1, + 'season_number': 1, + 'age_limit': 0, + }, + }], + 'params': { + 'skip_download': True, + }, }, # RSS feed with enclosures and unsupported link URLs { @@ -1581,31 +1631,6 @@ class GenericIE(InfoExtractor): 'upload_date': '20160409', }, }, - # LiveLeak embed - { - 'url': 'http://www.wykop.pl/link/3088787/', - 'md5': '7619da8c820e835bef21a1efa2a0fc71', - 'info_dict': { - 'id': '874_1459135191', - 'ext': 'mp4', - 'title': 'Man shows poor quality of new apartment building', - 'description': 'The wall is like a sand pile.', - 'uploader': 'Lake8737', - }, - 'add_ie': [LiveLeakIE.ie_key()], - }, - # Another LiveLeak embed pattern (#13336) - { - 'url': 'https://milo.yiannopoulos.net/2017/06/concealed-carry-robbery/', - 'info_dict': { - 'id': '2eb_1496309988', - 'ext': 'mp4', - 'title': 'Thief robs place where everyone was armed', - 'description': 'md5:694d73ee79e535953cf2488562288eee', - 'uploader': 'brazilwtf', - }, - 'add_ie': [LiveLeakIE.ie_key()], - }, # Duplicated embedded video URLs { 'url': 'http://www.hudl.com/athlete/2538180/highlights/149298443', @@ -1948,7 +1973,7 @@ class GenericIE(InfoExtractor): }, { # vshare embed - 'url': 'https://youtube-dlc-demo.neocities.org/vshare.html', + 'url': 'https://youtube-dl-demo.neocities.org/vshare.html', 'md5': '17b39f55b5497ae8b59f5fbce8e35886', 'info_dict': { 'id': '0f64ce6', @@ -1983,22 +2008,6 @@ class GenericIE(InfoExtractor): }, 'add_ie': [SpringboardPlatformIE.ie_key()], }, - { - 'url': 'https://www.youtube.com/shared?ci=1nEzmT-M4fU', - 'info_dict': { - 'id': 'uPDB5I9wfp8', - 'ext': 'webm', - 'title': 'Pocoyo: 90 minutos de episódios completos Português para crianças - PARTE 3', - 'description': 'md5:d9e4d9346a2dfff4c7dc4c8cec0f546d', - 'upload_date': '20160219', - 'uploader': 'Pocoyo - Português (BR)', - 'uploader_id': 'PocoyoBrazil', - }, - 'add_ie': [YoutubeIE.ie_key()], - 'params': { - 'skip_download': True, - }, - }, { 'url': 'https://www.yapfiles.ru/show/1872528/690b05d3054d2dbe1e69523aa21bb3b1.mp4.html', 'info_dict': { @@ -2103,23 +2112,23 @@ class GenericIE(InfoExtractor): 'skip_download': True, }, }, - { - # Zype embed - 'url': 'https://www.cookscountry.com/episode/554-smoky-barbecue-favorites', - 'info_dict': { - 'id': '5b400b834b32992a310622b9', - 'ext': 'mp4', - 'title': 'Smoky Barbecue Favorites', - 'thumbnail': r're:^https?://.*\.jpe?g', - 'description': 'md5:5ff01e76316bd8d46508af26dc86023b', - 'upload_date': '20170909', - 'timestamp': 1504915200, - }, - 'add_ie': [ZypeIE.ie_key()], - 'params': { - 'skip_download': True, - }, - }, + # { + # # Zype embed + # 'url': 'https://www.cookscountry.com/episode/554-smoky-barbecue-favorites', + # 'info_dict': { + # 'id': '5b400b834b32992a310622b9', + # 'ext': 'mp4', + # 'title': 'Smoky Barbecue Favorites', + # 'thumbnail': r're:^https?://.*\.jpe?g', + # 'description': 'md5:5ff01e76316bd8d46508af26dc86023b', + # 'upload_date': '20170909', + # 'timestamp': 1504915200, + # }, + # 'add_ie': [ZypeIE.ie_key()], + # 'params': { + # 'skip_download': True, + # }, + # }, { # videojs embed 'url': 'https://video.sibnet.ru/shell.php?videoid=3422904', @@ -2168,7 +2177,68 @@ class GenericIE(InfoExtractor): # 'params': { # 'force_generic_extractor': True, # }, - # } + # }, + { + # VHX Embed + 'url': 'https://demo.vhx.tv/category-c/videos/file-example-mp4-480-1-5mg-copy', + 'info_dict': { + 'id': '858208', + 'ext': 'mp4', + 'title': 'Untitled', + 'uploader_id': 'user80538407', + 'uploader': 'OTT Videos', + }, + }, + { + # ArcPublishing PoWa video player + 'url': 'https://www.adn.com/politics/2020/11/02/video-senate-candidates-campaign-in-anchorage-on-eve-of-election-day/', + 'md5': 'b03b2fac8680e1e5a7cc81a5c27e71b3', + 'info_dict': { + 'id': '8c99cb6e-b29c-4bc9-9173-7bf9979225ab', + 'ext': 'mp4', + 'title': 'Senate candidates wave to voters on Anchorage streets', + 'description': 'md5:91f51a6511f090617353dc720318b20e', + 'timestamp': 1604378735, + 'upload_date': '20201103', + 'duration': 1581, + }, + }, + { + # MyChannels SDK embed + # https://www.24kitchen.nl/populair/deskundige-dit-waarom-sommigen-gevoelig-zijn-voor-voedselallergieen + 'url': 'https://www.demorgen.be/nieuws/burgemeester-rotterdam-richt-zich-in-videoboodschap-tot-relschoppers-voelt-het-goed~b0bcfd741/', + 'md5': '90c0699c37006ef18e198c032d81739c', + 'info_dict': { + 'id': '194165', + 'ext': 'mp4', + 'title': 'Burgemeester Aboutaleb spreekt relschoppers toe', + 'timestamp': 1611740340, + 'upload_date': '20210127', + 'duration': 159, + }, + }, + { + # Simplecast player embed + 'url': 'https://www.bio.org/podcast', + 'info_dict': { + 'id': 'podcast', + 'title': 'I AM BIO Podcast | BIO', + }, + 'playlist_mincount': 52, + }, + { + # Sibnet embed (https://help.sibnet.ru/?sibnet_video_embed) + 'url': 'https://phpbb3.x-tk.ru/bbcode-video-sibnet-t24.html', + 'only_matching': True, + }, { + # WimTv embed player + 'url': 'http://www.msmotor.tv/wearefmi-pt-2-2021/', + 'info_dict': { + 'id': 'wearefmi-pt-2-2021', + 'title': '#WEAREFMI – PT.2 – 2021 – MsMotorTV', + }, + 'playlist_count': 1, + }, ] def report_following_redirect(self, new_url): @@ -2180,6 +2250,10 @@ class GenericIE(InfoExtractor): playlist_desc_el = doc.find('./channel/description') playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text + NS_MAP = { + 'itunes': 'http://www.itunes.com/dtds/podcast-1.0.dtd', + } + entries = [] for it in doc.findall('./channel/item'): next_url = None @@ -2195,10 +2269,33 @@ class GenericIE(InfoExtractor): if not next_url: continue + def itunes(key): + return xpath_text( + it, xpath_with_ns('./itunes:%s' % key, NS_MAP), + default=None) + + duration = itunes('duration') + explicit = (itunes('explicit') or '').lower() + if explicit in ('true', 'yes'): + age_limit = 18 + elif explicit in ('false', 'no'): + age_limit = 0 + else: + age_limit = None + entries.append({ '_type': 'url_transparent', 'url': next_url, 'title': it.find('title').text, + 'description': xpath_text(it, 'description', default=None), + 'timestamp': unified_timestamp( + xpath_text(it, 'pubDate', default=None)), + 'duration': int_or_none(duration) or parse_duration(duration), + 'thumbnail': url_or_none(xpath_attr(it, xpath_with_ns('./itunes:image', NS_MAP), 'href')), + 'episode': itunes('title'), + 'episode_number': int_or_none(itunes('episode')), + 'season_number': int_or_none(itunes('season')), + 'age_limit': age_limit, }) return { @@ -2252,29 +2349,29 @@ class GenericIE(InfoExtractor): parsed_url = compat_urlparse.urlparse(url) if not parsed_url.scheme: - default_search = self._downloader.params.get('default_search') + default_search = self.get_param('default_search') if default_search is None: default_search = 'fixup_error' if default_search in ('auto', 'auto_warning', 'fixup_error'): if re.match(r'^[^\s/]+\.[^\s/]+/', url): - self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http') + self.report_warning('The url doesn\'t specify the protocol, trying with http') return self.url_result('http://' + url) elif default_search != 'fixup_error': if default_search == 'auto_warning': if re.match(r'^(?:url|URL)$', url): raise ExtractorError( - 'Invalid URL: %r . Call youtube-dlc like this: youtube-dlc -v "https://www.youtube.com/watch?v=BaW_jenozKc" ' % url, + 'Invalid URL: %r . Call yt-dlp like this: yt-dlp -v "https://www.youtube.com/watch?v=BaW_jenozKc" ' % url, expected=True) else: - self._downloader.report_warning( + self.report_warning( 'Falling back to youtube search for %s . Set --default-search "auto" to suppress this warning.' % url) return self.url_result('ytsearch:' + url) if default_search in ('error', 'fixup_error'): raise ExtractorError( '%r is not a valid URL. ' - 'Set --default-search "ytsearch" (or run youtube-dlc "ytsearch:%s" ) to search YouTube' + 'Set --default-search "ytsearch" (or run yt-dlp "ytsearch:%s" ) to search YouTube' % (url, url), expected=True) else: if ':' not in default_search: @@ -2318,7 +2415,7 @@ class GenericIE(InfoExtractor): info_dict = { 'id': video_id, 'title': self._generic_title(url), - 'upload_date': unified_strdate(head_response.headers.get('Last-Modified')) + 'timestamp': unified_timestamp(head_response.headers.get('Last-Modified')) } # Check for direct link to a video @@ -2326,8 +2423,9 @@ class GenericIE(InfoExtractor): m = re.match(r'^(?Paudio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P[^;\s]+)', content_type) if m: format_id = compat_str(m.group('format_id')) + subtitles = {} if format_id.endswith('mpegurl'): - formats = self._extract_m3u8_formats(url, video_id, 'mp4') + formats, subtitles = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4') elif format_id == 'f4m': formats = self._extract_f4m_formats(url, video_id) else: @@ -2339,18 +2437,19 @@ class GenericIE(InfoExtractor): info_dict['direct'] = True self._sort_formats(formats) info_dict['formats'] = formats + info_dict['subtitles'] = subtitles return info_dict - if not self._downloader.params.get('test', False) and not is_intentional: - force = self._downloader.params.get('force_generic_extractor', False) - self._downloader.report_warning( + if not self.get_param('test', False) and not is_intentional: + force = self.get_param('force_generic_extractor', False) + self.report_warning( '%s on generic information extractor.' % ('Forcing' if force else 'Falling back')) if not full_response: request = sanitized_Request(url) # Some webservers may serve compressed content of rather big size (e.g. gzipped flac) # making it impossible to download only chunk of the file (yet we need only 512kB to - # test whether it's HTML or not). According to youtube-dlc default Accept-Encoding + # test whether it's HTML or not). According to yt-dlp default Accept-Encoding # that will always result in downloading the whole file that is not desirable. # Therefore for extraction pass we have to override Accept-Encoding to any in order # to accept raw bytes and being able to download only a chunk. @@ -2363,14 +2462,14 @@ class GenericIE(InfoExtractor): # Is it an M3U playlist? if first_bytes.startswith(b'#EXTM3U'): - info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4') + info_dict['formats'], info_dict['subtitles'] = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4') self._sort_formats(info_dict['formats']) return info_dict # Maybe it's a direct link to a video? # Be careful not to download the whole thing! if not is_html(first_bytes): - self._downloader.report_warning( + self.report_warning( 'URL could be a direct video link, returning it as such.') info_dict.update({ 'direct': True, @@ -2381,15 +2480,21 @@ class GenericIE(InfoExtractor): webpage = self._webpage_read_content( full_response, url, video_id, prefix=first_bytes) + if 'DPG Media Privacy Gate' in webpage: + webpage = self._download_webpage(url, video_id) + self.report_extraction(video_id) # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest? try: - doc = compat_etree_fromstring(webpage.encode('utf-8')) + try: + doc = compat_etree_fromstring(webpage) + except compat_xml_parse_error: + doc = compat_etree_fromstring(webpage.encode('utf-8')) if doc.tag == 'rss': return self._extract_rss(url, video_id, doc) elif doc.tag == 'SmoothStreamingMedia': - info_dict['formats'] = self._parse_ism_formats(doc, url) + info_dict['formats'], info_dict['subtitles'] = self._parse_ism_formats_and_subtitles(doc, url) self._sort_formats(info_dict['formats']) return info_dict elif re.match(r'^(?:{[^}]+})?smil$', doc.tag): @@ -2403,7 +2508,7 @@ class GenericIE(InfoExtractor): xspf_base_url=full_response.geturl()), video_id) elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag): - info_dict['formats'] = self._parse_mpd_formats( + info_dict['formats'], info_dict['subtitles'] = self._parse_mpd_formats_and_subtitles( doc, mpd_base_url=full_response.geturl().rpartition('/')[0], mpd_url=url) @@ -2424,7 +2529,9 @@ class GenericIE(InfoExtractor): # Sometimes embedded video player is hidden behind percent encoding # (e.g. https://github.com/ytdl-org/youtube-dl/issues/2448) # Unescaping the whole page allows to handle those cases in a generic way - webpage = compat_urllib_parse_unquote(webpage) + # FIXME: unescaping the whole page may break URLs, commenting out for now. + # There probably should be a second run of generic extractor on unescaped webpage. + # webpage = compat_urllib_parse_unquote(webpage) # Unescape squarespace embeds to be detected by generic extractor, # see https://github.com/ytdl-org/youtube-dl/issues/21294 @@ -2506,6 +2613,15 @@ class GenericIE(InfoExtractor): if tp_urls: return self.playlist_from_matches(tp_urls, video_id, video_title, ie='ThePlatform') + arc_urls = ArcPublishingIE._extract_urls(webpage) + if arc_urls: + return self.playlist_from_matches(arc_urls, video_id, video_title, ie=ArcPublishingIE.ie_key()) + + mychannels_urls = MedialaanIE._extract_urls(webpage) + if mychannels_urls: + return self.playlist_from_matches( + mychannels_urls, video_id, video_title, ie=MedialaanIE.ie_key()) + # Look for embedded rtl.nl player matches = re.findall( r']+?src="((?:https?:)?//(?:(?:www|static)\.)?rtl\.nl/(?:system/videoplayer/[^"]+(?:video_)?)?embed[^"]+)"', @@ -2517,12 +2633,25 @@ class GenericIE(InfoExtractor): if vimeo_urls: return self.playlist_from_matches(vimeo_urls, video_id, video_title, ie=VimeoIE.ie_key()) + vhx_url = VHXEmbedIE._extract_url(webpage) + if vhx_url: + return self.url_result(vhx_url, VHXEmbedIE.ie_key()) + vid_me_embed_url = self._search_regex( r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]', webpage, 'vid.me embed', default=None) if vid_me_embed_url is not None: return self.url_result(vid_me_embed_url, 'Vidme') + # Invidious Instances + # https://github.com/yt-dlp/yt-dlp/issues/195 + # https://github.com/iv-org/invidious/pull/1730 + youtube_url = self._search_regex( + r']+?src=(["\'])(?Phttps?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage) if mobj is not None: @@ -2678,6 +2812,12 @@ class GenericIE(InfoExtractor): return self.playlist_from_matches( matches, video_id, video_title, getter=unescapeHTML, ie='FunnyOrDie') + # Look for Simplecast embeds + simplecast_urls = SimplecastIE._extract_urls(webpage) + if simplecast_urls: + return self.playlist_from_matches( + simplecast_urls, video_id, video_title) + # Look for BBC iPlayer embed matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage) if matches: @@ -2772,11 +2912,6 @@ class GenericIE(InfoExtractor): if mobj is not None: return self.url_result(mobj.group('url')) - # Look for embedded smotri.com player - smotri_url = SmotriIE._extract_url(webpage) - if smotri_url: - return self.url_result(smotri_url, 'Smotri') - # Look for embedded Myvi.ru player myvi_url = MyviIE._extract_url(webpage) if myvi_url: @@ -2828,7 +2963,7 @@ class GenericIE(InfoExtractor): webpage) if not mobj: mobj = re.search( - r'data-video-link=["\'](?Phttp://m.mlb.com/video/[^"\']+)', + r'data-video-link=["\'](?Phttp://m\.mlb\.com/video/[^"\']+)', webpage) if mobj is not None: return self.url_result(mobj.group('url'), 'MLB') @@ -3043,11 +3178,6 @@ class GenericIE(InfoExtractor): return self.url_result( self._proto_relative_url(instagram_embed_url), InstagramIE.ie_key()) - # Look for LiveLeak embeds - liveleak_urls = LiveLeakIE._extract_urls(webpage) - if liveleak_urls: - return self.playlist_from_matches(liveleak_urls, video_id, video_title) - # Look for 3Q SDN embeds threeqsdn_url = ThreeQSDNIE._extract_url(webpage) if threeqsdn_url: @@ -3212,6 +3342,34 @@ class GenericIE(InfoExtractor): return self.playlist_from_matches( zype_urls, video_id, video_title, ie=ZypeIE.ie_key()) + gedi_urls = GediDigitalIE._extract_urls(webpage) + if gedi_urls: + return self.playlist_from_matches( + gedi_urls, video_id, video_title, ie=GediDigitalIE.ie_key()) + + # Look for RCS media group embeds + rcs_urls = RCSEmbedsIE._extract_urls(webpage) + if rcs_urls: + return self.playlist_from_matches( + rcs_urls, video_id, video_title, ie=RCSEmbedsIE.ie_key()) + + wimtv_urls = WimTVIE._extract_urls(webpage) + if wimtv_urls: + return self.playlist_from_matches( + wimtv_urls, video_id, video_title, ie=WimTVIE.ie_key()) + + bitchute_urls = BitChuteIE._extract_urls(webpage) + if bitchute_urls: + return self.playlist_from_matches( + bitchute_urls, video_id, video_title, ie=BitChuteIE.ie_key()) + + rumble_urls = RumbleEmbedIE._extract_urls(webpage) + if len(rumble_urls) == 1: + return self.url_result(rumble_urls[0], RumbleEmbedIE.ie_key()) + if rumble_urls: + return self.playlist_from_matches( + rumble_urls, video_id, video_title, ie=RumbleEmbedIE.ie_key()) + # Look for HTML5 media entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls') if entries: @@ -3252,6 +3410,7 @@ class GenericIE(InfoExtractor): if not isinstance(sources, list): sources = [sources] formats = [] + subtitles = {} for source in sources: src = source.get('src') if not src or not isinstance(src, compat_str): @@ -3264,21 +3423,29 @@ class GenericIE(InfoExtractor): if src_type == 'video/youtube': return self.url_result(src, YoutubeIE.ie_key()) if src_type == 'application/dash+xml' or ext == 'mpd': - formats.extend(self._extract_mpd_formats( - src, video_id, mpd_id='dash', fatal=False)) + fmts, subs = self._extract_mpd_formats_and_subtitles( + src, video_id, mpd_id='dash', fatal=False) + formats.extend(fmts) + self._merge_subtitles(subs, target=subtitles) elif src_type == 'application/x-mpegurl' or ext == 'm3u8': - formats.extend(self._extract_m3u8_formats( + fmts, subs = self._extract_m3u8_formats_and_subtitles( src, video_id, 'mp4', entry_protocol='m3u8_native', - m3u8_id='hls', fatal=False)) + m3u8_id='hls', fatal=False) + formats.extend(fmts) + self._merge_subtitles(subs, target=subtitles) else: formats.append({ 'url': src, 'ext': (mimetype2ext(src_type) or ext if ext in KNOWN_EXTENSIONS else 'mp4'), + 'http_headers': { + 'Referer': full_response.geturl(), + }, }) - if formats: + if formats or subtitles: self._sort_formats(formats) info_dict['formats'] = formats + info_dict['subtitles'] = subtitles return info_dict # Looking for http://schema.org/VideoObject @@ -3343,7 +3510,7 @@ class GenericIE(InfoExtractor): m_video_type = re.findall(r'%s)\.)?go| - (?Pabc|freeform|disneynow) + (?P + (?:%s\.)?go|fxnow\.fxnetworks| + (?:www\.)?(?:abc|freeform|disneynow) )\.com/ (?: (?:[^/]+/)*(?P[Vv][Dd][Kk][Aa]\w+)| (?:[^/]+/)*(?P[^/?\#]+) ) - ''' % '|'.join(list(_SITE_INFO.keys())) + ''' % r'\.|'.join(list(_SITE_INFO.keys())) _TESTS = [{ 'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643', 'info_dict': { @@ -99,6 +107,31 @@ class GoIE(AdobePassIE): # m3u8 download 'skip_download': True, }, + }, { + 'url': 'https://fxnow.fxnetworks.com/shows/better-things/video/vdka12782841', + 'info_dict': { + 'id': 'VDKA12782841', + 'ext': 'mp4', + 'title': 'First Look: Better Things - Season 2', + 'description': 'md5:fa73584a95761c605d9d54904e35b407', + }, + 'params': { + 'geo_bypass_ip_block': '3.244.239.0/24', + # m3u8 download + 'skip_download': True, + }, + }, { + 'url': 'https://abc.com/shows/modern-family/episode-guide/season-01/101-pilot', + 'info_dict': { + 'id': 'VDKA22600213', + 'ext': 'mp4', + 'title': 'Pilot', + 'description': 'md5:74306df917cfc199d76d061d66bebdb4', + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, }, { 'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding', 'only_matching': True, @@ -116,6 +149,9 @@ class GoIE(AdobePassIE): }, { 'url': 'https://disneynow.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013', 'only_matching': True, + }, { + 'url': 'https://www.freeform.com/shows/cruel-summer/episode-guide/season-01/01-happy-birthday-jeanette-turner', + 'only_matching': True, }] def _extract_videos(self, brand, video_id='-1', show_id='-1'): @@ -126,24 +162,36 @@ class GoIE(AdobePassIE): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - sub_domain = mobj.group('sub_domain') or mobj.group('sub_domain_2') + sub_domain = remove_start(remove_end(mobj.group('sub_domain') or '', '.go'), 'www.') video_id, display_id = mobj.group('id', 'display_id') site_info = self._SITE_INFO.get(sub_domain, {}) brand = site_info.get('brand') if not video_id or not site_info: webpage = self._download_webpage(url, display_id or video_id) - video_id = self._search_regex( - ( - # There may be inner quotes, e.g. data-video-id="'VDKA3609139'" - # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood - r'data-video-id=["\']*(VDKA\w+)', - # https://github.com/ytdl-org/youtube-dl/pull/25216/files - # The following is based on the pull request on the line above. Changed the ABC.com URL to a show available now. - # https://abc.com/shows/the-rookie/episode-guide/season-02/19-the-q-word - r'\bvideoIdCode["\']\s*:\s*["\'](vdka\w+)', - # Deprecated fallback pattern - r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)' - ), webpage, 'video id', default=video_id) + data = self._parse_json( + self._search_regex( + r'["\']__abc_com__["\']\s*\]\s*=\s*({.+?})\s*;', webpage, + 'data', default='{}'), + display_id or video_id, fatal=False) + # https://abc.com/shows/modern-family/episode-guide/season-01/101-pilot + layout = try_get(data, lambda x: x['page']['content']['video']['layout'], dict) + video_id = None + if layout: + video_id = try_get( + layout, + (lambda x: x['videoid'], lambda x: x['video']['id']), + compat_str) + if not video_id: + video_id = self._search_regex( + ( + # There may be inner quotes, e.g. data-video-id="'VDKA3609139'" + # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood + r'data-video-id=["\']*(VDKA\w+)', + # page.analytics.videoIdCode + r'\bvideoIdCode["\']\s*:\s*["\']((?:vdka|VDKA)\w+)', + # https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet + r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)' + ), webpage, 'video id', default=video_id) if not site_info: brand = self._search_regex( (r'data-brand=\s*["\']\s*(\d+)', @@ -219,7 +267,7 @@ class GoIE(AdobePassIE): if re.search(r'(?:/mp4/source/|_source\.mp4)', asset_url): f.update({ 'format_id': ('%s-' % format_id if format_id else '') + 'SOURCE', - 'preference': 1, + 'quality': 1, }) else: mobj = re.search(r'/(\d+)x(\d+)/', asset_url) diff --git a/youtube_dlc/extractor/godtube.py b/yt_dlp/extractor/godtube.py similarity index 100% rename from youtube_dlc/extractor/godtube.py rename to yt_dlp/extractor/godtube.py diff --git a/youtube_dlc/extractor/golem.py b/yt_dlp/extractor/golem.py similarity index 100% rename from youtube_dlc/extractor/golem.py rename to yt_dlp/extractor/golem.py diff --git a/youtube_dlc/extractor/googledrive.py b/yt_dlp/extractor/googledrive.py similarity index 95% rename from youtube_dlc/extractor/googledrive.py rename to yt_dlp/extractor/googledrive.py index fdb15795a..7b5bf280f 100644 --- a/youtube_dlc/extractor/googledrive.py +++ b/yt_dlp/extractor/googledrive.py @@ -7,6 +7,7 @@ from ..compat import compat_parse_qs from ..utils import ( determine_ext, ExtractorError, + get_element_by_class, int_or_none, lowercase_escape, try_get, @@ -237,7 +238,7 @@ class GoogleDriveIE(InfoExtractor): if confirmation_webpage: confirm = self._search_regex( r'confirm=([^&"\']+)', confirmation_webpage, - 'confirmation code', fatal=False) + 'confirmation code', default=None) if confirm: confirmed_source_url = update_url_query(source_url, { 'confirm': confirm, @@ -245,9 +246,14 @@ class GoogleDriveIE(InfoExtractor): urlh = request_source_file(confirmed_source_url, 'confirmed source') if urlh and urlh.headers.get('Content-Disposition'): add_source_format(urlh) + else: + self.report_warning( + get_element_by_class('uc-error-subcaption', confirmation_webpage) + or get_element_by_class('uc-error-caption', confirmation_webpage) + or 'unable to extract confirmation code') if not formats and reason: - raise ExtractorError(reason, expected=True) + self.raise_no_formats(reason, expected=True) self._sort_formats(formats) diff --git a/yt_dlp/extractor/googlepodcasts.py b/yt_dlp/extractor/googlepodcasts.py new file mode 100644 index 000000000..31ad79907 --- /dev/null +++ b/yt_dlp/extractor/googlepodcasts.py @@ -0,0 +1,88 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import json +import re + +from .common import InfoExtractor +from ..utils import ( + clean_podcast_url, + int_or_none, + try_get, + urlencode_postdata, +) + + +class GooglePodcastsBaseIE(InfoExtractor): + _VALID_URL_BASE = r'https?://podcasts\.google\.com/feed/' + + def _batch_execute(self, func_id, video_id, params): + return json.loads(self._download_json( + 'https://podcasts.google.com/_/PodcastsUi/data/batchexecute', + video_id, data=urlencode_postdata({ + 'f.req': json.dumps([[[func_id, json.dumps(params), None, '1']]]), + }), transform_source=lambda x: self._search_regex(r'(?s)(\[.+\])', x, 'data'))[0][2]) + + def _extract_episode(self, episode): + return { + 'id': episode[4][3], + 'title': episode[8], + 'url': clean_podcast_url(episode[13]), + 'thumbnail': episode[2], + 'description': episode[9], + 'creator': try_get(episode, lambda x: x[14]), + 'timestamp': int_or_none(episode[11]), + 'duration': int_or_none(episode[12]), + 'series': episode[1], + } + + +class GooglePodcastsIE(GooglePodcastsBaseIE): + IE_NAME = 'google:podcasts' + _VALID_URL = GooglePodcastsBaseIE._VALID_URL_BASE + r'(?P[^/]+)/episode/(?P[^/?&#]+)' + _TEST = { + 'url': 'https://podcasts.google.com/feed/aHR0cHM6Ly9mZWVkcy5ucHIub3JnLzM0NDA5ODUzOS9wb2RjYXN0LnhtbA/episode/MzBlNWRlN2UtOWE4Yy00ODcwLTk2M2MtM2JlMmUyNmViOTRh', + 'md5': 'fa56b2ee8bd0703e27e42d4b104c4766', + 'info_dict': { + 'id': '30e5de7e-9a8c-4870-963c-3be2e26eb94a', + 'ext': 'mp3', + 'title': 'WWDTM New Year 2021', + 'description': 'We say goodbye to 2020 with Christine Baranksi, Doug Jones, Jonna Mendez, and Kellee Edwards.', + 'upload_date': '20210102', + 'timestamp': 1609606800, + 'duration': 2901, + 'series': "Wait Wait... Don't Tell Me!", + } + } + + def _real_extract(self, url): + b64_feed_url, b64_guid = re.match(self._VALID_URL, url).groups() + episode = self._batch_execute( + 'oNjqVe', b64_guid, [b64_feed_url, b64_guid])[1] + return self._extract_episode(episode) + + +class GooglePodcastsFeedIE(GooglePodcastsBaseIE): + IE_NAME = 'google:podcasts:feed' + _VALID_URL = GooglePodcastsBaseIE._VALID_URL_BASE + r'(?P[^/?&#]+)/?(?:[?#&]|$)' + _TEST = { + 'url': 'https://podcasts.google.com/feed/aHR0cHM6Ly9mZWVkcy5ucHIub3JnLzM0NDA5ODUzOS9wb2RjYXN0LnhtbA', + 'info_dict': { + 'title': "Wait Wait... Don't Tell Me!", + 'description': "NPR's weekly current events quiz. Have a laugh and test your news knowledge while figuring out what's real and what we've made up.", + }, + 'playlist_mincount': 20, + } + + def _real_extract(self, url): + b64_feed_url = self._match_id(url) + data = self._batch_execute('ncqJEe', b64_feed_url, [b64_feed_url]) + + entries = [] + for episode in (try_get(data, lambda x: x[1][0]) or []): + entries.append(self._extract_episode(episode)) + + feed = try_get(data, lambda x: x[3]) or [] + return self.playlist_result( + entries, playlist_title=try_get(feed, lambda x: x[0]), + playlist_description=try_get(feed, lambda x: x[2])) diff --git a/youtube_dlc/extractor/googlesearch.py b/yt_dlp/extractor/googlesearch.py similarity index 100% rename from youtube_dlc/extractor/googlesearch.py rename to yt_dlp/extractor/googlesearch.py diff --git a/youtube_dlc/extractor/goshgay.py b/yt_dlp/extractor/goshgay.py similarity index 100% rename from youtube_dlc/extractor/goshgay.py rename to yt_dlp/extractor/goshgay.py diff --git a/youtube_dlc/extractor/gputechconf.py b/yt_dlp/extractor/gputechconf.py similarity index 100% rename from youtube_dlc/extractor/gputechconf.py rename to yt_dlp/extractor/gputechconf.py diff --git a/youtube_dlc/extractor/groupon.py b/yt_dlp/extractor/groupon.py similarity index 100% rename from youtube_dlc/extractor/groupon.py rename to yt_dlp/extractor/groupon.py diff --git a/youtube_dlc/extractor/hbo.py b/yt_dlp/extractor/hbo.py similarity index 100% rename from youtube_dlc/extractor/hbo.py rename to yt_dlp/extractor/hbo.py diff --git a/youtube_dlc/extractor/hearthisat.py b/yt_dlp/extractor/hearthisat.py similarity index 98% rename from youtube_dlc/extractor/hearthisat.py rename to yt_dlp/extractor/hearthisat.py index 18c252012..aeb216c71 100644 --- a/youtube_dlc/extractor/hearthisat.py +++ b/yt_dlp/extractor/hearthisat.py @@ -115,7 +115,7 @@ class HearThisAtIE(InfoExtractor): 'vcodec': 'none', 'ext': ext, 'url': download_url, - 'preference': 2, # Usually better quality + 'quality': 2, # Usually better quality }) self._sort_formats(formats) diff --git a/youtube_dlc/extractor/heise.py b/yt_dlp/extractor/heise.py similarity index 100% rename from youtube_dlc/extractor/heise.py rename to yt_dlp/extractor/heise.py diff --git a/youtube_dlc/extractor/hellporno.py b/yt_dlp/extractor/hellporno.py similarity index 100% rename from youtube_dlc/extractor/hellporno.py rename to yt_dlp/extractor/hellporno.py diff --git a/youtube_dlc/extractor/helsinki.py b/yt_dlp/extractor/helsinki.py similarity index 100% rename from youtube_dlc/extractor/helsinki.py rename to yt_dlp/extractor/helsinki.py diff --git a/youtube_dlc/extractor/hentaistigma.py b/yt_dlp/extractor/hentaistigma.py similarity index 100% rename from youtube_dlc/extractor/hentaistigma.py rename to yt_dlp/extractor/hentaistigma.py diff --git a/youtube_dlc/extractor/hgtv.py b/yt_dlp/extractor/hgtv.py similarity index 100% rename from youtube_dlc/extractor/hgtv.py rename to yt_dlp/extractor/hgtv.py diff --git a/youtube_dlc/extractor/hidive.py b/yt_dlp/extractor/hidive.py similarity index 100% rename from youtube_dlc/extractor/hidive.py rename to yt_dlp/extractor/hidive.py diff --git a/youtube_dlc/extractor/historicfilms.py b/yt_dlp/extractor/historicfilms.py similarity index 100% rename from youtube_dlc/extractor/historicfilms.py rename to yt_dlp/extractor/historicfilms.py diff --git a/youtube_dlc/extractor/hitbox.py b/yt_dlp/extractor/hitbox.py similarity index 100% rename from youtube_dlc/extractor/hitbox.py rename to yt_dlp/extractor/hitbox.py diff --git a/youtube_dlc/extractor/hitrecord.py b/yt_dlp/extractor/hitrecord.py similarity index 100% rename from youtube_dlc/extractor/hitrecord.py rename to yt_dlp/extractor/hitrecord.py diff --git a/youtube_dlc/extractor/hketv.py b/yt_dlp/extractor/hketv.py similarity index 100% rename from youtube_dlc/extractor/hketv.py rename to yt_dlp/extractor/hketv.py diff --git a/youtube_dlc/extractor/hornbunny.py b/yt_dlp/extractor/hornbunny.py similarity index 100% rename from youtube_dlc/extractor/hornbunny.py rename to yt_dlp/extractor/hornbunny.py diff --git a/youtube_dlc/extractor/hotnewhiphop.py b/yt_dlp/extractor/hotnewhiphop.py similarity index 100% rename from youtube_dlc/extractor/hotnewhiphop.py rename to yt_dlp/extractor/hotnewhiphop.py diff --git a/youtube_dlc/extractor/hotstar.py b/yt_dlp/extractor/hotstar.py similarity index 76% rename from youtube_dlc/extractor/hotstar.py rename to yt_dlp/extractor/hotstar.py index 1fb4d2d41..fc7756d5f 100644 --- a/youtube_dlc/extractor/hotstar.py +++ b/yt_dlp/extractor/hotstar.py @@ -27,8 +27,8 @@ from ..utils import ( class HotStarBaseIE(InfoExtractor): _AKAMAI_ENCRYPTION_KEY = b'\x05\xfc\x1a\x01\xca\xc9\x4b\xc4\x12\xfc\x53\x12\x07\x75\xf9\xee' - def _call_api_impl(self, path, video_id, query): - st = int(time.time()) + def _call_api_impl(self, path, video_id, query, st=None): + st = int_or_none(st) or int(time.time()) exp = st + 6000 auth = 'st=%d~exp=%d~acl=/*' % (st, exp) auth += '~hmac=' + hmac.new(self._AKAMAI_ENCRYPTION_KEY, auth.encode(), hashlib.sha256).hexdigest() @@ -75,9 +75,9 @@ class HotStarBaseIE(InfoExtractor): 'tas': 10000, }) - def _call_api_v2(self, path, video_id): + def _call_api_v2(self, path, video_id, st=None): return self._call_api_impl( - '%s/content/%s' % (path, video_id), video_id, { + '%s/content/%s' % (path, video_id), video_id, st=st, query={ 'desired-config': 'audio_channel:stereo|dynamic_range:sdr|encryption:plain|ladder:tv|package:dash|resolution:hd|subs-tag:HotstarVIP|video_codec:vp9', 'device-id': compat_str(uuid.uuid4()), 'os-name': 'Windows', @@ -87,7 +87,14 @@ class HotStarBaseIE(InfoExtractor): class HotStarIE(HotStarBaseIE): IE_NAME = 'hotstar' - _VALID_URL = r'https?://(?:www\.)?hotstar\.com/.*(?P\d{10})' + _VALID_URL = r'''(?x) + https?://(?:www\.)?hotstar\.com(?:/in)?/(?!in/) + (?: + tv/(?:[^/?#]+/){3}| + (?!tv/)[^?#]+/ + )? + (?P\d{10}) + ''' _TESTS = [{ # contentData 'url': 'https://www.hotstar.com/can-you-not-spread-rumours/1000076273', @@ -124,7 +131,8 @@ class HotStarIE(HotStarBaseIE): def _real_extract(self, url): video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) + webpage, urlh = self._download_webpage_handle(url, video_id) + st = urlh.headers.get('x-origin-date') app_state = self._parse_json(self._search_regex( r'', webpage, 'app state'), video_id) @@ -141,14 +149,14 @@ class HotStarIE(HotStarBaseIE): title = video_data['title'] - if video_data.get('drmProtected'): + if not self.get_param('allow_unplayable_formats') and video_data.get('drmProtected'): raise ExtractorError('This video is DRM protected.', expected=True) headers = {'Referer': url} formats = [] geo_restricted = False # change to v2 in the future - playback_sets = self._call_api_v2('play/v1/playback', video_id)['playBackSets'] + playback_sets = self._call_api_v2('play/v1/playback', video_id, st=st)['playBackSets'] for playback_set in playback_sets: if not isinstance(playback_set, dict): continue @@ -184,7 +192,7 @@ class HotStarIE(HotStarBaseIE): geo_restricted = True continue if not formats and geo_restricted: - self.raise_geo_restricted(countries=['IN']) + self.raise_geo_restricted(countries=['IN'], metadata_available=True) self._sort_formats(formats) for f in formats: @@ -235,3 +243,47 @@ class HotStarPlaylistIE(HotStarBaseIE): if video.get('contentId')] return self.playlist_result(entries, playlist_id) + + +class HotStarSeriesIE(HotStarBaseIE): + IE_NAME = 'hotstar:series' + _VALID_URL = r'(?:https?://)(?:www\.)?hotstar\.com(?:/in)?/tv/[^/]+/(?P\d+)' + _TESTS = [{ + 'url': 'https://www.hotstar.com/in/tv/radhakrishn/1260000646', + 'info_dict': { + 'id': '1260000646', + }, + 'playlist_mincount': 690, + }, { + 'url': 'https://www.hotstar.com/tv/dancee-/1260050431', + 'info_dict': { + 'id': '1260050431', + }, + 'playlist_mincount': 43, + }, { + 'url': 'https://www.hotstar.com/in/tv/mahabharat/435/', + 'info_dict': { + 'id': '435', + }, + 'playlist_mincount': 269, + }] + + def _real_extract(self, url): + series_id = self._match_id(url) + headers = { + 'x-country-code': 'IN', + 'x-platform-code': 'PCTV', + } + detail_json = self._download_json('https://api.hotstar.com/o/v1/show/detail?contentId=' + series_id, + video_id=series_id, headers=headers) + id = compat_str(try_get(detail_json, lambda x: x['body']['results']['item']['id'], int)) + item_json = self._download_json('https://api.hotstar.com/o/v1/tray/g/1/items?etid=0&tao=0&tas=10000&eid=' + id, + video_id=series_id, headers=headers) + entries = [ + self.url_result( + 'https://www.hotstar.com/%d' % video['contentId'], + ie=HotStarIE.ie_key(), video_id=video['contentId']) + for video in item_json['body']['results']['items'] + if video.get('contentId')] + + return self.playlist_result(entries, series_id) diff --git a/youtube_dlc/extractor/howcast.py b/yt_dlp/extractor/howcast.py similarity index 100% rename from youtube_dlc/extractor/howcast.py rename to yt_dlp/extractor/howcast.py diff --git a/youtube_dlc/extractor/howstuffworks.py b/yt_dlp/extractor/howstuffworks.py similarity index 100% rename from youtube_dlc/extractor/howstuffworks.py rename to yt_dlp/extractor/howstuffworks.py diff --git a/youtube_dlc/extractor/hrfensehen.py b/yt_dlp/extractor/hrfensehen.py similarity index 98% rename from youtube_dlc/extractor/hrfensehen.py rename to yt_dlp/extractor/hrfensehen.py index 805345e69..2a994d471 100644 --- a/youtube_dlc/extractor/hrfensehen.py +++ b/yt_dlp/extractor/hrfensehen.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals import json import re -from youtube_dlc.utils import int_or_none, unified_timestamp, unescapeHTML +from ..utils import int_or_none, unified_timestamp, unescapeHTML from .common import InfoExtractor diff --git a/youtube_dlc/extractor/hrti.py b/yt_dlp/extractor/hrti.py similarity index 100% rename from youtube_dlc/extractor/hrti.py rename to yt_dlp/extractor/hrti.py diff --git a/youtube_dlc/extractor/huajiao.py b/yt_dlp/extractor/huajiao.py similarity index 100% rename from youtube_dlc/extractor/huajiao.py rename to yt_dlp/extractor/huajiao.py diff --git a/youtube_dlc/extractor/huffpost.py b/yt_dlp/extractor/huffpost.py similarity index 100% rename from youtube_dlc/extractor/huffpost.py rename to yt_dlp/extractor/huffpost.py diff --git a/youtube_dlc/extractor/hungama.py b/yt_dlp/extractor/hungama.py similarity index 100% rename from youtube_dlc/extractor/hungama.py rename to yt_dlp/extractor/hungama.py diff --git a/youtube_dlc/extractor/hypem.py b/yt_dlp/extractor/hypem.py similarity index 100% rename from youtube_dlc/extractor/hypem.py rename to yt_dlp/extractor/hypem.py diff --git a/yt_dlp/extractor/ign.py b/yt_dlp/extractor/ign.py new file mode 100644 index 000000000..c826eb3ba --- /dev/null +++ b/yt_dlp/extractor/ign.py @@ -0,0 +1,257 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..compat import ( + compat_parse_qs, + compat_urllib_parse_urlparse, +) +from ..utils import ( + HEADRequest, + determine_ext, + int_or_none, + parse_iso8601, + strip_or_none, + try_get, +) + + +class IGNBaseIE(InfoExtractor): + def _call_api(self, slug): + return self._download_json( + 'http://apis.ign.com/{0}/v3/{0}s/slug/{1}'.format(self._PAGE_TYPE, slug), slug) + + +class IGNIE(IGNBaseIE): + """ + Extractor for some of the IGN sites, like www.ign.com, es.ign.com de.ign.com. + Some videos of it.ign.com are also supported + """ + + _VALID_URL = r'https?://(?:.+?\.ign|www\.pcmag)\.com/videos/(?:\d{4}/\d{2}/\d{2}/)?(?P[^/?&#]+)' + IE_NAME = 'ign.com' + _PAGE_TYPE = 'video' + + _TESTS = [{ + 'url': 'http://www.ign.com/videos/2013/06/05/the-last-of-us-review', + 'md5': 'd2e1586d9987d40fad7867bf96a018ea', + 'info_dict': { + 'id': '8f862beef863986b2785559b9e1aa599', + 'ext': 'mp4', + 'title': 'The Last of Us Review', + 'description': 'md5:c8946d4260a4d43a00d5ae8ed998870c', + 'timestamp': 1370440800, + 'upload_date': '20130605', + 'tags': 'count:9', + } + }, { + 'url': 'http://www.pcmag.com/videos/2015/01/06/010615-whats-new-now-is-gogo-snooping-on-your-data', + 'md5': 'f1581a6fe8c5121be5b807684aeac3f6', + 'info_dict': { + 'id': 'ee10d774b508c9b8ec07e763b9125b91', + 'ext': 'mp4', + 'title': 'What\'s New Now: Is GoGo Snooping on Your Data?', + 'description': 'md5:817a20299de610bd56f13175386da6fa', + 'timestamp': 1420571160, + 'upload_date': '20150106', + 'tags': 'count:4', + } + }, { + 'url': 'https://www.ign.com/videos/is-a-resident-evil-4-remake-on-the-way-ign-daily-fix', + 'only_matching': True, + }] + + def _real_extract(self, url): + display_id = self._match_id(url) + video = self._call_api(display_id) + video_id = video['videoId'] + metadata = video['metadata'] + title = metadata.get('longTitle') or metadata.get('title') or metadata['name'] + + formats = [] + refs = video.get('refs') or {} + + m3u8_url = refs.get('m3uUrl') + if m3u8_url: + formats.extend(self._extract_m3u8_formats( + m3u8_url, video_id, 'mp4', 'm3u8_native', + m3u8_id='hls', fatal=False)) + + f4m_url = refs.get('f4mUrl') + if f4m_url: + formats.extend(self._extract_f4m_formats( + f4m_url, video_id, f4m_id='hds', fatal=False)) + + for asset in (video.get('assets') or []): + asset_url = asset.get('url') + if not asset_url: + continue + formats.append({ + 'url': asset_url, + 'tbr': int_or_none(asset.get('bitrate'), 1000), + 'fps': int_or_none(asset.get('frame_rate')), + 'height': int_or_none(asset.get('height')), + 'width': int_or_none(asset.get('width')), + }) + + mezzanine_url = try_get(video, lambda x: x['system']['mezzanineUrl']) + if mezzanine_url: + formats.append({ + 'ext': determine_ext(mezzanine_url, 'mp4'), + 'format_id': 'mezzanine', + 'quality': 1, + 'url': mezzanine_url, + }) + + self._sort_formats(formats) + + thumbnails = [] + for thumbnail in (video.get('thumbnails') or []): + thumbnail_url = thumbnail.get('url') + if not thumbnail_url: + continue + thumbnails.append({ + 'url': thumbnail_url, + }) + + tags = [] + for tag in (video.get('tags') or []): + display_name = tag.get('displayName') + if not display_name: + continue + tags.append(display_name) + + return { + 'id': video_id, + 'title': title, + 'description': strip_or_none(metadata.get('description')), + 'timestamp': parse_iso8601(metadata.get('publishDate')), + 'duration': int_or_none(metadata.get('duration')), + 'display_id': display_id, + 'thumbnails': thumbnails, + 'formats': formats, + 'tags': tags, + } + + +class IGNVideoIE(InfoExtractor): + _VALID_URL = r'https?://.+?\.ign\.com/(?:[a-z]{2}/)?[^/]+/(?P\d+)/(?:video|trailer)/' + _TESTS = [{ + 'url': 'http://me.ign.com/en/videos/112203/video/how-hitman-aims-to-be-different-than-every-other-s', + 'md5': 'dd9aca7ed2657c4e118d8b261e5e9de1', + 'info_dict': { + 'id': 'e9be7ea899a9bbfc0674accc22a36cc8', + 'ext': 'mp4', + 'title': 'How Hitman Aims to Be Different Than Every Other Stealth Game - NYCC 2015', + 'description': 'Taking out assassination targets in Hitman has never been more stylish.', + 'timestamp': 1444665600, + 'upload_date': '20151012', + } + }, { + 'url': 'http://me.ign.com/ar/angry-birds-2/106533/video/lrd-ldyy-lwl-lfylm-angry-birds', + 'only_matching': True, + }, { + # Youtube embed + 'url': 'https://me.ign.com/ar/ratchet-clank-rift-apart/144327/trailer/embed', + 'only_matching': True, + }, { + # Twitter embed + 'url': 'http://adria.ign.com/sherlock-season-4/9687/trailer/embed', + 'only_matching': True, + }, { + # Vimeo embed + 'url': 'https://kr.ign.com/bic-2018/3307/trailer/embed', + 'only_matching': True, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + req = HEADRequest(url.rsplit('/', 1)[0] + '/embed') + url = self._request_webpage(req, video_id).geturl() + ign_url = compat_parse_qs( + compat_urllib_parse_urlparse(url).query).get('url', [None])[0] + if ign_url: + return self.url_result(ign_url, IGNIE.ie_key()) + return self.url_result(url) + + +class IGNArticleIE(IGNBaseIE): + _VALID_URL = r'https?://.+?\.ign\.com/(?:articles(?:/\d{4}/\d{2}/\d{2})?|(?:[a-z]{2}/)?feature/\d+)/(?P[^/?&#]+)' + _PAGE_TYPE = 'article' + _TESTS = [{ + 'url': 'http://me.ign.com/en/feature/15775/100-little-things-in-gta-5-that-will-blow-your-mind', + 'info_dict': { + 'id': '524497489e4e8ff5848ece34', + 'title': '100 Little Things in GTA 5 That Will Blow Your Mind', + }, + 'playlist': [ + { + 'info_dict': { + 'id': '5ebbd138523268b93c9141af17bec937', + 'ext': 'mp4', + 'title': 'GTA 5 Video Review', + 'description': 'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.', + 'timestamp': 1379339880, + 'upload_date': '20130916', + }, + }, + { + 'info_dict': { + 'id': '638672ee848ae4ff108df2a296418ee2', + 'ext': 'mp4', + 'title': '26 Twisted Moments from GTA 5 in Slow Motion', + 'description': 'The twisted beauty of GTA 5 in stunning slow motion.', + 'timestamp': 1386878820, + 'upload_date': '20131212', + }, + }, + ], + 'params': { + 'playlist_items': '2-3', + 'skip_download': True, + }, + }, { + 'url': 'http://www.ign.com/articles/2014/08/15/rewind-theater-wild-trailer-gamescom-2014?watch', + 'info_dict': { + 'id': '53ee806780a81ec46e0790f8', + 'title': 'Rewind Theater - Wild Trailer Gamescom 2014', + }, + 'playlist_count': 2, + }, { + # videoId pattern + 'url': 'http://www.ign.com/articles/2017/06/08/new-ducktales-short-donalds-birthday-doesnt-go-as-planned', + 'only_matching': True, + }, { + # Youtube embed + 'url': 'https://www.ign.com/articles/2021-mvp-named-in-puppy-bowl-xvii', + 'only_matching': True, + }, { + # IMDB embed + 'url': 'https://www.ign.com/articles/2014/08/07/sons-of-anarchy-final-season-trailer', + 'only_matching': True, + }, { + # Facebook embed + 'url': 'https://www.ign.com/articles/2017/09/20/marvels-the-punisher-watch-the-new-trailer-for-the-netflix-series', + 'only_matching': True, + }, { + # Brightcove embed + 'url': 'https://www.ign.com/articles/2016/01/16/supergirl-goes-flying-with-martian-manhunter-in-new-clip', + 'only_matching': True, + }] + + def _real_extract(self, url): + display_id = self._match_id(url) + article = self._call_api(display_id) + + def entries(): + media_url = try_get(article, lambda x: x['mediaRelations'][0]['media']['metadata']['url']) + if media_url: + yield self.url_result(media_url, IGNIE.ie_key()) + for content in (article.get('content') or []): + for video_url in re.findall(r'(?:\[(?:ignvideo\s+url|youtube\s+clip_id)|]+src)="([^"]+)"', content): + yield self.url_result(video_url) + + return self.playlist_result( + entries(), article.get('articleId'), + strip_or_none(try_get(article, lambda x: x['metadata']['headline']))) diff --git a/yt_dlp/extractor/iheart.py b/yt_dlp/extractor/iheart.py new file mode 100644 index 000000000..b54c05eeb --- /dev/null +++ b/yt_dlp/extractor/iheart.py @@ -0,0 +1,97 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import ( + clean_html, + clean_podcast_url, + int_or_none, + str_or_none, +) + + +class IHeartRadioBaseIE(InfoExtractor): + def _call_api(self, path, video_id, fatal=True, query=None): + return self._download_json( + 'https://api.iheart.com/api/v3/podcast/' + path, + video_id, fatal=fatal, query=query) + + def _extract_episode(self, episode): + return { + 'thumbnail': episode.get('imageUrl'), + 'description': clean_html(episode.get('description')), + 'timestamp': int_or_none(episode.get('startDate'), 1000), + 'duration': int_or_none(episode.get('duration')), + } + + +class IHeartRadioIE(IHeartRadioBaseIE): + IENAME = 'iheartradio' + _VALID_URL = r'(?:https?://(?:www\.)?iheart\.com/podcast/[^/]+/episode/(?P[^/?&#]+)-|iheartradio:)(?P\d+)' + _TEST = { + 'url': 'https://www.iheart.com/podcast/105-behind-the-bastards-29236323/episode/part-one-alexander-lukashenko-the-dictator-70346499/?embed=true', + 'md5': 'c8609c92c8688dcb69d8541042b8abca', + 'info_dict': { + 'id': '70346499', + 'ext': 'mp3', + 'title': 'Part One: Alexander Lukashenko: The Dictator of Belarus', + 'description': 'md5:96cc7297b3a5a9ebae28643801c96fae', + 'timestamp': 1597741200, + 'upload_date': '20200818', + } + } + + def _real_extract(self, url): + episode_id = self._match_id(url) + episode = self._call_api( + 'episodes/' + episode_id, episode_id)['episode'] + info = self._extract_episode(episode) + info.update({ + 'id': episode_id, + 'title': episode['title'], + 'url': clean_podcast_url(episode['mediaUrl']), + }) + return info + + +class IHeartRadioPodcastIE(IHeartRadioBaseIE): + IE_NAME = 'iheartradio:podcast' + _VALID_URL = r'https?://(?:www\.)?iheart(?:podcastnetwork)?\.com/podcast/[^/?&#]+-(?P\d+)/?(?:[?#&]|$)' + _TESTS = [{ + 'url': 'https://www.iheart.com/podcast/1119-it-could-happen-here-30717896/', + 'info_dict': { + 'id': '30717896', + 'title': 'It Could Happen Here', + 'description': 'md5:5842117412a967eb0b01f8088eb663e2', + }, + 'playlist_mincount': 11, + }, { + 'url': 'https://www.iheartpodcastnetwork.com/podcast/105-stuff-you-should-know-26940277', + 'only_matching': True, + }] + + def _real_extract(self, url): + podcast_id = self._match_id(url) + path = 'podcasts/' + podcast_id + episodes = self._call_api( + path + '/episodes', podcast_id, query={'limit': 1000000000})['data'] + + entries = [] + for episode in episodes: + episode_id = str_or_none(episode.get('id')) + if not episode_id: + continue + info = self._extract_episode(episode) + info.update({ + '_type': 'url', + 'id': episode_id, + 'title': episode.get('title'), + 'url': 'iheartradio:' + episode_id, + 'ie_key': IHeartRadioIE.ie_key(), + }) + entries.append(info) + + podcast = self._call_api(path, podcast_id, False) or {} + + return self.playlist_result( + entries, podcast_id, podcast.get('title'), podcast.get('description')) diff --git a/youtube_dlc/extractor/imdb.py b/yt_dlp/extractor/imdb.py similarity index 100% rename from youtube_dlc/extractor/imdb.py rename to yt_dlp/extractor/imdb.py diff --git a/youtube_dlc/extractor/imggaming.py b/yt_dlp/extractor/imggaming.py similarity index 98% rename from youtube_dlc/extractor/imggaming.py rename to yt_dlp/extractor/imggaming.py index e11f92053..1e43ec95b 100644 --- a/youtube_dlc/extractor/imggaming.py +++ b/yt_dlp/extractor/imggaming.py @@ -65,7 +65,7 @@ class ImgGamingBaseIE(InfoExtractor): domain, media_type, media_id, playlist_id = re.match(self._VALID_URL, url).groups() if playlist_id: - if self._downloader.params.get('noplaylist'): + if self.get_param('noplaylist'): self.to_screen('Downloading just video %s because of --no-playlist' % media_id) else: self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % playlist_id) diff --git a/youtube_dlc/extractor/imgur.py b/yt_dlp/extractor/imgur.py similarity index 96% rename from youtube_dlc/extractor/imgur.py rename to yt_dlp/extractor/imgur.py index 4dc7b0b5c..dfa473752 100644 --- a/youtube_dlc/extractor/imgur.py +++ b/yt_dlp/extractor/imgur.py @@ -60,7 +60,7 @@ class ImgurIE(InfoExtractor): 'width': width, 'height': height, 'http_headers': { - 'User-Agent': 'youtube-dlc (like wget)', + 'User-Agent': 'yt-dlp (like wget)', }, }) @@ -72,7 +72,7 @@ class ImgurIE(InfoExtractor): gif_json, video_id, transform_source=js_to_json) formats.append({ 'format_id': 'gif', - 'preference': -10, + 'preference': -10, # gifs are worse than videos 'width': width, 'height': height, 'ext': 'gif', @@ -82,7 +82,7 @@ class ImgurIE(InfoExtractor): 'url': self._proto_relative_url(gifd['gifUrl']), 'filesize': gifd.get('size'), 'http_headers': { - 'User-Agent': 'youtube-dlc (like wget)', + 'User-Agent': 'yt-dlp (like wget)', }, }) diff --git a/youtube_dlc/extractor/ina.py b/yt_dlp/extractor/ina.py similarity index 93% rename from youtube_dlc/extractor/ina.py rename to yt_dlp/extractor/ina.py index 12695af27..b3b2683cb 100644 --- a/youtube_dlc/extractor/ina.py +++ b/yt_dlp/extractor/ina.py @@ -12,7 +12,7 @@ from ..utils import ( class InaIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?ina\.fr/(?:video|audio)/(?P[A-Z0-9_]+)' + _VALID_URL = r'https?://(?:(?:www|m)\.)?ina\.fr/(?:video|audio)/(?P[A-Z0-9_]+)' _TESTS = [{ 'url': 'http://www.ina.fr/video/I12055569/francois-hollande-je-crois-que-c-est-clair-video.html', 'md5': 'a667021bf2b41f8dc6049479d9bb38a3', @@ -31,6 +31,9 @@ class InaIE(InfoExtractor): }, { 'url': 'https://www.ina.fr/video/P16173408-video.html', 'only_matching': True, + }, { + 'url': 'http://m.ina.fr/video/I12055569', + 'only_matching': True, }] def _real_extract(self, url): diff --git a/youtube_dlc/extractor/inc.py b/yt_dlp/extractor/inc.py similarity index 100% rename from youtube_dlc/extractor/inc.py rename to yt_dlp/extractor/inc.py diff --git a/youtube_dlc/extractor/indavideo.py b/yt_dlp/extractor/indavideo.py similarity index 100% rename from youtube_dlc/extractor/indavideo.py rename to yt_dlp/extractor/indavideo.py diff --git a/youtube_dlc/extractor/infoq.py b/yt_dlp/extractor/infoq.py similarity index 100% rename from youtube_dlc/extractor/infoq.py rename to yt_dlp/extractor/infoq.py diff --git a/youtube_dlc/extractor/instagram.py b/yt_dlp/extractor/instagram.py similarity index 72% rename from youtube_dlc/extractor/instagram.py rename to yt_dlp/extractor/instagram.py index b061850a1..1261f438e 100644 --- a/youtube_dlc/extractor/instagram.py +++ b/yt_dlp/extractor/instagram.py @@ -12,17 +12,19 @@ from ..compat import ( ) from ..utils import ( ExtractorError, + float_or_none, get_element_by_attribute, int_or_none, lowercase_escape, std_headers, try_get, url_or_none, + variadic, ) class InstagramIE(InfoExtractor): - _VALID_URL = r'(?Phttps?://(?:www\.)?instagram\.com/(?:p|tv)/(?P[^/?#&]+))' + _VALID_URL = r'(?Phttps?://(?:www\.)?instagram\.com/(?:p|tv|reel)/(?P[^/?#&]+))' _TESTS = [{ 'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc', 'md5': '0d2da106a9d2631273e192b372806516', @@ -32,10 +34,11 @@ class InstagramIE(InfoExtractor): 'title': 'Video by naomipq', 'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8', 'thumbnail': r're:^https?://.*\.jpg', + 'duration': 0, 'timestamp': 1371748545, 'upload_date': '20130620', 'uploader_id': 'naomipq', - 'uploader': 'Naomi Leonor Phan-Quang', + 'uploader': 'B E A U T Y F O R A S H E S', 'like_count': int, 'comment_count': int, 'comments': list, @@ -48,6 +51,7 @@ class InstagramIE(InfoExtractor): 'ext': 'mp4', 'title': 'Video by britneyspears', 'thumbnail': r're:^https?://.*\.jpg', + 'duration': 0, 'timestamp': 1453760977, 'upload_date': '20160125', 'uploader_id': 'britneyspears', @@ -86,6 +90,24 @@ class InstagramIE(InfoExtractor): 'title': 'Post by instagram', 'description': 'md5:0f9203fc6a2ce4d228da5754bcf54957', }, + }, { + # IGTV + 'url': 'https://www.instagram.com/tv/BkfuX9UB-eK/', + 'info_dict': { + 'id': 'BkfuX9UB-eK', + 'ext': 'mp4', + 'title': 'Fingerboarding Tricks with @cass.fb', + 'thumbnail': r're:^https?://.*\.jpg', + 'duration': 53.83, + 'timestamp': 1530032919, + 'upload_date': '20180626', + 'uploader_id': 'instagram', + 'uploader': 'Instagram', + 'like_count': int, + 'comment_count': int, + 'comments': list, + 'description': 'Meet Cass Hirst (@cass.fb), a fingerboarding pro who can perform tiny ollies and kickflips while blindfolded.', + } }, { 'url': 'https://instagram.com/p/-Cmh1cukG2/', 'only_matching': True, @@ -95,6 +117,9 @@ class InstagramIE(InfoExtractor): }, { 'url': 'https://www.instagram.com/tv/aye83DjauH/', 'only_matching': True, + }, { + 'url': 'https://www.instagram.com/reel/CDUMkliABpa/', + 'only_matching': True, }] @staticmethod @@ -122,9 +147,9 @@ class InstagramIE(InfoExtractor): webpage = self._download_webpage(url, video_id) - (video_url, description, thumbnail, timestamp, uploader, + (media, video_url, description, thumbnail, timestamp, uploader, uploader_id, like_count, comment_count, comments, height, - width) = [None] * 11 + width) = [None] * 12 shared_data = self._parse_json( self._search_regex( @@ -137,59 +162,78 @@ class InstagramIE(InfoExtractor): (lambda x: x['entry_data']['PostPage'][0]['graphql']['shortcode_media'], lambda x: x['entry_data']['PostPage'][0]['media']), dict) - if media: - video_url = media.get('video_url') - height = int_or_none(media.get('dimensions', {}).get('height')) - width = int_or_none(media.get('dimensions', {}).get('width')) - description = try_get( - media, lambda x: x['edge_media_to_caption']['edges'][0]['node']['text'], - compat_str) or media.get('caption') - thumbnail = media.get('display_src') - timestamp = int_or_none(media.get('taken_at_timestamp') or media.get('date')) - uploader = media.get('owner', {}).get('full_name') - uploader_id = media.get('owner', {}).get('username') + # _sharedData.entry_data.PostPage is empty when authenticated (see + # https://github.com/ytdl-org/youtube-dl/pull/22880) + if not media: + additional_data = self._parse_json( + self._search_regex( + r'window\.__additionalDataLoaded\s*\(\s*[^,]+,\s*({.+?})\s*\)\s*;', + webpage, 'additional data', default='{}'), + video_id, fatal=False) + if additional_data: + media = try_get( + additional_data, lambda x: x['graphql']['shortcode_media'], + dict) + if media: + video_url = media.get('video_url') + height = int_or_none(media.get('dimensions', {}).get('height')) + width = int_or_none(media.get('dimensions', {}).get('width')) + description = try_get( + media, lambda x: x['edge_media_to_caption']['edges'][0]['node']['text'], + compat_str) or media.get('caption') + title = media.get('title') + thumbnail = media.get('display_src') or media.get('display_url') + duration = float_or_none(media.get('video_duration')) + timestamp = int_or_none(media.get('taken_at_timestamp') or media.get('date')) + uploader = media.get('owner', {}).get('full_name') + uploader_id = media.get('owner', {}).get('username') - def get_count(key, kind): - return int_or_none(try_get( + def get_count(keys, kind): + for key in variadic(keys): + count = int_or_none(try_get( media, (lambda x: x['edge_media_%s' % key]['count'], lambda x: x['%ss' % kind]['count']))) - like_count = get_count('preview_like', 'like') - comment_count = get_count('to_comment', 'comment') + if count is not None: + return count + like_count = get_count('preview_like', 'like') + comment_count = get_count( + ('preview_comment', 'to_comment', 'to_parent_comment'), 'comment') - comments = [{ - 'author': comment.get('user', {}).get('username'), - 'author_id': comment.get('user', {}).get('id'), - 'id': comment.get('id'), - 'text': comment.get('text'), - 'timestamp': int_or_none(comment.get('created_at')), - } for comment in media.get( - 'comments', {}).get('nodes', []) if comment.get('text')] - if not video_url: - edges = try_get( - media, lambda x: x['edge_sidecar_to_children']['edges'], - list) or [] - if edges: - entries = [] - for edge_num, edge in enumerate(edges, start=1): - node = try_get(edge, lambda x: x['node'], dict) - if not node: - continue - node_video_url = url_or_none(node.get('video_url')) - if not node_video_url: - continue - entries.append({ - 'id': node.get('shortcode') or node['id'], - 'title': 'Video %d' % edge_num, - 'url': node_video_url, - 'thumbnail': node.get('display_url'), - 'width': int_or_none(try_get(node, lambda x: x['dimensions']['width'])), - 'height': int_or_none(try_get(node, lambda x: x['dimensions']['height'])), - 'view_count': int_or_none(node.get('video_view_count')), - }) - return self.playlist_result( - entries, video_id, - 'Post by %s' % uploader_id if uploader_id else None, - description) + comments = [{ + 'author': comment.get('user', {}).get('username'), + 'author_id': comment.get('user', {}).get('id'), + 'id': comment.get('id'), + 'text': comment.get('text'), + 'timestamp': int_or_none(comment.get('created_at')), + } for comment in media.get( + 'comments', {}).get('nodes', []) if comment.get('text')] + if not video_url: + edges = try_get( + media, lambda x: x['edge_sidecar_to_children']['edges'], + list) or [] + if edges: + entries = [] + for edge_num, edge in enumerate(edges, start=1): + node = try_get(edge, lambda x: x['node'], dict) + if not node: + continue + node_video_url = url_or_none(node.get('video_url')) + if not node_video_url: + continue + entries.append({ + 'id': node.get('shortcode') or node['id'], + 'title': node.get('title') or 'Video %d' % edge_num, + 'url': node_video_url, + 'thumbnail': node.get('display_url'), + 'duration': float_or_none(node.get('video_duration')), + 'width': int_or_none(try_get(node, lambda x: x['dimensions']['width'])), + 'height': int_or_none(try_get(node, lambda x: x['dimensions']['height'])), + 'view_count': int_or_none(node.get('video_view_count')), + }) + return self.playlist_result( + entries, video_id, + 'Post by %s' % uploader_id if uploader_id else None, + description) if not video_url: video_url = self._og_search_video_url(webpage, secure=False) @@ -218,8 +262,9 @@ class InstagramIE(InfoExtractor): 'id': video_id, 'formats': formats, 'ext': 'mp4', - 'title': 'Video by %s' % uploader_id, + 'title': title or 'Video by %s' % uploader_id, 'description': description, + 'duration': duration, 'thumbnail': thumbnail, 'timestamp': timestamp, 'uploader_id': uploader_id, diff --git a/youtube_dlc/extractor/internazionale.py b/yt_dlp/extractor/internazionale.py similarity index 100% rename from youtube_dlc/extractor/internazionale.py rename to yt_dlp/extractor/internazionale.py diff --git a/youtube_dlc/extractor/internetvideoarchive.py b/yt_dlp/extractor/internetvideoarchive.py similarity index 100% rename from youtube_dlc/extractor/internetvideoarchive.py rename to yt_dlp/extractor/internetvideoarchive.py diff --git a/youtube_dlc/extractor/iprima.py b/yt_dlp/extractor/iprima.py similarity index 98% rename from youtube_dlc/extractor/iprima.py rename to yt_dlp/extractor/iprima.py index 648ae6741..28e660972 100644 --- a/youtube_dlc/extractor/iprima.py +++ b/yt_dlp/extractor/iprima.py @@ -136,7 +136,7 @@ class IPrimaIE(InfoExtractor): extract_formats(src) if not formats and '>GEO_IP_NOT_ALLOWED<' in playerpage: - self.raise_geo_restricted(countries=['CZ']) + self.raise_geo_restricted(countries=['CZ'], metadata_available=True) self._sort_formats(formats) diff --git a/youtube_dlc/extractor/iqiyi.py b/yt_dlp/extractor/iqiyi.py similarity index 98% rename from youtube_dlc/extractor/iqiyi.py rename to yt_dlp/extractor/iqiyi.py index 5df674daf..e33e23f08 100644 --- a/youtube_dlc/extractor/iqiyi.py +++ b/yt_dlp/extractor/iqiyi.py @@ -280,7 +280,7 @@ class IqiyiIE(InfoExtractor): msg = 'error %s' % code if validation_result.get('msg'): msg += ': ' + validation_result['msg'] - self._downloader.report_warning('unable to log in: ' + msg) + self.report_warning('unable to log in: ' + msg) return False return True @@ -373,7 +373,7 @@ class IqiyiIE(InfoExtractor): 'url': stream['m3utx'], 'format_id': vd, 'ext': 'mp4', - 'preference': self._FORMATS_MAP.get(vd, -1), + 'quality': self._FORMATS_MAP.get(vd, -1), 'protocol': 'm3u8_native', }) diff --git a/youtube_dlc/extractor/ir90tv.py b/yt_dlp/extractor/ir90tv.py similarity index 100% rename from youtube_dlc/extractor/ir90tv.py rename to yt_dlp/extractor/ir90tv.py diff --git a/yt_dlp/extractor/itv.py b/yt_dlp/extractor/itv.py new file mode 100644 index 000000000..4122ac880 --- /dev/null +++ b/yt_dlp/extractor/itv.py @@ -0,0 +1,195 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import json + +from .common import InfoExtractor +from .brightcove import BrightcoveNewIE +from ..utils import ( + clean_html, + determine_ext, + extract_attributes, + get_element_by_class, + JSON_LD_RE, + merge_dicts, + parse_duration, + smuggle_url, + try_get, + url_or_none, +) + + +class ITVIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P[0-9a-zA-Z]+)' + _GEO_COUNTRIES = ['GB'] + _TESTS = [{ + 'url': 'https://www.itv.com/hub/liar/2a4547a0012', + 'info_dict': { + 'id': '2a4547a0012', + 'ext': 'mp4', + 'title': 'Liar - Series 2 - Episode 6', + 'description': 'md5:d0f91536569dec79ea184f0a44cca089', + 'series': 'Liar', + 'season_number': 2, + 'episode_number': 6, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, { + # unavailable via data-playlist-url + 'url': 'https://www.itv.com/hub/through-the-keyhole/2a2271a0033', + 'only_matching': True, + }, { + # InvalidVodcrid + 'url': 'https://www.itv.com/hub/james-martins-saturday-morning/2a5159a0034', + 'only_matching': True, + }, { + # ContentUnavailable + 'url': 'https://www.itv.com/hub/whos-doing-the-dishes/2a2898a0024', + 'only_matching': True, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + params = extract_attributes(self._search_regex( + r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params')) + + ios_playlist_url = params.get('data-video-playlist') or params['data-video-id'] + hmac = params['data-video-hmac'] + headers = self.geo_verification_headers() + headers.update({ + 'Accept': 'application/vnd.itv.vod.playlist.v2+json', + 'Content-Type': 'application/json', + 'hmac': hmac.upper(), + }) + ios_playlist = self._download_json( + ios_playlist_url, video_id, data=json.dumps({ + 'user': { + 'itvUserId': '', + 'entitlements': [], + 'token': '' + }, + 'device': { + 'manufacturer': 'Safari', + 'model': '5', + 'os': { + 'name': 'Windows NT', + 'version': '6.1', + 'type': 'desktop' + } + }, + 'client': { + 'version': '4.1', + 'id': 'browser' + }, + 'variantAvailability': { + 'featureset': { + 'min': ['hls', 'aes', 'outband-webvtt'], + 'max': ['hls', 'aes', 'outband-webvtt'] + }, + 'platformTag': 'dotcom' + } + }).encode(), headers=headers) + video_data = ios_playlist['Playlist']['Video'] + ios_base_url = video_data.get('Base') + + formats = [] + for media_file in (video_data.get('MediaFiles') or []): + href = media_file.get('Href') + if not href: + continue + if ios_base_url: + href = ios_base_url + href + ext = determine_ext(href) + if ext == 'm3u8': + formats.extend(self._extract_m3u8_formats( + href, video_id, 'mp4', entry_protocol='m3u8_native', + m3u8_id='hls', fatal=False)) + else: + formats.append({ + 'url': href, + }) + self._sort_formats(formats) + + subtitles = {} + subs = video_data.get('Subtitles') or [] + for sub in subs: + if not isinstance(sub, dict): + continue + href = url_or_none(sub.get('Href')) + if not href: + continue + subtitles.setdefault('en', []).append({ + 'url': href, + 'ext': determine_ext(href, 'vtt'), + }) + + info = self._search_json_ld(webpage, video_id, default={}) + if not info: + json_ld = self._parse_json(self._search_regex( + JSON_LD_RE, webpage, 'JSON-LD', '{}', + group='json_ld'), video_id, fatal=False) + if json_ld and json_ld.get('@type') == 'BreadcrumbList': + for ile in (json_ld.get('itemListElement:') or []): + item = ile.get('item:') or {} + if item.get('@type') == 'TVEpisode': + item['@context'] = 'http://schema.org' + info = self._json_ld(item, video_id, fatal=False) or {} + break + + return merge_dicts({ + 'id': video_id, + 'title': self._html_search_meta(['og:title', 'twitter:title'], webpage), + 'formats': formats, + 'subtitles': subtitles, + 'duration': parse_duration(video_data.get('Duration')), + 'description': clean_html(get_element_by_class('episode-info__synopsis', webpage)), + }, info) + + +class ITVBTCCIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?itv\.com/btcc/(?:[^/]+/)*(?P[^/?#&]+)' + _TEST = { + 'url': 'https://www.itv.com/btcc/articles/btcc-2019-brands-hatch-gp-race-action', + 'info_dict': { + 'id': 'btcc-2019-brands-hatch-gp-race-action', + 'title': 'BTCC 2019: Brands Hatch GP race action', + }, + 'playlist_count': 12, + } + BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1582188683001/HkiHLnNRx_default/index.html?videoId=%s' + + def _real_extract(self, url): + playlist_id = self._match_id(url) + + webpage = self._download_webpage(url, playlist_id) + + json_map = try_get(self._parse_json(self._html_search_regex( + '(?s)]+id=[\'"]__NEXT_DATA__[^>]*>([^<]+)', webpage, 'json_map'), playlist_id), + lambda x: x['props']['pageProps']['article']['body']['content']) or [] + + # Discard empty objects + video_ids = [] + for video in json_map: + if video['data'].get('id'): + video_ids.append(video['data']['id']) + + entries = [ + self.url_result( + smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, { + # ITV does not like some GB IP ranges, so here are some + # IP blocks it accepts + 'geo_ip_blocks': [ + '193.113.0.0/16', '54.36.162.0/23', '159.65.16.0/21' + ], + 'referrer': url, + }), + ie=BrightcoveNewIE.ie_key(), video_id=video_id) + for video_id in video_ids] + + title = self._og_search_title(webpage, fatal=False) + + return self.playlist_result(entries, playlist_id, title) diff --git a/youtube_dlc/extractor/ivi.py b/yt_dlp/extractor/ivi.py similarity index 97% rename from youtube_dlc/extractor/ivi.py rename to yt_dlp/extractor/ivi.py index b9cb5a8e6..c167ee500 100644 --- a/youtube_dlc/extractor/ivi.py +++ b/yt_dlp/extractor/ivi.py @@ -142,11 +142,11 @@ class IviIE(InfoExtractor): continue elif bundled: raise ExtractorError( - 'This feature does not work from bundled exe. Run youtube-dlc from sources.', + 'This feature does not work from bundled exe. Run yt-dlp from sources.', expected=True) elif not pycryptodomex_found: raise ExtractorError( - 'pycryptodomex not found. Please install it.', + 'pycryptodomex not found. Please install', expected=True) elif message: extractor_msg += ': ' + message @@ -163,7 +163,10 @@ class IviIE(InfoExtractor): for f in result.get('files', []): f_url = f.get('url') content_format = f.get('content_format') - if not f_url or '-MDRM-' in content_format or '-FPS-' in content_format: + if not f_url: + continue + if (not self.get_param('allow_unplayable_formats') + and ('-MDRM-' in content_format or '-FPS-' in content_format)): continue formats.append({ 'url': f_url, diff --git a/youtube_dlc/extractor/ivideon.py b/yt_dlp/extractor/ivideon.py similarity index 100% rename from youtube_dlc/extractor/ivideon.py rename to yt_dlp/extractor/ivideon.py diff --git a/youtube_dlc/extractor/iwara.py b/yt_dlp/extractor/iwara.py similarity index 100% rename from youtube_dlc/extractor/iwara.py rename to yt_dlp/extractor/iwara.py diff --git a/youtube_dlc/extractor/izlesene.py b/yt_dlp/extractor/izlesene.py similarity index 100% rename from youtube_dlc/extractor/izlesene.py rename to yt_dlp/extractor/izlesene.py diff --git a/youtube_dlc/extractor/jamendo.py b/yt_dlp/extractor/jamendo.py similarity index 79% rename from youtube_dlc/extractor/jamendo.py rename to yt_dlp/extractor/jamendo.py index 490efa8fb..1db7c64af 100644 --- a/youtube_dlc/extractor/jamendo.py +++ b/yt_dlp/extractor/jamendo.py @@ -29,34 +29,51 @@ class JamendoIE(InfoExtractor): 'id': '196219', 'display_id': 'stories-from-emona-i', 'ext': 'flac', - 'title': 'Maya Filipič - Stories from Emona I', - 'artist': 'Maya Filipič', + # 'title': 'Maya Filipič - Stories from Emona I', + 'title': 'Stories from Emona I', + # 'artist': 'Maya Filipič', 'track': 'Stories from Emona I', 'duration': 210, 'thumbnail': r're:^https?://.*\.jpg', 'timestamp': 1217438117, 'upload_date': '20080730', + 'license': 'by-nc-nd', + 'view_count': int, + 'like_count': int, + 'average_rating': int, + 'tags': ['piano', 'peaceful', 'newage', 'strings', 'upbeat'], } }, { 'url': 'https://licensing.jamendo.com/en/track/1496667/energetic-rock', 'only_matching': True, }] + def _call_api(self, resource, resource_id): + path = '/api/%ss' % resource + rand = compat_str(random.random()) + return self._download_json( + 'https://www.jamendo.com' + path, resource_id, query={ + 'id[]': resource_id, + }, headers={ + 'X-Jam-Call': '$%s*%s~' % (hashlib.sha1((path + rand).encode()).hexdigest(), rand) + })[0] + def _real_extract(self, url): track_id, display_id = self._VALID_URL_RE.match(url).groups() - webpage = self._download_webpage( - 'https://www.jamendo.com/track/' + track_id, track_id) - models = self._parse_json(self._html_search_regex( - r"data-bundled-models='([^']+)", - webpage, 'bundled models'), track_id) - track = models['track']['models'][0] + # webpage = self._download_webpage( + # 'https://www.jamendo.com/track/' + track_id, track_id) + # models = self._parse_json(self._html_search_regex( + # r"data-bundled-models='([^']+)", + # webpage, 'bundled models'), track_id) + # track = models['track']['models'][0] + track = self._call_api('track', track_id) title = track_name = track['name'] - get_model = lambda x: try_get(models, lambda y: y[x]['models'][0], dict) or {} - artist = get_model('artist') - artist_name = artist.get('name') - if artist_name: - title = '%s - %s' % (artist_name, title) - album = get_model('album') + # get_model = lambda x: try_get(models, lambda y: y[x]['models'][0], dict) or {} + # artist = get_model('artist') + # artist_name = artist.get('name') + # if artist_name: + # title = '%s - %s' % (artist_name, title) + # album = get_model('album') formats = [{ 'url': 'https://%s.jamendo.com/?trackid=%s&format=%s&from=app-97dab294' @@ -74,7 +91,7 @@ class JamendoIE(InfoExtractor): urls = [] thumbnails = [] - for _, covers in track.get('cover', {}).items(): + for covers in (track.get('cover') or {}).values(): for cover_id, cover_url in covers.items(): if not cover_url or cover_url in urls: continue @@ -88,13 +105,14 @@ class JamendoIE(InfoExtractor): }) tags = [] - for tag in track.get('tags', []): + for tag in (track.get('tags') or []): tag_name = tag.get('name') if not tag_name: continue tags.append(tag_name) stats = track.get('stats') or {} + license = track.get('licenseCC') or [] return { 'id': track_id, @@ -103,11 +121,11 @@ class JamendoIE(InfoExtractor): 'title': title, 'description': track.get('description'), 'duration': int_or_none(track.get('duration')), - 'artist': artist_name, + # 'artist': artist_name, 'track': track_name, - 'album': album.get('name'), + # 'album': album.get('name'), 'formats': formats, - 'license': '-'.join(track.get('licenseCC', [])) or None, + 'license': '-'.join(license) if license else None, 'timestamp': int_or_none(track.get('dateCreated')), 'view_count': int_or_none(stats.get('listenedAll')), 'like_count': int_or_none(stats.get('favorited')), @@ -116,9 +134,9 @@ class JamendoIE(InfoExtractor): } -class JamendoAlbumIE(InfoExtractor): +class JamendoAlbumIE(JamendoIE): _VALID_URL = r'https?://(?:www\.)?jamendo\.com/album/(?P[0-9]+)' - _TEST = { + _TESTS = [{ 'url': 'https://www.jamendo.com/album/121486/duck-on-cover', 'info_dict': { 'id': '121486', @@ -151,17 +169,7 @@ class JamendoAlbumIE(InfoExtractor): 'params': { 'playlistend': 2 } - } - - def _call_api(self, resource, resource_id): - path = '/api/%ss' % resource - rand = compat_str(random.random()) - return self._download_json( - 'https://www.jamendo.com' + path, resource_id, query={ - 'id[]': resource_id, - }, headers={ - 'X-Jam-Call': '$%s*%s~' % (hashlib.sha1((path + rand).encode()).hexdigest(), rand) - })[0] + }] def _real_extract(self, url): album_id = self._match_id(url) @@ -169,7 +177,7 @@ class JamendoAlbumIE(InfoExtractor): album_name = album.get('name') entries = [] - for track in album.get('tracks', []): + for track in (album.get('tracks') or []): track_id = track.get('id') if not track_id: continue diff --git a/youtube_dlc/extractor/jeuxvideo.py b/yt_dlp/extractor/jeuxvideo.py similarity index 100% rename from youtube_dlc/extractor/jeuxvideo.py rename to yt_dlp/extractor/jeuxvideo.py diff --git a/youtube_dlc/extractor/joj.py b/yt_dlp/extractor/joj.py similarity index 100% rename from youtube_dlc/extractor/joj.py rename to yt_dlp/extractor/joj.py diff --git a/youtube_dlc/extractor/jove.py b/yt_dlp/extractor/jove.py similarity index 100% rename from youtube_dlc/extractor/jove.py rename to yt_dlp/extractor/jove.py diff --git a/youtube_dlc/extractor/jwplatform.py b/yt_dlp/extractor/jwplatform.py similarity index 78% rename from youtube_dlc/extractor/jwplatform.py rename to yt_dlp/extractor/jwplatform.py index c34b5f5e6..5aa508bf9 100644 --- a/youtube_dlc/extractor/jwplatform.py +++ b/yt_dlp/extractor/jwplatform.py @@ -32,9 +32,14 @@ class JWPlatformIE(InfoExtractor): @staticmethod def _extract_urls(webpage): - return re.findall( - r'<(?:script|iframe)[^>]+?src=["\']((?:https?:)?//(?:content\.jwplatform|cdn\.jwplayer)\.com/players/[a-zA-Z0-9]{8})', - webpage) + for tag, key in ((r'(?:script|iframe)', 'src'), ('input', 'value')): + # is used by hyland.com + # if we find