diff --git a/.gitignore b/.gitignore index 36a3190..e63e7bd 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ unshackle/PRDs/ temp/ logs/ services/ +/.[^/]*/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a948da..3862c08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,735 +2,529 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [2.3.1] - 2026-01-22 +This changelog is automatically generated using [git-cliff](https://git-cliff.org). -### Fixed +## [3.1.0] - 2026-02-23 -- **Vulnerable Dependencies**: Upgraded dependencies to address security alerts - - urllib3: 2.5.0 → 2.6.3 (CVE-2025-66418, CVE-2025-66471, CVE-2026-21441) - - aiohttp: 3.13.2 → 3.13.3 (8 CVEs including CVE-2025-69223, CVE-2025-69227) - - fonttools: 4.60.1 → 4.61.1 (CVE-2025-66034) - - filelock: 3.19.1 → 3.20.3 (CVE-2025-68146, CVE-2026-22701) - - virtualenv: 20.34.0 → 20.36.1 (CVE-2026-22702) -- **HLS Key Selection**: Prefer media playlist keys over session keys for accurate KID matching - - Session keys from master playlists often contain PSSHs with multiple KIDs covering all tracks - - Unified DRM licensing logic for all downloaders - - Added `filter_keys_for_cdm()` to select keys matching configured CDM type - - Added `get_track_kid_from_init()` to extract KID from init segment with fallback - - Fixed PlayReady keyformat matching using strict `PR_PSSH.SYSTEM_ID` URN - - Fixes download failures where track KID was null or mismatched -- **DASH Audio Track Selection**: Include language in N_m3u8DL-RE track selection - - Fixes duplicate audio downloads when DASH manifests have multiple adaptation sets with same representation IDs -- **SubtitleEdit Compatibility**: Update CLI syntax for SubtitleEdit 4.x - - Use lowercase format names (subrip, webvtt, advancedsubstationalpha) - - Respect `conversion_method` config setting when stripping SDH +### Features -## [2.3.0] - 2026-01-18 +- *hybrid*: Add L5 active area and dynamic L6 luminance metadata +- *debug*: Add JSONL debug logging to decryption, muxing, and all downloaders +- *debug*: Log binary tool versions at session start +- *dl*: Add --repack flag to insert REPACK tag in output filenames +- *core*: Add TrackRequest system for multi-codec/multi-range support -### Added +### Bug Fixes -- **Unicode Filenames Option**: New `unicode_filenames` config option to preserve native characters - - Allows disabling ASCII transliteration in filenames - - Preserves Korean, Japanese, Chinese, and other native language characters - - Closes #49 +- *n_m3u8dl_re*: Pass all content keys for DualKey DRM decryption +- *hybrid*: Skip bitrate filter for DV tracks in HYBRID mode +- *attachment*: Sanitize filenames with illegal Windows characters +- *hybrid*: Accept HDR10+ tracks as valid base layer for HYBRID mode +- *dl*: Allow selection of audio tracks for 'all' languages in addition to 'best' +- *dl*: Overwrite existing files on re-download and use atomic replace +- *dl*: Handle cross-device moves when temp and downloads differ -### Fixed +### Changes -- **WebVTT Cue Handling**: Handle WebVTT cue identifiers and overlapping multi-line cues - - Added detection and sanitization for cue identifiers (Q0, Q1, etc.) before timing lines - - Added merging of overlapping cues with different line positions into multi-line subtitles - - Fixes parsing issues with pysubs2/pycaption on certain WebVTT files -- **Widevine PSSH Filtering**: Filter Widevine PSSH by system ID instead of sorting - - Fixes KeyError crash when unsupported DRM systems are present in init segments -- **TTML Negative Values**: Handle negative values in multi-value TTML attributes - - Fixes pycaption parse errors for attributes like `tts:extent="-5% 7.5%"` - - Closes #47 -- **ASS Font Names**: Strip whitespace from ASS font names - - Handles ASS subtitle files with spaces after commas in Style definitions - - Fixes #57 -- **Shaka-Packager Error Messages**: Include shaka-packager binary path in error messages -- **N_m3u8DL-RE Merge and Decryption**: Handle merge and decryption properly - - Prevents audio corruption ("Box 'OG 2' size is too large") with DASH manifests - - Fixes duplicate init segment writing when using N_m3u8DL-RE -- **DASH Placeholder KIDs**: Handle placeholder KIDs and improve DRM init from segments - - Detects and replaces placeholder/test KIDs in Widevine PSSH - - Adds CENC namespace support for kid/default_KID attributes -- **PlayReady PSSH Comparison**: Correct PSSH system ID comparison in PlayReady - - Removes erroneous `.bytes` accessor from PSSH.SYSTEM_ID comparisons +- *hybrid*: Replace log.info with console status and add JSONL debug logging +- *dl*: Remove legacy multi-fetch loop for unmigrated services -## [2.2.0] - 2026-01-15 +## [3.0.0] - 2026-02-15 -### Added +### Features -- **CDM-Aware PlayReady Fallback Detection**: Intelligent DRM fallback based on selected CDM - - Adds PlayReady PSSH/KID extraction from track and init data with CDM-aware ordering - - When PlayReady CDM is selected, tries PlayReady first then falls back to Widevine - - When Widevine CDM is selected (default), tries Widevine first then falls back to PlayReady -- **Comprehensive Debug Logging**: Enhanced debug logging for downloaders and muxing - - Added detailed debug logging to aria2c, curl_impersonate, n_m3u8dl_re, and requests downloaders - - Enhanced manifest parsers (DASH, HLS, ISM) with debug logging - - Added debug logging to track muxing operations +- *titles*: Use track source attribute for service name in filenames +- *debug*: Add download output verification logging +- Gluetun VPN integration and remote service enhancements +- *gluetun*: Improve VPN connection display and Windscribe support +- *serve*: Add PlayReady CDM support alongside Widevine +- *cdm*: Add remote PlayReady CDM support via pyplayready RemoteCdm +- *env*: Add ML-Worker binary for DRM licensing +- *video*: Detect interlaced scan type from MPD manifests +- *drm*: Add MonaLisa DRM support to core infrastructure +- *audio*: Codec lists and split muxing +- *proxy*: Add specific server selection for WindscribeVPN +- *cdm*: Normalize CDM detection for local and remote implementations +- *HLS*: Improve audio codec handling with error handling for codec extraction +- *tracks*: Prioritize Atmos audio tracks over higher bitrate non-Atmos -### Fixed +### Bug Fixes -- **Hybrid DV+HDR10 Filename Detection**: Fixed HDR10 detection in hybrid Dolby Vision filenames - - Hybrid DV+HDR10 files were incorrectly named "DV.H.265" instead of "DV.HDR.H.265" - - Now checks both `hdr_format_full` and `hdr_format_commercial` fields for HDR10 indicators -- **Vault Adaptive Batch Sizing**: Improved bulk key operations with adaptive batch sizing - - Prevents query limit issues when retrieving large numbers of keys from vaults - - Dynamically adjusts batch sizes based on vault response characteristics -- **Test Command Improvements**: Enhanced test command error detection and sorting - - Improved error detection in test command output - - Added natural sorting for test results +- *subs*: Update SubtitleEdit CLI syntax and respect conversion_method +- *n_m3u8dl_re*: Include language in DASH audio track selection +- *hls*: Prefer media playlist keys over session keys for accurate KID matching +- *deps*: Upgrade vulnerable dependencies for security alerts +- *serve*: Use correct pywidevine users config format +- *cdm*: Correct error key casing in Decrypt Labs API response parsing +- *api*: Validate Bearer prefix before extracting API key +- *serve*: Correct PlayReady RemoteCDM server validation +- *n_m3u8dl_re*: Remove duplicate --write-meta-json argument causing download failures +- *manifests*: Correct DRM type selection for remote PlayReady CDMs +- *proxies*: Fixes WindscribeVPN server authentication +- *subs*: Route pycaption-unsupported formats to pysubs2 in auto mode +- *proxy*: Remove regional restrictions from WindscribeVPN +- *proxy*: Collect servers from all locations in WindscribeVPN +- *downloader*: Correct progress bar tracking for segmented downloads +- *binaries*: Search subdirectories for binary files +- *dash*: Handle high startNumber in SegmentTimeline for DVR manifests +- *drm*: Hide Shaka Packager message for MonaLisa decryption +- *dash*: Add CENC namespace support for PSSH extraction +- *dash*: Preserve MPD DRM instead of overwriting from init segment +- *subtitles*: Preserve sidecar originals +- *mux*: Avoid audio codec suffix on split-audio outputs +- *dl*: Prevent attachment downloads during --skip-dl +- *progress*: Force track bar completion on terminal states +- *progress*: Bind per-track bars and force terminal completion +- *dl*: Keep descriptive and standard audio for requested langs +- *dl*: Always clean up hybrid temp hevc outputs +- *hls*: Finalize n_m3u8dl_re outputs +- *downloader*: Restore requests progress for single-url downloads +- *dl*: Invert audio codec suffixing when splitting +- *dl*: Support snake_case keys for RemoteCdm +- *aria2c*: Warn on config mismatch and wait for RPC ready +- *serve*: [**breaking**] Make PlayReady users config consistently a mapping +- *dl*: Preserve proxy_query selector (not resolved URI) +- *gluetun*: Stop leaking proxy/vpn secrets to process list +- *monalisa*: Avoid leaking secrets and add worker safety +- *dl*: Avoid selecting all variants when multiple audio codecs requested +- *hls*: Keep range offset numeric and align MonaLisa licensing +- *titles*: Remove trailing space from HDR dynamic range label +- *config*: Normalize playready_remote remote_cdm keys +- *titles*: Avoid None/double spaces in HDR tokens +- *naming*: Keep technical tokens with scene_naming off +- *api*: Log PSSH extraction failures +- *proxies*: Harden surfshark and windscribe selection +- *service*: Redact proxy credentials in logs +- *monalisa*: Harden wasm calls and license handling +- *hls*: Remove no-op encryption_data reassignment +- *serve*: Default PlayReady access to none +- *tracks*: Close temp session and improve path type error +- *main*: Update copyright year dynamically in version display -## [2.1.0] - 2025-11-27 +### Reverts -### Added - -- **Per-Track Quality-Based CDM Selection**: Dynamic CDM switching during runtime DRM operations - - Enables quality-based CDM selection during runtime DRM switching - - Different CDMs can be used for different video quality levels within the same download session - - Example: Use Widevine L3 for SD/HD and PlayReady SL3 for 4K content -- **Enhanced Track Export**: Improved export functionality with additional metadata - - Added URL field to track export for easier identification - - Added descriptor information in export output - - Keys now exported in hex-formatted strings - -### Changed - -- **Dependencies**: Upgraded to latest compatible versions - - Updated various dependencies to their latest versions - -### Fixed - -- **Attachment Preservation**: Fixed attachments being dropped during track filtering - - Attachments (screenshots, fonts) were being lost when track list was rebuilt - - Fixes image files remaining in temp directory after muxing -- **DASH BaseURL Resolution**: Added AdaptationSet-level BaseURL support per DASH spec - - URL resolution chain now properly follows: MPD → Period → AdaptationSet → Representation -- **WindscribeVPN Region Support**: Restricted to supported regions with proper error handling - - Added error handling for unsupported regions in get_proxy method - - Prevents cryptic errors when using unsupported region codes -- **Filename Sanitization**: Fixed space-hyphen-space handling in filenames - - Pre-process space-hyphen-space patterns (e.g., "Title - Episode") before other replacements - - Made space-hyphen-space handling conditional on scene_naming setting - - Addresses PR #44 by fixing the root cause -- **CICP Enum Values**: Corrected values to match ITU-T H.273 specification - - Added Primaries.Unspecified (value 2) per H.273 spec - - Renamed Primaries/Transfer value 0 from Unspecified to Reserved for spec accuracy - - Simplified Transfer value 2 from Unspecified_Image to Unspecified - - Verified against ITU-T H.273, ISO/IEC 23091-2, H.264/H.265 specs, and FFmpeg enums -- **HLS Byte Range Parsing**: Fixed TypeError in range_offset conversion - - Converted range_offset to int to prevent TypeError in calculate_byte_range -- **pyplayready Compatibility**: Pinned to <0.7 to avoid KID extraction bug - -## [2.0.0] - 2025-11-10 - -### Breaking Changes - -- **REST API Integration**: Core architecture modified to support REST API functionality - - Changes to internal APIs for download management and tracking - - Title and track classes updated with API integration points - - Core component interfaces modified for queue management support -- **Configuration Changes**: New required configuration options for API and enhanced features - - Added `simkl_client_id` now required for Simkl functionality - - Service-specific configuration override structure introduced - - Debug logging configuration options added -- **Forced Subtitles**: Behavior change for forced subtitle inclusion - - Forced subs no longer auto-included, requires explicit `--forced-subs` flag - -### Added - -- **REST API Server**: Complete download management via REST API (early development) - - Implemented download queue management and worker system - - Added OpenAPI/Swagger documentation for easy API exploration - - Included download progress tracking and status endpoints - - API authentication and comprehensive error handling - - Updated core components to support API integration - - Early development work with more changes planned -- **CustomRemoteCDM**: Highly configurable custom CDM API support - - Support for third-party CDM API providers with maximum configurability - - Full configuration through YAML without code changes - - Addresses GitHub issue #26 for flexible CDM integration -- **WindscribeVPN Proxy Provider**: New VPN provider support - - Added WindscribeVPN following NordVPN and SurfsharkVPN patterns - - Fixes GitHub issue #29 -- **Latest Episode Download**: New `--latest-episode` CLI option - - `-le, --latest-episode` flag to download only the most recent episode - - Automatically selects the single most recent episode regardless of season - - Fixes GitHub issue #28 -- **Video Track Exclusion**: New `--no-video` CLI option - - `-nv, --no-video` flag to skip downloading video tracks - - Allows downloading only audio, subtitles, attachments, and chapters - - Useful for audio-only or subtitle extraction workflows - - Fixes GitHub issue #39 -- **Service-Specific Configuration Overrides**: Per-service fine-tuned control - - Support for per-service configuration overrides in YAML - - Fine-tuned control of downloader and command options per service - - Fixes GitHub issue #13 -- **Comprehensive JSON Debug Logging**: Structured logging for troubleshooting - - Binary toggle via `--debug` flag or `debug: true` in config - - JSON Lines (.jsonl) format for easy parsing and analysis - - Comprehensive logging of all operations (session info, CLI params, CDM details, auth status, title/track metadata, DRM operations, vault queries) - - Configurable key logging via `debug_keys` option with smart redaction - - Error logging for all critical operations - - Removed old text logging system -- **curl_cffi Retry Handler**: Enhanced session reliability - - Added automatic retry mechanism to curl_cffi Session - - Improved download reliability with configurable retries -- **Simkl API Configuration**: New API key support - - Added `simkl_client_id` configuration option - - Simkl now requires client_id from https://simkl.com/settings/developer/ -- **Custom Session Fingerprints**: Enhanced browser impersonation capabilities - - Added custom fingerprint and preset support for better service compatibility - - Configurable fingerprint presets for different device types - - Improved success rate with services using advanced bot detection -- **TMDB and Simkl Metadata Caching**: Enhanced title cache system - - Added metadata caching to title cache to reduce API calls - - Caches movie/show metadata alongside title information - - Improves performance for repeated title lookups and reduces API rate limiting -- **API Enhancements**: Improved REST API functionality - - Added default parameter handling for better request processing - - Added URL field to services endpoint response for easier service identification - - Complete API enhancements for production readiness - - Improved error responses with better detail and debugging information - -### Changed - -- **Binary Search Enhancement**: Improved binary discovery - - Refactored to search for binaries in root of binary folder or subfolder named after the binary - - Better organization of binary dependencies -- **Type Annotations**: Modernized to PEP 604 syntax - - Updated session.py type annotations to use modern Python syntax - - Improved code readability and type checking - -### Fixed - -- **Audio Description Track Support**: Added option to download audio description tracks - - Added `--audio-description/-ad` flag to optionally include descriptive audio tracks - - Previously, audio description tracks were always filtered out - - Users can now choose to download AD tracks when needed - - Fixes GitHub issue #33 -- **Config Directory Support**: Cross-platform user config directory support - - Fixed config loading to properly support user config directories across all platforms - - Fixes GitHub issue #23 -- **HYBRID Mode Validation**: Pre-download validation for hybrid processing - - Added validation to check both HDR10 and DV tracks are available before download - - Prevents wasted downloads when hybrid processing would fail -- **TMDB/Simkl API Keys**: Graceful handling of missing API keys - - Improved error handling when TMDB or Simkl API keys are not configured - - Better user messaging for API configuration requirements -- **Forced Subtitles Behavior**: Correct forced subtitle filtering - - Fixed forced subtitles being incorrectly included without `--forced-subs` flag - - Forced subs now only included when explicitly requested -- **Font Attachment Constructor**: Fixed ASS/SSA font attachment - - Use keyword arguments for Attachment constructor in font attachment - - Fixes "Invalid URL: No scheme supplied" error - - Fixes GitHub issue #24 -- **Binary Subdirectory Checking**: Enhanced binary location discovery (by @TPD94, PR #19) - - Updated binaries.py to check subdirectories in binaries folders named after the binary - - Improved binary detection and loading -- **HLS Manifest Processing**: Minor HLS parser fix (by @TPD94, PR #19) -- **lxml and pyplayready**: Updated dependencies (by @Sp5rky) - - Updated lxml constraint and pyplayready import path for compatibility -- **DASH Segment Calculation**: Corrected segment handling - - Fixed segment count calculation for DASH manifests with startNumber=0 - - Ensures accurate segment processing for all DASH manifest configurations - - Prevents off-by-one errors in segment downloads -- **HDR Detection and Naming**: Comprehensive HDR format support - - Improved HDR detection with comprehensive transfer characteristics checks - - Added hybrid DV+HDR10 support for accurate file naming - - Better identification of HDR formats across different streaming services - - More accurate HDR/DV detection in filename generation -- **Subtitle Processing**: VTT subtitle handling improvements - - Resolved SDH (Subtitles for Deaf and Hard of hearing) stripping crash when processing VTT files - - More robust subtitle processing pipeline with better error handling - - Fixes crashes when filtering specific VTT subtitle formats -- **DRM Processing**: Enhanced encoding handling - - Added explicit UTF-8 encoding to mp4decrypt subprocess calls - - Prevents encoding issues on systems with non-UTF-8 default encodings - - Improves cross-platform compatibility for Windows and some Linux configurations -- **Session Fingerprints**: Updated OkHttp presets - - Updated OkHttp fingerprint presets for better Android TV compatibility - - Improved success rate with services using fingerprint-based detection +- *monalisa*: Pass key via argv again ### Documentation -- **GitHub Issue Templates**: Enhanced issue reporting - - Improved bug report template with better structure and required fields - - Enhanced feature request template for clearer specifications - - Added helpful guidance for contributors to provide complete information +- Add configuration documentation WIP +- *changelog*: Add 2.4.0 release notes +- *changelog*: Update cliff config and regenerate changelog +- *changelog*: Complete 2.4.0 notes +- *config*: Clarify sdh_method uses subtitle-filter -### Refactored +### Performance Improvements -- **Import Cleanup**: Removed unused imports - - Removed unused mypy import from binaries.py - - Fixed import ordering in API download_manager and handlers +- *aria2c*: Improve download performance with singleton manager -### Contributors +### Changes -This release includes contributions from: +- *remote_auth*: Remove unused requests.Session +- Remove remote-service code until feature is more complete -- @Sp5rky - REST API server implementation, dependency updates -- @stabbedbybrick - curl_cffi retry handler (PR #31) -- @stabbedbybrick - n_m3u8dl-re refactor (PR #38) -- @TPD94 - Binary search enhancements, manifest parser fixes (PR #19) -- @scene (Andy) - Core features, configuration system, bug fixes +### Maintenance + +- *api*: Remove remote services + +### Chore + +- *release*: [**breaking**] Bump version to 3.0.0 + +## [2.3.0] - 2026-01-18 + +### Features + +- *config*: Add unicode_filenames option to preserve native characters + +### Bug Fixes + +- *drm*: Correct PSSH system ID comparison in PlayReady +- *dash*: Handle placeholder KIDs and improve DRM init from segments +- *dash*: Handle N_m3u8DL-RE merge and decryption +- *drm*: Include shaka-packager binary in error messages +- *subs*: Strip whitespace from ASS font names +- *subs*: Handle negative TTML values in multi-value attributes +- *drm*: Filter Widevine PSSH by system ID instead of sorting +- *subs*: Handle WebVTT cue identifiers and overlapping multi-line cues + +## [2.2.0] - 2026-01-15 + +### Features + +- *debug*: Add comprehensive debug logging for downloaders and muxing +- *drm*: Add CDM-aware PlayReady fallback detection + +### Bug Fixes + +- *util*: Improve test command error detection and add natural sorting +- *vaults*: Batch bulk key operations to avoid query limits +- *titles*: Detect HDR10 in hybrid DV filenames correctly +- *vaults*: Adaptive batch sizing for bulk key operations + +## [2.1.0] - 2025-11-27 + +### Features + +- *export*: Enhance track export with URL, descriptor, and hex-formatted keys +- *cdm*: Add per-track quality-based CDM selection during runtime DRM switching +- Merge upstream dev branch + +### Bug Fixes + +- *deps*: Pin pyplayready to <0.7 to avoid KID extraction bug +- *hls*: Convert range_offset to int to prevent TypeError +- *video*: Correct CICP enum values to match ITU-T H.273 specification +- *utilities*: Handle space-hyphen-space separators in sanitize_filename +- *utilities*: Make space-hyphen-space handling conditional on scene_naming +- *windscribevpn*: Add error handling for unsupported regions in get_proxy method +- Restrict WindscribeVPN to supported regions +- *dash*: Add AdaptationSet-level BaseURL resolution +- *dl*: Preserve attachments when rebuilding track list + +## [2.0.0] - 2025-11-10 + +### Features + +- Add REST API server with download management +- Add comprehensive JSON debug logging system +- *cdm*: Add highly configurable CustomRemoteCDM for flexible API support +- *proxies*: Add WindscribeVPN proxy provider support +- *dl*: Add --latest-episode option to download only the most recent episode +- Add service-specific configuration overrides +- Add retry handler to curl_cffi Session +- *dl*: Add --audio-description flag to download AD tracks +- *api*: Add url field to services endpoint response +- *api*: Complete API enhancements for v2.0.0 +- *api*: Add default parameter handling and improved error responses +- *cache*: Add TMDB and Simkl metadata caching to title cache +- *session*: Add custom fingerprint and preset support +- *fonts*: Add Linux font support for ASS/SSA subtitles +- *subtitle*: Preserve original formatting when no conversion requested +- *dl*: Add --no-video flag to skip video track downloads + +### Bug Fixes + +- Use keyword arguments for Attachment constructor in font attachment +- Only exclude forced subs when --forced-subs flag is not set +- Update lxml constraint and pyplayready import path +- *tags*: Gracefully handle missing TMDB/Simkl API keys +- *config*: Support config in user config directory across platforms +- *dl*: Validate HYBRID mode requirements before download +- *drm*: Add explicit UTF-8 encoding to mp4decrypt subprocess calls +- *subtitle*: Resolve SDH stripping crash with VTT files +- *naming*: Improve HDR detection with comprehensive transfer checks and hybrid DV+HDR10 support +- *dash*: Correct segment count calculation for startNumber=0 +- *session*: Update OkHttp fingerprint presets +- *session*: Remove padding extension from OkHttp JA3 fingerprints +- *dl*: Prevent vault loading when --cdm-only flag is set +- *cdm*: Resolve session key handling for partial cached keys +- *cdm*: Apply session key fix to custom_remote_cdm +- *n_m3u8dl_re*: Read lang attribute from DASH manifests correctly +- *subtitles*: Fix closure bug preventing SDH subtitle stripping +- Ensure subtitles use requests downloader instead of n_m3u8dl_re if Descriptor.URL +- Suppress verbose fontTools logging when scanning system fonts +- *tags*: Skip metadata lookup when API keys not configured + +### Documentation + +- Improve GitHub issue templates for better bug reports and feature requests +- Add dev branch and update README +- Update CHANGELOG for audio description feature +- *changelog*: Complete v2.0.0 release documentation +- *changelog*: Add --no-video flag and PR #38 credit +- *changelog*: Set release date for version 2.0.0 +- *readme*: Remove dev branch warning for main merge + +### Changes + +- *session*: Modernize type annotations to PEP 604 syntax +- *binaries*: Remove unused mypy import +- Remove unnecessary underscore prefixes from function names +- *tags*: Remove environment variable fallbacks for API keys + +### Maintenance + +- *api*: Fix import ordering in download_manager and handlers +- Update CHANGELOG.md for version 2.0.0 ## [1.4.8] - 2025-10-08 -### Added +### Features -- **Exact Language Matching**: New `--exact-lang` flag for precise language matching - - Enables strict language code matching without fallbacks -- **No-Mux Flag**: New `--no-mux` flag to skip muxing tracks into container files - - Useful for keeping individual track files separate -- **DecryptLabs API Integration for HTTP Vault**: Enhanced vault support - - Added DecryptLabs API support to HTTP vault for improved key retrieval -- **AC4 Audio Codec Support**: Enhanced audio format handling - - Added AC4 codec support in Audio class with updated mime/profile handling -- **pysubs2 Subtitle Conversion**: Extended subtitle format support - - Added pysubs2 subtitle conversion with extended format support - - Configurable conversion method in configuration +- Add AC4 codec support in Audio class and update mime/profile handling +- Add pysubs2 subtitle conversion with extended format support +- Add --no-mux flag to skip muxing tracks into container files +- *vaults*: Add DecryptLabs API support to HTTP vault +- Add --exact-lang flag for precise language matching -### Changed +### Bug Fixes -- **Audio Track Sorting**: Optimized audio track selection logic - - Improved audio track sorting by grouping descriptive tracks and sorting by bitrate - - Better identification of ATMOS and DD+ as highest quality for filenaming -- **pyplayready Update**: Upgraded to version 0.6.3 - - Updated import paths to resolve compatibility issues - - Fixed lxml constraints for better dependency management -- **pysubs2 Conversion Method**: Moved from auto to manual configuration - - pysubs2 no longer auto-selected during testing phase +- Optimize audio track sorting by grouping descriptive tracks and sorting by bitrate, fixes bug that does not identify ATMOS or DD+ as the highest quality available in filenaming. +- Update lxml constraint and pyplayready import path +- Dl.py +- Upgrade pyplayready to 0.6.3 and resolve import compatibility issues +- Suppress tinycss SyntaxWarning by initializing filter before imports +- (subtitle): Move pysubs2 to not be auto while in "testing" phase. -### Fixed +### Reverts -- **Remote CDM**: Fixed curl_cffi compatibility - - Added curl_cffi to instance checks in RemoteCDM -- **Temporary File Handling**: Improved encoding handling - - Specified UTF-8 encoding when opening temporary files +- Remove tinycss SyntaxWarning suppression and fix isort -### Reverted +### Documentation -- **tinycss SyntaxWarning Suppression**: Removed ineffective warning filter - - Reverted warnings filter that didn't work as expected for suppressing tinycss warnings +- Add pysubs2 conversion_method to configuration documentation + +### Maintenance + +- Bump version to 1.4.8 ## [1.4.7] - 2025-09-25 -### Added +### Features -- **curl_cffi Session Support**: Enhanced anti-bot protection with browser impersonation - - Added new session utility with curl_cffi support for bypassing anti-bot measures - - Browser impersonation support for Chrome, Firefox, and Safari user agents - - Full backward compatibility with requests.Session maintained - - Suppressed HTTPS proxy warnings for improved user experience -- **Download Retry Functionality**: Configurable retry mechanism for failed downloads - - Added retry count option to download function for improved reliability -- **Subtitle Requirements Options**: Enhanced subtitle download control - - Added options for required subtitles in download command - - Better control over subtitle track selection and requirements -- **Quality Selection Enhancement**: Improved quality selection options - - Added best available quality option in download command for optimal track selection -- **DecryptLabs API Integration**: Enhanced remote CDM configuration - - Added decrypt_labs_api_key to Config initialization for better API integration - -### Changed - -- **Manifest Parser Updates**: Enhanced compatibility across all parsers - - Updated DASH, HLS, ISM, and M3U8 parsers to accept curl_cffi sessions - - Improved cookie handling compatibility between requests and curl_cffi -- **Logging Improvements**: Reduced log verbosity for better user experience - - Changed duplicate track log level to debug to reduce console noise - - Dynamic CDM selection messages moved to debug-only output - -### Fixed - -- **Remote CDM Reuse**: Fixed KeyError in dynamic CDM selection - - Prevents KeyError when reusing remote CDMs in dynamic selection process - - Creates copy of CDM dictionary before modification to prevent configuration mutation - - Allows same CDM to be selected multiple times within session without errors +- Add options for required subtitles and best available quality in download command +- Add download retry count option to download function +- Add decrypt_labs_api_key to Config initialization and change duplicate track log level to debug +- Add curl_cffi session support with browser impersonation +- Update changelog for version 1.4.7 ## [1.4.6] - 2025-09-13 -### Added +### Features -- **Quality-Based CDM Selection**: Dynamic CDM selection based on video resolution - - Automatically selects appropriate CDM (L3/L1) based on video track quality - - Supports quality thresholds in configuration (>=, >, <=, <, exact match) - - Pre-selects optimal CDM based on highest quality across all video tracks - - Maintains backward compatibility with existing CDM configurations -- **Automatic Audio Language Metadata**: Intelligent embedded audio language detection - - Automatically sets audio language metadata when no separate audio tracks exist - - Smart video track selection based on title language with fallbacks - - Enhanced FFmpeg repackaging with audio stream metadata injection -- **Lazy DRM Loading**: Deferred DRM loading for multi-track key retrieval optimization - - Add deferred DRM loading to M3U8 parser to mark tracks for later processing - - Just-in-time DRM loading during download process for better performance +- Automatic audio language metadata for embedded audio tracks +- Add quality-based CDM selection for dynamic CDM switching -### Changed +### Bug Fixes -- **Enhanced CDM Management**: Improved CDM switching logic for multi-quality downloads - - CDM selection now based on highest quality track to avoid inefficient switching - - Quality-based selection only within same DRM type (Widevine-to-Widevine, PlayReady-to-PlayReady) - - Single CDM used per session for better performance and reliability +- Resolve service name transmission and vault case sensitivity issues +- Improve import ordering and code formatting -### Fixed +### Maintenance -- **Vault Caching Issues**: Fixed vault count display and NoneType iteration errors - - Fix 'NoneType' object is not iterable error in DecryptLabsRemoteCDM - - Fix vault count display showing 0/3 instead of actual successful vault count -- **Service Name Transmission**: Resolved DecryptLabsRemoteCDM service name issues - - Fixed DecryptLabsRemoteCDM sending 'generic' instead of proper service names - - Added case-insensitive vault lookups for SQLite/MySQL vaults - - Added local vault integration to DecryptLabsRemoteCDM -- **Import Organization**: Improved import ordering and code formatting - - Reorder imports in decrypt_labs_remote_cdm.py for better organization - - Clean up trailing whitespace in vault files - -### Configuration - -- **New CDM Configuration Format**: Extended `cdm:` section supports quality-based selection - ```yaml - cdm: - SERVICE_NAME: - "<=1080": l3_cdm_name - ">1080": l1_cdm_name - default: l3_cdm_name - ``` +- Bump version to 1.4.6 and update changelog ## [1.4.5] - 2025-09-09 -### Added +### Features -- **Enhanced CDM Key Caching**: Improved key caching and session management for L1/L2 devices - - Optimized `get_cached_keys_if_exists` functionality for better performance with L1/L2 devices - - Enhanced cached key retrieval logic with improved session handling -- **Widevine Common Certificate Fallback**: Added fallback to Widevine common certificate for L1 devices - - Improved compatibility for L1 devices when service certificates are unavailable -- **Enhanced Vault Loading**: Improved vault loading and key copying logic - - Better error handling and key management in vault operations -- **PSSH Display Optimization**: Truncated PSSH string display in non-debug mode for cleaner output -- **CDM Error Messaging**: Added error messages for missing service certificates in CDM sessions +- *changelog*: Update changelog for version 1.4.4 with enhanced CDM support, configuration options, and various improvements +- *cdm*: Enhance key retrieval logic and improve cached keys handling +- Implement intelligent caching system for CDM license requests +- *tags*: Enhance tag handling for TV shows and movies from Simkl data +- *kv*: Enhance vault loading and key copying logic +- *dl*: Truncate PSSH string for display in non-debug mode +- *cdm*: Add fallback to Widevine common cert for L1 devices +- *cdm*: Optimize get_cached_keys_if_exists for L1/L2 devices +- *cdm*: Update User-Agent to use dynamic version -### Changed +### Bug Fixes -- **Dynamic Version Headers**: Updated User-Agent headers to use dynamic version strings - - DecryptLabsRemoteCDM now uses dynamic version import instead of hardcoded version -- **Intelligent CDM Caching**: Implemented intelligent caching system for CDM license requests - - Enhanced caching logic reduces redundant license requests and improves performance -- **Enhanced Tag Handling**: Improved tag handling for TV shows and movies from Simkl data - - Better metadata processing and formatting for improved media tagging +- *tags*: Fix import order. +- *cdm*: Add error message for missing service certificate in CDM session +- *tags*: Fix formatting issues -### Fixed +### Maintenance -- **CDM Session Management**: Clean up session data when retrieving cached keys - - Remove decrypt_labs_session_id and challenge from session when cached keys exist but there are missing kids - - Ensures clean state for subsequent requests and prevents session conflicts -- **Tag Formatting**: Fixed formatting issues in tag processing -- **Import Order**: Fixed import order issues in tags module +- Bump version to 1.4.5 and update changelog ## [1.4.4] - 2025-09-02 -### Added +### Features -- **Enhanced DecryptLabs CDM Support**: Comprehensive remote CDM functionality - - Full support for Widevine, PlayReady, and ChromeCDM through DecryptLabsRemoteCDM - - Enhanced session management and caching support for remote WV/PR operations - - Support for cached keys and improved license handling - - New CDM configurations for Chrome and PlayReady devices with updated User-Agent and service certificate -- **Advanced Configuration Options**: New device and language preferences - - Added configuration options for device certificate status list - - Enhanced language preference settings +- *ip-info*: Add cached IP info retrieval with fallback tester to avoid rate limiting +- *ip-info*: Fix few more issues with the get_ip_info make sure we failover to different provider on 429 errors and allow future for more API providers to be added later. +- *release*: Bump version to 1.4.3 and update changelog with new features and improvements +- *config*: Add new configuration options for device certificate status list and language preferences +- *cdm*: Enhance DecryptLabsRemoteCDM to support cached keys and improve license handling +- *cdm*: Enhance DecryptLabsRemoteCDM with improved session management and caching support and better support for remote WV/PR +- *cdm*: Add DecryptLabs CDM configurations for Chrome and PlayReady devices with updated User-Agent and service certificate +- *cdm*: Refactor DecryptLabsRemoteCDM full support for Widevine/Playready and ChromeCDM -### Changed +### Bug Fixes -- **DRM Decryption Enhancements**: Streamlined decryption process - - Simplified decrypt method by removing unused parameter and streamlined logic - - Improved DecryptLabs CDM configurations with better device support +- *dependencies*: Remove unnecessary data extra requirement from langcodes +- *main*: As requested old devine version removed from banner to avoid any confusion the developer of this software. Original GNU is still applys. +- *tags*: Fix Matroska tag compliance with official specification -### Fixed +### Changes -- **Matroska Tag Compliance**: Enhanced media container compatibility - - Fixed Matroska tag compliance with official specification -- **Application Branding**: Cleaned up version display - - Removed old devine version reference from banner to avoid developer confusion - - Updated branding while maintaining original GNU license compliance -- **IP Information Handling**: Improved geolocation services - - Enhanced get_ip_info functionality with better failover handling - - Added support for 429 error handling and multiple API provider fallback - - Implemented cached IP info retrieval with fallback tester to avoid rate limiting -- **Dependencies**: Streamlined package requirements - - Removed unnecessary data extra requirement from langcodes - -### Removed - -- Deprecated version references in application banner for clarity - -## [1.4.3] - 2025-08-20 - -### Added - -- Cached IP info helper for region detection - - New `get_cached_ip_info()` with 24h cache and provider rotation (ipinfo/ipapi) with 429 handling. - - Reduces external calls and stabilizes non-proxy region lookups for caching/logging. - -### Changed - -- DRM decryption selection is fully configuration-driven - - Widevine and PlayReady now select the decrypter based solely on `decryption` in YAML (including per-service mapping). - - Shaka Packager remains the default decrypter when not specified. - - `dl.py` logs the chosen tool based on the resolved configuration. -- Geofencing and proxy verification improvements - - Safer geofence checks with error handling and clearer logs. - - Always verify proxy exit region via live IP lookup; fallback to proxy parsing on failure. -- Example config updated to default to Shaka - - `unshackle.yaml`/example now sets `decryption.default: shaka` (service overrides still supported). - -### Removed - -- Deprecated parameter `use_mp4decrypt` - - Removed from `Widevine.decrypt()` and `PlayReady.decrypt()` and all callsites. - - Internal naming switched from mp4decrypt-specific flags to generic `decrypter` selection. +- *drm*: Simplify decrypt method by removing unused parameter and streamline logic ## [1.4.2] - 2025-08-14 -### Added +### Features -- **Session Management for API Requests**: Enhanced API reliability with retry logic - - Implemented session management for tags functionality with automatic retry mechanisms - - Improved API request stability and error handling -- **Series Year Configuration**: New `series_year` option for title naming control - - Added configurable `series_year` option to control year inclusion in series titles - - Enhanced YAML configuration with series year handling options -- **Audio Language Override**: New audio language selection option - - Added `audio_language` option to override default language selection for audio tracks - - Provides more granular control over audio track selection -- **Vault Key Reception Control**: Enhanced vault security options - - Added `no_push` option to Vault and its subclasses to control key reception - - Improved key management security and flexibility +- *dl*: Add audio language option to override language for audio tracks +- *vault*: Add no_push option to Vault and its subclasses to control key reception +- *hls*: Enhance segment merging with recursive file search and fallback to binary concatenation +- *hls*: Enhance segment retrieval by allowing all file types and clean up empty segment directories. Fixes issues with VTT files from HLS not being found correctly due to new HLS "changes" +- *config*: Add series_year option to control year inclusion in titles and YAML configuration +- *tags*: Implement session management for API requests with retry logic +- *release*: Bump version to 1.4.2 and update changelog with new features and fixes -### Changed +### Bug Fixes -- **HLS Segment Processing**: Enhanced segment retrieval and merging capabilities - - Enhanced segment retrieval to allow all file types for better compatibility - - Improved segment merging with recursive file search and fallback to binary concatenation - - Fixed issues with VTT files from HLS not being found correctly due to format changes - - Added cleanup of empty segment directories after processing -- **Documentation**: Updated README.md with latest information - -### Fixed - -- **Audio Track Selection**: Improved per-language logic for audio tracks - - Adjusted `per_language` logic to ensure correct audio track selection - - Fixed issue where all tracks for selected language were being downloaded instead of just the intended ones +- *dl*: Adjust per_language logic to ensure correct audio track selection and not download all tracks for selected language. ## [1.4.1] - 2025-08-08 -### Added +### Features -- **Title Caching System**: Intelligent title caching to reduce redundant API calls - - Configurable title caching with 30-minute default cache duration - - 24-hour fallback cache on API failures for improved reliability - - Region-aware caching to handle geo-restricted content properly - - SHA256 hashing for cache keys to handle complex title IDs - - Added `--no-cache` CLI flag to bypass caching when needed - - Added `--reset-cache` CLI flag to clear existing cache data - - New cache configuration variables in config system - - Documented caching options in example configuration file - - Significantly improves performance when debugging or modifying CLI parameters -- **Enhanced Tagging Configuration**: New options for customizing tag behavior - - Added `tag_group_name` config option to control group name inclusion in tags - - Added `tag_imdb_tmdb` config option to control IMDB/TMDB details in tags - - Added Simkl API endpoint support as fallback when no TMDB API key is provided - - Enhanced tag_file function to prioritize provided TMDB ID when `--tmdb` flag is used - - Improved TMDB ID handling with better prioritization logic +- Implement title caching system to reduce API calls +- *dl*: Update language option default to 'orig' if no -l is set, avoids hardcoded en +- *config*: Add options for tagging with group name and IMDB/TMDB details and new API endpoint of simkl if no tmdb api key is added. +- *tags*: Enhance tag_file function to prioritize provided TMDB ID if --tmdb is used +- *changelog*: Update changelog with enhanced tagging configuration and improvements -### Changed +### Bug Fixes -- **Language Selection Enhancement**: Improved default language handling - - Updated language option default to 'orig' when no `-l` flag is set - - Avoids hardcoded 'en' default and respects original content language -- **Tagging Logic Improvements**: Simplified and enhanced tagging functionality - - Simplified Simkl search logic with soft-fail when no results found - - Enhanced tag_file function with better TMDB ID prioritization - - Improved error handling in tagging operations +- *subtitle*: Handle ValueError in subtitle filtering for multiple colons in time references fixes issues with subtitles that contain multiple colons -### Fixed +### Changes -- **Subtitle Processing**: Enhanced subtitle filtering for edge cases - - Fixed ValueError in subtitle filtering for multiple colons in time references - - Improved handling of subtitles containing complex time formatting - - Better error handling for malformed subtitle timestamps - -### Removed - -- **Docker Support**: Removed Docker configuration from repository - - Removed Dockerfile and .dockerignore files - - Cleaned up README.md Docker-related documentation - - Focuses on direct installation methods +- Remove Dockerfile and .dockerignore from the repository +- *tags*: Simplify Simkl search logic and soft-fail when no results found ## [1.4.0] - 2025-08-05 -### Added +### Features -- **HLG Transfer Characteristics Preservation**: Enhanced video muxing to preserve HLG color metadata - - Added automatic detection of HLG video tracks during muxing process - - Implemented `--color-transfer-characteristics 0:18` argument for mkvmerge when processing HLG content - - Prevents incorrect conversion from HLG (18) to BT.2020 (14) transfer characteristics - - Ensures proper HLG playback support on compatible hardware without manual editing -- **Original Language Support**: Enhanced language selection with 'orig' keyword support - - Added support for 'orig' language selector for both video and audio tracks - - Automatically detects and uses the title's original language when 'orig' is specified - - Improved language processing logic with better duplicate handling - - Enhanced help text to document original language selection usage -- **Forced Subtitle Support**: Added option to include forced subtitle tracks - - New functionality to download and include forced subtitle tracks alongside regular subtitles -- **WebVTT Subtitle Filtering**: Enhanced subtitle processing capabilities - - Added filtering for unwanted cues in WebVTT subtitles - - Improved subtitle quality by removing unnecessary metadata +- *update_checker*: Enhance update checking logic and cache handling +- *dl*: Add option to include forced subtitle tracks +- *subtitle*: Add filtering for unwanted cues in WebVTT subtitles +- *tracks*: Add support for HLG color transfer characteristics in video arguments +- *dl*: Enhance language selection for video and audio tracks, including original language support +- *dl*: Improve DRM track decryption handling +- *series*: Enhance tree representation with season breakdown +- *hybrid*: Enhance extraction and conversion processes with dymanic spinning bars to follow the rest of the codebase. +- *dl*: Fix track selection to support combining -V, -A, -S flags +- *titles*: Better detection of DV across all codecs in Episode and Movie classes dvhe.05.06 was not being detected correctly. +- *dl*: Add support for services that do not support subtitle downloads +- *playready*: Enhance KID extraction from PSSH with base64 support and XML parsing +- Bump version to 1.4.0 and update changelog with new features and fixes -### Changed +### Maintenance -- **DRM Track Decryption**: Improved DRM decryption track selection logic - - Enhanced `get_drm_for_cdm()` method usage for better DRM-CDM matching - - Added warning messages when no matching DRM is found for tracks - - Improved error handling and logging for DRM decryption failures -- **Series Tree Representation**: Enhanced episode tree display formatting - - Updated series tree to show season breakdown with episode counts - - Improved visual representation with "S{season}({count})" format - - Better organization of series information in console output -- **Hybrid Processing UI**: Enhanced extraction and conversion processes - - Added dynamic spinning bars to follow the rest of the codebase design - - Improved visual feedback during hybrid HDR processing operations -- **Track Selection Logic**: Enhanced multi-track selection capabilities - - Fixed track selection to support combining -V, -A, -S flags properly - - Improved flexibility in selecting multiple track types simultaneously -- **Service Subtitle Support**: Added configuration for services without subtitle support - - Services can now indicate if they don't support subtitle downloads - - Prevents unnecessary subtitle download attempts for unsupported services -- **Update Checker**: Enhanced update checking logic and cache handling - - Improved rate limiting and caching mechanisms for update checks - - Better performance and reduced API calls to GitHub - -### Fixed - -- **PlayReady KID Extraction**: Enhanced KID extraction from PSSH data - - Added base64 support and XML parsing for better KID detection - - Fixed issue where only one KID was being extracted for certain services - - Improved multi-KID support for PlayReady protected content -- **Dolby Vision Detection**: Improved DV codec detection across all formats - - Fixed detection of dvhe.05.06 codec which was not being recognized correctly - - Enhanced detection logic in Episode and Movie title classes - - Better support for various Dolby Vision codec variants +- Update changelog with new features, enhancements, and fixes for version 1.3.0 +- Bump unshackle version to 1.3.0 in uv.lock ## [1.3.0] - 2025-08-03 -### Added +### Features -- **mp4decrypt Support**: Alternative DRM decryption method using mp4decrypt from Bento4 - - Added `mp4decrypt` binary detection and support in binaries module - - New `decryption` configuration option in unshackle.yaml for service-specific decryption methods - - Enhanced PlayReady and Widevine DRM classes with mp4decrypt decryption support - - Service-specific decryption mapping allows choosing between `shaka` and `mp4decrypt` per service - - Improved error handling and progress reporting for mp4decrypt operations -- **Scene Naming Configuration**: New `scene_naming` option for controlling file naming conventions - - Added scene naming logic to movie, episode, and song title classes - - Configurable through unshackle.yaml to enable/disable scene naming standards -- **Terminal Cleanup and Signal Handling**: Enhanced console management - - Implemented proper terminal cleanup on application exit - - Added signal handling for graceful shutdown in ComfyConsole -- **Configuration Template**: New `unshackle-example.yaml` template file - - Replaced main `unshackle.yaml` with example template to prevent git conflicts - - Users can now modify their local config without affecting repository updates -- **Enhanced Credential Management**: Improved CDM and vault configuration - - Expanded credential management documentation in configuration - - Enhanced CDM configuration examples and guidelines -- **Video Transfer Standards**: Added `Unspecified_Image` option to Transfer enum - - Implements ITU-T H.Sup19 standard value 2 for image characteristics - - Supports still image coding systems and unknown transfer characteristics -- **Update Check Rate Limiting**: Enhanced update checking system - - Added configurable update check intervals to prevent excessive API calls - - Improved rate limiting for GitHub API requests +- Add update check interval configuration and implement rate limiting for update checks +- Implement terminal cleanup on exit and signal handling in ComfyConsole +- Add Unspecified_Image option to Transfer enum in Video class. +- Enhance credential management and CDM configuration in unshackle.yaml +- Update path of update_check.json to .gitignore +- Add scene naming option to configuration and update naming logic in titles +- Add unshackle-example.yaml to replace the unshackle.yaml file, you can now make changes to the unshackle.yaml file and pull from the the repo without issues. +- *drm*: Add support for mp4decrypt as a decryption method -### Changed +### Bug Fixes -- **DRM Decryption Architecture**: Enhanced decryption system with dual method support - - Updated `dl.py` to handle service-specific decryption method selection - - Refactored `Config` class to manage decryption method mapping per service - - Enhanced DRM decrypt methods with `use_mp4decrypt` parameter for method selection -- **Error Handling**: Improved exception handling in Hybrid class - - Replaced log.exit calls with ValueError exceptions for better error propagation - - Enhanced error handling consistency across hybrid processing +- Correct URL handling and improve key retrieval logic in HTTP vault +- Rename 'servers' to 'server_map' for proxy configuration in unshackle.yaml to resolve nord/surfshark incorrect named config -### Fixed +### Changes -- **Proxy Configuration**: Fixed proxy server mapping in configuration - - Renamed 'servers' to 'server_map' in proxy configuration to resolve Nord/Surfshark naming conflicts - - Updated configuration structure for better compatibility with proxy providers -- **HTTP Vault**: Improved URL handling and key retrieval logic - - Fixed URL processing issues in HTTP-based key vaults - - Enhanced key retrieval reliability and error handling +- Replace log.exit calls with ValueError exceptions for error handling in Hybrid class + +### Maintenance + +- Bump version to 1.3.0 and update changelog with mp4decrypt support and enhancements ## [1.2.0] - 2025-07-30 -### Added +### Features -- **Update Checker**: Automatic GitHub release version checking on startup - - Configurable update notifications via `update_checks` setting in unshackle.yaml - - Non-blocking HTTP requests with 5-second timeout for performance - - Smart semantic version comparison supporting all version formats (x.y.z, x.y, x) - - Graceful error handling for network issues and API failures - - User-friendly update notifications with current → latest version display - - Direct links to GitHub releases page for easy updates -- **HDR10+ Support**: Enhanced HDR10+ metadata processing for hybrid tracks - - HDR10+ tool binary support (`hdr10plus_tool`) added to binaries module - - HDR10+ to Dolby Vision conversion capabilities in hybrid processing - - Enhanced metadata extraction for HDR10+ content -- **Duration Fix Handling**: Added duration correction for video and hybrid tracks -- **Temporary Directory Management**: Automatic creation of temp directories for attachment downloads +- *dl*: Enhance hybrid processing to handle HDR10 and DV tracks separately by resolution, Hotfix for -q 2160,1080 both tracks will have Hybrid correctly now. +- *hybrid*: Display resolution of HDR10 track in hybrid mode console output and clean up unused code +- *subtitle*: Add information into unshackle.yaml on how to use new Subby subtitle conversion. +- *vaults*: Enhance vault loading with success status +- *attachment*: Ensure temporary directory is created for downloads +- *tracks*: Add duration fix handling for video and hybrid tracks +- *hybrid*: Add HDR10+ support for conversion to Dolby Vision and enhance metadata extraction +- Update version to 1.1.1 and add update checking functionality +- Bump version to 1.2.0 and update changelog, I'll eventually learn symantic versioning. -### Changed +### Changes -- Enhanced configuration system with new `update_checks` boolean option (defaults to true) -- Updated sample unshackle.yaml with update checker configuration documentation -- Improved console styling consistency using `bright_black` for dimmed text -- **Environment Dependency Check**: Complete overhaul with detailed categorization and status summary - - Organized dependencies by category (Core, HDR, Download, Subtitle, Player, Network) - - Enhanced status reporting with compact summary display - - Improved tool requirement tracking and missing dependency alerts -- **Hybrid Track Processing**: Significant improvements to HDR10+ and Dolby Vision handling - - Enhanced metadata extraction and processing workflows - - Better integration with HDR processing tools +- *env*: Enhance dependency check with detailed categorization and status summary -### Removed +### Maintenance -- **Docker Workflow**: Removed Docker build and publish GitHub Actions workflow for manual builds +- *workflow*: Remove Docker build and publish workflow, its too messy at the moment doing manual builds for now. -## [1.1.0] - 2025-07-29 +## [1.1.0] - 2025-07-30 -### Added +### Features -- **HDR10+DV Hybrid Processing**: New `-r HYBRID` command for processing HDR10 and Dolby Vision tracks - - Support for hybrid HDR processing and injection using dovi_tool - - New hybrid track processing module for seamless HDR10/DV conversion - - Automatic detection and handling of HDR10 and DV metadata -- Support for HDR10 and DV tracks in hybrid mode for EXAMPLE service -- Binary availability check for dovi_tool in hybrid mode operations -- Enhanced track processing capabilities for HDR content +- Update version display in main.py +- *proxies*: Add SurfsharkVPN support +- *binaries*: Add support for `MKVToolNix` and `mkvpropedit` +- *subtitles*: Integrate `subby` library for enhanced subtitle processing and conversion methods +- *hybrid*: Implement HDR10+DV hybrid processing and injection support +- *EXAMPLE*: Add support for HDR10 and DV tracks in hybrid mode -### Fixed +### Bug Fixes -- Import order issues and missing json import in hybrid processing -- UV installation process and error handling improvements -- Binary search functionality updated to use `binaries.find` +- *cfg*: Update services directory handling +- *binaries*: Improve local binary search functionality +- *env*: Update binary search functionality to use `binaries.find` +- *env*: Update `Shaka-Packager` binary retrieval method +- *env*: Improve handling of directory paths in `info` command +- *install*: Improve UV installation process and error handling +- *download*: Skip Content-Length validation for compressed responses in curl_impersonate and requests. The fix ensures that when Content-Encoding indicates compression, we skip the validation by setting content_length = 0, allowing the downloads to complete successfully. +- *dl*: Check for dovi_tool availability in hybrid mode +- *download*: Skip Content-Length validation for compressed responses in curl_impersonate and requests -### Changed +### Maintenance -- Updated package version from 1.0.2 to 1.1.0 -- Enhanced dl.py command processing for hybrid mode support -- Improved core titles (episode/movie) processing for HDR content -- Extended tracks module with hybrid processing capabilities +- Bump version to 1.1.0 in pyproject.toml, __init__.py, and uv.lock to follow correct Semantic Versioning. +- Add CHANGELOG.md to document notable changes and version history + +## [1.0.1] - 2025-07-20 + +### Features + +- Enhance CONFIG.md with new configuration options for curl_impersonate, filenames, n_m3u8dl_re, and nordvpn +- Update .gitignore and enhance README with planned features +- Add .github/ to .gitignore to exclude GitHub-related files +- Implement VideoCodecChoice for enhanced codec selection +- Add Dockerfile and GitHub Actions workflow for building and publishing Docker image +- Update GitHub Actions workflow for Docker image build and add Docker installation instructions to README + +### Bug Fixes + +- Change default value of set_terminal_bg to False +- Add video_only condition to subtitle track selection logic fixes issues where ccextractor would run even with -V enabled +- Add SubtitleCodecChoice for resolving issues with config clicktype selection, using names like VTT or SRT was not working as expected +- Update shaka packager version and enhance Docker run command with additional volume mounts + +### Changes + +- Streamline README by removing outdated service and legal information and moved it directly to the WIKI +- Reorganize Planned Features section in README for clarity +- Improve track selection logic in dl.py + +[3.1.0]: https://github.com/unshackle-dl/unshackle/compare/3.0.0..3.1.0 +[3.0.0]: https://github.com/unshackle-dl/unshackle/compare/2.3.0..3.0.0 +[2.3.0]: https://github.com/unshackle-dl/unshackle/compare/2.2.0..2.3.0 +[2.2.0]: https://github.com/unshackle-dl/unshackle/compare/2.1.0..2.2.0 +[2.1.0]: https://github.com/unshackle-dl/unshackle/compare/2.0.0..2.1.0 +[2.0.0]: https://github.com/unshackle-dl/unshackle/compare/1.4.8..2.0.0 +[1.4.8]: https://github.com/unshackle-dl/unshackle/compare/1.4.7..1.4.8 +[1.4.7]: https://github.com/unshackle-dl/unshackle/compare/1.4.6..1.4.7 +[1.4.6]: https://github.com/unshackle-dl/unshackle/compare/1.4.5..1.4.6 +[1.4.5]: https://github.com/unshackle-dl/unshackle/compare/1.4.4..1.4.5 +[1.4.4]: https://github.com/unshackle-dl/unshackle/compare/1.4.2..1.4.4 +[1.4.2]: https://github.com/unshackle-dl/unshackle/compare/1.4.1..1.4.2 +[1.4.1]: https://github.com/unshackle-dl/unshackle/compare/1.4.0..1.4.1 +[1.4.0]: https://github.com/unshackle-dl/unshackle/compare/1.3.0..1.4.0 +[1.3.0]: https://github.com/unshackle-dl/unshackle/compare/1.2.0..1.3.0 +[1.2.0]: https://github.com/unshackle-dl/unshackle/compare/1.1.0..1.2.0 +[1.1.0]: https://github.com/unshackle-dl/unshackle/compare/1.0.1..1.1.0 diff --git a/CONFIG.md b/CONFIG.md deleted file mode 100644 index 942657a..0000000 --- a/CONFIG.md +++ /dev/null @@ -1,1482 +0,0 @@ -# Config Documentation - -This page documents configuration values and what they do. You begin with an empty configuration file. -You may alter your configuration with `unshackle cfg --help`, or find the direct location with `unshackle env info`. -Configuration values are listed in alphabetical order. - -Avoid putting comments in the config file as they may be removed. Comments are currently kept only thanks -to the usage of `ruamel.yaml` to parse and write YAML files. In the future `yaml` may be used instead, -which does not keep comments. - -## aria2c (dict) - -- `max_concurrent_downloads` - Maximum number of parallel downloads. Default: `min(32,(cpu_count+4))` - Note: Overrides the `max_workers` parameter of the aria2(c) downloader function. -- `max_connection_per_server` - Maximum number of connections to one server for each download. Default: `1` -- `split` - Split a file into N chunks and download each chunk on its own connection. Default: `5` -- `file_allocation` - Specify file allocation method. Default: `"prealloc"` - - - `"none"` doesn't pre-allocate file space. - - `"prealloc"` pre-allocates file space before download begins. This may take some time depending on the size of the - file. - - `"falloc"` is your best choice if you are using newer file systems such as ext4 (with extents support), btrfs, xfs - or NTFS (MinGW build only). It allocates large(few GiB) files almost instantly. Don't use falloc with legacy file - systems such as ext3 and FAT32 because it takes almost same time as prealloc, and it blocks aria2 entirely until - allocation finishes. falloc may not be available if your system doesn't have posix_fallocate(3) function. - - `"trunc"` uses ftruncate(2) system call or platform-specific counterpart to truncate a file to a specified length. - -## cdm (dict) - -Pre-define which Widevine or PlayReady device to use for each Service by Service Tag as Key (case-sensitive). -The value should be a WVD or PRD filename without the file extension, or a remote CDM name defined in `remote_cdm`. -When loading a local device, unshackle will look in both the `WVDs` and `PRDs` directories for a matching file. - -### Basic CDM Selection - -For example, - -```yaml -AMZN: chromecdm_903_l3 -NF: nexus_6_l1 -``` - -### Profile-Based CDM Selection - -You may also specify this device based on the profile used. - -For example, - -```yaml -AMZN: chromecdm_903_l3 -NF: nexus_6_l1 -DSNP: - john_sd: chromecdm_903_l3 - jane_uhd: nexus_5_l1 -``` - -### Default Fallback - -You can also specify a fallback value to predefine if a match was not made. -This can be done using `default` key. This can help reduce redundancy in your specifications. - -For example, the following has the same result as the previous example, as well as all other -services and profiles being pre-defined to use `chromecdm_903_l3`. - -```yaml -NF: nexus_6_l1 -DSNP: - jane_uhd: nexus_5_l1 -default: chromecdm_903_l3 -``` - -### Quality-Based CDM Selection - -**NEW:** You can now select different CDMs based on video resolution (quality). This allows you to use local CDMs -for lower qualities and automatically switch to remote CDMs for higher qualities that require L1/L2 security levels. - -unshackle automatically detects the highest quality video track and selects the appropriate CDM before downloading. - -#### Supported Quality Operators - -- **Exact match**: `"480"`, `"720"`, `"1080"`, `"2160"` - Matches exact resolution -- **Greater than or equal**: `">=1080"` - Matches 1080p and above (1440p, 2160p, etc.) -- **Greater than**: `">720"` - Matches above 720p (1080p, 1440p, 2160p, etc.) -- **Less than or equal**: `"<=1080"` - Matches 1080p and below -- **Less than**: `"<1080"` - Matches below 1080p - -**Note**: Quality keys must be quoted strings to preserve operators in YAML. - -#### Example: Local for SD/HD, Remote for 4K - -```yaml -cdm: - NETFLIX: - "<=1080": local_l3 # Use local CDM for 1080p and below - ">=1440": remote_l1 # Use remote L1 CDM for 1440p and above - default: local_l3 # Fallback if no quality match - - DISNEY: - "480": local_l3_mobile # Use mobile L3 for 480p - "720": local_l3 # Use local L3 for 720p - "1080": local_l3_hd # Use local L3 for 1080p - ">1080": remote_l1 # Use remote L1 for above 1080p -``` - -#### Example: Mixed Profile and Quality Selection - -```yaml -cdm: - AMAZON: - # Profile-based selection - john_account: johns_local_l3 - jane_account: janes_remote_l1 - - # Quality-based selection (for default profile) - "<=720": local_l3 - ">=1080": remote_l1 - - default: local_l3 -``` - -#### Example: Switching Between Widevine and PlayReady Based on Quality - -Some services may use different DRM systems for different quality levels. For example, AMAZON might use -Widevine (ChromeCDM) for SD/HD content but require PlayReady (SL3) for UHD content: - -```yaml -cdm: - AMAZON: - # Use local/remote Widevine ChromeCDM for 1080p and below - "<=1080": local_chromecdm - - # Use remote PlayReady SL3 for above 1080p (1440p, 2160p) - ">1080": remote_sl3 - - default: local_chromecdm - -# If using local CDMs, place chromecdm.wvd in your WVDs directory -# If using remote CDMs, configure them below: -remote_cdm: - - name: remote_chromecdm - type: decrypt_labs # Or custom_api - device_name: ChromeCDM - host: https://your-cdm-api.com - secret: YOUR_API_KEY - - - name: remote_sl3 - type: decrypt_labs # Or custom_api - device_name: SL3 - device_type: PLAYREADY - host: https://your-cdm-api.com - secret: YOUR_API_KEY -``` - -**How it works:** -- When downloading 720p or 1080p content → Uses `local_chromecdm` (local Widevine L3) -- When downloading 1440p or 2160p content → Uses `remote_sl3` (remote PlayReady SL3) -- unshackle automatically detects the video quality and selects the appropriate CDM -- The DRM type is verified against the content's actual DRM system - -**Note:** This configuration assumes the service uses different DRM systems for different qualities. -Most services use a single DRM system across all qualities, but some (like AMAZON) may vary by region or quality tier. - -### DRM-Specific CDM Selection (Widevine/PlayReady) - -For services that support multiple DRM systems, you can specify different CDMs based on the DRM type. -unshackle automatically detects the DRM system used by content and switches to the appropriate CDM. - -#### Example: Separate Widevine and PlayReady CDMs - -```yaml -cdm: - DISNEY: - widevine: - default: local_wv # Local Widevine CDM - ">=2160": remote_l1 # Remote L1 for 4K Widevine - - playready: - default: local_pr # Local PlayReady CDM - ">=1080": remote_sl2 # Remote SL2 for HD+ PlayReady -``` - -#### Example: AMAZON - Quality-Based with DRM Type Override - -For AMAZON, you might want to use ChromeCDM (Widevine) for SD/HD content and PlayReady SL3 for UHD content. -Here's a more explicit configuration using DRM-specific overrides: - -```yaml -cdm: - AMAZON: - # DRM-specific configuration with quality-based selection - widevine: - "<=1080": local_chromecdm # Local ChromeCDM for 1080p and below - default: local_chromecdm - - playready: - ">1080": remote_sl3 # Remote PlayReady SL3 for above 1080p - "<=1080": local_pr # Optional: Local PlayReady for lower quality - default: remote_sl3 - - # Fallback for unknown DRM types - default: local_chromecdm - -# Define remote CDMs (if using remote for high quality) -remote_cdm: - - name: remote_sl3 - type: decrypt_labs # Or custom_api - device_name: SL3 - device_type: PLAYREADY - host: https://your-cdm-api.com - secret: YOUR_API_KEY - - - name: remote_chromecdm - type: decrypt_labs # Or custom_api - device_name: ChromeCDM - host: https://your-cdm-api.com - secret: YOUR_API_KEY -``` - -**How it works:** -- If content uses **Widevine** → Uses `local_chromecdm` for all qualities up to 1080p -- If content uses **PlayReady** and quality > 1080p → Uses `remote_sl3` (remote SL3) -- If content uses **PlayReady** and quality ≤ 1080p → Uses `local_pr` (local, optional) -- Fallback for unknown DRM → Uses `local_chromecdm` - -**Alternative: Simple quality-based approach** (when DRM type varies by quality): - -```yaml -cdm: - AMAZON: - "<=1080": local_chromecdm # Local Widevine for SD/HD - ">1080": remote_sl3 # Remote PlayReady for UHD - default: local_chromecdm -``` - -This simpler approach works when the service consistently uses Widevine for SD/HD and PlayReady for UHD. - -### How Automatic DRM Switching Works - -When downloading content, unshackle: - -1. **Detects video quality** - Analyzes all video tracks and determines the highest resolution -2. **Applies quality rules** - Matches resolution against your quality-based CDM configuration -3. **Detects DRM type** - Identifies whether content uses Widevine or PlayReady -4. **Switches CDM automatically** - Loads the appropriate CDM based on DRM type and quality -5. **Falls back if needed** - Uses local CDM if remote CDM is unavailable - -For example, if you download 4K content that uses Widevine: -- System detects 2160p resolution -- Matches `">=2160": remote_l1` rule -- Detects Widevine DRM -- Automatically loads `remote_l1` remote CDM -- If remote CDM fails, falls back to local CDM (if available) - -### Local to Remote CDM Fallback - -When you configure both local and remote CDMs, unshackle follows this priority order: - -1. **Remote CDM** (if defined in `remote_cdm` and matched by quality/DRM rules) -2. **Local PlayReady** (.prd files in `PRDs` directory) -3. **Local Widevine** (.wvd files in `WVDs` directory) - -This ensures that if a remote CDM API is unavailable, unshackle can still use local devices as fallback. - -#### Example: Complete Configuration with Fallback - -```yaml -cdm: - NETFLIX: - # Use local for low quality, remote for high quality - "<=720": local_l3_sd # Local WVD file - "1080": local_l3_hd # Local WVD file - ">=1440": remote_l1 # Remote L1 API - default: local_l3_sd - -# Define remote CDMs -remote_cdm: - - name: remote_l1 - type: decrypt_labs # Or custom_api - device_name: L1 - host: https://your-cdm-api.com - secret: YOUR_API_KEY - - - name: remote_sl2 - type: decrypt_labs # Or custom_api - device_name: SL2 # PlayReady SL2000 - device_type: PLAYREADY - host: https://your-cdm-api.com - secret: YOUR_API_KEY -``` - -**Result:** -- **480p/720p content** → Uses `local_l3_sd` (local .wvd file) -- **1080p content** → Uses `local_l3_hd` (local .wvd file) -- **1440p/2160p content** → Uses `remote_l1` (remote API) -- **If remote API fails** → Falls back to local .wvd files if available - -### Advanced: Service Certificate Configuration - -Some services require L1/L2 security levels for high-quality content. When using remote L1/L2 CDMs, -you may need to configure the service certificate in the `services` section. See the [services](#services-dict) -section for certificate configuration details. - -### Configuration Priority Order - -When multiple configuration types are defined, unshackle follows this selection hierarchy: - -1. **Profile-specific** (if `-p/--profile` specified on command line) -2. **DRM-specific** (widevine/playready keys) -3. **Quality-based** (resolution with operators: >=, >, <=, <, exact) -4. **Service-level default** (default key under service) -5. **Global default** (top-level default key) - -### Summary - -- **Basic**: Simple service → CDM mapping -- **Profile**: Different CDMs per user profile -- **Quality**: Automatic CDM selection based on video resolution -- **DRM Type**: Separate CDMs for Widevine vs PlayReady -- **Fallback**: Local CDM fallback if remote CDM unavailable -- **Automatic**: Zero manual intervention - unshackle handles all switching - -## chapter_fallback_name (str) - -The Chapter Name to use when exporting a Chapter without a Name. -The default is no fallback name at all and no Chapter name will be set. - -The fallback name can use the following variables in f-string style: - -- `{i}`: The Chapter number starting at 1. - E.g., `"Chapter {i}"`: "Chapter 1", "Intro", "Chapter 3". -- `{j}`: A number starting at 1 that increments any time a Chapter has no title. - E.g., `"Chapter {j}"`: "Chapter 1", "Intro", "Chapter 2". - -These are formatted with f-strings, directives are supported. -For example, `"Chapter {i:02}"` will result in `"Chapter 01"`. - -## credentials (dict[str, str|list|dict]) - -Specify login credentials to use for each Service, and optionally per-profile. - -For example, - -```yaml -ALL4: jane@gmail.com:LoremIpsum100 # directly -AMZN: # or per-profile, optionally with a default - default: jane@example.tld:LoremIpsum99 # <-- used by default if -p/--profile is not used - james: james@gmail.com:TheFriend97 - john: john@example.tld:LoremIpsum98 -NF: # the `default` key is not necessary, but no credential will be used by default - john: john@gmail.com:TheGuyWhoPaysForTheNetflix69420 -``` - -The value should be in string form, i.e. `john@gmail.com:password123` or `john:password123`. -Any arbitrary values can be used on the left (username/password/phone) and right (password/secret). -You can also specify these in list form, i.e., `["john@gmail.com", ":PasswordWithAColon"]`. - -If you specify multiple credentials with keys like the `AMZN` and `NF` example above, then you should -use a `default` key or no credential will be loaded automatically unless you use `-p/--profile`. You -do not have to use a `default` key at all. - -Please be aware that this information is sensitive and to keep it safe. Do not share your config. - -## curl_impersonate (dict) - -Configuration for curl_cffi browser impersonation and custom fingerprinting. - -- `browser` - The Browser to impersonate as OR a fingerprint preset name. A list of available Browsers and Versions - are listed here: - - Default: `"chrome124"` - -### Available Fingerprint Presets - -- `okhttp4` - Android TV OkHttp 4.x fingerprint preset (for better Android TV compatibility) -- `okhttp5` - Android TV OkHttp 5.x fingerprint preset (for better Android TV compatibility) - -### Custom Fingerprinting - -For advanced users, you can specify custom TLS and HTTP/2 fingerprints: - -- `ja3` (str): Custom JA3 TLS fingerprint string (format: "SSLVersion,Ciphers,Extensions,Curves,PointFormats") -- `akamai` (str): Custom Akamai HTTP/2 fingerprint string (format: "SETTINGS|WINDOW_UPDATE|PRIORITY|PSEUDO_HEADERS") -- `extra_fp` (dict): Additional fingerprint parameters for advanced customization - -For example, using a browser preset: - -```yaml -curl_impersonate: - browser: "chrome120" -``` - -Using an Android TV preset: - -```yaml -curl_impersonate: - browser: "okhttp4" -``` - -Using custom fingerprints: - -```yaml -curl_impersonate: - browser: "chrome120" - ja3: "custom_ja3_fingerprint_string" - akamai: "custom_akamai_fingerprint_string" -``` - -## directories (dict) - -Override the default directories used across unshackle. -The directories are set to common values by default. - -The following directories are available and may be overridden, - -- `commands` - CLI Command Classes. -- `services` - Service Classes. -- `vaults` - Vault Classes. -- `fonts` - Font files (ttf or otf). -- `downloads` - Downloads. -- `temp` - Temporary files or conversions during download. -- `cache` - Expiring data like Authorization tokens, or other misc data. -- `cookies` - Expiring Cookie data. -- `logs` - Logs. -- `wvds` - Widevine Devices. -- `prds` - PlayReady Devices. -- `dcsl` - Device Certificate Status List. - -Notes: - -- `services` accepts either a single directory or a list of directories to search for service modules. - -For example, - -```yaml -downloads: "D:/Downloads/unshackle" -temp: "D:/Temp/unshackle" -``` - -There are directories not listed that cannot be modified as they are crucial to the operation of unshackle. - -## dl (dict) - -Pre-define default options and switches of the `dl` command. -The values will be ignored if explicitly set in the CLI call. - -The Key must be the same value Python click would resolve it to as an argument. -E.g., `@click.option("-r", "--range", "range_", type=...` actually resolves as `range_` variable. - -### Common Options - -For example to set the default primary language to download to German, - -```yaml -lang: de -``` - -You can also set multiple preferred languages using a list, e.g., - -```yaml -lang: - - en - - fr -``` - -to set how many tracks to download concurrently to 4 and download threads to 16, - -```yaml -downloads: 4 -workers: 16 -``` - -to set `--bitrate=CVBR` for the AMZN service, - -```yaml -lang: de -AMZN: - bitrate: CVBR -``` - -or to change the output subtitle format from the default (original format) to WebVTT, - -```yaml -sub_format: vtt -``` - -### Additional Available Options - -The following additional flags can be pre-configured as defaults: - -- `latest_episode` (bool): Download only the most recent episode (corresponds to `--latest-episode` / `-le` flag) -- `no_video` (bool): Skip downloading video tracks (corresponds to `--no-video` / `-nv` flag) -- `audio_description` (bool): Download audio description tracks (corresponds to `--audio-description` / `-ad` flag) -- `forced_subs` (bool): Include forced subtitle tracks (corresponds to `--forced-subs` / `-fs` flag) -- `no_cache` (bool): Bypass title cache (corresponds to `--no-cache` flag) -- `reset_cache` (bool): Clear title cache before fetching (corresponds to `--reset-cache` flag) -- `best_available` (bool): Continue with best quality if requested unavailable (corresponds to `--best-available` flag) - -For example, - -```yaml -dl: - latest_episode: true # Always download only the latest episode - audio_description: true # Include audio description tracks by default - best_available: true # Use best available quality as fallback -``` - -**Note**: These options can also be set per-service by nesting them under a service tag. - -## downloader (str | dict) - -Choose what software to use to download data throughout unshackle where needed. -You may provide a single downloader globally or a mapping of service tags to -downloaders. - -Options: - -- `requests` (default) - -- `aria2c` - -- `curl_impersonate` - (via ) -- `n_m3u8dl_re` - - -Note that aria2c can reach the highest speeds as it utilizes threading and more connections than the other downloaders. However, aria2c can also be one of the more unstable downloaders. It will work one day, then not another day. It also does not support HTTP(S) proxies while the other downloaders do. - -Example mapping: - -```yaml -downloader: - NF: requests - AMZN: n_m3u8dl_re - DSNP: n_m3u8dl_re - default: requests -``` - -The `default` entry is optional. If omitted, `requests` will be used for services not listed. - -## debug (bool) - -Enable comprehensive JSON-based debug logging for troubleshooting and service development. -When enabled, creates JSON Lines (`.jsonl`) log files with complete debugging context. - -Default: `false` - -When enabled (via `--debug` flag or `debug: true` in config): - -- Creates structured JSON Lines log files: `logs/unshackle_debug_{service}_{timestamp}.jsonl` -- Logs session info, CLI parameters, service configuration, CDM details, authentication status -- Logs title/track metadata, DRM operations, vault queries -- Logs errors with full stack traces -- Also creates text log: `logs/unshackle_root_{timestamp}.log` - -For example, - -```yaml -debug: true -``` - -**Security Note**: Passwords, tokens, cookies, and session tokens are ALWAYS redacted regardless of this setting. - -## debug_keys (bool) - -Control whether actual decryption keys (CEKs) are logged in debug logs. - -Default: `false` - -When set to `true`, includes actual content encryption keys in debug logs. This is useful for debugging -key retrieval and decryption issues. - -For example, - -```yaml -debug_keys: true -``` - -**Security Notes**: - -- Only affects content_key and key fields (the actual CEKs) -- Key metadata (kid, keys_count, key_id) is always logged regardless of this setting -- Passwords, tokens, cookies, and session tokens remain redacted even when this is enabled -- Use with caution and ensure debug logs are stored securely - -## decryption (str | dict) - -Choose what software to use to decrypt DRM-protected content throughout unshackle where needed. -You may provide a single decryption method globally or a mapping of service tags to -decryption methods. - -Options: - -- `shaka` (default) - Shaka Packager - -- `mp4decrypt` - mp4decrypt from Bento4 - - -Note that Shaka Packager is the traditional method and works with most services. mp4decrypt -is an alternative that may work better with certain services that have specific encryption formats. - -Example mapping: - -```yaml -decryption: - ATVP: mp4decrypt - AMZN: shaka - default: shaka -``` - -The `default` entry is optional. If omitted, `shaka` will be used for services not listed. - -Simple configuration (single method for all services): - -```yaml -decryption: mp4decrypt -``` - -## decrypt_labs_api_key (str) - -API key for DecryptLabs CDM service integration. - -When set, enables the use of DecryptLabs remote CDM services in your `remote_cdm` configuration. -This is used specifically for `type: "decrypt_labs"` entries in the remote CDM list. - -For example, - -```yaml -decrypt_labs_api_key: "your_api_key_here" -``` - -**Note**: This is different from the per-CDM `secret` field in `remote_cdm` entries. This provides a global -API key that can be referenced across multiple DecryptLabs CDM configurations. - -## filenames (dict) - -Override the default filenames used across unshackle. -The filenames use various variables that are replaced during runtime. - -The following filenames are available and may be overridden: - -- `log` - Log filenames. Uses `{name}` and `{time}` variables. -- `debug_log` - Debug log filenames in JSON Lines format. Uses `{service}` and `{time}` variables. -- `config` - Service configuration filenames. -- `root_config` - Root configuration filename. -- `chapters` - Chapter export filenames. Uses `{title}` and `{random}` variables. -- `subtitle` - Subtitle export filenames. Uses `{id}` and `{language}` variables. - -For example, - -```yaml -filenames: - log: "unshackle_{name}_{time}.log" - debug_log: "unshackle_debug_{service}_{time}.jsonl" - config: "config.yaml" - root_config: "unshackle.yaml" - chapters: "Chapters_{title}_{random}.txt" - subtitle: "Subtitle_{id}_{language}.srt" -``` - -## headers (dict) - -Case-Insensitive dictionary of headers that all Services begin their Request Session state with. -All requests will use these unless changed explicitly or implicitly via a Server response. -These should be sane defaults and anything that would only be useful for some Services should not -be put here. - -Avoid headers like 'Accept-Encoding' as that would be a compatibility header that Python-requests will -set for you. - -I recommend using, - -```yaml -Accept-Language: "en-US,en;q=0.8" -User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36" -``` - -## key_vaults (list\[dict]) - -Key Vaults store your obtained Content Encryption Keys (CEKs) and Key IDs per-service. - -This can help reduce unnecessary License calls even during the first download. This is because a Service may -provide the same Key ID and CEK for both Video and Audio, as well as for multiple resolutions or bitrates. - -You can have as many Key Vaults as you would like. It's nice to share Key Vaults or use a unified Vault on -Teams as sharing CEKs immediately can help reduce License calls drastically. - -Three types of Vaults are in the Core codebase, API, SQLite and MySQL. API makes HTTP requests to a RESTful API, -whereas SQLite and MySQL directly connect to an SQLite or MySQL Database. - -Note: SQLite and MySQL vaults have to connect directly to the Host/IP. It cannot be in front of a PHP API or such. -Beware that some Hosting Providers do not let you access the MySQL server outside their intranet and may not be -accessible outside their hosting platform. - -Additional behavior: - -- `no_push` (bool): Optional per-vault flag. When `true`, the vault will not receive pushed keys (writes) but - will still be queried and can provide keys for lookups. Useful for read-only/backup vaults. - -### Using an API Vault - -API vaults use a specific HTTP request format, therefore API or HTTP Key Vault APIs from other projects or services may -not work in unshackle. The API format can be seen in the [API Vault Code](unshackle/vaults/API.py). - -```yaml -- type: API - name: "John#0001's Vault" # arbitrary vault name - uri: "https://key-vault.example.com" # api base uri (can also be an IP or IP:Port) - # uri: "127.0.0.1:80/key-vault" - # uri: "https://api.example.com/key-vault" - token: "random secret key" # authorization token - # no_push: true # optional; make this API vault read-only (lookups only) -``` - -### Using a MySQL Vault - -MySQL vaults can be either MySQL or MariaDB servers. I recommend MariaDB. -A MySQL Vault can be on a local or remote network, but I recommend SQLite for local Vaults. - -```yaml -- type: MySQL - name: "John#0001's Vault" # arbitrary vault name - host: "127.0.0.1" # host/ip - # port: 3306 # port (defaults to 3306) - database: vault # database used for unshackle - username: jane11 - password: Doe123 - # no_push: false # optional; defaults to false -``` - -I recommend giving only a trustable user (or yourself) CREATE permission and then use unshackle to cache at least one CEK -per Service to have it create the tables. If you don't give any user permissions to create tables, you will need to -make tables yourself. - -- Use a password on all user accounts. -- Never use the root account with unshackle (even if it's you). -- Do not give multiple users the same username and/or password. -- Only give users access to the database used for unshackle. -- You may give trusted users CREATE permission so unshackle can create tables if needed. -- Other uses should only be given SELECT and INSERT permissions. - -### Using an SQLite Vault - -SQLite Vaults are usually only used for locally stored vaults. This vault may be stored on a mounted Cloud storage -drive, but I recommend using SQLite exclusively as an offline-only vault. Effectively this is your backup vault in -case something happens to your MySQL Vault. - -```yaml -- type: SQLite - name: "My Local Vault" # arbitrary vault name - path: "C:/Users/Jane11/Documents/unshackle/data/key_vault.db" - # no_push: true # optional; commonly true for local backup vaults -``` - -**Note**: You do not need to create the file at the specified path. -SQLite will create a new SQLite database at that path if one does not exist. -Try not to accidentally move the `db` file once created without reflecting the change in the config, or you will end -up with multiple databases. - -If you work on a Team I recommend every team member having their own SQLite Vault even if you all use a MySQL vault -together. - -## muxing (dict) - -- `set_title` - Set the container title to `Show SXXEXX Episode Name` or `Movie (Year)`. Default: `true` - -## n_m3u8dl_re (dict) - -Configuration for N_m3u8DL-RE downloader. This downloader is particularly useful for HLS streams. - -- `thread_count` - Number of threads to use for downloading. Default: Uses the same value as max_workers from the command. -- `ad_keyword` - Keyword to identify and potentially skip advertisement segments. Default: `None` -- `use_proxy` - Whether to use proxy when downloading. Default: `true` - -For example, - -```yaml -n_m3u8dl_re: - thread_count: 16 - ad_keyword: "advertisement" - use_proxy: true -``` - -## proxy_providers (dict) - -Enable external proxy provider services. These proxies will be used automatically where needed as defined by the -Service's GEOFENCE class property, but can also be explicitly used with `--proxy`. You can specify which provider -to use by prefixing it with the provider key name, e.g., `--proxy basic:de` or `--proxy nordvpn:de`. Some providers -support specific query formats for selecting a country/server. - -### basic (dict[str, str|list]) - -Define a mapping of country to proxy to use where required. -The keys are region Alpha 2 Country Codes. Alpha 2 Country Codes are `[a-z]{2}` codes, e.g., `us`, `gb`, and `jp`. -Don't get this mixed up with language codes like `en` vs. `gb`, or `ja` vs. `jp`. - -Do note that each key's value can be a list of strings, or a string. For example, - -```yaml -us: - - "http://john%40email.tld:password123@proxy-us.domain.tld:8080" - - "http://jane%40email.tld:password456@proxy-us.domain2.tld:8080" -de: "https://127.0.0.1:8080" -``` - -Note that if multiple proxies are defined for a region, then by default one will be randomly chosen. -You can choose a specific one by specifying it's number, e.g., `--proxy basic:us2` will choose the -second proxy of the US list. - -### nordvpn (dict) - -Set your NordVPN Service credentials with `username` and `password` keys to automate the use of NordVPN as a Proxy -system where required. - -You can also specify specific servers to use per-region with the `server_map` key. -Sometimes a specific server works best for a service than others, so hard-coding one for a day or two helps. - -For example, - -```yaml -username: zxqsR7C5CyGwmGb6KSvk8qsZ # example of the login format -password: wXVHmht22hhRKUEQ32PQVjCZ -server_map: - us: 12 # force US server #12 for US proxies -``` - -The username and password should NOT be your normal NordVPN Account Credentials. -They should be the `Service credentials` which can be found on your Nord Account Dashboard. - -Once set, you can also specifically opt in to use a NordVPN proxy by specifying `--proxy=gb` or such. -You can even set a specific server number this way, e.g., `--proxy=gb2366`. - -Note that `gb` is used instead of `uk` to be more consistent across regional systems. - -### surfsharkvpn (dict) - -Enable Surfshark VPN proxy service using Surfshark Service credentials (not your login password). -You may pin specific server IDs per region using `server_map`. - -```yaml -username: your_surfshark_service_username # https://my.surfshark.com/vpn/manual-setup/main/openvpn -password: your_surfshark_service_password # service credentials, not account password -server_map: - us: 3844 # force US server #3844 - gb: 2697 # force GB server #2697 - au: 4621 # force AU server #4621 -``` - -### windscribevpn (dict) - -Enable Windscribe VPN proxy service using Windscribe Service credentials (not your login password). -You may pin specific server hostnames per region using `server_map`. - -```yaml -username: your_windscribe_username # From https://windscribe.com/getconfig/openvpn -password: your_windscribe_password # Service credentials (not your login password) -server_map: - us: "us-central-096.totallyacdn.com" # Force specific US server - gb: "uk-london-055.totallyacdn.com" # Force specific GB server - de: "de-frankfurt-001.totallyacdn.com" # Force specific DE server -``` - -**Note**: The username and password should be your Windscribe OpenVPN credentials, which can be obtained from -the Windscribe configuration generator. The `server_map` uses full server hostnames (not just numbers like NordVPN). - -You can use Windscribe proxies by specifying `--proxy=windscribevpn:us` or such. Server selection works similar -to other providers - use `--proxy=windscribevpn:us` for automatic server or specify the full hostname if needed. - -### hola (dict) - -Enable Hola VPN proxy service for datacenter and residential proxies. - -This provider uses the open-source `hola-proxy` tool and requires no account credentials. -Simply include an empty configuration to enable it. - -For example, - -```yaml -proxy_providers: - hola: {} -``` - -**Requirements**: The `hola-proxy` binary must be installed and available in your system PATH or in the -unshackle binaries directory. - -**Note**: Hola uses a peer-to-peer VPN network. Consider the privacy implications before using this provider. - -### gluetun (dict) - -Enable Gluetun VPN proxy service, which creates Docker containers running Gluetun to bridge VPN connections -to HTTP proxies. This supports 50+ VPN providers through a single, unified interface. - -**Requirements**: Docker must be installed and running. Check with `unshackle env check`. - -```yaml -gluetun: - base_port: 8888 # Starting port for HTTP proxies - auto_cleanup: true # Remove containers when done - container_prefix: "unshackle-gluetun" # Docker container name prefix - verify_ip: true # Verify VPN IP matches expected region - - providers: - windscribe: - vpn_type: wireguard - credentials: - private_key: "YOUR_WIREGUARD_PRIVATE_KEY" - addresses: "YOUR_WIREGUARD_ADDRESS" # e.g., "10.x.x.x/32" - server_countries: - us: US - uk: GB - ca: CA -``` - -**Usage**: Use the format `--proxy gluetun::`, e.g.: -- `--proxy gluetun:windscribe:us` - Connect via Windscribe to US -- `--proxy gluetun:nordvpn:de` - Connect via NordVPN to Germany - -**Supported VPN Types**: -- `wireguard` - For providers like Windscribe, NordVPN, Surfshark (recommended) -- `openvpn` - For providers like ExpressVPN, PIA - -See the example config file for more provider configurations. - -## remote_cdm (list\[dict]) - -Configure remote CDM (Content Decryption Module) APIs to use for decrypting DRM-protected content. -Remote CDMs allow you to use high-security CDMs (L1/L2 for Widevine, SL2000/SL3000 for PlayReady) without -having the physical device files locally. - -unshackle supports multiple types of remote CDM providers: - -1. **DecryptLabs CDM** - Official DecryptLabs KeyXtractor API with intelligent caching -2. **Custom API CDM** - Highly configurable adapter for any third-party CDM API -3. **Legacy PyWidevine Serve** - Standard pywidevine serve-compliant APIs - -The name of each defined remote CDM can be referenced in the `cdm` configuration as if it was a local device file. - -### DecryptLabs Remote CDM - -DecryptLabs provides a professional CDM API service with support for multiple device types and intelligent key caching. - -**Supported Devices:** -- **Widevine**: `ChromeCDM` (L3), `L1` (Security Level 1), `L2` (Security Level 2) -- **PlayReady**: `SL2` (SL2000), `SL3` (SL3000) - -**Configuration:** - -```yaml -remote_cdm: - # Widevine L1 Device - - name: decrypt_labs_l1 - type: decrypt_labs # Required: identifies as DecryptLabs CDM - device_name: L1 # Required: must match exactly (L1, L2, ChromeCDM, SL2, SL3) - host: https://keyxtractor.decryptlabs.com - secret: YOUR_API_KEY # Your DecryptLabs API key - - # Widevine L2 Device - - name: decrypt_labs_l2 - type: decrypt_labs - device_name: L2 - host: https://keyxtractor.decryptlabs.com - secret: YOUR_API_KEY - - # Chrome CDM (L3) - - name: decrypt_labs_chrome - type: decrypt_labs - device_name: ChromeCDM - host: https://keyxtractor.decryptlabs.com - secret: YOUR_API_KEY - - # PlayReady SL2000 - - name: decrypt_labs_playready_sl2 - type: decrypt_labs - device_name: SL2 - device_type: PLAYREADY # Required for PlayReady - host: https://keyxtractor.decryptlabs.com - secret: YOUR_API_KEY - - # PlayReady SL3000 - - name: decrypt_labs_playready_sl3 - type: decrypt_labs - device_name: SL3 - device_type: PLAYREADY - host: https://keyxtractor.decryptlabs.com - secret: YOUR_API_KEY -``` - -**Features:** -- Intelligent key caching system (reduces API calls) -- Automatic integration with unshackle's vault system -- Support for both Widevine and PlayReady -- Multiple security levels (L1, L2, L3, SL2000, SL3000) - -**Note:** The `device_type` and `security_level` fields are optional metadata. They don't affect API communication -but are used for internal device identification. - -### Custom API Remote CDM - -A highly configurable CDM adapter that can work with virtually any third-party CDM API through YAML configuration. -This allows you to integrate custom CDM services without writing code. - -**Configuration Philosophy:** -- **90%** of new CDM providers: Only YAML config needed -- **9%** of cases: Add new transform type -- **1%** of cases: Add new auth strategy - -**Basic Example:** - -```yaml -remote_cdm: - - name: custom_chrome_cdm - type: custom_api # Required: identifies as Custom API CDM - host: https://your-cdm-api.com - timeout: 30 # Optional: request timeout in seconds - - device: - name: ChromeCDM - type: CHROME # CHROME, ANDROID, PLAYREADY - system_id: 27175 - security_level: 3 - - auth: - type: bearer # bearer, header, basic, body - key: YOUR_API_TOKEN - - endpoints: - get_request: - path: /get-challenge - method: POST - decrypt_response: - path: /get-keys - method: POST - - caching: - enabled: true # Enable key caching - use_vaults: true # Integrate with vault system -``` - -**Advanced Example with Field Mapping:** - -```yaml -remote_cdm: - - name: advanced_custom_api - type: custom_api - host: https://api.example.com - device: - name: L1 - type: ANDROID - security_level: 1 - - # Authentication configuration - auth: - type: header - header_name: X-API-Key - key: YOUR_SECRET_KEY - custom_headers: - User-Agent: Unshackle/2.0.0 - X-Client-Version: "1.0" - - # Endpoint configuration - endpoints: - get_request: - path: /v2/challenge - method: POST - timeout: 30 - decrypt_response: - path: /v2/decrypt - method: POST - timeout: 30 - - # Request parameter mapping - request_mapping: - get_request: - param_names: - init_data: pssh # Rename 'init_data' to 'pssh' - scheme: device_type # Rename 'scheme' to 'device_type' - static_params: - api_version: "2.0" # Add static parameter - decrypt_response: - param_names: - license_request: challenge - license_response: license - - # Response field mapping - response_mapping: - get_request: - fields: - challenge: data.challenge # Deep field access - session_id: session.id - success_conditions: - - status == 'ok' # Validate response - decrypt_response: - fields: - keys: data.keys - key_fields: - kid: key_id # Map 'kid' field - key: content_key # Map 'key' field - - caching: - enabled: true - use_vaults: true - check_cached_first: true # Check cache before API calls -``` - -**Supported Authentication Types:** -- `bearer` - Bearer token authentication -- `header` - Custom header authentication -- `basic` - HTTP Basic authentication -- `body` - Credentials in request body - -### Legacy PyWidevine Serve Format - -Standard pywidevine serve-compliant remote CDM configuration (backwards compatibility). - -```yaml -remote_cdm: - - name: legacy_chrome_cdm - device_name: chrome - device_type: CHROME - system_id: 27175 - security_level: 3 - host: https://domain.com/api - secret: secret_key -``` - -**Note:** If `type` is not specified, unshackle assumes legacy format. For DecryptLabs or Custom API, -always specify `type: decrypt_labs` or `type: custom_api`. - -### Integration with Quality-Based CDM Selection - -Remote CDMs can be used in quality-based and DRM-specific CDM configurations: - -```yaml -cdm: - NETFLIX: - "<=1080": local_l3 # Local for SD/HD - ">=1440": remote_l1 # Remote for 4K+ - - widevine: - ">=2160": remote_l1 # Remote L1 for 4K Widevine - default: local_wv - - playready: - ">=1080": remote_sl2 # Remote SL2 for HD+ PlayReady - default: local_pr - -remote_cdm: - - name: remote_l1 - type: decrypt_labs # Or custom_api - device_name: L1 - host: https://your-cdm-api.com - secret: YOUR_API_KEY - - - name: remote_sl2 - type: decrypt_labs # Or custom_api - device_name: SL2 - device_type: PLAYREADY - host: https://your-cdm-api.com - secret: YOUR_API_KEY -``` - -### Key Features - -**Intelligent Caching:** -- Remote CDMs integrate with unshackle's vault system -- Keys are cached locally to reduce API calls -- Cached keys are checked before making license requests -- Multiple vault sources supported (SQLite, MySQL, API) - -**Automatic Fallback:** -- If remote CDM fails, unshackle falls back to local devices (if available) -- Priority: Remote CDM → Local PRD → Local WVD - -**DRM Type Detection:** -- Automatically switches between Widevine and PlayReady remote CDMs -- Based on content DRM system detection - -**Quality-Based Selection:** -- Use different remote CDMs based on video resolution -- Combine with local CDMs for cost-effective downloads - -[pywidevine]: https://github.com/rlaphoenix/pywidevine - -## scene_naming (bool) - -Set scene-style naming for titles. When `true` uses scene naming patterns (e.g., `Prime.Suspect.S07E01...`), when -`false` uses a more human-readable style (e.g., `Prime Suspect S07E01 ...`). Default: `true`. - -## series_year (bool) - -Whether to include the series year in series names for episodes and folders. Default: `true`. - -## serve (dict) - -Configuration data for pywidevine's serve functionality run through unshackle. -This effectively allows you to run `unshackle serve` to start serving pywidevine Serve-compliant CDMs right from your -local widevine device files. - -- `api_secret` - Secret key for REST API authentication. When set, enables the REST API server alongside the CDM serve functionality. This key is required for authenticating API requests. - -For example, - -```yaml -api_secret: "your-secret-key-here" -users: - secret_key_for_jane: # 32bit hex recommended, case-sensitive - devices: # list of allowed devices for this user - - generic_nexus_4464_l3 - username: jane # only for internal logging, users will not see this name - secret_key_for_james: - devices: - - generic_nexus_4464_l3 - username: james - secret_key_for_john: - devices: - - generic_nexus_4464_l3 - username: john -# devices can be manually specified by path if you don't want to add it to -# unshackle's WVDs directory for whatever reason -# devices: -# - 'C:\Users\john\Devices\test_devices_001.wvd' -``` - -## services (dict) - -Configuration data for each Service. The Service will have the data within this section merged into the `config.yaml` -before provided to the Service class. - -This configuration serves two purposes: - -1. **Service-specific data**: Sensitive configuration like user or device-specific API keys, IDs, device attributes, and - so on. A `config.yaml` file is typically shared and not meant to be modified, so use this for any sensitive data. - -2. **Per-service configuration overrides**: Override any global configuration option on a per-service basis for fine-tuned - control. This allows you to customize behavior for services with special requirements. - -The Key is the Service Tag, and the value can take any form (typically a dictionary or list). - -### Basic Service Configuration - -For example, - -```yaml -services: - NOW: - client: - auth_scheme: MESSO - # ... more sensitive data -``` - -### Service-Specific Configuration Overrides - -**New in v2.0.0**: You can override ANY global configuration option on a per-service basis. Supported overrides include: - -- `dl` - Download command defaults -- `aria2c` - aria2c downloader settings -- `n_m3u8dl_re` - N_m3u8DL-RE downloader settings -- `curl_impersonate` - Browser impersonation settings -- `subtitle` - Subtitle processing options -- `muxing` - Muxing behavior -- `headers` - HTTP headers -- And more... - -### Comprehensive Example - -```yaml -services: - EXAMPLE: - # Standard service configuration - api_key: "service_api_key" - - # Service certificate for Widevine L1/L2 (base64 encoded) - certificate: | - CAUSwwUKvQIIAxIQ5US6QAvBDzfTtjb4tU/7QxiH8c+TBSKOAjCCAQoCggEBAObzvlu2hZRs... - # (full base64 certificate) - - # Profile-specific configurations - profiles: - john_sd: - device: - app_name: "AIV" - device_model: "SHIELD Android TV" - jane_uhd: - device: - app_name: "AIV" - device_model: "Fire TV Stick 4K" - - # Override dl command defaults for this service - dl: - downloads: 4 # Limit concurrent track downloads - workers: 8 # Reduce workers per track - lang: ["en", "es-419"] # Different language priority - sub_format: srt # Force SRT subtitle format - - # Override n_m3u8dl_re downloader settings - n_m3u8dl_re: - thread_count: 8 # Lower thread count for rate-limited service - use_proxy: true # Force proxy usage - retry_count: 10 # More retries for unstable connections - - # Override aria2c downloader settings - aria2c: - max_concurrent_downloads: 2 - max_connection_per_server: 1 - split: 3 - - # Override subtitle processing - subtitle: - conversion_method: pycaption - sdh_method: auto - - # Service-specific headers - headers: - User-Agent: "Service-specific user agent string" - Accept-Language: "en-US,en;q=0.9" - - # Override muxing options - muxing: - set_title: true - - # Example: Rate-limited service requiring conservative settings - RATE_LIMITED_SERVICE: - dl: - downloads: 2 - workers: 4 - n_m3u8dl_re: - thread_count: 4 - retry_count: 20 - aria2c: - max_concurrent_downloads: 1 - max_connection_per_server: 1 -``` - -### Important Notes - -- Overrides are merged with global config, not replaced -- Only specified keys are overridden; others use global defaults -- Reserved keys (`profiles`, `api_key`, `certificate`, etc.) are NOT treated as overrides -- Any dict-type config option can be overridden -- CLI arguments always take priority over service-specific config -- This feature enables fine-tuned control without modifying global settings - -## set_terminal_bg (bool) - -Controls whether unshackle should set the terminal background color. Default: `false` - -For example, - -```yaml -set_terminal_bg: true -``` - -## simkl_client_id (str) - -Client ID for SIMKL API integration. SIMKL is used as a metadata source for improved title matching and tagging, -especially when a TMDB API key is not configured. - -To obtain a SIMKL Client ID: - -1. Create an account at -2. Go to -3. Register a new application to receive your Client ID - -For example, - -```yaml -simkl_client_id: "your_client_id_here" -``` - -**Note**: While optional, having a SIMKL Client ID improves metadata lookup reliability and reduces the chance -of rate limiting. SIMKL serves as an alternative or fallback metadata source to TMDB. - -## tag (str) - -Group or Username to postfix to the end of all download filenames following a dash. -For example, `tag: "J0HN"` will have `-J0HN` at the end of all download filenames. - -## tag_group_name (bool) - -Enable/disable tagging downloads with your group name when `tag` is set. Default: `true`. - -## tag_imdb_tmdb (bool) - -Enable/disable tagging downloaded files with IMDB/TMDB/TVDB identifiers (when available). Default: `true`. - -## title_cache_enabled (bool) - -Enable/disable caching of title metadata to reduce redundant API calls. Default: `true`. - -## title_cache_time (int) - -Cache duration in seconds for title metadata. Default: `1800` (30 minutes). - -## title_cache_max_retention (int) - -Maximum retention time in seconds for serving slightly stale cached title metadata when API calls fail. -Default: `86400` (24 hours). Effective retention is `min(title_cache_time + grace, title_cache_max_retention)`. - -## tmdb_api_key (str) - -API key for The Movie Database (TMDB). This is used for tagging downloaded files with TMDB, -IMDB and TVDB identifiers. Leave empty to disable automatic lookups. - -To obtain a TMDB API key: - -1. Create an account at -2. Go to to register for API access -3. Fill out the API application form with your project details -4. Once approved, you'll receive your API key - -For example, - -```yaml -tmdb_api_key: cf66bf18956kca5311ada3bebb84eb9a # Not a real key -``` - -**Note**: Keep your API key secure and do not share it publicly. This key is used by the core/utils/tags.py module to fetch metadata from TMDB for proper file tagging. - -## subtitle (dict) - -Control subtitle conversion, SDH (hearing-impaired) stripping behavior, and formatting preservation. - -### Conversion and Processing Options - -- `conversion_method`: How to convert subtitles between formats. Default: `auto`. - - - `auto`: Smart routing - use subby for WebVTT/SAMI, pycaption for others. - - `subby`: Always use subby with CommonIssuesFixer for advanced processing. - - `subtitleedit`: Prefer SubtitleEdit when available; otherwise fallback to standard conversion. - - `pycaption`: Use only the pycaption library (no SubtitleEdit, no subby). - - `pysubs2`: Use pysubs2 library (supports SRT, SSA, ASS, WebVTT, TTML, SAMI, MicroDVD, MPL2, TMP formats). - -- `sdh_method`: How to strip SDH cues. Default: `auto`. - - - `auto`: Try subby for SRT first, then SubtitleEdit, then subtitle-filter. - - `subby`: Use subby's SDHStripper (SRT only). - - `subtitleedit`: Use SubtitleEdit's RemoveTextForHI when available. - - `filter-subs`: Use the subtitle-filter library directly. - -- `strip_sdh`: Automatically create stripped (non-SDH) versions of SDH subtitles. Default: `true`. - - Set to `false` to disable automatic SDH stripping entirely. When `true`, unshackle will automatically - detect SDH subtitles and create clean versions alongside the originals. - -- `convert_before_strip`: Auto-convert VTT/other formats to SRT before using subtitle-filter. Default: `true`. - - This ensures compatibility when subtitle-filter is used as the fallback SDH stripping method, as - subtitle-filter works best with SRT format. - -- `preserve_formatting`: Preserve original subtitle formatting (tags, positioning, styling). Default: `true`. - - When `true`, skips pycaption processing for WebVTT files to keep tags like ``, ``, positioning, - and other formatting intact. Combined with no `sub_format` setting, ensures subtitles remain in their - original format. - -### Example Configuration - -```yaml -subtitle: - conversion_method: auto - sdh_method: auto - strip_sdh: true - convert_before_strip: true - preserve_formatting: true -``` - -### Minimal Configuration (Disable Processing) - -```yaml -subtitle: - strip_sdh: false # Don't strip SDH - preserve_formatting: true # Keep all formatting intact -``` - -## update_checks (bool) - -Check for updates from the GitHub repository on startup. Default: `true`. - -## update_check_interval (int) - -How often to check for updates, in hours. Default: `24`. diff --git a/cliff.toml b/cliff.toml index f8552cb..286a0f1 100644 --- a/cliff.toml +++ b/cliff.toml @@ -1,4 +1,4 @@ -# git-cliff ~ default configuration file +# git-cliff ~ configuration file # https://git-cliff.org/docs/configuration [changelog] @@ -8,8 +8,7 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -Versions [3.0.0] and older use a format based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -but versions thereafter use a custom changelog format using [git-cliff](https://git-cliff.org).\n +This changelog is automatically generated using [git-cliff](https://git-cliff.org).\n """ body = """ {% if version -%} @@ -17,9 +16,11 @@ body = """ {% else -%} ## [Unreleased] {% endif -%} -{% for group, commits in commits | group_by(attribute="group") %} +{% for group, commits in commits + | filter(attribute="merge_commit", value=false) + | group_by(attribute="group") %} ### {{ group | striptags | trim | upper_first }} - {% for commit in commits %} + {% for commit in commits | unique(attribute="message") %} - {% if commit.scope %}*{{ commit.scope }}*: {% endif %}\ {% if commit.breaking %}[**breaking**] {% endif %}\ {{ commit.message | upper_first }}\ @@ -38,34 +39,39 @@ footer = """ [unreleased]: https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}\ /compare/{{ release.previous.version }}..HEAD {% endif -%} -{% endfor %} +{% endfor -%} """ trim = true -postprocessors = [ - # { pattern = '', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL -] +postprocessors = [] [git] conventional_commits = true filter_unconventional = true split_commits = false -commit_preprocessors = [] +commit_preprocessors = [ + # Strip emoji (both UTF-8 and :shortcode: styles) from commit messages + { pattern = ' *(:\w+:|[\p{Emoji_Presentation}\p{Extended_Pictographic}]\x{FE0F}?\x{200D}?) *', replace = "" }, + # Remove trailing PR/issue numbers like (#123) from commit messages + { pattern = '\((\w+\s)?#([0-9]+)\)', replace = "" }, +] commit_parsers = [ { message = "^feat", group = "Features" }, - { message = "^fix|revert", group = "Bug Fixes" }, - { message = "^docs", group = "Documentation" }, - { message = "^style", skip = true }, - { message = "^refactor", group = "Changes" }, + { message = "^fix", group = "Bug Fixes" }, + { message = "^revert", group = "Reverts" }, + { message = "^docs", group = "Documentation" }, { message = "^perf", group = "Performance Improvements" }, + { message = "^refactor", group = "Changes" }, + { message = "^style", skip = true }, { message = "^test", skip = true }, - { message = "^build", group = "Builds" }, + { message = "^chore\\(release\\)", skip = true }, + { message = "^chore\\(deps.*\\)", skip = true }, + { message = "^chore\\(pr\\)", skip = true }, + { message = "^chore", group = "Maintenance" }, { message = "^ci", skip = true }, - { message = "^chore", skip = true }, + { message = "^build", group = "Builds" }, + { body = ".*security", group = "Security" }, ] -protect_breaking_commits = false -filter_commits = false -# tag_pattern = "v[0-9].*" -# skip_tags = "" -# ignore_tags = "" +protect_breaking_commits = true +filter_commits = true topo_order = false sort_commits = "oldest" diff --git a/docs/DOWNLOAD_CONFIG.md b/docs/DOWNLOAD_CONFIG.md index c5f2b47..8f5664d 100644 --- a/docs/DOWNLOAD_CONFIG.md +++ b/docs/DOWNLOAD_CONFIG.md @@ -54,7 +54,9 @@ Options: - `curl_impersonate` - (via ) - `n_m3u8dl_re` - -Note that aria2c can reach the highest speeds as it utilizes threading and more connections than the other downloaders. However, aria2c can also be one of the more unstable downloaders. It will work one day, then not another day. It also does not support HTTP(S) proxies while the other downloaders do. +Note that aria2c can reach the highest speeds as it utilizes threading and more connections than the other downloaders. However, aria2c can also be one of the more unstable downloaders. It will work one day, then not another day. It also does not support HTTP(S) proxies natively (non-HTTP proxies are bridged via pproxy). + +Note that `n_m3u8dl_re` will automatically fall back to `requests` for track types it does not support, specifically: direct URL downloads, Subtitle tracks, and Attachment tracks. Example mapping: @@ -72,10 +74,12 @@ The `default` entry is optional. If omitted, `requests` will be used for service ## n_m3u8dl_re (dict) -Configuration for N_m3u8DL-RE downloader. This downloader is particularly useful for HLS streams. +Configuration for N_m3u8DL-RE downloader. This downloader supports HLS, DASH, and ISM (Smooth Streaming) manifests. +It will automatically fall back to the `requests` downloader for unsupported track types (direct URLs, subtitles, attachments). - `thread_count` - Number of threads to use for downloading. Default: Uses the same value as max_workers from the command. + Number of threads to use for downloading. Default: Uses the same value as max_workers from the command + (which defaults to `min(32,(cpu_count+4))`). - `ad_keyword` Keyword to identify and potentially skip advertisement segments. Default: `None` - `use_proxy` @@ -83,6 +87,9 @@ Configuration for N_m3u8DL-RE downloader. This downloader is particularly useful - `retry_count` Number of times to retry failed downloads. Default: `10` +N_m3u8DL-RE also respects the `decryption` config setting. When content keys are provided, it will use +the configured decryption engine (`shaka` or `mp4decrypt`) and automatically locate the corresponding binary. + For example, ```yaml @@ -140,6 +147,57 @@ sub_format: vtt --- +## subtitle (dict) + +Configuration for subtitle processing and conversion. + +- `conversion_method` + Method to use for converting subtitles between formats. Default: `"auto"` + - `"auto"` — Smart routing: uses subby for WebVTT/SAMI, pycaption for others. + - `"subby"` — Always use subby with advanced processing. + - `"pycaption"` — Use only pycaption library (no SubtitleEdit, no subby). + - `"subtitleedit"` — Prefer SubtitleEdit when available, fall back to pycaption. + - `"pysubs2"` — Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP). +- `sdh_method` + Method to use for SDH (hearing impaired) stripping. Default: `"auto"` + - `"auto"` — Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter. + - `"subby"` — Use subby library (SRT only). + - `"subtitleedit"` — Use SubtitleEdit tool (Windows only, falls back to subtitle-filter). + - `"filter-subs"` — Use subtitle-filter library directly. +- `strip_sdh` + Automatically create stripped (non-SDH) versions of SDH subtitles. Default: `true` +- `convert_before_strip` + Auto-convert VTT/other formats to SRT before using subtitle-filter for SDH stripping. + Ensures compatibility when subtitle-filter is used as fallback. Default: `true` +- `preserve_formatting` + Preserve original subtitle formatting (tags, positioning, styling). + When `true`, skips pycaption processing for WebVTT files to keep tags like ``, ``, + positioning intact. Combined with no `sub_format` setting, ensures subtitles remain in + their original format. Default: `true` +- `output_mode` + Output mode for subtitles. Default: `"mux"` + - `"mux"` — Embed subtitles in MKV container only. + - `"sidecar"` — Save subtitles as separate files only. + - `"both"` — Embed in MKV and save as sidecar files. +- `sidecar_format` + Format for sidecar subtitle files when `output_mode` is `"sidecar"` or `"both"`. Default: `"srt"` + Options: `srt`, `vtt`, `ass`, `original` (keep current format). + +For example, + +```yaml +subtitle: + conversion_method: auto + sdh_method: auto + strip_sdh: true + convert_before_strip: true + preserve_formatting: true + output_mode: mux + sidecar_format: srt +``` + +--- + ## decryption (str | dict) Choose what software to use to decrypt DRM-protected content throughout unshackle where needed. diff --git a/docs/DRM_CONFIG.md b/docs/DRM_CONFIG.md index 2981a38..4b2b9f6 100644 --- a/docs/DRM_CONFIG.md +++ b/docs/DRM_CONFIG.md @@ -41,6 +41,29 @@ DSNP: default: chromecdm_903_l3 ``` +You can also select CDMs based on video resolution using comparison operators (`>=`, `>`, `<=`, `<`) +or exact match on the resolution height. + +For example, + +```yaml +EXAMPLE: + "<=1080": generic_android_l3 # Use L3 for 1080p and below + ">1080": nexus_5_l1 # Use L1 for above 1080p (1440p, 2160p) + default: generic_android_l3 # Fallback if no quality match +``` + +You can mix profiles and quality thresholds in the same service: + +```yaml +NETFLIX: + john: netflix_l3_profile # Profile-based selection + "<=720": netflix_mobile_l3 # Quality-based selection + "1080": netflix_standard_l3 # Exact match for 1080p + ">=1440": netflix_premium_l1 # Quality-based selection + default: netflix_standard_l3 # Fallback +``` + --- ## remote_cdm (list\[dict]) @@ -113,8 +136,10 @@ remote_cdm: - Support for both Widevine and PlayReady - Multiple security levels (L1, L2, L3, SL2000, SL3000) -**Note:** The `device_type` and `security_level` fields are optional metadata. They don't affect API communication -but are used for internal device identification. +**Note:** The `device_type` field determines whether the CDM operates in PlayReady or Widevine mode. +Setting `device_type: PLAYREADY` (or using `device_name: SL2` / `SL3`) activates PlayReady mode. +The `security_level` field is auto-computed from `device_name` when not specified (e.g., SL2 defaults +to 2000, SL3 to 3000, and Widevine devices default to 3). You can override these if needed. ### Custom API Remote CDM @@ -171,7 +196,7 @@ remote_cdm: header_name: X-API-Key key: YOUR_SECRET_KEY custom_headers: - User-Agent: Unshackle/2.0.0 + User-Agent: Unshackle/3.1.0 X-Client-Version: "1.0" # Endpoint configuration @@ -224,6 +249,7 @@ remote_cdm: - `header` - Custom header authentication - `basic` - HTTP Basic authentication - `body` - Credentials in request body +- `query` - Authentication added to query string parameters ### Legacy PyWidevine Serve Format @@ -266,6 +292,58 @@ used as a fallback. --- +## decryption (str|dict) + +Configure which decryption tool to use for DRM-protected content. Default: `shaka`. + +Supported values: +- `shaka` - Shaka Packager (default) +- `mp4decrypt` - Bento4 mp4decrypt + +You can specify a single decrypter for all services: + +```yaml +decryption: shaka +``` + +Or configure per-service with a `DEFAULT` fallback: + +```yaml +decryption: + DEFAULT: shaka + AMZN: mp4decrypt + NF: shaka +``` + +Service keys are case-insensitive (normalized to uppercase internally). + +--- + +## MonaLisa DRM + +MonaLisa is a WASM-based DRM system that uses local key extraction and two-stage segment decryption. +Unlike Widevine and PlayReady, MonaLisa does not use a challenge/response flow with a license server. +Instead, the PSSH value (ticket) is provided directly by the service API, and keys are extracted +locally via a WASM module. + +### Requirements + +- **ML-Worker binary**: Must be available on your system `PATH` (discovered via `binaries.ML_Worker`). + This is the binary that performs stage-1 decryption. + +### Decryption stages + +1. **ML-Worker binary**: Removes MonaLisa encryption layer (bbts -> ents). The key is passed via command-line argument. +2. **AES-ECB decryption**: Final decryption with service-provided key. + +MonaLisa uses per-segment decryption during download (not post-download like Widevine/PlayReady), +so segments are decrypted as they are downloaded. + +**Note:** MonaLisa is configured per-service rather than through global config options. Services +that use MonaLisa handle ticket/key retrieval and CDM initialization internally. + +--- + ## key_vaults (list\[dict]) Key Vaults store your obtained Content Encryption Keys (CEKs) and Key IDs per-service. diff --git a/docs/GLUETUN.md b/docs/GLUETUN.md index 07c39d1..ebc1315 100644 --- a/docs/GLUETUN.md +++ b/docs/GLUETUN.md @@ -47,7 +47,7 @@ Format: `gluetun:provider:region` **OpenVPN (Recommended)**: Most providers support OpenVPN with just `username` and `password` - the simplest setup. -**WireGuard**: Requires private keys and varies by provider. See the [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) for provider-specific requirements. +**WireGuard**: Requires private keys and varies by provider. See the [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) for provider-specific requirements. Note that `vpn_type` defaults to `wireguard` if not specified. ## Getting Your Credentials @@ -104,6 +104,13 @@ providers: credentials: private_key: YOUR_PRIVATE_KEY +# Surfshark/Mullvad/IVPN (private_key AND addresses required) + surfshark: + vpn_type: wireguard + credentials: + private_key: YOUR_PRIVATE_KEY + addresses: 10.x.x.x/32 + # Windscribe (all three credentials required) windscribe: vpn_type: wireguard @@ -124,6 +131,46 @@ Most providers use `SERVER_COUNTRIES`, but some use `SERVER_REGIONS`: Unshackle handles this automatically - just use 2-letter country codes. +### Per-Provider Server Mapping + +You can explicitly map region codes to country names, cities, or hostnames per provider: + +```yaml +providers: + nordvpn: + vpn_type: openvpn + credentials: + username: YOUR_USERNAME + password: YOUR_PASSWORD + server_countries: + us: "United States" + uk: "United Kingdom" + server_cities: + us: "New York" + server_hostnames: + us: "us1239.nordvpn.com" +``` + +### Specific Server Selection + +Use `--proxy gluetun:nordvpn:us1239` for specific server selection. Unshackle builds the hostname +automatically based on the provider (e.g., `us1239.nordvpn.com` for NordVPN). + +### Extra Environment Variables + +You can pass additional Gluetun environment variables per provider using `extra_env`: + +```yaml +providers: + nordvpn: + vpn_type: openvpn + credentials: + username: YOUR_USERNAME + password: YOUR_PASSWORD + extra_env: + LOG_LEVEL: debug +``` + ## Global Settings ```yaml @@ -133,17 +180,19 @@ proxy_providers: base_port: 8888 # Starting port (default: 8888) auto_cleanup: true # Remove containers on exit (default: true) verify_ip: true # Verify IP matches region (default: true) - container_prefix: "unshackle-gluetun" + container_prefix: "unshackle-gluetun" # Docker container name prefix (default: "unshackle-gluetun") auth_user: username # Proxy auth (optional) - auth_password: password + auth_password: password # Proxy auth (optional) ``` ## Features -- **Container Reuse**: First request takes 10-30s; subsequent requests are instant -- **IP Verification**: Automatically verifies VPN exit IP matches requested region +- **Container Reuse**: First request takes 10-30s; subsequent requests are instant. Containers from other sessions are also detected and reused. +- **IP Verification**: Automatically verifies VPN exit IP matches requested region (configurable via `verify_ip`) - **Concurrent Sessions**: Multiple downloads share the same container - **Specific Servers**: Use `--proxy gluetun:nordvpn:us1239` for specific server selection +- **Automatic Image Pull**: The Gluetun Docker image (`qmcgaw/gluetun:latest`) is pulled automatically on first use +- **Secure Credentials**: Credentials are passed via temporary env files (mode 0600) rather than command-line arguments ## Container Management @@ -174,8 +223,9 @@ docker logs unshackle-gluetun-nordvpn-us Common issues: - Invalid/missing credentials -- Windscribe requires `preshared_key` (can be empty string) +- Windscribe WireGuard requires `preshared_key` (can be empty string, but must be set in credentials) - VPN provider server issues +- Container startup timeout (default 60 seconds) ## Resources diff --git a/docs/NETWORK_CONFIG.md b/docs/NETWORK_CONFIG.md index 8e7c131..96b7c39 100644 --- a/docs/NETWORK_CONFIG.md +++ b/docs/NETWORK_CONFIG.md @@ -36,6 +36,8 @@ system where required. You can also specify specific servers to use per-region with the `server_map` key. Sometimes a specific server works best for a service than others, so hard-coding one for a day or two helps. +You can also select servers by city using the format `--proxy nordvpn:us:seattle` or `--proxy nordvpn:ca:calgary`. + For example, ```yaml @@ -48,8 +50,8 @@ server_map: The username and password should NOT be your normal NordVPN Account Credentials. They should be the `Service credentials` which can be found on your Nord Account Dashboard. -Once set, you can also specifically opt in to use a NordVPN proxy by specifying `--proxy=gb` or such. -You can even set a specific server number this way, e.g., `--proxy=gb2366`. +Once set, you can also specifically opt in to use a NordVPN proxy by specifying `--proxy nordvpn:gb` or such. +You can even set a specific server number this way, e.g., `--proxy nordvpn:gb2366`. Note that `gb` is used instead of `uk` to be more consistent across regional systems. @@ -58,6 +60,8 @@ Note that `gb` is used instead of `uk` to be more consistent across regional sys Enable Surfshark VPN proxy service using Surfshark Service credentials (not your login password). You may pin specific server IDs per region using `server_map`. +You can also select servers by city using the format `--proxy surfsharkvpn:us:seattle`. + ```yaml username: your_surfshark_service_username # https://my.surfshark.com/vpn/manual-setup/main/openvpn password: your_surfshark_service_password # service credentials, not account password @@ -67,16 +71,13 @@ server_map: au: 4621 # force AU server #4621 ``` -### hola (dict) +### hola Enable Hola VPN proxy service. Requires the `hola-proxy` binary to be installed and available in your PATH. +No configuration is needed under `proxy_providers`. Hola is loaded automatically when the `hola-proxy` binary +is detected. -```yaml -proxy_providers: - hola: {} -``` - -Once configured, use `--proxy hola:us` or similar to connect through Hola. +Once available, use `--proxy hola:us` or similar to connect through Hola. ### windscribevpn (dict) @@ -105,32 +106,28 @@ proxy_providers: gb: uk-london-001.totallyacdn.com # Force specific UK server ``` -Once configured, use `--proxy windscribe:us` or `--proxy windscribe:gb` etc. to connect through Windscribe. +Once configured, use `--proxy windscribevpn:us` or `--proxy windscribevpn:gb` etc. to connect through Windscribe. -### Legacy nordvpn Configuration +You can also select specific servers by number (e.g., `--proxy windscribevpn:sg007`) or filter by city +(e.g., `--proxy windscribevpn:ca:toronto`). -**Legacy configuration. Use `proxy_providers.nordvpn` instead.** +### gluetun (dict) -Set your NordVPN Service credentials with `username` and `password` keys to automate the use of NordVPN as a Proxy -system where required. - -You can also specify specific servers to use per-region with the `server_map` key. -Sometimes a specific server works best for a service than others, so hard-coding one for a day or two helps. - -For example, +Docker-managed VPN proxy supporting 50+ VPN providers via Gluetun. See [GLUETUN.md](GLUETUN.md) for full +configuration and usage details. ```yaml -nordvpn: - username: zxqsR7C5CyGwmGb6KSvk8qsZ # example of the login format - password: wXVHmht22hhRKUEQ32PQVjCZ - server_map: - us: 12 # force US server #12 for US proxies +proxy_providers: + gluetun: + providers: + windscribe: + vpn_type: openvpn + credentials: + username: "YOUR_OPENVPN_USERNAME" + password: "YOUR_OPENVPN_PASSWORD" ``` -The username and password should NOT be your normal NordVPN Account Credentials. -They should be the `Service credentials` which can be found on your Nord Account Dashboard. - -Note that `gb` is used instead of `uk` to be more consistent across regional systems. +Usage: `--proxy gluetun:windscribe:us` --- @@ -141,14 +138,14 @@ All requests will use these unless changed explicitly or implicitly via a Server These should be sane defaults and anything that would only be useful for some Services should not be put here. -Avoid headers like 'Accept-Encoding' as that would be a compatibility header that Python-requests will +Avoid headers like 'Accept-Encoding' as that would be a compatibility header that curl_cffi will set for you. I recommend using, ```yaml Accept-Language: "en-US,en;q=0.8" -User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36" +User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" ``` --- diff --git a/docs/OUTPUT_CONFIG.md b/docs/OUTPUT_CONFIG.md index c7c209a..c09123e 100644 --- a/docs/OUTPUT_CONFIG.md +++ b/docs/OUTPUT_CONFIG.md @@ -37,6 +37,19 @@ Set scene-style naming for titles. When `true` uses scene naming patterns (e.g., --- +## dash_naming (bool) + +Use dash-separated naming convention for output files. Default: `false`. + +--- + +## unicode_filenames (bool) + +Allow Unicode characters in output filenames. When `false`, Unicode characters are transliterated +to ASCII equivalents. Default: `false`. + +--- + ## series_year (bool) Whether to include the series year in series names for episodes and folders. Default: `true`. @@ -67,6 +80,14 @@ Enable/disable tagging downloaded files with IMDB/TMDB/TVDB identifiers (when av - `set_title` Set the container title to `Show SXXEXX Episode Name` or `Movie (Year)`. Default: `true` +- `merge_audio` + Merge all audio tracks into each output file. Default: `true` + - `true`: All selected audio tracks are muxed into one MKV per quality. + - `false`: Separate MKV per (quality, audio_codec) combination. + For example: `Title.1080p.AAC.mkv`, `Title.1080p.EC3.mkv`. + + Note: The `--split-audio` CLI flag overrides this setting. When `--split-audio` is passed, + `merge_audio` is effectively set to `false` for that run. --- @@ -114,8 +135,9 @@ Notes: For example, ```yaml -downloads: "D:/Downloads/unshackle" -temp: "D:/Temp/unshackle" +directories: + downloads: "D:/Downloads/unshackle" + temp: "D:/Temp/unshackle" ``` There are directories not listed that cannot be modified as they are crucial to the operation of unshackle. diff --git a/docs/SERVICE_CONFIG.md b/docs/SERVICE_CONFIG.md index 849cb91..0d108ad 100644 --- a/docs/SERVICE_CONFIG.md +++ b/docs/SERVICE_CONFIG.md @@ -4,12 +4,12 @@ This document covers service-specific configuration, authentication, and metadat ## services (dict) -Configuration data for each Service. The Service will have the data within this section merged into the `config.yaml` -before provided to the Service class. +Configuration data for each Service. The Service will have the data within this section merged into the per-service +`config.yaml` (located in the service's directory) before being provided to the Service class. Think of this config to be used for more sensitive configuration data, like user or device-specific API keys, IDs, -device attributes, and so on. A `config.yaml` file is typically shared and not meant to be modified, so use this for -any sensitive configuration data. +device attributes, and so on. A per-service `config.yaml` file is typically shared and not meant to be modified, +so use this for any sensitive configuration data. The Key is the Service Tag, but can take any arbitrary form for its value. It's expected to begin as either a list or a dictionary. @@ -23,6 +23,29 @@ NOW: # ... more sensitive data ``` +### Per-Service Configuration Overrides + +You can override many global configuration options on a per-service basis by nesting them under the +service tag in the `services` section. Supported override keys include: `dl`, `aria2c`, `n_m3u8dl_re`, +`curl_impersonate`, `subtitle`, `muxing`, `headers`, and more. + +Overrides are merged with global config (not replaced) -- only specified keys are overridden, others +use global defaults. CLI arguments always take priority over service-specific config. + +For example, + +```yaml +services: + RATE_LIMITED_SERVICE: + dl: + downloads: 2 # Limit concurrent track downloads + workers: 4 # Reduce workers to avoid rate limits + n_m3u8dl_re: + thread_count: 4 # Very low thread count + aria2c: + max_concurrent_downloads: 1 +``` + --- ## credentials (dict[str, str|list|dict]) @@ -114,3 +137,28 @@ Maximum retention time in seconds for serving slightly stale cached title metada Default: `86400` (24 hours). Effective retention is `min(title_cache_time + grace, title_cache_max_retention)`. --- + +## debug (bool) + +Enable structured JSON debug logging for troubleshooting and service development. Default: `false`. + +When enabled (via config or the `--debug` CLI flag): +- Creates JSON Lines (`.jsonl`) log files with complete debugging context +- Logs: session info, CLI params, service config, CDM details, authentication, titles, tracks metadata, + DRM operations, vault queries, errors with stack traces +- File location: `logs/unshackle_debug_{service}_{timestamp}.jsonl` + +--- + +## debug_keys (bool) + +Log decryption keys in debug logs. Default: `false`. + +When `true`, actual content encryption keys (CEKs) are included in debug log output. Useful for +debugging key retrieval and decryption issues. + +**Security note:** Passwords, tokens, cookies, and session tokens are always redacted regardless +of this setting. Only content keys (`content_key`, `key` fields) are affected. Key IDs (`kid`), +key counts, and other metadata are always logged. + +--- diff --git a/docs/SUBTITLE_CONFIG.md b/docs/SUBTITLE_CONFIG.md index cc27870..29d16db 100644 --- a/docs/SUBTITLE_CONFIG.md +++ b/docs/SUBTITLE_CONFIG.md @@ -6,8 +6,8 @@ This document covers subtitle processing and formatting options. Control subtitle conversion and SDH (hearing-impaired) stripping behavior. -- `conversion_method`: How to convert subtitles between formats. Default: `pysubs2`. - - `auto`: Use subby for WebVTT/SAMI, standard for others. +- `conversion_method`: How to convert subtitles between formats. Default: `auto`. + - `auto`: Smart routing - use subby for WebVTT/SAMI, pycaption for others. - `subby`: Always use subby with CommonIssuesFixer. - `subtitleedit`: Prefer SubtitleEdit when available; otherwise fallback to standard conversion. - `pycaption`: Use only the pycaption library (no SubtitleEdit, no subby). @@ -23,17 +23,30 @@ Control subtitle conversion and SDH (hearing-impaired) stripping behavior. - `convert_before_strip`: When using `filter-subs` SDH method, automatically convert subtitles to SRT format first for better compatibility. Default: `true`. -- `preserve_formatting`: Keep original subtitle tags and positioning during conversion. Default: `true`. +- `preserve_formatting`: Keep original subtitle tags and positioning during conversion. When true, skips pycaption processing for WebVTT files to keep tags like ``, ``, and positioning intact. Default: `true`. + +- `output_mode`: Controls how subtitles are included in the output. Default: `mux`. + - `mux`: Embed subtitles in the MKV container only. + - `sidecar`: Save subtitles as separate files only (not muxed into the container). + - `both`: Embed subtitles in the MKV container and save as sidecar files. + +- `sidecar_format`: Format for sidecar subtitle files (used when `output_mode` is `sidecar` or `both`). Default: `srt`. + - `srt`: SubRip format. + - `vtt`: WebVTT format. + - `ass`: Advanced SubStation Alpha format. + - `original`: Keep the subtitle in its current format without conversion. Example: ```yaml subtitle: - conversion_method: pysubs2 + conversion_method: auto sdh_method: auto strip_sdh: true convert_before_strip: true preserve_formatting: true + output_mode: mux + sidecar_format: srt ``` --- diff --git a/pyproject.toml b/pyproject.toml index 8c128d6..f60d503 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "unshackle" -version = "2.3.1" +version = "3.1.0" description = "Modular Movie, TV, and Music Archival Software." authors = [{ name = "unshackle team" }] requires-python = ">=3.10,<3.13" diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index cde73cc..ab6e6df 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -3,6 +3,7 @@ from __future__ import annotations import html import logging import math +import os import random import re import shutil @@ -43,6 +44,7 @@ from rich.tree import Tree from unshackle.core import binaries from unshackle.core.cdm import CustomRemoteCDM, DecryptLabsRemoteCDM +from unshackle.core.cdm.detect import is_playready_cdm, is_widevine_cdm from unshackle.core.config import config from unshackle.core.console import console from unshackle.core.constants import DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings @@ -62,8 +64,10 @@ from unshackle.core.utilities import (find_font_with_fallbacks, get_debug_logger is_close_match, suggest_font_packages, time_elapsed_since) from unshackle.core.utils import tags from unshackle.core.utils.click_types import (AUDIO_CODEC_LIST, LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE, - ContextData, MultipleChoice, SubtitleCodecChoice, VideoCodecChoice) + ContextData, MultipleChoice, MultipleVideoCodecChoice, + SubtitleCodecChoice) from unshackle.core.utils.collections import merge_dict +from unshackle.core.utils.selector import select_multiple from unshackle.core.utils.subprocess import ffprobe from unshackle.core.vaults import Vaults @@ -193,12 +197,7 @@ class dl: sdh_suffix = ".sdh" if (subtitle.sdh or subtitle.cc) else "" extension = (target_codec or subtitle.codec or Subtitle.Codec.SubRip).extension - if ( - not target_codec - and not subtitle.codec - and source_path - and source_path.suffix - ): + if not target_codec and not subtitle.codec and source_path and source_path.suffix: extension = source_path.suffix.lstrip(".") filename = f"{base_filename}.{lang_suffix}{forced_suffix}{sdh_suffix}.{extension}" @@ -290,9 +289,9 @@ class dl: @click.option( "-v", "--vcodec", - type=VideoCodecChoice(Video.Codec), - default=None, - help="Video Codec to download, defaults to any codec.", + type=MultipleVideoCodecChoice(Video.Codec), + default=[], + help="Video Codec(s) to download, defaults to any codec.", ) @click.option( "-a", @@ -345,6 +344,12 @@ class dl: default=None, help="Create separate output files per audio codec instead of merging all audio.", ) + @click.option( + "--select-titles", + is_flag=True, + default=False, + help="Interactively select downloads from a list. Only use with Series to select Episodes", + ) @click.option( "-w", "--wanted", @@ -402,6 +407,7 @@ class dl: @click.option( "--tag", type=str, default=None, help="Set the Group Tag to be used, overriding the one in config if any." ) + @click.option("--repack", is_flag=True, default=False, help="Add REPACK tag to the output filename.") @click.option( "--tmdb", "tmdb_id", @@ -481,6 +487,14 @@ class dl: help="Max workers/threads to download with per-track. Default depends on the downloader.", ) @click.option("--downloads", type=int, default=1, help="Amount of tracks to download concurrently.") + @click.option( + "-o", + "--output", + "output_dir", + type=Path, + default=None, + help="Override the output directory for this download, instead of the one in config.", + ) @click.option("--no-cache", "no_cache", is_flag=True, default=False, help="Bypass title cache for this download.") @click.option( "--reset-cache", "reset_cache", is_flag=True, default=False, help="Clear title cache before fetching." @@ -504,10 +518,12 @@ class dl: no_proxy: bool, profile: Optional[str] = None, proxy: Optional[str] = None, + repack: bool = False, tag: Optional[str] = None, tmdb_id: Optional[int] = None, tmdb_name: bool = False, tmdb_year: bool = False, + output_dir: Optional[Path] = None, *_: Any, **__: Any, ): @@ -553,6 +569,7 @@ class dl: self.tmdb_id = tmdb_id self.tmdb_name = tmdb_name self.tmdb_year = tmdb_year + self.output_dir = output_dir # Initialize debug logger with service name if debug logging is enabled if config.debug or logging.root.level == logging.DEBUG: @@ -585,6 +602,59 @@ class dl: }, }, ) + + # Log binary versions for diagnostics + binary_versions = {} + for name, binary in [ + ("shaka_packager", binaries.ShakaPackager), + ("mp4decrypt", binaries.Mp4decrypt), + ("n_m3u8dl_re", binaries.N_m3u8DL_RE), + ("mkvmerge", binaries.MKVToolNix), + ("ffmpeg", binaries.FFMPEG), + ("ffprobe", binaries.FFProbe), + ]: + if binary: + version = None + try: + if name == "shaka_packager": + r = subprocess.run( + [str(binary), "--version"], capture_output=True, text=True, timeout=5 + ) + version = (r.stdout or r.stderr or "").strip() + elif name in ("ffmpeg", "ffprobe"): + r = subprocess.run( + [str(binary), "-version"], capture_output=True, text=True, timeout=5 + ) + version = (r.stdout or "").split("\n")[0].strip() + elif name == "mkvmerge": + r = subprocess.run( + [str(binary), "--version"], capture_output=True, text=True, timeout=5 + ) + version = (r.stdout or "").strip() + elif name == "mp4decrypt": + r = subprocess.run( + [str(binary)], capture_output=True, text=True, timeout=5 + ) + output = (r.stdout or "") + (r.stderr or "") + lines = [line.strip() for line in output.split("\n") if line.strip()] + version = " | ".join(lines[:2]) if lines else None + elif name == "n_m3u8dl_re": + r = subprocess.run( + [str(binary), "--version"], capture_output=True, text=True, timeout=5 + ) + version = (r.stdout or r.stderr or "").strip().split("\n")[0] + except Exception: + version = "" + binary_versions[name] = {"path": str(binary), "version": version} + else: + binary_versions[name] = None + + self.debug_logger.log( + level="DEBUG", + operation="binary_versions", + message="Binary tool versions", + context=binary_versions, + ) else: self.debug_logger = None @@ -777,6 +847,9 @@ class dl: r"^[a-z]+:[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE ): proxy = proxy.lower() + # Preserve the original user query (region code) for service-specific proxy_map overrides. + # NOTE: `proxy` may be overwritten with the resolved proxy URI later. + proxy_query = proxy status_msg = ( f"Connecting to VPN ({proxy})..." if requested_provider == "gluetun" @@ -791,7 +864,6 @@ class dl: if not proxy_provider: self.log.error(f"The proxy provider '{requested_provider}' was not recognised.") sys.exit(1) - proxy_query = proxy # Save query before overwriting with URI proxy_uri = proxy_provider.get_proxy(proxy) if not proxy_uri: self.log.error(f"The proxy provider {requested_provider} had no proxy for {proxy}") @@ -810,7 +882,6 @@ class dl: self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") else: for proxy_provider in self.proxy_providers: - proxy_query = proxy # Save query before overwriting with URI proxy_uri = proxy_provider.get_proxy(proxy) if proxy_uri: proxy = ctx.params["proxy"] = proxy_uri @@ -827,7 +898,7 @@ class dl: self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") break # Store proxy query info for service-specific overrides - ctx.params["proxy_query"] = proxy + ctx.params["proxy_query"] = proxy_query ctx.params["proxy_provider"] = requested_provider else: self.log.info(f"Using explicit Proxy: {proxy}") @@ -839,6 +910,9 @@ class dl: config=self.service_config, cdm=self.cdm, proxy_providers=self.proxy_providers, profile=self.profile ) + if repack: + config.repack = True + if tag: config.tag = tag @@ -850,13 +924,14 @@ class dl: self, service: Service, quality: list[int], - vcodec: Optional[Video.Codec], + vcodec: list[Video.Codec], acodec: list[Audio.Codec], vbitrate: int, abitrate: int, range_: list[Video.Range], channels: float, no_atmos: bool, + select_titles: bool, wanted: list[str], latest_episode: bool, lang: list[str], @@ -897,6 +972,9 @@ class dl: self.search_source = None start_time = time.time() + if skip_dl: + DOWNLOAD_LICENCE_ONLY.set() + if not acodec: acodec = [] elif isinstance(acodec, Audio.Codec): @@ -1042,6 +1120,78 @@ class dl: if list_titles: return + # Enables manual selection for Series when --select-titles is set + if select_titles and isinstance(titles, Series): + console.print(Padding(Rule("[rule.text]Select Titles"), (1, 2))) + + selection_titles = [] + dependencies = {} + original_indices = {} + + current_season = None + current_season_header_idx = -1 + + unique_seasons = {t.season for t in titles} + multiple_seasons = len(unique_seasons) > 1 + + # Build selection options + for i, t in enumerate(titles): + # Insert season header only if multiple seasons exist + if multiple_seasons and t.season != current_season: + current_season = t.season + header_text = f"Season {t.season}" + selection_titles.append(header_text) + current_season_header_idx = len(selection_titles) - 1 + dependencies[current_season_header_idx] = [] + # Note: Headers are not mapped to actual title indices + + # Format display name + display_name = ((t.name[:35].rstrip() + "…") if len(t.name) > 35 else t.name) if t.name else None + + # Apply indentation only for multiple seasons + prefix = " " if multiple_seasons else "" + option_text = f"{prefix}{t.number}" + (f". {display_name}" if t.name else "") + + selection_titles.append(option_text) + current_ui_idx = len(selection_titles) - 1 + + # Map UI index to actual title index + original_indices[current_ui_idx] = i + + # Link episode to season header for group selection + if current_season_header_idx != -1: + dependencies[current_season_header_idx].append(current_ui_idx) + + selection_start = time.time() + + # Execute selector with dependencies (headers select all children) + selected_ui_idx = select_multiple( + selection_titles, minimal_count=1, page_size=8, return_indices=True, dependencies=dependencies + ) + + selection_end = time.time() + start_time += selection_end - selection_start + + # Map UI indices back to title indices (excluding headers) + selected_idx = [] + for idx in selected_ui_idx: + if idx in original_indices: + selected_idx.append(original_indices[idx]) + + # Ensure indices are unique and ordered + selected_idx = sorted(set(selected_idx)) + keep = set(selected_idx) + + # In-place filter: remove unselected items (iterate backwards) + for i in range(len(titles) - 1, -1, -1): + if i not in keep: + del titles[i] + + # Show selected count + if titles: + count = len(titles) + console.print(Padding(f"[text]Total selected: {count}[/]", (0, 5))) + # Determine the latest episode if --latest-episode is set latest_episode_id = None if latest_episode and isinstance(titles, Series) and len(titles) > 0: @@ -1245,9 +1395,12 @@ class dl: if isinstance(title, (Movie, Episode)): # filter video tracks if vcodec: - title.tracks.select_video(lambda x: x.codec == vcodec) + title.tracks.select_video(lambda x: x.codec in vcodec) + missing_codecs = [c for c in vcodec if not any(x.codec == c for x in title.tracks.videos)] + for codec in missing_codecs: + self.log.warning(f"Skipping {codec.name} video tracks as none are available.") if not title.tracks.videos: - self.log.error(f"There's no {vcodec.name} Video Track...") + self.log.error(f"There's no {', '.join(c.name for c in vcodec)} Video Track...") sys.exit(1) if range_: @@ -1259,10 +1412,20 @@ class dl: self.log.warning(f"Skipping {color_range.name} video tracks as none are available.") if vbitrate: - title.tracks.select_video(lambda x: x.bitrate and x.bitrate // 1000 == vbitrate) - if not title.tracks.videos: - self.log.error(f"There's no {vbitrate}kbps Video Track...") - sys.exit(1) + if any(r == Video.Range.HYBRID for r in range_): + # In HYBRID mode, only apply bitrate filter to non-DV tracks + # DV tracks are kept regardless since they're only used for RPU metadata + title.tracks.select_video( + lambda x: x.range == Video.Range.DV or (x.bitrate and x.bitrate // 1000 == vbitrate) + ) + if not any(x.range != Video.Range.DV for x in title.tracks.videos): + self.log.error(f"There's no {vbitrate}kbps Video Track...") + sys.exit(1) + else: + title.tracks.select_video(lambda x: x.bitrate and x.bitrate // 1000 == vbitrate) + if not title.tracks.videos: + self.log.error(f"There's no {vbitrate}kbps Video Track...") + sys.exit(1) video_languages = [lang for lang in (v_lang or lang) if lang != "best"] if video_languages and "all" not in video_languages: @@ -1289,10 +1452,38 @@ class dl: self.log.error(f"There's no {processed_video_lang} Video Track...") sys.exit(1) + has_hybrid = any(r == Video.Range.HYBRID for r in range_) + non_hybrid_ranges = [r for r in range_ if r != Video.Range.HYBRID] + if quality: missing_resolutions = [] - if any(r == Video.Range.HYBRID for r in range_): - title.tracks.select_video(title.tracks.select_hybrid(title.tracks.videos, quality)) + if has_hybrid: + # Split tracks: hybrid candidates vs non-hybrid + hybrid_candidate_tracks = [ + v for v in title.tracks.videos + if v.range in (Video.Range.HDR10, Video.Range.HDR10P, Video.Range.DV) + ] + non_hybrid_tracks = [ + v for v in title.tracks.videos + if v.range not in (Video.Range.HDR10, Video.Range.HDR10P, Video.Range.DV) + ] + + # Apply hybrid selection to HDR10+DV tracks + hybrid_filter = title.tracks.select_hybrid(hybrid_candidate_tracks, quality) + hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) + + if non_hybrid_ranges and non_hybrid_tracks: + # Also filter non-hybrid tracks by resolution + non_hybrid_selected = [ + v for v in non_hybrid_tracks + if any( + v.height == res or int(v.width * (9 / 16)) == res + for res in quality + ) + ] + title.tracks.videos = hybrid_selected + non_hybrid_selected + else: + title.tracks.videos = hybrid_selected else: title.tracks.by_resolutions(quality) @@ -1319,21 +1510,63 @@ class dl: sys.exit(1) # choose best track by range and quality - if any(r == Video.Range.HYBRID for r in range_): - # For hybrid mode, always apply hybrid selection - # If no quality specified, use only the best (highest) resolution + if has_hybrid: + # Apply hybrid selection for HYBRID tracks + hybrid_candidate_tracks = [ + v for v in title.tracks.videos + if v.range in (Video.Range.HDR10, Video.Range.HDR10P, Video.Range.DV) + ] + non_hybrid_tracks = [ + v for v in title.tracks.videos + if v.range not in (Video.Range.HDR10, Video.Range.HDR10P, Video.Range.DV) + ] + if not quality: - # Get the highest resolution available - best_resolution = max((v.height for v in title.tracks.videos), default=None) + best_resolution = max( + (v.height for v in hybrid_candidate_tracks), default=None + ) if best_resolution: - # Use the hybrid selection logic with only the best resolution - title.tracks.select_video( - title.tracks.select_hybrid(title.tracks.videos, [best_resolution]) + hybrid_filter = title.tracks.select_hybrid( + hybrid_candidate_tracks, [best_resolution] ) - # If quality was specified, hybrid selection was already applied above + hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) + else: + hybrid_selected = [] + else: + hybrid_filter = title.tracks.select_hybrid( + hybrid_candidate_tracks, quality + ) + hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) + + # For non-hybrid ranges, apply Cartesian product selection + non_hybrid_selected: list[Video] = [] + if non_hybrid_ranges and non_hybrid_tracks: + for resolution, color_range, codec in product( + quality or [None], non_hybrid_ranges, vcodec or [None] + ): + match = next( + ( + t + for t in non_hybrid_tracks + if ( + not resolution + or t.height == resolution + or int(t.width * (9 / 16)) == resolution + ) + and (not color_range or t.range == color_range) + and (not codec or t.codec == codec) + ), + None, + ) + if match and match not in non_hybrid_selected: + non_hybrid_selected.append(match) + + title.tracks.videos = hybrid_selected + non_hybrid_selected else: selected_videos: list[Video] = [] - for resolution, color_range in product(quality or [None], range_ or [None]): + for resolution, color_range, codec in product( + quality or [None], range_ or [None], vcodec or [None] + ): match = next( ( t @@ -1344,6 +1577,7 @@ class dl: or int(t.width * (9 / 16)) == resolution ) and (not color_range or t.range == color_range) + and (not codec or t.codec == codec) ), None, ) @@ -1353,26 +1587,44 @@ class dl: # validate hybrid mode requirements if any(r == Video.Range.HYBRID for r in range_): - hdr10_tracks = [v for v in title.tracks.videos if v.range == Video.Range.HDR10] + base_tracks = [ + v for v in title.tracks.videos + if v.range in (Video.Range.HDR10, Video.Range.HDR10P) + ] dv_tracks = [v for v in title.tracks.videos if v.range == Video.Range.DV] - if not hdr10_tracks and not dv_tracks: + hybrid_failed = False + if not base_tracks and not dv_tracks: available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) - self.log.error("HYBRID mode requires both HDR10 and DV tracks, but neither is available") - self.log.error( + msg = "HYBRID mode requires both HDR10/HDR10+ and DV tracks, but neither is available" + msg_detail = ( f"Available ranges: {', '.join(available_ranges) if available_ranges else 'none'}" ) - sys.exit(1) - elif not hdr10_tracks: + hybrid_failed = True + elif not base_tracks: available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) - self.log.error("HYBRID mode requires both HDR10 and DV tracks, but only DV is available") - self.log.error(f"Available ranges: {', '.join(available_ranges)}") - sys.exit(1) + msg = "HYBRID mode requires both HDR10/HDR10+ and DV tracks, but only DV is available" + msg_detail = f"Available ranges: {', '.join(available_ranges)}" + hybrid_failed = True elif not dv_tracks: available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) - self.log.error("HYBRID mode requires both HDR10 and DV tracks, but only HDR10 is available") - self.log.error(f"Available ranges: {', '.join(available_ranges)}") - sys.exit(1) + msg = "HYBRID mode requires both HDR10/HDR10+ and DV tracks, but only HDR10 is available" + msg_detail = f"Available ranges: {', '.join(available_ranges)}" + hybrid_failed = True + + if hybrid_failed: + other_ranges = [r for r in range_ if r != Video.Range.HYBRID] + if best_available and other_ranges: + self.log.warning(msg) + self.log.warning( + f"Continuing with remaining range(s): " + f"{', '.join(r.name for r in other_ranges)}" + ) + range_ = other_ranges + else: + self.log.error(msg) + self.log.error(msg_detail) + sys.exit(1) # filter subtitle tracks if require_subs: @@ -1456,33 +1708,81 @@ class dl: if language not in processed_lang: processed_lang.append(language) - if "best" in processed_lang: + if "best" in processed_lang or "all" in processed_lang: unique_languages = {track.language for track in title.tracks.audio} selected_audio = [] + for language in unique_languages: + codecs_to_check = acodec if (acodec and len(acodec) > 1) else [None] + for codec in codecs_to_check: + base_candidates = [ + t + for t in title.tracks.audio + if t.language == language and (codec is None or t.codec == codec) + ] + if not base_candidates: + continue + if audio_description: + standards = [t for t in base_candidates if not t.descriptive] + if standards: + selected_audio.append(max(standards, key=lambda x: x.bitrate or 0)) + descs = [t for t in base_candidates if t.descriptive] + if descs: + selected_audio.append(max(descs, key=lambda x: x.bitrate or 0)) + else: + selected_audio.append(max(base_candidates, key=lambda x: x.bitrate or 0)) + title.tracks.audio = selected_audio + else: + # If multiple codecs were explicitly requested, pick the best track per codec per + # requested language instead of selecting *all* bitrate variants of a codec. if acodec and len(acodec) > 1: - for language in unique_languages: + selected_audio: list[Audio] = [] + + for language in processed_lang: for codec in acodec: - candidates = [ - track - for track in title.tracks.audio - if track.language == language and track.codec == codec - ] + codec_tracks = [a for a in title.tracks.audio if a.codec == codec] + if not codec_tracks: + continue + + candidates = title.tracks.by_language( + codec_tracks, [language], per_language=0, exact_match=exact_lang + ) if not candidates: continue - selected_audio.append(max(candidates, key=lambda x: x.bitrate or 0)) + + if audio_description: + standards = [t for t in candidates if not t.descriptive] + if standards: + selected_audio.append(max(standards, key=lambda x: x.bitrate or 0)) + descs = [t for t in candidates if t.descriptive] + if descs: + selected_audio.append(max(descs, key=lambda x: x.bitrate or 0)) + else: + selected_audio.append(max(candidates, key=lambda x: x.bitrate or 0)) + + title.tracks.audio = selected_audio else: - for language in unique_languages: - highest_quality = max( - (track for track in title.tracks.audio if track.language == language), - key=lambda x: x.bitrate or 0, + per_language = 1 + if audio_description: + standard_audio = [a for a in title.tracks.audio if not a.descriptive] + selected_standards = title.tracks.by_language( + standard_audio, + processed_lang, + per_language=per_language, + exact_match=exact_lang, + ) + desc_audio = [a for a in title.tracks.audio if a.descriptive] + # Include all descriptive tracks for the requested languages. + selected_descs = title.tracks.by_language( + desc_audio, processed_lang, per_language=0, exact_match=exact_lang + ) + title.tracks.audio = selected_standards + selected_descs + else: + title.tracks.audio = title.tracks.by_language( + title.tracks.audio, + processed_lang, + per_language=per_language, + exact_match=exact_lang, ) - selected_audio.append(highest_quality) - title.tracks.audio = selected_audio - elif "all" not in processed_lang: - per_language = 0 if acodec and len(acodec) > 1 else 1 - title.tracks.audio = title.tracks.by_language( - title.tracks.audio, processed_lang, per_language=per_language, exact_match=exact_lang - ) if not title.tracks.audio: self.log.error(f"There's no {processed_lang} Audio Track, cannot continue...") sys.exit(1) @@ -1552,9 +1852,7 @@ class dl: if video_tracks: highest_quality = max((track.height for track in video_tracks if track.height), default=0) if highest_quality > 0: - if isinstance(self.cdm, (WidevineCdm, DecryptLabsRemoteCDM)) and not ( - isinstance(self.cdm, DecryptLabsRemoteCDM) and self.cdm.is_playready - ): + if is_widevine_cdm(self.cdm): quality_based_cdm = self.get_cdm( self.service, self.profile, drm="widevine", quality=highest_quality ) @@ -1563,9 +1861,7 @@ class dl: f"Pre-selecting Widevine CDM based on highest quality {highest_quality}p across all video tracks" ) self.cdm = quality_based_cdm - elif isinstance(self.cdm, (PlayReadyCdm, DecryptLabsRemoteCDM)) and ( - isinstance(self.cdm, DecryptLabsRemoteCDM) and self.cdm.is_playready - ): + elif is_playready_cdm(self.cdm): quality_based_cdm = self.get_cdm( self.service, self.profile, drm="playready", quality=highest_quality ) @@ -1577,9 +1873,6 @@ class dl: dl_start_time = time.time() - if skip_dl: - DOWNLOAD_LICENCE_ONLY.set() - try: with Live(Padding(download_table, (1, 5)), console=console, refresh_per_second=5): with ThreadPoolExecutor(downloads) as pool: @@ -1599,12 +1892,7 @@ class dl: ), licence=partial( service.get_playready_license - if ( - isinstance(self.cdm, PlayReadyCdm) - or ( - isinstance(self.cdm, DecryptLabsRemoteCDM) and self.cdm.is_playready - ) - ) + if (is_playready_cdm(self.cdm)) and hasattr(service, "get_playready_license") else service.get_widevine_license, title=title, @@ -1722,9 +2010,7 @@ class dl: # Subtitle output mode configuration (for sidecar originals) subtitle_output_mode = config.subtitle.get("output_mode", "mux") sidecar_format = config.subtitle.get("sidecar_format", "srt") - skip_subtitle_mux = ( - subtitle_output_mode == "sidecar" and (title.tracks.videos or title.tracks.audio) - ) + skip_subtitle_mux = subtitle_output_mode == "sidecar" and (title.tracks.videos or title.tracks.audio) sidecar_subtitles: list[Subtitle] = [] sidecar_original_paths: dict[str, Path] = {} if subtitle_output_mode in ("sidecar", "both") and not no_mux: @@ -1800,6 +2086,7 @@ class dl: muxed_paths = [] muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {} + append_audio_codec_suffix = True if no_mux: # Skip muxing, handle individual track files @@ -1816,12 +2103,16 @@ class dl: console=console, ) - if split_audio is not None: - merge_audio = not split_audio - else: - merge_audio = config.muxing.get("merge_audio", True) + merge_audio = ( + (not split_audio) if split_audio is not None else config.muxing.get("merge_audio", True) + ) + # When we split audio (merge_audio=False), multiple outputs may exist per title, so suffix codec. + append_audio_codec_suffix = not merge_audio multiplex_tasks: list[tuple[TaskID, Tracks, Optional[Audio.Codec]]] = [] + # Track hybrid-processing outputs explicitly so we can always clean them up, + # even if muxing fails early (e.g. SystemExit) before the normal delete loop. + hybrid_temp_paths: list[Path] = [] def clone_tracks_for_audio(base_tracks: Tracks, audio_tracks: list[Audio]) -> Tracks: task_tracks = Tracks() @@ -1856,12 +2147,15 @@ class dl: # Hybrid mode: process DV and HDR10 tracks separately for each resolution self.log.info("Processing Hybrid HDR10+DV tracks...") - # Group video tracks by resolution + # Group video tracks by resolution (prefer HDR10+ over HDR10 as base) resolutions_processed = set() - hdr10_tracks = [v for v in title.tracks.videos if v.range == Video.Range.HDR10] + base_tracks_list = [ + v for v in title.tracks.videos + if v.range in (Video.Range.HDR10P, Video.Range.HDR10) + ] dv_tracks = [v for v in title.tracks.videos if v.range == Video.Range.DV] - for hdr10_track in hdr10_tracks: + for hdr10_track in base_tracks_list: resolution = hdr10_track.height if resolution in resolutions_processed: continue @@ -1883,10 +2177,13 @@ class dl: # Create unique output filename for this resolution hybrid_filename = f"HDR10-DV-{resolution}p.hevc" hybrid_output_path = config.directories.temp / hybrid_filename + hybrid_temp_paths.append(hybrid_output_path) # The Hybrid class creates HDR10-DV.hevc, rename it for this resolution default_output = config.directories.temp / "HDR10-DV.hevc" if default_output.exists(): + # If a previous run left this behind, replace it to avoid move() failures. + hybrid_output_path.unlink(missing_ok=True) shutil.move(str(default_output), str(hybrid_output_path)) # Create tracks with the hybrid video output for this resolution @@ -1895,9 +2192,11 @@ class dl: # Create a new video track for the hybrid output hybrid_track = deepcopy(hdr10_track) + hybrid_track.id = f"hybrid_{hdr10_track.id}_{resolution}" hybrid_track.path = hybrid_output_path hybrid_track.range = Video.Range.DV # It's now a DV track hybrid_track.needs_duration_fix = True + title.tracks.add(hybrid_track) task_tracks.videos = [hybrid_track] enqueue_mux_tasks(task_description, task_tracks) @@ -1912,6 +2211,8 @@ class dl: task_description += f" {video_track.height}p" if len(range_) > 1: task_description += f" {video_track.range.name}" + if len(vcodec) > 1: + task_description += f" {video_track.codec.name}" task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments if video_track: @@ -1919,80 +2220,97 @@ class dl: enqueue_mux_tasks(task_description, task_tracks) - with Live(Padding(progress, (0, 5, 1, 5)), console=console): - mux_index = 0 - for task_id, task_tracks, audio_codec in multiplex_tasks: - progress.start_task(task_id) # TODO: Needed? - audio_expected = not video_only and not no_audio - muxed_path, return_code, errors = task_tracks.mux( - str(title), - progress=partial(progress.update, task_id=task_id), - delete=False, - audio_expected=audio_expected, - title_language=title.language, - skip_subtitles=skip_subtitle_mux, - ) - if muxed_path.exists(): - mux_index += 1 - unique_path = muxed_path.with_name( - f"{muxed_path.stem}.{mux_index}{muxed_path.suffix}" + try: + with Live(Padding(progress, (0, 5, 1, 5)), console=console): + mux_index = 0 + for task_id, task_tracks, audio_codec in multiplex_tasks: + progress.start_task(task_id) # TODO: Needed? + audio_expected = not video_only and not no_audio + muxed_path, return_code, errors = task_tracks.mux( + str(title), + progress=partial(progress.update, task_id=task_id), + delete=False, + audio_expected=audio_expected, + title_language=title.language, + skip_subtitles=skip_subtitle_mux, ) - if unique_path != muxed_path: - shutil.move(muxed_path, unique_path) - muxed_path = unique_path - muxed_paths.append(muxed_path) - muxed_audio_codecs[muxed_path] = audio_codec - if return_code >= 2: - self.log.error(f"Failed to Mux video to Matroska file ({return_code}):") - elif return_code == 1 or errors: - self.log.warning("mkvmerge had at least one warning or error, continuing anyway...") - for line in errors: - if line.startswith("#GUI#error"): - self.log.error(line) + if muxed_path.exists(): + mux_index += 1 + unique_path = muxed_path.with_name( + f"{muxed_path.stem}.{mux_index}{muxed_path.suffix}" + ) + if unique_path != muxed_path: + shutil.move(muxed_path, unique_path) + muxed_path = unique_path + muxed_paths.append(muxed_path) + muxed_audio_codecs[muxed_path] = audio_codec + if return_code >= 2: + self.log.error(f"Failed to Mux video to Matroska file ({return_code}):") + elif return_code == 1 or errors: + self.log.warning("mkvmerge had at least one warning or error, continuing anyway...") + for line in errors: + if line.startswith("#GUI#error"): + self.log.error(line) + else: + self.log.warning(line) + if return_code >= 2: + sys.exit(1) + + # Output sidecar subtitles before deleting track files + if sidecar_subtitles and not no_mux: + media_info = MediaInfo.parse(muxed_paths[0]) if muxed_paths else None + if media_info: + base_filename = title.get_filename(media_info, show_service=not no_source) else: - self.log.warning(line) - if return_code >= 2: - sys.exit(1) + base_filename = str(title) - # Output sidecar subtitles before deleting track files - if sidecar_subtitles and not no_mux: - media_info = MediaInfo.parse(muxed_paths[0]) if muxed_paths else None - if media_info: - base_filename = title.get_filename(media_info, show_service=not no_source) - else: - base_filename = str(title) + sidecar_dir = self.output_dir or config.directories.downloads + if not no_folder and isinstance(title, (Episode, Song)) and media_info: + sidecar_dir /= title.get_filename( + media_info, show_service=not no_source, folder=True + ) + sidecar_dir.mkdir(parents=True, exist_ok=True) - sidecar_dir = config.directories.downloads - if not no_folder and isinstance(title, (Episode, Song)) and media_info: - sidecar_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) - sidecar_dir.mkdir(parents=True, exist_ok=True) + with console.status("Saving subtitle sidecar files..."): + created = self.output_subtitle_sidecars( + sidecar_subtitles, + base_filename, + sidecar_dir, + sidecar_format, + original_paths=sidecar_original_paths or None, + ) + if created: + self.log.info(f"Saved {len(created)} sidecar subtitle files") - with console.status("Saving subtitle sidecar files..."): - created = self.output_subtitle_sidecars( - sidecar_subtitles, - base_filename, - sidecar_dir, - sidecar_format, - original_paths=sidecar_original_paths or None, - ) - if created: - self.log.info(f"Saved {len(created)} sidecar subtitle files") + for track in title.tracks: + track.delete() - for track in title.tracks: - track.delete() + # Clear temp font attachment paths and delete other attachments + for attachment in title.tracks.attachments: + if attachment.path and attachment.path in temp_font_files: + attachment.path = None + else: + attachment.delete() - # Clear temp font attachment paths and delete other attachments - for attachment in title.tracks.attachments: - if attachment.path and attachment.path in temp_font_files: - attachment.path = None - else: - attachment.delete() - - # Clean up temp fonts - for temp_path in temp_font_files: - temp_path.unlink(missing_ok=True) - for temp_path in sidecar_original_paths.values(): - temp_path.unlink(missing_ok=True) + # Clean up temp fonts + for temp_path in temp_font_files: + temp_path.unlink(missing_ok=True) + for temp_path in sidecar_original_paths.values(): + temp_path.unlink(missing_ok=True) + finally: + # Hybrid() produces a temp HEVC output we rename; make sure it's never left behind. + # Also attempt to remove the default hybrid output name if it still exists. + for temp_path in hybrid_temp_paths: + try: + temp_path.unlink(missing_ok=True) + except PermissionError: + self.log.warning(f"Failed to delete temp file (in use?): {temp_path}") + try: + (config.directories.temp / "HDR10-DV.hevc").unlink(missing_ok=True) + except PermissionError: + self.log.warning( + f"Failed to delete temp file (in use?): {config.directories.temp / 'HDR10-DV.hevc'}" + ) else: # dont mux @@ -2000,7 +2318,7 @@ class dl: if no_mux: # Handle individual track files without muxing - final_dir = config.directories.downloads + final_dir = self.output_dir or config.directories.downloads if not no_folder and isinstance(title, (Episode, Song)): # Create folder based on title # Use first available track for filename generation @@ -2050,21 +2368,37 @@ class dl: self.log.debug(f"Saved: {final_path.name}") else: # Handle muxed files + used_final_paths: set[Path] = set() for muxed_path in muxed_paths: media_info = MediaInfo.parse(muxed_path) - final_dir = config.directories.downloads + final_dir = self.output_dir or config.directories.downloads final_filename = title.get_filename(media_info, show_service=not no_source) audio_codec_suffix = muxed_audio_codecs.get(muxed_path) - if audio_codec_suffix: - final_filename = f"{final_filename}.{audio_codec_suffix.name}" if not no_folder and isinstance(title, (Episode, Song)): final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) final_dir.mkdir(parents=True, exist_ok=True) final_path = final_dir / f"{final_filename}{muxed_path.suffix}" + if final_path.exists() and audio_codec_suffix and append_audio_codec_suffix: + sep = "." if config.scene_naming else " " + final_filename = f"{final_filename.rstrip()}{sep}{audio_codec_suffix.name}" + final_path = final_dir / f"{final_filename}{muxed_path.suffix}" - shutil.move(muxed_path, final_path) + if final_path in used_final_paths: + sep = "." if config.scene_naming else " " + i = 2 + while final_path in used_final_paths: + final_path = final_dir / f"{final_filename.rstrip()}{sep}{i}{muxed_path.suffix}" + i += 1 + + try: + os.replace(muxed_path, final_path) + except OSError: + if final_path.exists(): + final_path.unlink() + shutil.move(muxed_path, final_path) + used_final_paths.add(final_path) tags.tag_file(final_path, title, self.tmdb_id) title_dl_time = time_elapsed_since(dl_start_time) @@ -2106,9 +2440,7 @@ class dl: track_quality = track.height if isinstance(drm, Widevine): - if not isinstance(self.cdm, (WidevineCdm, DecryptLabsRemoteCDM)) or ( - isinstance(self.cdm, DecryptLabsRemoteCDM) and self.cdm.is_playready - ): + if not is_widevine_cdm(self.cdm): widevine_cdm = self.get_cdm(self.service, self.profile, drm="widevine", quality=track_quality) if widevine_cdm: if track_quality: @@ -2118,9 +2450,7 @@ class dl: self.cdm = widevine_cdm elif isinstance(drm, PlayReady): - if not isinstance(self.cdm, (PlayReadyCdm, DecryptLabsRemoteCDM)) or ( - isinstance(self.cdm, DecryptLabsRemoteCDM) and not self.cdm.is_playready - ): + if not is_playready_cdm(self.cdm): playready_cdm = self.get_cdm(self.service, self.profile, drm="playready", quality=track_quality) if playready_cdm: if track_quality: @@ -2673,12 +3003,12 @@ class dl: ) else: return RemoteCdm( - device_type=cdm_api["Device Type"], - system_id=cdm_api["System ID"], - security_level=cdm_api["Security Level"], - host=cdm_api["Host"], - secret=cdm_api["Secret"], - device_name=cdm_api["Device Name"], + device_type=cdm_api.get("Device Type", cdm_api.get("device_type", "")), + system_id=cdm_api.get("System ID", cdm_api.get("system_id", "")), + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), ) prd_path = config.directories.prds / f"{cdm_name}.prd" diff --git a/unshackle/commands/serve.py b/unshackle/commands/serve.py index 9276976..3df962a 100644 --- a/unshackle/commands/serve.py +++ b/unshackle/commands/serve.py @@ -123,7 +123,7 @@ def serve( log.info("Starting REST API server (pywidevine/pyplayready CDM disabled)") if no_key: app = web.Application(middlewares=[cors_middleware]) - app["config"] = {"users": []} + app["config"] = {"users": {}} else: app = web.Application(middlewares=[cors_middleware, pywidevine_serve.authentication]) app["config"] = {"users": {api_secret: {"devices": [], "username": "api_user"}}} @@ -164,7 +164,12 @@ def serve( for user_key, user_config in serve_config["users"].items(): if "playready_devices" not in user_config: - user_config["playready_devices"] = prd_device_names + # Require explicit PlayReady device access per user (default: no access). + user_config["playready_devices"] = [] + log.warning( + f'User "{user_key}" has no "playready_devices" configured; PlayReady access disabled for this user. ' + f"Available PlayReady devices: {prd_device_names}" + ) def create_serve_authentication(serve_playready_flag: bool): @web.middleware @@ -206,13 +211,13 @@ def serve( "devices": prd_devices, "users": { user_key: { - "devices": user_cfg.get("playready_devices", prd_device_names), + "devices": user_cfg.get("playready_devices", []), "username": user_cfg.get("username", "user"), } for user_key, user_cfg in serve_config["users"].items() } if not no_key - else [], + else {}, } playready_app["config"] = playready_config playready_app.on_startup.append(pyplayready_serve._startup) diff --git a/unshackle/core/__init__.py b/unshackle/core/__init__.py index 3a5935a..f5f41e5 100644 --- a/unshackle/core/__init__.py +++ b/unshackle/core/__init__.py @@ -1 +1 @@ -__version__ = "2.3.1" +__version__ = "3.1.0" diff --git a/unshackle/core/__main__.py b/unshackle/core/__main__.py index 6cf2fac..a475233 100644 --- a/unshackle/core/__main__.py +++ b/unshackle/core/__main__.py @@ -1,5 +1,6 @@ import atexit import logging +from datetime import datetime import click import urllib3 @@ -58,7 +59,7 @@ def main(version: bool, debug: bool) -> None: r" ▀▀▀ ▀▀ █▪ ▀▀▀▀ ▀▀▀ · ▀ ▀ ·▀▀▀ ·▀ ▀.▀▀▀ ▀▀▀ ", style="ascii.art", ), - f"v [repr.number]{__version__}[/] - © 2025 - github.com/unshackle-dl/unshackle", + f"v [repr.number]{__version__}[/] - © 2025-{datetime.now().year} - github.com/unshackle-dl/unshackle", ), (1, 11, 1, 10), expand=True, diff --git a/unshackle/core/api/api_keys.py b/unshackle/core/api/api_keys.py deleted file mode 100644 index 8d868b9..0000000 --- a/unshackle/core/api/api_keys.py +++ /dev/null @@ -1,145 +0,0 @@ -"""API key tier management for remote services.""" - -import logging -from typing import Any, Dict, List, Optional - -from aiohttp import web - -log = logging.getLogger("api.keys") - - -def get_api_key_from_request(request: web.Request) -> Optional[str]: - """ - Extract API key from request headers. - - Args: - request: aiohttp request object - - Returns: - API key string or None - """ - api_key = request.headers.get("X-API-Key") - if api_key: - return api_key - - auth_header = request.headers.get("Authorization", "") - if auth_header.startswith("Bearer "): - return auth_header[7:] # len("Bearer ") == 7 - - return None - - -def get_api_key_config(app: web.Application, api_key: str) -> Optional[Dict[str, Any]]: - """ - Get configuration for a specific API key. - - Args: - app: aiohttp application - api_key: API key to look up - - Returns: - API key configuration dict or None if not found - """ - config = app.get("config", {}) - - # Check new-style tiered API keys - api_keys = config.get("api_keys", []) - for key_config in api_keys: - if isinstance(key_config, dict) and key_config.get("key") == api_key: - return key_config - - # Check legacy users list (backward compatibility) - users = config.get("users", []) - if api_key in users: - return { - "key": api_key, - "tier": "basic", - "allowed_cdms": [] - } - - return None - - -def is_premium_user(app: web.Application, api_key: str) -> bool: - """ - Check if an API key belongs to a premium user. - - Premium users can use server-side CDM for decryption. - - Args: - app: aiohttp application - api_key: API key to check - - Returns: - True if premium user, False otherwise - """ - key_config = get_api_key_config(app, api_key) - if not key_config: - return False - - tier = key_config.get("tier", "basic") - return tier == "premium" - - -def get_allowed_cdms(app: web.Application, api_key: str) -> List[str]: - """ - Get list of CDMs that an API key is allowed to use. - - Args: - app: aiohttp application - api_key: API key to check - - Returns: - List of allowed CDM names, or empty list if not premium - """ - key_config = get_api_key_config(app, api_key) - if not key_config: - return [] - - allowed_cdms = key_config.get("allowed_cdms", []) - - # Handle wildcard - if allowed_cdms == "*" or allowed_cdms == ["*"]: - return ["*"] - - return allowed_cdms if isinstance(allowed_cdms, list) else [] - - -def get_default_cdm(app: web.Application, api_key: str) -> Optional[str]: - """ - Get default CDM for an API key. - - Args: - app: aiohttp application - api_key: API key to check - - Returns: - Default CDM name or None - """ - key_config = get_api_key_config(app, api_key) - if not key_config: - return None - - return key_config.get("default_cdm") - - -def can_use_cdm(app: web.Application, api_key: str, cdm_name: str) -> bool: - """ - Check if an API key can use a specific CDM. - - Args: - app: aiohttp application - api_key: API key to check - cdm_name: CDM name to check access for - - Returns: - True if allowed, False otherwise - """ - allowed_cdms = get_allowed_cdms(app, api_key) - - # Wildcard access - if "*" in allowed_cdms: - return True - - # Specific CDM access - return cdm_name in allowed_cdms diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 2704f9c..ca69476 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -207,6 +207,7 @@ def serialize_drm(drm_list) -> Optional[List[Dict[str, Any]]]: # Get PSSH - handle both Widevine and PlayReady if hasattr(drm, "_pssh") and drm._pssh: + pssh_obj = None try: pssh_obj = drm._pssh # Try to get base64 representation @@ -225,8 +226,24 @@ def serialize_drm(drm_list) -> Optional[List[Dict[str, Any]]]: # Check if it's already base64-like or an object repr if not pssh_str.startswith("<"): drm_info["pssh"] = pssh_str + except (ValueError, TypeError, KeyError): + # Some PSSH implementations can fail to parse/serialize; log and continue. + pssh_type = type(pssh_obj).__name__ if pssh_obj is not None else None + log.warning( + "Failed to extract/serialize PSSH for DRM type=%s pssh_type=%s", + drm_class, + pssh_type, + exc_info=True, + ) except Exception: - pass + # Don't silently swallow unexpected failures; make them visible and propagate. + pssh_type = type(pssh_obj).__name__ if pssh_obj is not None else None + log.exception( + "Unexpected error while extracting/serializing PSSH for DRM type=%s pssh_type=%s", + drm_class, + pssh_type, + ) + raise # Get KIDs if hasattr(drm, "kids") and drm.kids: diff --git a/unshackle/core/api/remote_handlers.py b/unshackle/core/api/remote_handlers.py deleted file mode 100644 index db60ee2..0000000 --- a/unshackle/core/api/remote_handlers.py +++ /dev/null @@ -1,2195 +0,0 @@ -"""API handlers for remote service functionality.""" - -import http.cookiejar -import inspect -import logging -import tempfile -import time -from pathlib import Path -from typing import Any, Dict, Optional - -import click -import yaml -from aiohttp import web - -from unshackle.commands.dl import dl -from unshackle.core.api.api_keys import can_use_cdm, get_api_key_from_request, get_default_cdm, is_premium_user -from unshackle.core.api.handlers import (serialize_audio_track, serialize_subtitle_track, serialize_title, - serialize_video_track, validate_service) -from unshackle.core.api.session_serializer import deserialize_session, serialize_session -from unshackle.core.config import config -from unshackle.core.credential import Credential -from unshackle.core.search_result import SearchResult -from unshackle.core.services import Services -from unshackle.core.titles import Episode -from unshackle.core.utils.click_types import ContextData -from unshackle.core.utils.collections import merge_dict - -log = logging.getLogger("api.remote") - -# Session expiry time in seconds (24 hours) -SESSION_EXPIRY_TIME = 86400 - - -class CDMProxy: - """ - Lightweight CDM proxy that holds CDM properties sent from client. - - This allows services to check CDM properties (like security_level) - without needing an actual CDM loaded on the server. - """ - - def __init__(self, cdm_info: Dict[str, Any]): - """ - Initialize CDM proxy from client-provided info. - - Args: - cdm_info: Dictionary with CDM properties (type, security_level, etc.) - """ - self.cdm_type = cdm_info.get("type", "widevine") - self.security_level = cdm_info.get("security_level", 3) - self.is_playready = self.cdm_type == "playready" - self.device_type = cdm_info.get("device_type") - self.is_remote = cdm_info.get("is_remote", False) - - def __repr__(self): - return f"CDMProxy(type={self.cdm_type}, L{self.security_level})" - - -def load_cookies_from_content(cookies_content: Optional[str]) -> Optional[http.cookiejar.MozillaCookieJar]: - """ - Load cookies from raw cookie file content. - - Args: - cookies_content: Raw content of a Netscape/Mozilla format cookie file - - Returns: - MozillaCookieJar object or None - """ - if not cookies_content: - return None - - # Write to temporary file - with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: - f.write(cookies_content) - temp_path = f.name - - try: - # Load using standard cookie jar - cookie_jar = http.cookiejar.MozillaCookieJar(temp_path) - cookie_jar.load(ignore_discard=True, ignore_expires=True) - return cookie_jar - finally: - # Clean up temp file - Path(temp_path).unlink(missing_ok=True) - - -def create_credential_from_dict(cred_data: Optional[Dict[str, str]]) -> Optional[Credential]: - """ - Create a Credential object from dictionary. - - Args: - cred_data: Dictionary with 'username' and 'password' keys - - Returns: - Credential object or None - """ - if not cred_data or "username" not in cred_data or "password" not in cred_data: - return None - - return Credential(username=cred_data["username"], password=cred_data["password"]) - - -def validate_session_expiry(session_data: Dict[str, Any]) -> Optional[str]: - """ - Validate if a session is expired. - - Args: - session_data: Session data with cached_at timestamp - - Returns: - Error code if session is expired, None if valid - """ - if not session_data: - return None - - cached_at = session_data.get("cached_at") - if not cached_at: - # No timestamp - assume valid (backward compatibility) - return None - - age = time.time() - cached_at - if age > SESSION_EXPIRY_TIME: - log.warning(f"Session expired (age: {age:.0f}s, limit: {SESSION_EXPIRY_TIME}s)") - return "SESSION_EXPIRED" - - # Warn if session is close to expiry (within 1 hour) - if age > (SESSION_EXPIRY_TIME - 3600): - remaining = SESSION_EXPIRY_TIME - age - log.info(f"Session expires soon (remaining: {remaining:.0f}s)") - - return None - - -def get_auth_from_request(data: Dict[str, Any], service_tag: str, profile: Optional[str] = None): - """ - Get authentication from request data or fallback to server config. - - Server is STATELESS - it never stores sessions. - Client sends pre-authenticated session with each request. - - Priority order: - 1. Pre-authenticated session from client (sent with request) - 2. Client-provided credentials/cookies in request - 3. Server-side credentials/cookies from config (fallback) - - Args: - data: Request data - service_tag: Service tag - profile: Profile name - - Returns: - Tuple of (cookies, credential, pre_authenticated_session, session_error) - where session_error is an error code if session is expired - """ - # First priority: Check for pre-authenticated session sent by client - pre_authenticated_session = data.get("pre_authenticated_session") - - if pre_authenticated_session: - log.info(f"Using client's pre-authenticated session for {service_tag}") - - # Validate session expiry - session_error = validate_session_expiry(pre_authenticated_session) - if session_error: - log.warning(f"Session validation failed: {session_error}") - return None, None, None, session_error - - # Return None, None to indicate we'll use the pre-authenticated session - return None, None, pre_authenticated_session, None - - # Second priority: Try to get from client request - cookies_content = data.get("cookies") - credential_data = data.get("credential") - - if cookies_content: - cookies = load_cookies_from_content(cookies_content) - else: - # Fallback to server-side cookies if not provided by client - cookies = dl.get_cookie_jar(service_tag, profile) - - if credential_data: - credential = create_credential_from_dict(credential_data) - else: - # Fallback to server-side credentials if not provided by client - credential = dl.get_credentials(service_tag, profile) - - return cookies, credential, None, None - - -async def remote_list_services(request: web.Request) -> web.Response: - """ - List all available services on this remote server. - --- - summary: List remote services - description: Get all available services that can be accessed remotely - responses: - '200': - description: List of available services - content: - application/json: - schema: - type: object - properties: - status: - type: string - example: success - services: - type: array - items: - type: object - properties: - tag: - type: string - aliases: - type: array - items: - type: string - geofence: - type: array - items: - type: string - help: - type: string - '500': - description: Server error - """ - try: - service_tags = Services.get_tags() - services_info = [] - - for tag in service_tags: - service_data = { - "tag": tag, - "aliases": [], - "geofence": [], - "help": None, - } - - try: - service_module = Services.load(tag) - - if hasattr(service_module, "ALIASES"): - service_data["aliases"] = list(service_module.ALIASES) - - if hasattr(service_module, "GEOFENCE"): - service_data["geofence"] = list(service_module.GEOFENCE) - - if service_module.__doc__: - service_data["help"] = service_module.__doc__.strip() - - except Exception as e: - log.warning(f"Could not load details for service {tag}: {e}") - - services_info.append(service_data) - - return web.json_response({"status": "success", "services": services_info}) - - except Exception: - log.exception("Error listing remote services") - return web.json_response({"status": "error", "message": "Internal server error while listing services"}, status=500) - - -async def remote_search(request: web.Request) -> web.Response: - """ - Search for content on a remote service. - --- - summary: Search remote service - description: Search for content using a remote service - parameters: - - name: service - in: path - required: true - schema: - type: string - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - query - properties: - query: - type: string - description: Search query - profile: - type: string - description: Profile to use for credentials - responses: - '200': - description: Search results - '400': - description: Invalid request - '500': - description: Server error - """ - service_tag = request.match_info.get("service") - - try: - data = await request.json() - except Exception: - return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) - - query = data.get("query") - if not query: - return web.json_response({"status": "error", "message": "Missing required parameter: query"}, status=400) - - normalized_service = validate_service(service_tag) - if not normalized_service: - return web.json_response( - {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, status=400 - ) - - try: - profile = data.get("profile") - - service_config_path = Services.get_path(normalized_service) / config.filenames.config - if service_config_path.exists(): - service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) - else: - service_config = {} - merge_dict(config.services.get(normalized_service), service_config) - - @click.command() - @click.pass_context - def dummy_service(ctx: click.Context) -> None: - pass - - # Handle proxy configuration - # Client MUST send resolved proxy with credentials (e.g., http://user:pass@host:port) - # Server does NOT resolve proxy providers - client must do that - proxy_param = data.get("proxy") - no_proxy = data.get("no_proxy", False) - - if proxy_param and not no_proxy: - import re - - # Validate that client sent a fully resolved proxy URL - if re.match(r"^https?://", proxy_param): - log.info("Using client-resolved proxy with credentials") - else: - # Reject unresolved proxy parameters - log.error(f"[SECURITY] Client sent unresolved proxy parameter: {proxy_param}") - return web.json_response({ - "status": "error", - "error_code": "INVALID_PROXY", - "message": f"Proxy must be a fully resolved URL (http://... or https://...). " - f"Cannot use proxy provider shortcuts like '{proxy_param}'. " - f"Please resolve the proxy on the client side before sending to server." - }, status=400) - - ctx = click.Context(dummy_service) - ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=[], profile=profile) - ctx.params = {"proxy": proxy_param, "no_proxy": no_proxy} - - service_module = Services.load(normalized_service) - - dummy_service.name = normalized_service - ctx.invoked_subcommand = normalized_service - - service_ctx = click.Context(dummy_service, parent=ctx) - service_ctx.obj = ctx.obj - - # Get service initialization parameters - service_init_params = inspect.signature(service_module.__init__).parameters - service_kwargs = {} - - # Extract defaults from click command - if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): - for param in service_module.cli.params: - if hasattr(param, "name") and param.name not in service_kwargs: - if hasattr(param, "default") and param.default is not None: - service_kwargs[param.name] = param.default - - # Add query parameter - if "query" in service_init_params: - service_kwargs["query"] = query - - # Filter to only valid parameters - filtered_kwargs = {k: v for k, v in service_kwargs.items() if k in service_init_params} - - service_instance = service_module(service_ctx, **filtered_kwargs) - - # Authenticate with client-provided or server-side auth - cookies, credential, pre_authenticated_session, session_error = get_auth_from_request(data, normalized_service, profile) - - # Check for session expiry - if session_error == "SESSION_EXPIRED": - return web.json_response({ - "status": "error", - "error_code": "SESSION_EXPIRED", - "message": f"Session expired for {normalized_service}. Please re-authenticate." - }, status=401) - - try: - if pre_authenticated_session: - # Use pre-authenticated session sent by client (server is stateless) - deserialize_session(pre_authenticated_session, service_instance.session) - else: - # Authenticate with credentials/cookies - if not cookies and not credential: - # No auth data available - tell client to authenticate - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication required for {normalized_service}. No credentials or session available." - }, status=401) - - service_instance.authenticate(cookies, credential) - except Exception as auth_error: - # Authentication failed - tell client to re-authenticate - log.warning(f"Authentication failed for {normalized_service}: {auth_error}") - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication failed for {normalized_service}. Please authenticate locally." - }, status=401) - - # Perform search - search_results = [] - if hasattr(service_instance, "search"): - for result in service_instance.search(): - if isinstance(result, SearchResult): - search_results.append( - { - "id": str(result.id_), - "title": result.title, - "description": result.description, - "label": result.label, - "url": result.url, - } - ) - - # Serialize session data - session_data = serialize_session(service_instance.session) - - return web.json_response({"status": "success", "results": search_results, "session": session_data}) - - except Exception: - log.exception("Error performing remote search") - return web.json_response({"status": "error", "message": "Internal server error while performing search"}, status=500) - - -async def remote_get_titles(request: web.Request) -> web.Response: - """ - Get titles from a remote service. - --- - summary: Get titles from remote service - description: Get available titles for content from a remote service - parameters: - - name: service - in: path - required: true - schema: - type: string - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - title - properties: - title: - type: string - description: Title identifier, URL, or any format accepted by the service - profile: - type: string - description: Profile to use for credentials - proxy: - type: string - description: Proxy region code (e.g., "ca", "us") or full proxy URL - uses server's proxy configuration - no_proxy: - type: boolean - description: Disable proxy usage - cookies: - type: string - description: Raw Netscape/Mozilla format cookie file content (optional - uses server cookies if not provided) - credential: - type: object - description: Credentials object with username and password (optional - uses server credentials if not provided) - properties: - username: - type: string - password: - type: string - responses: - '200': - description: Titles and session data - '400': - description: Invalid request - '500': - description: Server error - """ - service_tag = request.match_info.get("service") - - try: - data = await request.json() - except Exception: - return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) - - # Accept 'title', 'title_id', or 'url' for flexibility - title = data.get("title") or data.get("title_id") or data.get("url") - if not title: - return web.json_response( - { - "status": "error", - "message": "Missing required parameter: title (can be URL, ID, or any format accepted by the service)", - }, - status=400, - ) - - normalized_service = validate_service(service_tag) - if not normalized_service: - return web.json_response( - {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, status=400 - ) - - try: - profile = data.get("profile") - - service_config_path = Services.get_path(normalized_service) / config.filenames.config - if service_config_path.exists(): - service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) - else: - service_config = {} - merge_dict(config.services.get(normalized_service), service_config) - - @click.command() - @click.pass_context - def dummy_service(ctx: click.Context) -> None: - pass - - # Handle proxy configuration - # Client MUST send resolved proxy with credentials (e.g., http://user:pass@host:port) - # Server does NOT resolve proxy providers - client must do that - proxy_param = data.get("proxy") - no_proxy = data.get("no_proxy", False) - - if proxy_param and not no_proxy: - import re - - # Validate that client sent a fully resolved proxy URL - if re.match(r"^https?://", proxy_param): - log.info("Using client-resolved proxy with credentials") - else: - # Reject unresolved proxy parameters - log.error(f"[SECURITY] Client sent unresolved proxy parameter: {proxy_param}") - return web.json_response({ - "status": "error", - "error_code": "INVALID_PROXY", - "message": f"Proxy must be a fully resolved URL (http://... or https://...). " - f"Cannot use proxy provider shortcuts like '{proxy_param}'. " - f"Please resolve the proxy on the client side before sending to server." - }, status=400) - - ctx = click.Context(dummy_service) - ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=[], profile=profile) - ctx.params = {"proxy": proxy_param, "no_proxy": no_proxy} - - service_module = Services.load(normalized_service) - - dummy_service.name = normalized_service - dummy_service.params = [click.Argument([title], type=str)] - ctx.invoked_subcommand = normalized_service - - service_ctx = click.Context(dummy_service, parent=ctx) - service_ctx.obj = ctx.obj - - service_kwargs = {"title": title} - - # Add additional parameters from request data - for key, value in data.items(): - if key not in ["title", "title_id", "url", "profile", "proxy", "no_proxy"]: - service_kwargs[key] = value - - # Get service parameter info and click command defaults - service_init_params = inspect.signature(service_module.__init__).parameters - - # Extract default values from the click command - if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): - for param in service_module.cli.params: - if hasattr(param, "name") and param.name not in service_kwargs: - if hasattr(param, "default") and param.default is not None: - service_kwargs[param.name] = param.default - - # Handle required parameters - for param_name, param_info in service_init_params.items(): - if param_name not in service_kwargs and param_name not in ["self", "ctx"]: - if param_info.default is inspect.Parameter.empty: - if param_name == "meta_lang": - service_kwargs[param_name] = None - elif param_name == "movie": - service_kwargs[param_name] = False - - # Filter to only valid parameters - filtered_kwargs = {k: v for k, v in service_kwargs.items() if k in service_init_params} - - service_instance = service_module(service_ctx, **filtered_kwargs) - - # Authenticate with client-provided or server-side auth - cookies, credential, pre_authenticated_session, session_error = get_auth_from_request(data, normalized_service, profile) - - # Check for session expiry - if session_error == "SESSION_EXPIRED": - return web.json_response({ - "status": "error", - "error_code": "SESSION_EXPIRED", - "message": f"Session expired for {normalized_service}. Please re-authenticate." - }, status=401) - - try: - if pre_authenticated_session: - # Use pre-authenticated session sent by client (server is stateless) - deserialize_session(pre_authenticated_session, service_instance.session) - else: - # Authenticate with credentials/cookies - if not cookies and not credential: - # No auth data available - tell client to authenticate - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication required for {normalized_service}. No credentials or session available." - }, status=401) - - service_instance.authenticate(cookies, credential) - except Exception as auth_error: - # Authentication failed - tell client to re-authenticate - log.warning(f"Authentication failed for {normalized_service}: {auth_error}") - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication failed for {normalized_service}. Please authenticate locally." - }, status=401) - - # Get titles - titles = service_instance.get_titles() - - if hasattr(titles, "__iter__") and not isinstance(titles, str): - title_list = [serialize_title(t) for t in titles] - else: - title_list = [serialize_title(titles)] - - # Serialize session data - session_data = serialize_session(service_instance.session) - - # Include geofence info so client knows to activate VPN - geofence = [] - if hasattr(service_module, "GEOFENCE"): - geofence = list(service_module.GEOFENCE) - - return web.json_response({ - "status": "success", - "titles": title_list, - "session": session_data, - "geofence": geofence - }) - - except Exception: - log.exception("Error getting remote titles") - return web.json_response({"status": "error", "message": "Internal server error while getting titles"}, status=500) - - -async def remote_get_tracks(request: web.Request) -> web.Response: - """ - Get tracks from a remote service. - --- - summary: Get tracks from remote service - description: Get available tracks for a title from a remote service - parameters: - - name: service - in: path - required: true - schema: - type: string - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - title - properties: - title: - type: string - description: Title identifier, URL, or any format accepted by the service - wanted: - type: string - description: Specific episodes/seasons - profile: - type: string - description: Profile to use for credentials - proxy: - type: string - description: Proxy region code (e.g., "ca", "us") or full proxy URL - uses server's proxy configuration - no_proxy: - type: boolean - description: Disable proxy usage - cookies: - type: string - description: Raw Netscape/Mozilla format cookie file content (optional - uses server cookies if not provided) - credential: - type: object - description: Credentials object with username and password (optional - uses server credentials if not provided) - properties: - username: - type: string - password: - type: string - responses: - '200': - description: Tracks and session data - '400': - description: Invalid request - '500': - description: Server error - """ - service_tag = request.match_info.get("service") - - try: - data = await request.json() - except Exception: - return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) - - # Accept 'title', 'title_id', or 'url' for flexibility - title = data.get("title") or data.get("title_id") or data.get("url") - if not title: - return web.json_response( - { - "status": "error", - "message": "Missing required parameter: title (can be URL, ID, or any format accepted by the service)", - }, - status=400, - ) - - normalized_service = validate_service(service_tag) - if not normalized_service: - return web.json_response( - {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, status=400 - ) - - try: - profile = data.get("profile") - - service_config_path = Services.get_path(normalized_service) / config.filenames.config - if service_config_path.exists(): - service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) - else: - service_config = {} - merge_dict(config.services.get(normalized_service), service_config) - - @click.command() - @click.pass_context - def dummy_service(ctx: click.Context) -> None: - pass - - # Handle proxy configuration - # Client MUST send resolved proxy with credentials (e.g., http://user:pass@host:port) - # Server does NOT resolve proxy providers - client must do that - proxy_param = data.get("proxy") - no_proxy = data.get("no_proxy", False) - - if proxy_param and not no_proxy: - import re - - # Validate that client sent a fully resolved proxy URL - if re.match(r"^https?://", proxy_param): - log.info("Using client-resolved proxy with credentials") - else: - # Reject unresolved proxy parameters - log.error(f"[SECURITY] Client sent unresolved proxy parameter: {proxy_param}") - return web.json_response({ - "status": "error", - "error_code": "INVALID_PROXY", - "message": f"Proxy must be a fully resolved URL (http://... or https://...). " - f"Cannot use proxy provider shortcuts like '{proxy_param}'. " - f"Please resolve the proxy on the client side before sending to server." - }, status=400) - - # Create CDM proxy from client-provided info (default to L3 Widevine if not provided) - cdm_info = data.get("cdm_info") or {"type": "widevine", "security_level": 3} - cdm = CDMProxy(cdm_info) - - ctx = click.Context(dummy_service) - ctx.obj = ContextData(config=service_config, cdm=cdm, proxy_providers=[], profile=profile) - ctx.params = {"proxy": proxy_param, "no_proxy": no_proxy} - - service_module = Services.load(normalized_service) - - dummy_service.name = normalized_service - dummy_service.params = [click.Argument([title], type=str)] - ctx.invoked_subcommand = normalized_service - - service_ctx = click.Context(dummy_service, parent=ctx) - service_ctx.obj = ctx.obj - - service_kwargs = {"title": title} - - # Add additional parameters - for key, value in data.items(): - if key not in ["title", "title_id", "url", "profile", "wanted", "season", "episode", "proxy", "no_proxy", "cdm_info"]: - service_kwargs[key] = value - - # Get service parameters - service_init_params = inspect.signature(service_module.__init__).parameters - - # Extract defaults from click command - if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): - for param in service_module.cli.params: - if hasattr(param, "name") and param.name not in service_kwargs: - if hasattr(param, "default") and param.default is not None: - service_kwargs[param.name] = param.default - - # Handle required parameters - for param_name, param_info in service_init_params.items(): - if param_name not in service_kwargs and param_name not in ["self", "ctx"]: - if param_info.default is inspect.Parameter.empty: - if param_name == "meta_lang": - service_kwargs[param_name] = None - elif param_name == "movie": - service_kwargs[param_name] = False - - # Filter to valid parameters - filtered_kwargs = {k: v for k, v in service_kwargs.items() if k in service_init_params} - - service_instance = service_module(service_ctx, **filtered_kwargs) - - # Authenticate with client-provided or server-side auth - cookies, credential, pre_authenticated_session, session_error = get_auth_from_request(data, normalized_service, profile) - - # Check for session expiry - if session_error == "SESSION_EXPIRED": - return web.json_response({ - "status": "error", - "error_code": "SESSION_EXPIRED", - "message": f"Session expired for {normalized_service}. Please re-authenticate." - }, status=401) - - try: - if pre_authenticated_session: - # Use pre-authenticated session sent by client (server is stateless) - deserialize_session(pre_authenticated_session, service_instance.session) - else: - # Authenticate with credentials/cookies - if not cookies and not credential: - # No auth data available - tell client to authenticate - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication required for {normalized_service}. No credentials or session available." - }, status=401) - - service_instance.authenticate(cookies, credential) - except Exception as auth_error: - # Authentication failed - tell client to re-authenticate - log.warning(f"Authentication failed for {normalized_service}: {auth_error}") - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication failed for {normalized_service}. Please authenticate locally." - }, status=401) - - # Get titles - titles = service_instance.get_titles() - - wanted_param = data.get("wanted") - season = data.get("season") - episode = data.get("episode") - - if hasattr(titles, "__iter__") and not isinstance(titles, str): - titles_list = list(titles) - - wanted = None - if wanted_param: - from unshackle.core.utils.click_types import SeasonRange - - try: - season_range = SeasonRange() - wanted = season_range.parse_tokens(wanted_param) - except Exception as e: - return web.json_response( - {"status": "error", "message": f"Invalid wanted parameter: {e}"}, status=400 - ) - elif season is not None and episode is not None: - wanted = [f"{season}x{episode}"] - - if wanted: - matching_titles = [] - for title in titles_list: - if isinstance(title, Episode): - episode_key = f"{title.season}x{title.number}" - if episode_key in wanted: - matching_titles.append(title) - else: - matching_titles.append(title) - - if not matching_titles: - return web.json_response( - {"status": "error", "message": "No episodes found matching wanted criteria"}, status=404 - ) - - # Handle multiple episodes - if len(matching_titles) > 1 and all(isinstance(t, Episode) for t in matching_titles): - episodes_data = [] - failed_episodes = [] - - sorted_titles = sorted(matching_titles, key=lambda t: (t.season, t.number)) - - for title in sorted_titles: - try: - tracks = service_instance.get_tracks(title) - video_tracks = sorted(tracks.videos, key=lambda t: t.bitrate or 0, reverse=True) - audio_tracks = sorted(tracks.audio, key=lambda t: t.bitrate or 0, reverse=True) - - episode_data = { - "title": serialize_title(title), - "video": [serialize_video_track(t) for t in video_tracks], - "audio": [serialize_audio_track(t) for t in audio_tracks], - "subtitles": [serialize_subtitle_track(t) for t in tracks.subtitles], - } - episodes_data.append(episode_data) - except (SystemExit, Exception): - failed_episodes.append(f"S{title.season}E{title.number:02d}") - continue - - if episodes_data: - session_data = serialize_session(service_instance.session) - - # Include geofence info - geofence = [] - if hasattr(service_module, "GEOFENCE"): - geofence = list(service_module.GEOFENCE) - - response = { - "status": "success", - "episodes": episodes_data, - "session": session_data, - "geofence": geofence - } - if failed_episodes: - response["unavailable_episodes"] = failed_episodes - return web.json_response(response) - else: - return web.json_response( - { - "status": "error", - "message": f"No available episodes. Unavailable: {', '.join(failed_episodes)}", - }, - status=404, - ) - else: - first_title = matching_titles[0] - else: - first_title = titles_list[0] - else: - first_title = titles - - # Get tracks for single title - tracks = service_instance.get_tracks(first_title) - - video_tracks = sorted(tracks.videos, key=lambda t: t.bitrate or 0, reverse=True) - audio_tracks = sorted(tracks.audio, key=lambda t: t.bitrate or 0, reverse=True) - - # Serialize session data - session_data = serialize_session(service_instance.session) - - # Include geofence info - geofence = [] - if hasattr(service_module, "GEOFENCE"): - geofence = list(service_module.GEOFENCE) - - # Try to extract license URL from service (for remote licensing) - license_url = None - title_id = first_title.id if hasattr(first_title, "id") else str(first_title) - - # Check playback_data for license URL - if hasattr(service_instance, "playback_data") and title_id in service_instance.playback_data: - playback_data = service_instance.playback_data[title_id] - # DSNP pattern - if "drm" in playback_data and "licenseServerUrl" in playback_data.get("drm", {}): - license_url = playback_data["drm"]["licenseServerUrl"] - elif "stream" in playback_data and "drm" in playback_data["stream"]: - drm_info = playback_data["stream"]["drm"] - if isinstance(drm_info, dict) and "licenseServerUrl" in drm_info: - license_url = drm_info["licenseServerUrl"] - - # Check service config for license URL - if not license_url and hasattr(service_instance, "config"): - if "license_url" in service_instance.config: - license_url = service_instance.config["license_url"] - - response_data = { - "status": "success", - "title": serialize_title(first_title), - "video": [serialize_video_track(t, include_url=True) for t in video_tracks], - "audio": [serialize_audio_track(t, include_url=True) for t in audio_tracks], - "subtitles": [serialize_subtitle_track(t, include_url=True) for t in tracks.subtitles], - "session": session_data, - "geofence": geofence, - "license_url": license_url, - } - - return web.json_response(response_data) - - except Exception: - log.exception("Error getting remote tracks") - return web.json_response({"status": "error", "message": "Internal server error while getting tracks"}, status=500) - - -async def remote_get_manifest(request: web.Request) -> web.Response: - """ - Get manifest URL and session from a remote service. - - This endpoint returns the manifest URL and authenticated session, - allowing the client to fetch and parse the manifest locally. - --- - summary: Get manifest info from remote service - description: Get manifest URL and session for client-side parsing - parameters: - - name: service - in: path - required: true - schema: - type: string - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - title - properties: - title: - type: string - description: Title identifier - cdm_info: - type: object - description: Client CDM info (type, security_level) - responses: - '200': - description: Manifest info - """ - service_tag = request.match_info.get("service") - - try: - data = await request.json() - except Exception: - return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) - - title = data.get("title") or data.get("title_id") or data.get("url") - if not title: - return web.json_response( - {"status": "error", "message": "Missing required parameter: title"}, - status=400, - ) - - normalized_service = validate_service(service_tag) - if not normalized_service: - return web.json_response( - {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, status=400 - ) - - try: - profile = data.get("profile") - - service_config_path = Services.get_path(normalized_service) / config.filenames.config - if service_config_path.exists(): - service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) - else: - service_config = {} - merge_dict(config.services.get(normalized_service), service_config) - - @click.command() - @click.pass_context - def dummy_service(ctx: click.Context) -> None: - pass - - proxy_param = data.get("proxy") - no_proxy = data.get("no_proxy", False) - - if proxy_param and not no_proxy: - import re - if not re.match(r"^https?://", proxy_param): - return web.json_response({ - "status": "error", - "error_code": "INVALID_PROXY", - "message": "Proxy must be a fully resolved URL" - }, status=400) - - # Create CDM proxy from client-provided info - cdm_info = data.get("cdm_info") or {"type": "widevine", "security_level": 3} - cdm = CDMProxy(cdm_info) - - ctx = click.Context(dummy_service) - ctx.obj = ContextData(config=service_config, cdm=cdm, proxy_providers=[], profile=profile) - ctx.params = {"proxy": proxy_param, "no_proxy": no_proxy} - - service_module = Services.load(normalized_service) - - dummy_service.name = normalized_service - dummy_service.params = [click.Argument([title], type=str)] - ctx.invoked_subcommand = normalized_service - - service_ctx = click.Context(dummy_service, parent=ctx) - service_ctx.obj = ctx.obj - - service_kwargs = {"title": title} - - for key, value in data.items(): - if key not in ["title", "title_id", "url", "profile", "proxy", "no_proxy", "cdm_info"]: - service_kwargs[key] = value - - service_init_params = inspect.signature(service_module.__init__).parameters - - if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): - for param in service_module.cli.params: - if hasattr(param, "name") and param.name not in service_kwargs: - if hasattr(param, "default") and param.default is not None: - service_kwargs[param.name] = param.default - - for param_name, param_info in service_init_params.items(): - if param_name not in service_kwargs and param_name not in ["self", "ctx"]: - if param_info.default is inspect.Parameter.empty: - if param_name == "meta_lang": - service_kwargs[param_name] = None - elif param_name == "movie": - service_kwargs[param_name] = False - - filtered_kwargs = {k: v for k, v in service_kwargs.items() if k in service_init_params} - - service_instance = service_module(service_ctx, **filtered_kwargs) - - # Authenticate - cookies, credential, pre_authenticated_session, session_error = get_auth_from_request(data, normalized_service, profile) - - if session_error == "SESSION_EXPIRED": - return web.json_response({ - "status": "error", - "error_code": "SESSION_EXPIRED", - "message": f"Session expired for {normalized_service}. Please re-authenticate." - }, status=401) - - try: - if pre_authenticated_session: - deserialize_session(pre_authenticated_session, service_instance.session) - else: - if not cookies and not credential: - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication required for {normalized_service}" - }, status=401) - service_instance.authenticate(cookies, credential) - except Exception as e: - log.error(f"Authentication failed: {e}") - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication failed for {normalized_service}" - }, status=401) - - # Get titles - titles = service_instance.get_titles() - - if hasattr(titles, "__iter__") and not isinstance(titles, str): - titles_list = list(titles) - else: - titles_list = [titles] if titles else [] - - if not titles_list: - return web.json_response({"status": "error", "message": "No titles found"}, status=404) - - # Handle episode filtering (wanted parameter) - wanted_param = data.get("wanted") - season = data.get("season") - episode = data.get("episode") - target_title = None - - if wanted_param or (season is not None and episode is not None): - # Filter to matching episode - wanted = None - if wanted_param: - from unshackle.core.utils.click_types import SeasonRange - try: - season_range = SeasonRange() - wanted = season_range.parse_tokens(wanted_param) - except Exception: - pass - elif season is not None and episode is not None: - wanted = [f"{season}x{episode}"] - - if wanted: - for t in titles_list: - if isinstance(t, Episode): - episode_key = f"{t.season}x{t.number}" - if episode_key in wanted: - target_title = t - break - - if not target_title: - target_title = titles_list[0] - - # Now we need to get the manifest URL - # This is service-specific, so we call get_tracks but extract manifest info - - # Call get_tracks to populate playback_data - try: - _ = service_instance.get_tracks(target_title) - except Exception as e: - log.warning(f"get_tracks failed, trying to extract manifest anyway: {e}") - - # Extract manifest URL from service's playback_data - manifest_url = None - manifest_type = "hls" # Default - playback_data = {} - - # Check for playback_data (DSNP, HMAX, etc.) - if hasattr(service_instance, "playback_data"): - title_id = target_title.id if hasattr(target_title, "id") else str(target_title) - if title_id in service_instance.playback_data: - playback_data = service_instance.playback_data[title_id] - - # Try to extract manifest URL from common patterns - # Pattern 1: DSNP style - stream.sources[0].complete.url - if "stream" in playback_data and "sources" in playback_data["stream"]: - sources = playback_data["stream"]["sources"] - if sources and "complete" in sources[0]: - manifest_url = sources[0]["complete"].get("url") - - # Pattern 2: Direct manifest_url field - if not manifest_url and "manifest_url" in playback_data: - manifest_url = playback_data["manifest_url"] - - # Pattern 3: url field at top level - if not manifest_url and "url" in playback_data: - manifest_url = playback_data["url"] - - # Check for manifest attribute on service - if not manifest_url and hasattr(service_instance, "manifest"): - manifest_url = service_instance.manifest - - # Check for manifest_url attribute on service - if not manifest_url and hasattr(service_instance, "manifest_url"): - manifest_url = service_instance.manifest_url - - # Detect manifest type from URL - if manifest_url: - if manifest_url.endswith(".mpd") or "dash" in manifest_url.lower(): - manifest_type = "dash" - elif manifest_url.endswith(".m3u8") or manifest_url.endswith(".m3u"): - manifest_type = "hls" - - # Serialize session - session_data = serialize_session(service_instance.session) - - # Serialize title info - title_info = serialize_title(target_title) - - response_data = { - "status": "success", - "title": title_info, - "manifest_url": manifest_url, - "manifest_type": manifest_type, - "playback_data": playback_data, - "session": session_data, - } - - return web.json_response(response_data) - - except Exception: - log.exception("Error getting remote manifest") - return web.json_response({"status": "error", "message": "Internal server error while getting manifest"}, status=500) - - -async def remote_get_chapters(request: web.Request) -> web.Response: - """ - Get chapters from a remote service. - --- - summary: Get chapters from remote service - description: Get available chapters for a title from a remote service - parameters: - - name: service - in: path - required: true - schema: - type: string - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - title - properties: - title: - type: string - description: Title identifier, URL, or any format accepted by the service - profile: - type: string - description: Profile to use for credentials - proxy: - type: string - description: Proxy region code (e.g., "ca", "us") or full proxy URL - uses server's proxy configuration - no_proxy: - type: boolean - description: Disable proxy usage - cookies: - type: string - description: Raw Netscape/Mozilla format cookie file content (optional - uses server cookies if not provided) - credential: - type: object - description: Credentials object with username and password (optional - uses server credentials if not provided) - properties: - username: - type: string - password: - type: string - responses: - '200': - description: Chapters and session data - '400': - description: Invalid request - '500': - description: Server error - """ - service_tag = request.match_info.get("service") - - try: - data = await request.json() - except Exception: - return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) - - # Accept 'title', 'title_id', or 'url' for flexibility - title = data.get("title") or data.get("title_id") or data.get("url") - if not title: - return web.json_response( - { - "status": "error", - "message": "Missing required parameter: title (can be URL, ID, or any format accepted by the service)", - }, - status=400, - ) - - normalized_service = validate_service(service_tag) - if not normalized_service: - return web.json_response( - {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, status=400 - ) - - try: - profile = data.get("profile") - - service_config_path = Services.get_path(normalized_service) / config.filenames.config - if service_config_path.exists(): - service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) - else: - service_config = {} - merge_dict(config.services.get(normalized_service), service_config) - - @click.command() - @click.pass_context - def dummy_service(ctx: click.Context) -> None: - pass - - # Handle proxy configuration - # Client MUST send resolved proxy with credentials (e.g., http://user:pass@host:port) - # Server does NOT resolve proxy providers - client must do that - proxy_param = data.get("proxy") - no_proxy = data.get("no_proxy", False) - - if proxy_param and not no_proxy: - import re - - # Validate that client sent a fully resolved proxy URL - if re.match(r"^https?://", proxy_param): - log.info("Using client-resolved proxy with credentials") - else: - # Reject unresolved proxy parameters - log.error(f"[SECURITY] Client sent unresolved proxy parameter: {proxy_param}") - return web.json_response({ - "status": "error", - "error_code": "INVALID_PROXY", - "message": f"Proxy must be a fully resolved URL (http://... or https://...). " - f"Cannot use proxy provider shortcuts like '{proxy_param}'. " - f"Please resolve the proxy on the client side before sending to server." - }, status=400) - - ctx = click.Context(dummy_service) - ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=[], profile=profile) - ctx.params = {"proxy": proxy_param, "no_proxy": no_proxy} - - service_module = Services.load(normalized_service) - - dummy_service.name = normalized_service - dummy_service.params = [click.Argument([title], type=str)] - ctx.invoked_subcommand = normalized_service - - service_ctx = click.Context(dummy_service, parent=ctx) - service_ctx.obj = ctx.obj - - service_kwargs = {"title": title} - - # Add additional parameters - for key, value in data.items(): - if key not in ["title", "title_id", "url", "profile", "proxy", "no_proxy"]: - service_kwargs[key] = value - - # Get service parameters - service_init_params = inspect.signature(service_module.__init__).parameters - - # Extract defaults - if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): - for param in service_module.cli.params: - if hasattr(param, "name") and param.name not in service_kwargs: - if hasattr(param, "default") and param.default is not None: - service_kwargs[param.name] = param.default - - # Handle required parameters - for param_name, param_info in service_init_params.items(): - if param_name not in service_kwargs and param_name not in ["self", "ctx"]: - if param_info.default is inspect.Parameter.empty: - if param_name == "meta_lang": - service_kwargs[param_name] = None - elif param_name == "movie": - service_kwargs[param_name] = False - - # Filter to valid parameters - filtered_kwargs = {k: v for k, v in service_kwargs.items() if k in service_init_params} - - service_instance = service_module(service_ctx, **filtered_kwargs) - - # Authenticate with client-provided or server-side auth - cookies, credential, pre_authenticated_session, session_error = get_auth_from_request(data, normalized_service, profile) - - # Check for session expiry - if session_error == "SESSION_EXPIRED": - return web.json_response({ - "status": "error", - "error_code": "SESSION_EXPIRED", - "message": f"Session expired for {normalized_service}. Please re-authenticate." - }, status=401) - - try: - if pre_authenticated_session: - # Use pre-authenticated session sent by client (server is stateless) - deserialize_session(pre_authenticated_session, service_instance.session) - else: - # Authenticate with credentials/cookies - if not cookies and not credential: - # No auth data available - tell client to authenticate - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication required for {normalized_service}. No credentials or session available." - }, status=401) - - service_instance.authenticate(cookies, credential) - except Exception as auth_error: - # Authentication failed - tell client to re-authenticate - log.warning(f"Authentication failed for {normalized_service}: {auth_error}") - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication failed for {normalized_service}. Please authenticate locally." - }, status=401) - - # Get titles - titles = service_instance.get_titles() - - if hasattr(titles, "__iter__") and not isinstance(titles, str): - first_title = list(titles)[0] - else: - first_title = titles - - # Get chapters if service supports it - chapters_data = [] - if hasattr(service_instance, "get_chapters"): - chapters = service_instance.get_chapters(first_title) - if chapters: - for chapter in chapters: - chapters_data.append( - { - "timestamp": chapter.timestamp, - "name": chapter.name if hasattr(chapter, "name") else None, - } - ) - - # Serialize session data - session_data = serialize_session(service_instance.session) - - return web.json_response({"status": "success", "chapters": chapters_data, "session": session_data}) - - except Exception: - log.exception("Error getting remote chapters") - return web.json_response({"status": "error", "message": "Internal server error while getting chapters"}, status=500) - - -async def remote_get_license(request: web.Request) -> web.Response: - """ - Get DRM license from a remote service using client's CDM. - - The server does NOT need a CDM - it just facilitates the license request - using the client's pre-authenticated session. The client decrypts using - their own CDM. - --- - summary: Get DRM license from remote service - description: Request license acquisition using client session (server does not need CDM) - parameters: - - name: service - in: path - required: true - schema: - type: string - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - title - - track_id - - challenge - properties: - title: - type: string - description: Title identifier - track_id: - type: string - description: Track ID for license - challenge: - type: string - description: Base64-encoded license challenge from client's CDM - session: - type: integer - description: CDM session ID - profile: - type: string - description: Profile to use - pre_authenticated_session: - type: object - description: Client's pre-authenticated session - responses: - '200': - description: License response - content: - application/json: - schema: - type: object - properties: - status: - type: string - example: success - license: - type: string - description: Base64-encoded license response - session: - type: object - description: Updated session data - '400': - description: Invalid request - '401': - description: Authentication required - '500': - description: Server error - """ - service_tag = request.match_info.get("service") - - try: - data = await request.json() - except Exception: - return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) - - # Validate required parameters - title = data.get("title") - track_id = data.get("track_id") - challenge = data.get("challenge") - - if not all([title, track_id, challenge]): - return web.json_response( - { - "status": "error", - "message": "Missing required parameters: title, track_id, challenge" - }, - status=400 - ) - - normalized_service = validate_service(service_tag) - if not normalized_service: - return web.json_response( - {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, - status=400 - ) - - try: - profile = data.get("profile") - - service_config_path = Services.get_path(normalized_service) / config.filenames.config - if service_config_path.exists(): - service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) - else: - service_config = {} - merge_dict(config.services.get(normalized_service), service_config) - - @click.command() - @click.pass_context - def dummy_service(ctx: click.Context) -> None: - pass - - # Handle proxy configuration - # Client MUST send resolved proxy with credentials (e.g., http://user:pass@host:port) - # Server does NOT resolve proxy providers - client must do that - proxy_param = data.get("proxy") - no_proxy = data.get("no_proxy", False) - - if proxy_param and not no_proxy: - import re - - # Validate that client sent a fully resolved proxy URL - if re.match(r"^https?://", proxy_param): - log.info("Using client-resolved proxy with credentials") - else: - # Reject unresolved proxy parameters - log.error(f"[SECURITY] Client sent unresolved proxy parameter: {proxy_param}") - return web.json_response({ - "status": "error", - "error_code": "INVALID_PROXY", - "message": f"Proxy must be a fully resolved URL (http://... or https://...). " - f"Cannot use proxy provider shortcuts like '{proxy_param}'. " - f"Please resolve the proxy on the client side before sending to server." - }, status=400) - - ctx = click.Context(dummy_service) - ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=[], profile=profile) - ctx.params = {"proxy": proxy_param, "no_proxy": no_proxy} - - service_module = Services.load(normalized_service) - - dummy_service.name = normalized_service - ctx.invoked_subcommand = normalized_service - - service_ctx = click.Context(dummy_service, parent=ctx) - service_ctx.obj = ctx.obj - - service_kwargs = {"title": title} - - # Add additional parameters - for key, value in data.items(): - if key not in ["title", "track_id", "challenge", "session", "profile", "proxy", "no_proxy", "pre_authenticated_session", "credential", "cookies"]: - service_kwargs[key] = value - - # Get service parameters - service_init_params = inspect.signature(service_module.__init__).parameters - - # Extract defaults - if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): - for param in service_module.cli.params: - if hasattr(param, "name") and param.name not in service_kwargs: - if hasattr(param, "default") and param.default is not None: - service_kwargs[param.name] = param.default - - # Handle required parameters - for param_name, param_info in service_init_params.items(): - if param_name not in service_kwargs and param_name not in ["self", "ctx"]: - if param_info.default is inspect.Parameter.empty: - if param_name == "meta_lang": - service_kwargs[param_name] = None - elif param_name == "movie": - service_kwargs[param_name] = False - - # Filter to valid parameters - filtered_kwargs = {k: v for k, v in service_kwargs.items() if k in service_init_params} - - service_instance = service_module(service_ctx, **filtered_kwargs) - - # Authenticate with client-provided or server-side auth - cookies, credential, pre_authenticated_session, session_error = get_auth_from_request(data, normalized_service, profile) - - # Check for session expiry - if session_error == "SESSION_EXPIRED": - return web.json_response({ - "status": "error", - "error_code": "SESSION_EXPIRED", - "message": f"Session expired for {normalized_service}. Please re-authenticate." - }, status=401) - - try: - if pre_authenticated_session: - # Use pre-authenticated session sent by client (server is stateless) - deserialize_session(pre_authenticated_session, service_instance.session) - else: - # Authenticate with credentials/cookies - if not cookies and not credential: - # No auth data available - tell client to authenticate - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication required for {normalized_service}. No credentials or session available." - }, status=401) - - service_instance.authenticate(cookies, credential) - except Exception as auth_error: - # Authentication failed - tell client to re-authenticate - log.warning(f"Authentication failed for {normalized_service}: {auth_error}") - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication failed for {normalized_service}. Please authenticate locally." - }, status=401) - - # Get titles to find the correct one - titles = service_instance.get_titles() - if hasattr(titles, "__iter__") and not isinstance(titles, str): - first_title = list(titles)[0] - else: - first_title = titles - - # Get tracks to find license URL - tracks = service_instance.get_tracks(first_title) - - # Find the track with the matching ID - target_track = None - for track in tracks.videos + tracks.audio: - if str(track.id) == str(track_id) or track.id == track_id: - target_track = track - break - - if not target_track: - return web.json_response({ - "status": "error", - "message": f"Track {track_id} not found" - }, status=404) - - # Get license URL and headers from track - if not hasattr(target_track, "drm") or not target_track.drm: - return web.json_response({ - "status": "error", - "message": f"Track {track_id} is not DRM-protected" - }, status=400) - - # Extract license information - license_url = None - license_headers = {} - - # Try to get license URL from DRM info - for drm_info in target_track.drm: - if hasattr(drm_info, "license_url"): - license_url = drm_info.license_url - if hasattr(drm_info, "license_headers"): - license_headers = drm_info.license_headers or {} - break - - if not license_url: - return web.json_response({ - "status": "error", - "message": "No license URL found for track" - }, status=400) - - # Make license request using service session - import base64 - challenge_data = base64.b64decode(challenge) - - license_response = service_instance.session.post( - license_url, - data=challenge_data, - headers=license_headers - ) - - if license_response.status_code != 200: - return web.json_response({ - "status": "error", - "message": f"License request failed: {license_response.status_code}" - }, status=500) - - # Return base64-encoded license - license_b64 = base64.b64encode(license_response.content).decode("utf-8") - - # Serialize session data - session_data = serialize_session(service_instance.session) - - return web.json_response({ - "status": "success", - "license": license_b64, - "session": session_data - }) - - except Exception: - log.exception("Error getting remote license") - return web.json_response({"status": "error", "message": "Internal server error while getting license"}, status=500) - - -async def remote_decrypt(request: web.Request) -> web.Response: - """ - Decrypt DRM content using server's CDM (premium users only). - - This endpoint is for premium API key holders who can use the server's - CDM infrastructure. Regular users must use their own CDM with the - license endpoint. - - --- - summary: Decrypt DRM content using server CDM - description: Use server's CDM to decrypt content (premium tier only) - parameters: - - name: service - in: path - required: true - schema: - type: string - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - title - - track_id - - pssh - properties: - title: - type: string - description: Title identifier - track_id: - type: string - description: Track ID for decryption - pssh: - type: string - description: Base64-encoded PSSH box - cdm: - type: string - description: Specific CDM to use (optional, uses default if not specified) - license_url: - type: string - description: License server URL (optional, extracted from track if not provided) - profile: - type: string - description: Profile to use - pre_authenticated_session: - type: object - description: Client's pre-authenticated session - responses: - '200': - description: Decryption keys - content: - application/json: - schema: - type: object - properties: - status: - type: string - example: success - keys: - type: array - items: - type: object - properties: - kid: - type: string - key: - type: string - type: - type: string - session: - type: object - description: Updated session data - '400': - description: Invalid request - '401': - description: Authentication required - '403': - description: Not authorized for premium features - '500': - description: Server error - """ - service_tag = request.match_info.get("service") - - # Check if user is premium - api_key = get_api_key_from_request(request) - if not api_key: - return web.json_response({ - "status": "error", - "error_code": "NO_API_KEY", - "message": "API key required" - }, status=401) - - if not is_premium_user(request.app, api_key): - return web.json_response({ - "status": "error", - "error_code": "PREMIUM_REQUIRED", - "message": "This endpoint requires a premium API key. Use /api/remote/{service}/license with your own CDM instead." - }, status=403) - - try: - data = await request.json() - except Exception: - return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) - - # Validate required parameters - title = data.get("title") - track_id = data.get("track_id") - pssh = data.get("pssh") - - if not all([title, track_id, pssh]): - return web.json_response( - { - "status": "error", - "message": "Missing required parameters: title, track_id, pssh" - }, - status=400 - ) - - # Determine which CDM to use - requested_cdm = data.get("cdm") - if not requested_cdm: - # Use default CDM for this API key - requested_cdm = get_default_cdm(request.app, api_key) - - if not requested_cdm: - return web.json_response({ - "status": "error", - "message": "No CDM specified and no default CDM configured for your API key" - }, status=400) - - # Check if user can use this CDM - if not can_use_cdm(request.app, api_key, requested_cdm): - return web.json_response({ - "status": "error", - "error_code": "CDM_NOT_ALLOWED", - "message": f"Your API key is not authorized to use CDM: {requested_cdm}" - }, status=403) - - normalized_service = validate_service(service_tag) - if not normalized_service: - return web.json_response( - {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, - status=400 - ) - - try: - from pywidevine.cdm import Cdm as WidevineCdm - from pywidevine.device import Device - - # Load the requested CDM - log.info(f"Premium user using server CDM: {requested_cdm}") - - # Get CDM device path - cdm_device_path = None - if requested_cdm.endswith(".wvd"): - # Direct path to WVD file - cdm_device_path = Path(requested_cdm) - else: - # Look in configured CDM directory - cdm_dir = config.directories.wvds - potential_path = cdm_dir / f"{requested_cdm}.wvd" - if potential_path.exists(): - cdm_device_path = potential_path - - if not cdm_device_path or not cdm_device_path.exists(): - return web.json_response({ - "status": "error", - "message": f"CDM device not found: {requested_cdm}" - }, status=404) - - # Initialize CDM - device = Device.load(cdm_device_path) - cdm = WidevineCdm.from_device(device) - - # Open CDM session - session_id = cdm.open() - - # Parse PSSH - import base64 - pssh_data = base64.b64decode(pssh) - - # Set service certificate if needed (some services require it) - # This would be service-specific - - # Get challenge - challenge = cdm.get_license_challenge(session_id, pssh_data) - - # Get license URL - license_url = data.get("license_url") - - # If no license URL provided, get it from track - if not license_url: - profile = data.get("profile") - - service_config_path = Services.get_path(normalized_service) / config.filenames.config - if service_config_path.exists(): - service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) - else: - service_config = {} - merge_dict(config.services.get(normalized_service), service_config) - - @click.command() - @click.pass_context - def dummy_service(ctx: click.Context) -> None: - pass - - # Handle proxy configuration - # Client MUST send resolved proxy with credentials - # Server does NOT resolve proxy providers - client must do that - proxy_param = data.get("proxy") - no_proxy = data.get("no_proxy", False) - - if proxy_param and not no_proxy: - import re - - # Validate that client sent a fully resolved proxy URL - if re.match(r"^https?://", proxy_param): - log.info("Using client-resolved proxy with credentials") - else: - # Reject unresolved proxy parameters - log.error(f"[SECURITY] Client sent unresolved proxy parameter: {proxy_param}") - cdm.close(session_id) - return web.json_response({ - "status": "error", - "error_code": "INVALID_PROXY", - "message": f"Proxy must be a fully resolved URL (http://... or https://...). " - f"Cannot use proxy provider shortcuts like '{proxy_param}'. " - f"Please resolve the proxy on the client side before sending to server." - }, status=400) - - ctx = click.Context(dummy_service) - ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=[], profile=profile) - ctx.params = {"proxy": proxy_param, "no_proxy": no_proxy} - - service_module = Services.load(normalized_service) - dummy_service.name = normalized_service - ctx.invoked_subcommand = normalized_service - - service_ctx = click.Context(dummy_service, parent=ctx) - service_ctx.obj = ctx.obj - - service_kwargs = {"title": title} - - # Get service parameters - service_init_params = inspect.signature(service_module.__init__).parameters - - # Extract defaults - if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): - for param in service_module.cli.params: - if hasattr(param, "name") and param.name not in service_kwargs: - if hasattr(param, "default") and param.default is not None: - service_kwargs[param.name] = param.default - - # Handle required parameters - for param_name, param_info in service_init_params.items(): - if param_name not in service_kwargs and param_name not in ["self", "ctx"]: - if param_info.default is inspect.Parameter.empty: - if param_name == "meta_lang": - service_kwargs[param_name] = None - elif param_name == "movie": - service_kwargs[param_name] = False - - # Filter to valid parameters - filtered_kwargs = {k: v for k, v in service_kwargs.items() if k in service_init_params} - - service_instance = service_module(service_ctx, **filtered_kwargs) - - # Authenticate - cookies, credential, pre_authenticated_session, session_error = get_auth_from_request(data, normalized_service, profile) - - if session_error == "SESSION_EXPIRED": - cdm.close(session_id) - return web.json_response({ - "status": "error", - "error_code": "SESSION_EXPIRED", - "message": f"Session expired for {normalized_service}. Please re-authenticate." - }, status=401) - - try: - if pre_authenticated_session: - deserialize_session(pre_authenticated_session, service_instance.session) - else: - if not cookies and not credential: - cdm.close(session_id) - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication required for {normalized_service}." - }, status=401) - service_instance.authenticate(cookies, credential) - except Exception as auth_error: - cdm.close(session_id) - log.warning(f"Authentication failed for {normalized_service}: {auth_error}") - return web.json_response({ - "status": "error", - "error_code": "AUTH_REQUIRED", - "message": f"Authentication failed for {normalized_service}.", - "details": str(auth_error) - }, status=401) - - # Get titles and tracks to find license URL - titles = service_instance.get_titles() - if hasattr(titles, "__iter__") and not isinstance(titles, str): - first_title = list(titles)[0] - else: - first_title = titles - - tracks = service_instance.get_tracks(first_title) - - # Find the track - target_track = None - for track in tracks.videos + tracks.audio: - if str(track.id) == str(track_id) or track.id == track_id: - target_track = track - break - - if not target_track: - cdm.close(session_id) - return web.json_response({ - "status": "error", - "message": f"Track {track_id} not found" - }, status=404) - - if not hasattr(target_track, "drm") or not target_track.drm: - cdm.close(session_id) - return web.json_response({ - "status": "error", - "message": f"Track {track_id} is not DRM-protected" - }, status=400) - - # Extract license URL - license_headers = {} - for drm_info in target_track.drm: - if hasattr(drm_info, "license_url"): - license_url = drm_info.license_url - if hasattr(drm_info, "license_headers"): - license_headers = drm_info.license_headers or {} - break - - if not license_url: - cdm.close(session_id) - return web.json_response({ - "status": "error", - "message": "No license URL found for track" - }, status=400) - - # Make license request - license_response = service_instance.session.post( - license_url, - data=challenge, - headers=license_headers - ) - - if license_response.status_code != 200: - cdm.close(session_id) - return web.json_response({ - "status": "error", - "message": f"License request failed: {license_response.status_code}" - }, status=500) - - # Parse license - cdm.parse_license(session_id, license_response.content) - - # Get keys - keys = [] - for key in cdm.get_keys(session_id): - if key.type == "CONTENT": - keys.append({ - "kid": key.kid.hex(), - "key": key.key.hex(), - "type": key.type - }) - - # Close CDM session - cdm.close(session_id) - - # Serialize session - session_data = serialize_session(service_instance.session) - - return web.json_response({ - "status": "success", - "keys": keys, - "session": session_data, - "cdm_used": requested_cdm - }) - - else: - # License URL provided directly - # Make license request (need to provide session for this) - cdm.close(session_id) - return web.json_response({ - "status": "error", - "message": "Direct license URL not yet supported, omit license_url to auto-detect from service" - }, status=400) - - except Exception: - log.exception("Error in server-side decryption") - return web.json_response({"status": "error", "message": "Internal server error during decryption"}, status=500) diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 164c7c8..9adf8be 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -8,9 +8,6 @@ from unshackle.core import __version__ from unshackle.core.api.errors import APIError, APIErrorCode, build_error_response, handle_api_exception from unshackle.core.api.handlers import (cancel_download_job_handler, download_handler, get_download_job_handler, list_download_jobs_handler, list_titles_handler, list_tracks_handler) -from unshackle.core.api.remote_handlers import (remote_decrypt, remote_get_chapters, remote_get_license, - remote_get_manifest, remote_get_titles, remote_get_tracks, - remote_list_services, remote_search) from unshackle.core.services import Services from unshackle.core.update_checker import UpdateChecker @@ -733,16 +730,6 @@ def setup_routes(app: web.Application) -> None: app.router.add_get("/api/download/jobs/{job_id}", download_job_detail) app.router.add_delete("/api/download/jobs/{job_id}", cancel_download_job) - # Remote service endpoints - app.router.add_get("/api/remote/services", remote_list_services) - app.router.add_post("/api/remote/{service}/search", remote_search) - app.router.add_post("/api/remote/{service}/titles", remote_get_titles) - app.router.add_post("/api/remote/{service}/tracks", remote_get_tracks) - app.router.add_post("/api/remote/{service}/manifest", remote_get_manifest) - app.router.add_post("/api/remote/{service}/chapters", remote_get_chapters) - app.router.add_post("/api/remote/{service}/license", remote_get_license) - app.router.add_post("/api/remote/{service}/decrypt", remote_decrypt) - def setup_swagger(app: web.Application) -> None: """Setup Swagger UI documentation.""" @@ -767,14 +754,5 @@ def setup_swagger(app: web.Application) -> None: web.get("/api/download/jobs", download_jobs), web.get("/api/download/jobs/{job_id}", download_job_detail), web.delete("/api/download/jobs/{job_id}", cancel_download_job), - # Remote service routes - web.get("/api/remote/services", remote_list_services), - web.post("/api/remote/{service}/search", remote_search), - web.post("/api/remote/{service}/titles", remote_get_titles), - web.post("/api/remote/{service}/tracks", remote_get_tracks), - web.post("/api/remote/{service}/manifest", remote_get_manifest), - web.post("/api/remote/{service}/chapters", remote_get_chapters), - web.post("/api/remote/{service}/license", remote_get_license), - web.post("/api/remote/{service}/decrypt", remote_decrypt), ] ) diff --git a/unshackle/core/api/session_serializer.py b/unshackle/core/api/session_serializer.py deleted file mode 100644 index 733b179..0000000 --- a/unshackle/core/api/session_serializer.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Session serialization helpers for remote services.""" - -from http.cookiejar import CookieJar -from typing import Any, Dict, Optional - -import requests - -from unshackle.core.credential import Credential - - -def serialize_session(session: requests.Session) -> Dict[str, Any]: - """ - Serialize a requests.Session into a JSON-serializable dictionary. - - Extracts cookies, headers, and other session data that can be - transferred to a remote client for downloading. - - Args: - session: The requests.Session to serialize - - Returns: - Dictionary containing serialized session data - """ - session_data = { - "cookies": {}, - "headers": {}, - "proxies": session.proxies.copy() if session.proxies else {}, - } - - # Serialize cookies - if session.cookies: - for cookie in session.cookies: - session_data["cookies"][cookie.name] = { - "value": cookie.value, - "domain": cookie.domain, - "path": cookie.path, - "secure": cookie.secure, - "expires": cookie.expires, - } - - # Serialize headers (exclude proxy-authorization for security) - if session.headers: - for key, value in session.headers.items(): - # Skip proxy-related headers as they're server-specific - if key.lower() not in ["proxy-authorization"]: - session_data["headers"][key] = value - - return session_data - - -def deserialize_session( - session_data: Dict[str, Any], target_session: Optional[requests.Session] = None -) -> requests.Session: - """ - Deserialize session data into a requests.Session. - - Applies cookies, headers, and other session data from a remote server - to a local session for downloading. - - Args: - session_data: Dictionary containing serialized session data - target_session: Optional existing session to update (creates new if None) - - Returns: - requests.Session with applied session data - """ - if target_session is None: - target_session = requests.Session() - - # Apply cookies - if "cookies" in session_data: - for cookie_name, cookie_data in session_data["cookies"].items(): - target_session.cookies.set( - name=cookie_name, - value=cookie_data["value"], - domain=cookie_data.get("domain"), - path=cookie_data.get("path", "/"), - secure=cookie_data.get("secure", False), - expires=cookie_data.get("expires"), - ) - - # Apply headers - if "headers" in session_data: - target_session.headers.update(session_data["headers"]) - - # Note: We don't apply proxies from remote as the local client - # should use its own proxy configuration - - return target_session - - -def extract_session_tokens(session: requests.Session) -> Dict[str, Any]: - """ - Extract authentication tokens and similar data from a session. - - Looks for common authentication patterns like Bearer tokens, - API keys in headers, etc. - - Args: - session: The requests.Session to extract tokens from - - Returns: - Dictionary containing extracted tokens - """ - tokens = {} - - # Check for Authorization header - if "Authorization" in session.headers: - tokens["authorization"] = session.headers["Authorization"] - - # Check for common API key headers - for key in ["X-API-Key", "Api-Key", "X-Auth-Token"]: - if key in session.headers: - tokens[key.lower().replace("-", "_")] = session.headers[key] - - return tokens - - -def apply_session_tokens(tokens: Dict[str, Any], target_session: requests.Session) -> None: - """ - Apply authentication tokens to a session. - - Args: - tokens: Dictionary containing tokens to apply - target_session: Session to apply tokens to - """ - # Apply Authorization header - if "authorization" in tokens: - target_session.headers["Authorization"] = tokens["authorization"] - - # Apply other token headers - token_header_map = { - "x_api_key": "X-API-Key", - "api_key": "Api-Key", - "x_auth_token": "X-Auth-Token", - } - - for token_key, header_name in token_header_map.items(): - if token_key in tokens: - target_session.headers[header_name] = tokens[token_key] - - -def serialize_cookies(cookie_jar: Optional[CookieJar]) -> Dict[str, Any]: - """ - Serialize a CookieJar into a JSON-serializable dictionary. - - Args: - cookie_jar: The CookieJar to serialize - - Returns: - Dictionary containing serialized cookies - """ - if not cookie_jar: - return {} - - cookies = {} - for cookie in cookie_jar: - cookies[cookie.name] = { - "value": cookie.value, - "domain": cookie.domain, - "path": cookie.path, - "secure": cookie.secure, - "expires": cookie.expires, - } - - return cookies - - -def deserialize_cookies(cookies_data: Dict[str, Any]) -> CookieJar: - """ - Deserialize cookies into a CookieJar. - - Args: - cookies_data: Dictionary containing serialized cookies - - Returns: - CookieJar with cookies - """ - import http.cookiejar - - cookie_jar = http.cookiejar.CookieJar() - - for cookie_name, cookie_data in cookies_data.items(): - cookie = http.cookiejar.Cookie( - version=0, - name=cookie_name, - value=cookie_data["value"], - port=None, - port_specified=False, - domain=cookie_data.get("domain", ""), - domain_specified=bool(cookie_data.get("domain")), - domain_initial_dot=cookie_data.get("domain", "").startswith("."), - path=cookie_data.get("path", "/"), - path_specified=True, - secure=cookie_data.get("secure", False), - expires=cookie_data.get("expires"), - discard=False, - comment=None, - comment_url=None, - rest={}, - ) - cookie_jar.set_cookie(cookie) - - return cookie_jar - - -def serialize_credential(credential: Optional[Credential]) -> Optional[Dict[str, str]]: - """ - Serialize a Credential into a JSON-serializable dictionary. - - Args: - credential: The Credential to serialize - - Returns: - Dictionary containing username and password, or None - """ - if not credential: - return None - - return {"username": credential.username, "password": credential.password} - - -def deserialize_credential(credential_data: Optional[Dict[str, str]]) -> Optional[Credential]: - """ - Deserialize credential data into a Credential object. - - Args: - credential_data: Dictionary containing username and password - - Returns: - Credential object or None - """ - if not credential_data: - return None - - return Credential(username=credential_data["username"], password=credential_data["password"]) diff --git a/unshackle/core/cdm/__init__.py b/unshackle/core/cdm/__init__.py index 349099b..addf676 100644 --- a/unshackle/core/cdm/__init__.py +++ b/unshackle/core/cdm/__init__.py @@ -1,5 +1,57 @@ -from .custom_remote_cdm import CustomRemoteCDM -from .decrypt_labs_remote_cdm import DecryptLabsRemoteCDM -from .monalisa import MonaLisaCDM +""" +CDM helpers and implementations. -__all__ = ["DecryptLabsRemoteCDM", "CustomRemoteCDM", "MonaLisaCDM"] +Keep this module import-light: downstream code frequently imports helpers from +`unshackle.core.cdm.detect`, which requires importing this package first. +Some CDM implementations pull in optional/heavy dependencies, so we lazily +import them via `__getattr__` (PEP 562). +""" + +from __future__ import annotations + +from typing import Any + +__all__ = [ + "DecryptLabsRemoteCDM", + "CustomRemoteCDM", + "MonaLisaCDM", + "is_remote_cdm", + "is_local_cdm", + "cdm_location", + "is_playready_cdm", + "is_widevine_cdm", +] + + +def __getattr__(name: str) -> Any: + if name == "DecryptLabsRemoteCDM": + from .decrypt_labs_remote_cdm import DecryptLabsRemoteCDM + + return DecryptLabsRemoteCDM + if name == "CustomRemoteCDM": + from .custom_remote_cdm import CustomRemoteCDM + + return CustomRemoteCDM + if name == "MonaLisaCDM": + from .monalisa import MonaLisaCDM + + return MonaLisaCDM + + if name in { + "is_remote_cdm", + "is_local_cdm", + "cdm_location", + "is_playready_cdm", + "is_widevine_cdm", + }: + from .detect import cdm_location, is_local_cdm, is_playready_cdm, is_remote_cdm, is_widevine_cdm + + return { + "is_remote_cdm": is_remote_cdm, + "is_local_cdm": is_local_cdm, + "cdm_location": cdm_location, + "is_playready_cdm": is_playready_cdm, + "is_widevine_cdm": is_widevine_cdm, + }[name] + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/unshackle/core/cdm/detect.py b/unshackle/core/cdm/detect.py new file mode 100644 index 0000000..767ba1e --- /dev/null +++ b/unshackle/core/cdm/detect.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +from typing import Any + + +def is_remote_cdm(cdm: Any) -> bool: + """ + Return True if the CDM instance is backed by a remote/service CDM. + + This is useful for service logic that needs to know whether the CDM runs + locally (in-process) vs over HTTP/RPC (remote). + """ + + if cdm is None: + return False + + if hasattr(cdm, "is_remote_cdm"): + try: + return bool(getattr(cdm, "is_remote_cdm")) + except Exception: + pass + + try: + from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm + except Exception: + PlayReadyRemoteCdm = None + + if PlayReadyRemoteCdm is not None: + try: + if isinstance(cdm, PlayReadyRemoteCdm): + return True + except Exception: + pass + + try: + from pywidevine.remotecdm import RemoteCdm as WidevineRemoteCdm + except Exception: + WidevineRemoteCdm = None + + if WidevineRemoteCdm is not None: + try: + if isinstance(cdm, WidevineRemoteCdm): + return True + except Exception: + pass + + cls = getattr(cdm, "__class__", None) + mod = getattr(cls, "__module__", "") or "" + name = getattr(cls, "__name__", "") or "" + + if mod == "unshackle.core.cdm.decrypt_labs_remote_cdm" and name == "DecryptLabsRemoteCDM": + return True + if mod == "unshackle.core.cdm.custom_remote_cdm" and name == "CustomRemoteCDM": + return True + + if mod.startswith("pyplayready.remote") or mod.startswith("pywidevine.remote"): + return True + if "remote" in mod.lower() and name.lower().endswith("cdm"): + return True + if name.lower().endswith("remotecdm"): + return True + + return False + + +def is_local_cdm(cdm: Any) -> bool: + """ + Return True if the CDM instance is local/in-process. + + Unknown CDM types return False (use `cdm_location()` if you need 3-state). + """ + + if cdm is None: + return False + + if is_remote_cdm(cdm): + return False + + if is_playready_cdm(cdm) or is_widevine_cdm(cdm): + return True + + cls = getattr(cdm, "__class__", None) + mod = getattr(cls, "__module__", "") or "" + name = getattr(cls, "__name__", "") or "" + if mod == "unshackle.core.cdm.monalisa.monalisa_cdm" and name == "MonaLisaCDM": + return True + + return False + + +def cdm_location(cdm: Any) -> str: + """ + Return one of: "local", "remote", "unknown". + """ + + if is_remote_cdm(cdm): + return "remote" + if is_local_cdm(cdm): + return "local" + return "unknown" + + +def is_playready_cdm(cdm: Any) -> bool: + """ + Return True if the given CDM should be treated as PlayReady. + + This intentionally supports both: + - Local PlayReady CDMs (pyplayready.cdm.Cdm) + - Remote/wrapper CDMs (e.g. DecryptLabsRemoteCDM) that expose `is_playready` + """ + + if cdm is None: + return False + + if hasattr(cdm, "is_playready"): + try: + return bool(getattr(cdm, "is_playready")) + except Exception: + pass + + try: + from pyplayready.cdm import Cdm as PlayReadyCdm + except Exception: + PlayReadyCdm = None + + if PlayReadyCdm is not None: + try: + return isinstance(cdm, PlayReadyCdm) + except Exception: + pass + + try: + from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm + except Exception: + PlayReadyRemoteCdm = None + + if PlayReadyRemoteCdm is not None: + try: + return isinstance(cdm, PlayReadyRemoteCdm) + except Exception: + pass + + mod = getattr(getattr(cdm, "__class__", None), "__module__", "") or "" + return "pyplayready" in mod + + +def is_widevine_cdm(cdm: Any) -> bool: + """ + Return True if the given CDM should be treated as Widevine. + + Note: for remote/wrapper CDMs that expose `is_playready`, Widevine is treated + as the logical opposite. + """ + + if cdm is None: + return False + + if hasattr(cdm, "is_playready"): + try: + return not bool(getattr(cdm, "is_playready")) + except Exception: + pass + + try: + from pywidevine.cdm import Cdm as WidevineCdm + except Exception: + WidevineCdm = None + + if WidevineCdm is not None: + try: + return isinstance(cdm, WidevineCdm) + except Exception: + pass + + try: + from pywidevine.remotecdm import RemoteCdm as WidevineRemoteCdm + except Exception: + WidevineRemoteCdm = None + + if WidevineRemoteCdm is not None: + try: + return isinstance(cdm, WidevineRemoteCdm) + except Exception: + pass + + mod = getattr(getattr(cdm, "__class__", None), "__module__", "") or "" + return "pywidevine" in mod diff --git a/unshackle/core/cdm/monalisa/monalisa_cdm.py b/unshackle/core/cdm/monalisa/monalisa_cdm.py index c5880e1..ec9214e 100644 --- a/unshackle/core/cdm/monalisa/monalisa_cdm.py +++ b/unshackle/core/cdm/monalisa/monalisa_cdm.py @@ -7,8 +7,11 @@ a WebAssembly module that runs locally via wasmtime. import base64 import ctypes +import hashlib import json +import logging import re +import sys import uuid from pathlib import Path from typing import Dict, Optional, Union @@ -17,6 +20,8 @@ import wasmtime from unshackle.core import binaries +logger = logging.getLogger(__name__) + class MonaLisaCDM: """ @@ -128,10 +133,27 @@ class MonaLisaCDM: } self.exports["___wasm_call_ctors"](self.store) - self.ctx = self.exports["_monalisa_context_alloc"](self.store) + ctx = self.exports["_monalisa_context_alloc"](self.store) + self.ctx = ctx + + # _monalisa_context_alloc is expected to return a positive pointer/handle. + # Treat 0/negative/non-int-like values as allocation failure. + try: + ctx_int = int(ctx) + except Exception: + ctx_int = None + + if ctx_int is None or ctx_int <= 0: + # Ensure we don't leave a partially-initialized instance around. + self.close() + raise RuntimeError(f"Failed to allocate MonaLisa context (ctx={ctx!r})") return 1 except Exception as e: - raise RuntimeError(f"Failed to initialize session: {e}") + # Clean up partial state (e.g., store/memory/instance) before propagating failure. + self.close() + if isinstance(e, RuntimeError): + raise + raise RuntimeError(f"Failed to initialize session: {e}") from e def close(self, session_id: int = 1) -> None: """ @@ -188,7 +210,9 @@ class MonaLisaCDM: # Extract DCID from license to generate KID try: decoded = base64.b64decode(license_b64).decode("ascii", errors="ignore") - except Exception: + except Exception as e: + # Avoid logging raw license content; log only safe metadata. + logger.exception("Failed to base64-decode MonaLisa license (len=%s): %s", len(license_b64), e) decoded = "" m = re.search( @@ -198,7 +222,14 @@ class MonaLisaCDM: if m: kid_bytes = uuid.uuid5(uuid.NAMESPACE_DNS, m.group()).bytes else: - kid_bytes = uuid.UUID(int=0).bytes + # No DCID in the license: derive a deterministic per-license KID to avoid collisions. + try: + license_raw = base64.b64decode(license_b64) + except Exception: + license_raw = license_b64.encode("utf-8", errors="replace") + + license_hash = hashlib.sha256(license_raw).hexdigest() + kid_bytes = uuid.uuid5(uuid.NAMESPACE_DNS, f"monalisa:license:{license_hash}").bytes return {"kid": kid_bytes.hex(), "key": key_bytes.hex(), "type": "CONTENT"} @@ -221,21 +252,29 @@ class MonaLisaCDM: stack = 0 converted_args = [] - for arg in args: - if isinstance(arg, str): - if stack == 0: - stack = self.exports["stackSave"](self.store) - max_length = (len(arg) << 2) + 1 - ptr = self.exports["stackAlloc"](self.store, max_length) - self._string_to_utf8(arg, ptr, max_length) - converted_args.append(ptr) - else: - converted_args.append(arg) + try: + for arg in args: + if isinstance(arg, str): + if stack == 0: + stack = self.exports["stackSave"](self.store) + max_length = (len(arg) << 2) + 1 + ptr = self.exports["stackAlloc"](self.store, max_length) + self._string_to_utf8(arg, ptr, max_length) + converted_args.append(ptr) + else: + converted_args.append(arg) - result = self.exports[func_name](self.store, *converted_args) - - if stack != 0: - self.exports["stackRestore"](self.store, stack) + result = self.exports[func_name](self.store, *converted_args) + finally: + # stackAlloc pointers live on the WASM stack; always restore even if the call throws. + if stack != 0: + exc = sys.exc_info()[1] + try: + self.exports["stackRestore"](self.store, stack) + except Exception: + # If we're already failing, don't mask the original exception. + if exc is None: + raise if return_type is bool: return bool(result) @@ -243,6 +282,13 @@ class MonaLisaCDM: def _write_i32(self, addr: int, value: int) -> None: """Write a 32-bit integer to WASM memory.""" + if addr % 4 != 0: + raise ValueError(f"Unaligned i32 write: addr={addr} (must be 4-byte aligned)") + + data_len = self.memory.data_len(self.store) + if addr < 0 or addr + 4 > data_len: + raise IndexError(f"i32 write out of bounds: addr={addr}, mem_len={data_len}") + data = self.memory.data_ptr(self.store) mem_ptr = ctypes.cast(data, ctypes.POINTER(ctypes.c_int32)) mem_ptr[addr >> 2] = value diff --git a/unshackle/core/downloaders/aria2c.py b/unshackle/core/downloaders/aria2c.py index af7d34f..c4125a7 100644 --- a/unshackle/core/downloaders/aria2c.py +++ b/unshackle/core/downloaders/aria2c.py @@ -1,3 +1,4 @@ +import logging import os import subprocess import textwrap @@ -54,11 +55,13 @@ class _Aria2Manager: """Singleton manager to run one aria2c process and enqueue downloads via RPC.""" def __init__(self) -> None: + self._logger = logging.getLogger(__name__) self._proc: Optional[subprocess.Popen] = None self._rpc_port: Optional[int] = None self._rpc_secret: Optional[str] = None self._rpc_uri: Optional[str] = None self._session: Session = Session() + self._max_workers: Optional[int] = None self._max_concurrent_downloads: int = 0 self._max_connection_per_server: int = 1 self._split_default: int = 5 @@ -66,6 +69,47 @@ class _Aria2Manager: self._proxy: Optional[str] = None self._lock: threading.Lock = threading.Lock() + def _wait_for_rpc_ready(self, timeout_s: float = 8.0, interval_s: float = 0.1) -> None: + assert self._proc is not None + assert self._rpc_uri is not None + assert self._rpc_secret is not None + + deadline = time.monotonic() + timeout_s + + payload = { + "jsonrpc": "2.0", + "id": get_random_bytes(16).hex(), + "method": "aria2.getVersion", + "params": [f"token:{self._rpc_secret}"], + } + + while time.monotonic() < deadline: + if self._proc.poll() is not None: + raise RuntimeError( + f"aria2c exited before RPC became ready (exit code {self._proc.returncode})" + ) + try: + res = self._session.post(self._rpc_uri, json=payload, timeout=0.25) + data = res.json() + if isinstance(data, dict) and data.get("result") is not None: + return + except (requests.exceptions.RequestException, ValueError): + # Not ready yet (connection refused / bad response / etc.) + pass + time.sleep(interval_s) + + # Timed out: ensure we don't leave a zombie/stray aria2c process behind. + try: + self._proc.terminate() + self._proc.wait(timeout=2) + except Exception: + try: + self._proc.kill() + self._proc.wait(timeout=2) + except Exception: + pass + raise TimeoutError(f"aria2c RPC did not become ready within {timeout_s:.1f}s") + def _build_args(self) -> list[str]: args = [ "--continue=true", @@ -95,9 +139,6 @@ class _Aria2Manager: max_workers: Optional[int], ) -> None: with self._lock: - if self._proc and self._proc.poll() is None: - return - if not binaries.Aria2: debug_logger = get_debug_logger() if debug_logger: @@ -109,27 +150,45 @@ class _Aria2Manager: ) raise EnvironmentError("Aria2c executable not found...") + effective_proxy = proxy or None + if not max_workers: - max_workers = min(32, (os.cpu_count() or 1) + 4) + effective_max_workers = min(32, (os.cpu_count() or 1) + 4) elif not isinstance(max_workers, int): raise TypeError(f"Expected max_workers to be {int}, not {type(max_workers)}") + else: + effective_max_workers = max_workers + + if self._proc and self._proc.poll() is None: + if effective_proxy != self._proxy or effective_max_workers != self._max_workers: + self._logger.warning( + "aria2c process is already running; requested proxy=%r, max_workers=%r, " + "but running process will continue with proxy=%r, max_workers=%r", + effective_proxy, + effective_max_workers, + self._proxy, + self._max_workers, + ) + return self._rpc_port = get_free_port() self._rpc_secret = get_random_bytes(16).hex() self._rpc_uri = f"http://127.0.0.1:{self._rpc_port}/jsonrpc" - self._max_concurrent_downloads = int(config.aria2c.get("max_concurrent_downloads", max_workers)) + self._max_workers = effective_max_workers + self._max_concurrent_downloads = int( + config.aria2c.get("max_concurrent_downloads", effective_max_workers) + ) self._max_connection_per_server = int(config.aria2c.get("max_connection_per_server", 1)) self._split_default = int(config.aria2c.get("split", 5)) self._file_allocation = config.aria2c.get("file_allocation", "prealloc") - self._proxy = proxy or None + self._proxy = effective_proxy args = self._build_args() self._proc = subprocess.Popen( [binaries.Aria2, *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) - # Give aria2c a moment to start up and bind to the RPC port - time.sleep(0.5) + self._wait_for_rpc_ready() @property def rpc_uri(self) -> str: diff --git a/unshackle/core/downloaders/n_m3u8dl_re.py b/unshackle/core/downloaders/n_m3u8dl_re.py index d6b7d08..7789d10 100644 --- a/unshackle/core/downloaders/n_m3u8dl_re.py +++ b/unshackle/core/downloaders/n_m3u8dl_re.py @@ -192,8 +192,10 @@ def build_download_args( if ad_keyword: args["--ad-keyword"] = ad_keyword + key_args = [] if content_keys: - args["--key"] = next((f"{kid.hex}:{key.lower()}" for kid, key in content_keys.items()), None) + for kid, key in content_keys.items(): + key_args.extend(["--key", f"{kid.hex}:{key.lower()}"]) decryption_config = config.decryption.lower() engine_name = DECRYPTION_ENGINE.get(decryption_config) or "SHAKA_PACKAGER" @@ -221,6 +223,9 @@ def build_download_args( elif value is not False and value is not None: command.extend([flag, str(value)]) + # Append all content keys (multiple --key flags supported by N_m3u8DL-RE) + command.extend(key_args) + if headers: for key, value in headers.items(): if key.lower() not in ("accept-encoding", "cookie"): diff --git a/unshackle/core/downloaders/requests.py b/unshackle/core/downloaders/requests.py index cb3ce5a..0cb6b4e 100644 --- a/unshackle/core/downloaders/requests.py +++ b/unshackle/core/downloaders/requests.py @@ -260,11 +260,18 @@ def requests( }, ) - yield dict(total=len(urls)) + # If we're downloading more than one URL, treat them as "segments" for progress purposes. + # For single-URL downloads we want per-chunk progress (and the inner `download()` will yield + # a chunk-based `total`), so don't set a segment total of 1 here. + segmented_batch = len(urls) > 1 + if segmented_batch: + yield dict(total=len(urls)) try: with ThreadPoolExecutor(max_workers=max_workers) as pool: - for future in as_completed(pool.submit(download, session=session, segmented=True, **url) for url in urls): + for future in as_completed( + pool.submit(download, session=session, segmented=segmented_batch, **url) for url in urls + ): try: yield from future.result() except KeyboardInterrupt: diff --git a/unshackle/core/drm/monalisa.py b/unshackle/core/drm/monalisa.py index f89d764..3b6f29a 100644 --- a/unshackle/core/drm/monalisa.py +++ b/unshackle/core/drm/monalisa.py @@ -7,6 +7,7 @@ segment decryption (ML-Worker binary + AES-ECB). from __future__ import annotations +import logging import os import subprocess import sys @@ -17,6 +18,8 @@ from uuid import UUID from Cryptodome.Cipher import AES from Cryptodome.Util.Padding import unpad +log = logging.getLogger(__name__) + class MonaLisa: """ @@ -142,7 +145,16 @@ class MonaLisa: The raw PSSH value as a base64 string. """ if isinstance(self._ticket, bytes): - return self._ticket.decode("utf-8") + try: + return self._ticket.decode("utf-8") + except UnicodeDecodeError: + # Tickets are typically base64, so ASCII is a reasonable fallback. + try: + return self._ticket.decode("ascii") + except UnicodeDecodeError as e: + raise ValueError( + f"Ticket bytes must be UTF-8 text or ASCII base64; got undecodable bytes (len={len(self._ticket)})" + ) from e return self._ticket @property @@ -222,19 +234,21 @@ class MonaLisa: raise MonaLisa.Exceptions.DecryptionFailed(f"Segment file does not exist: {segment_path}") # Stage 1: ML-Worker decryption - cmd = [str(worker_path), self._key, str(bbts_path), str(ents_path)] + cmd = [str(worker_path), str(self._key), str(bbts_path), str(ents_path)] startupinfo = None if sys.platform == "win32": startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + worker_timeout_s = 60 process = subprocess.run( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, startupinfo=startupinfo, + timeout=worker_timeout_s, ) if process.returncode != 0: @@ -260,6 +274,11 @@ class MonaLisa: except MonaLisa.Exceptions.DecryptionFailed: raise + except subprocess.TimeoutExpired as e: + log.error("ML-Worker timed out after %ss for %s", worker_timeout_s, segment_path.name) + raise MonaLisa.Exceptions.DecryptionFailed( + f"ML-Worker timed out after {worker_timeout_s}s for {segment_path.name}" + ) from e except Exception as e: raise MonaLisa.Exceptions.DecryptionFailed(f"Failed to decrypt segment {segment_path.name}: {e}") finally: diff --git a/unshackle/core/manifests/dash.py b/unshackle/core/manifests/dash.py index c8a7ef7..9585ee0 100644 --- a/unshackle/core/manifests/dash.py +++ b/unshackle/core/manifests/dash.py @@ -19,12 +19,12 @@ import requests from curl_cffi.requests import Session as CurlSession from langcodes import Language, tag_is_valid from lxml.etree import Element, ElementTree -from pyplayready.cdm import Cdm as PlayReadyCdm from pyplayready.system.pssh import PSSH as PR_PSSH from pywidevine.cdm import Cdm as WidevineCdm from pywidevine.pssh import PSSH from requests import Session +from unshackle.core.cdm.detect import is_playready_cdm from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack from unshackle.core.downloaders import requests as requests_downloader from unshackle.core.drm import DRM_T, PlayReady, Widevine @@ -477,7 +477,7 @@ class DASH: track.data["dash"]["segment_durations"] = segment_durations if not track.drm and init_data and isinstance(track, (Video, Audio)): - prefers_playready = isinstance(cdm, PlayReadyCdm) or (hasattr(cdm, "is_playready") and cdm.is_playready) + prefers_playready = is_playready_cdm(cdm) if prefers_playready: try: track.drm = [PlayReady.from_init_data(init_data)] diff --git a/unshackle/core/manifests/hls.py b/unshackle/core/manifests/hls.py index 86133c0..122b85c 100644 --- a/unshackle/core/manifests/hls.py +++ b/unshackle/core/manifests/hls.py @@ -28,6 +28,7 @@ from pywidevine.pssh import PSSH as WV_PSSH from requests import Session from unshackle.core import binaries +from unshackle.core.cdm.detect import is_playready_cdm, is_widevine_cdm from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack from unshackle.core.downloaders import requests as requests_downloader from unshackle.core.drm import DRM_T, ClearKey, MonaLisa, PlayReady, Widevine @@ -115,9 +116,14 @@ class HLS: for playlist in self.manifest.playlists: audio_group = playlist.stream_info.audio - if audio_group: - audio_codec = Audio.Codec.from_codecs(playlist.stream_info.codecs) - audio_codecs_by_group_id[audio_group] = audio_codec + audio_codec: Optional[Audio.Codec] = None + if audio_group and playlist.stream_info.codecs: + try: + audio_codec = Audio.Codec.from_codecs(playlist.stream_info.codecs) + except ValueError: + audio_codec = None + if audio_codec: + audio_codecs_by_group_id[audio_group] = audio_codec try: # TODO: Any better way to figure out the primary track type? @@ -225,6 +231,39 @@ class HLS: return tracks + @staticmethod + def _finalize_n_m3u8dl_re_output(*, track: AnyTrack, save_dir: Path, save_path: Path) -> Path: + """ + Finalize output from N_m3u8DL-RE. + + We call N_m3u8DL-RE with `--save-name track.id`, so the final file should be `{track.id}.*` under `save_dir`. + This moves that output to `save_path` (preserving the real suffix) and, for subtitles, updates `track.codec` + to match the produced file extension. + """ + matches = [p for p in save_dir.rglob(f"{track.id}.*") if p.is_file()] + if not matches: + raise FileNotFoundError(f"No output files produced by N_m3u8DL-RE for save-name={track.id} in: {save_dir}") + + primary = max(matches, key=lambda p: p.stat().st_size) + + final_save_path = save_path.with_suffix(primary.suffix) if primary.suffix else save_path + + final_save_path.parent.mkdir(parents=True, exist_ok=True) + if primary.absolute() != final_save_path.absolute(): + final_save_path.unlink(missing_ok=True) + shutil.move(str(primary), str(final_save_path)) + + if isinstance(track, Subtitle): + ext = final_save_path.suffix.lower().lstrip(".") + try: + track.codec = Subtitle.Codec.from_mime(ext) + except ValueError: + pass + + shutil.rmtree(save_dir, ignore_errors=True) + + return final_save_path + @staticmethod def download_track( track: AnyTrack, @@ -255,7 +294,7 @@ class HLS: else: # Get the playlist text and handle both session types response = session.get(track.url) - if isinstance(response, requests.Response): + if isinstance(response, requests.Response) or isinstance(response, CurlResponse): if not response.ok: log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}") sys.exit(1) @@ -317,8 +356,16 @@ class HLS: raise if not initial_drm_licensed and session_drm and isinstance(session_drm, MonaLisa): - if license_widevine: + try: + if not license_widevine: + raise ValueError("license_widevine func must be supplied to use DRM") + progress(downloaded="LICENSING") license_widevine(session_drm) + progress(downloaded="[yellow]LICENSED") + except Exception: # noqa + DOWNLOAD_CANCELLED.set() # skip pending track downloads + progress(downloaded="[red]FAILED") + raise if DOWNLOAD_LICENCE_ONLY.is_set(): progress(downloaded="[yellow]SKIPPED") @@ -420,222 +467,228 @@ class HLS: for control_file in segment_save_dir.glob("*.aria2__temp"): control_file.unlink() - if not skip_merge: - progress(total=total_segments, completed=0, downloaded="Merging") + if skip_merge: + final_save_path = HLS._finalize_n_m3u8dl_re_output(track=track, save_dir=save_dir, save_path=save_path) + progress(downloaded="Downloaded") + track.path = final_save_path + events.emit(events.Types.TRACK_DOWNLOADED, track=track) + return - name_len = len(str(total_segments)) - discon_i = 0 - range_offset = 0 - map_data: Optional[tuple[m3u8.model.InitializationSection, bytes]] = None - if session_drm: - encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = (initial_drm_key, session_drm) - else: - encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = None + progress(total=total_segments, completed=0, downloaded="Merging") - i = -1 - for real_i, segment in enumerate(master.segments): - if segment not in unwanted_segments: - i += 1 + name_len = len(str(total_segments)) + discon_i = 0 + range_offset = 0 + map_data: Optional[tuple[m3u8.model.InitializationSection, bytes]] = None + if session_drm: + encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = (initial_drm_key, session_drm) + else: + encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = None - is_last_segment = (real_i + 1) == len(master.segments) + i = -1 + for real_i, segment in enumerate(master.segments): + if segment not in unwanted_segments: + i += 1 - def merge(to: Path, via: list[Path], delete: bool = False, include_map_data: bool = False): - """ - Merge all files to a given path, optionally including map data. + is_last_segment = (real_i + 1) == len(master.segments) - Parameters: - to: The output file with all merged data. - via: List of files to merge, in sequence. - delete: Delete the file once it's been merged. - include_map_data: Whether to include the init map data. - """ - with open(to, "wb") as x: - if include_map_data and map_data and map_data[1]: - x.write(map_data[1]) - for file in via: - x.write(file.read_bytes()) - x.flush() - if delete: - file.unlink() + def merge(to: Path, via: list[Path], delete: bool = False, include_map_data: bool = False): + """ + Merge all files to a given path, optionally including map data. - def decrypt(include_this_segment: bool) -> Path: - """ - Decrypt all segments that uses the currently set DRM. + Parameters: + to: The output file with all merged data. + via: List of files to merge, in sequence. + delete: Delete the file once it's been merged. + include_map_data: Whether to include the init map data. + """ + with open(to, "wb") as x: + if include_map_data and map_data and map_data[1]: + x.write(map_data[1]) + for file in via: + x.write(file.read_bytes()) + x.flush() + if delete: + file.unlink() - All segments that will be decrypted with this DRM will be merged together - in sequence, prefixed with the init data (if any), and then deleted. Once - merged they will be decrypted. The merged and decrypted file names state - the range of segments that were used. + def decrypt(include_this_segment: bool) -> Path: + """ + Decrypt all segments that uses the currently set DRM. - Parameters: - include_this_segment: Whether to include the current segment in the - list of segments to merge and decrypt. This should be False if - decrypting on EXT-X-KEY changes, or True when decrypting on the - last segment. + All segments that will be decrypted with this DRM will be merged together + in sequence, prefixed with the init data (if any), and then deleted. Once + merged they will be decrypted. The merged and decrypted file names state + the range of segments that were used. - Returns the decrypted path. - """ - drm = encryption_data[1] - first_segment_i = next( - int(file.stem) for file in sorted(segment_save_dir.iterdir()) if file.stem.isdigit() - ) - last_segment_i = max(0, i - int(not include_this_segment)) - range_len = (last_segment_i - first_segment_i) + 1 + Parameters: + include_this_segment: Whether to include the current segment in the + list of segments to merge and decrypt. This should be False if + decrypting on EXT-X-KEY changes, or True when decrypting on the + last segment. - segment_range = f"{str(first_segment_i).zfill(name_len)}-{str(last_segment_i).zfill(name_len)}" - merged_path = ( - segment_save_dir / f"{segment_range}{get_extension(master.segments[last_segment_i].uri)}" - ) - decrypted_path = segment_save_dir / f"{merged_path.stem}_decrypted{merged_path.suffix}" + Returns the decrypted path. + """ + drm = encryption_data[1] + first_segment_i = next( + int(file.stem) for file in sorted(segment_save_dir.iterdir()) if file.stem.isdigit() + ) + last_segment_i = max(0, i - int(not include_this_segment)) + range_len = (last_segment_i - first_segment_i) + 1 - files = [ - file - for file in sorted(segment_save_dir.iterdir()) - if file.stem.isdigit() and first_segment_i <= int(file.stem) <= last_segment_i - ] - if not files: - raise ValueError(f"None of the segment files for {segment_range} exist...") - elif len(files) != range_len: - raise ValueError(f"Missing {range_len - len(files)} segment files for {segment_range}...") + segment_range = f"{str(first_segment_i).zfill(name_len)}-{str(last_segment_i).zfill(name_len)}" + merged_path = segment_save_dir / f"{segment_range}{get_extension(master.segments[last_segment_i].uri)}" + decrypted_path = segment_save_dir / f"{merged_path.stem}_decrypted{merged_path.suffix}" - if isinstance(drm, (Widevine, PlayReady)): - # with widevine we can merge all segments and decrypt once - merge(to=merged_path, via=files, delete=True, include_map_data=True) - drm.decrypt(merged_path) - merged_path.rename(decrypted_path) - else: - # with other drm we must decrypt separately and then merge them - # for aes this is because each segment likely has 16-byte padding - for file in files: - drm.decrypt(file) - merge(to=merged_path, via=files, delete=True, include_map_data=True) + files = [ + file + for file in sorted(segment_save_dir.iterdir()) + if file.stem.isdigit() and first_segment_i <= int(file.stem) <= last_segment_i + ] + if not files: + raise ValueError(f"None of the segment files for {segment_range} exist...") + elif len(files) != range_len: + raise ValueError(f"Missing {range_len - len(files)} segment files for {segment_range}...") - events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=decrypted_path) + if isinstance(drm, (Widevine, PlayReady)): + # with widevine we can merge all segments and decrypt once + merge(to=merged_path, via=files, delete=True, include_map_data=True) + drm.decrypt(merged_path) + merged_path.rename(decrypted_path) + else: + # with other drm we must decrypt separately and then merge them + # for aes this is because each segment likely has 16-byte padding + for file in files: + drm.decrypt(file) + merge(to=merged_path, via=files, delete=True, include_map_data=True) - return decrypted_path + events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=decrypted_path) - def merge_discontinuity(include_this_segment: bool, include_map_data: bool = True): - """ - Merge all segments of the discontinuity. + return decrypted_path - All segment files for this discontinuity must already be downloaded and - already decrypted (if it needs to be decrypted). + def merge_discontinuity(include_this_segment: bool, include_map_data: bool = True): + """ + Merge all segments of the discontinuity. - Parameters: - include_this_segment: Whether to include the current segment in the - list of segments to merge and decrypt. This should be False if - decrypting on EXT-X-KEY changes, or True when decrypting on the - last segment. - include_map_data: Whether to prepend the init map data before the - segment files when merging. - """ - last_segment_i = max(0, i - int(not include_this_segment)) + All segment files for this discontinuity must already be downloaded and + already decrypted (if it needs to be decrypted). - files = [ - file - for file in sorted(segment_save_dir.iterdir()) - if int(file.stem.replace("_decrypted", "").split("-")[-1]) <= last_segment_i - ] - if files: - to_dir = segment_save_dir.parent - to_path = to_dir / f"{str(discon_i).zfill(name_len)}{files[-1].suffix}" - merge(to=to_path, via=files, delete=True, include_map_data=include_map_data) + Parameters: + include_this_segment: Whether to include the current segment in the + list of segments to merge and decrypt. This should be False if + decrypting on EXT-X-KEY changes, or True when decrypting on the + last segment. + include_map_data: Whether to prepend the init map data before the + segment files when merging. + """ + last_segment_i = max(0, i - int(not include_this_segment)) - if segment not in unwanted_segments: - if isinstance(track, Subtitle): - segment_file_ext = get_extension(segment.uri) - segment_file_path = segment_save_dir / f"{str(i).zfill(name_len)}{segment_file_ext}" - segment_data = try_ensure_utf8(segment_file_path.read_bytes()) - if track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML): - segment_data = ( - segment_data.decode("utf8") - .replace("‎", html.unescape("‎")) - .replace("‏", html.unescape("‏")) - .encode("utf8") - ) - segment_file_path.write_bytes(segment_data) + files = [ + file + for file in sorted(segment_save_dir.iterdir()) + if int(file.stem.replace("_decrypted", "").split("-")[-1]) <= last_segment_i + ] + if files: + to_dir = segment_save_dir.parent + to_path = to_dir / f"{str(discon_i).zfill(name_len)}{files[-1].suffix}" + merge(to=to_path, via=files, delete=True, include_map_data=include_map_data) - if segment.discontinuity and i != 0: - if encryption_data: - decrypt(include_this_segment=False) - merge_discontinuity( - include_this_segment=False, include_map_data=not encryption_data or not encryption_data[1] + if segment not in unwanted_segments: + if isinstance(track, Subtitle): + segment_file_ext = get_extension(segment.uri) + segment_file_path = segment_save_dir / f"{str(i).zfill(name_len)}{segment_file_ext}" + segment_data = try_ensure_utf8(segment_file_path.read_bytes()) + if track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML): + segment_data = ( + segment_data.decode("utf8") + .replace("‎", html.unescape("‎")) + .replace("‏", html.unescape("‏")) + .encode("utf8") ) + segment_file_path.write_bytes(segment_data) - discon_i += 1 - range_offset = 0 # TODO: Should this be reset or not? - map_data = None - if encryption_data: - encryption_data = (encryption_data[0], encryption_data[1]) - - if segment.init_section and (not map_data or segment.init_section != map_data[0]): - if segment.init_section.byterange: - init_byte_range = HLS.calculate_byte_range(segment.init_section.byterange, range_offset) - range_offset = init_byte_range.split("-")[0] - init_range_header = {"Range": f"bytes={init_byte_range}"} - else: - init_range_header = {} - - # Handle both session types for init section request - res = session.get( - url=urljoin(segment.init_section.base_uri, segment.init_section.uri), - headers=init_range_header, - ) - - # Check response based on session type - if isinstance(res, requests.Response): - res.raise_for_status() - init_content = res.content - else: - raise TypeError( - f"Expected response to be requests.Response or curl_cffi.Response, not {type(res)}" - ) - - map_data = (segment.init_section, init_content) - - segment_keys = getattr(segment, "keys", None) - if segment_keys: - if cdm: - cdm_segment_keys = HLS.filter_keys_for_cdm(segment_keys, cdm) - key = HLS.get_supported_key(cdm_segment_keys) if cdm_segment_keys else HLS.get_supported_key(segment_keys) - else: - key = HLS.get_supported_key(segment_keys) - if encryption_data and encryption_data[0] != key and i != 0 and segment not in unwanted_segments: - decrypt(include_this_segment=False) - - if key is None: - encryption_data = None - elif not encryption_data or encryption_data[0] != key: - drm = HLS.get_drm(key, session) - if isinstance(drm, (Widevine, PlayReady)): - try: - if map_data: - track_kid = track.get_key_id(map_data[1]) - else: - track_kid = None - if not track_kid: - track_kid = drm.kid - progress(downloaded="LICENSING") - license_widevine(drm, track_kid=track_kid) - progress(downloaded="[yellow]LICENSED") - except Exception: # noqa - DOWNLOAD_CANCELLED.set() # skip pending track downloads - progress(downloaded="[red]FAILED") - raise - encryption_data = (key, drm) - - if DOWNLOAD_LICENCE_ONLY.is_set(): - continue - - if is_last_segment: - # required as it won't end with EXT-X-DISCONTINUITY nor a new key + if segment.discontinuity and i != 0: if encryption_data: - decrypt(include_this_segment=True) + decrypt(include_this_segment=False) merge_discontinuity( - include_this_segment=True, include_map_data=not encryption_data or not encryption_data[1] + include_this_segment=False, include_map_data=not encryption_data or not encryption_data[1] ) - progress(advance=1) + discon_i += 1 + range_offset = 0 # TODO: Should this be reset or not? + map_data = None + + if segment.init_section and (not map_data or segment.init_section != map_data[0]): + if segment.init_section.byterange: + init_byte_range = HLS.calculate_byte_range(segment.init_section.byterange, range_offset) + range_offset = int(init_byte_range.split("-")[0]) + init_range_header = {"Range": f"bytes={init_byte_range}"} + else: + init_range_header = {} + + # Handle both session types for init section request + res = session.get( + url=urljoin(segment.init_section.base_uri, segment.init_section.uri), + headers=init_range_header, + ) + + # Check response based on session type + if isinstance(res, requests.Response) or isinstance(res, CurlResponse): + res.raise_for_status() + init_content = res.content + else: + raise TypeError( + f"Expected response to be requests.Response or curl_cffi.Response, not {type(res)}" + ) + + map_data = (segment.init_section, init_content) + + segment_keys = getattr(segment, "keys", None) + if segment_keys: + if cdm: + cdm_segment_keys = HLS.filter_keys_for_cdm(segment_keys, cdm) + key = ( + HLS.get_supported_key(cdm_segment_keys) + if cdm_segment_keys + else HLS.get_supported_key(segment_keys) + ) + else: + key = HLS.get_supported_key(segment_keys) + if encryption_data and encryption_data[0] != key and i != 0 and segment not in unwanted_segments: + decrypt(include_this_segment=False) + + if key is None: + encryption_data = None + elif not encryption_data or encryption_data[0] != key: + drm = HLS.get_drm(key, session) + if isinstance(drm, (Widevine, PlayReady)): + try: + if map_data: + track_kid = track.get_key_id(map_data[1]) + else: + track_kid = None + if not track_kid: + track_kid = drm.kid + progress(downloaded="LICENSING") + license_widevine(drm, track_kid=track_kid) + progress(downloaded="[yellow]LICENSED") + except Exception: # noqa + DOWNLOAD_CANCELLED.set() # skip pending track downloads + progress(downloaded="[red]FAILED") + raise + encryption_data = (key, drm) + + if DOWNLOAD_LICENCE_ONLY.is_set(): + continue + + if is_last_segment: + # required as it won't end with EXT-X-DISCONTINUITY nor a new key + if encryption_data: + decrypt(include_this_segment=True) + merge_discontinuity( + include_this_segment=True, include_map_data=not encryption_data or not encryption_data[1] + ) + + progress(advance=1) if DOWNLOAD_LICENCE_ONLY.is_set(): return @@ -865,15 +918,10 @@ class HLS: """ playready_urn = f"urn:uuid:{PR_PSSH.SYSTEM_ID}" playready_keyformats = {playready_urn, "com.microsoft.playready"} - if isinstance(cdm, WidevineCdm): + if is_widevine_cdm(cdm): return [k for k in keys if k.keyformat and k.keyformat.lower() == WidevineCdm.urn] - elif isinstance(cdm, PlayReadyCdm): + elif is_playready_cdm(cdm): return [k for k in keys if k.keyformat and k.keyformat.lower() in playready_keyformats] - elif hasattr(cdm, "is_playready"): - if cdm.is_playready: - return [k for k in keys if k.keyformat and k.keyformat.lower() in playready_keyformats] - else: - return [k for k in keys if k.keyformat and k.keyformat.lower() == WidevineCdm.urn] return keys @staticmethod diff --git a/unshackle/core/proxies/gluetun.py b/unshackle/core/proxies/gluetun.py index 9063816..f9be0f3 100644 --- a/unshackle/core/proxies/gluetun.py +++ b/unshackle/core/proxies/gluetun.py @@ -2,7 +2,9 @@ import atexit import logging import os import re +import stat import subprocess +import tempfile import threading import time from typing import Optional @@ -750,7 +752,8 @@ class Gluetun(Proxy): # Debug log environment variables (redact sensitive values) if debug_logger: - safe_env = {k: ("***" if "KEY" in k or "PASSWORD" in k else v) for k, v in env_vars.items()} + redact_markers = ("KEY", "PASSWORD", "PASS", "TOKEN", "SECRET", "USER") + safe_env = {k: ("***" if any(m in k for m in redact_markers) else v) for k, v in env_vars.items()} debug_logger.log( level="DEBUG", operation="gluetun_env_vars", @@ -771,23 +774,62 @@ class Gluetun(Proxy): f"127.0.0.1:{port}:8888/tcp", ] - # Add environment variables - for key, value in env_vars.items(): - cmd.extend(["-e", f"{key}={value}"]) - - # Add Gluetun image - cmd.append("qmcgaw/gluetun:latest") - - # Execute docker run + # Avoid exposing credentials in process listings by using --env-file instead of many "-e KEY=VALUE". + env_file_path: str | None = None try: - result = subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=30, - encoding="utf-8", - errors="replace", - ) + fd, env_file_path = tempfile.mkstemp(prefix=f"unshackle-{container_name}-", suffix=".env") + try: + # Best-effort restrictive permissions. + if os.name != "nt": + if hasattr(os, "fchmod"): + os.fchmod(fd, 0o600) + else: + os.chmod(env_file_path, 0o600) + else: + os.chmod(env_file_path, stat.S_IREAD | stat.S_IWRITE) + + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as f: + for key, value in env_vars.items(): + if "=" in key: + raise ValueError(f"Invalid env var name for docker env-file: {key!r}") + v = "" if value is None else str(value) + if "\n" in v or "\r" in v: + raise ValueError(f"Invalid env var value (contains newline) for {key!r}") + f.write(f"{key}={v}\n") + except Exception: + # If we fail before fdopen closes the descriptor, make sure it's not leaked. + try: + os.close(fd) + except Exception: + pass + raise + + cmd.extend(["--env-file", env_file_path]) + + # Add Gluetun image + cmd.append(gluetun_image) + + # Execute docker run + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=30, + encoding="utf-8", + errors="replace", + ) + except subprocess.TimeoutExpired: + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_container_create_timeout", + message=f"Docker run timed out for {container_name}", + context={"container_name": container_name}, + success=False, + duration_ms=(time.time() - start_time) * 1000, + ) + raise RuntimeError("Docker run command timed out") if result.returncode != 0: error_msg = result.stderr or "unknown error" @@ -826,29 +868,51 @@ class Gluetun(Proxy): success=True, duration_ms=duration_ms, ) - - except subprocess.TimeoutExpired: - if debug_logger: - debug_logger.log( - level="ERROR", - operation="gluetun_container_create_timeout", - message=f"Docker run timed out for {container_name}", - context={"container_name": container_name}, - success=False, - duration_ms=(time.time() - start_time) * 1000, - ) - raise RuntimeError("Docker run command timed out") + finally: + if env_file_path: + # Best-effort "secure delete": overwrite then unlink (not guaranteed on all filesystems). + try: + with open(env_file_path, "r+b") as f: + try: + f.seek(0, os.SEEK_END) + length = f.tell() + f.seek(0) + if length > 0: + f.write(b"\x00" * length) + f.flush() + os.fsync(f.fileno()) + except Exception: + pass + except Exception: + pass + try: + os.remove(env_file_path) + except FileNotFoundError: + pass + except Exception: + pass def _is_container_running(self, container_name: str) -> bool: """Check if a Docker container is running.""" try: result = subprocess.run( - ["docker", "ps", "--filter", f"name={container_name}", "--format", "{{.Names}}"], + [ + "docker", + "ps", + "--filter", + f"name=^{re.escape(container_name)}$", + "--format", + "{{.Names}}", + ], capture_output=True, text=True, timeout=5, ) - return result.returncode == 0 and container_name in result.stdout + if result.returncode != 0: + return False + + names = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()] + return any(name == container_name for name in names) except (subprocess.TimeoutExpired, FileNotFoundError): return False @@ -1132,98 +1196,104 @@ class Gluetun(Proxy): # Create a session with the proxy configured session = requests.Session() - session.proxies = {"http": proxy_url, "https": proxy_url} + try: + session.proxies = {"http": proxy_url, "https": proxy_url} - # Retry with exponential backoff - for attempt in range(max_retries): - try: - # Get external IP through the proxy using shared utility - ip_info = get_ip_info(session) + # Retry with exponential backoff + for attempt in range(max_retries): + try: + # Get external IP through the proxy using shared utility + ip_info = get_ip_info(session) - if ip_info: - actual_country = ip_info.get("country", "").upper() + if ip_info: + actual_country = ip_info.get("country", "").upper() - # Check if country matches (if we have an expected country) - # ipinfo.io returns country codes (CA), but we may have full names (Canada) - # Normalize both to country codes for comparison using shared utility - if expected_country: - # Convert expected country name to code if it's a full name - expected_code = get_country_code(expected_country) or expected_country - expected_code = expected_code.upper() + # Check if country matches (if we have an expected country) + # ipinfo.io returns country codes (CA), but we may have full names (Canada) + # Normalize both to country codes for comparison using shared utility + if expected_country: + # Convert expected country name to code if it's a full name + expected_code = get_country_code(expected_country) or expected_country + expected_code = expected_code.upper() - if actual_country != expected_code: - duration_ms = (time.time() - start_time) * 1000 - if debug_logger: - debug_logger.log( - level="ERROR", - operation="gluetun_verify_mismatch", - message=f"Region mismatch for {query_key}", - context={ - "query_key": query_key, - "expected_country": expected_code, - "actual_country": actual_country, - "ip": ip_info.get("ip"), - "city": ip_info.get("city"), - "org": ip_info.get("org"), - }, - success=False, - duration_ms=duration_ms, - ) - raise RuntimeError( + if actual_country != expected_code: + duration_ms = (time.time() - start_time) * 1000 + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_verify_mismatch", + message=f"Region mismatch for {query_key}", + context={ + "query_key": query_key, + "expected_country": expected_code, + "actual_country": actual_country, + "ip": ip_info.get("ip"), + "city": ip_info.get("city"), + "org": ip_info.get("org"), + }, + success=False, + duration_ms=duration_ms, + ) + raise RuntimeError( f"Region mismatch for {container['provider']}:{container['region']}: " f"Expected '{expected_code}' but got '{actual_country}' " f"(IP: {ip_info.get('ip')}, City: {ip_info.get('city')})" ) - # Verification successful - store IP info in container record - if query_key in self.active_containers: - self.active_containers[query_key]["public_ip"] = ip_info.get("ip") - self.active_containers[query_key]["ip_country"] = actual_country - self.active_containers[query_key]["ip_city"] = ip_info.get("city") - self.active_containers[query_key]["ip_org"] = ip_info.get("org") + # Verification successful - store IP info in container record + if query_key in self.active_containers: + self.active_containers[query_key]["public_ip"] = ip_info.get("ip") + self.active_containers[query_key]["ip_country"] = actual_country + self.active_containers[query_key]["ip_city"] = ip_info.get("city") + self.active_containers[query_key]["ip_org"] = ip_info.get("org") - duration_ms = (time.time() - start_time) * 1000 + duration_ms = (time.time() - start_time) * 1000 + if debug_logger: + debug_logger.log( + level="INFO", + operation="gluetun_verify_success", + message=f"VPN IP verified for: {query_key}", + context={ + "query_key": query_key, + "ip": ip_info.get("ip"), + "country": actual_country, + "city": ip_info.get("city"), + "org": ip_info.get("org"), + "attempts": attempt + 1, + }, + success=True, + duration_ms=duration_ms, + ) + return + + # ip_info was None, retry + last_error = "Failed to get IP info from ipinfo.io" + + except RuntimeError: + raise # Re-raise region mismatch errors immediately + except Exception as e: + last_error = str(e) if debug_logger: debug_logger.log( - level="INFO", - operation="gluetun_verify_success", - message=f"VPN IP verified for: {query_key}", + level="DEBUG", + operation="gluetun_verify_retry", + message=f"Verification attempt {attempt + 1} failed, retrying", context={ "query_key": query_key, - "ip": ip_info.get("ip"), - "country": actual_country, - "city": ip_info.get("city"), - "org": ip_info.get("org"), - "attempts": attempt + 1, + "attempt": attempt + 1, + "error": last_error, }, - success=True, - duration_ms=duration_ms, ) - return - # ip_info was None, retry - last_error = "Failed to get IP info from ipinfo.io" - - except RuntimeError: - raise # Re-raise region mismatch errors immediately - except Exception as e: - last_error = str(e) - if debug_logger: - debug_logger.log( - level="DEBUG", - operation="gluetun_verify_retry", - message=f"Verification attempt {attempt + 1} failed, retrying", - context={ - "query_key": query_key, - "attempt": attempt + 1, - "error": last_error, - }, - ) - - # Wait before retry (exponential backoff) - if attempt < max_retries - 1: - wait_time = 2**attempt # 1, 2, 4 seconds - time.sleep(wait_time) + # Wait before retry (exponential backoff) + if attempt < max_retries - 1: + wait_time = 2**attempt # 1, 2, 4 seconds + time.sleep(wait_time) + finally: + try: + session.close() + except Exception: + pass # All retries exhausted duration_ms = (time.time() - start_time) * 1000 diff --git a/unshackle/core/proxies/surfsharkvpn.py b/unshackle/core/proxies/surfsharkvpn.py index 491906d..c1988f3 100644 --- a/unshackle/core/proxies/surfsharkvpn.py +++ b/unshackle/core/proxies/surfsharkvpn.py @@ -142,12 +142,17 @@ class SurfsharkVPN(Proxy): ) # Get connection names from filtered servers - connection_names = [x["connectionName"] for x in servers] + if not servers: + raise ValueError(f"Could not get random server for country '{country_id}': no servers found.") - try: - return random.choice(connection_names) - except (IndexError, KeyError): - raise ValueError(f"Could not get random server for country '{country_id}'.") + # Only include servers that actually have a connection name to avoid KeyError. + connection_names = [x["connectionName"] for x in servers if "connectionName" in x] + if not connection_names: + raise ValueError( + f"Could not get random server for country '{country_id}': no servers with connectionName found." + ) + + return random.choice(connection_names) @staticmethod def get_countries() -> list[dict]: diff --git a/unshackle/core/proxies/windscribevpn.py b/unshackle/core/proxies/windscribevpn.py index 25d5af0..2233cca 100644 --- a/unshackle/core/proxies/windscribevpn.py +++ b/unshackle/core/proxies/windscribevpn.py @@ -47,6 +47,7 @@ class WindscribeVPN(Proxy): Supports: - Country code: "us", "ca", "gb" + - Specific server: "sg007", "us150" - City selection: "us:seattle", "ca:toronto" """ query = query.lower() @@ -61,10 +62,20 @@ class WindscribeVPN(Proxy): server_map_key = f"{query}:{city}" if city else query if server_map_key in self.server_map: hostname = self.server_map[server_map_key] - elif query in self.server_map and not city: + elif query in self.server_map: hostname = self.server_map[query] else: - if re.match(r"^[a-z]+$", query): + server_match = re.match(r"^([a-z]{2})(\d+)$", query) + if server_match: + # Specific server selection, e.g., sg007, us150 + country_code, server_num = server_match.groups() + hostname = self.get_specific_server(country_code, server_num) + if not hostname: + raise ValueError( + f"No WindscribeVPN server found matching '{query}'. " + f"Check the server number or use just '{country_code}' for a random server." + ) + elif re.match(r"^[a-z]+$", query): hostname = self.get_random_server(query, city) else: raise ValueError(f"The query provided is unsupported and unrecognized: {query}") @@ -75,6 +86,38 @@ class WindscribeVPN(Proxy): hostname = hostname.split(':')[0] return f"https://{self.username}:{self.password}@{hostname}:443" + def get_specific_server(self, country_code: str, server_num: str) -> Optional[str]: + """ + Find a specific server by country code and server number. + + Matches against hostnames like "sg-007.totallyacdn.com" for query "sg007". + Tries both the raw number and zero-padded variants. + + Args: + country_code: Two-letter country code (e.g., "sg", "us") + server_num: Server number as string (e.g., "007", "7", "150") + + Returns: + The matching hostname, or None if not found. + """ + num_stripped = server_num.lstrip("0") or "0" + candidates = { + f"{country_code}-{server_num}.", + f"{country_code}-{num_stripped}.", + f"{country_code}-{server_num.zfill(3)}.", + } + + for location in self.countries: + if location.get("country_code", "").lower() != country_code: + continue + for group in location.get("groups", []): + for host in group.get("hosts", []): + hostname = host.get("hostname", "") + if any(hostname.startswith(prefix) for prefix in candidates): + return hostname + + return None + def get_random_server(self, country_code: str, city: Optional[str] = None) -> Optional[str]: """ Get a random server hostname for a country, optionally filtered by city. diff --git a/unshackle/core/service.py b/unshackle/core/service.py index d39cb55..ae32797 100644 --- a/unshackle/core/service.py +++ b/unshackle/core/service.py @@ -1,11 +1,12 @@ import base64 import logging from abc import ABCMeta, abstractmethod -from collections.abc import Generator +from collections.abc import Callable, Generator +from dataclasses import dataclass, field from http.cookiejar import CookieJar from pathlib import Path from typing import Optional, Union -from urllib.parse import urlparse +from urllib.parse import urlparse, urlunparse import click import m3u8 @@ -24,9 +25,65 @@ from unshackle.core.search_result import SearchResult from unshackle.core.title_cacher import TitleCacher, get_account_hash, get_region_from_proxy from unshackle.core.titles import Title_T, Titles_T from unshackle.core.tracks import Chapters, Tracks +from unshackle.core.tracks.video import Video from unshackle.core.utilities import get_cached_ip_info, get_ip_info +@dataclass +class TrackRequest: + """Holds what the user requested for video codec and range selection. + + Services read from this instead of ctx.parent.params for vcodec/range. + + Attributes: + codecs: Requested codecs from CLI. Empty list means no filter (accept any). + ranges: Requested ranges from CLI. Defaults to [SDR]. + """ + + codecs: list[Video.Codec] = field(default_factory=list) + ranges: list[Video.Range] = field(default_factory=lambda: [Video.Range.SDR]) + best_available: bool = False + + +def sanitize_proxy_for_log(uri: Optional[str]) -> Optional[str]: + """ + Sanitize a proxy URI for logs by redacting any embedded userinfo (username/password). + + Examples: + - http://user:pass@host:8080 -> http://REDACTED@host:8080 + - socks5h://user@host:1080 -> socks5h://REDACTED@host:1080 + """ + if uri is None: + return None + if not isinstance(uri, str): + return str(uri) + if not uri: + return uri + + try: + parsed = urlparse(uri) + + # Handle schemeless proxies like "user:pass@host:port" + if not parsed.scheme and not parsed.netloc and "@" in uri and "://" not in uri: + # Parse as netloc using a dummy scheme, then strip scheme back out. + dummy = urlparse(f"http://{uri}") + netloc = dummy.netloc + if "@" in netloc: + netloc = f"REDACTED@{netloc.split('@', 1)[1]}" + # urlparse("http://...") sets path to "" for typical netloc-only strings; keep it just in case. + return f"{netloc}{dummy.path}" + + netloc = parsed.netloc + if "@" in netloc: + netloc = f"REDACTED@{netloc.split('@', 1)[1]}" + + return urlunparse(parsed._replace(netloc=netloc)) + except Exception: + if "@" in uri: + return f"REDACTED@{uri.split('@', 1)[1]}" + return uri + + class Service(metaclass=ABCMeta): """The Service Base Class.""" @@ -50,6 +107,16 @@ class Service(metaclass=ABCMeta): self.credential = None # Will be set in authenticate() self.current_region = None # Will be set based on proxy/geolocation + # Set track request from CLI params - services can read/override in their __init__ + vcodec = ctx.parent.params.get("vcodec") if ctx.parent else None + range_ = ctx.parent.params.get("range_") if ctx.parent else None + best_available = ctx.parent.params.get("best_available", False) if ctx.parent else False + self.track_request = TrackRequest( + codecs=list(vcodec) if vcodec else [], + ranges=list(range_) if range_ else [Video.Range.SDR], + best_available=bool(best_available), + ) + if not ctx.parent or not ctx.parent.params.get("no_proxy"): if ctx.parent: proxy = ctx.parent.params["proxy"] @@ -75,7 +142,9 @@ class Service(metaclass=ABCMeta): # Check if there's a mapping for this query mapped_value = proxy_map.get(full_proxy_key) if mapped_value: - self.log.info(f"Found service-specific proxy mapping: {full_proxy_key} -> {mapped_value}") + self.log.info( + f"Found service-specific proxy mapping: {full_proxy_key} -> {sanitize_proxy_for_log(mapped_value)}" + ) # Query the proxy provider with the mapped value if proxy_provider_name: # Specific provider requested @@ -87,9 +156,13 @@ class Service(metaclass=ABCMeta): mapped_proxy_uri = proxy_provider.get_proxy(mapped_value) if mapped_proxy_uri: proxy = mapped_proxy_uri - self.log.info(f"Using mapped proxy from {proxy_provider.__class__.__name__}: {proxy}") + self.log.info( + f"Using mapped proxy from {proxy_provider.__class__.__name__}: {sanitize_proxy_for_log(proxy)}" + ) else: - self.log.warning(f"Failed to get proxy for mapped value '{mapped_value}', using default") + self.log.warning( + f"Failed to get proxy for mapped value '{sanitize_proxy_for_log(mapped_value)}', using default" + ) else: self.log.warning(f"Proxy provider '{proxy_provider_name}' not found, using default proxy") else: @@ -98,10 +171,14 @@ class Service(metaclass=ABCMeta): mapped_proxy_uri = proxy_provider.get_proxy(mapped_value) if mapped_proxy_uri: proxy = mapped_proxy_uri - self.log.info(f"Using mapped proxy from {proxy_provider.__class__.__name__}: {proxy}") + self.log.info( + f"Using mapped proxy from {proxy_provider.__class__.__name__}: {sanitize_proxy_for_log(proxy)}" + ) break else: - self.log.warning(f"No provider could resolve mapped value '{mapped_value}', using default") + self.log.warning( + f"No provider could resolve mapped value '{sanitize_proxy_for_log(mapped_value)}', using default" + ) if not proxy: # don't override the explicit proxy set by the user, even if they may be geoblocked @@ -156,6 +233,76 @@ class Service(metaclass=ABCMeta): self.log.debug(f"Failed to get cached IP info: {e}") self.current_region = None + def _get_tracks_for_variants( + self, + title: Title_T, + fetch_fn: Callable[..., Tracks], + ) -> Tracks: + """Call fetch_fn for each codec/range combo in track_request, merge results. + + Services that need separate API calls per codec/range combo can use this + helper from their get_tracks() implementation. + + The fetch_fn signature should be: (title, codec, range_) -> Tracks + + For HYBRID range, fetch_fn is called with HDR10 and DV separately and + the DV video tracks are merged into the HDR10 result. + + Args: + title: The title being processed. + fetch_fn: A callable that fetches tracks for a specific codec/range. + """ + all_tracks = Tracks() + first = True + + codecs = self.track_request.codecs or [None] + ranges = self.track_request.ranges or [Video.Range.SDR] + + for range_val in ranges: + if range_val == Video.Range.HYBRID: + # HYBRID: fetch HDR10 first (full tracks), then DV (video only) + for codec_val in codecs: + try: + hdr_tracks = fetch_fn(title, codec=codec_val, range_=Video.Range.HDR10) + except (ValueError, SystemExit) as e: + if self.track_request.best_available: + self.log.warning(f" - HDR10 not available for HYBRID, skipping ({e})") + continue + raise + if first: + all_tracks.add(hdr_tracks, warn_only=True) + first = False + else: + for video in hdr_tracks.videos: + all_tracks.add(video, warn_only=True) + + try: + dv_tracks = fetch_fn(title, codec=codec_val, range_=Video.Range.DV) + for video in dv_tracks.videos: + all_tracks.add(video, warn_only=True) + except (ValueError, SystemExit): + self.log.info(" - No DolbyVision manifest available for HYBRID") + else: + for codec_val in codecs: + try: + tracks = fetch_fn(title, codec=codec_val, range_=range_val) + except (ValueError, SystemExit) as e: + if self.track_request.best_available: + codec_name = codec_val.name if codec_val else "default" + self.log.warning( + f" - {range_val.name}/{codec_name} not available, skipping ({e})" + ) + continue + raise + if first: + all_tracks.add(tracks, warn_only=True) + first = False + else: + for video in tracks.videos: + all_tracks.add(video, warn_only=True) + + return all_tracks + # Optional Abstract functions # The following functions may be implemented by the Service. # Otherwise, the base service code (if any) of the function will be executed on call. @@ -173,7 +320,7 @@ class Service(metaclass=ABCMeta): session.mount( "https://", HTTPAdapter( - max_retries=Retry(total=15, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504]), + max_retries=Retry(total=5, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504]), pool_block=True, ), ) @@ -412,4 +559,4 @@ class Service(metaclass=ABCMeta): """ -__all__ = ("Service",) +__all__ = ("Service", "TrackRequest") diff --git a/unshackle/core/session.py b/unshackle/core/session.py index 3a4f704..051dce4 100644 --- a/unshackle/core/session.py +++ b/unshackle/core/session.py @@ -56,7 +56,7 @@ class MaxRetriesError(exceptions.RequestException): class CurlSession(Session): def __init__( self, - max_retries: int = 10, + max_retries: int = 5, backoff_factor: float = 0.2, max_backoff: float = 60.0, status_forcelist: list[int] | None = None, @@ -150,7 +150,7 @@ def session( browser: Browser to impersonate (e.g. "chrome124", "firefox", "safari") OR fingerprint preset name (e.g. "okhttp4"). Uses the configured default from curl_impersonate.browser if not specified. - Available presets: okhttp4 + Available presets: okhttp4, okhttp5 See https://github.com/lexiforest/curl_cffi#sessions for browser options. ja3: Custom JA3 TLS fingerprint string (format: "SSLVersion,Ciphers,Extensions,Curves,PointFormats"). When provided, curl_cffi will use this exact TLS fingerprint instead of the browser's default. @@ -172,7 +172,7 @@ def session( - cert: Client certificate (str or tuple) Extra arguments for retry handler: - - max_retries: Maximum number of retries (int, default 10) + - max_retries: Maximum number of retries (int, default 5) - backoff_factor: Backoff factor (float, default 0.2) - max_backoff: Maximum backoff time (float, default 60.0) - status_forcelist: List of status codes to force retry (list, default [429, 500, 502, 503, 504]) diff --git a/unshackle/core/titles/episode.py b/unshackle/core/titles/episode.py index be6cc15..869bfb8 100644 --- a/unshackle/core/titles/episode.py +++ b/unshackle/core/titles/episode.py @@ -286,4 +286,4 @@ class Series(SortedKeyList, ABC): return tree -__all__ = ("Episode", "Series") +__all__ = ("Episode", "Series") \ No newline at end of file diff --git a/unshackle/core/titles/movie.py b/unshackle/core/titles/movie.py index 2346a82..d14df3e 100644 --- a/unshackle/core/titles/movie.py +++ b/unshackle/core/titles/movie.py @@ -67,109 +67,122 @@ class Movie(Title): primary_audio_track = sorted_audio[0] unique_audio_languages = len({x.language.split("-")[0] for x in media_info.audio_tracks if x.language}) + def _get_resolution_token(track: Any) -> str: + if not track or not getattr(track, "height", None): + return "" + resolution = track.height + try: + dar = getattr(track, "other_display_aspect_ratio", None) or [] + if dar and dar[0]: + aspect_ratio = [int(float(plane)) for plane in str(dar[0]).split(":")] + if len(aspect_ratio) == 1: + aspect_ratio.append(1) + if aspect_ratio[0] / aspect_ratio[1] not in (16 / 9, 4 / 3): + resolution = int(track.width * (9 / 16)) + except Exception: + pass + + scan_suffix = "p" + scan_type = getattr(track, "scan_type", None) + if scan_type and str(scan_type).lower() == "interlaced": + scan_suffix = "i" + return f"{resolution}{scan_suffix}" + # Name (Year) name = str(self).replace("$", "S") # e.g., Arli$$ - if config.scene_naming: - # Resolution - if primary_video_track: - resolution = primary_video_track.height - aspect_ratio = [ - int(float(plane)) for plane in primary_video_track.other_display_aspect_ratio[0].split(":") - ] - if len(aspect_ratio) == 1: - # e.g., aspect ratio of 2 (2.00:1) would end up as `(2.0,)`, add 1 - aspect_ratio.append(1) - if aspect_ratio[0] / aspect_ratio[1] not in (16 / 9, 4 / 3): - # We want the resolution represented in a 4:3 or 16:9 canvas. - # If it's not 4:3 or 16:9, calculate as if it's inside a 16:9 canvas, - # otherwise the track's height value is fine. - # We are assuming this title is some weird aspect ratio so most - # likely a movie or HD source, so it's most likely widescreen so - # 16:9 canvas makes the most sense. - resolution = int(primary_video_track.width * (9 / 16)) - # Determine scan type suffix - default to "p", use "i" only if explicitly interlaced - scan_suffix = "p" - scan_type = getattr(primary_video_track, 'scan_type', None) - if scan_type and str(scan_type).lower() == "interlaced": - scan_suffix = "i" - name += f" {resolution}{scan_suffix}" + if getattr(config, "repack", False): + name += " REPACK" - # Service (use track source if available) - if show_service: - source_name = None - if self.tracks: - first_track = next(iter(self.tracks), None) - if first_track and hasattr(first_track, "source") and first_track.source: - source_name = first_track.source - name += f" {source_name or self.service.__name__}" + if primary_video_track: + resolution_token = _get_resolution_token(primary_video_track) + if resolution_token: + name += f" {resolution_token}" - # 'WEB-DL' - name += " WEB-DL" + # Service (use track source if available) + if show_service: + source_name = None + if self.tracks: + first_track = next(iter(self.tracks), None) + if first_track and hasattr(first_track, "source") and first_track.source: + source_name = first_track.source + name += f" {source_name or self.service.__name__}" - # DUAL - if unique_audio_languages == 2: - name += " DUAL" + # 'WEB-DL' + name += " WEB-DL" - # MULTi - if unique_audio_languages > 2: - name += " MULTi" + # DUAL + if unique_audio_languages == 2: + name += " DUAL" - # Audio Codec + Channels (+ feature) - if primary_audio_track: - codec = primary_audio_track.format - channel_layout = primary_audio_track.channel_layout or primary_audio_track.channellayout_original - if channel_layout: - channels = float( - sum({"LFE": 0.1}.get(position.upper(), 1) for position in channel_layout.split(" ")) - ) - else: - channel_count = primary_audio_track.channel_s or primary_audio_track.channels or 0 - channels = float(channel_count) + # MULTi + if unique_audio_languages > 2: + name += " MULTi" - features = primary_audio_track.format_additionalfeatures or "" - name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}" - if "JOC" in features or primary_audio_track.joc: - name += " Atmos" - - # Video (dynamic range + hfr +) Codec - if primary_video_track: - codec = primary_video_track.format - hdr_format = primary_video_track.hdr_format_commercial - hdr_format_full = primary_video_track.hdr_format or "" - trc = ( - primary_video_track.transfer_characteristics - or primary_video_track.transfer_characteristics_original - or "" + # Audio Codec + Channels (+ feature) + if primary_audio_track: + codec = primary_audio_track.format + channel_layout = primary_audio_track.channel_layout or primary_audio_track.channellayout_original + if channel_layout: + channels = float( + sum({"LFE": 0.1}.get(position.upper(), 1) for position in channel_layout.split(" ")) ) - frame_rate = float(primary_video_track.frame_rate) + else: + channel_count = primary_audio_track.channel_s or primary_audio_track.channels or 0 + channels = float(channel_count) - # Primary HDR format detection - if hdr_format: - if hdr_format_full.startswith("Dolby Vision"): - name += " DV" - if any( - indicator in (hdr_format_full + " " + hdr_format) - for indicator in ["HDR10", "SMPTE ST 2086"] - ): - name += " HDR" - else: - name += f" {DYNAMIC_RANGE_MAP.get(hdr_format)} " - elif "HLG" in trc or "Hybrid Log-Gamma" in trc or "ARIB STD-B67" in trc or "arib-std-b67" in trc.lower(): - name += " HLG" - elif any(indicator in trc for indicator in ["PQ", "SMPTE ST 2084", "BT.2100"]) or "smpte2084" in trc.lower() or "bt.2020-10" in trc.lower(): - name += " HDR" - if frame_rate > 30: - name += " HFR" - name += f" {VIDEO_CODEC_MAP.get(codec, codec)}" + features = primary_audio_track.format_additionalfeatures or "" + name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}" + if "JOC" in features or primary_audio_track.joc: + name += " Atmos" - if config.tag: - name += f"-{config.tag}" + # Video (dynamic range + hfr +) Codec + if primary_video_track: + codec = primary_video_track.format + hdr_format = primary_video_track.hdr_format_commercial + hdr_format_full = primary_video_track.hdr_format or "" + trc = ( + primary_video_track.transfer_characteristics + or primary_video_track.transfer_characteristics_original + or "" + ) + frame_rate = float(primary_video_track.frame_rate) - return sanitize_filename(name) - else: - # Simple naming style without technical details - use spaces instead of dots - return sanitize_filename(name, " ") + def _append_token(current: str, token: Optional[str]) -> str: + token = (token or "").strip() + current = current.rstrip() + if not token: + return current + if current.endswith(f" {token}"): + return current + return f"{current} {token}" + + # Primary HDR format detection + if hdr_format: + if hdr_format_full.startswith("Dolby Vision"): + name = _append_token(name, "DV") + if any( + indicator in (hdr_format_full + " " + hdr_format) + for indicator in ["HDR10", "SMPTE ST 2086"] + ): + name = _append_token(name, "HDR") + elif "HDR Vivid" in hdr_format: + name = _append_token(name, "HDR") + else: + dynamic_range = DYNAMIC_RANGE_MAP.get(hdr_format) or hdr_format or "" + name = _append_token(name, dynamic_range) + elif "HLG" in trc or "Hybrid Log-Gamma" in trc or "ARIB STD-B67" in trc or "arib-std-b67" in trc.lower(): + name += " HLG" + elif any(indicator in trc for indicator in ["PQ", "SMPTE ST 2084", "BT.2100"]) or "smpte2084" in trc.lower() or "bt.2020-10" in trc.lower(): + name += " HDR" + if frame_rate > 30: + name += " HFR" + name += f" {VIDEO_CODEC_MAP.get(codec, codec)}" + + if config.tag: + name += f"-{config.tag}" + + return sanitize_filename(name, "." if config.scene_naming else " ") class Movies(SortedKeyList, ABC): diff --git a/unshackle/core/titles/song.py b/unshackle/core/titles/song.py index 303b336..580e33e 100644 --- a/unshackle/core/titles/song.py +++ b/unshackle/core/titles/song.py @@ -100,31 +100,30 @@ class Song(Title): # NN. Song Name name = str(self).split(" / ")[1] - if config.scene_naming: - # Service (use track source if available) - if show_service: - source_name = None - if self.tracks: - first_track = next(iter(self.tracks), None) - if first_track and hasattr(first_track, "source") and first_track.source: - source_name = first_track.source - name += f" {source_name or self.service.__name__}" + if getattr(config, "repack", False): + name += " REPACK" - # 'WEB-DL' - name += " WEB-DL" + # Service (use track source if available) + if show_service: + source_name = None + if self.tracks: + first_track = next(iter(self.tracks), None) + if first_track and hasattr(first_track, "source") and first_track.source: + source_name = first_track.source + name += f" {source_name or self.service.__name__}" - # Audio Codec + Channels (+ feature) - name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}" - if "JOC" in features or audio_track.joc: - name += " Atmos" + # 'WEB-DL' + name += " WEB-DL" - if config.tag: - name += f"-{config.tag}" + # Audio Codec + Channels (+ feature) + name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}" + if "JOC" in features or audio_track.joc: + name += " Atmos" - return sanitize_filename(name, " ") - else: - # Simple naming style without technical details - return sanitize_filename(name, " ") + if config.tag: + name += f"-{config.tag}" + + return sanitize_filename(name, " ") class Album(SortedKeyList, ABC): diff --git a/unshackle/core/tracks/attachment.py b/unshackle/core/tracks/attachment.py index 3ca6260..01f6acc 100644 --- a/unshackle/core/tracks/attachment.py +++ b/unshackle/core/tracks/attachment.py @@ -2,6 +2,7 @@ from __future__ import annotations import mimetypes import os +import re from pathlib import Path from typing import Optional, Union from urllib.parse import urlparse @@ -10,6 +11,7 @@ from zlib import crc32 import requests from unshackle.core.config import config +from unshackle.core.constants import DOWNLOAD_LICENCE_ONLY class Attachment: @@ -43,6 +45,8 @@ class Attachment: if path is None and url is None: raise ValueError("Either path or url must be provided.") + self.url = url + if url: if not isinstance(url, str): raise ValueError("The attachment URL must be a string.") @@ -53,45 +57,60 @@ class Attachment: # Use provided name for the file if available if name: - file_name = f"{name.replace(' ', '_')}{os.path.splitext(file_name)[1]}" + safe_name = re.sub(r'[<>:"/\\|?*]', "", name).replace(" ", "_") + file_name = f"{safe_name}{os.path.splitext(file_name)[1]}" download_path = config.directories.temp / file_name - # Download the file - try: - session = session or requests.Session() - response = session.get(url, stream=True) - response.raise_for_status() - config.directories.temp.mkdir(parents=True, exist_ok=True) - download_path.parent.mkdir(parents=True, exist_ok=True) + # Download the file unless we're in license-only mode + if DOWNLOAD_LICENCE_ONLY.is_set(): + path = None + else: + try: + if session is None: + with requests.Session() as session: + response = session.get(url, stream=True) + response.raise_for_status() + else: + response = session.get(url, stream=True) + response.raise_for_status() + config.directories.temp.mkdir(parents=True, exist_ok=True) + download_path.parent.mkdir(parents=True, exist_ok=True) - with open(download_path, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) + with open(download_path, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) - path = download_path - except Exception as e: - raise ValueError(f"Failed to download attachment from URL: {e}") + path = download_path + except Exception as e: + raise ValueError(f"Failed to download attachment from URL: {e}") - if not isinstance(path, (str, Path)): - raise ValueError("The attachment path must be provided.") + if path is not None and not isinstance(path, (str, Path)): + raise ValueError( + f"Invalid attachment path type: expected str or Path, got {type(path).__name__}." + ) - path = Path(path) - if not path.exists(): - raise ValueError("The attachment file does not exist.") + if path is not None: + path = Path(path) + if not path.exists(): + raise ValueError("The attachment file does not exist.") - name = (name or path.stem).strip() + if path is not None: + name = (name or path.stem).strip() + else: + name = (name or Path(file_name).stem).strip() mime_type = (mime_type or "").strip() or None description = (description or "").strip() or None if not mime_type: + suffix = path.suffix.lower() if path is not None else Path(file_name).suffix.lower() mime_type = { ".ttf": "application/x-truetype-font", ".otf": "application/vnd.ms-opentype", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png", - }.get(path.suffix.lower(), mimetypes.guess_type(path)[0]) + }.get(suffix, mimetypes.guess_type(file_name if path is None else path)[0]) if not mime_type: raise ValueError("The attachment mime-type could not be automatically detected.") @@ -111,13 +130,18 @@ class Attachment: @property def id(self) -> str: """Compute an ID from the attachment data.""" - checksum = crc32(self.path.read_bytes()) + if self.path and self.path.exists(): + checksum = crc32(self.path.read_bytes()) + elif self.url: + checksum = crc32(self.url.encode("utf8")) + else: + checksum = crc32(self.name.encode("utf8")) return hex(checksum) def delete(self) -> None: - if self.path: + if self.path and self.path.exists(): self.path.unlink() - self.path = None + self.path = None @classmethod def from_url( diff --git a/unshackle/core/tracks/hybrid.py b/unshackle/core/tracks/hybrid.py index 316db97..ecd44b6 100644 --- a/unshackle/core/tracks/hybrid.py +++ b/unshackle/core/tracks/hybrid.py @@ -1,6 +1,8 @@ import json import logging import os +import random +import re import subprocess import sys from pathlib import Path @@ -8,14 +10,16 @@ from pathlib import Path from rich.padding import Padding from rich.rule import Rule -from unshackle.core.binaries import FFMPEG, DoviTool, HDR10PlusTool +from unshackle.core.binaries import FFMPEG, DoviTool, FFProbe, HDR10PlusTool from unshackle.core.config import config from unshackle.core.console import console +from unshackle.core.utilities import get_debug_logger class Hybrid: def __init__(self, videos, source) -> None: self.log = logging.getLogger("hybrid") + self.debug_logger = get_debug_logger() """ Takes the Dolby Vision and HDR10(+) streams out of the VideoTracks. @@ -41,6 +45,19 @@ class Hybrid: console.print(Padding(Rule(f"[rule.text]HDR10+DV Hybrid ({self.resolution})"), (1, 2))) + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_init", + message="Starting HDR10+DV hybrid processing", + context={ + "source": source, + "resolution": self.resolution, + "video_count": len(videos), + "video_ranges": [str(v.range) for v in videos], + }, + ) + for video in self.videos: if not video.path or not os.path.exists(video.path): raise ValueError(f"Video track {video.id} was not downloaded before injection.") @@ -50,18 +67,18 @@ class Hybrid: has_hdr10 = any(video.range == Video.Range.HDR10 for video in self.videos) has_hdr10p = any(video.range == Video.Range.HDR10P for video in self.videos) - if not has_hdr10: - raise ValueError("No HDR10 track available for hybrid processing.") + if not has_hdr10 and not has_hdr10p: + raise ValueError("No HDR10 or HDR10+ track available for hybrid processing.") # If we have HDR10+ but no DV, we can convert HDR10+ to DV if not has_dv and has_hdr10p: - self.log.info("✓ No DV track found, but HDR10+ is available. Will convert HDR10+ to DV.") + console.status("No DV track found, but HDR10+ is available. Will convert HDR10+ to DV.") self.hdr10plus_to_dv = True elif not has_dv: raise ValueError("No DV track available and no HDR10+ to convert.") if os.path.isfile(config.directories.temp / self.hevc_file): - self.log.info("✓ Already Injected") + console.status("Already Injected") return for video in videos: @@ -89,14 +106,34 @@ class Hybrid: self.extract_rpu(dv_video) if os.path.isfile(config.directories.temp / "RPU_UNT.bin"): self.rpu_file = "RPU_UNT.bin" - self.level_6() # Mode 3 conversion already done during extraction when not untouched elif os.path.isfile(config.directories.temp / "RPU.bin"): # RPU already extracted with mode 3 pass + # Edit L6 with actual luminance values from RPU, then L5 active area + self.level_6() + base_video = next( + (v for v in videos if v.range in (Video.Range.HDR10, Video.Range.HDR10P)), None + ) + if base_video and base_video.path: + self.level_5(base_video.path) + self.injecting() + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="hybrid_complete", + message="Injection Completed", + context={ + "hdr_type": self.hdr_type, + "resolution": self.resolution, + "hdr10plus_to_dv": self.hdr10plus_to_dv, + "rpu_file": self.rpu_file, + "output_file": self.hevc_file, + }, + ) self.log.info("✓ Injection Completed") if self.source == ("itunes" or "appletvplus"): Path.unlink(config.directories.temp / "hdr10.mkv") @@ -104,6 +141,10 @@ class Hybrid: Path.unlink(config.directories.temp / "HDR10.hevc", missing_ok=True) Path.unlink(config.directories.temp / "DV.hevc", missing_ok=True) Path.unlink(config.directories.temp / f"{self.rpu_file}", missing_ok=True) + Path.unlink(config.directories.temp / "RPU_L6.bin", missing_ok=True) + Path.unlink(config.directories.temp / "RPU_L5.bin", missing_ok=True) + Path.unlink(config.directories.temp / "L5.json", missing_ok=True) + Path.unlink(config.directories.temp / "L6.json", missing_ok=True) def ffmpeg_simple(self, save_path, output): """Simple ffmpeg execution without progress tracking""" @@ -121,20 +162,41 @@ class Hybrid: stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - return p.returncode + return p def extract_stream(self, save_path, type_): output = Path(config.directories.temp / f"{type_}.hevc") with console.status(f"Extracting {type_} stream...", spinner="dots"): - returncode = self.ffmpeg_simple(save_path, output) + result = self.ffmpeg_simple(save_path, output) - if returncode: + if result.returncode: output.unlink(missing_ok=True) + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_extract_stream", + message=f"Failed extracting {type_} stream", + context={ + "type": type_, + "input": str(save_path), + "output": str(output), + "returncode": result.returncode, + "stderr": (result.stderr or b"").decode(errors="replace"), + "stdout": (result.stdout or b"").decode(errors="replace"), + }, + ) self.log.error(f"x Failed extracting {type_} stream") sys.exit(1) - self.log.info(f"Extracted {type_} stream") + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_extract_stream", + message=f"Extracted {type_} stream", + context={"type": type_, "input": str(save_path), "output": str(output)}, + success=True, + ) def extract_rpu(self, video, untouched=False): if os.path.isfile(config.directories.temp / "RPU.bin") or os.path.isfile( @@ -161,58 +223,326 @@ class Hybrid: stderr=subprocess.PIPE, ) + rpu_name = "RPU" if not untouched else "RPU_UNT" if rpu_extraction.returncode: - Path.unlink(config.directories.temp / f"{'RPU' if not untouched else 'RPU_UNT'}.bin") + Path.unlink(config.directories.temp / f"{rpu_name}.bin") + stderr_text = rpu_extraction.stderr.decode(errors="replace") if rpu_extraction.stderr else "" + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_extract_rpu", + message=f"Failed extracting{' untouched ' if untouched else ' '}RPU", + context={ + "untouched": untouched, + "returncode": rpu_extraction.returncode, + "stderr": stderr_text, + "args": [str(a) for a in extraction_args], + }, + ) if b"MAX_PQ_LUMINANCE" in rpu_extraction.stderr: self.extract_rpu(video, untouched=True) elif b"Invalid PPS index" in rpu_extraction.stderr: raise ValueError("Dolby Vision VideoTrack seems to be corrupt") else: raise ValueError(f"Failed extracting{' untouched ' if untouched else ' '}RPU from Dolby Vision stream") + elif self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_extract_rpu", + message=f"Extracted{' untouched ' if untouched else ' '}RPU from Dolby Vision stream", + context={"untouched": untouched, "output": f"{rpu_name}.bin"}, + success=True, + ) - self.log.info(f"Extracted{' untouched ' if untouched else ' '}RPU from Dolby Vision stream") + def level_5(self, input_video): + """Generate Level 5 active area metadata via crop detection on the HDR10 stream. - def level_6(self): - """Edit RPU Level 6 values""" - with open(config.directories.temp / "L6.json", "w+") as level6_file: - level6 = { - "cm_version": "V29", - "length": 0, - "level6": { - "max_display_mastering_luminance": 1000, - "min_display_mastering_luminance": 1, - "max_content_light_level": 0, - "max_frame_average_light_level": 0, - }, - } + This resolves mismatches where DV has no black bars but HDR10 does (or vice versa) + by telling the display the correct active area. + """ + if os.path.isfile(config.directories.temp / "RPU_L5.bin"): + return - json.dump(level6, level6_file, indent=3) + ffprobe_bin = str(FFProbe) if FFProbe else "ffprobe" + ffmpeg_bin = str(FFMPEG) if FFMPEG else "ffmpeg" - if not os.path.isfile(config.directories.temp / "RPU_L6.bin"): - with console.status("Editing RPU Level 6 values...", spinner="dots"): - level6 = subprocess.run( + # Get video duration for random sampling + with console.status("Detecting active area (crop detection)...", spinner="dots"): + result_duration = subprocess.run( + [ffprobe_bin, "-v", "error", "-show_entries", "format=duration", "-of", "json", str(input_video)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + if result_duration.returncode != 0: + if self.debug_logger: + self.debug_logger.log( + level="WARNING", + operation="hybrid_level5", + message="Could not probe video duration", + context={"returncode": result_duration.returncode, "stderr": (result_duration.stderr or "")}, + ) + self.log.warning("Could not probe video duration, skipping L5 crop detection") + return + + duration_info = json.loads(result_duration.stdout) + duration = float(duration_info["format"]["duration"]) + + # Get video resolution for proper border calculation + result_streams = subprocess.run( + [ + ffprobe_bin, + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "stream=width,height", + "-of", + "json", + str(input_video), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + if result_streams.returncode != 0: + if self.debug_logger: + self.debug_logger.log( + level="WARNING", + operation="hybrid_level5", + message="Could not probe video resolution", + context={"returncode": result_streams.returncode, "stderr": (result_streams.stderr or "")}, + ) + self.log.warning("Could not probe video resolution, skipping L5 crop detection") + return + + stream_info = json.loads(result_streams.stdout) + original_width = int(stream_info["streams"][0]["width"]) + original_height = int(stream_info["streams"][0]["height"]) + + # Sample 10 random timestamps and run cropdetect on each + random_times = sorted(random.uniform(0, duration) for _ in range(10)) + + crop_results = [] + for t in random_times: + result_cropdetect = subprocess.run( [ - str(DoviTool), - "editor", + ffmpeg_bin, + "-y", + "-nostdin", + "-loglevel", + "info", + "-ss", + f"{t:.2f}", "-i", - config.directories.temp / self.rpu_file, - "-j", - config.directories.temp / "L6.json", - "-o", - config.directories.temp / "RPU_L6.bin", + str(input_video), + "-vf", + "cropdetect=round=2", + "-vframes", + "10", + "-f", + "null", + "-", ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, ) - if level6.returncode: - Path.unlink(config.directories.temp / "RPU_L6.bin") - raise ValueError("Failed editing RPU Level 6 values") + # cropdetect outputs crop=w:h:x:y + crop_match = re.search( + r"crop=(\d+):(\d+):(\d+):(\d+)", + (result_cropdetect.stdout or "") + (result_cropdetect.stderr or ""), + ) + if crop_match: + w, h = int(crop_match.group(1)), int(crop_match.group(2)) + x, y = int(crop_match.group(3)), int(crop_match.group(4)) + # Calculate actual border sizes from crop geometry + left = x + top = y + right = original_width - w - x + bottom = original_height - h - y + crop_results.append((left, top, right, bottom)) - self.log.info("Edited RPU Level 6 values") + if not crop_results: + if self.debug_logger: + self.debug_logger.log( + level="WARNING", + operation="hybrid_level5", + message="No crop data detected, skipping L5", + context={"samples": len(random_times)}, + ) + self.log.warning("No crop data detected, skipping L5") + return - # Update rpu_file to use the edited version - self.rpu_file = "RPU_L6.bin" + # Find the most common crop values + crop_counts = {} + for crop in crop_results: + crop_counts[crop] = crop_counts.get(crop, 0) + 1 + most_common = max(crop_counts, key=crop_counts.get) + left, top, right, bottom = most_common + + # If all borders are 0 there's nothing to correct + if left == 0 and top == 0 and right == 0 and bottom == 0: + return + + l5_json = { + "active_area": { + "crop": False, + "presets": [{"id": 0, "left": left, "right": right, "top": top, "bottom": bottom}], + "edits": {"all": 0}, + } + } + + l5_path = config.directories.temp / "L5.json" + with open(l5_path, "w") as f: + json.dump(l5_json, f, indent=4) + + with console.status("Editing RPU Level 5 active area...", spinner="dots"): + result = subprocess.run( + [ + str(DoviTool), + "editor", + "-i", + str(config.directories.temp / self.rpu_file), + "-j", + str(l5_path), + "-o", + str(config.directories.temp / "RPU_L5.bin"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + if result.returncode: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_level5", + message="Failed editing RPU Level 5 values", + context={"returncode": result.returncode, "stderr": (result.stderr or b"").decode(errors="replace")}, + ) + Path.unlink(config.directories.temp / "RPU_L5.bin", missing_ok=True) + raise ValueError("Failed editing RPU Level 5 values") + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_level5", + message="Edited RPU Level 5 active area", + context={"crop": {"left": left, "right": right, "top": top, "bottom": bottom}, "samples": len(crop_results)}, + success=True, + ) + self.rpu_file = "RPU_L5.bin" + + def level_6(self): + """Edit RPU Level 6 values using actual luminance data from the RPU.""" + if os.path.isfile(config.directories.temp / "RPU_L6.bin"): + return + + with console.status("Reading RPU luminance metadata...", spinner="dots"): + result = subprocess.run( + [str(DoviTool), "info", "-i", str(config.directories.temp / self.rpu_file), "-s"], + capture_output=True, + text=True, + ) + + if result.returncode != 0: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_level6", + message="Failed reading RPU metadata for Level 6 values", + context={"returncode": result.returncode, "stderr": (result.stderr or "")}, + ) + raise ValueError("Failed reading RPU metadata for Level 6 values") + + max_cll = None + max_fall = None + max_mdl = None + min_mdl = None + + for line in result.stdout.splitlines(): + if "RPU content light level (L1):" in line: + parts = line.split("MaxCLL:")[1].split(",") + max_cll = int(float(parts[0].strip().split()[0])) + if len(parts) > 1 and "MaxFALL:" in parts[1]: + max_fall = int(float(parts[1].split("MaxFALL:")[1].strip().split()[0])) + elif "RPU mastering display:" in line: + mastering = line.split(":", 1)[1].strip() + min_lum, max_lum = mastering.split("/")[0], mastering.split("/")[1].split(" ")[0] + min_mdl = int(float(min_lum) * 10000) + max_mdl = int(float(max_lum)) + + if any(v is None for v in (max_cll, max_fall, max_mdl, min_mdl)): + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_level6", + message="Could not extract Level 6 luminance data from RPU", + context={"max_cll": max_cll, "max_fall": max_fall, "max_mdl": max_mdl, "min_mdl": min_mdl}, + ) + raise ValueError("Could not extract Level 6 luminance data from RPU") + + level6_data = { + "level6": { + "remove_cmv4": False, + "remove_mapping": False, + "max_display_mastering_luminance": max_mdl, + "min_display_mastering_luminance": min_mdl, + "max_content_light_level": max_cll, + "max_frame_average_light_level": max_fall, + } + } + + l6_path = config.directories.temp / "L6.json" + with open(l6_path, "w") as f: + json.dump(level6_data, f, indent=4) + + with console.status("Editing RPU Level 6 values...", spinner="dots"): + result = subprocess.run( + [ + str(DoviTool), + "editor", + "-i", + str(config.directories.temp / self.rpu_file), + "-j", + str(l6_path), + "-o", + str(config.directories.temp / "RPU_L6.bin"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + if result.returncode: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_level6", + message="Failed editing RPU Level 6 values", + context={"returncode": result.returncode, "stderr": (result.stderr or b"").decode(errors="replace")}, + ) + Path.unlink(config.directories.temp / "RPU_L6.bin", missing_ok=True) + raise ValueError("Failed editing RPU Level 6 values") + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_level6", + message="Edited RPU Level 6 luminance values", + context={ + "max_cll": max_cll, + "max_fall": max_fall, + "max_mdl": max_mdl, + "min_mdl": min_mdl, + }, + success=True, + ) + self.rpu_file = "RPU_L6.bin" def injecting(self): if os.path.isfile(config.directories.temp / self.hevc_file): @@ -228,12 +558,6 @@ class Hybrid: config.directories.temp / self.rpu_file, ] - # If we converted from HDR10+, optionally remove HDR10+ metadata during injection - # Default to removing HDR10+ metadata since we're converting to DV - if self.hdr10plus_to_dv: - inject_cmd.append("--drop-hdr10plus") - self.log.info(" - Removing HDR10+ metadata during injection") - inject_cmd.extend(["-o", config.directories.temp / self.hevc_file]) inject = subprocess.run( @@ -243,10 +567,29 @@ class Hybrid: ) if inject.returncode: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_inject_rpu", + message="Failed injecting Dolby Vision metadata into HDR10 stream", + context={ + "returncode": inject.returncode, + "stderr": (inject.stderr or b"").decode(errors="replace"), + "stdout": (inject.stdout or b"").decode(errors="replace"), + "cmd": [str(a) for a in inject_cmd], + }, + ) Path.unlink(config.directories.temp / self.hevc_file) raise ValueError("Failed injecting Dolby Vision metadata into HDR10 stream") - self.log.info(f"Injected Dolby Vision metadata into {self.hdr_type} stream") + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_inject_rpu", + message=f"Injected Dolby Vision metadata into {self.hdr_type} stream", + context={"hdr_type": self.hdr_type, "rpu_file": self.rpu_file, "output": self.hevc_file, "drop_hdr10plus": self.hdr10plus_to_dv}, + success=True, + ) def extract_hdr10plus(self, _video): """Extract HDR10+ metadata from the video stream""" @@ -271,13 +614,39 @@ class Hybrid: ) if extraction.returncode: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_extract_hdr10plus", + message="Failed extracting HDR10+ metadata", + context={ + "returncode": extraction.returncode, + "stderr": (extraction.stderr or b"").decode(errors="replace"), + "stdout": (extraction.stdout or b"").decode(errors="replace"), + }, + ) raise ValueError("Failed extracting HDR10+ metadata") # Check if the extracted file has content - if os.path.getsize(config.directories.temp / self.hdr10plus_file) == 0: + file_size = os.path.getsize(config.directories.temp / self.hdr10plus_file) + if file_size == 0: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_extract_hdr10plus", + message="No HDR10+ metadata found in the stream", + context={"file_size": 0}, + ) raise ValueError("No HDR10+ metadata found in the stream") - self.log.info("Extracted HDR10+ metadata") + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_extract_hdr10plus", + message="Extracted HDR10+ metadata", + context={"output": self.hdr10plus_file, "file_size": file_size}, + success=True, + ) def convert_hdr10plus_to_dv(self): """Convert HDR10+ metadata to Dolby Vision RPU""" @@ -317,10 +686,26 @@ class Hybrid: ) if conversion.returncode: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_convert_hdr10plus", + message="Failed converting HDR10+ to Dolby Vision", + context={ + "returncode": conversion.returncode, + "stderr": (conversion.stderr or b"").decode(errors="replace"), + "stdout": (conversion.stdout or b"").decode(errors="replace"), + }, + ) raise ValueError("Failed converting HDR10+ to Dolby Vision") - self.log.info("Converted HDR10+ metadata to Dolby Vision") - self.log.info("✓ HDR10+ successfully converted to Dolby Vision Profile 8") + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_convert_hdr10plus", + message="Converted HDR10+ metadata to Dolby Vision Profile 8", + success=True, + ) # Clean up temporary files Path.unlink(config.directories.temp / "extra.json") diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 21af79d..a6beb7f 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -15,17 +15,16 @@ from zlib import crc32 from curl_cffi.requests import Session as CurlSession from langcodes import Language -from pyplayready.cdm import Cdm as PlayReadyCdm -from pywidevine.cdm import Cdm as WidevineCdm from requests import Session from unshackle.core import binaries +from unshackle.core.cdm.detect import is_playready_cdm, is_widevine_cdm from unshackle.core.config import config from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY from unshackle.core.downloaders import aria2c, curl_impersonate, n_m3u8dl_re, requests from unshackle.core.drm import DRM_T, PlayReady, Widevine from unshackle.core.events import events -from unshackle.core.utilities import get_boxes, get_extension, try_ensure_utf8 +from unshackle.core.utilities import get_boxes, try_ensure_utf8 from unshackle.core.utils.subprocess import ffprobe @@ -211,23 +210,12 @@ class Track: save_path = config.directories.temp / f"{track_type}_{self.id}.mp4" if track_type == "Subtitle": save_path = save_path.with_suffix(f".{self.codec.extension}") - # n_m3u8dl_re doesn't support directly downloading subtitles from URLs - # or when the subtitle has a direct file extension - if self.downloader.__name__ == "n_m3u8dl_re" and ( - self.descriptor == self.Descriptor.URL - or get_extension(self.url) - in { - ".srt", - ".vtt", - ".ttml", - ".ssa", - ".ass", - ".stpp", - ".wvtt", - ".xml", - } - ): - self.downloader = requests + + if self.downloader.__name__ == "n_m3u8dl_re" and ( + self.descriptor == self.Descriptor.URL + or track_type in ("Subtitle", "Attachment") + ): + self.downloader = requests if self.descriptor != self.Descriptor.URL: save_dir = save_path.with_name(save_path.name + "_segments") @@ -297,7 +285,7 @@ class Track: if not self.drm and track_type in ("Video", "Audio"): # the service might not have explicitly defined the `drm` property # try find DRM information from the init data of URL based on CDM type - if isinstance(cdm, PlayReadyCdm): + if is_playready_cdm(cdm): try: self.drm = [PlayReady.from_track(self, session)] except PlayReady.Exceptions.PSSHNotFound: @@ -451,23 +439,14 @@ class Track: if not self.drm: return None - if isinstance(cdm, WidevineCdm): + if is_widevine_cdm(cdm): for drm in self.drm: if isinstance(drm, Widevine): return drm - elif isinstance(cdm, PlayReadyCdm): + elif is_playready_cdm(cdm): for drm in self.drm: if isinstance(drm, PlayReady): return drm - elif hasattr(cdm, "is_playready"): - if cdm.is_playready: - for drm in self.drm: - if isinstance(drm, PlayReady): - return drm - else: - for drm in self.drm: - if isinstance(drm, Widevine): - return drm return self.drm[0] diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index ce41d10..e1770a5 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -95,7 +95,7 @@ class Tracks: return rep - def tree(self, add_progress: bool = False) -> tuple[Tree, list[partial]]: + def tree(self, add_progress: bool = False) -> tuple[Tree, list[Callable[..., None]]]: all_tracks = [*list(self), *self.chapters, *self.attachments] progress_callables = [] @@ -121,7 +121,29 @@ class Tracks: speed_estimate_period=10, ) task = progress.add_task("", downloaded="-") - progress_callables.append(partial(progress.update, task_id=task)) + state = {"total": 100.0} + + def update_track_progress( + task_id: int = task, + _state: dict[str, float] = state, + _progress: Progress = progress, + **kwargs, + ) -> None: + """ + Ensure terminal status states render as a fully completed bar. + + Some downloaders can report completed slightly below total + before emitting the final "Downloaded" state. + """ + if "total" in kwargs and kwargs["total"] is not None: + _state["total"] = kwargs["total"] + + downloaded_state = kwargs.get("downloaded") + if downloaded_state in {"Downloaded", "Decrypted", "[yellow]SKIPPED"}: + kwargs["completed"] = _state["total"] + _progress.update(task_id=task_id, **kwargs) + + progress_callables.append(update_track_progress) track_table = Table.grid() track_table.add_row(str(track)[6:], style="text2") track_table.add_row(progress) @@ -199,13 +221,15 @@ class Tracks: self.videos.sort(key=lambda x: not is_close_match(language, [x.language])) def sort_audio(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None: - """Sort audio tracks by bitrate, descriptive, and optionally language.""" + """Sort audio tracks by bitrate, Atmos, descriptive, and optionally language.""" if not self.audio: return - # descriptive - self.audio.sort(key=lambda x: x.descriptive) - # bitrate (within each descriptive group) + # bitrate (highest first) self.audio.sort(key=lambda x: float(x.bitrate or 0.0), reverse=True) + # Atmos tracks first (prioritize over higher bitrate non-Atmos) + self.audio.sort(key=lambda x: not x.atmos) + # descriptive tracks last + self.audio.sort(key=lambda x: x.descriptive) # language for language in reversed(by_language or []): if str(language) in ("all", "best"): @@ -254,23 +278,30 @@ class Tracks: self.subtitles = list(filter(x, self.subtitles)) def select_hybrid(self, tracks, quality): - hdr10_tracks = [ - v - for v in tracks - if v.range == Video.Range.HDR10 and (v.height in quality or int(v.width * 9 / 16) in quality) - ] - hdr10 = [] + # Prefer HDR10+ over HDR10 as the base layer (preserves dynamic metadata) + base_ranges = (Video.Range.HDR10P, Video.Range.HDR10) + base_tracks = [] + for range_type in base_ranges: + base_tracks = [ + v + for v in tracks + if v.range == range_type and (v.height in quality or int(v.width * 9 / 16) in quality) + ] + if base_tracks: + break + + base_selected = [] for res in quality: - candidates = [v for v in hdr10_tracks if v.height == res or int(v.width * 9 / 16) == res] + candidates = [v for v in base_tracks if v.height == res or int(v.width * 9 / 16) == res] if candidates: - best = max(candidates, key=lambda v: v.bitrate) # assumes .bitrate exists - hdr10.append(best) + best = max(candidates, key=lambda v: v.bitrate) + base_selected.append(best) dv_tracks = [v for v in tracks if v.range == Video.Range.DV] lowest_dv = min(dv_tracks, key=lambda v: v.height) if dv_tracks else None def select(x): - if x in hdr10: + if x in base_selected: return True if lowest_dv and x is lowest_dv: return True diff --git a/unshackle/core/utils/click_types.py b/unshackle/core/utils/click_types.py index 64cea97..890ce25 100644 --- a/unshackle/core/utils/click_types.py +++ b/unshackle/core/utils/click_types.py @@ -44,6 +44,33 @@ class VideoCodecChoice(click.Choice): self.fail(f"'{value}' is not a valid video codec", param, ctx) +class MultipleVideoCodecChoice(VideoCodecChoice): + """ + A multiple-value variant of VideoCodecChoice that accepts comma-separated codecs. + + Accepts both enum names and values, e.g.: ``-v hevc,avc`` or ``-v H.264,H.265`` + """ + + name = "multiple_video_codec_choice" + + def convert( + self, value: Any, param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None + ) -> list[Any]: + if not value: + return [] + if isinstance(value, list): + values = value + elif isinstance(value, str): + values = value.split(",") + else: + self.fail(f"{value!r} is not a supported value.", param, ctx) + + chosen_values: list[Any] = [] + for v in values: + chosen_values.append(super().convert(v.strip(), param, ctx)) + return chosen_values + + class SubtitleCodecChoice(click.Choice): """ A custom Choice type for subtitle codecs that accepts both enum names, values, and common aliases. diff --git a/unshackle/core/utils/selector.py b/unshackle/core/utils/selector.py new file mode 100644 index 0000000..b6e00a5 --- /dev/null +++ b/unshackle/core/utils/selector.py @@ -0,0 +1,310 @@ +import sys + +import click +from rich.console import Group +from rich.live import Live +from rich.padding import Padding +from rich.table import Table +from rich.text import Text + +from unshackle.core.console import console + +IS_WINDOWS = sys.platform == "win32" +if IS_WINDOWS: + import msvcrt + + +class Selector: + """ + A custom interactive selector class using the Rich library. + Allows for multi-selection of items with pagination. + """ + + def __init__( + self, + options: list[str], + cursor_style: str = "pink", + text_style: str = "text", + page_size: int = 8, + minimal_count: int = 0, + dependencies: dict[int, list[int]] = None, + prefixes: list[str] = None, + ): + """ + Initialize the Selector. + + Args: + options: List of strings to select from. + cursor_style: Rich style for the highlighted cursor item. + text_style: Rich style for normal items. + page_size: Number of items to show per page. + minimal_count: Minimum number of items that must be selected. + dependencies: Dictionary mapping parent index to list of child indices. + """ + self.options = options + self.cursor_style = cursor_style + self.text_style = text_style + self.page_size = page_size + self.minimal_count = minimal_count + self.dependencies = dependencies or {} + + self.cursor_index = 0 + self.selected_indices = set() + self.scroll_offset = 0 + + def get_renderable(self): + """ + Constructs and returns the renderable object (Table + Info) for the current state. + """ + table = Table(show_header=False, show_edge=False, box=None, pad_edge=False, padding=(0, 1, 0, 0)) + table.add_column("Indicator", justify="right", no_wrap=True) + table.add_column("Option", overflow="ellipsis", no_wrap=True) + + for i in range(self.page_size): + idx = self.scroll_offset + i + + if idx < len(self.options): + option = self.options[idx] + is_cursor = idx == self.cursor_index + is_selected = idx in self.selected_indices + + symbol = "[X]" if is_selected else "[ ]" + style = self.cursor_style if is_cursor else self.text_style + indicator_text = Text(f"{symbol}", style=style) + + content_text = Text.from_markup(option) + content_text.style = style + + table.add_row(indicator_text, content_text) + else: + table.add_row(Text(" "), Text(" ")) + + total_pages = (len(self.options) + self.page_size - 1) // self.page_size + current_page = (self.scroll_offset // self.page_size) + 1 + + info_text = Text( + f"\n[Space]: Toggle [a]: All [←/→]: Page [Enter]: Confirm (Page {current_page}/{total_pages})", + style="gray", + ) + + return Padding(Group(table, info_text), (0, 5)) + + def move_cursor(self, delta: int): + """ + Moves the cursor up or down by the specified delta. + Updates the scroll offset if the cursor moves out of the current view. + """ + self.cursor_index = (self.cursor_index + delta) % len(self.options) + new_page_idx = self.cursor_index // self.page_size + self.scroll_offset = new_page_idx * self.page_size + + def change_page(self, delta: int): + """ + Changes the current page view by the specified delta (previous/next page). + Also moves the cursor to the first item of the new page. + """ + current_page = self.scroll_offset // self.page_size + total_pages = (len(self.options) + self.page_size - 1) // self.page_size + new_page = current_page + delta + + if 0 <= new_page < total_pages: + self.scroll_offset = new_page * self.page_size + first_idx_of_page = self.scroll_offset + if first_idx_of_page < len(self.options): + self.cursor_index = first_idx_of_page + else: + self.cursor_index = len(self.options) - 1 + + def toggle_selection(self): + """ + Toggles the selection state of the item currently under the cursor. + Propagates selection to children if defined in dependencies. + """ + target_indices = {self.cursor_index} + + if self.cursor_index in self.dependencies: + target_indices.update(self.dependencies[self.cursor_index]) + + should_select = self.cursor_index not in self.selected_indices + + if should_select: + self.selected_indices.update(target_indices) + else: + self.selected_indices.difference_update(target_indices) + + def toggle_all(self): + """ + Toggles the selection of all items. + If all are selected, clears selection. Otherwise, selects all. + """ + if len(self.selected_indices) == len(self.options): + self.selected_indices.clear() + else: + self.selected_indices = set(range(len(self.options))) + + def get_input_windows(self): + """ + Captures and parses keyboard input on Windows systems using msvcrt. + Returns command strings like 'UP', 'DOWN', 'ENTER', etc. + """ + key = msvcrt.getch() + if key == b"\x03" or key == b"\x1b": + return "CANCEL" + if key == b"\xe0" or key == b"\x00": + try: + key = msvcrt.getch() + if key == b"H": + return "UP" + if key == b"P": + return "DOWN" + if key == b"K": + return "LEFT" + if key == b"M": + return "RIGHT" + except Exception: + pass + + try: + char = key.decode("utf-8", errors="ignore") + except Exception: + return None + + if char in ("\r", "\n"): + return "ENTER" + if char == " ": + return "SPACE" + if char in ("q", "Q"): + return "QUIT" + if char in ("a", "A"): + return "ALL" + if char in ("w", "W", "k", "K"): + return "UP" + if char in ("s", "S", "j", "J"): + return "DOWN" + if char in ("h", "H"): + return "LEFT" + if char in ("d", "D", "l", "L"): + return "RIGHT" + return None + + def get_input_unix(self): + """ + Captures and parses keyboard input on Unix/Linux systems using click.getchar(). + Returns command strings like 'UP', 'DOWN', 'ENTER', etc. + """ + char = click.getchar() + if char == "\x03": + return "CANCEL" + mapping = { + "\x1b[A": "UP", + "\x1b[B": "DOWN", + "\x1b[C": "RIGHT", + "\x1b[D": "LEFT", + } + if char in mapping: + return mapping[char] + if char == "\x1b": + try: + next1 = click.getchar() + if next1 in ("[", "O"): + next2 = click.getchar() + if next2 == "A": + return "UP" + if next2 == "B": + return "DOWN" + if next2 == "C": + return "RIGHT" + if next2 == "D": + return "LEFT" + return "CANCEL" + except Exception: + return "CANCEL" + + if char in ("\r", "\n"): + return "ENTER" + if char == " ": + return "SPACE" + if char in ("q", "Q"): + return "QUIT" + if char in ("a", "A"): + return "ALL" + if char in ("w", "W", "k", "K"): + return "UP" + if char in ("s", "S", "j", "J"): + return "DOWN" + if char in ("h", "H"): + return "LEFT" + if char in ("d", "D", "l", "L"): + return "RIGHT" + return None + + def run(self) -> list[int]: + """ + Starts the main event loop for the selector. + Renders the UI and processes input until confirmed or cancelled. + + Returns: + list[int]: A sorted list of selected indices. + """ + try: + with Live(self.get_renderable(), console=console, auto_refresh=False, transient=True) as live: + while True: + live.update(self.get_renderable(), refresh=True) + if IS_WINDOWS: + action = self.get_input_windows() + else: + action = self.get_input_unix() + + if action == "UP": + self.move_cursor(-1) + elif action == "DOWN": + self.move_cursor(1) + elif action == "LEFT": + self.change_page(-1) + elif action == "RIGHT": + self.change_page(1) + elif action == "SPACE": + self.toggle_selection() + elif action == "ALL": + self.toggle_all() + elif action in ("ENTER", "QUIT"): + if len(self.selected_indices) >= self.minimal_count: + return sorted(list(self.selected_indices)) + elif action == "CANCEL": + raise KeyboardInterrupt + except KeyboardInterrupt: + return [] + + +def select_multiple( + options: list[str], + minimal_count: int = 1, + page_size: int = 8, + return_indices: bool = True, + cursor_style: str = "pink", + **kwargs, +) -> list[int]: + """ + Drop-in replacement using custom Selector with global console. + + Args: + options: List of options to display. + minimal_count: Minimum number of selections required. + page_size: Number of items per page. + return_indices: If True, returns indices; otherwise returns the option strings. + cursor_style: Style color for the cursor. + """ + selector = Selector( + options=options, + cursor_style=cursor_style, + text_style="text", + page_size=page_size, + minimal_count=minimal_count, + **kwargs, + ) + + selected_indices = selector.run() + + if return_indices: + return selected_indices + return [options[i] for i in selected_indices] diff --git a/unshackle/core/utils/webvtt.py b/unshackle/core/utils/webvtt.py index 9379fc6..68cc52d 100644 --- a/unshackle/core/utils/webvtt.py +++ b/unshackle/core/utils/webvtt.py @@ -168,6 +168,16 @@ def merge_segmented_webvtt(vtt_raw: str, segment_durations: Optional[list[int]] duplicate_index: list[int] = [] captions = vtt.get_captions(lang) + # Some providers can produce "segment_index" values that are + # outside the provided segment_durations list after normalization/merge. + # This used to crash with IndexError and abort the entire download. + if segment_durations and captions: + max_idx = max(getattr(c, "segment_index", 0) for c in captions) + if max_idx >= len(segment_durations): + # Pad with the last known duration (or 0 if empty) so indexing is safe. + pad_val = segment_durations[-1] if segment_durations else 0 + segment_durations = segment_durations + [pad_val] * (max_idx - len(segment_durations) + 1) + if captions[0].segment_index == 0: first_segment_mpegts = captions[0].mpegts else: @@ -179,6 +189,9 @@ def merge_segmented_webvtt(vtt_raw: str, segment_durations: Optional[list[int]] # calculate the timestamp from SegmentTemplate/SegmentList duration. likely_dash = first_segment_mpegts == 0 and caption.mpegts == 0 if likely_dash and segment_durations: + # Defensive: segment_index can still be out of range if captions are malformed. + if caption.segment_index < 0 or caption.segment_index >= len(segment_durations): + continue duration = segment_durations[caption.segment_index] caption.mpegts = MPEG_TIMESCALE * (duration / timescale) diff --git a/unshackle/services/EXAMPLE/__init__.py b/unshackle/services/EXAMPLE/__init__.py index 2590e87..c296cd0 100644 --- a/unshackle/services/EXAMPLE/__init__.py +++ b/unshackle/services/EXAMPLE/__init__.py @@ -13,6 +13,7 @@ from langcodes import Language from unshackle.core.constants import AnyTrack from unshackle.core.credential import Credential from unshackle.core.manifests import DASH +# from unshackle.core.manifests import HLS from unshackle.core.search_result import SearchResult from unshackle.core.service import Service from unshackle.core.titles import Episode, Movie, Movies, Series, Title_T, Titles_T @@ -35,6 +36,12 @@ class EXAMPLE(Service): GEOFENCE = ("US", "UK") NO_SUBTITLES = True + VIDEO_RANGE_MAP = { + "SDR": "sdr", + "HDR10": "hdr10", + "DV": "dolby_vision", + } + @staticmethod @click.command(name="EXAMPLE", short_help="https://domain.com") @click.argument("title", type=str) @@ -52,17 +59,23 @@ class EXAMPLE(Service): self.device = device self.cdm = ctx.obj.cdm - # Get range parameter for HDR support - range_param = ctx.parent.params.get("range_") - self.range = range_param[0].name if range_param else "SDR" + # self.track_request is set by Service.__init__() from CLI params + # Contains: codecs (list[Video.Codec]), ranges (list[Video.Range]), best_available (bool) + + # Override codec for HDR ranges (HDR requires HEVC) + if any(r != Video.Range.SDR for r in self.track_request.ranges): + self.track_request.codecs = [Video.Codec.HEVC] + + # Override for L3 CDM limitations + if self.cdm and self.cdm.security_level == 3: + self.track_request.codecs = [Video.Codec.AVC] + self.track_request.ranges = [Video.Range.SDR] if self.config is None: raise Exception("Config is missing!") - else: - profile_name = ctx.parent.params.get("profile") - if profile_name is None: - profile_name = "default" - self.profile = profile_name + + profile_name = ctx.parent.params.get("profile") + self.profile = profile_name or "default" def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None: super().authenticate(cookies, credential) @@ -165,78 +178,83 @@ class EXAMPLE(Service): ) return Series(episodes) + # DASH Example: Service requires separate API calls per codec/range. + # Uses _get_tracks_for_variants() which iterates codecs x ranges, + # handles HYBRID (HDR10+DV), and best_available fallback. + def get_tracks(self, title: Title_T) -> Tracks: - # Handle HYBRID mode by fetching both HDR10 and DV tracks separately - if self.range == "HYBRID" and self.cdm.security_level != 3: - tracks = Tracks() + def _fetch_variant( + title: Title_T, + codec: Optional[Video.Codec], + range_: Video.Range, + ) -> Tracks: + vcodec_str = "H265" if codec == Video.Codec.HEVC else "H264" + range_str = range_.name + video_format = self.VIDEO_RANGE_MAP.get(range_str, "sdr") - # Get HDR10 tracks - hdr10_tracks = self._get_tracks_for_range(title, "HDR10") - tracks.add(hdr10_tracks, warn_only=True) + self.log.info(f" + Fetching {vcodec_str} {range_str} manifest") + tracks = self._fetch_dash_manifest(title, vcodec=vcodec_str, video_format=video_format) - # Get DV tracks - dv_tracks = self._get_tracks_for_range(title, "DV") - tracks.add(dv_tracks, warn_only=True) + expected_range = { + "HDR10": Video.Range.HDR10, + "DV": Video.Range.DV, + }.get(range_str) + if expected_range and not any(v.range == expected_range for v in tracks.videos): + raise ValueError(f"{range_str} requested but no {range_str} tracks available") return tracks - else: - # Normal single-range behavior - return self._get_tracks_for_range(title, self.range) - def _get_tracks_for_range(self, title: Title_T, range_override: str = None) -> Tracks: - # Use range_override if provided, otherwise use self.range - current_range = range_override if range_override else self.range + return self._get_tracks_for_variants(title, _fetch_variant) - # Build API request parameters - params = { - "token": self.token, - "guid": title.id, - } - - data = { - "type": self.config["client"][self.device]["type"], - } - - # Add range-specific parameters - if current_range == "HDR10": - data["video_format"] = "hdr10" - elif current_range == "DV": - data["video_format"] = "dolby_vision" - else: - data["video_format"] = "sdr" - - # Only request high-quality HDR content with L1 CDM - if current_range in ("HDR10", "DV") and self.cdm.security_level == 3: - # L3 CDM - skip HDR content - return Tracks() + # HLS Example: Service returns all codecs/ranges in one master playlist. + # No need for _get_tracks_for_variants, dl.py filters by user selection. + # + # def get_tracks(self, title: Title_T) -> Tracks: + # playback = self.session.get( + # url=self.config["endpoints"]["playback"].format(title_id=title.id), + # params={"token": self.token}, + # ).json() + # return HLS.from_url( + # url=playback["manifest_url"], + # session=self.session, + # ).to_tracks(title.language) + def _fetch_dash_manifest( + self, + title: Title_T, + vcodec: str = "H264", + video_format: str = "sdr", + ) -> Tracks: streams = self.session.post( url=self.config["endpoints"]["streams"], - params=params, - data=data, + params={ + "token": self.token, + "guid": title.id, + }, + data={ + "type": self.config["client"][self.device]["type"], + "video_format": video_format, + "video_codec": vcodec, + }, ).json()["media"] - self.license = { + self.license_data = { "url": streams["drm"]["url"], "data": streams["drm"]["data"], "session": streams["drm"]["session"], } manifest_url = streams["url"].split("?")[0] - self.log.debug(f"Manifest URL: {manifest_url}") tracks = DASH.from_url(url=manifest_url, session=self.session).to_tracks(language=title.language) - # Set range attributes on video tracks + range_enum = { + "hdr10": Video.Range.HDR10, + "dolby_vision": Video.Range.DV, + }.get(video_format, Video.Range.SDR) for video in tracks.videos: - if current_range == "HDR10": - video.range = Video.Range.HDR10 - elif current_range == "DV": - video.range = Video.Range.DV - else: - video.range = Video.Range.SDR + video.range = range_enum - # Remove DRM-free ("clear") audio tracks tracks.audio = [ track for track in tracks.audio if "clear" not in track.data["dash"]["representation"].get("id") ] @@ -257,14 +275,14 @@ class EXAMPLE(Service): url=subtitle["url"], codec=Subtitle.Codec.from_mime("vtt"), language=Language.get(subtitle["language"]), - # cc=True if '(cc)' in subtitle['name'] else False, sdh=True, ) ) if not self.movie: title.data["chapters"] = self.session.get( - url=self.config["endpoints"]["metadata"].format(title_id=title.id), params={"token": self.token} + url=self.config["endpoints"]["metadata"].format(title_id=title.id), + params={"token": self.token}, ).json()["chapters"] return tracks @@ -283,12 +301,9 @@ class EXAMPLE(Service): return chapters def get_widevine_service_certificate(self, **_: any) -> str: - """Return the Widevine service certificate from config, if available.""" return self.config.get("certificate") def get_playready_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[bytes]: - """Retrieve a PlayReady license for a given track.""" - license_url = self.config["endpoints"].get("playready_license") if not license_url: raise ValueError("PlayReady license endpoint not configured") @@ -304,7 +319,7 @@ class EXAMPLE(Service): return response.content def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]: - license_url = self.license.get("url") or self.config["endpoints"].get("widevine_license") + license_url = self.license_data.get("url") or self.config["endpoints"].get("widevine_license") if not license_url: raise ValueError("Widevine license endpoint not configured") @@ -312,11 +327,11 @@ class EXAMPLE(Service): url=license_url, data=challenge, params={ - "session": self.license.get("session"), + "session": self.license_data.get("session"), "userId": self.user_id, }, headers={ - "dt-custom-data": self.license.get("data"), + "dt-custom-data": self.license_data.get("data"), "user-agent": self.config["client"][self.device]["license_user_agent"], }, ) diff --git a/unshackle/unshackle-example.yaml b/unshackle/unshackle-example.yaml index 5a04481..2a248c2 100644 --- a/unshackle/unshackle-example.yaml +++ b/unshackle/unshackle-example.yaml @@ -275,12 +275,12 @@ remote_cdm: # PyPlayReady RemoteCdm - connects to an unshackle serve instance - name: "playready_remote" - Device Type: PLAYREADY - System ID: 0 - Security Level: 3000 # 2000 for SL2000, 3000 for SL3000 - Host: "http://127.0.0.1:8786/playready" # Include /playready path - Secret: "your-api-secret-key" - Device Name: "my_prd_device" # Device name on the serve instance + device_name: "my_prd_device" # Device name on the serve instance + device_type: PLAYREADY + system_id: 0 + security_level: 3000 # 2000 for SL2000, 3000 for SL3000 + host: "http://127.0.0.1:8786/playready" # Include /playready path + secret: "your-api-secret-key" # Key Vaults store your obtained Content Encryption Keys (CEKs) # Use 'no_push: true' to prevent a vault from receiving pushed keys diff --git a/uv.lock b/uv.lock index 21487dd..25ca049 100644 --- a/uv.lock +++ b/uv.lock @@ -1627,7 +1627,7 @@ wheels = [ [[package]] name = "unshackle" -version = "2.3.1" +version = "3.1.0" source = { editable = "." } dependencies = [ { name = "aiohttp" },