From 2afc59624d2159081af8126f1c7933f27f3b694a Mon Sep 17 00:00:00 2001 From: Sp5rky Date: Sun, 28 Sep 2025 21:49:00 -0600 Subject: [PATCH 01/62] feat: add REST API server with download management Very early dev work, more changes will be active in this branch. - Implement download queue management and worker system - Add OpenAPI/Swagger documentation - Include download progress tracking and status endpoints - Add API authentication and error handling - Update core components to support API integration --- CONFIG.md | 3 + pyproject.toml | 1 + unshackle/commands/serve.py | 78 ++- unshackle/core/__init__.py | 2 +- unshackle/core/api/__init__.py | 3 + unshackle/core/api/download_manager.py | 630 ++++++++++++++++++++++++ unshackle/core/api/download_worker.py | 84 ++++ unshackle/core/api/handlers.py | 653 +++++++++++++++++++++++++ unshackle/core/api/routes.py | 375 ++++++++++++++ unshackle/core/titles/episode.py | 12 +- unshackle/core/titles/movie.py | 12 +- unshackle/core/tracks/audio.py | 5 + unshackle/core/tracks/tracks.py | 7 +- unshackle/unshackle-example.yaml | 1 + uv.lock | 48 ++ 15 files changed, 1902 insertions(+), 12 deletions(-) create mode 100644 unshackle/core/api/__init__.py create mode 100644 unshackle/core/api/download_manager.py create mode 100644 unshackle/core/api/download_worker.py create mode 100644 unshackle/core/api/handlers.py create mode 100644 unshackle/core/api/routes.py diff --git a/CONFIG.md b/CONFIG.md index 880370a..5a5aae8 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -547,9 +547,12 @@ Configuration data for pywidevine's serve functionality run through unshackle. This effectively allows you to run `unshackle serve` to start serving pywidevine Serve-compliant CDMs right from your local widevine device files. +- `api_secret` - Secret key for REST API authentication. When set, enables the REST API server alongside the CDM serve functionality. This key is required for authenticating API requests. + For example, ```yaml +api_secret: "your-secret-key-here" users: secret_key_for_jane: # 32bit hex recommended, case-sensitive devices: # list of allowed devices for this user diff --git a/pyproject.toml b/pyproject.toml index 1b872a7..cfa5d0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ dependencies = [ "httpx>=0.28.1,<0.29", "cryptography>=45.0.0", "subby", + "aiohttp-swagger3>=0.9.0,<1", ] [project.urls] diff --git a/unshackle/commands/serve.py b/unshackle/commands/serve.py index 85c9739..eaad5fe 100644 --- a/unshackle/commands/serve.py +++ b/unshackle/commands/serve.py @@ -1,19 +1,26 @@ +import logging import subprocess import click +from aiohttp import web from unshackle.core import binaries +from unshackle.core.api import setup_routes, setup_swagger from unshackle.core.config import config from unshackle.core.constants import context_settings -@click.command(short_help="Serve your Local Widevine Devices for Remote Access.", context_settings=context_settings) +@click.command( + short_help="Serve your Local Widevine Devices and REST API for Remote Access.", context_settings=context_settings +) @click.option("-h", "--host", type=str, default="0.0.0.0", help="Host to serve from.") @click.option("-p", "--port", type=int, default=8786, help="Port to serve from.") @click.option("--caddy", is_flag=True, default=False, help="Also serve with Caddy.") -def serve(host: str, port: int, caddy: bool) -> None: +@click.option("--api-only", is_flag=True, default=False, help="Serve only the REST API, not pywidevine CDM.") +@click.option("--no-key", is_flag=True, default=False, help="Disable API key authentication (allows all requests).") +def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool) -> None: """ - Serve your Local Widevine Devices for Remote Access. + Serve your Local Widevine Devices and REST API for Remote Access. \b Host as 127.0.0.1 may block remote access even if port-forwarded. @@ -23,8 +30,25 @@ def serve(host: str, port: int, caddy: bool) -> None: You may serve with Caddy at the same time with --caddy. You can use Caddy as a reverse-proxy to serve with HTTPS. The config used will be the Caddyfile next to the unshackle config. + + \b + The REST API provides programmatic access to unshackle functionality. + Configure authentication in your config under serve.users and serve.api_secret. """ - from pywidevine import serve + from pywidevine import serve as pywidevine_serve + + log = logging.getLogger("serve") + + # Validate API secret for REST API routes (unless --no-key is used) + if not no_key: + api_secret = config.serve.get("api_secret") + if not api_secret: + raise click.ClickException( + "API secret key is not configured. Please add 'api_secret' to the 'serve' section in your config." + ) + else: + api_secret = None + log.warning("Running with --no-key: Authentication is DISABLED for all API endpoints!") if caddy: if not binaries.Caddy: @@ -39,7 +63,51 @@ def serve(host: str, port: int, caddy: bool) -> None: if not config.serve.get("devices"): config.serve["devices"] = [] config.serve["devices"].extend(list(config.directories.wvds.glob("*.wvd"))) - serve.run(config.serve, host, port) + + if api_only: + # API-only mode: serve just the REST API + log.info("Starting REST API server (pywidevine CDM disabled)") + if no_key: + app = web.Application() + app["config"] = {"users": []} + else: + app = web.Application(middlewares=[pywidevine_serve.authentication]) + app["config"] = {"users": [api_secret]} + setup_routes(app) + setup_swagger(app) + log.info(f"REST API endpoints available at http://{host}:{port}/api/") + log.info(f"Swagger UI available at http://{host}:{port}/api/docs/") + log.info("(Press CTRL+C to quit)") + web.run_app(app, host=host, port=port, print=None) + else: + # Integrated mode: serve both pywidevine + REST API + log.info("Starting integrated server (pywidevine CDM + REST API)") + + # Create integrated app with both pywidevine and API routes + if no_key: + app = web.Application() + app["config"] = dict(config.serve) + app["config"]["users"] = [] + else: + app = web.Application(middlewares=[pywidevine_serve.authentication]) + # Setup config - add API secret to users for authentication + serve_config = dict(config.serve) + if not serve_config.get("users"): + serve_config["users"] = [] + if api_secret not in serve_config["users"]: + serve_config["users"].append(api_secret) + app["config"] = serve_config + + app.on_startup.append(pywidevine_serve._startup) + app.on_cleanup.append(pywidevine_serve._cleanup) + app.add_routes(pywidevine_serve.routes) + setup_routes(app) + setup_swagger(app) + + log.info(f"REST API endpoints available at http://{host}:{port}/api/") + log.info(f"Swagger UI available at http://{host}:{port}/api/docs/") + log.info("(Press CTRL+C to quit)") + web.run_app(app, host=host, port=port, print=None) finally: if caddy_p: caddy_p.kill() diff --git a/unshackle/core/__init__.py b/unshackle/core/__init__.py index ac329c9..8c0d5d5 100644 --- a/unshackle/core/__init__.py +++ b/unshackle/core/__init__.py @@ -1 +1 @@ -__version__ = "1.4.7" +__version__ = "2.0.0" diff --git a/unshackle/core/api/__init__.py b/unshackle/core/api/__init__.py new file mode 100644 index 0000000..1fa2c9b --- /dev/null +++ b/unshackle/core/api/__init__.py @@ -0,0 +1,3 @@ +from unshackle.core.api.routes import setup_routes, setup_swagger + +__all__ = ["setup_routes", "setup_swagger"] diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py new file mode 100644 index 0000000..7e2f0a4 --- /dev/null +++ b/unshackle/core/api/download_manager.py @@ -0,0 +1,630 @@ +import asyncio +import json +import logging +import os +import sys +import tempfile +import threading +import uuid +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Callable, Dict, List, Optional +from datetime import datetime, timedelta +from contextlib import suppress + +log = logging.getLogger("download_manager") + + +class JobStatus(Enum): + QUEUED = "queued" + DOWNLOADING = "downloading" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +@dataclass +class DownloadJob: + """Represents a download job with all its parameters and status.""" + + job_id: str + status: JobStatus + created_time: datetime + service: str + title_id: str + parameters: Dict[str, Any] + + # Progress tracking + started_time: Optional[datetime] = None + completed_time: Optional[datetime] = None + progress: float = 0.0 + + # Results and error info + output_files: List[str] = field(default_factory=list) + error_message: Optional[str] = None + error_details: Optional[str] = None + + # Cancellation support + cancel_event: threading.Event = field(default_factory=threading.Event) + + def to_dict(self, include_full_details: bool = False) -> Dict[str, Any]: + """Convert job to dictionary for JSON response.""" + result = { + "job_id": self.job_id, + "status": self.status.value, + "created_time": self.created_time.isoformat(), + "service": self.service, + "title_id": self.title_id, + "progress": self.progress, + } + + if include_full_details: + result.update( + { + "parameters": self.parameters, + "started_time": self.started_time.isoformat() if self.started_time else None, + "completed_time": self.completed_time.isoformat() if self.completed_time else None, + "output_files": self.output_files, + "error_message": self.error_message, + "error_details": self.error_details, + } + ) + + return result + + +def _perform_download( + job_id: str, + service: str, + title_id: str, + params: Dict[str, Any], + cancel_event: Optional[threading.Event] = None, + progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None, +) -> List[str]: + """Execute the synchronous download logic for a job.""" + + def _check_cancel(stage: str): + if cancel_event and cancel_event.is_set(): + raise Exception(f"Job was cancelled {stage}") + + from io import StringIO + from contextlib import redirect_stdout, redirect_stderr + + _check_cancel("before execution started") + + # Import dl.py components lazily to avoid circular deps during module import + import click + import yaml + from unshackle.commands.dl import dl + from unshackle.core.config import config + from unshackle.core.services import Services + from unshackle.core.utils.click_types import ContextData + from unshackle.core.utils.collections import merge_dict + + log.info(f"Starting sync download for job {job_id}") + + # Load service configuration + service_config_path = Services.get_path(service) / config.filenames.config + if service_config_path.exists(): + service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) + else: + service_config = {} + merge_dict(config.services.get(service), service_config) + + from unshackle.commands.dl import dl as dl_command + + ctx = click.Context(dl_command.cli) + ctx.invoked_subcommand = service + ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=[], profile=params.get("profile")) + ctx.params = { + "proxy": params.get("proxy"), + "no_proxy": params.get("no_proxy", False), + "profile": params.get("profile"), + "tag": params.get("tag"), + "tmdb_id": params.get("tmdb_id"), + "tmdb_name": params.get("tmdb_name", False), + "tmdb_year": params.get("tmdb_year", False), + } + + dl_instance = dl( + ctx=ctx, + no_proxy=params.get("no_proxy", False), + profile=params.get("profile"), + proxy=params.get("proxy"), + tag=params.get("tag"), + tmdb_id=params.get("tmdb_id"), + tmdb_name=params.get("tmdb_name", False), + tmdb_year=params.get("tmdb_year", False), + ) + + service_module = Services.load(service) + + _check_cancel("before service instantiation") + + try: + import inspect + + service_init_params = inspect.signature(service_module.__init__).parameters + + service_ctx = click.Context(click.Command(service)) + service_ctx.parent = ctx + service_ctx.obj = ctx.obj + + service_kwargs = {} + + if "title" in service_init_params: + service_kwargs["title"] = title_id + + for key, value in params.items(): + if key in service_init_params and key not in ["service", "title_id"]: + service_kwargs[key] = value + + for param_name, param_info in service_init_params.items(): + if param_name not in service_kwargs and param_name not in ["self", "ctx"]: + if param_info.default is inspect.Parameter.empty: + if param_name == "movie": + service_kwargs[param_name] = "/movies/" in title_id + elif param_name == "meta_lang": + service_kwargs[param_name] = None + else: + log.warning(f"Unknown required parameter '{param_name}' for service {service}, using None") + service_kwargs[param_name] = None + + service_instance = service_module(service_ctx, **service_kwargs) + + except Exception as exc: # noqa: BLE001 - propagate meaningful failure + log.error(f"Failed to create service instance: {exc}") + raise + + original_download_dir = config.directories.downloads + + _check_cancel("before download execution") + + stdout_capture = StringIO() + stderr_capture = StringIO() + + # Simple progress tracking if callback provided + if progress_callback: + # Report initial progress + progress_callback({"progress": 0.0, "status": "starting"}) + + # Simple approach: report progress at key points + original_result = dl_instance.result + + def result_with_progress(*args, **kwargs): + try: + # Report that download started + progress_callback({"progress": 5.0, "status": "downloading"}) + + # Call original method + result = original_result(*args, **kwargs) + + # Report completion + progress_callback({"progress": 100.0, "status": "completed"}) + return result + except Exception as e: + progress_callback({"progress": 0.0, "status": "failed", "error": str(e)}) + raise + + dl_instance.result = result_with_progress + + try: + with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture): + dl_instance.result( + service=service_instance, + quality=params.get("quality", []), + vcodec=params.get("vcodec"), + acodec=params.get("acodec"), + vbitrate=params.get("vbitrate"), + abitrate=params.get("abitrate"), + range_=params.get("range", []), + channels=params.get("channels"), + no_atmos=params.get("no_atmos", False), + wanted=params.get("wanted", []), + lang=params.get("lang", ["orig"]), + v_lang=params.get("v_lang", []), + a_lang=params.get("a_lang", []), + s_lang=params.get("s_lang", ["all"]), + require_subs=params.get("require_subs", []), + forced_subs=params.get("forced_subs", False), + sub_format=params.get("sub_format"), + video_only=params.get("video_only", False), + audio_only=params.get("audio_only", False), + subs_only=params.get("subs_only", False), + chapters_only=params.get("chapters_only", False), + no_subs=params.get("no_subs", False), + no_audio=params.get("no_audio", False), + no_chapters=params.get("no_chapters", False), + slow=params.get("slow", False), + list_=False, + list_titles=False, + skip_dl=params.get("skip_dl", False), + export=params.get("export"), + cdm_only=params.get("cdm_only"), + no_proxy=params.get("no_proxy", False), + no_folder=params.get("no_folder", False), + no_source=params.get("no_source", False), + workers=params.get("workers"), + downloads=params.get("downloads", 1), + best_available=params.get("best_available", False), + ) + + except SystemExit as exc: + if exc.code != 0: + stdout_str = stdout_capture.getvalue() + stderr_str = stderr_capture.getvalue() + log.error(f"Download exited with code {exc.code}") + log.error(f"Stdout: {stdout_str}") + log.error(f"Stderr: {stderr_str}") + raise Exception(f"Download failed with exit code {exc.code}") + + except Exception as exc: # noqa: BLE001 - propagate to caller + stdout_str = stdout_capture.getvalue() + stderr_str = stderr_capture.getvalue() + log.error(f"Download execution failed: {exc}") + log.error(f"Stdout: {stdout_str}") + log.error(f"Stderr: {stderr_str}") + raise + + log.info(f"Download completed for job {job_id}, files in {original_download_dir}") + + return [] + + +class DownloadQueueManager: + """Manages download job queue with configurable concurrency limits.""" + + def __init__(self, max_concurrent_downloads: int = 2, job_retention_hours: int = 24): + self.max_concurrent_downloads = max_concurrent_downloads + self.job_retention_hours = job_retention_hours + + self._jobs: Dict[str, DownloadJob] = {} + self._job_queue: asyncio.Queue = asyncio.Queue() + self._active_downloads: Dict[str, asyncio.Task] = {} + self._download_processes: Dict[str, asyncio.subprocess.Process] = {} + self._job_temp_files: Dict[str, Dict[str, str]] = {} + self._workers_started = False + self._shutdown_event = asyncio.Event() + + log.info( + f"Initialized download queue manager: max_concurrent={max_concurrent_downloads}, retention_hours={job_retention_hours}" + ) + + def create_job(self, service: str, title_id: str, **parameters) -> DownloadJob: + """Create a new download job and add it to the queue.""" + job_id = str(uuid.uuid4()) + job = DownloadJob( + job_id=job_id, + status=JobStatus.QUEUED, + created_time=datetime.now(), + service=service, + title_id=title_id, + parameters=parameters, + ) + + self._jobs[job_id] = job + self._job_queue.put_nowait(job) + + log.info(f"Created download job {job_id} for {service}:{title_id}") + return job + + def get_job(self, job_id: str) -> Optional[DownloadJob]: + """Get job by ID.""" + return self._jobs.get(job_id) + + def list_jobs(self) -> List[DownloadJob]: + """List all jobs.""" + return list(self._jobs.values()) + + def cancel_job(self, job_id: str) -> bool: + """Cancel a job if it's queued or downloading.""" + job = self._jobs.get(job_id) + if not job: + return False + + if job.status == JobStatus.QUEUED: + job.status = JobStatus.CANCELLED + job.cancel_event.set() # Signal cancellation + log.info(f"Cancelled queued job {job_id}") + return True + elif job.status == JobStatus.DOWNLOADING: + # Set the cancellation event first - this will be checked by the download thread + job.cancel_event.set() + job.status = JobStatus.CANCELLED + log.info(f"Signaled cancellation for downloading job {job_id}") + + # Cancel the active download task + task = self._active_downloads.get(job_id) + if task: + task.cancel() + log.info(f"Cancelled download task for job {job_id}") + + process = self._download_processes.get(job_id) + if process: + try: + process.terminate() + log.info(f"Terminated worker process for job {job_id}") + except ProcessLookupError: + log.debug(f"Worker process for job {job_id} already exited") + + return True + + return False + + def cleanup_old_jobs(self) -> int: + """Remove jobs older than retention period.""" + cutoff_time = datetime.now() - timedelta(hours=self.job_retention_hours) + jobs_to_remove = [] + + for job_id, job in self._jobs.items(): + if job.status in [JobStatus.COMPLETED, JobStatus.FAILED, JobStatus.CANCELLED]: + if job.completed_time and job.completed_time < cutoff_time: + jobs_to_remove.append(job_id) + elif not job.completed_time and job.created_time < cutoff_time: + jobs_to_remove.append(job_id) + + for job_id in jobs_to_remove: + del self._jobs[job_id] + + if jobs_to_remove: + log.info(f"Cleaned up {len(jobs_to_remove)} old jobs") + + return len(jobs_to_remove) + + async def start_workers(self): + """Start worker tasks to process the download queue.""" + if self._workers_started: + return + + self._workers_started = True + + # Start worker tasks + for i in range(self.max_concurrent_downloads): + asyncio.create_task(self._download_worker(f"worker-{i}")) + + # Start cleanup task + asyncio.create_task(self._cleanup_worker()) + + log.info(f"Started {self.max_concurrent_downloads} download workers") + + async def shutdown(self): + """Shutdown the queue manager and cancel all active downloads.""" + log.info("Shutting down download queue manager") + self._shutdown_event.set() + + # Cancel all active downloads + for task in self._active_downloads.values(): + task.cancel() + + # Terminate worker processes + for job_id, process in list(self._download_processes.items()): + try: + process.terminate() + except ProcessLookupError: + log.debug(f"Worker process for job {job_id} already exited during shutdown") + + for job_id, process in list(self._download_processes.items()): + try: + await asyncio.wait_for(process.wait(), timeout=5) + except asyncio.TimeoutError: + log.warning(f"Worker process for job {job_id} did not exit, killing") + process.kill() + await process.wait() + finally: + self._download_processes.pop(job_id, None) + + # Clean up any remaining temp files + for paths in self._job_temp_files.values(): + for path in paths.values(): + try: + os.remove(path) + except OSError: + pass + self._job_temp_files.clear() + + # Wait for workers to finish + if self._active_downloads: + await asyncio.gather(*self._active_downloads.values(), return_exceptions=True) + + async def _download_worker(self, worker_name: str): + """Worker task that processes jobs from the queue.""" + log.debug(f"Download worker {worker_name} started") + + while not self._shutdown_event.is_set(): + try: + # Wait for a job or shutdown signal + job = await asyncio.wait_for(self._job_queue.get(), timeout=1.0) + + if job.status == JobStatus.CANCELLED: + continue + + # Start processing the job + job.status = JobStatus.DOWNLOADING + job.started_time = datetime.now() + + log.info(f"Worker {worker_name} starting job {job.job_id}") + + # Create download task + download_task = asyncio.create_task(self._execute_download(job)) + self._active_downloads[job.job_id] = download_task + + try: + await download_task + except asyncio.CancelledError: + job.status = JobStatus.CANCELLED + log.info(f"Job {job.job_id} was cancelled") + except Exception as e: + job.status = JobStatus.FAILED + job.error_message = str(e) + log.error(f"Job {job.job_id} failed: {e}") + finally: + job.completed_time = datetime.now() + if job.job_id in self._active_downloads: + del self._active_downloads[job.job_id] + + except asyncio.TimeoutError: + continue + except Exception as e: + log.error(f"Worker {worker_name} error: {e}") + + async def _execute_download(self, job: DownloadJob): + """Execute the actual download for a job.""" + log.info(f"Executing download for job {job.job_id}") + + try: + output_files = await self._run_download_async(job) + job.status = JobStatus.COMPLETED + job.output_files = output_files + job.progress = 100.0 + log.info(f"Download completed for job {job.job_id}: {len(output_files)} files") + except Exception as e: + job.status = JobStatus.FAILED + job.error_message = str(e) + job.error_details = str(e) + log.error(f"Download failed for job {job.job_id}: {e}") + raise + + async def _run_download_async(self, job: DownloadJob) -> List[str]: + """Invoke a worker subprocess to execute the download.""" + + payload = { + "job_id": job.job_id, + "service": job.service, + "title_id": job.title_id, + "parameters": job.parameters, + } + + payload_fd, payload_path = tempfile.mkstemp(prefix=f"unshackle_job_{job.job_id}_", suffix="_payload.json") + os.close(payload_fd) + result_fd, result_path = tempfile.mkstemp(prefix=f"unshackle_job_{job.job_id}_", suffix="_result.json") + os.close(result_fd) + progress_fd, progress_path = tempfile.mkstemp(prefix=f"unshackle_job_{job.job_id}_", suffix="_progress.json") + os.close(progress_fd) + + with open(payload_path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) + + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "unshackle.core.api.download_worker", + payload_path, + result_path, + progress_path, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + self._download_processes[job.job_id] = process + self._job_temp_files[job.job_id] = {"payload": payload_path, "result": result_path, "progress": progress_path} + + communicate_task = asyncio.create_task(process.communicate()) + + stdout_bytes = b"" + stderr_bytes = b"" + + try: + while True: + done, _ = await asyncio.wait({communicate_task}, timeout=0.5) + if communicate_task in done: + stdout_bytes, stderr_bytes = communicate_task.result() + break + + # Check for progress updates + try: + if os.path.exists(progress_path): + with open(progress_path, "r", encoding="utf-8") as handle: + progress_data = json.load(handle) + if "progress" in progress_data: + new_progress = float(progress_data["progress"]) + if new_progress != job.progress: + job.progress = new_progress + log.info(f"Job {job.job_id} progress updated: {job.progress}%") + except (FileNotFoundError, json.JSONDecodeError, ValueError) as e: + log.debug(f"Could not read progress for job {job.job_id}: {e}") + + if job.cancel_event.is_set() or job.status == JobStatus.CANCELLED: + log.info(f"Cancellation detected for job {job.job_id}, terminating worker process") + process.terminate() + try: + await asyncio.wait_for(communicate_task, timeout=5) + except asyncio.TimeoutError: + log.warning(f"Worker process for job {job.job_id} did not terminate, killing") + process.kill() + await asyncio.wait_for(communicate_task, timeout=5) + raise asyncio.CancelledError("Job was cancelled") + + returncode = process.returncode + stdout = stdout_bytes.decode("utf-8", errors="ignore") + stderr = stderr_bytes.decode("utf-8", errors="ignore") + + if stdout.strip(): + log.debug(f"Worker stdout for job {job.job_id}: {stdout.strip()}") + if stderr.strip(): + log.warning(f"Worker stderr for job {job.job_id}: {stderr.strip()}") + + result_data: Optional[Dict[str, Any]] = None + try: + with open(result_path, "r", encoding="utf-8") as handle: + result_data = json.load(handle) + except FileNotFoundError: + log.error(f"Result file missing for job {job.job_id}") + except json.JSONDecodeError as exc: + log.error(f"Failed to parse worker result for job {job.job_id}: {exc}") + + if returncode != 0: + message = result_data.get("message") if result_data else "unknown error" + raise Exception(f"Worker exited with code {returncode}: {message}") + + if not result_data or result_data.get("status") != "success": + message = result_data.get("message") if result_data else "worker did not report success" + raise Exception(f"Worker failure: {message}") + + return result_data.get("output_files", []) + + finally: + if not communicate_task.done(): + communicate_task.cancel() + with suppress(asyncio.CancelledError): + await communicate_task + + self._download_processes.pop(job.job_id, None) + + temp_paths = self._job_temp_files.pop(job.job_id, {}) + for path in temp_paths.values(): + try: + os.remove(path) + except OSError: + pass + + def _execute_download_sync(self, job: DownloadJob) -> List[str]: + """Execute download synchronously using existing dl.py logic.""" + return _perform_download(job.job_id, job.service, job.title_id, job.parameters.copy(), job.cancel_event) + + async def _cleanup_worker(self): + """Worker that periodically cleans up old jobs.""" + while not self._shutdown_event.is_set(): + try: + await asyncio.sleep(3600) # Run every hour + self.cleanup_old_jobs() + except Exception as e: + log.error(f"Cleanup worker error: {e}") + + +# Global instance +download_manager: Optional[DownloadQueueManager] = None + + +def get_download_manager() -> DownloadQueueManager: + """Get the global download manager instance.""" + global download_manager + if download_manager is None: + # Load configuration from unshackle config + from unshackle.core.config import config + + max_concurrent = getattr(config, "max_concurrent_downloads", 2) + retention_hours = getattr(config, "download_job_retention_hours", 24) + + download_manager = DownloadQueueManager(max_concurrent, retention_hours) + + return download_manager diff --git a/unshackle/core/api/download_worker.py b/unshackle/core/api/download_worker.py new file mode 100644 index 0000000..08810d4 --- /dev/null +++ b/unshackle/core/api/download_worker.py @@ -0,0 +1,84 @@ +"""Standalone worker process entry point for executing download jobs.""" + +from __future__ import annotations + +import json +import logging +import sys +import traceback +from pathlib import Path +from typing import Any, Dict + +from .download_manager import _perform_download + +log = logging.getLogger("download_worker") + + +def _read_payload(path: Path) -> Dict[str, Any]: + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def _write_result(path: Path, payload: Dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(payload, handle) + + +def main(argv: list[str]) -> int: + if len(argv) not in [3, 4]: + print( + "Usage: python -m unshackle.core.api.download_worker [progress_path]", + file=sys.stderr, + ) + return 2 + + payload_path = Path(argv[1]) + result_path = Path(argv[2]) + progress_path = Path(argv[3]) if len(argv) > 3 else None + + result: Dict[str, Any] = {} + exit_code = 0 + + try: + payload = _read_payload(payload_path) + job_id = payload["job_id"] + service = payload["service"] + title_id = payload["title_id"] + params = payload.get("parameters", {}) + + log.info(f"Worker starting job {job_id} ({service}:{title_id})") + + def progress_callback(progress_data: Dict[str, Any]) -> None: + """Write progress updates to file for main process to read.""" + if progress_path: + try: + log.info(f"Writing progress update: {progress_data}") + _write_result(progress_path, progress_data) + log.info(f"Progress update written to {progress_path}") + except Exception as e: + log.error(f"Failed to write progress update: {e}") + + output_files = _perform_download( + job_id, service, title_id, params, cancel_event=None, progress_callback=progress_callback + ) + + result = {"status": "success", "output_files": output_files} + + except Exception as exc: # noqa: BLE001 - capture for parent process + exit_code = 1 + tb = traceback.format_exc() + log.error(f"Worker failed with error: {exc}") + result = {"status": "error", "message": str(exc), "traceback": tb} + + finally: + try: + _write_result(result_path, result) + except Exception as exc: # noqa: BLE001 - last resort logging + log.error(f"Failed to write worker result file: {exc}") + + return exit_code + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py new file mode 100644 index 0000000..60261a6 --- /dev/null +++ b/unshackle/core/api/handlers.py @@ -0,0 +1,653 @@ +import logging +from typing import Any, Dict, List, Optional + +from aiohttp import web + +from unshackle.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_CODEC_MAP +from unshackle.core.proxies.basic import Basic +from unshackle.core.proxies.hola import Hola +from unshackle.core.proxies.nordvpn import NordVPN +from unshackle.core.proxies.surfsharkvpn import SurfsharkVPN +from unshackle.core.services import Services +from unshackle.core.titles import Episode, Movie, Title_T +from unshackle.core.tracks import Audio, Subtitle, Video + +log = logging.getLogger("api") + + +def initialize_proxy_providers() -> List[Any]: + """Initialize and return available proxy providers.""" + proxy_providers = [] + try: + from unshackle.core import binaries + + # Load the main unshackle config to get proxy provider settings + from unshackle.core.config import config as main_config + + log.debug(f"Main config proxy providers: {getattr(main_config, 'proxy_providers', {})}") + log.debug(f"Available proxy provider configs: {list(getattr(main_config, 'proxy_providers', {}).keys())}") + + # Use main_config instead of the service-specific config for proxy providers + proxy_config = getattr(main_config, "proxy_providers", {}) + + if proxy_config.get("basic"): + log.debug("Loading Basic proxy provider") + proxy_providers.append(Basic(**proxy_config["basic"])) + if proxy_config.get("nordvpn"): + log.debug("Loading NordVPN proxy provider") + proxy_providers.append(NordVPN(**proxy_config["nordvpn"])) + if proxy_config.get("surfsharkvpn"): + log.debug("Loading SurfsharkVPN proxy provider") + proxy_providers.append(SurfsharkVPN(**proxy_config["surfsharkvpn"])) + if hasattr(binaries, "HolaProxy") and binaries.HolaProxy: + log.debug("Loading Hola proxy provider") + proxy_providers.append(Hola()) + + for proxy_provider in proxy_providers: + log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}") + + if not proxy_providers: + log.warning("No proxy providers were loaded. Check your proxy provider configuration in unshackle.yaml") + + except Exception as e: + log.warning(f"Failed to initialize some proxy providers: {e}") + + return proxy_providers + + +def resolve_proxy(proxy: str, proxy_providers: List[Any]) -> str: + """Resolve proxy parameter to actual proxy URI.""" + import re + + if not proxy: + return proxy + + # Check if explicit proxy URI + if re.match(r"^https?://", proxy): + return proxy + + # Handle provider:country format (e.g., "nordvpn:us") + requested_provider = None + if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): + requested_provider, proxy = proxy.split(":", maxsplit=1) + + # Handle country code format (e.g., "us", "uk") + if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE): + proxy = proxy.lower() + + if requested_provider: + # Find specific provider (case-insensitive matching) + proxy_provider = next( + (x for x in proxy_providers if x.__class__.__name__.lower() == requested_provider.lower()), + None, + ) + if not proxy_provider: + available_providers = [x.__class__.__name__ for x in proxy_providers] + raise ValueError( + f"The proxy provider '{requested_provider}' was not recognized. Available providers: {available_providers}" + ) + + proxy_uri = proxy_provider.get_proxy(proxy) + if not proxy_uri: + raise ValueError(f"The proxy provider {requested_provider} had no proxy for {proxy}") + + log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy_uri}") + return proxy_uri + else: + # Try all providers + for proxy_provider in proxy_providers: + proxy_uri = proxy_provider.get_proxy(proxy) + if proxy_uri: + log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy_uri}") + return proxy_uri + + raise ValueError(f"No proxy provider had a proxy for {proxy}") + + # Return as-is if not recognized format + log.info(f"Using explicit Proxy: {proxy}") + return proxy + + +def validate_service(service_tag: str) -> Optional[str]: + """Validate and normalize service tag.""" + try: + normalized = Services.get_tag(service_tag) + service_path = Services.get_path(normalized) + if not service_path.exists(): + return None + return normalized + except Exception: + return None + + +def serialize_title(title: Title_T) -> Dict[str, Any]: + """Convert a title object to JSON-serializable dict.""" + if isinstance(title, Episode): + episode_name = title.name if title.name else f"Episode {title.number:02d}" + result = { + "type": "episode", + "name": episode_name, + "series_title": str(title.title), + "season": title.season, + "number": title.number, + "year": title.year, + "id": str(title.id) if hasattr(title, "id") else None, + } + elif isinstance(title, Movie): + result = { + "type": "movie", + "name": str(title.name) if hasattr(title, "name") else str(title), + "year": title.year, + "id": str(title.id) if hasattr(title, "id") else None, + } + else: + result = { + "type": "other", + "name": str(title.name) if hasattr(title, "name") else str(title), + "id": str(title.id) if hasattr(title, "id") else None, + } + + return result + + +def serialize_video_track(track: Video) -> Dict[str, Any]: + """Convert video track to JSON-serializable dict.""" + codec_name = track.codec.name if hasattr(track.codec, "name") else str(track.codec) + range_name = track.range.name if hasattr(track.range, "name") else str(track.range) + + return { + "id": str(track.id), + "codec": codec_name, + "codec_display": VIDEO_CODEC_MAP.get(codec_name, codec_name), + "bitrate": int(track.bitrate / 1000) if track.bitrate else None, + "width": track.width, + "height": track.height, + "resolution": f"{track.width}x{track.height}" if track.width and track.height else None, + "fps": track.fps if track.fps else None, + "range": range_name, + "range_display": DYNAMIC_RANGE_MAP.get(range_name, range_name), + "language": str(track.language) if track.language else None, + "drm": str(track.drm) if hasattr(track, "drm") and track.drm else None, + } + + +def serialize_audio_track(track: Audio) -> Dict[str, Any]: + """Convert audio track to JSON-serializable dict.""" + codec_name = track.codec.name if hasattr(track.codec, "name") else str(track.codec) + + return { + "id": str(track.id), + "codec": codec_name, + "codec_display": AUDIO_CODEC_MAP.get(codec_name, codec_name), + "bitrate": int(track.bitrate / 1000) if track.bitrate else None, + "channels": track.channels if track.channels else None, + "language": str(track.language) if track.language else None, + "atmos": track.atmos if hasattr(track, "atmos") else False, + "descriptive": track.descriptive if hasattr(track, "descriptive") else False, + "drm": str(track.drm) if hasattr(track, "drm") and track.drm else None, + } + + +def serialize_subtitle_track(track: Subtitle) -> Dict[str, Any]: + """Convert subtitle track to JSON-serializable dict.""" + return { + "id": str(track.id), + "codec": track.codec.name if hasattr(track.codec, "name") else str(track.codec), + "language": str(track.language) if track.language else None, + "forced": track.forced if hasattr(track, "forced") else False, + "sdh": track.sdh if hasattr(track, "sdh") else False, + "cc": track.cc if hasattr(track, "cc") else False, + } + + +async def list_titles_handler(data: Dict[str, Any]) -> web.Response: + """Handle list-titles request.""" + service_tag = data.get("service") + title_id = data.get("title_id") + profile = data.get("profile") + + if not service_tag: + return web.json_response({"status": "error", "message": "Missing required parameter: service"}, status=400) + + if not title_id: + return web.json_response({"status": "error", "message": "Missing required parameter: title_id"}, status=400) + + normalized_service = validate_service(service_tag) + if not normalized_service: + return web.json_response( + {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, status=400 + ) + + try: + import inspect + + import click + import yaml + + from unshackle.commands.dl import dl + from unshackle.core.config import config + from unshackle.core.utils.click_types import ContextData + from unshackle.core.utils.collections import merge_dict + + service_config_path = Services.get_path(normalized_service) / config.filenames.config + if service_config_path.exists(): + service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) + else: + service_config = {} + merge_dict(config.services.get(normalized_service), service_config) + + @click.command() + @click.pass_context + def dummy_service(ctx: click.Context) -> None: + pass + + # Handle proxy configuration + proxy_param = data.get("proxy") + no_proxy = data.get("no_proxy", False) + proxy_providers = [] + + if not no_proxy: + proxy_providers = initialize_proxy_providers() + + if proxy_param and not no_proxy: + try: + resolved_proxy = resolve_proxy(proxy_param, proxy_providers) + proxy_param = resolved_proxy + except ValueError as e: + return web.json_response({"status": "error", "message": f"Proxy error: {e}"}, status=400) + + ctx = click.Context(dummy_service) + ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=proxy_providers, profile=profile) + ctx.params = {"proxy": proxy_param, "no_proxy": no_proxy} + + service_module = Services.load(normalized_service) + + dummy_service.name = normalized_service + dummy_service.params = [click.Argument([title_id], type=str)] + ctx.invoked_subcommand = normalized_service + + service_ctx = click.Context(dummy_service, parent=ctx) + service_ctx.obj = ctx.obj + + service_kwargs = {"title": title_id} + + # Add additional parameters from request data + for key, value in data.items(): + if key not in ["service", "title_id", "profile", "season", "episode", "wanted", "proxy", "no_proxy"]: + service_kwargs[key] = value + + # Get service parameter info and click command defaults + service_init_params = inspect.signature(service_module.__init__).parameters + + # Extract default values from the click command + if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): + for param in service_module.cli.params: + if hasattr(param, "name") and param.name not in service_kwargs: + # Add default value if parameter is not already provided + if hasattr(param, "default") and param.default is not None: + service_kwargs[param.name] = param.default + + # Handle required parameters that don't have click defaults + for param_name, param_info in service_init_params.items(): + if param_name not in service_kwargs and param_name not in ["self", "ctx"]: + # Check if parameter is required (no default value in signature) + if param_info.default is inspect.Parameter.empty: + # Provide sensible defaults for common required parameters + if param_name == "meta_lang": + service_kwargs[param_name] = None + elif param_name == "movie": + service_kwargs[param_name] = False + else: + # Log warning for unknown required parameters + log.warning(f"Unknown required parameter '{param_name}' for service {normalized_service}") + + # Filter out any parameters that the service doesn't accept + filtered_kwargs = {} + for key, value in service_kwargs.items(): + if key in service_init_params: + filtered_kwargs[key] = value + + service_instance = service_module(service_ctx, **filtered_kwargs) + + cookies = dl.get_cookie_jar(normalized_service, profile) + credential = dl.get_credentials(normalized_service, profile) + service_instance.authenticate(cookies, credential) + + titles = service_instance.get_titles() + + if hasattr(titles, "__iter__") and not isinstance(titles, str): + title_list = [serialize_title(t) for t in titles] + else: + title_list = [serialize_title(titles)] + + return web.json_response({"titles": title_list}) + + except Exception as e: + log.exception("Error listing titles") + return web.json_response({"status": "error", "message": str(e)}, status=500) + + +async def list_tracks_handler(data: Dict[str, Any]) -> web.Response: + """Handle list-tracks request.""" + service_tag = data.get("service") + title_id = data.get("title_id") + profile = data.get("profile") + + if not service_tag: + return web.json_response({"status": "error", "message": "Missing required parameter: service"}, status=400) + + if not title_id: + return web.json_response({"status": "error", "message": "Missing required parameter: title_id"}, status=400) + + normalized_service = validate_service(service_tag) + if not normalized_service: + return web.json_response( + {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, status=400 + ) + + try: + import inspect + + import click + import yaml + + from unshackle.commands.dl import dl + from unshackle.core.config import config + from unshackle.core.utils.click_types import ContextData + from unshackle.core.utils.collections import merge_dict + + service_config_path = Services.get_path(normalized_service) / config.filenames.config + if service_config_path.exists(): + service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) + else: + service_config = {} + merge_dict(config.services.get(normalized_service), service_config) + + @click.command() + @click.pass_context + def dummy_service(ctx: click.Context) -> None: + pass + + # Handle proxy configuration + proxy_param = data.get("proxy") + no_proxy = data.get("no_proxy", False) + proxy_providers = [] + + if not no_proxy: + proxy_providers = initialize_proxy_providers() + + if proxy_param and not no_proxy: + try: + resolved_proxy = resolve_proxy(proxy_param, proxy_providers) + proxy_param = resolved_proxy + except ValueError as e: + return web.json_response({"status": "error", "message": f"Proxy error: {e}"}, status=400) + + ctx = click.Context(dummy_service) + ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=proxy_providers, profile=profile) + ctx.params = {"proxy": proxy_param, "no_proxy": no_proxy} + + service_module = Services.load(normalized_service) + + dummy_service.name = normalized_service + dummy_service.params = [click.Argument([title_id], type=str)] + ctx.invoked_subcommand = normalized_service + + service_ctx = click.Context(dummy_service, parent=ctx) + service_ctx.obj = ctx.obj + + service_kwargs = {"title": title_id} + + # Add additional parameters from request data + for key, value in data.items(): + if key not in ["service", "title_id", "profile", "season", "episode", "wanted", "proxy", "no_proxy"]: + service_kwargs[key] = value + + # Get service parameter info and click command defaults + service_init_params = inspect.signature(service_module.__init__).parameters + + # Extract default values from the click command + if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): + for param in service_module.cli.params: + if hasattr(param, "name") and param.name not in service_kwargs: + # Add default value if parameter is not already provided + if hasattr(param, "default") and param.default is not None: + service_kwargs[param.name] = param.default + + # Handle required parameters that don't have click defaults + for param_name, param_info in service_init_params.items(): + if param_name not in service_kwargs and param_name not in ["self", "ctx"]: + # Check if parameter is required (no default value in signature) + if param_info.default is inspect.Parameter.empty: + # Provide sensible defaults for common required parameters + if param_name == "meta_lang": + service_kwargs[param_name] = None + elif param_name == "movie": + service_kwargs[param_name] = False + else: + # Log warning for unknown required parameters + log.warning(f"Unknown required parameter '{param_name}' for service {normalized_service}") + + # Filter out any parameters that the service doesn't accept + filtered_kwargs = {} + for key, value in service_kwargs.items(): + if key in service_init_params: + filtered_kwargs[key] = value + + service_instance = service_module(service_ctx, **filtered_kwargs) + + cookies = dl.get_cookie_jar(normalized_service, profile) + credential = dl.get_credentials(normalized_service, profile) + service_instance.authenticate(cookies, credential) + + titles = service_instance.get_titles() + + wanted_param = data.get("wanted") + season = data.get("season") + episode = data.get("episode") + + if hasattr(titles, "__iter__") and not isinstance(titles, str): + titles_list = list(titles) + + wanted = None + if wanted_param: + from unshackle.core.utils.click_types import SeasonRange + + try: + season_range = SeasonRange() + wanted = season_range.parse_tokens(wanted_param) + log.debug(f"Parsed wanted '{wanted_param}' into {len(wanted)} episodes: {wanted[:10]}...") + except Exception as e: + return web.json_response( + {"status": "error", "message": f"Invalid wanted parameter: {e}"}, status=400 + ) + elif season is not None and episode is not None: + wanted = [f"{season}x{episode}"] + + if wanted: + # Filter titles based on wanted episodes, similar to how dl.py does it + matching_titles = [] + log.debug(f"Filtering {len(titles_list)} titles with {len(wanted)} wanted episodes") + for title in titles_list: + if isinstance(title, Episode): + episode_key = f"{title.season}x{title.number}" + if episode_key in wanted: + log.debug(f"Episode {episode_key} matches wanted list") + matching_titles.append(title) + else: + log.debug(f"Episode {episode_key} not in wanted list") + else: + matching_titles.append(title) + + log.debug(f"Found {len(matching_titles)} matching titles") + + if not matching_titles: + return web.json_response( + {"status": "error", "message": "No episodes found matching wanted criteria"}, status=404 + ) + + # If multiple episodes match, return tracks for all episodes + if len(matching_titles) > 1 and all(isinstance(t, Episode) for t in matching_titles): + episodes_data = [] + failed_episodes = [] + + # Sort matching titles by season and episode number for consistent ordering + sorted_titles = sorted(matching_titles, key=lambda t: (t.season, t.number)) + + for title in sorted_titles: + try: + tracks = service_instance.get_tracks(title) + video_tracks = sorted(tracks.videos, key=lambda t: t.bitrate or 0, reverse=True) + audio_tracks = sorted(tracks.audio, key=lambda t: t.bitrate or 0, reverse=True) + + episode_data = { + "title": serialize_title(title), + "video": [serialize_video_track(t) for t in video_tracks], + "audio": [serialize_audio_track(t) for t in audio_tracks], + "subtitles": [serialize_subtitle_track(t) for t in tracks.subtitles], + } + episodes_data.append(episode_data) + log.debug(f"Successfully got tracks for {title.season}x{title.number}") + except SystemExit: + # Service calls sys.exit() for unavailable episodes - catch and skip + failed_episodes.append(f"S{title.season}E{title.number:02d}") + log.debug(f"Episode {title.season}x{title.number} not available, skipping") + continue + except Exception as e: + # Handle other errors gracefully + failed_episodes.append(f"S{title.season}E{title.number:02d}") + log.debug(f"Error getting tracks for {title.season}x{title.number}: {e}") + continue + + if episodes_data: + response = {"episodes": episodes_data} + if failed_episodes: + response["unavailable_episodes"] = failed_episodes + return web.json_response(response) + else: + return web.json_response( + { + "status": "error", + "message": f"No available episodes found. Unavailable: {', '.join(failed_episodes)}", + }, + status=404, + ) + else: + # Single episode or movie + first_title = matching_titles[0] + else: + first_title = titles_list[0] + else: + first_title = titles + + tracks = service_instance.get_tracks(first_title) + + video_tracks = sorted(tracks.videos, key=lambda t: t.bitrate or 0, reverse=True) + audio_tracks = sorted(tracks.audio, key=lambda t: t.bitrate or 0, reverse=True) + + response = { + "title": serialize_title(first_title), + "video": [serialize_video_track(t) for t in video_tracks], + "audio": [serialize_audio_track(t) for t in audio_tracks], + "subtitles": [serialize_subtitle_track(t) for t in tracks.subtitles], + } + + return web.json_response(response) + + except Exception as e: + log.exception("Error listing tracks") + return web.json_response({"status": "error", "message": str(e)}, status=500) + + +async def download_handler(data: Dict[str, Any]) -> web.Response: + """Handle download request - create and queue a download job.""" + from unshackle.core.api.download_manager import get_download_manager + + service_tag = data.get("service") + title_id = data.get("title_id") + + if not service_tag: + return web.json_response({"status": "error", "message": "Missing required parameter: service"}, status=400) + + if not title_id: + return web.json_response({"status": "error", "message": "Missing required parameter: title_id"}, status=400) + + normalized_service = validate_service(service_tag) + if not normalized_service: + return web.json_response( + {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, status=400 + ) + + try: + # Get download manager and start workers if needed + manager = get_download_manager() + await manager.start_workers() + + # Create download job with filtered parameters (exclude service and title_id as they're already passed) + filtered_params = {k: v for k, v in data.items() if k not in ["service", "title_id"]} + job = manager.create_job(normalized_service, title_id, **filtered_params) + + return web.json_response( + {"job_id": job.job_id, "status": job.status.value, "created_time": job.created_time.isoformat()}, status=202 + ) + + except Exception as e: + log.exception("Error creating download job") + return web.json_response({"status": "error", "message": str(e)}, status=500) + + +async def list_download_jobs_handler(data: Dict[str, Any]) -> web.Response: + """Handle list download jobs request.""" + from unshackle.core.api.download_manager import get_download_manager + + try: + manager = get_download_manager() + jobs = manager.list_jobs() + + job_list = [job.to_dict(include_full_details=False) for job in jobs] + + return web.json_response({"jobs": job_list}) + + except Exception as e: + log.exception("Error listing download jobs") + return web.json_response({"status": "error", "message": str(e)}, status=500) + + +async def get_download_job_handler(job_id: str) -> web.Response: + """Handle get specific download job request.""" + from unshackle.core.api.download_manager import get_download_manager + + try: + manager = get_download_manager() + job = manager.get_job(job_id) + + if not job: + return web.json_response({"status": "error", "message": "Job not found"}, status=404) + + return web.json_response(job.to_dict(include_full_details=True)) + + except Exception as e: + log.exception(f"Error getting download job {job_id}") + return web.json_response({"status": "error", "message": str(e)}, status=500) + + +async def cancel_download_job_handler(job_id: str) -> web.Response: + """Handle cancel download job request.""" + from unshackle.core.api.download_manager import get_download_manager + + try: + manager = get_download_manager() + + if not manager.get_job(job_id): + return web.json_response({"status": "error", "message": "Job not found"}, status=404) + + success = manager.cancel_job(job_id) + + if success: + return web.json_response({"status": "success", "message": "Job cancelled"}) + else: + return web.json_response({"status": "error", "message": "Job cannot be cancelled"}, status=400) + + except Exception as e: + log.exception(f"Error cancelling download job {job_id}") + return web.json_response({"status": "error", "message": str(e)}, status=500) diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py new file mode 100644 index 0000000..c8dfa7a --- /dev/null +++ b/unshackle/core/api/routes.py @@ -0,0 +1,375 @@ +import logging + +from aiohttp import web +from aiohttp_swagger3 import SwaggerDocs, SwaggerInfo, SwaggerUiSettings + +from unshackle.core import __version__ +from unshackle.core.api.handlers import ( + download_handler, + list_titles_handler, + list_tracks_handler, + list_download_jobs_handler, + get_download_job_handler, + cancel_download_job_handler, +) +from unshackle.core.services import Services +from unshackle.core.update_checker import UpdateChecker + +log = logging.getLogger("api") + + +async def health(request: web.Request) -> web.Response: + """ + Health check endpoint. + --- + summary: Health check + description: Get server health status, version info, and update availability + responses: + '200': + description: Health status + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: ok + version: + type: string + example: "2.0.0" + update_check: + type: object + properties: + update_available: + type: boolean + nullable: true + current_version: + type: string + latest_version: + type: string + nullable: true + """ + try: + latest_version = await UpdateChecker.check_for_updates(__version__) + update_info = { + "update_available": latest_version is not None, + "current_version": __version__, + "latest_version": latest_version, + } + except Exception as e: + log.warning(f"Failed to check for updates: {e}") + update_info = {"update_available": None, "current_version": __version__, "latest_version": None} + + return web.json_response({"status": "ok", "version": __version__, "update_check": update_info}) + + +async def services(request: web.Request) -> web.Response: + """ + List available services. + --- + summary: List services + description: Get all available streaming services with their details + responses: + '200': + description: List of services + content: + application/json: + schema: + type: object + properties: + services: + type: array + items: + type: object + properties: + tag: + type: string + aliases: + type: array + items: + type: string + geofence: + type: array + items: + type: string + title_regex: + type: string + nullable: true + help: + type: string + nullable: true + '500': + description: Server error + """ + try: + service_tags = Services.get_tags() + services_info = [] + + for tag in service_tags: + service_data = {"tag": tag, "aliases": [], "geofence": [], "title_regex": None, "help": None} + + try: + service_module = Services.load(tag) + + if hasattr(service_module, "ALIASES"): + service_data["aliases"] = list(service_module.ALIASES) + + if hasattr(service_module, "GEOFENCE"): + service_data["geofence"] = list(service_module.GEOFENCE) + + if hasattr(service_module, "TITLE_RE"): + service_data["title_regex"] = service_module.TITLE_RE + + if service_module.__doc__: + service_data["help"] = service_module.__doc__.strip() + + except Exception as e: + log.warning(f"Could not load details for service {tag}: {e}") + + services_info.append(service_data) + + return web.json_response({"services": services_info}) + except Exception as e: + log.exception("Error listing services") + return web.json_response({"status": "error", "message": str(e)}, status=500) + + +async def list_titles(request: web.Request) -> web.Response: + """ + List titles for a service and title ID. + --- + summary: List titles + description: Get available titles for a service and title ID + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - service + - title_id + properties: + service: + type: string + description: Service tag + title_id: + type: string + description: Title identifier + responses: + '200': + description: List of titles + '400': + description: Invalid request + """ + try: + data = await request.json() + except Exception: + return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) + + return await list_titles_handler(data) + + +async def list_tracks(request: web.Request) -> web.Response: + """ + List tracks for a title, separated by type. + --- + summary: List tracks + description: Get available video, audio, and subtitle tracks for a title + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - service + - title_id + properties: + service: + type: string + description: Service tag + title_id: + type: string + description: Title identifier + wanted: + type: string + description: Specific episode/season (optional) + proxy: + type: string + description: Proxy configuration (optional) + responses: + '200': + description: Track information + '400': + description: Invalid request + """ + try: + data = await request.json() + except Exception: + return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) + + return await list_tracks_handler(data) + + +async def download(request: web.Request) -> web.Response: + """ + Download content based on provided parameters. + --- + summary: Download content + description: Download video content based on specified parameters + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - service + - title_id + properties: + service: + type: string + description: Service tag + title_id: + type: string + description: Title identifier + responses: + '200': + description: Download started + '400': + description: Invalid request + """ + try: + data = await request.json() + except Exception: + return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) + + return await download_handler(data) + + +async def download_jobs(request: web.Request) -> web.Response: + """ + List all download jobs. + --- + summary: List download jobs + description: Get list of all download jobs with their status + responses: + '200': + description: List of download jobs + content: + application/json: + schema: + type: object + properties: + jobs: + type: array + items: + type: object + properties: + job_id: + type: string + status: + type: string + created_time: + type: string + service: + type: string + title_id: + type: string + progress: + type: number + '500': + description: Server error + """ + return await list_download_jobs_handler({}) + + +async def download_job_detail(request: web.Request) -> web.Response: + """ + Get download job details. + --- + summary: Get download job + description: Get detailed information about a specific download job + parameters: + - name: job_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Download job details + '404': + description: Job not found + '500': + description: Server error + """ + job_id = request.match_info["job_id"] + return await get_download_job_handler(job_id) + + +async def cancel_download_job(request: web.Request) -> web.Response: + """ + Cancel download job. + --- + summary: Cancel download job + description: Cancel a queued or running download job + parameters: + - name: job_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Job cancelled successfully + '400': + description: Job cannot be cancelled + '404': + description: Job not found + '500': + description: Server error + """ + job_id = request.match_info["job_id"] + return await cancel_download_job_handler(job_id) + + +def setup_routes(app: web.Application) -> None: + """Setup all API routes.""" + app.router.add_get("/api/health", health) + app.router.add_get("/api/services", services) + app.router.add_post("/api/list-titles", list_titles) + app.router.add_post("/api/list-tracks", list_tracks) + app.router.add_post("/api/download", download) + app.router.add_get("/api/download/jobs", download_jobs) + app.router.add_get("/api/download/jobs/{job_id}", download_job_detail) + app.router.add_delete("/api/download/jobs/{job_id}", cancel_download_job) + + +def setup_swagger(app: web.Application) -> None: + """Setup Swagger UI documentation.""" + swagger = SwaggerDocs( + app, + swagger_ui_settings=SwaggerUiSettings(path="/api/docs/"), + info=SwaggerInfo( + title="Unshackle REST API", + version=__version__, + description="REST API for Unshackle - Modular Movie, TV, and Music Archival Software", + ), + ) + + # Add routes with OpenAPI documentation + swagger.add_routes( + [ + web.get("/api/health", health), + web.get("/api/services", services), + web.post("/api/list-titles", list_titles), + web.post("/api/list-tracks", list_tracks), + web.post("/api/download", download), + web.get("/api/download/jobs", download_jobs), + web.get("/api/download/jobs/{job_id}", download_job_detail), + web.delete("/api/download/jobs/{job_id}", cancel_download_job), + ] + ) diff --git a/unshackle/core/titles/episode.py b/unshackle/core/titles/episode.py index f66bcb7..16ecab6 100644 --- a/unshackle/core/titles/episode.py +++ b/unshackle/core/titles/episode.py @@ -89,7 +89,17 @@ class Episode(Title): def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str: primary_video_track = next(iter(media_info.video_tracks), None) - primary_audio_track = next(iter(media_info.audio_tracks), None) + primary_audio_track = None + if media_info.audio_tracks: + sorted_audio = sorted( + media_info.audio_tracks, + key=lambda x: ( + float(x.bit_rate) if x.bit_rate else 0, + bool(x.format_additionalfeatures and "JOC" in x.format_additionalfeatures) + ), + reverse=True + ) + primary_audio_track = sorted_audio[0] unique_audio_languages = len({x.language.split("-")[0] for x in media_info.audio_tracks if x.language}) # Title [Year] SXXEXX Name (or Title [Year] SXX if folder) diff --git a/unshackle/core/titles/movie.py b/unshackle/core/titles/movie.py index 3d552d2..2e1d8bb 100644 --- a/unshackle/core/titles/movie.py +++ b/unshackle/core/titles/movie.py @@ -52,7 +52,17 @@ class Movie(Title): def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str: primary_video_track = next(iter(media_info.video_tracks), None) - primary_audio_track = next(iter(media_info.audio_tracks), None) + primary_audio_track = None + if media_info.audio_tracks: + sorted_audio = sorted( + media_info.audio_tracks, + key=lambda x: ( + float(x.bit_rate) if x.bit_rate else 0, + bool(x.format_additionalfeatures and "JOC" in x.format_additionalfeatures) + ), + reverse=True + ) + primary_audio_track = sorted_audio[0] unique_audio_languages = len({x.language.split("-")[0] for x in media_info.audio_tracks if x.language}) # Name (Year) diff --git a/unshackle/core/tracks/audio.py b/unshackle/core/tracks/audio.py index ff5de9f..0069efa 100644 --- a/unshackle/core/tracks/audio.py +++ b/unshackle/core/tracks/audio.py @@ -12,6 +12,7 @@ class Audio(Track): AAC = "AAC" # https://wikipedia.org/wiki/Advanced_Audio_Coding AC3 = "DD" # https://wikipedia.org/wiki/Dolby_Digital EC3 = "DD+" # https://wikipedia.org/wiki/Dolby_Digital_Plus + AC4 = "AC-4" # https://wikipedia.org/wiki/Dolby_AC-4 OPUS = "OPUS" # https://wikipedia.org/wiki/Opus_(audio_format) OGG = "VORB" # https://wikipedia.org/wiki/Vorbis DTS = "DTS" # https://en.wikipedia.org/wiki/DTS_(company)#DTS_Digital_Surround @@ -31,6 +32,8 @@ class Audio(Track): return Audio.Codec.AC3 if mime == "ec-3": return Audio.Codec.EC3 + if mime == "ac-4": + return Audio.Codec.AC4 if mime == "opus": return Audio.Codec.OPUS if mime == "dtsc": @@ -60,6 +63,8 @@ class Audio(Track): return Audio.Codec.AC3 if profile.startswith("ddplus"): return Audio.Codec.EC3 + if profile.startswith("ac4"): + return Audio.Codec.AC4 if profile.startswith("playready-oggvorbis"): return Audio.Codec.OGG raise ValueError(f"The Content Profile '{profile}' is not a supported Audio Codec") diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index 34c9da6..cf691b7 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -202,17 +202,16 @@ class Tracks: """Sort audio tracks by bitrate, descriptive, and optionally language.""" if not self.audio: return - # bitrate - self.audio.sort(key=lambda x: float(x.bitrate or 0.0), reverse=True) # descriptive - self.audio.sort(key=lambda x: str(x.language) if x.descriptive else "") + self.audio.sort(key=lambda x: x.descriptive) + # bitrate (within each descriptive group) + self.audio.sort(key=lambda x: float(x.bitrate or 0.0), reverse=True) # language for language in reversed(by_language or []): if str(language) in ("all", "best"): language = next((x.language for x in self.audio if x.is_original_lang), "") if not language: continue - self.audio.sort(key=lambda x: str(x.language)) self.audio.sort(key=lambda x: not is_close_match(language, [x.language])) def sort_subtitles(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None: diff --git a/unshackle/unshackle-example.yaml b/unshackle/unshackle-example.yaml index 4ad46ff..ca4f031 100644 --- a/unshackle/unshackle-example.yaml +++ b/unshackle/unshackle-example.yaml @@ -259,6 +259,7 @@ subtitle: # Configuration for pywidevine's serve functionality serve: + api_secret: "your-secret-key-here" users: secret_key_for_user: devices: diff --git a/uv.lock b/uv.lock index 1cea081..9ae2600 100644 --- a/uv.lock +++ b/uv.lock @@ -80,6 +80,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/d8/fa65d2a349fe938b76d309db1a56a75c4fb8cc7b17a398b698488a939903/aiohttp-3.12.15-cp312-cp312-win_amd64.whl", hash = "sha256:b390ef5f62bb508a9d67cb3bba9b8356e23b3996da7062f1a57ce1a79d2b3d34", size = 450266, upload-time = "2025-07-29T05:51:17.239Z" }, ] +[[package]] +name = "aiohttp-swagger3" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "attrs" }, + { name = "fastjsonschema" }, + { name = "pyyaml" }, + { name = "rfc3339-validator" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/06/00ccb2c8afdde4ca7c3cac424d54715c7d90cdd4e13e1ca71d68f5b2e665/aiohttp_swagger3-0.10.0.tar.gz", hash = "sha256:a333c59328f64dd64587e5f276ee84dc256f587d09f2da6ddaae3812fa4d4f33", size = 1839028, upload-time = "2025-02-11T10:51:26.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/8f/db4cb843999a3088846d170f38eda2182b50b5733387be8102fed171c53f/aiohttp_swagger3-0.10.0-py3-none-any.whl", hash = "sha256:0ae2d2ba7dbd8ea8fe1cffe8f0197db5d0aa979eb9679bd699ecd87923912509", size = 1826491, upload-time = "2025-02-11T10:51:25.174Z" }, +] + [[package]] name = "aiosignal" version = "1.4.0" @@ -468,6 +484,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, ] +[[package]] +name = "fastjsonschema" +version = "2.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/7f/cedf77ace50aa60c566deaca9066750f06e1fcf6ad24f254d255bb976dd6/fastjsonschema-2.19.1.tar.gz", hash = "sha256:e3126a94bdc4623d3de4485f8d468a12f02a67921315ddc87836d6e456dc789d", size = 372732, upload-time = "2023-12-28T14:02:06.823Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/b9/79691036d4a8f9857e74d1728b23f34f583b81350a27492edda58d5604e1/fastjsonschema-2.19.1-py3-none-any.whl", hash = "sha256:3672b47bc94178c9f23dbb654bf47440155d4db9df5f7bc47643315f9c405cd0", size = 23388, upload-time = "2023-12-28T14:02:04.512Z" }, +] + [[package]] name = "filelock" version = "3.19.1" @@ -1252,6 +1277,18 @@ socks = [ { name = "pysocks" }, ] +[[package]] +name = "rfc3339-validator" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", size = 5513, upload-time = "2021-05-12T16:37:54.178Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa", size = 3490, upload-time = "2021-05-12T16:37:52.536Z" }, +] + [[package]] name = "rich" version = "13.9.4" @@ -1358,6 +1395,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, ] +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -1502,6 +1548,7 @@ name = "unshackle" version = "1.4.6" source = { editable = "." } dependencies = [ + { name = "aiohttp-swagger3" }, { name = "appdirs" }, { name = "brotli" }, { name = "chardet" }, @@ -1551,6 +1598,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "aiohttp-swagger3", specifier = ">=0.9.0,<1" }, { name = "appdirs", specifier = ">=1.4.4,<2" }, { name = "brotli", specifier = ">=1.1.0,<2" }, { name = "chardet", specifier = ">=5.2.0,<6" }, From 2e2f8f5099555dd4f44e8919f53ed095af65e287 Mon Sep 17 00:00:00 2001 From: TPD94 Date: Mon, 29 Sep 2025 20:48:59 -0400 Subject: [PATCH 02/62] Fix remoteCDM, add curl_cffi to instance check --- .gitignore | 6 ++++++ .idea/.gitignore | 8 ++++++++ unshackle/commands/dl.py | 12 ++++++++---- unshackle/core/manifests/dash.py | 2 +- 4 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 .idea/.gitignore diff --git a/.gitignore b/.gitignore index 26a73b6..b84292e 100644 --- a/.gitignore +++ b/.gitignore @@ -235,3 +235,9 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ +.idea/vcs.xml +.idea/unshackle.iml +.idea/modules.xml +.idea/misc.xml +.idea/inspectionProfiles/Project_Default.xml +.idea/inspectionProfiles/profiles_settings.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 9a99bfc..06d1b71 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -1701,10 +1701,14 @@ class dl: # All DecryptLabs CDMs use DecryptLabsRemoteCDM return DecryptLabsRemoteCDM(service_name=service, vaults=self.vaults, **cdm_api) else: - del cdm_api["name"] - if "type" in cdm_api: - del cdm_api["type"] - return RemoteCdm(**cdm_api) + return RemoteCdm( + device_type=cdm_api['Device Type'], + system_id=cdm_api['System ID'], + security_level=cdm_api['Security Level'], + host=cdm_api['Host'], + secret=cdm_api['Secret'], + device_name=cdm_api['Device Name'], + ) prd_path = config.directories.prds / f"{cdm_name}.prd" if not prd_path.is_file(): diff --git a/unshackle/core/manifests/dash.py b/unshackle/core/manifests/dash.py index ec19e25..67ef362 100644 --- a/unshackle/core/manifests/dash.py +++ b/unshackle/core/manifests/dash.py @@ -253,7 +253,7 @@ class DASH: ): if not session: session = Session() - elif not isinstance(session, Session): + elif not isinstance(session, (Session, CurlSession)): raise TypeError(f"Expected session to be a {Session}, not {session!r}") if proxy: From 55f116f1e8d7b510b4bd9861ebd8cb3f6c23a4e5 Mon Sep 17 00:00:00 2001 From: TPD94 <39639333+TPD94@users.noreply.github.com> Date: Mon, 29 Sep 2025 20:53:16 -0400 Subject: [PATCH 03/62] Delete .idea directory --- .idea/.gitignore | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .idea/.gitignore diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b8..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml From bade3f8c09bb00862788b6995adcc214627b49d5 Mon Sep 17 00:00:00 2001 From: TPD94 <39639333+TPD94@users.noreply.github.com> Date: Mon, 29 Sep 2025 20:53:38 -0400 Subject: [PATCH 04/62] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b84292e..b7c4e73 100644 --- a/.gitignore +++ b/.gitignore @@ -202,7 +202,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ # Abstra # Abstra is an AI-powered process automation framework. From 4f3d0f1f7ab1c14e7f4fd5744447392bae14e970 Mon Sep 17 00:00:00 2001 From: TPD94 <39639333+TPD94@users.noreply.github.com> Date: Mon, 29 Sep 2025 20:54:42 -0400 Subject: [PATCH 05/62] Update .gitignore --- .gitignore | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitignore b/.gitignore index b7c4e73..317b448 100644 --- a/.gitignore +++ b/.gitignore @@ -235,9 +235,3 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ -.idea/vcs.xml -.idea/unshackle.iml -.idea/modules.xml -.idea/misc.xml -.idea/inspectionProfiles/Project_Default.xml -.idea/inspectionProfiles/profiles_settings.xml From 724703d14b4a8a11cb0915ee2902630b5744eba6 Mon Sep 17 00:00:00 2001 From: TPD94 <39639333+TPD94@users.noreply.github.com> Date: Mon, 29 Sep 2025 20:56:25 -0400 Subject: [PATCH 06/62] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 317b448..26a73b6 100644 --- a/.gitignore +++ b/.gitignore @@ -202,7 +202,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -.idea/ +#.idea/ # Abstra # Abstra is an AI-powered process automation framework. From 03f08159b45f4f9cf44fa7a039309d18ee9f7bfc Mon Sep 17 00:00:00 2001 From: TPD94 <39639333+TPD94@users.noreply.github.com> Date: Mon, 29 Sep 2025 21:01:55 -0400 Subject: [PATCH 07/62] Update dash.py --- unshackle/core/manifests/dash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unshackle/core/manifests/dash.py b/unshackle/core/manifests/dash.py index 67ef362..56fec08 100644 --- a/unshackle/core/manifests/dash.py +++ b/unshackle/core/manifests/dash.py @@ -254,7 +254,7 @@ class DASH: if not session: session = Session() elif not isinstance(session, (Session, CurlSession)): - raise TypeError(f"Expected session to be a {Session}, not {session!r}") + raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {session!r}") if proxy: session.proxies.update({"all": proxy}) From 97f7eb06742e1aa39a937022c08b80c88eca53b1 Mon Sep 17 00:00:00 2001 From: Andy Date: Tue, 30 Sep 2025 02:14:14 +0000 Subject: [PATCH 08/62] Changes for API/UI --- unshackle/commands/serve.py | 10 +++++----- unshackle/core/api/__init__.py | 4 ++-- unshackle/core/api/routes.py | 29 +++++++++++++++++++++-------- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/unshackle/commands/serve.py b/unshackle/commands/serve.py index eaad5fe..515cd45 100644 --- a/unshackle/commands/serve.py +++ b/unshackle/commands/serve.py @@ -5,7 +5,7 @@ import click from aiohttp import web from unshackle.core import binaries -from unshackle.core.api import setup_routes, setup_swagger +from unshackle.core.api import cors_middleware, setup_routes, setup_swagger from unshackle.core.config import config from unshackle.core.constants import context_settings @@ -68,10 +68,10 @@ def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool) -> No # API-only mode: serve just the REST API log.info("Starting REST API server (pywidevine CDM disabled)") if no_key: - app = web.Application() + app = web.Application(middlewares=[cors_middleware]) app["config"] = {"users": []} else: - app = web.Application(middlewares=[pywidevine_serve.authentication]) + app = web.Application(middlewares=[cors_middleware, pywidevine_serve.authentication]) app["config"] = {"users": [api_secret]} setup_routes(app) setup_swagger(app) @@ -85,11 +85,11 @@ def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool) -> No # Create integrated app with both pywidevine and API routes if no_key: - app = web.Application() + app = web.Application(middlewares=[cors_middleware]) app["config"] = dict(config.serve) app["config"]["users"] = [] else: - app = web.Application(middlewares=[pywidevine_serve.authentication]) + app = web.Application(middlewares=[cors_middleware, pywidevine_serve.authentication]) # Setup config - add API secret to users for authentication serve_config = dict(config.serve) if not serve_config.get("users"): diff --git a/unshackle/core/api/__init__.py b/unshackle/core/api/__init__.py index 1fa2c9b..8369876 100644 --- a/unshackle/core/api/__init__.py +++ b/unshackle/core/api/__init__.py @@ -1,3 +1,3 @@ -from unshackle.core.api.routes import setup_routes, setup_swagger +from unshackle.core.api.routes import cors_middleware, setup_routes, setup_swagger -__all__ = ["setup_routes", "setup_swagger"] +__all__ = ["setup_routes", "setup_swagger", "cors_middleware"] diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index c8dfa7a..5445c87 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -4,17 +4,30 @@ from aiohttp import web from aiohttp_swagger3 import SwaggerDocs, SwaggerInfo, SwaggerUiSettings from unshackle.core import __version__ -from unshackle.core.api.handlers import ( - download_handler, - list_titles_handler, - list_tracks_handler, - list_download_jobs_handler, - get_download_job_handler, - cancel_download_job_handler, -) +from unshackle.core.api.handlers import (cancel_download_job_handler, download_handler, get_download_job_handler, + list_download_jobs_handler, list_titles_handler, list_tracks_handler) from unshackle.core.services import Services from unshackle.core.update_checker import UpdateChecker + +@web.middleware +async def cors_middleware(request: web.Request, handler): + """Add CORS headers to all responses.""" + # Handle preflight requests + if request.method == "OPTIONS": + response = web.Response() + else: + response = await handler(request) + + # Add CORS headers + response.headers["Access-Control-Allow-Origin"] = "*" + response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, OPTIONS" + response.headers["Access-Control-Allow-Headers"] = "Content-Type, X-API-Key, Authorization" + response.headers["Access-Control-Max-Age"] = "3600" + + return response + + log = logging.getLogger("api") From e1e2e35ff43656ec8fbacb5db56478e2e66c6208 Mon Sep 17 00:00:00 2001 From: TPD94 Date: Tue, 30 Sep 2025 00:14:44 -0400 Subject: [PATCH 09/62] Update binaries.py to check subdirs in binaries folders named after the binary --- .gitignore | 2 ++ unshackle/core/binaries.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 26a73b6..e38a05e 100644 --- a/.gitignore +++ b/.gitignore @@ -235,3 +235,5 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ +/unshackle/binaries +/.idea diff --git a/unshackle/core/binaries.py b/unshackle/core/binaries.py index da31fb5..878a36f 100644 --- a/unshackle/core/binaries.py +++ b/unshackle/core/binaries.py @@ -3,6 +3,8 @@ import sys from pathlib import Path from typing import Optional +from mypy.types import names + __shaka_platform = {"win32": "win", "darwin": "osx"}.get(sys.platform, sys.platform) @@ -15,16 +17,17 @@ def find(*names: str) -> Optional[Path]: for name in names: # First check local binaries folder if local_binaries_dir.exists(): - local_path = local_binaries_dir / name - if local_path.is_file() and local_path.stat().st_mode & 0o111: # Check if executable - return local_path - - # Also check with .exe extension on Windows + # On Windows, check for .exe extension first if sys.platform == "win32": - local_path_exe = local_binaries_dir / f"{name}.exe" + local_path_exe = local_binaries_dir / f"{name}" / f"{name}.exe" if local_path_exe.is_file(): return local_path_exe + # Check for exact name match with executable bit on Unix-like systems + local_path = local_binaries_dir / f"{name}" / f"{name}" + if local_path.is_file() and local_path.stat().st_mode & 0o111: # Check if executable + return local_path + # Fall back to system PATH path = shutil.which(name) if path: From 0f4a68ca622610741b5ba3d4ee3ba15e1947a251 Mon Sep 17 00:00:00 2001 From: Sp5rky Date: Tue, 30 Sep 2025 12:53:27 -0600 Subject: [PATCH 10/62] fix: update lxml constraint and pyplayready import path - Update lxml dependency to allow version 6.x (required by subby 0.3.23) - Fix pyplayready exception import path (moved to misc.exceptions in 0.6.3) fixes #17 --- pyproject.toml | 2 +- unshackle/commands/prd.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cfa5d0a..d9bd604 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "crccheck>=1.3.0,<2", "jsonpickle>=3.0.4,<4", "langcodes>=3.4.0,<4", - "lxml>=5.2.1,<6", + "lxml>=5.2.1,<7", "pproxy>=2.7.9,<3", "protobuf>=4.25.3,<5", "pycaption>=2.2.6,<3", diff --git a/unshackle/commands/prd.py b/unshackle/commands/prd.py index ad46950..443efec 100644 --- a/unshackle/commands/prd.py +++ b/unshackle/commands/prd.py @@ -8,7 +8,7 @@ from Crypto.Random import get_random_bytes from pyplayready.cdm import Cdm from pyplayready.crypto.ecc_key import ECCKey from pyplayready.device import Device -from pyplayready.exceptions import InvalidCertificateChain, OutdatedDevice +from pyplayready.misc.exceptions import InvalidCertificateChain, OutdatedDevice from pyplayready.system.bcert import Certificate, CertificateChain from pyplayready.system.pssh import PSSH From 8437ba24d537877694a67d77595d04855b26c72c Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 13 Oct 2025 23:49:01 +0000 Subject: [PATCH 11/62] feat: Add comprehensive JSON debug logging system Implements a complete structured logging system for troubleshooting and service development. Features: - Binary toggle via --debug flag or debug: true in config - JSON Lines (.jsonl) format for easy parsing and analysis - Comprehensive logging of all operations: * Session info (version, platform, Python version) * CLI parameters and service configuration * CDM details (Widevine/PlayReady, security levels) * Authentication status * Title and track metadata * DRM operations (PSSH, KIDs, license requests) * Vault queries with key retrieval * Full error traces with context - Configurable key logging via debug_keys option - Smart redaction (passwords, tokens, cookies always redacted) - Error logging for all critical operations: * Authentication failures * Title fetching errors * Track retrieval errors * License request failures (Widevine & PlayReady) * Vault operation errors - Removed old text logging system --- unshackle/commands/dl.py | 329 ++++++++++++++++++++++++++++-- unshackle/core/__main__.py | 32 +-- unshackle/core/config.py | 4 + unshackle/core/utilities.py | 340 ++++++++++++++++++++++++++++++- unshackle/unshackle-example.yaml | 20 +- 5 files changed, 687 insertions(+), 38 deletions(-) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 4e4ad8d..cba5ee9 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -56,7 +56,8 @@ from unshackle.core.titles.episode import Episode from unshackle.core.tracks import Audio, Subtitle, Tracks, Video from unshackle.core.tracks.attachment import Attachment from unshackle.core.tracks.hybrid import Hybrid -from unshackle.core.utilities import get_system_fonts, is_close_match, time_elapsed_since +from unshackle.core.utilities import (get_debug_logger, get_system_fonts, init_debug_logger, is_close_match, + time_elapsed_since) from unshackle.core.utils import tags from unshackle.core.utils.click_types import (LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE, ContextData, MultipleChoice, SubtitleCodecChoice, VideoCodecChoice) @@ -313,6 +314,40 @@ class dl: self.tmdb_name = tmdb_name self.tmdb_year = tmdb_year + # Initialize debug logger with service name if debug logging is enabled + if config.debug or logging.root.level == logging.DEBUG: + from collections import defaultdict + from datetime import datetime + + debug_log_path = config.directories.logs / config.filenames.debug_log.format_map( + defaultdict(str, service=self.service, time=datetime.now().strftime("%Y%m%d-%H%M%S")) + ) + init_debug_logger( + log_path=debug_log_path, + enabled=True, + log_keys=config.debug_keys + ) + self.debug_logger = get_debug_logger() + + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="download_init", + message=f"Download command initialized for service {self.service}", + service=self.service, + context={ + "profile": profile, + "proxy": proxy, + "tag": tag, + "tmdb_id": tmdb_id, + "tmdb_name": tmdb_name, + "tmdb_year": tmdb_year, + "cli_params": {k: v for k, v in ctx.params.items() if k not in ['profile', 'proxy', 'tag', 'tmdb_id', 'tmdb_name', 'tmdb_year']} + } + ) + else: + self.debug_logger = None + if self.profile: self.log.info(f"Using profile: '{self.profile}'") @@ -321,6 +356,13 @@ class dl: if service_config_path.exists(): self.service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) self.log.info("Service Config loaded") + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="load_service_config", + service=self.service, + context={"config_path": str(service_config_path), "config": self.service_config} + ) else: self.service_config = {} merge_dict(config.services.get(self.service), self.service_config) @@ -384,18 +426,32 @@ class dl: self.cdm = self.get_cdm(self.service, self.profile) except ValueError as e: self.log.error(f"Failed to load CDM, {e}") + if self.debug_logger: + self.debug_logger.log_error("load_cdm", e, service=self.service) sys.exit(1) if self.cdm: + cdm_info = {} if isinstance(self.cdm, DecryptLabsRemoteCDM): drm_type = "PlayReady" if self.cdm.is_playready else "Widevine" self.log.info(f"Loaded {drm_type} Remote CDM: DecryptLabs (L{self.cdm.security_level})") + cdm_info = {"type": "DecryptLabs", "drm_type": drm_type, "security_level": self.cdm.security_level} elif hasattr(self.cdm, "device_type") and self.cdm.device_type.name in ["ANDROID", "CHROME"]: self.log.info(f"Loaded Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})") + cdm_info = {"type": "Widevine", "system_id": self.cdm.system_id, "security_level": self.cdm.security_level, "device_type": self.cdm.device_type.name} else: self.log.info( f"Loaded PlayReady CDM: {self.cdm.certificate_chain.get_name()} (L{self.cdm.security_level})" ) + cdm_info = {"type": "PlayReady", "certificate": self.cdm.certificate_chain.get_name(), "security_level": self.cdm.security_level} + + if self.debug_logger and cdm_info: + self.debug_logger.log( + level="INFO", + operation="load_cdm", + service=self.service, + context={"cdm": cdm_info} + ) self.proxy_providers = [] if no_proxy: @@ -521,18 +577,83 @@ class dl: else: vaults_only = not cdm_only + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="drm_mode_config", + service=self.service, + context={ + "cdm_only": cdm_only, + "vaults_only": vaults_only, + "mode": "CDM only" if cdm_only else ("Vaults only" if vaults_only else "Both CDM and Vaults") + } + ) + with console.status("Authenticating with Service...", spinner="dots"): - cookies = self.get_cookie_jar(self.service, self.profile) - credential = self.get_credentials(self.service, self.profile) - service.authenticate(cookies, credential) - if cookies or credential: - self.log.info("Authenticated with Service") + try: + cookies = self.get_cookie_jar(self.service, self.profile) + credential = self.get_credentials(self.service, self.profile) + service.authenticate(cookies, credential) + if cookies or credential: + self.log.info("Authenticated with Service") + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="authenticate", + service=self.service, + context={ + "has_cookies": bool(cookies), + "has_credentials": bool(credential), + "profile": self.profile + } + ) + except Exception as e: + if self.debug_logger: + self.debug_logger.log_error( + "authenticate", + e, + service=self.service, + context={"profile": self.profile} + ) + raise with console.status("Fetching Title Metadata...", spinner="dots"): - titles = service.get_titles_cached() - if not titles: - self.log.error("No titles returned, nothing to download...") - sys.exit(1) + try: + titles = service.get_titles_cached() + if not titles: + self.log.error("No titles returned, nothing to download...") + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="get_titles", + service=self.service, + message="No titles returned from service", + success=False + ) + sys.exit(1) + except Exception as e: + if self.debug_logger: + self.debug_logger.log_error( + "get_titles", + e, + service=self.service + ) + raise + + if self.debug_logger: + titles_info = { + "type": titles.__class__.__name__, + "count": len(titles) if hasattr(titles, "__len__") else 1, + "title": str(titles) + } + if hasattr(titles, "seasons"): + titles_info["seasons"] = len(titles.seasons) if hasattr(titles, "seasons") else 0 + self.debug_logger.log( + level="INFO", + operation="get_titles", + service=self.service, + context={"titles": titles_info} + ) if self.tmdb_year and self.tmdb_id: sample_title = titles[0] if hasattr(titles, "__getitem__") else titles @@ -621,8 +742,55 @@ class dl: title.tracks.subtitles = [] with console.status("Getting tracks...", spinner="dots"): - title.tracks.add(service.get_tracks(title), warn_only=True) - title.tracks.chapters = service.get_chapters(title) + try: + title.tracks.add(service.get_tracks(title), warn_only=True) + title.tracks.chapters = service.get_chapters(title) + except Exception as e: + if self.debug_logger: + self.debug_logger.log_error( + "get_tracks", + e, + service=self.service, + context={"title": str(title)} + ) + raise + + if self.debug_logger: + tracks_info = { + "title": str(title), + "video_tracks": len(title.tracks.videos), + "audio_tracks": len(title.tracks.audio), + "subtitle_tracks": len(title.tracks.subtitles), + "has_chapters": bool(title.tracks.chapters), + "videos": [{ + "codec": str(v.codec), + "resolution": f"{v.width}x{v.height}" if v.width and v.height else "unknown", + "bitrate": v.bitrate, + "range": str(v.range), + "language": str(v.language) if v.language else None, + "drm": [str(type(d).__name__) for d in v.drm] if v.drm else [] + } for v in title.tracks.videos], + "audio": [{ + "codec": str(a.codec), + "bitrate": a.bitrate, + "channels": a.channels, + "language": str(a.language) if a.language else None, + "descriptive": a.descriptive, + "drm": [str(type(d).__name__) for d in a.drm] if a.drm else [] + } for a in title.tracks.audio], + "subtitles": [{ + "codec": str(s.codec), + "language": str(s.language) if s.language else None, + "forced": s.forced, + "sdh": s.sdh + } for s in title.tracks.subtitles] + } + self.debug_logger.log( + level="INFO", + operation="get_tracks", + service=self.service, + context=tracks_info + ) # strip SDH subs to non-SDH if no equivalent same-lang non-SDH is available # uses a loose check, e.g, wont strip en-US SDH sub if a non-SDH en-GB is available @@ -1009,6 +1177,14 @@ class dl: download.result() except KeyboardInterrupt: console.print(Padding(":x: Download Cancelled...", (0, 5, 1, 5))) + if self.debug_logger: + self.debug_logger.log( + level="WARNING", + operation="download_tracks", + service=self.service, + message="Download cancelled by user", + context={"title": str(title)} + ) return except Exception as e: # noqa error_messages = [ @@ -1031,6 +1207,19 @@ class dl: # CalledProcessError already lists the exception trace console.print_exception() console.print(Padding(Group(*error_messages), (1, 5))) + + if self.debug_logger: + self.debug_logger.log_error( + "download_tracks", + e, + service=self.service, + context={ + "title": str(title), + "error_type": type(e).__name__, + "tracks_count": len(title.tracks), + "returncode": getattr(e, "returncode", None) + } + ) return if skip_dl: @@ -1394,6 +1583,20 @@ class dl: self.cdm = playready_cdm if isinstance(drm, Widevine): + if self.debug_logger: + self.debug_logger.log_drm_operation( + drm_type="Widevine", + operation="prepare_drm", + service=self.service, + context={ + "track": str(track), + "title": str(title), + "pssh": drm.pssh.dumps() if drm.pssh else None, + "kids": [k.hex for k in drm.kids], + "track_kid": track_kid.hex if track_kid else None + } + ) + with self.DRM_TABLE_LOCK: pssh_display = self._truncate_pssh_for_display(drm.pssh.dumps(), "Widevine") cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="fold")) @@ -1422,11 +1625,32 @@ class dl: if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): cek_tree.add(label) self.vaults.add_key(kid, content_key, excluding=vault_used) + + if self.debug_logger: + self.debug_logger.log_vault_query( + vault_name=vault_used, + operation="get_key_success", + service=self.service, + context={ + "kid": kid.hex, + "content_key": content_key, + "track": str(track), + "from_cache": True + } + ) elif vaults_only: msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used" cek_tree.add(f"[logging.level.error]{msg}") if not pre_existing_tree: table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="vault_key_not_found", + service=self.service, + message=msg, + context={"kid": kid.hex, "track": str(track)} + ) raise Widevine.Exceptions.CEKNotFound(msg) else: need_license = True @@ -1437,6 +1661,18 @@ class dl: if need_license and not vaults_only: from_vaults = drm.content_keys.copy() + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="get_license", + service=self.service, + message="Requesting Widevine license from service", + context={ + "track": str(track), + "kids_needed": [k.hex for k in all_kids if k not in drm.content_keys] + } + ) + try: if self.service == "NF": drm.get_NF_content_keys(cdm=self.cdm, licence=licence, certificate=certificate) @@ -1450,8 +1686,30 @@ class dl: cek_tree.add(f"[logging.level.error]{msg}") if not pre_existing_tree: table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log_error( + "get_license", + e, + service=self.service, + context={ + "track": str(track), + "exception_type": type(e).__name__ + } + ) raise e + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="license_keys_retrieved", + service=self.service, + context={ + "track": str(track), + "keys_count": len(drm.content_keys), + "kids": [k.hex for k in drm.content_keys.keys()] + } + ) + for kid_, key in drm.content_keys.items(): if key == "0" * 32: key = f"[red]{key}[/]" @@ -1497,6 +1755,20 @@ class dl: export.write_text(jsonpickle.dumps(keys, indent=4), encoding="utf8") elif isinstance(drm, PlayReady): + if self.debug_logger: + self.debug_logger.log_drm_operation( + drm_type="PlayReady", + operation="prepare_drm", + service=self.service, + context={ + "track": str(track), + "title": str(title), + "pssh": drm.pssh_b64 or "", + "kids": [k.hex for k in drm.kids], + "track_kid": track_kid.hex if track_kid else None + } + ) + with self.DRM_TABLE_LOCK: pssh_display = self._truncate_pssh_for_display(drm.pssh_b64 or "", "PlayReady") cek_tree = Tree( @@ -1531,11 +1803,33 @@ class dl: if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): cek_tree.add(label) self.vaults.add_key(kid, content_key, excluding=vault_used) + + if self.debug_logger: + self.debug_logger.log_vault_query( + vault_name=vault_used, + operation="get_key_success", + service=self.service, + context={ + "kid": kid.hex, + "content_key": content_key, + "track": str(track), + "from_cache": True, + "drm_type": "PlayReady" + } + ) elif vaults_only: msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used" cek_tree.add(f"[logging.level.error]{msg}") if not pre_existing_tree: table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="vault_key_not_found", + service=self.service, + message=msg, + context={"kid": kid.hex, "track": str(track), "drm_type": "PlayReady"} + ) raise PlayReady.Exceptions.CEKNotFound(msg) else: need_license = True @@ -1556,6 +1850,17 @@ class dl: cek_tree.add(f"[logging.level.error]{msg}") if not pre_existing_tree: table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log_error( + "get_license_playready", + e, + service=self.service, + context={ + "track": str(track), + "exception_type": type(e).__name__, + "drm_type": "PlayReady" + } + ) raise e for kid_, key in drm.content_keys.items(): diff --git a/unshackle/core/__main__.py b/unshackle/core/__main__.py index e4717fa..6cf2fac 100644 --- a/unshackle/core/__main__.py +++ b/unshackle/core/__main__.py @@ -1,6 +1,5 @@ import atexit import logging -from pathlib import Path import click import urllib3 @@ -16,23 +15,16 @@ from unshackle.core.config import config from unshackle.core.console import ComfyRichHandler, console from unshackle.core.constants import context_settings from unshackle.core.update_checker import UpdateChecker -from unshackle.core.utilities import rotate_log_file - -LOGGING_PATH = None +from unshackle.core.utilities import close_debug_logger, init_debug_logger @click.command(cls=Commands, invoke_without_command=True, context_settings=context_settings) @click.option("-v", "--version", is_flag=True, default=False, help="Print version information.") -@click.option("-d", "--debug", is_flag=True, default=False, help="Enable DEBUG level logs.") -@click.option( - "--log", - "log_path", - type=Path, - default=config.directories.logs / config.filenames.log, - help="Log path (or filename). Path can contain the following f-string args: {name} {time}.", -) -def main(version: bool, debug: bool, log_path: Path) -> None: +@click.option("-d", "--debug", is_flag=True, default=False, help="Enable DEBUG level logs and JSON debug logging.") +def main(version: bool, debug: bool) -> None: """unshackle—Modular Movie, TV, and Music Archival Software.""" + debug_logging_enabled = debug or config.debug + logging.basicConfig( level=logging.DEBUG if debug else logging.INFO, format="%(message)s", @@ -48,11 +40,8 @@ def main(version: bool, debug: bool, log_path: Path) -> None: ], ) - if log_path: - global LOGGING_PATH - console.record = True - new_log_path = rotate_log_file(log_path) - LOGGING_PATH = new_log_path + if debug_logging_enabled: + init_debug_logger(enabled=True) urllib3.disable_warnings(InsecureRequestWarning) @@ -98,10 +87,9 @@ def main(version: bool, debug: bool, log_path: Path) -> None: @atexit.register -def save_log(): - if console.record and LOGGING_PATH: - # TODO: Currently semi-bust. Everything that refreshes gets duplicated. - console.save_text(LOGGING_PATH) +def cleanup(): + """Clean up resources on exit.""" + close_debug_logger() if __name__ == "__main__": diff --git a/unshackle/core/config.py b/unshackle/core/config.py index 79483ce..6ac5f29 100644 --- a/unshackle/core/config.py +++ b/unshackle/core/config.py @@ -31,6 +31,7 @@ class Config: class _Filenames: # default filenames, do not modify here, set via config log = "unshackle_{name}_{time}.log" # Directories.logs + debug_log = "unshackle_debug_{service}_{time}.jsonl" # Directories.logs config = "config.yaml" # Directories.services / tag root_config = "unshackle.yaml" # Directories.user_configs chapters = "Chapters_{title}_{random}.txt" # Directories.temp @@ -98,6 +99,9 @@ class Config: self.title_cache_max_retention: int = kwargs.get("title_cache_max_retention", 86400) # 24 hours default self.title_cache_enabled: bool = kwargs.get("title_cache_enabled", True) + self.debug: bool = kwargs.get("debug", False) + self.debug_keys: bool = kwargs.get("debug_keys", False) + @classmethod def from_yaml(cls, path: Path) -> Config: if not path.exists(): diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index 9302e0d..d97e9bd 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -1,19 +1,22 @@ import ast import contextlib import importlib.util +import json import logging import os import re import socket import sys import time +import traceback import unicodedata from collections import defaultdict -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from types import ModuleType -from typing import Optional, Sequence, Union +from typing import Any, Optional, Sequence, Union from urllib.parse import ParseResult, urlparse +from uuid import uuid4 import chardet import requests @@ -122,7 +125,7 @@ def is_exact_match(language: Union[str, Language], languages: Sequence[Union[str return closest_match(language, list(map(str, languages)))[1] <= LANGUAGE_EXACT_DISTANCE -def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: +def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: # type: ignore """ Scan a byte array for a wanted MP4/ISOBMFF box, then parse and yield each find. @@ -457,3 +460,334 @@ class FPS(ast.NodeVisitor): @classmethod def parse(cls, expr: str) -> float: return cls().visit(ast.parse(expr).body[0]) + + +""" +Structured JSON debug logging for unshackle. + +Provides comprehensive debugging information for service developers and troubleshooting. +When enabled, logs all operations, requests, responses, DRM operations, and errors in JSON format. +""" + + +class DebugLogger: + """ + Structured JSON debug logger for unshackle. + + Outputs JSON Lines format where each line is a complete JSON object. + This makes it easy to parse, filter, and analyze logs programmatically. + """ + + def __init__(self, log_path: Optional[Path] = None, enabled: bool = False, log_keys: bool = False): + """ + Initialize the debug logger. + + Args: + log_path: Path to the log file. If None, logging is disabled. + enabled: Whether debug logging is enabled. + log_keys: Whether to log decryption keys (for debugging key issues). + """ + self.enabled = enabled and log_path is not None + self.log_path = log_path + self.session_id = str(uuid4())[:8] + self.file_handle = None + self.log_keys = log_keys + + if self.enabled: + self.log_path.parent.mkdir(parents=True, exist_ok=True) + self.file_handle = open(self.log_path, "a", encoding="utf-8") + self._log_session_start() + + def _log_session_start(self): + """Log the start of a new session with environment information.""" + import platform + + from unshackle.core import __version__ + + self.log( + level="INFO", + operation="session_start", + message="Debug logging session started", + context={ + "unshackle_version": __version__, + "python_version": sys.version, + "platform": platform.platform(), + "platform_system": platform.system(), + "platform_release": platform.release(), + }, + ) + + def log( + self, + level: str = "DEBUG", + operation: str = "", + message: str = "", + context: Optional[dict[str, Any]] = None, + service: Optional[str] = None, + error: Optional[Exception] = None, + request: Optional[dict[str, Any]] = None, + response: Optional[dict[str, Any]] = None, + duration_ms: Optional[float] = None, + success: Optional[bool] = None, + **kwargs, + ): + """ + Log a structured JSON entry. + + Args: + level: Log level (DEBUG, INFO, WARNING, ERROR) + operation: Name of the operation being performed + message: Human-readable message + context: Additional context information + service: Service name (e.g., DSNP, NF) + error: Exception object if an error occurred + request: Request details (URL, method, headers, body) + response: Response details (status, headers, body) + duration_ms: Operation duration in milliseconds + success: Whether the operation succeeded + **kwargs: Additional fields to include in the log entry + """ + if not self.enabled or not self.file_handle: + return + + entry = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "session_id": self.session_id, + "level": level, + } + + if operation: + entry["operation"] = operation + if message: + entry["message"] = message + if service: + entry["service"] = service + if context: + entry["context"] = self._sanitize_data(context) + if request: + entry["request"] = self._sanitize_data(request) + if response: + entry["response"] = self._sanitize_data(response) + if duration_ms is not None: + entry["duration_ms"] = duration_ms + if success is not None: + entry["success"] = success + + if error: + entry["error"] = { + "type": type(error).__name__, + "message": str(error), + "traceback": traceback.format_exception(type(error), error, error.__traceback__), + } + + for key, value in kwargs.items(): + if key not in entry: + entry[key] = self._sanitize_data(value) + + try: + self.file_handle.write(json.dumps(entry, default=str) + "\n") + self.file_handle.flush() + except Exception as e: + print(f"Failed to write debug log: {e}", file=sys.stderr) + + def _sanitize_data(self, data: Any) -> Any: + """ + Sanitize data for JSON serialization. + Handles complex objects and removes sensitive information. + """ + if data is None: + return None + + if isinstance(data, (str, int, float, bool)): + return data + + if isinstance(data, (list, tuple)): + return [self._sanitize_data(item) for item in data] + + if isinstance(data, dict): + sanitized = {} + for key, value in data.items(): + key_lower = str(key).lower() + has_prefix = key_lower.startswith("has_") + + is_always_sensitive = not has_prefix and any( + sensitive in key_lower for sensitive in ["password", "token", "secret", "auth", "cookie"] + ) + + is_key_field = ( + "key" in key_lower + and not has_prefix + and not any(safe in key_lower for safe in ["_count", "_id", "_type", "kid", "keys_", "key_found"]) + ) + + should_redact = is_always_sensitive or (is_key_field and not self.log_keys) + + if should_redact: + sanitized[key] = "[REDACTED]" + else: + sanitized[key] = self._sanitize_data(value) + return sanitized + + if isinstance(data, bytes): + try: + return data.hex() + except Exception: + return "[BINARY_DATA]" + + if isinstance(data, Path): + return str(data) + + try: + return str(data) + except Exception: + return f"[{type(data).__name__}]" + + def log_operation_start(self, operation: str, **kwargs) -> str: + """ + Log the start of an operation and return an operation ID. + + Args: + operation: Name of the operation + **kwargs: Additional context + + Returns: + Operation ID that can be used to log the end of the operation + """ + op_id = str(uuid4())[:8] + self.log( + level="DEBUG", + operation=f"{operation}_start", + message=f"Starting operation: {operation}", + operation_id=op_id, + **kwargs, + ) + return op_id + + def log_operation_end( + self, operation: str, operation_id: str, success: bool = True, duration_ms: Optional[float] = None, **kwargs + ): + """ + Log the end of an operation. + + Args: + operation: Name of the operation + operation_id: Operation ID from log_operation_start + success: Whether the operation succeeded + duration_ms: Operation duration in milliseconds + **kwargs: Additional context + """ + self.log( + level="INFO" if success else "ERROR", + operation=f"{operation}_end", + message=f"Finished operation: {operation}", + operation_id=operation_id, + success=success, + duration_ms=duration_ms, + **kwargs, + ) + + def log_service_call(self, method: str, url: str, **kwargs): + """ + Log a service API call. + + Args: + method: HTTP method (GET, POST, etc.) + url: Request URL + **kwargs: Additional request details (headers, body, etc.) + """ + self.log(level="DEBUG", operation="service_call", request={"method": method, "url": url, **kwargs}) + + def log_drm_operation(self, drm_type: str, operation: str, **kwargs): + """ + Log a DRM operation (PSSH extraction, license request, key retrieval). + + Args: + drm_type: DRM type (Widevine, PlayReady, etc.) + operation: DRM operation name + **kwargs: Additional context (PSSH, KIDs, keys, etc.) + """ + self.log( + level="DEBUG", operation=f"drm_{operation}", message=f"{drm_type} {operation}", drm_type=drm_type, **kwargs + ) + + def log_vault_query(self, vault_name: str, operation: str, **kwargs): + """ + Log a vault query operation. + + Args: + vault_name: Name of the vault + operation: Vault operation (get_key, add_key, etc.) + **kwargs: Additional context (KID, key, success, etc.) + """ + self.log( + level="DEBUG", + operation=f"vault_{operation}", + message=f"Vault {vault_name}: {operation}", + vault=vault_name, + **kwargs, + ) + + def log_error(self, operation: str, error: Exception, **kwargs): + """ + Log an error with full context. + + Args: + operation: Operation that failed + error: Exception that occurred + **kwargs: Additional context + """ + self.log( + level="ERROR", + operation=operation, + message=f"Error in {operation}: {str(error)}", + error=error, + success=False, + **kwargs, + ) + + def close(self): + """Close the log file and clean up resources.""" + if self.file_handle: + self.log(level="INFO", operation="session_end", message="Debug logging session ended") + self.file_handle.close() + self.file_handle = None + + +# Global debug logger instance +_debug_logger: Optional[DebugLogger] = None + + +def get_debug_logger() -> Optional[DebugLogger]: + """Get the global debug logger instance.""" + return _debug_logger + + +def init_debug_logger(log_path: Optional[Path] = None, enabled: bool = False, log_keys: bool = False): + """ + Initialize the global debug logger. + + Args: + log_path: Path to the log file + enabled: Whether debug logging is enabled + log_keys: Whether to log decryption keys (for debugging key issues) + """ + global _debug_logger + if _debug_logger: + _debug_logger.close() + _debug_logger = DebugLogger(log_path=log_path, enabled=enabled, log_keys=log_keys) + + +def close_debug_logger(): + """Close the global debug logger.""" + global _debug_logger + if _debug_logger: + _debug_logger.close() + _debug_logger = None + + +__all__ = ( + "DebugLogger", + "get_debug_logger", + "init_debug_logger", + "close_debug_logger", +) diff --git a/unshackle/unshackle-example.yaml b/unshackle/unshackle-example.yaml index 74447c1..2b2685e 100644 --- a/unshackle/unshackle-example.yaml +++ b/unshackle/unshackle-example.yaml @@ -32,6 +32,24 @@ title_cache_enabled: true # Enable/disable title caching globally (default: true title_cache_time: 1800 # Cache duration in seconds (default: 1800 = 30 minutes) title_cache_max_retention: 86400 # Maximum cache retention for fallback when API fails (default: 86400 = 24 hours) +# Debug logging configuration +# Comprehensive JSON-based debug logging for troubleshooting and service development +debug: false # Enable structured JSON debug logging (default: false) + # When enabled with --debug flag or set to true: + # - Creates JSON Lines (.jsonl) log files with complete debugging context + # - Logs: session info, CLI params, service config, CDM details, authentication, + # titles, tracks metadata, DRM operations, vault queries, errors with stack traces + # - File location: logs/unshackle_debug_{service}_{timestamp}.jsonl + # - Also creates text log: logs/unshackle_root_{timestamp}.log + +debug_keys: false # Log decryption keys in debug logs (default: false) + # Set to true to include actual decryption keys in logs + # Useful for debugging key retrieval and decryption issues + # SECURITY NOTE: Passwords, tokens, cookies, and session tokens + # are ALWAYS redacted regardless of this setting + # Only affects: content_key, key fields (the actual CEKs) + # Never affects: kid, keys_count, key_id (metadata is always logged) + # Muxing configuration muxing: set_title: false @@ -239,7 +257,7 @@ headers: # Override default filenames used across unshackle filenames: - log: "unshackle_{name}_{time}.log" + debug_log: "unshackle_debug_{service}_{time}.jsonl" # JSON Lines debug log file config: "config.yaml" root_config: "unshackle.yaml" chapters: "Chapters_{title}_{random}.txt" From 133f91a2e850324a769deb296c010ac91bd65417 Mon Sep 17 00:00:00 2001 From: Andy Date: Fri, 17 Oct 2025 00:28:43 +0000 Subject: [PATCH 12/62] feat(cdm): add highly configurable CustomRemoteCDM for flexible API support Add new CustomRemoteCDM class to support custom CDM API providers with maximum configurability through YAML configuration alone. This addresses GitHub issue #26 by enabling integration with third-party CDM APIs. --- unshackle/commands/dl.py | 17 +- unshackle/core/cdm/__init__.py | 3 +- unshackle/core/cdm/custom_remote_cdm.py | 1085 +++++++++++++++++++++++ unshackle/unshackle-example.yaml | 68 ++ 4 files changed, 1168 insertions(+), 5 deletions(-) create mode 100644 unshackle/core/cdm/custom_remote_cdm.py diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index cba5ee9..94ca39b 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -41,7 +41,7 @@ from rich.text import Text from rich.tree import Tree from unshackle.core import binaries -from unshackle.core.cdm import DecryptLabsRemoteCDM +from unshackle.core.cdm import CustomRemoteCDM, DecryptLabsRemoteCDM from unshackle.core.config import config from unshackle.core.console import console from unshackle.core.constants import DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings @@ -988,7 +988,7 @@ class dl: sys.exit(1) if not forced_subs: - title.tracks.select_subtitles(lambda x: not x.forced or is_close_match(x.language, lang)) + title.tracks.select_subtitles(lambda x: not x.forced) # filter audio tracks # might have no audio tracks if part of the video, e.g. transport stream hls @@ -2055,8 +2055,9 @@ class dl: cdm_api = next(iter(x.copy() for x in config.remote_cdm if x["name"] == cdm_name), None) if cdm_api: - is_decrypt_lab = True if cdm_api.get("type") == "decrypt_labs" else False - if is_decrypt_lab: + cdm_type = cdm_api.get("type") + + if cdm_type == "decrypt_labs": del cdm_api["name"] del cdm_api["type"] @@ -2071,6 +2072,14 @@ class dl: # All DecryptLabs CDMs use DecryptLabsRemoteCDM return DecryptLabsRemoteCDM(service_name=service, vaults=self.vaults, **cdm_api) + + elif cdm_type == "custom_api": + del cdm_api["name"] + del cdm_api["type"] + + # All Custom API CDMs use CustomRemoteCDM + return CustomRemoteCDM(service_name=service, vaults=self.vaults, **cdm_api) + else: return RemoteCdm( device_type=cdm_api['Device Type'], diff --git a/unshackle/core/cdm/__init__.py b/unshackle/core/cdm/__init__.py index 10c0131..226f9ea 100644 --- a/unshackle/core/cdm/__init__.py +++ b/unshackle/core/cdm/__init__.py @@ -1,3 +1,4 @@ +from .custom_remote_cdm import CustomRemoteCDM from .decrypt_labs_remote_cdm import DecryptLabsRemoteCDM -__all__ = ["DecryptLabsRemoteCDM"] +__all__ = ["DecryptLabsRemoteCDM", "CustomRemoteCDM"] diff --git a/unshackle/core/cdm/custom_remote_cdm.py b/unshackle/core/cdm/custom_remote_cdm.py new file mode 100644 index 0000000..5ae6a17 --- /dev/null +++ b/unshackle/core/cdm/custom_remote_cdm.py @@ -0,0 +1,1085 @@ +from __future__ import annotations + +import base64 +import secrets +from typing import Any, Dict, List, Optional, Union +from uuid import UUID + +import requests +from pywidevine.cdm import Cdm as WidevineCdm +from pywidevine.device import DeviceTypes +from requests import Session + +from unshackle.core import __version__ +from unshackle.core.vaults import Vaults + + +class MockCertificateChain: + """Mock certificate chain for PlayReady compatibility.""" + + def __init__(self, name: str): + self._name = name + + def get_name(self) -> str: + return self._name + + +class Key: + """Key object compatible with pywidevine.""" + + def __init__(self, kid: str, key: str, type_: str = "CONTENT"): + if isinstance(kid, str): + clean_kid = kid.replace("-", "") + if len(clean_kid) == 32: + self.kid = UUID(hex=clean_kid) + else: + self.kid = UUID(hex=clean_kid.ljust(32, "0")) + else: + self.kid = kid + + if isinstance(key, str): + self.key = bytes.fromhex(key) + else: + self.key = key + + self.type = type_ + + +class CustomRemoteCDMExceptions: + """Exception classes for compatibility with pywidevine CDM.""" + + class InvalidSession(Exception): + """Raised when session ID is invalid.""" + + class TooManySessions(Exception): + """Raised when session limit is reached.""" + + class InvalidInitData(Exception): + """Raised when PSSH/init data is invalid.""" + + class InvalidLicenseType(Exception): + """Raised when license type is invalid.""" + + class InvalidLicenseMessage(Exception): + """Raised when license message is invalid.""" + + class InvalidContext(Exception): + """Raised when session has no context data.""" + + class SignatureMismatch(Exception): + """Raised when signature verification fails.""" + + +class CustomRemoteCDM: + """ + Highly Configurable Custom Remote CDM implementation. + + This class provides a maximally flexible CDM interface that can adapt to + ANY CDM API format through YAML configuration alone. It's designed to support + both current and future CDM providers without requiring code changes. + + Key Features: + - Fully configuration-driven behavior (all logic controlled via YAML) + - Pluggable authentication strategies (header, body, bearer, basic, custom) + - Flexible endpoint configuration (custom paths, methods, timeouts) + - Advanced parameter mapping (rename, add static, conditional, nested) + - Powerful response parsing (deep field access, type detection, transforms) + - Transform engine (base64, hex, JSON, custom key formats) + - Condition evaluation (response type detection, success validation) + - Compatible with both Widevine and PlayReady DRM schemes + - Vault integration for intelligent key caching + + Configuration Philosophy: + - 90% of new CDM providers: YAML config only + - 9% of cases: Add new transform type (minimal code) + - 1% of cases: Add new auth strategy (minimal code) + - 0% need to modify core request/response logic + + The class is designed to handle diverse API patterns including: + - Different authentication mechanisms (headers vs body vs tokens) + - Custom endpoint paths and HTTP methods + - Parameter name variations (scheme vs device, init_data vs pssh) + - Nested JSON structures in requests/responses + - Various key formats (JSON objects, colon-separated strings, etc.) + - Different response success indicators and error messages + - Conditional parameters based on device type or other factors + """ + + service_certificate_challenge = b"\x08\x04" + + def __init__( + self, + host: str, + service_name: Optional[str] = None, + vaults: Optional[Vaults] = None, + device: Optional[Dict[str, Any]] = None, + auth: Optional[Dict[str, Any]] = None, + endpoints: Optional[Dict[str, Any]] = None, + request_mapping: Optional[Dict[str, Any]] = None, + response_mapping: Optional[Dict[str, Any]] = None, + caching: Optional[Dict[str, Any]] = None, + legacy: Optional[Dict[str, Any]] = None, + timeout: int = 30, + **kwargs, + ): + """ + Initialize Custom Remote CDM with highly configurable options. + + Args: + host: Base URL for the CDM API + service_name: Service name for key caching and vault operations + vaults: Vaults instance for local key caching + device: Device configuration (name, type, system_id, security_level) + auth: Authentication configuration (type, credentials, headers) + endpoints: Endpoint configuration (paths, methods, timeouts) + request_mapping: Request transformation rules (param names, static params, transforms) + response_mapping: Response parsing rules (field locations, type detection, success conditions) + caching: Caching configuration (enabled, use_vaults, etc.) + legacy: Legacy mode configuration + timeout: Default request timeout in seconds + **kwargs: Additional configuration options for future extensibility + """ + self.host = host.rstrip("/") + self.service_name = service_name or "" + self.vaults = vaults + self.timeout = timeout + + # Device configuration + device = device or {} + self.device_name = device.get("name", "ChromeCDM") + self.device_type_str = device.get("type", "CHROME") + self.system_id = device.get("system_id", 26830) + self.security_level = device.get("security_level", 3) + + # Determine if this is a PlayReady CDM + self._is_playready = self.device_type_str.upper() == "PLAYREADY" or self.device_name in ["SL2", "SL3"] + + # Get device type enum for compatibility + if self.device_type_str: + self.device_type = self._get_device_type_enum(self.device_type_str) + + # Authentication configuration + self.auth_config = auth or {"type": "header", "header_name": "Authorization", "key": ""} + + # Endpoints configuration with defaults + endpoints = endpoints or {} + self.endpoints = { + "get_request": { + "path": endpoints.get("get_request", {}).get("path", "/get-challenge") + if isinstance(endpoints.get("get_request"), dict) + else endpoints.get("get_request", "/get-challenge"), + "method": ( + endpoints.get("get_request", {}).get("method", "POST") + if isinstance(endpoints.get("get_request"), dict) + else "POST" + ), + "timeout": ( + endpoints.get("get_request", {}).get("timeout", self.timeout) + if isinstance(endpoints.get("get_request"), dict) + else self.timeout + ), + }, + "decrypt_response": { + "path": endpoints.get("decrypt_response", {}).get("path", "/get-keys") + if isinstance(endpoints.get("decrypt_response"), dict) + else endpoints.get("decrypt_response", "/get-keys"), + "method": ( + endpoints.get("decrypt_response", {}).get("method", "POST") + if isinstance(endpoints.get("decrypt_response"), dict) + else "POST" + ), + "timeout": ( + endpoints.get("decrypt_response", {}).get("timeout", self.timeout) + if isinstance(endpoints.get("decrypt_response"), dict) + else self.timeout + ), + }, + } + + # Request mapping configuration + self.request_mapping = request_mapping or {} + + # Response mapping configuration + self.response_mapping = response_mapping or {} + + # Caching configuration + caching = caching or {} + self.caching_enabled = caching.get("enabled", True) + self.use_vaults = caching.get("use_vaults", True) and self.vaults is not None + self.check_cached_first = caching.get("check_cached_first", True) + + # Legacy configuration + self.legacy_config = legacy or {"enabled": False} + + # Session management + self._sessions: Dict[bytes, Dict[str, Any]] = {} + self._pssh_b64 = None + self._required_kids: Optional[List[str]] = None + + # HTTP session setup + self._http_session = Session() + self._http_session.headers.update( + {"Content-Type": "application/json", "User-Agent": f"unshackle-custom-cdm/{__version__}"} + ) + + # Apply custom headers from auth config + custom_headers = self.auth_config.get("custom_headers", {}) + if custom_headers: + self._http_session.headers.update(custom_headers) + + def _get_device_type_enum(self, device_type: str): + """Convert device type string to enum for compatibility.""" + device_type_upper = device_type.upper() + if device_type_upper == "ANDROID": + return DeviceTypes.ANDROID + elif device_type_upper == "CHROME": + return DeviceTypes.CHROME + else: + return DeviceTypes.CHROME + + @property + def is_playready(self) -> bool: + """Check if this CDM is in PlayReady mode.""" + return self._is_playready + + @property + def certificate_chain(self) -> MockCertificateChain: + """Mock certificate chain for PlayReady compatibility.""" + return MockCertificateChain(f"{self.device_name}_Custom_Remote") + + def set_pssh_b64(self, pssh_b64: str) -> None: + """Store base64-encoded PSSH data for PlayReady compatibility.""" + self._pssh_b64 = pssh_b64 + + def set_required_kids(self, kids: List[Union[str, UUID]]) -> None: + """ + Set the required Key IDs for intelligent caching decisions. + + This method enables the CDM to make smart decisions about when to request + additional keys via license challenges. When cached keys are available, + the CDM will compare them against the required KIDs to determine if a + license request is still needed for missing keys. + + Args: + kids: List of required Key IDs as UUIDs or hex strings + + Note: + Should be called by DRM classes (PlayReady/Widevine) before making + license challenge requests to enable optimal caching behavior. + """ + self._required_kids = [] + for kid in kids: + if isinstance(kid, UUID): + self._required_kids.append(str(kid).replace("-", "").lower()) + else: + self._required_kids.append(str(kid).replace("-", "").lower()) + + def _generate_session_id(self) -> bytes: + """Generate a unique session ID.""" + return secrets.token_bytes(16) + + def _get_init_data_from_pssh(self, pssh: Any) -> str: + """Extract init data from various PSSH formats.""" + if self.is_playready and self._pssh_b64: + return self._pssh_b64 + + if hasattr(pssh, "dumps"): + dumps_result = pssh.dumps() + + if isinstance(dumps_result, str): + try: + base64.b64decode(dumps_result) + return dumps_result + except Exception: + return base64.b64encode(dumps_result.encode("utf-8")).decode("utf-8") + else: + return base64.b64encode(dumps_result).decode("utf-8") + elif hasattr(pssh, "raw"): + raw_data = pssh.raw + if isinstance(raw_data, str): + raw_data = raw_data.encode("utf-8") + return base64.b64encode(raw_data).decode("utf-8") + elif hasattr(pssh, "__class__") and "WrmHeader" in pssh.__class__.__name__: + if self.is_playready: + raise ValueError("PlayReady WRM header received but no PSSH B64 was set via set_pssh_b64()") + + if hasattr(pssh, "raw_bytes"): + return base64.b64encode(pssh.raw_bytes).decode("utf-8") + elif hasattr(pssh, "bytes"): + return base64.b64encode(pssh.bytes).decode("utf-8") + else: + raise ValueError(f"Cannot extract PSSH data from WRM header type: {type(pssh)}") + else: + raise ValueError(f"Unsupported PSSH type: {type(pssh)}") + + def _get_nested_field(self, data: Dict[str, Any], field_path: str, default: Any = None) -> Any: + """ + Get a nested field from a dictionary using dot notation. + + Args: + data: Dictionary to extract field from + field_path: Field path using dot notation (e.g., "data.cached_keys") + default: Default value if field not found + + Returns: + Field value or default + + Examples: + _get_nested_field({"data": {"keys": [1,2,3]}}, "data.keys") -> [1,2,3] + _get_nested_field({"message": "success"}, "message") -> "success" + """ + if not field_path: + return default + + keys = field_path.split(".") + current = data + + for key in keys: + if isinstance(current, dict) and key in current: + current = current[key] + else: + return default + + return current + + def _apply_transform(self, value: Any, transform_type: str) -> Any: + """ + Apply a transformation to a value. + + Args: + value: Value to transform + transform_type: Type of transformation to apply + + Returns: + Transformed value + + Supported transforms: + - base64_encode: Encode bytes/string to base64 + - base64_decode: Decode base64 string to bytes + - hex_encode: Encode bytes to hex string + - hex_decode: Decode hex string to bytes + - json_stringify: Convert object to JSON string + - json_parse: Parse JSON string to object + - parse_key_string: Parse "kid:key" format strings + """ + if transform_type == "base64_encode": + if isinstance(value, str): + value = value.encode("utf-8") + return base64.b64encode(value).decode("utf-8") + + elif transform_type == "base64_decode": + if isinstance(value, str): + return base64.b64decode(value) + return value + + elif transform_type == "hex_encode": + if isinstance(value, bytes): + return value.hex() + elif isinstance(value, str): + return value.encode("utf-8").hex() + return value + + elif transform_type == "hex_decode": + if isinstance(value, str): + return bytes.fromhex(value) + return value + + elif transform_type == "json_stringify": + import json + + return json.dumps(value) + + elif transform_type == "json_parse": + import json + + if isinstance(value, str): + return json.loads(value) + return value + + elif transform_type == "parse_key_string": + # Handle key formats like "kid:key" or "--key kid:key" + if isinstance(value, str): + keys = [] + for line in value.split("\n"): + line = line.strip() + if line.startswith("--key "): + line = line[6:] + if ":" in line: + kid, key = line.split(":", 1) + keys.append({"kid": kid.strip(), "key": key.strip(), "type": "CONTENT"}) + return keys + return value + + # Unknown transform type - return value unchanged + return value + + def _evaluate_condition(self, condition: str, context: Dict[str, Any]) -> bool: + """ + Evaluate a simple condition against a context. + + Args: + condition: Condition string (e.g., "message == 'success'") + context: Context dictionary with values to check + + Returns: + True if condition is met, False otherwise + + Supported conditions: + - "field == value": Equality check + - "field != value": Inequality check + - "field == null": Null check + - "field != null": Not null check + - "field exists": Existence check + """ + condition = condition.strip() + + # Check for existence + if " exists" in condition: + field = condition.replace(" exists", "").strip() + return self._get_nested_field(context, field) is not None + + # Check for null comparisons + if " == null" in condition: + field = condition.replace(" == null", "").strip() + return self._get_nested_field(context, field) is None + + if " != null" in condition: + field = condition.replace(" != null", "").strip() + return self._get_nested_field(context, field) is not None + + # Check for equality + if " == " in condition: + parts = condition.split(" == ", 1) + field = parts[0].strip() + expected_value = parts[1].strip().strip("'\"") + actual_value = self._get_nested_field(context, field) + return str(actual_value) == expected_value + + # Check for inequality + if " != " in condition: + parts = condition.split(" != ", 1) + field = parts[0].strip() + expected_value = parts[1].strip().strip("'\"") + actual_value = self._get_nested_field(context, field) + return str(actual_value) != expected_value + + # Unknown condition format - return False + return False + + def _build_request_params( + self, endpoint_name: str, base_params: Dict[str, Any], session: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + """ + Build request parameters with mapping and transformations. + + Args: + endpoint_name: Name of the endpoint (e.g., "get_request", "decrypt_response") + base_params: Base parameters to transform + session: Optional session data for context + + Returns: + Transformed parameters dictionary + + This method applies the following transformations in order: + 1. Parameter name mappings (rename parameters) + 2. Static parameters (add fixed values) + 3. Conditional parameters (add based on conditions) + 4. Parameter transforms (apply data transformations) + 5. Nested parameter structure (create nested objects) + 6. Parameter exclusions (remove unwanted params) + """ + # Get mapping config for this endpoint + mapping_config = self.request_mapping.get(endpoint_name, {}) + + # Start with base parameters + params = base_params.copy() + + # 1. Apply parameter name mappings + param_names = mapping_config.get("param_names", {}) + if param_names: + renamed_params = {} + for old_name, new_name in param_names.items(): + if old_name in params: + renamed_params[new_name] = params.pop(old_name) + params.update(renamed_params) + + # 2. Add static parameters + static_params = mapping_config.get("static_params", {}) + if static_params: + params.update(static_params) + + # 3. Add conditional parameters + conditional_params = mapping_config.get("conditional_params", []) + for condition_block in conditional_params: + condition = condition_block.get("condition", "") + # Create context for condition evaluation + context = { + "device_type": self.device_type_str, + "device_name": self.device_name, + "is_playready": self._is_playready, + } + if session: + context.update(session) + + if self._evaluate_condition(condition, context): + params.update(condition_block.get("params", {})) + + # 4. Apply parameter transforms + transforms = mapping_config.get("transforms", []) + for transform in transforms: + param_name = transform.get("param") + transform_type = transform.get("type") + if param_name in params: + params[param_name] = self._apply_transform(params[param_name], transform_type) + + # 5. Handle nested parameter structure + nested_params = mapping_config.get("nested_params", {}) + if nested_params: + for parent_key, child_keys in nested_params.items(): + nested_obj = {} + for child_key in child_keys: + if child_key in params: + nested_obj[child_key] = params.pop(child_key) + if nested_obj: + params[parent_key] = nested_obj + + # 6. Exclude unwanted parameters + exclude_params = mapping_config.get("exclude_params", []) + for param_name in exclude_params: + params.pop(param_name, None) + + return params + + def _apply_authentication(self, session: Session) -> None: + """ + Apply authentication to the HTTP session based on auth configuration. + + Args: + session: requests.Session to apply authentication to + + Supported auth types: + - header: Add authentication header (e.g., x-api-key, Authorization) + - body: Authentication will be added to request body (handled in request building) + - bearer: Add Bearer token to Authorization header + - basic: Add HTTP Basic authentication + - query: Authentication will be added to query string (handled in request) + """ + auth_type = self.auth_config.get("type", "header") + + if auth_type == "header": + header_name = self.auth_config.get("header_name", "Authorization") + key = self.auth_config.get("key", "") + if key: + session.headers[header_name] = key + + elif auth_type == "bearer": + token = self.auth_config.get("bearer_token") or self.auth_config.get("key", "") + if token: + session.headers["Authorization"] = f"Bearer {token}" + + elif auth_type == "basic": + username = self.auth_config.get("username", "") + password = self.auth_config.get("password", "") + if username and password: + from requests.auth import HTTPBasicAuth + + session.auth = HTTPBasicAuth(username, password) + + def _parse_response_data(self, endpoint_name: str, response_data: Dict[str, Any]) -> Dict[str, Any]: + """ + Parse response data based on response mapping configuration. + + Args: + endpoint_name: Name of the endpoint (e.g., "get_request", "decrypt_response") + response_data: Raw response data from API + + Returns: + Parsed response with standardized field names + + This method extracts fields from the response using the response_mapping + configuration, handling nested fields, type detection, and transformations. + """ + # Get mapping config for this endpoint + mapping_config = self.response_mapping.get(endpoint_name, {}) + + # Extract fields based on mapping + fields_config = mapping_config.get("fields", {}) + parsed = {} + + for standard_name, field_path in fields_config.items(): + value = self._get_nested_field(response_data, field_path) + if value is not None: + parsed[standard_name] = value + + # Apply response transforms + transforms = mapping_config.get("transforms", []) + for transform in transforms: + field_name = transform.get("field") + transform_type = transform.get("type") + if field_name in parsed: + parsed[field_name] = self._apply_transform(parsed[field_name], transform_type) + + # Determine response type + response_types = mapping_config.get("response_types", []) + for response_type_config in response_types: + condition = response_type_config.get("condition", "") + if self._evaluate_condition(condition, parsed): + parsed["_response_type"] = response_type_config.get("type") + break + + # Check success conditions + success_conditions = mapping_config.get("success_conditions", []) + is_success = True + if success_conditions: + is_success = all(self._evaluate_condition(cond, parsed) for cond in success_conditions) + parsed["_is_success"] = is_success + + # Extract error messages if not successful + if not is_success: + error_fields = mapping_config.get("error_fields", ["error", "message", "details"]) + error_messages = [] + for error_field in error_fields: + error_msg = self._get_nested_field(response_data, error_field) + if error_msg and error_msg not in error_messages: + error_messages.append(str(error_msg)) + parsed["_error_message"] = " - ".join(error_messages) if error_messages else "Unknown error" + + return parsed + + def _parse_keys_from_response(self, endpoint_name: str, response_data: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Parse keys from response data using key field mapping. + + Args: + endpoint_name: Name of the endpoint + response_data: Parsed response data + + Returns: + List of key dictionaries with standardized format + """ + mapping_config = self.response_mapping.get(endpoint_name, {}) + key_fields = mapping_config.get("key_fields", {"kid": "kid", "key": "key", "type": "type"}) + + keys = [] + keys_data = response_data.get("keys", []) + + if isinstance(keys_data, list): + for key_obj in keys_data: + if isinstance(key_obj, dict): + kid = key_obj.get(key_fields.get("kid", "kid")) + key = key_obj.get(key_fields.get("key", "key")) + key_type = key_obj.get(key_fields.get("type", "type"), "CONTENT") + + if kid and key: + keys.append({"kid": str(kid), "key": str(key), "type": str(key_type)}) + + # Handle string format keys (e.g., "kid:key" format) + elif isinstance(keys_data, str): + keys = self._apply_transform(keys_data, "parse_key_string") + + return keys + + def open(self) -> bytes: + """ + Open a new CDM session. + + Returns: + Session identifier as bytes + """ + session_id = self._generate_session_id() + self._sessions[session_id] = { + "service_certificate": None, + "keys": [], + "pssh": None, + "challenge": None, + "remote_session_id": None, + "tried_cache": False, + "cached_keys": None, + } + return session_id + + def close(self, session_id: bytes) -> None: + """ + Close a CDM session and perform comprehensive cleanup. + + Args: + session_id: Session identifier + + Raises: + ValueError: If session ID is invalid + """ + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + session = self._sessions[session_id] + session.clear() + del self._sessions[session_id] + + def get_service_certificate(self, session_id: bytes) -> Optional[bytes]: + """ + Get the service certificate for a session. + + Args: + session_id: Session identifier + + Returns: + Service certificate if set, None otherwise + + Raises: + ValueError: If session ID is invalid + """ + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + return self._sessions[session_id]["service_certificate"] + + def set_service_certificate(self, session_id: bytes, certificate: Optional[Union[bytes, str]]) -> str: + """ + Set the service certificate for a session. + + Args: + session_id: Session identifier + certificate: Service certificate (bytes or base64 string) + + Returns: + Certificate status message + + Raises: + ValueError: If session ID is invalid + """ + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + if certificate is None: + if not self._is_playready and self.device_name == "L1": + certificate = WidevineCdm.common_privacy_cert + self._sessions[session_id]["service_certificate"] = base64.b64decode(certificate) + return "Using default Widevine common privacy certificate for L1" + else: + self._sessions[session_id]["service_certificate"] = None + return "No certificate set (not required for this device type)" + + if isinstance(certificate, str): + certificate = base64.b64decode(certificate) + + self._sessions[session_id]["service_certificate"] = certificate + return "Successfully set Service Certificate" + + def has_cached_keys(self, session_id: bytes) -> bool: + """ + Check if cached keys are available for the session. + + Args: + session_id: Session identifier + + Returns: + True if cached keys are available + + Raises: + ValueError: If session ID is invalid + """ + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + session = self._sessions[session_id] + session_keys = session.get("keys", []) + return len(session_keys) > 0 + + def get_license_challenge( + self, session_id: bytes, pssh_or_wrm: Any, license_type: str = "STREAMING", privacy_mode: bool = True + ) -> bytes: + """ + Generate a license challenge using the custom CDM API. + + This method implements intelligent caching logic that checks vaults first, + then attempts to retrieve cached keys from the API, and only makes a + license request if keys are missing. + + Args: + session_id: Session identifier + pssh_or_wrm: PSSH object or WRM header (for PlayReady compatibility) + license_type: Type of license (STREAMING, OFFLINE, AUTOMATIC) - for compatibility only + privacy_mode: Whether to use privacy mode - for compatibility only + + Returns: + License challenge as bytes, or empty bytes if available keys satisfy requirements + + Raises: + InvalidSession: If session ID is invalid + requests.RequestException: If API request fails + """ + _ = license_type, privacy_mode + + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + session = self._sessions[session_id] + session["pssh"] = pssh_or_wrm + init_data = self._get_init_data_from_pssh(pssh_or_wrm) + + # Check vaults for cached keys first + if self.use_vaults and self._required_kids: + vault_keys = [] + for kid_str in self._required_kids: + try: + clean_kid = kid_str.replace("-", "") + if len(clean_kid) == 32: + kid_uuid = UUID(hex=clean_kid) + else: + kid_uuid = UUID(hex=clean_kid.ljust(32, "0")) + key, _ = self.vaults.get_key(kid_uuid) + if key and key.count("0") != len(key): + vault_keys.append({"kid": kid_str, "key": key, "type": "CONTENT"}) + except (ValueError, TypeError): + continue + + if vault_keys: + vault_kids = set(k["kid"] for k in vault_keys) + required_kids = set(self._required_kids) + + if required_kids.issubset(vault_kids): + session["keys"] = vault_keys + return b"" + else: + session["vault_keys"] = vault_keys + + # Build request parameters + base_params = { + "scheme": self.device_name, + "init_data": init_data, + } + + if self.service_name: + base_params["service"] = self.service_name + + if session["service_certificate"]: + base_params["service_certificate"] = base64.b64encode(session["service_certificate"]).decode("utf-8") + + # Transform parameters based on configuration + request_params = self._build_request_params("get_request", base_params, session) + + # Apply authentication + self._apply_authentication(self._http_session) + + # Make API request + endpoint_config = self.endpoints["get_request"] + url = f"{self.host}{endpoint_config['path']}" + timeout = endpoint_config["timeout"] + + response = self._http_session.post(url, json=request_params, timeout=timeout) + + if response.status_code != 200: + raise requests.RequestException(f"API request failed: {response.status_code} {response.text}") + + # Parse response + response_data = response.json() + parsed_response = self._parse_response_data("get_request", response_data) + + # Check if request was successful + if not parsed_response.get("_is_success", False): + error_msg = parsed_response.get("_error_message", "Unknown error") + raise requests.RequestException(f"API error: {error_msg}") + + # Determine response type + response_type = parsed_response.get("_response_type") + + # Handle cached keys response + if response_type == "cached_keys" or "cached_keys" in parsed_response: + cached_keys = self._parse_keys_from_response("get_request", parsed_response) + + all_available_keys = list(cached_keys) + if "vault_keys" in session: + all_available_keys.extend(session["vault_keys"]) + + session["keys"] = all_available_keys + session["tried_cache"] = True + + # Check if we have all required keys + if self._required_kids: + available_kids = set() + for key in all_available_keys: + if isinstance(key, dict) and "kid" in key: + available_kids.add(key["kid"].replace("-", "").lower()) + + required_kids = set(self._required_kids) + missing_kids = required_kids - available_kids + + if not missing_kids: + return b"" + + # Store cached keys for later combination + session["cached_keys"] = cached_keys + + # Handle license request response or fetch license if keys missing + challenge = parsed_response.get("challenge") + remote_session_id = parsed_response.get("session_id") + + if challenge and remote_session_id: + # Decode challenge if it's base64 + if isinstance(challenge, str): + try: + challenge = base64.b64decode(challenge) + except Exception: + challenge = challenge.encode("utf-8") + + session["challenge"] = challenge + session["remote_session_id"] = remote_session_id + return challenge + + # If we have some keys but not all, return empty to skip license parsing + if session.get("keys"): + return b"" + + raise requests.RequestException("API response did not contain challenge or cached keys") + + def parse_license(self, session_id: bytes, license_message: Union[bytes, str]) -> None: + """ + Parse license response using the custom CDM API. + + This method intelligently combines cached keys with newly obtained license keys, + avoiding duplicates while ensuring all required keys are available. + + Args: + session_id: Session identifier + license_message: License response from license server + + Raises: + ValueError: If session ID is invalid or no challenge available + requests.RequestException: If API request fails + """ + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + session = self._sessions[session_id] + + # If we already have keys and no cached keys to combine, skip + if session["keys"] and not session.get("cached_keys"): + return + + # Ensure we have a challenge and session ID + if not session.get("challenge") or not session.get("remote_session_id"): + raise ValueError("No challenge available - call get_license_challenge first") + + # Prepare license message + if isinstance(license_message, str): + if self.is_playready and license_message.strip().startswith(" List[Key]: + """ + Get keys from the session. + + Args: + session_id: Session identifier + type_: Optional key type filter (CONTENT, SIGNING, etc.) + + Returns: + List of Key objects + + Raises: + InvalidSession: If session ID is invalid + """ + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + key_dicts = self._sessions[session_id]["keys"] + keys = [Key(kid=k["kid"], key=k["key"], type_=k["type"]) for k in key_dicts] + + if type_: + keys = [key for key in keys if key.type == type_] + + return keys + + +__all__ = ["CustomRemoteCDM"] diff --git a/unshackle/unshackle-example.yaml b/unshackle/unshackle-example.yaml index 0e45840..2b837fb 100644 --- a/unshackle/unshackle-example.yaml +++ b/unshackle/unshackle-example.yaml @@ -127,6 +127,74 @@ cdm: default: netflix_standard_l3 # Use pywidevine Serve-compliant Remote CDMs + + # Example: Custom CDM API Configuration + # This demonstrates the highly configurable custom_api type that can adapt to any CDM API format + # - name: "chrome" + # type: "custom_api" + # host: "http://remotecdm.test/" + # timeout: 30 + # device: + # name: "ChromeCDM" + # type: "CHROME" + # system_id: 34312 + # security_level: 3 + # auth: + # type: "header" + # header_name: "x-api-key" + # key: "YOUR_API_KEY_HERE" + # custom_headers: + # User-Agent: "Unshackle/2.0.0" + # endpoints: + # get_request: + # path: "/get-challenge" + # method: "POST" + # timeout: 30 + # decrypt_response: + # path: "/get-keys" + # method: "POST" + # timeout: 30 + # request_mapping: + # get_request: + # param_names: + # scheme: "device" + # init_data: "init_data" + # static_params: + # scheme: "Widevine" + # decrypt_response: + # param_names: + # scheme: "device" + # license_request: "license_request" + # license_response: "license_response" + # static_params: + # scheme: "Widevine" + # response_mapping: + # get_request: + # fields: + # challenge: "challenge" + # session_id: "session_id" + # message: "message" + # message_type: "message_type" + # response_types: + # - condition: "message_type == 'license-request'" + # type: "license_request" + # success_conditions: + # - "message == 'success'" + # decrypt_response: + # fields: + # keys: "keys" + # message: "message" + # key_fields: + # kid: "kid" + # key: "key" + # type: "type" + # success_conditions: + # - "message == 'success'" + # caching: + # enabled: true + # use_vaults: true + # check_cached_first: true + remote_cdm: - name: "chrome" device_name: chrome From 888647ad64f55fa315e5303943850784a2fab12b Mon Sep 17 00:00:00 2001 From: Andy Date: Fri, 17 Oct 2025 20:21:47 +0000 Subject: [PATCH 13/62] feat(proxies): add WindscribeVPN proxy provider support Add WindscribeVPN as a new proxy provider option, following the same pattern as NordVPN and SurfsharkVPN implementations. Fixes: #29 --- pyproject.toml | 2 +- unshackle/commands/dl.py | 4 +- unshackle/core/proxies/__init__.py | 3 +- unshackle/core/proxies/windscribevpn.py | 99 +++++++++++++++ unshackle/unshackle-example.yaml | 6 + uv.lock | 153 +++++++++++++----------- 6 files changed, 196 insertions(+), 71 deletions(-) create mode 100644 unshackle/core/proxies/windscribevpn.py diff --git a/pyproject.toml b/pyproject.toml index dcf9d0a..cd1552d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "unshackle" -version = "1.4.8" +version = "2.0.0" description = "Modular Movie, TV, and Music Archival Software." authors = [{ name = "unshackle team" }] requires-python = ">=3.10,<3.13" diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 94ca39b..c6cd61d 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -48,7 +48,7 @@ from unshackle.core.constants import DOWNLOAD_LICENCE_ONLY, AnyTrack, context_se from unshackle.core.credential import Credential from unshackle.core.drm import DRM_T, PlayReady, Widevine from unshackle.core.events import events -from unshackle.core.proxies import Basic, Hola, NordVPN, SurfsharkVPN +from unshackle.core.proxies import Basic, Hola, NordVPN, SurfsharkVPN, WindscribeVPN from unshackle.core.service import Service from unshackle.core.services import Services from unshackle.core.titles import Movie, Movies, Series, Song, Title_T @@ -464,6 +464,8 @@ class dl: self.proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"])) if config.proxy_providers.get("surfsharkvpn"): self.proxy_providers.append(SurfsharkVPN(**config.proxy_providers["surfsharkvpn"])) + if config.proxy_providers.get("windscribevpn"): + self.proxy_providers.append(WindscribeVPN(**config.proxy_providers["windscribevpn"])) if binaries.HolaProxy: self.proxy_providers.append(Hola()) for proxy_provider in self.proxy_providers: diff --git a/unshackle/core/proxies/__init__.py b/unshackle/core/proxies/__init__.py index 10008c1..ecb97de 100644 --- a/unshackle/core/proxies/__init__.py +++ b/unshackle/core/proxies/__init__.py @@ -2,5 +2,6 @@ from .basic import Basic from .hola import Hola from .nordvpn import NordVPN from .surfsharkvpn import SurfsharkVPN +from .windscribevpn import WindscribeVPN -__all__ = ("Basic", "Hola", "NordVPN", "SurfsharkVPN") +__all__ = ("Basic", "Hola", "NordVPN", "SurfsharkVPN", "WindscribeVPN") diff --git a/unshackle/core/proxies/windscribevpn.py b/unshackle/core/proxies/windscribevpn.py new file mode 100644 index 0000000..de4458e --- /dev/null +++ b/unshackle/core/proxies/windscribevpn.py @@ -0,0 +1,99 @@ +import json +import random +import re +from typing import Optional + +import requests + +from unshackle.core.proxies.proxy import Proxy + + +class WindscribeVPN(Proxy): + def __init__(self, username: str, password: str, server_map: Optional[dict[str, str]] = None): + """ + Proxy Service using WindscribeVPN Service Credentials. + + A username and password must be provided. These are Service Credentials, not your Login Credentials. + The Service Credentials can be found here: https://windscribe.com/getconfig/openvpn + """ + if not username: + raise ValueError("No Username was provided to the WindscribeVPN Proxy Service.") + if not password: + raise ValueError("No Password was provided to the WindscribeVPN Proxy Service.") + + if server_map is not None and not isinstance(server_map, dict): + raise TypeError(f"Expected server_map to be a dict mapping a region to a hostname, not '{server_map!r}'.") + + self.username = username + self.password = password + self.server_map = server_map or {} + + self.countries = self.get_countries() + + def __repr__(self) -> str: + countries = len(set(x.get("country_code") for x in self.countries if x.get("country_code"))) + servers = sum( + len(host) + for location in self.countries + for group in location.get("groups", []) + for host in group.get("hosts", []) + ) + + return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})" + + def get_proxy(self, query: str) -> Optional[str]: + """ + Get an HTTPS proxy URI for a WindscribeVPN server. + """ + query = query.lower() + + if query in self.server_map: + hostname = self.server_map[query] + else: + if re.match(r"^[a-z]+$", query): + hostname = self.get_random_server(query) + else: + raise ValueError(f"The query provided is unsupported and unrecognized: {query}") + + if not hostname: + return None + + return f"https://{self.username}:{self.password}@{hostname}:443" + + def get_random_server(self, country_code: str) -> Optional[str]: + """ + Get a random server hostname for a country. + + Returns None if no servers are available for the country. + """ + for location in self.countries: + if location.get("country_code", "").lower() == country_code.lower(): + hostnames = [] + for group in location.get("groups", []): + for host in group.get("hosts", []): + if hostname := host.get("hostname"): + hostnames.append(hostname) + + if hostnames: + return random.choice(hostnames) + + return None + + @staticmethod + def get_countries() -> list[dict]: + """Get a list of available Countries and their metadata.""" + res = requests.get( + url="https://assets.windscribe.com/serverlist/firefox/1/1", + headers={ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Content-Type": "application/json", + }, + ) + if not res.ok: + raise ValueError(f"Failed to get a list of WindscribeVPN locations [{res.status_code}]") + + try: + data = res.json() + return data.get("data", []) + except json.JSONDecodeError: + raise ValueError("Could not decode list of WindscribeVPN locations, not JSON data.") diff --git a/unshackle/unshackle-example.yaml b/unshackle/unshackle-example.yaml index 2b837fb..8056d47 100644 --- a/unshackle/unshackle-example.yaml +++ b/unshackle/unshackle-example.yaml @@ -407,6 +407,12 @@ proxy_providers: us: 3844 # force US server #3844 for US proxies gb: 2697 # force GB server #2697 for GB proxies au: 4621 # force AU server #4621 for AU proxies + windscribevpn: + username: your_windscribe_username # Service credentials from https://windscribe.com/getconfig/openvpn + password: your_windscribe_password # Service credentials (not your login password) + server_map: + us: "us-central-096.totallyacdn.com" # force US server + gb: "uk-london-055.totallyacdn.com" # force GB server basic: GB: - "socks5://username:password@bhx.socks.ipvanish.com:1080" # 1 (Birmingham) diff --git a/uv.lock b/uv.lock index 1bdb091..b9b0fd2 100644 --- a/uv.lock +++ b/uv.lock @@ -670,67 +670,72 @@ wheels = [ [[package]] name = "lxml" -version = "5.4.0" +version = "6.0.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/76/3d/14e82fc7c8fb1b7761f7e748fd47e2ec8276d137b6acfe5a4bb73853e08f/lxml-5.4.0.tar.gz", hash = "sha256:d12832e1dbea4be280b22fd0ea7c9b87f0d8fc51ba06e92dc62d52f804f78ebd", size = 3679479, upload-time = "2025-04-23T01:50:29.322Z" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/88/262177de60548e5a2bfc46ad28232c9e9cbde697bd94132aeb80364675cb/lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62", size = 4073426, upload-time = "2025-09-22T04:04:59.287Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/1f/a3b6b74a451ceb84b471caa75c934d2430a4d84395d38ef201d539f38cd1/lxml-5.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e7bc6df34d42322c5289e37e9971d6ed114e3776b45fa879f734bded9d1fea9c", size = 8076838, upload-time = "2025-04-23T01:44:29.325Z" }, - { url = "https://files.pythonhosted.org/packages/36/af/a567a55b3e47135b4d1f05a1118c24529104c003f95851374b3748139dc1/lxml-5.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6854f8bd8a1536f8a1d9a3655e6354faa6406621cf857dc27b681b69860645c7", size = 4381827, upload-time = "2025-04-23T01:44:33.345Z" }, - { url = "https://files.pythonhosted.org/packages/50/ba/4ee47d24c675932b3eb5b6de77d0f623c2db6dc466e7a1f199792c5e3e3a/lxml-5.4.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:696ea9e87442467819ac22394ca36cb3d01848dad1be6fac3fb612d3bd5a12cf", size = 5204098, upload-time = "2025-04-23T01:44:35.809Z" }, - { url = "https://files.pythonhosted.org/packages/f2/0f/b4db6dfebfefe3abafe360f42a3d471881687fd449a0b86b70f1f2683438/lxml-5.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ef80aeac414f33c24b3815ecd560cee272786c3adfa5f31316d8b349bfade28", size = 4930261, upload-time = "2025-04-23T01:44:38.271Z" }, - { url = "https://files.pythonhosted.org/packages/0b/1f/0bb1bae1ce056910f8db81c6aba80fec0e46c98d77c0f59298c70cd362a3/lxml-5.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b9c2754cef6963f3408ab381ea55f47dabc6f78f4b8ebb0f0b25cf1ac1f7609", size = 5529621, upload-time = "2025-04-23T01:44:40.921Z" }, - { url = "https://files.pythonhosted.org/packages/21/f5/e7b66a533fc4a1e7fa63dd22a1ab2ec4d10319b909211181e1ab3e539295/lxml-5.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a62cc23d754bb449d63ff35334acc9f5c02e6dae830d78dab4dd12b78a524f4", size = 4983231, upload-time = "2025-04-23T01:44:43.871Z" }, - { url = "https://files.pythonhosted.org/packages/11/39/a38244b669c2d95a6a101a84d3c85ba921fea827e9e5483e93168bf1ccb2/lxml-5.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f82125bc7203c5ae8633a7d5d20bcfdff0ba33e436e4ab0abc026a53a8960b7", size = 5084279, upload-time = "2025-04-23T01:44:46.632Z" }, - { url = "https://files.pythonhosted.org/packages/db/64/48cac242347a09a07740d6cee7b7fd4663d5c1abd65f2e3c60420e231b27/lxml-5.4.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:b67319b4aef1a6c56576ff544b67a2a6fbd7eaee485b241cabf53115e8908b8f", size = 4927405, upload-time = "2025-04-23T01:44:49.843Z" }, - { url = "https://files.pythonhosted.org/packages/98/89/97442835fbb01d80b72374f9594fe44f01817d203fa056e9906128a5d896/lxml-5.4.0-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:a8ef956fce64c8551221f395ba21d0724fed6b9b6242ca4f2f7beb4ce2f41997", size = 5550169, upload-time = "2025-04-23T01:44:52.791Z" }, - { url = "https://files.pythonhosted.org/packages/f1/97/164ca398ee654eb21f29c6b582685c6c6b9d62d5213abc9b8380278e9c0a/lxml-5.4.0-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:0a01ce7d8479dce84fc03324e3b0c9c90b1ece9a9bb6a1b6c9025e7e4520e78c", size = 5062691, upload-time = "2025-04-23T01:44:56.108Z" }, - { url = "https://files.pythonhosted.org/packages/d0/bc/712b96823d7feb53482d2e4f59c090fb18ec7b0d0b476f353b3085893cda/lxml-5.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:91505d3ddebf268bb1588eb0f63821f738d20e1e7f05d3c647a5ca900288760b", size = 5133503, upload-time = "2025-04-23T01:44:59.222Z" }, - { url = "https://files.pythonhosted.org/packages/d4/55/a62a39e8f9da2a8b6002603475e3c57c870cd9c95fd4b94d4d9ac9036055/lxml-5.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a3bcdde35d82ff385f4ede021df801b5c4a5bcdfb61ea87caabcebfc4945dc1b", size = 4999346, upload-time = "2025-04-23T01:45:02.088Z" }, - { url = "https://files.pythonhosted.org/packages/ea/47/a393728ae001b92bb1a9e095e570bf71ec7f7fbae7688a4792222e56e5b9/lxml-5.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:aea7c06667b987787c7d1f5e1dfcd70419b711cdb47d6b4bb4ad4b76777a0563", size = 5627139, upload-time = "2025-04-23T01:45:04.582Z" }, - { url = "https://files.pythonhosted.org/packages/5e/5f/9dcaaad037c3e642a7ea64b479aa082968de46dd67a8293c541742b6c9db/lxml-5.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a7fb111eef4d05909b82152721a59c1b14d0f365e2be4c742a473c5d7372f4f5", size = 5465609, upload-time = "2025-04-23T01:45:07.649Z" }, - { url = "https://files.pythonhosted.org/packages/a7/0a/ebcae89edf27e61c45023005171d0ba95cb414ee41c045ae4caf1b8487fd/lxml-5.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:43d549b876ce64aa18b2328faff70f5877f8c6dede415f80a2f799d31644d776", size = 5192285, upload-time = "2025-04-23T01:45:10.456Z" }, - { url = "https://files.pythonhosted.org/packages/42/ad/cc8140ca99add7d85c92db8b2354638ed6d5cc0e917b21d36039cb15a238/lxml-5.4.0-cp310-cp310-win32.whl", hash = "sha256:75133890e40d229d6c5837b0312abbe5bac1c342452cf0e12523477cd3aa21e7", size = 3477507, upload-time = "2025-04-23T01:45:12.474Z" }, - { url = "https://files.pythonhosted.org/packages/e9/39/597ce090da1097d2aabd2f9ef42187a6c9c8546d67c419ce61b88b336c85/lxml-5.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:de5b4e1088523e2b6f730d0509a9a813355b7f5659d70eb4f319c76beea2e250", size = 3805104, upload-time = "2025-04-23T01:45:15.104Z" }, - { url = "https://files.pythonhosted.org/packages/81/2d/67693cc8a605a12e5975380d7ff83020dcc759351b5a066e1cced04f797b/lxml-5.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:98a3912194c079ef37e716ed228ae0dcb960992100461b704aea4e93af6b0bb9", size = 8083240, upload-time = "2025-04-23T01:45:18.566Z" }, - { url = "https://files.pythonhosted.org/packages/73/53/b5a05ab300a808b72e848efd152fe9c022c0181b0a70b8bca1199f1bed26/lxml-5.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ea0252b51d296a75f6118ed0d8696888e7403408ad42345d7dfd0d1e93309a7", size = 4387685, upload-time = "2025-04-23T01:45:21.387Z" }, - { url = "https://files.pythonhosted.org/packages/d8/cb/1a3879c5f512bdcd32995c301886fe082b2edd83c87d41b6d42d89b4ea4d/lxml-5.4.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b92b69441d1bd39f4940f9eadfa417a25862242ca2c396b406f9272ef09cdcaa", size = 4991164, upload-time = "2025-04-23T01:45:23.849Z" }, - { url = "https://files.pythonhosted.org/packages/f9/94/bbc66e42559f9d04857071e3b3d0c9abd88579367fd2588a4042f641f57e/lxml-5.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20e16c08254b9b6466526bc1828d9370ee6c0d60a4b64836bc3ac2917d1e16df", size = 4746206, upload-time = "2025-04-23T01:45:26.361Z" }, - { url = "https://files.pythonhosted.org/packages/66/95/34b0679bee435da2d7cae895731700e519a8dfcab499c21662ebe671603e/lxml-5.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7605c1c32c3d6e8c990dd28a0970a3cbbf1429d5b92279e37fda05fb0c92190e", size = 5342144, upload-time = "2025-04-23T01:45:28.939Z" }, - { url = "https://files.pythonhosted.org/packages/e0/5d/abfcc6ab2fa0be72b2ba938abdae1f7cad4c632f8d552683ea295d55adfb/lxml-5.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ecf4c4b83f1ab3d5a7ace10bafcb6f11df6156857a3c418244cef41ca9fa3e44", size = 4825124, upload-time = "2025-04-23T01:45:31.361Z" }, - { url = "https://files.pythonhosted.org/packages/5a/78/6bd33186c8863b36e084f294fc0a5e5eefe77af95f0663ef33809cc1c8aa/lxml-5.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cef4feae82709eed352cd7e97ae062ef6ae9c7b5dbe3663f104cd2c0e8d94ba", size = 4876520, upload-time = "2025-04-23T01:45:34.191Z" }, - { url = "https://files.pythonhosted.org/packages/3b/74/4d7ad4839bd0fc64e3d12da74fc9a193febb0fae0ba6ebd5149d4c23176a/lxml-5.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:df53330a3bff250f10472ce96a9af28628ff1f4efc51ccba351a8820bca2a8ba", size = 4765016, upload-time = "2025-04-23T01:45:36.7Z" }, - { url = "https://files.pythonhosted.org/packages/24/0d/0a98ed1f2471911dadfc541003ac6dd6879fc87b15e1143743ca20f3e973/lxml-5.4.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:aefe1a7cb852fa61150fcb21a8c8fcea7b58c4cb11fbe59c97a0a4b31cae3c8c", size = 5362884, upload-time = "2025-04-23T01:45:39.291Z" }, - { url = "https://files.pythonhosted.org/packages/48/de/d4f7e4c39740a6610f0f6959052b547478107967362e8424e1163ec37ae8/lxml-5.4.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ef5a7178fcc73b7d8c07229e89f8eb45b2908a9238eb90dcfc46571ccf0383b8", size = 4902690, upload-time = "2025-04-23T01:45:42.386Z" }, - { url = "https://files.pythonhosted.org/packages/07/8c/61763abd242af84f355ca4ef1ee096d3c1b7514819564cce70fd18c22e9a/lxml-5.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d2ed1b3cb9ff1c10e6e8b00941bb2e5bb568b307bfc6b17dffbbe8be5eecba86", size = 4944418, upload-time = "2025-04-23T01:45:46.051Z" }, - { url = "https://files.pythonhosted.org/packages/f9/c5/6d7e3b63e7e282619193961a570c0a4c8a57fe820f07ca3fe2f6bd86608a/lxml-5.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:72ac9762a9f8ce74c9eed4a4e74306f2f18613a6b71fa065495a67ac227b3056", size = 4827092, upload-time = "2025-04-23T01:45:48.943Z" }, - { url = "https://files.pythonhosted.org/packages/71/4a/e60a306df54680b103348545706a98a7514a42c8b4fbfdcaa608567bb065/lxml-5.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f5cb182f6396706dc6cc1896dd02b1c889d644c081b0cdec38747573db88a7d7", size = 5418231, upload-time = "2025-04-23T01:45:51.481Z" }, - { url = "https://files.pythonhosted.org/packages/27/f2/9754aacd6016c930875854f08ac4b192a47fe19565f776a64004aa167521/lxml-5.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:3a3178b4873df8ef9457a4875703488eb1622632a9cee6d76464b60e90adbfcd", size = 5261798, upload-time = "2025-04-23T01:45:54.146Z" }, - { url = "https://files.pythonhosted.org/packages/38/a2/0c49ec6941428b1bd4f280650d7b11a0f91ace9db7de32eb7aa23bcb39ff/lxml-5.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e094ec83694b59d263802ed03a8384594fcce477ce484b0cbcd0008a211ca751", size = 4988195, upload-time = "2025-04-23T01:45:56.685Z" }, - { url = "https://files.pythonhosted.org/packages/7a/75/87a3963a08eafc46a86c1131c6e28a4de103ba30b5ae903114177352a3d7/lxml-5.4.0-cp311-cp311-win32.whl", hash = "sha256:4329422de653cdb2b72afa39b0aa04252fca9071550044904b2e7036d9d97fe4", size = 3474243, upload-time = "2025-04-23T01:45:58.863Z" }, - { url = "https://files.pythonhosted.org/packages/fa/f9/1f0964c4f6c2be861c50db380c554fb8befbea98c6404744ce243a3c87ef/lxml-5.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd3be6481ef54b8cfd0e1e953323b7aa9d9789b94842d0e5b142ef4bb7999539", size = 3815197, upload-time = "2025-04-23T01:46:01.096Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4c/d101ace719ca6a4ec043eb516fcfcb1b396a9fccc4fcd9ef593df34ba0d5/lxml-5.4.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b5aff6f3e818e6bdbbb38e5967520f174b18f539c2b9de867b1e7fde6f8d95a4", size = 8127392, upload-time = "2025-04-23T01:46:04.09Z" }, - { url = "https://files.pythonhosted.org/packages/11/84/beddae0cec4dd9ddf46abf156f0af451c13019a0fa25d7445b655ba5ccb7/lxml-5.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:942a5d73f739ad7c452bf739a62a0f83e2578afd6b8e5406308731f4ce78b16d", size = 4415103, upload-time = "2025-04-23T01:46:07.227Z" }, - { url = "https://files.pythonhosted.org/packages/d0/25/d0d93a4e763f0462cccd2b8a665bf1e4343dd788c76dcfefa289d46a38a9/lxml-5.4.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:460508a4b07364d6abf53acaa0a90b6d370fafde5693ef37602566613a9b0779", size = 5024224, upload-time = "2025-04-23T01:46:10.237Z" }, - { url = "https://files.pythonhosted.org/packages/31/ce/1df18fb8f7946e7f3388af378b1f34fcf253b94b9feedb2cec5969da8012/lxml-5.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:529024ab3a505fed78fe3cc5ddc079464e709f6c892733e3f5842007cec8ac6e", size = 4769913, upload-time = "2025-04-23T01:46:12.757Z" }, - { url = "https://files.pythonhosted.org/packages/4e/62/f4a6c60ae7c40d43657f552f3045df05118636be1165b906d3423790447f/lxml-5.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ca56ebc2c474e8f3d5761debfd9283b8b18c76c4fc0967b74aeafba1f5647f9", size = 5290441, upload-time = "2025-04-23T01:46:16.037Z" }, - { url = "https://files.pythonhosted.org/packages/9e/aa/04f00009e1e3a77838c7fc948f161b5d2d5de1136b2b81c712a263829ea4/lxml-5.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a81e1196f0a5b4167a8dafe3a66aa67c4addac1b22dc47947abd5d5c7a3f24b5", size = 4820165, upload-time = "2025-04-23T01:46:19.137Z" }, - { url = "https://files.pythonhosted.org/packages/c9/1f/e0b2f61fa2404bf0f1fdf1898377e5bd1b74cc9b2cf2c6ba8509b8f27990/lxml-5.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00b8686694423ddae324cf614e1b9659c2edb754de617703c3d29ff568448df5", size = 4932580, upload-time = "2025-04-23T01:46:21.963Z" }, - { url = "https://files.pythonhosted.org/packages/24/a2/8263f351b4ffe0ed3e32ea7b7830f845c795349034f912f490180d88a877/lxml-5.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c5681160758d3f6ac5b4fea370495c48aac0989d6a0f01bb9a72ad8ef5ab75c4", size = 4759493, upload-time = "2025-04-23T01:46:24.316Z" }, - { url = "https://files.pythonhosted.org/packages/05/00/41db052f279995c0e35c79d0f0fc9f8122d5b5e9630139c592a0b58c71b4/lxml-5.4.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:2dc191e60425ad70e75a68c9fd90ab284df64d9cd410ba8d2b641c0c45bc006e", size = 5324679, upload-time = "2025-04-23T01:46:27.097Z" }, - { url = "https://files.pythonhosted.org/packages/1d/be/ee99e6314cdef4587617d3b3b745f9356d9b7dd12a9663c5f3b5734b64ba/lxml-5.4.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:67f779374c6b9753ae0a0195a892a1c234ce8416e4448fe1e9f34746482070a7", size = 4890691, upload-time = "2025-04-23T01:46:30.009Z" }, - { url = "https://files.pythonhosted.org/packages/ad/36/239820114bf1d71f38f12208b9c58dec033cbcf80101cde006b9bde5cffd/lxml-5.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:79d5bfa9c1b455336f52343130b2067164040604e41f6dc4d8313867ed540079", size = 4955075, upload-time = "2025-04-23T01:46:32.33Z" }, - { url = "https://files.pythonhosted.org/packages/d4/e1/1b795cc0b174efc9e13dbd078a9ff79a58728a033142bc6d70a1ee8fc34d/lxml-5.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d3c30ba1c9b48c68489dc1829a6eede9873f52edca1dda900066542528d6b20", size = 4838680, upload-time = "2025-04-23T01:46:34.852Z" }, - { url = "https://files.pythonhosted.org/packages/72/48/3c198455ca108cec5ae3662ae8acd7fd99476812fd712bb17f1b39a0b589/lxml-5.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1af80c6316ae68aded77e91cd9d80648f7dd40406cef73df841aa3c36f6907c8", size = 5391253, upload-time = "2025-04-23T01:46:37.608Z" }, - { url = "https://files.pythonhosted.org/packages/d6/10/5bf51858971c51ec96cfc13e800a9951f3fd501686f4c18d7d84fe2d6352/lxml-5.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4d885698f5019abe0de3d352caf9466d5de2baded00a06ef3f1216c1a58ae78f", size = 5261651, upload-time = "2025-04-23T01:46:40.183Z" }, - { url = "https://files.pythonhosted.org/packages/2b/11/06710dd809205377da380546f91d2ac94bad9ff735a72b64ec029f706c85/lxml-5.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aea53d51859b6c64e7c51d522c03cc2c48b9b5d6172126854cc7f01aa11f52bc", size = 5024315, upload-time = "2025-04-23T01:46:43.333Z" }, - { url = "https://files.pythonhosted.org/packages/f5/b0/15b6217834b5e3a59ebf7f53125e08e318030e8cc0d7310355e6edac98ef/lxml-5.4.0-cp312-cp312-win32.whl", hash = "sha256:d90b729fd2732df28130c064aac9bb8aff14ba20baa4aee7bd0795ff1187545f", size = 3486149, upload-time = "2025-04-23T01:46:45.684Z" }, - { url = "https://files.pythonhosted.org/packages/91/1e/05ddcb57ad2f3069101611bd5f5084157d90861a2ef460bf42f45cced944/lxml-5.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1dc4ca99e89c335a7ed47d38964abcb36c5910790f9bd106f2a8fa2ee0b909d2", size = 3817095, upload-time = "2025-04-23T01:46:48.521Z" }, - { url = "https://files.pythonhosted.org/packages/c6/b0/e4d1cbb8c078bc4ae44de9c6a79fec4e2b4151b1b4d50af71d799e76b177/lxml-5.4.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1b717b00a71b901b4667226bba282dd462c42ccf618ade12f9ba3674e1fabc55", size = 3892319, upload-time = "2025-04-23T01:49:22.069Z" }, - { url = "https://files.pythonhosted.org/packages/5b/aa/e2bdefba40d815059bcb60b371a36fbfcce970a935370e1b367ba1cc8f74/lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27a9ded0f0b52098ff89dd4c418325b987feed2ea5cc86e8860b0f844285d740", size = 4211614, upload-time = "2025-04-23T01:49:24.599Z" }, - { url = "https://files.pythonhosted.org/packages/3c/5f/91ff89d1e092e7cfdd8453a939436ac116db0a665e7f4be0cd8e65c7dc5a/lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b7ce10634113651d6f383aa712a194179dcd496bd8c41e191cec2099fa09de5", size = 4306273, upload-time = "2025-04-23T01:49:27.355Z" }, - { url = "https://files.pythonhosted.org/packages/be/7c/8c3f15df2ca534589717bfd19d1e3482167801caedfa4d90a575facf68a6/lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:53370c26500d22b45182f98847243efb518d268374a9570409d2e2276232fd37", size = 4208552, upload-time = "2025-04-23T01:49:29.949Z" }, - { url = "https://files.pythonhosted.org/packages/7d/d8/9567afb1665f64d73fc54eb904e418d1138d7f011ed00647121b4dd60b38/lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c6364038c519dffdbe07e3cf42e6a7f8b90c275d4d1617a69bb59734c1a2d571", size = 4331091, upload-time = "2025-04-23T01:49:32.842Z" }, - { url = "https://files.pythonhosted.org/packages/f1/ab/fdbbd91d8d82bf1a723ba88ec3e3d76c022b53c391b0c13cad441cdb8f9e/lxml-5.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b12cb6527599808ada9eb2cd6e0e7d3d8f13fe7bbb01c6311255a15ded4c7ab4", size = 3487862, upload-time = "2025-04-23T01:49:36.296Z" }, + { url = "https://files.pythonhosted.org/packages/db/8a/f8192a08237ef2fb1b19733f709db88a4c43bc8ab8357f01cb41a27e7f6a/lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388", size = 8590589, upload-time = "2025-09-22T04:00:10.51Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/27bcd07ae17ff5e5536e8d88f4c7d581b48963817a13de11f3ac3329bfa2/lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153", size = 4629671, upload-time = "2025-09-22T04:00:15.411Z" }, + { url = "https://files.pythonhosted.org/packages/02/5a/a7d53b3291c324e0b6e48f3c797be63836cc52156ddf8f33cd72aac78866/lxml-6.0.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f952dacaa552f3bb8834908dddd500ba7d508e6ea6eb8c52eb2d28f48ca06a31", size = 4999961, upload-time = "2025-09-22T04:00:17.619Z" }, + { url = "https://files.pythonhosted.org/packages/f5/55/d465e9b89df1761674d8672bb3e4ae2c47033b01ec243964b6e334c6743f/lxml-6.0.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71695772df6acea9f3c0e59e44ba8ac50c4f125217e84aab21074a1a55e7e5c9", size = 5157087, upload-time = "2025-09-22T04:00:19.868Z" }, + { url = "https://files.pythonhosted.org/packages/62/38/3073cd7e3e8dfc3ba3c3a139e33bee3a82de2bfb0925714351ad3d255c13/lxml-6.0.2-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f68764f35fd78d7c4cc4ef209a184c38b65440378013d24b8aecd327c3e0c8", size = 5067620, upload-time = "2025-09-22T04:00:21.877Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d3/1e001588c5e2205637b08985597827d3827dbaaece16348c8822bfe61c29/lxml-6.0.2-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:058027e261afed589eddcfe530fcc6f3402d7fd7e89bfd0532df82ebc1563dba", size = 5406664, upload-time = "2025-09-22T04:00:23.714Z" }, + { url = "https://files.pythonhosted.org/packages/20/cf/cab09478699b003857ed6ebfe95e9fb9fa3d3c25f1353b905c9b73cfb624/lxml-6.0.2-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8ffaeec5dfea5881d4c9d8913a32d10cfe3923495386106e4a24d45300ef79c", size = 5289397, upload-time = "2025-09-22T04:00:25.544Z" }, + { url = "https://files.pythonhosted.org/packages/a3/84/02a2d0c38ac9a8b9f9e5e1bbd3f24b3f426044ad618b552e9549ee91bd63/lxml-6.0.2-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:f2e3b1a6bb38de0bc713edd4d612969dd250ca8b724be8d460001a387507021c", size = 4772178, upload-time = "2025-09-22T04:00:27.602Z" }, + { url = "https://files.pythonhosted.org/packages/56/87/e1ceadcc031ec4aa605fe95476892d0b0ba3b7f8c7dcdf88fdeff59a9c86/lxml-6.0.2-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d6690ec5ec1cce0385cb20896b16be35247ac8c2046e493d03232f1c2414d321", size = 5358148, upload-time = "2025-09-22T04:00:29.323Z" }, + { url = "https://files.pythonhosted.org/packages/fe/13/5bb6cf42bb228353fd4ac5f162c6a84fd68a4d6f67c1031c8cf97e131fc6/lxml-6.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f2a50c3c1d11cad0ebebbac357a97b26aa79d2bcaf46f256551152aa85d3a4d1", size = 5112035, upload-time = "2025-09-22T04:00:31.061Z" }, + { url = "https://files.pythonhosted.org/packages/e4/e2/ea0498552102e59834e297c5c6dff8d8ded3db72ed5e8aad77871476f073/lxml-6.0.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:3efe1b21c7801ffa29a1112fab3b0f643628c30472d507f39544fd48e9549e34", size = 4799111, upload-time = "2025-09-22T04:00:33.11Z" }, + { url = "https://files.pythonhosted.org/packages/6a/9e/8de42b52a73abb8af86c66c969b3b4c2a96567b6ac74637c037d2e3baa60/lxml-6.0.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:59c45e125140b2c4b33920d21d83681940ca29f0b83f8629ea1a2196dc8cfe6a", size = 5351662, upload-time = "2025-09-22T04:00:35.237Z" }, + { url = "https://files.pythonhosted.org/packages/28/a2/de776a573dfb15114509a37351937c367530865edb10a90189d0b4b9b70a/lxml-6.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:452b899faa64f1805943ec1c0c9ebeaece01a1af83e130b69cdefeda180bb42c", size = 5314973, upload-time = "2025-09-22T04:00:37.086Z" }, + { url = "https://files.pythonhosted.org/packages/50/a0/3ae1b1f8964c271b5eec91db2043cf8c6c0bce101ebb2a633b51b044db6c/lxml-6.0.2-cp310-cp310-win32.whl", hash = "sha256:1e786a464c191ca43b133906c6903a7e4d56bef376b75d97ccbb8ec5cf1f0a4b", size = 3611953, upload-time = "2025-09-22T04:00:39.224Z" }, + { url = "https://files.pythonhosted.org/packages/d1/70/bd42491f0634aad41bdfc1e46f5cff98825fb6185688dc82baa35d509f1a/lxml-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:dacf3c64ef3f7440e3167aa4b49aa9e0fb99e0aa4f9ff03795640bf94531bcb0", size = 4032695, upload-time = "2025-09-22T04:00:41.402Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d0/05c6a72299f54c2c561a6c6cbb2f512e047fca20ea97a05e57931f194ac4/lxml-6.0.2-cp310-cp310-win_arm64.whl", hash = "sha256:45f93e6f75123f88d7f0cfd90f2d05f441b808562bf0bc01070a00f53f5028b5", size = 3680051, upload-time = "2025-09-22T04:00:43.525Z" }, + { url = "https://files.pythonhosted.org/packages/77/d5/becbe1e2569b474a23f0c672ead8a29ac50b2dc1d5b9de184831bda8d14c/lxml-6.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:13e35cbc684aadf05d8711a5d1b5857c92e5e580efa9a0d2be197199c8def607", size = 8634365, upload-time = "2025-09-22T04:00:45.672Z" }, + { url = "https://files.pythonhosted.org/packages/28/66/1ced58f12e804644426b85d0bb8a4478ca77bc1761455da310505f1a3526/lxml-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b1675e096e17c6fe9c0e8c81434f5736c0739ff9ac6123c87c2d452f48fc938", size = 4650793, upload-time = "2025-09-22T04:00:47.783Z" }, + { url = "https://files.pythonhosted.org/packages/11/84/549098ffea39dfd167e3f174b4ce983d0eed61f9d8d25b7bf2a57c3247fc/lxml-6.0.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac6e5811ae2870953390452e3476694196f98d447573234592d30488147404d", size = 4944362, upload-time = "2025-09-22T04:00:49.845Z" }, + { url = "https://files.pythonhosted.org/packages/ac/bd/f207f16abf9749d2037453d56b643a7471d8fde855a231a12d1e095c4f01/lxml-6.0.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5aa0fc67ae19d7a64c3fe725dc9a1bb11f80e01f78289d05c6f62545affec438", size = 5083152, upload-time = "2025-09-22T04:00:51.709Z" }, + { url = "https://files.pythonhosted.org/packages/15/ae/bd813e87d8941d52ad5b65071b1affb48da01c4ed3c9c99e40abb266fbff/lxml-6.0.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de496365750cc472b4e7902a485d3f152ecf57bd3ba03ddd5578ed8ceb4c5964", size = 5023539, upload-time = "2025-09-22T04:00:53.593Z" }, + { url = "https://files.pythonhosted.org/packages/02/cd/9bfef16bd1d874fbe0cb51afb00329540f30a3283beb9f0780adbb7eec03/lxml-6.0.2-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:200069a593c5e40b8f6fc0d84d86d970ba43138c3e68619ffa234bc9bb806a4d", size = 5344853, upload-time = "2025-09-22T04:00:55.524Z" }, + { url = "https://files.pythonhosted.org/packages/b8/89/ea8f91594bc5dbb879734d35a6f2b0ad50605d7fb419de2b63d4211765cc/lxml-6.0.2-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d2de809c2ee3b888b59f995625385f74629707c9355e0ff856445cdcae682b7", size = 5225133, upload-time = "2025-09-22T04:00:57.269Z" }, + { url = "https://files.pythonhosted.org/packages/b9/37/9c735274f5dbec726b2db99b98a43950395ba3d4a1043083dba2ad814170/lxml-6.0.2-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:b2c3da8d93cf5db60e8858c17684c47d01fee6405e554fb55018dd85fc23b178", size = 4677944, upload-time = "2025-09-22T04:00:59.052Z" }, + { url = "https://files.pythonhosted.org/packages/20/28/7dfe1ba3475d8bfca3878365075abe002e05d40dfaaeb7ec01b4c587d533/lxml-6.0.2-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:442de7530296ef5e188373a1ea5789a46ce90c4847e597856570439621d9c553", size = 5284535, upload-time = "2025-09-22T04:01:01.335Z" }, + { url = "https://files.pythonhosted.org/packages/e7/cf/5f14bc0de763498fc29510e3532bf2b4b3a1c1d5d0dff2e900c16ba021ef/lxml-6.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2593c77efde7bfea7f6389f1ab249b15ed4aa5bc5cb5131faa3b843c429fbedb", size = 5067343, upload-time = "2025-09-22T04:01:03.13Z" }, + { url = "https://files.pythonhosted.org/packages/1c/b0/bb8275ab5472f32b28cfbbcc6db7c9d092482d3439ca279d8d6fa02f7025/lxml-6.0.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3e3cb08855967a20f553ff32d147e14329b3ae70ced6edc2f282b94afbc74b2a", size = 4725419, upload-time = "2025-09-22T04:01:05.013Z" }, + { url = "https://files.pythonhosted.org/packages/25/4c/7c222753bc72edca3b99dbadba1b064209bc8ed4ad448af990e60dcce462/lxml-6.0.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2ed6c667fcbb8c19c6791bbf40b7268ef8ddf5a96940ba9404b9f9a304832f6c", size = 5275008, upload-time = "2025-09-22T04:01:07.327Z" }, + { url = "https://files.pythonhosted.org/packages/6c/8c/478a0dc6b6ed661451379447cdbec77c05741a75736d97e5b2b729687828/lxml-6.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b8f18914faec94132e5b91e69d76a5c1d7b0c73e2489ea8929c4aaa10b76bbf7", size = 5248906, upload-time = "2025-09-22T04:01:09.452Z" }, + { url = "https://files.pythonhosted.org/packages/2d/d9/5be3a6ab2784cdf9accb0703b65e1b64fcdd9311c9f007630c7db0cfcce1/lxml-6.0.2-cp311-cp311-win32.whl", hash = "sha256:6605c604e6daa9e0d7f0a2137bdc47a2e93b59c60a65466353e37f8272f47c46", size = 3610357, upload-time = "2025-09-22T04:01:11.102Z" }, + { url = "https://files.pythonhosted.org/packages/e2/7d/ca6fb13349b473d5732fb0ee3eec8f6c80fc0688e76b7d79c1008481bf1f/lxml-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e5867f2651016a3afd8dd2c8238baa66f1e2802f44bc17e236f547ace6647078", size = 4036583, upload-time = "2025-09-22T04:01:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a2/51363b5ecd3eab46563645f3a2c3836a2fc67d01a1b87c5017040f39f567/lxml-6.0.2-cp311-cp311-win_arm64.whl", hash = "sha256:4197fb2534ee05fd3e7afaab5d8bfd6c2e186f65ea7f9cd6a82809c887bd1285", size = 3680591, upload-time = "2025-09-22T04:01:14.874Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c8/8ff2bc6b920c84355146cd1ab7d181bc543b89241cfb1ebee824a7c81457/lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456", size = 8661887, upload-time = "2025-09-22T04:01:17.265Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/9aae1008083bb501ef63284220ce81638332f9ccbfa53765b2b7502203cf/lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924", size = 4667818, upload-time = "2025-09-22T04:01:19.688Z" }, + { url = "https://files.pythonhosted.org/packages/f1/ca/31fb37f99f37f1536c133476674c10b577e409c0a624384147653e38baf2/lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f", size = 4950807, upload-time = "2025-09-22T04:01:21.487Z" }, + { url = "https://files.pythonhosted.org/packages/da/87/f6cb9442e4bada8aab5ae7e1046264f62fdbeaa6e3f6211b93f4c0dd97f1/lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534", size = 5109179, upload-time = "2025-09-22T04:01:23.32Z" }, + { url = "https://files.pythonhosted.org/packages/c8/20/a7760713e65888db79bbae4f6146a6ae5c04e4a204a3c48896c408cd6ed2/lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564", size = 5023044, upload-time = "2025-09-22T04:01:25.118Z" }, + { url = "https://files.pythonhosted.org/packages/a2/b0/7e64e0460fcb36471899f75831509098f3fd7cd02a3833ac517433cb4f8f/lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f", size = 5359685, upload-time = "2025-09-22T04:01:27.398Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e1/e5df362e9ca4e2f48ed6411bd4b3a0ae737cc842e96877f5bf9428055ab4/lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0", size = 5654127, upload-time = "2025-09-22T04:01:29.629Z" }, + { url = "https://files.pythonhosted.org/packages/c6/d1/232b3309a02d60f11e71857778bfcd4acbdb86c07db8260caf7d008b08f8/lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192", size = 5253958, upload-time = "2025-09-22T04:01:31.535Z" }, + { url = "https://files.pythonhosted.org/packages/35/35/d955a070994725c4f7d80583a96cab9c107c57a125b20bb5f708fe941011/lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0", size = 4711541, upload-time = "2025-09-22T04:01:33.801Z" }, + { url = "https://files.pythonhosted.org/packages/1e/be/667d17363b38a78c4bd63cfd4b4632029fd68d2c2dc81f25ce9eb5224dd5/lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092", size = 5267426, upload-time = "2025-09-22T04:01:35.639Z" }, + { url = "https://files.pythonhosted.org/packages/ea/47/62c70aa4a1c26569bc958c9ca86af2bb4e1f614e8c04fb2989833874f7ae/lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f", size = 5064917, upload-time = "2025-09-22T04:01:37.448Z" }, + { url = "https://files.pythonhosted.org/packages/bd/55/6ceddaca353ebd0f1908ef712c597f8570cc9c58130dbb89903198e441fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8", size = 4788795, upload-time = "2025-09-22T04:01:39.165Z" }, + { url = "https://files.pythonhosted.org/packages/cf/e8/fd63e15da5e3fd4c2146f8bbb3c14e94ab850589beab88e547b2dbce22e1/lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f", size = 5676759, upload-time = "2025-09-22T04:01:41.506Z" }, + { url = "https://files.pythonhosted.org/packages/76/47/b3ec58dc5c374697f5ba37412cd2728f427d056315d124dd4b61da381877/lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6", size = 5255666, upload-time = "2025-09-22T04:01:43.363Z" }, + { url = "https://files.pythonhosted.org/packages/19/93/03ba725df4c3d72afd9596eef4a37a837ce8e4806010569bedfcd2cb68fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322", size = 5277989, upload-time = "2025-09-22T04:01:45.215Z" }, + { url = "https://files.pythonhosted.org/packages/c6/80/c06de80bfce881d0ad738576f243911fccf992687ae09fd80b734712b39c/lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849", size = 3611456, upload-time = "2025-09-22T04:01:48.243Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d7/0cdfb6c3e30893463fb3d1e52bc5f5f99684a03c29a0b6b605cfae879cd5/lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f", size = 4011793, upload-time = "2025-09-22T04:01:50.042Z" }, + { url = "https://files.pythonhosted.org/packages/ea/7b/93c73c67db235931527301ed3785f849c78991e2e34f3fd9a6663ffda4c5/lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6", size = 3672836, upload-time = "2025-09-22T04:01:52.145Z" }, + { url = "https://files.pythonhosted.org/packages/e7/9c/780c9a8fce3f04690b374f72f41306866b0400b9d0fdf3e17aaa37887eed/lxml-6.0.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e748d4cf8fef2526bb2a589a417eba0c8674e29ffcb570ce2ceca44f1e567bf6", size = 3939264, upload-time = "2025-09-22T04:04:32.892Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5a/1ab260c00adf645d8bf7dec7f920f744b032f69130c681302821d5debea6/lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4ddb1049fa0579d0cbd00503ad8c58b9ab34d1254c77bc6a5576d96ec7853dba", size = 4216435, upload-time = "2025-09-22T04:04:34.907Z" }, + { url = "https://files.pythonhosted.org/packages/f2/37/565f3b3d7ffede22874b6d86be1a1763d00f4ea9fc5b9b6ccb11e4ec8612/lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cb233f9c95f83707dae461b12b720c1af9c28c2d19208e1be03387222151daf5", size = 4325913, upload-time = "2025-09-22T04:04:37.205Z" }, + { url = "https://files.pythonhosted.org/packages/22/ec/f3a1b169b2fb9d03467e2e3c0c752ea30e993be440a068b125fc7dd248b0/lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc456d04db0515ce3320d714a1eac7a97774ff0849e7718b492d957da4631dd4", size = 4269357, upload-time = "2025-09-22T04:04:39.322Z" }, + { url = "https://files.pythonhosted.org/packages/77/a2/585a28fe3e67daa1cf2f06f34490d556d121c25d500b10082a7db96e3bcd/lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2613e67de13d619fd283d58bda40bff0ee07739f624ffee8b13b631abf33083d", size = 4412295, upload-time = "2025-09-22T04:04:41.647Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d9/a57dd8bcebd7c69386c20263830d4fa72d27e6b72a229ef7a48e88952d9a/lxml-6.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:24a8e756c982c001ca8d59e87c80c4d9dcd4d9b44a4cbeb8d9be4482c514d41d", size = 3516913, upload-time = "2025-09-22T04:04:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/0b/11/29d08bc103a62c0eba8016e7ed5aeebbf1e4312e83b0b1648dd203b0e87d/lxml-6.0.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1c06035eafa8404b5cf475bb37a9f6088b0aca288d4ccc9d69389750d5543700", size = 3949829, upload-time = "2025-09-22T04:04:45.608Z" }, + { url = "https://files.pythonhosted.org/packages/12/b3/52ab9a3b31e5ab8238da241baa19eec44d2ab426532441ee607165aebb52/lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c7d13103045de1bdd6fe5d61802565f1a3537d70cd3abf596aa0af62761921ee", size = 4226277, upload-time = "2025-09-22T04:04:47.754Z" }, + { url = "https://files.pythonhosted.org/packages/a0/33/1eaf780c1baad88224611df13b1c2a9dfa460b526cacfe769103ff50d845/lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a3c150a95fbe5ac91de323aa756219ef9cf7fde5a3f00e2281e30f33fa5fa4f", size = 4330433, upload-time = "2025-09-22T04:04:49.907Z" }, + { url = "https://files.pythonhosted.org/packages/7a/c1/27428a2ff348e994ab4f8777d3a0ad510b6b92d37718e5887d2da99952a2/lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60fa43be34f78bebb27812ed90f1925ec99560b0fa1decdb7d12b84d857d31e9", size = 4272119, upload-time = "2025-09-22T04:04:51.801Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d0/3020fa12bcec4ab62f97aab026d57c2f0cfd480a558758d9ca233bb6a79d/lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21c73b476d3cfe836be731225ec3421fa2f048d84f6df6a8e70433dff1376d5a", size = 4417314, upload-time = "2025-09-22T04:04:55.024Z" }, + { url = "https://files.pythonhosted.org/packages/6c/77/d7f491cbc05303ac6801651aabeb262d43f319288c1ea96c66b1d2692ff3/lxml-6.0.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:27220da5be049e936c3aca06f174e8827ca6445a4353a1995584311487fc4e3e", size = 3518768, upload-time = "2025-09-22T04:04:57.097Z" }, ] [[package]] @@ -1173,21 +1178,22 @@ wheels = [ [[package]] name = "pyplayready" -version = "0.6.0" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp" }, { name = "click" }, { name = "construct" }, + { name = "cryptography" }, { name = "ecpy" }, + { name = "lxml" }, { name = "pycryptodome" }, { name = "pyyaml" }, { name = "requests" }, { name = "xmltodict" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/39/5f/aba36faf0f7feafa4b82bb9e38a0d8c70048e068416a931ee54a565ee3db/pyplayready-0.6.0.tar.gz", hash = "sha256:2b874596a8532efa5d7f2380e8de2cdb611a96cd69b0da5182ab1902083566e9", size = 99157, upload-time = "2025-02-06T13:16:02.763Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/f2/6d75b6d10a8361b53a2acbe959d51aa586418e9af497381a9f5c436ca488/pyplayready-0.6.3.tar.gz", hash = "sha256:b9b82a32c2cced9c43f910eb1fb891545f1491dc063c1eb9c20634e2417eda76", size = 58019, upload-time = "2025-08-20T19:32:43.642Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/85/a5b7dba7d5420c8f5d133123376a135fda69973f3e8d7c05c58a516a54e5/pyplayready-0.6.0-py3-none-any.whl", hash = "sha256:7f85ba94f2ae0d0c964d2c84e3a4f99bfa947fb120069c70af6c17f83ed6a7f3", size = 114232, upload-time = "2025-02-06T13:16:01.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7f/64d5ff5d765f9f2138ee7cc196fd9401f9eae0fb514c66660ad4e56584fa/pyplayready-0.6.3-py3-none-any.whl", hash = "sha256:82f35434e790a7da21df57ec053a2924ceb63622c5a6c5ff9f0fa03db0531c57", size = 66162, upload-time = "2025-08-20T19:32:42.62Z" }, ] [[package]] @@ -1199,6 +1205,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" }, ] +[[package]] +name = "pysubs2" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/4a/becf78d9d3df56e6c4a9c50b83794e5436b6c5ab6dd8a3f934e94c89338c/pysubs2-1.8.0.tar.gz", hash = "sha256:3397bb58a4a15b1325ba2ae3fd4d7c214e2c0ddb9f33190d6280d783bb433b20", size = 1130048, upload-time = "2024-12-24T12:39:47.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/09/0fc0719162e5ad723f71d41cf336f18b6b5054d70dc0fe42ace6b4d2bdc9/pysubs2-1.8.0-py3-none-any.whl", hash = "sha256:05716f5039a9ebe32cd4d7673f923cf36204f3a3e99987f823ab83610b7035a0", size = 43516, upload-time = "2024-12-24T12:39:44.469Z" }, +] + [[package]] name = "pywidevine" version = "1.8.0" @@ -1439,8 +1454,8 @@ sdist = { url = "https://files.pythonhosted.org/packages/66/b7/4a1bc231e0681ebf3 [[package]] name = "subby" -version = "0.3.21" -source = { git = "https://github.com/vevv/subby.git#390cb2f4a55e98057cdd65314d8cbffd5d0a11f1" } +version = "0.3.23" +source = { git = "https://github.com/vevv/subby.git?rev=5a925c367ffb3f5e53fd114ae222d3be1fdff35d#5a925c367ffb3f5e53fd114ae222d3be1fdff35d" } dependencies = [ { name = "beautifulsoup4" }, { name = "click" }, @@ -1545,7 +1560,7 @@ wheels = [ [[package]] name = "unshackle" -version = "1.4.8" +version = "1.4.9" source = { editable = "." } dependencies = [ { name = "aiohttp-swagger3" }, @@ -1570,6 +1585,7 @@ dependencies = [ { name = "pymp4" }, { name = "pymysql" }, { name = "pyplayready" }, + { name = "pysubs2" }, { name = "pywidevine", extra = ["serve"] }, { name = "pyyaml" }, { name = "requests", extra = ["socks"] }, @@ -1619,7 +1635,8 @@ requires-dist = [ { name = "pymediainfo", specifier = ">=6.1.0,<7" }, { name = "pymp4", specifier = ">=1.4.0,<2" }, { name = "pymysql", specifier = ">=1.1.0,<2" }, - { name = "pyplayready", specifier = ">=0.6.0,<0.7" }, + { name = "pyplayready", specifier = ">=0.6.3,<0.7" }, + { name = "pysubs2", specifier = ">=1.7.0,<2" }, { name = "pywidevine", extras = ["serve"], specifier = ">=1.8.0,<2" }, { name = "pyyaml", specifier = ">=6.0.1,<7" }, { name = "requests", extras = ["socks"], specifier = ">=2.31.0,<3" }, @@ -1627,7 +1644,7 @@ requires-dist = [ { name = "rlaphoenix-m3u8", specifier = ">=3.4.0,<4" }, { name = "ruamel-yaml", specifier = ">=0.18.6,<0.19" }, { name = "sortedcontainers", specifier = ">=2.4.0,<3" }, - { name = "subby", git = "https://github.com/vevv/subby.git" }, + { name = "subby", git = "https://github.com/vevv/subby.git?rev=5a925c367ffb3f5e53fd114ae222d3be1fdff35d" }, { name = "subtitle-filter", specifier = ">=1.4.9,<2" }, { name = "unidecode", specifier = ">=1.3.8,<2" }, { name = "urllib3", specifier = ">=2.2.1,<3" }, From 7a49a6a4f989522c9c0ebccc80076ecabb247d00 Mon Sep 17 00:00:00 2001 From: Andy Date: Fri, 17 Oct 2025 20:41:09 +0000 Subject: [PATCH 14/62] docs: add dev branch and update README --- .gitignore | 1 + README.md | 3 +++ uv.lock | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 26a73b6..a7e3fe3 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ unshackle/certs/ unshackle/WVDs/ unshackle/PRDs/ temp/ +logs/ services/ # Byte-compiled / optimized / DLL files diff --git a/README.md b/README.md index a9b78c4..a5bd916 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,9 @@

+> [!WARNING] +> **Development Branch**: This is the `dev` branch containing bleeding-edge features and experimental changes. Use for testing only. For stable releases, use the [`main`](https://github.com/unshackle-dl/unshackle/tree/main) branch. + ## What is unshackle? unshackle is a fork of [Devine](https://github.com/devine-dl/devine/), a powerful archival tool for downloading movies, TV shows, and music from streaming services. Built with a focus on modularity and extensibility, it provides a robust framework for content acquisition with support for DRM-protected content. diff --git a/uv.lock b/uv.lock index b9b0fd2..bd17d09 100644 --- a/uv.lock +++ b/uv.lock @@ -1560,7 +1560,7 @@ wheels = [ [[package]] name = "unshackle" -version = "1.4.9" +version = "2.0.0" source = { editable = "." } dependencies = [ { name = "aiohttp-swagger3" }, From ed1314572b6d422dede203fece778a221397a4f9 Mon Sep 17 00:00:00 2001 From: Andy Date: Sat, 18 Oct 2025 07:04:11 +0000 Subject: [PATCH 15/62] feat(dl): add --latest-episode option to download only the most recent episode Adds a new CLI option `-le, --latest-episode` that automatically selects and downloads only the single most recent episode from a series, regardless of which season it's in. Fixes #28 --- unshackle/commands/dl.py | 219 +++++++++++++++++++++------------------ 1 file changed, 121 insertions(+), 98 deletions(-) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index c6cd61d..1c77ee7 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -152,6 +152,13 @@ class dl: default=None, help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, `S02-S02E03`, e.t.c, defaults to all.", ) + @click.option( + "-le", + "--latest-episode", + is_flag=True, + default=False, + help="Download only the single most recent episode available.", + ) @click.option( "-l", "--lang", @@ -322,11 +329,7 @@ class dl: debug_log_path = config.directories.logs / config.filenames.debug_log.format_map( defaultdict(str, service=self.service, time=datetime.now().strftime("%Y%m%d-%H%M%S")) ) - init_debug_logger( - log_path=debug_log_path, - enabled=True, - log_keys=config.debug_keys - ) + init_debug_logger(log_path=debug_log_path, enabled=True, log_keys=config.debug_keys) self.debug_logger = get_debug_logger() if self.debug_logger: @@ -342,8 +345,12 @@ class dl: "tmdb_id": tmdb_id, "tmdb_name": tmdb_name, "tmdb_year": tmdb_year, - "cli_params": {k: v for k, v in ctx.params.items() if k not in ['profile', 'proxy', 'tag', 'tmdb_id', 'tmdb_name', 'tmdb_year']} - } + "cli_params": { + k: v + for k, v in ctx.params.items() + if k not in ["profile", "proxy", "tag", "tmdb_id", "tmdb_name", "tmdb_year"] + }, + }, ) else: self.debug_logger = None @@ -361,7 +368,7 @@ class dl: level="DEBUG", operation="load_service_config", service=self.service, - context={"config_path": str(service_config_path), "config": self.service_config} + context={"config_path": str(service_config_path), "config": self.service_config}, ) else: self.service_config = {} @@ -438,19 +445,25 @@ class dl: cdm_info = {"type": "DecryptLabs", "drm_type": drm_type, "security_level": self.cdm.security_level} elif hasattr(self.cdm, "device_type") and self.cdm.device_type.name in ["ANDROID", "CHROME"]: self.log.info(f"Loaded Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})") - cdm_info = {"type": "Widevine", "system_id": self.cdm.system_id, "security_level": self.cdm.security_level, "device_type": self.cdm.device_type.name} + cdm_info = { + "type": "Widevine", + "system_id": self.cdm.system_id, + "security_level": self.cdm.security_level, + "device_type": self.cdm.device_type.name, + } else: self.log.info( f"Loaded PlayReady CDM: {self.cdm.certificate_chain.get_name()} (L{self.cdm.security_level})" ) - cdm_info = {"type": "PlayReady", "certificate": self.cdm.certificate_chain.get_name(), "security_level": self.cdm.security_level} + cdm_info = { + "type": "PlayReady", + "certificate": self.cdm.certificate_chain.get_name(), + "security_level": self.cdm.security_level, + } if self.debug_logger and cdm_info: self.debug_logger.log( - level="INFO", - operation="load_cdm", - service=self.service, - context={"cdm": cdm_info} + level="INFO", operation="load_cdm", service=self.service, context={"cdm": cdm_info} ) self.proxy_providers = [] @@ -526,6 +539,7 @@ class dl: channels: float, no_atmos: bool, wanted: list[str], + latest_episode: bool, lang: list[str], v_lang: list[str], a_lang: list[str], @@ -587,8 +601,8 @@ class dl: context={ "cdm_only": cdm_only, "vaults_only": vaults_only, - "mode": "CDM only" if cdm_only else ("Vaults only" if vaults_only else "Both CDM and Vaults") - } + "mode": "CDM only" if cdm_only else ("Vaults only" if vaults_only else "Both CDM and Vaults"), + }, ) with console.status("Authenticating with Service...", spinner="dots"): @@ -606,16 +620,13 @@ class dl: context={ "has_cookies": bool(cookies), "has_credentials": bool(credential), - "profile": self.profile - } + "profile": self.profile, + }, ) except Exception as e: if self.debug_logger: self.debug_logger.log_error( - "authenticate", - e, - service=self.service, - context={"profile": self.profile} + "authenticate", e, service=self.service, context={"profile": self.profile} ) raise @@ -630,31 +641,24 @@ class dl: operation="get_titles", service=self.service, message="No titles returned from service", - success=False + success=False, ) sys.exit(1) except Exception as e: if self.debug_logger: - self.debug_logger.log_error( - "get_titles", - e, - service=self.service - ) + self.debug_logger.log_error("get_titles", e, service=self.service) raise if self.debug_logger: titles_info = { "type": titles.__class__.__name__, "count": len(titles) if hasattr(titles, "__len__") else 1, - "title": str(titles) + "title": str(titles), } if hasattr(titles, "seasons"): titles_info["seasons"] = len(titles.seasons) if hasattr(titles, "seasons") else 0 self.debug_logger.log( - level="INFO", - operation="get_titles", - service=self.service, - context={"titles": titles_info} + level="INFO", operation="get_titles", service=self.service, context={"titles": titles_info} ) if self.tmdb_year and self.tmdb_id: @@ -674,8 +678,21 @@ class dl: if list_titles: return + # Determine the latest episode if --latest-episode is set + latest_episode_id = None + if latest_episode and isinstance(titles, Series) and len(titles) > 0: + # Series is already sorted by (season, number, year) + # The last episode in the sorted list is the latest + latest_ep = titles[-1] + latest_episode_id = f"{latest_ep.season}x{latest_ep.number}" + self.log.info(f"Latest episode mode: Selecting S{latest_ep.season:02}E{latest_ep.number:02}") + for i, title in enumerate(titles): - if isinstance(title, Episode) and wanted and f"{title.season}x{title.number}" not in wanted: + if isinstance(title, Episode) and latest_episode and latest_episode_id: + # If --latest-episode is set, only process the latest episode + if f"{title.season}x{title.number}" != latest_episode_id: + continue + elif isinstance(title, Episode) and wanted and f"{title.season}x{title.number}" not in wanted: continue console.print(Padding(Rule(f"[rule.text]{title}"), (1, 2))) @@ -750,10 +767,7 @@ class dl: except Exception as e: if self.debug_logger: self.debug_logger.log_error( - "get_tracks", - e, - service=self.service, - context={"title": str(title)} + "get_tracks", e, service=self.service, context={"title": str(title)} ) raise @@ -764,34 +778,40 @@ class dl: "audio_tracks": len(title.tracks.audio), "subtitle_tracks": len(title.tracks.subtitles), "has_chapters": bool(title.tracks.chapters), - "videos": [{ - "codec": str(v.codec), - "resolution": f"{v.width}x{v.height}" if v.width and v.height else "unknown", - "bitrate": v.bitrate, - "range": str(v.range), - "language": str(v.language) if v.language else None, - "drm": [str(type(d).__name__) for d in v.drm] if v.drm else [] - } for v in title.tracks.videos], - "audio": [{ - "codec": str(a.codec), - "bitrate": a.bitrate, - "channels": a.channels, - "language": str(a.language) if a.language else None, - "descriptive": a.descriptive, - "drm": [str(type(d).__name__) for d in a.drm] if a.drm else [] - } for a in title.tracks.audio], - "subtitles": [{ - "codec": str(s.codec), - "language": str(s.language) if s.language else None, - "forced": s.forced, - "sdh": s.sdh - } for s in title.tracks.subtitles] + "videos": [ + { + "codec": str(v.codec), + "resolution": f"{v.width}x{v.height}" if v.width and v.height else "unknown", + "bitrate": v.bitrate, + "range": str(v.range), + "language": str(v.language) if v.language else None, + "drm": [str(type(d).__name__) for d in v.drm] if v.drm else [], + } + for v in title.tracks.videos + ], + "audio": [ + { + "codec": str(a.codec), + "bitrate": a.bitrate, + "channels": a.channels, + "language": str(a.language) if a.language else None, + "descriptive": a.descriptive, + "drm": [str(type(d).__name__) for d in a.drm] if a.drm else [], + } + for a in title.tracks.audio + ], + "subtitles": [ + { + "codec": str(s.codec), + "language": str(s.language) if s.language else None, + "forced": s.forced, + "sdh": s.sdh, + } + for s in title.tracks.subtitles + ], } self.debug_logger.log( - level="INFO", - operation="get_tracks", - service=self.service, - context=tracks_info + level="INFO", operation="get_tracks", service=self.service, context=tracks_info ) # strip SDH subs to non-SDH if no equivalent same-lang non-SDH is available @@ -1185,7 +1205,7 @@ class dl: operation="download_tracks", service=self.service, message="Download cancelled by user", - context={"title": str(title)} + context={"title": str(title)}, ) return except Exception as e: # noqa @@ -1219,8 +1239,8 @@ class dl: "title": str(title), "error_type": type(e).__name__, "tracks_count": len(title.tracks), - "returncode": getattr(e, "returncode", None) - } + "returncode": getattr(e, "returncode", None), + }, ) return @@ -1475,9 +1495,13 @@ class dl: if not no_folder and isinstance(title, (Episode, Song)): # Create folder based on title # Use first available track for filename generation - sample_track = title.tracks.videos[0] if title.tracks.videos else ( - title.tracks.audio[0] if title.tracks.audio else ( - title.tracks.subtitles[0] if title.tracks.subtitles else None + sample_track = ( + title.tracks.videos[0] + if title.tracks.videos + else ( + title.tracks.audio[0] + if title.tracks.audio + else (title.tracks.subtitles[0] if title.tracks.subtitles else None) ) ) if sample_track and sample_track.path: @@ -1498,7 +1522,9 @@ class dl: track_suffix = f".{track.codec.name if hasattr(track.codec, 'name') else 'video'}" elif isinstance(track, Audio): lang_suffix = f".{track.language}" if track.language else "" - track_suffix = f"{lang_suffix}.{track.codec.name if hasattr(track.codec, 'name') else 'audio'}" + track_suffix = ( + f"{lang_suffix}.{track.codec.name if hasattr(track.codec, 'name') else 'audio'}" + ) elif isinstance(track, Subtitle): lang_suffix = f".{track.language}" if track.language else "" forced_suffix = ".forced" if track.forced else "" @@ -1595,8 +1621,8 @@ class dl: "title": str(title), "pssh": drm.pssh.dumps() if drm.pssh else None, "kids": [k.hex for k in drm.kids], - "track_kid": track_kid.hex if track_kid else None - } + "track_kid": track_kid.hex if track_kid else None, + }, ) with self.DRM_TABLE_LOCK: @@ -1637,8 +1663,8 @@ class dl: "kid": kid.hex, "content_key": content_key, "track": str(track), - "from_cache": True - } + "from_cache": True, + }, ) elif vaults_only: msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used" @@ -1651,7 +1677,7 @@ class dl: operation="vault_key_not_found", service=self.service, message=msg, - context={"kid": kid.hex, "track": str(track)} + context={"kid": kid.hex, "track": str(track)}, ) raise Widevine.Exceptions.CEKNotFound(msg) else: @@ -1671,8 +1697,8 @@ class dl: message="Requesting Widevine license from service", context={ "track": str(track), - "kids_needed": [k.hex for k in all_kids if k not in drm.content_keys] - } + "kids_needed": [k.hex for k in all_kids if k not in drm.content_keys], + }, ) try: @@ -1693,10 +1719,7 @@ class dl: "get_license", e, service=self.service, - context={ - "track": str(track), - "exception_type": type(e).__name__ - } + context={"track": str(track), "exception_type": type(e).__name__}, ) raise e @@ -1708,8 +1731,8 @@ class dl: context={ "track": str(track), "keys_count": len(drm.content_keys), - "kids": [k.hex for k in drm.content_keys.keys()] - } + "kids": [k.hex for k in drm.content_keys.keys()], + }, ) for kid_, key in drm.content_keys.items(): @@ -1767,8 +1790,8 @@ class dl: "title": str(title), "pssh": drm.pssh_b64 or "", "kids": [k.hex for k in drm.kids], - "track_kid": track_kid.hex if track_kid else None - } + "track_kid": track_kid.hex if track_kid else None, + }, ) with self.DRM_TABLE_LOCK: @@ -1816,8 +1839,8 @@ class dl: "content_key": content_key, "track": str(track), "from_cache": True, - "drm_type": "PlayReady" - } + "drm_type": "PlayReady", + }, ) elif vaults_only: msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used" @@ -1830,7 +1853,7 @@ class dl: operation="vault_key_not_found", service=self.service, message=msg, - context={"kid": kid.hex, "track": str(track), "drm_type": "PlayReady"} + context={"kid": kid.hex, "track": str(track), "drm_type": "PlayReady"}, ) raise PlayReady.Exceptions.CEKNotFound(msg) else: @@ -1860,8 +1883,8 @@ class dl: context={ "track": str(track), "exception_type": type(e).__name__, - "drm_type": "PlayReady" - } + "drm_type": "PlayReady", + }, ) raise e @@ -1937,7 +1960,7 @@ class dl: @staticmethod def save_cookies(path: Path, cookies: CookieJar): - if hasattr(cookies, 'jar'): + if hasattr(cookies, "jar"): cookies = cookies.jar cookie_jar = MozillaCookieJar(path) @@ -2084,12 +2107,12 @@ class dl: else: return RemoteCdm( - device_type=cdm_api['Device Type'], - system_id=cdm_api['System ID'], - security_level=cdm_api['Security Level'], - host=cdm_api['Host'], - secret=cdm_api['Secret'], - device_name=cdm_api['Device Name'], + device_type=cdm_api["Device Type"], + system_id=cdm_api["System ID"], + security_level=cdm_api["Security Level"], + host=cdm_api["Host"], + secret=cdm_api["Secret"], + device_name=cdm_api["Device Name"], ) prd_path = config.directories.prds / f"{cdm_name}.prd" From 3dd12b0cbe2138f4300fbe559e79e4c0fd0ab257 Mon Sep 17 00:00:00 2001 From: Andy Date: Sat, 18 Oct 2025 07:05:05 +0000 Subject: [PATCH 16/62] chore(api): fix import ordering in download_manager and handlers --- unshackle/core/api/download_manager.py | 7 ++++--- unshackle/core/api/handlers.py | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index 7e2f0a4..4b87c17 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -6,11 +6,11 @@ import sys import tempfile import threading import uuid +from contextlib import suppress from dataclasses import dataclass, field +from datetime import datetime, timedelta from enum import Enum from typing import Any, Callable, Dict, List, Optional -from datetime import datetime, timedelta -from contextlib import suppress log = logging.getLogger("download_manager") @@ -87,14 +87,15 @@ def _perform_download( if cancel_event and cancel_event.is_set(): raise Exception(f"Job was cancelled {stage}") + from contextlib import redirect_stderr, redirect_stdout from io import StringIO - from contextlib import redirect_stdout, redirect_stderr _check_cancel("before execution started") # Import dl.py components lazily to avoid circular deps during module import import click import yaml + from unshackle.commands.dl import dl from unshackle.core.config import config from unshackle.core.services import Services diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 60261a6..3b8dd1f 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -20,7 +20,6 @@ def initialize_proxy_providers() -> List[Any]: proxy_providers = [] try: from unshackle.core import binaries - # Load the main unshackle config to get proxy provider settings from unshackle.core.config import config as main_config From 9921690339711748437fba9d1c89fa9fde444464 Mon Sep 17 00:00:00 2001 From: Andy Date: Sat, 18 Oct 2025 07:32:17 +0000 Subject: [PATCH 17/62] feat: add service-specific configuration overrides Add support for per-service configuration overrides allowing fine-tuned control of downloader and command options on a service-by-service basis. Fixes #13 --- unshackle/commands/dl.py | 27 ++++ unshackle/unshackle-example.yaml | 213 +++++++++++++++++++------------ 2 files changed, 158 insertions(+), 82 deletions(-) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 1c77ee7..0436bed 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -380,6 +380,33 @@ class dl: if getattr(config, "decryption_map", None): config.decryption = config.decryption_map.get(self.service, config.decryption) + service_config = config.services.get(self.service, {}) + + reserved_keys = { + "profiles", + "api_key", + "certificate", + "api_endpoint", + "region", + "device", + "endpoints", + "client", + } + + for config_key, override_value in service_config.items(): + if config_key in reserved_keys: + continue + + if isinstance(override_value, dict) and hasattr(config, config_key): + current_config = getattr(config, config_key, {}) + if isinstance(current_config, dict): + merged_config = {**current_config, **override_value} + setattr(config, config_key, merged_config) + + self.log.debug( + f"Applied service-specific '{config_key}' overrides for {self.service}: {override_value}" + ) + with console.status("Loading Key Vaults...", spinner="dots"): self.vaults = Vaults(self.service) total_vaults = len(config.key_vaults) diff --git a/unshackle/unshackle-example.yaml b/unshackle/unshackle-example.yaml index 8056d47..d1dda47 100644 --- a/unshackle/unshackle-example.yaml +++ b/unshackle/unshackle-example.yaml @@ -34,21 +34,23 @@ title_cache_max_retention: 86400 # Maximum cache retention for fallback when API # Debug logging configuration # Comprehensive JSON-based debug logging for troubleshooting and service development -debug: false # Enable structured JSON debug logging (default: false) - # When enabled with --debug flag or set to true: - # - Creates JSON Lines (.jsonl) log files with complete debugging context - # - Logs: session info, CLI params, service config, CDM details, authentication, - # titles, tracks metadata, DRM operations, vault queries, errors with stack traces - # - File location: logs/unshackle_debug_{service}_{timestamp}.jsonl - # - Also creates text log: logs/unshackle_root_{timestamp}.log +debug: + false # Enable structured JSON debug logging (default: false) + # When enabled with --debug flag or set to true: + # - Creates JSON Lines (.jsonl) log files with complete debugging context + # - Logs: session info, CLI params, service config, CDM details, authentication, + # titles, tracks metadata, DRM operations, vault queries, errors with stack traces + # - File location: logs/unshackle_debug_{service}_{timestamp}.jsonl + # - Also creates text log: logs/unshackle_root_{timestamp}.log -debug_keys: false # Log decryption keys in debug logs (default: false) - # Set to true to include actual decryption keys in logs - # Useful for debugging key retrieval and decryption issues - # SECURITY NOTE: Passwords, tokens, cookies, and session tokens - # are ALWAYS redacted regardless of this setting - # Only affects: content_key, key fields (the actual CEKs) - # Never affects: kid, keys_count, key_id (metadata is always logged) +debug_keys: + false # Log decryption keys in debug logs (default: false) + # Set to true to include actual decryption keys in logs + # Useful for debugging key retrieval and decryption issues + # SECURITY NOTE: Passwords, tokens, cookies, and session tokens + # are ALWAYS redacted regardless of this setting + # Only affects: content_key, key fields (the actual CEKs) + # Never affects: kid, keys_count, key_id (metadata is always logged) # Muxing configuration muxing: @@ -128,72 +130,72 @@ cdm: # Use pywidevine Serve-compliant Remote CDMs - # Example: Custom CDM API Configuration - # This demonstrates the highly configurable custom_api type that can adapt to any CDM API format - # - name: "chrome" - # type: "custom_api" - # host: "http://remotecdm.test/" - # timeout: 30 - # device: - # name: "ChromeCDM" - # type: "CHROME" - # system_id: 34312 - # security_level: 3 - # auth: - # type: "header" - # header_name: "x-api-key" - # key: "YOUR_API_KEY_HERE" - # custom_headers: - # User-Agent: "Unshackle/2.0.0" - # endpoints: - # get_request: - # path: "/get-challenge" - # method: "POST" - # timeout: 30 - # decrypt_response: - # path: "/get-keys" - # method: "POST" - # timeout: 30 - # request_mapping: - # get_request: - # param_names: - # scheme: "device" - # init_data: "init_data" - # static_params: - # scheme: "Widevine" - # decrypt_response: - # param_names: - # scheme: "device" - # license_request: "license_request" - # license_response: "license_response" - # static_params: - # scheme: "Widevine" - # response_mapping: - # get_request: - # fields: - # challenge: "challenge" - # session_id: "session_id" - # message: "message" - # message_type: "message_type" - # response_types: - # - condition: "message_type == 'license-request'" - # type: "license_request" - # success_conditions: - # - "message == 'success'" - # decrypt_response: - # fields: - # keys: "keys" - # message: "message" - # key_fields: - # kid: "kid" - # key: "key" - # type: "type" - # success_conditions: - # - "message == 'success'" - # caching: - # enabled: true - # use_vaults: true - # check_cached_first: true +# Example: Custom CDM API Configuration +# This demonstrates the highly configurable custom_api type that can adapt to any CDM API format +# - name: "chrome" +# type: "custom_api" +# host: "http://remotecdm.test/" +# timeout: 30 +# device: +# name: "ChromeCDM" +# type: "CHROME" +# system_id: 34312 +# security_level: 3 +# auth: +# type: "header" +# header_name: "x-api-key" +# key: "YOUR_API_KEY_HERE" +# custom_headers: +# User-Agent: "Unshackle/2.0.0" +# endpoints: +# get_request: +# path: "/get-challenge" +# method: "POST" +# timeout: 30 +# decrypt_response: +# path: "/get-keys" +# method: "POST" +# timeout: 30 +# request_mapping: +# get_request: +# param_names: +# scheme: "device" +# init_data: "init_data" +# static_params: +# scheme: "Widevine" +# decrypt_response: +# param_names: +# scheme: "device" +# license_request: "license_request" +# license_response: "license_response" +# static_params: +# scheme: "Widevine" +# response_mapping: +# get_request: +# fields: +# challenge: "challenge" +# session_id: "session_id" +# message: "message" +# message_type: "message_type" +# response_types: +# - condition: "message_type == 'license-request'" +# type: "license_request" +# success_conditions: +# - "message == 'success'" +# decrypt_response: +# fields: +# keys: "keys" +# message: "message" +# key_fields: +# kid: "kid" +# key: "key" +# type: "type" +# success_conditions: +# - "message == 'success'" +# caching: +# enabled: true +# use_vaults: true +# check_cached_first: true remote_cdm: - name: "chrome" @@ -360,9 +362,13 @@ services: # Service-specific configuration goes here # Profile-specific configurations can be nested under service names - # Example: with profile-specific device configs + # You can override ANY global configuration option on a per-service basis + # This allows fine-tuned control for services with special requirements + # Supported overrides: dl, aria2c, n_m3u8dl_re, curl_impersonate, subtitle, muxing, headers, etc. + + # Example: Comprehensive service configuration showing all features EXAMPLE: - # Global service config + # Standard service config api_key: "service_api_key" # Service certificate for Widevine L1/L2 (base64 encoded) @@ -383,6 +389,42 @@ services: app_name: "AIV" device_model: "Fire TV Stick 4K" + # NEW: Configuration overrides (can be combined with profiles and certificates) + # Override dl command defaults for this service + dl: + downloads: 4 # Limit concurrent track downloads (global default: 6) + workers: 8 # Reduce workers per track (global default: 16) + lang: ["en", "es-419"] # Different language priority for this service + sub_format: srt # Force SRT subtitle format + + # Override n_m3u8dl_re downloader settings + n_m3u8dl_re: + thread_count: 8 # Lower thread count for rate-limited service (global default: 16) + use_proxy: true # Force proxy usage for this service + retry_count: 10 # More retries for unstable connections + ad_keyword: "advertisement" # Service-specific ad filtering + + # Override aria2c downloader settings + aria2c: + max_concurrent_downloads: 2 # Limit concurrent downloads (global default: 4) + max_connection_per_server: 1 # Single connection per server + split: 3 # Fewer splits (global default: 5) + file_allocation: none # Faster allocation for this service + + # Override subtitle processing for this service + subtitle: + conversion_method: pycaption # Use specific subtitle converter + sdh_method: auto + + # Service-specific headers + headers: + User-Agent: "Service-specific user agent string" + Accept-Language: "en-US,en;q=0.9" + + # Override muxing options + muxing: + set_title: true + # Example: Service with different regions per profile SERVICE_NAME: profiles: @@ -393,6 +435,13 @@ services: region: "GB" api_endpoint: "https://api.uk.service.com" + # Notes on service-specific overrides: + # - Overrides are merged with global config, not replaced + # - Only specified keys are overridden, others use global defaults + # - Reserved keys (profiles, api_key, certificate, etc.) are NOT treated as overrides + # - Any dict-type config option can be overridden (dl, aria2c, n_m3u8dl_re, etc.) + # - Use --debug flag to see which overrides are applied during downloads + # External proxy provider services proxy_providers: nordvpn: From d3ca8e70395e017a0e370fd534155523e9c1bde5 Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 20 Oct 2025 03:13:30 +0000 Subject: [PATCH 18/62] fix(tags): gracefully handle missing TMDB/Simkl API keys Simkl now requires a client_id from https://simkl.com/settings/developer/ --- unshackle/core/config.py | 1 + unshackle/core/utils/tags.py | 164 +++++++++++++++++++------------ unshackle/unshackle-example.yaml | 4 + 3 files changed, 104 insertions(+), 65 deletions(-) diff --git a/unshackle/core/config.py b/unshackle/core/config.py index 6ac5f29..08e620e 100644 --- a/unshackle/core/config.py +++ b/unshackle/core/config.py @@ -89,6 +89,7 @@ class Config: self.tag_group_name: bool = kwargs.get("tag_group_name", True) self.tag_imdb_tmdb: bool = kwargs.get("tag_imdb_tmdb", True) self.tmdb_api_key: str = kwargs.get("tmdb_api_key") or "" + self.simkl_client_id: str = kwargs.get("simkl_client_id") or "" self.decrypt_labs_api_key: str = kwargs.get("decrypt_labs_api_key") or "" self.update_checks: bool = kwargs.get("update_checks", True) self.update_check_interval: int = kwargs.get("update_check_interval", 24) diff --git a/unshackle/core/utils/tags.py b/unshackle/core/utils/tags.py index 5a5e616..f9570d0 100644 --- a/unshackle/core/utils/tags.py +++ b/unshackle/core/utils/tags.py @@ -47,6 +47,10 @@ def _api_key() -> Optional[str]: return config.tmdb_api_key or os.getenv("TMDB_API_KEY") +def _simkl_client_id() -> Optional[str]: + return config.simkl_client_id or os.getenv("SIMKL_CLIENT_ID") + + def _clean(s: str) -> str: return STRIP_RE.sub("", s).lower() @@ -63,9 +67,14 @@ def fuzzy_match(a: str, b: str, threshold: float = 0.8) -> bool: def search_simkl(title: str, year: Optional[int], kind: str) -> Tuple[Optional[dict], Optional[str], Optional[int]]: - """Search Simkl API for show information by filename (no auth required).""" + """Search Simkl API for show information by filename.""" log.debug("Searching Simkl for %r (%s, %s)", title, kind, year) + client_id = _simkl_client_id() + if not client_id: + log.debug("No SIMKL client ID configured; skipping SIMKL search") + return None, None, None + # Construct appropriate filename based on type filename = f"{title}" if year: @@ -78,7 +87,8 @@ def search_simkl(title: str, year: Optional[int], kind: str) -> Tuple[Optional[d try: session = _get_session() - resp = session.post("https://api.simkl.com/search/file", json={"file": filename}, timeout=30) + headers = {"simkl-api-key": client_id} + resp = session.post("https://api.simkl.com/search/file", json={"file": filename}, headers=headers, timeout=30) resp.raise_for_status() data = resp.json() log.debug("Simkl API response received") @@ -338,73 +348,97 @@ def tag_file(path: Path, title: Title, tmdb_id: Optional[int] | None = None) -> return if config.tag_imdb_tmdb: - # If tmdb_id is provided (via --tmdb), skip Simkl and use TMDB directly - if tmdb_id is not None: - log.debug("Using provided TMDB ID %s for tags", tmdb_id) - else: - # Try Simkl first for automatic lookup - simkl_data, simkl_title, simkl_tmdb_id = search_simkl(name, year, kind) - - if simkl_data and simkl_title and fuzzy_match(simkl_title, name): - log.debug("Using Simkl data for tags") - if simkl_tmdb_id: - tmdb_id = simkl_tmdb_id - - # Handle TV show data from Simkl - if simkl_data.get("type") == "episode" and "show" in simkl_data: - show_ids = simkl_data.get("show", {}).get("ids", {}) - if show_ids.get("imdb"): - standard_tags["IMDB"] = show_ids["imdb"] - if show_ids.get("tvdb"): - standard_tags["TVDB2"] = f"series/{show_ids['tvdb']}" - if show_ids.get("tmdbtv"): - standard_tags["TMDB"] = f"tv/{show_ids['tmdbtv']}" - - # Handle movie data from Simkl - elif simkl_data.get("type") == "movie" and "movie" in simkl_data: - movie_ids = simkl_data.get("movie", {}).get("ids", {}) - if movie_ids.get("imdb"): - standard_tags["IMDB"] = movie_ids["imdb"] - if movie_ids.get("tvdb"): - standard_tags["TVDB2"] = f"movies/{movie_ids['tvdb']}" - if movie_ids.get("tmdb"): - standard_tags["TMDB"] = f"movie/{movie_ids['tmdb']}" - - # Use TMDB API for additional metadata (either from provided ID or Simkl lookup) + # Check if we have any API keys available for metadata lookup api_key = _api_key() - if not api_key: - log.debug("No TMDB API key set; applying basic tags only") - _apply_tags(path, custom_tags) - return + simkl_client = _simkl_client_id() - tmdb_title: Optional[str] = None - if tmdb_id is None: - tmdb_id, tmdb_title = search_tmdb(name, year, kind) - log.debug("TMDB search result: %r (ID %s)", tmdb_title, tmdb_id) - if not tmdb_id or not tmdb_title or not fuzzy_match(tmdb_title, name): - log.debug("TMDB search did not match; skipping external ID lookup") - _apply_tags(path, custom_tags) - return - - prefix = "movie" if kind == "movie" else "tv" - standard_tags["TMDB"] = f"{prefix}/{tmdb_id}" - try: - ids = external_ids(tmdb_id, kind) - except requests.RequestException as exc: - log.debug("Failed to fetch external IDs: %s", exc) - ids = {} + if not api_key and not simkl_client: + log.debug("No TMDB API key or Simkl client ID configured; skipping IMDB/TMDB tag lookup") else: - log.debug("External IDs found: %s", ids) - - imdb_id = ids.get("imdb_id") - if imdb_id: - standard_tags["IMDB"] = imdb_id - tvdb_id = ids.get("tvdb_id") - if tvdb_id: - if kind == "movie": - standard_tags["TVDB2"] = f"movies/{tvdb_id}" + # If tmdb_id is provided (via --tmdb), skip Simkl and use TMDB directly + if tmdb_id is not None: + log.debug("Using provided TMDB ID %s for tags", tmdb_id) else: - standard_tags["TVDB2"] = f"series/{tvdb_id}" + # Try Simkl first for automatic lookup (only if client ID is available) + if simkl_client: + simkl_data, simkl_title, simkl_tmdb_id = search_simkl(name, year, kind) + + if simkl_data and simkl_title and fuzzy_match(simkl_title, name): + log.debug("Using Simkl data for tags") + if simkl_tmdb_id: + tmdb_id = simkl_tmdb_id + + # Handle TV show data from Simkl + if simkl_data.get("type") == "episode" and "show" in simkl_data: + show_ids = simkl_data.get("show", {}).get("ids", {}) + if show_ids.get("imdb"): + standard_tags["IMDB"] = show_ids["imdb"] + if show_ids.get("tvdb"): + standard_tags["TVDB2"] = f"series/{show_ids['tvdb']}" + if show_ids.get("tmdbtv"): + standard_tags["TMDB"] = f"tv/{show_ids['tmdbtv']}" + + # Handle movie data from Simkl + elif simkl_data.get("type") == "movie" and "movie" in simkl_data: + movie_ids = simkl_data.get("movie", {}).get("ids", {}) + if movie_ids.get("imdb"): + standard_tags["IMDB"] = movie_ids["imdb"] + if movie_ids.get("tvdb"): + standard_tags["TVDB2"] = f"movies/{movie_ids['tvdb']}" + if movie_ids.get("tmdb"): + standard_tags["TMDB"] = f"movie/{movie_ids['tmdb']}" + + # Use TMDB API for additional metadata (either from provided ID or Simkl lookup) + if api_key: + tmdb_title: Optional[str] = None + if tmdb_id is None: + tmdb_id, tmdb_title = search_tmdb(name, year, kind) + log.debug("TMDB search result: %r (ID %s)", tmdb_title, tmdb_id) + if not tmdb_id or not tmdb_title or not fuzzy_match(tmdb_title, name): + log.debug("TMDB search did not match; skipping external ID lookup") + else: + prefix = "movie" if kind == "movie" else "tv" + standard_tags["TMDB"] = f"{prefix}/{tmdb_id}" + try: + ids = external_ids(tmdb_id, kind) + except requests.RequestException as exc: + log.debug("Failed to fetch external IDs: %s", exc) + ids = {} + else: + log.debug("External IDs found: %s", ids) + + imdb_id = ids.get("imdb_id") + if imdb_id: + standard_tags["IMDB"] = imdb_id + tvdb_id = ids.get("tvdb_id") + if tvdb_id: + if kind == "movie": + standard_tags["TVDB2"] = f"movies/{tvdb_id}" + else: + standard_tags["TVDB2"] = f"series/{tvdb_id}" + elif tmdb_id is not None: + # tmdb_id was provided or found via Simkl + prefix = "movie" if kind == "movie" else "tv" + standard_tags["TMDB"] = f"{prefix}/{tmdb_id}" + try: + ids = external_ids(tmdb_id, kind) + except requests.RequestException as exc: + log.debug("Failed to fetch external IDs: %s", exc) + ids = {} + else: + log.debug("External IDs found: %s", ids) + + imdb_id = ids.get("imdb_id") + if imdb_id: + standard_tags["IMDB"] = imdb_id + tvdb_id = ids.get("tvdb_id") + if tvdb_id: + if kind == "movie": + standard_tags["TVDB2"] = f"movies/{tvdb_id}" + else: + standard_tags["TVDB2"] = f"series/{tvdb_id}" + else: + log.debug("No TMDB API key configured; skipping TMDB external ID lookup") merged_tags = { **custom_tags, diff --git a/unshackle/unshackle-example.yaml b/unshackle/unshackle-example.yaml index d1dda47..a56bb77 100644 --- a/unshackle/unshackle-example.yaml +++ b/unshackle/unshackle-example.yaml @@ -336,6 +336,10 @@ filenames: # API key for The Movie Database (TMDB) tmdb_api_key: "" +# Client ID for SIMKL API (optional, improves metadata matching) +# Get your free client ID at: https://simkl.com/settings/developer/ +simkl_client_id: "" + # conversion_method: # - auto (default): Smart routing - subby for WebVTT/SAMI, standard for others # - subby: Always use subby with advanced processing From 1409f93de52aaf5968e354fbe37d1c52686a400f Mon Sep 17 00:00:00 2001 From: stabbedbybrick Date: Mon, 20 Oct 2025 18:28:12 +0200 Subject: [PATCH 19/62] feat: add retry handler to curl_cffi Session --- unshackle/core/session.py | 135 +++++++++++++++++++++++++++++++------- 1 file changed, 113 insertions(+), 22 deletions(-) diff --git a/unshackle/core/session.py b/unshackle/core/session.py index 4cda472..d1a03e2 100644 --- a/unshackle/core/session.py +++ b/unshackle/core/session.py @@ -2,9 +2,16 @@ from __future__ import annotations +import logging +import random +import time import warnings +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from typing import Any, List, Optional, Set, Tuple +from urllib.parse import urlparse -from curl_cffi.requests import Session as CurlSession +from curl_cffi import Response, Session, exceptions from unshackle.core.config import config @@ -15,18 +22,91 @@ warnings.filterwarnings( ) -class Session(CurlSession): - """curl_cffi Session with warning suppression.""" +class MaxRetriesError(exceptions.RequestException): + def __init__(self, message, cause=None): + super().__init__(message) + self.__cause__ = cause - def request(self, method, url, **kwargs): - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", message="Make sure you are using https over https proxy.*", category=RuntimeWarning - ) +class CurlSession(Session): + def __init__( + self, + max_retries: int = 10, + backoff_factor: float = 0.2, + max_backoff: float = 60.0, + status_forcelist: Optional[List[int]] = None, + allowed_methods: Optional[Set[str]] = None, + catch_exceptions: Optional[Tuple[type[Exception], ...]] = None, + **session_kwargs: Any, + ): + super().__init__(**session_kwargs) + + self.max_retries = max_retries + self.backoff_factor = backoff_factor + self.max_backoff = max_backoff + self.status_forcelist = status_forcelist or [429, 500, 502, 503, 504] + self.allowed_methods = allowed_methods or {"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"} + self.catch_exceptions = catch_exceptions or ( + exceptions.ConnectionError, + exceptions.SSLError, + exceptions.Timeout, + ) + self.log = logging.getLogger(self.__class__.__name__) + + def _get_sleep_time(self, response: Response | None, attempt: int) -> float | None: + if response: + retry_after = response.headers.get("Retry-After") + if retry_after: + try: + return float(retry_after) + except ValueError: + if retry_date := parsedate_to_datetime(retry_after): + return (retry_date - datetime.now(timezone.utc)).total_seconds() + + if attempt == 0: + return 0.0 + + backoff_value = self.backoff_factor * (2 ** (attempt - 1)) + jitter = backoff_value * 0.1 + sleep_time = backoff_value + random.uniform(-jitter, jitter) + return min(sleep_time, self.max_backoff) + + def request(self, method: str, url: str, **kwargs: Any) -> Response: + if method.upper() not in self.allowed_methods: return super().request(method, url, **kwargs) + last_exception = None + response = None -def session(browser: str | None = None, **kwargs) -> Session: + for attempt in range(self.max_retries + 1): + try: + response = super().request(method, url, **kwargs) + if response.status_code not in self.status_forcelist: + return response + last_exception = exceptions.HTTPError(f"Received status code: {response.status_code}") + self.log.warning( + f"{response.status_code} {response.reason}({urlparse(url).path}). Retrying... " + f"({attempt + 1}/{self.max_retries})" + ) + + except self.catch_exceptions as e: + last_exception = e + response = None + self.log.warning( + f"{e.__class__.__name__}({urlparse(url).path}). Retrying... " + f"({attempt + 1}/{self.max_retries})" + ) + + if attempt < self.max_retries: + if sleep_duration := self._get_sleep_time(response, attempt + 1): + if sleep_duration > 0: + time.sleep(sleep_duration) + else: + break + + raise MaxRetriesError(f"Max retries exceeded for {method} {url}", cause=last_exception) + + +def session(browser: str | None = None, **kwargs) -> CurlSession: """ Create a curl_cffi session that impersonates a browser. @@ -48,32 +128,43 @@ def session(browser: str | None = None, **kwargs) -> Session: - allow_redirects: Follow redirects (bool, default True) - max_redirects: Maximum redirect count (int) - cert: Client certificate (str or tuple) + - ja3: JA3 fingerprint (str) + - akamai: Akamai fingerprint (str) + + Extra arguments for retry handler: + - max_retries: Maximum number of retries (int, default 10) + - backoff_factor: Backoff factor (float, default 0.2) + - max_backoff: Maximum backoff time (float, default 60.0) + - status_forcelist: List of status codes to force retry (list, default [429, 500, 502, 503, 504]) + - allowed_methods: List of allowed HTTP methods (set, default {"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"}) + - catch_exceptions: List of exceptions to catch (tuple, default (exceptions.ConnectionError, exceptions.SSLError, exceptions.Timeout)) Returns: curl_cffi.requests.Session configured with browser impersonation, common headers, and equivalent retry behavior to requests.Session. Example: - from unshackle.core.session import session + from unshackle.core.session import session as CurlSession class MyService(Service): @staticmethod - def get_session(): - return session() # Uses config default browser + def get_session() -> CurlSession: + session = CurlSession( + impersonate="chrome", + ja3="...", + akamai="...", + max_retries=5, + status_forcelist=[429, 500], + allowed_methods={"GET", "HEAD", "OPTIONS"}, + ) + return session # Uses config default browser """ - if browser is None: - browser = config.curl_impersonate.get("browser", "chrome124") session_config = { - "impersonate": browser, - "timeout": 30.0, - "allow_redirects": True, - "max_redirects": 15, - "verify": True, + "impersonate": browser or config.curl_impersonate.get("browser", "chrome"), + **kwargs, } - session_config.update(kwargs) - session_obj = Session(**session_config) + session_obj = CurlSession(**session_config) session_obj.headers.update(config.headers) - return session_obj From 5384b775a4d9cadc92e4493132c27c86fb2168b5 Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 20 Oct 2025 21:09:19 +0000 Subject: [PATCH 20/62] refactor(session): modernize type annotations to PEP 604 syntax --- unshackle/core/session.py | 14 +++++++------- uv.lock | 11 +++++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/unshackle/core/session.py b/unshackle/core/session.py index d1a03e2..bb08ca2 100644 --- a/unshackle/core/session.py +++ b/unshackle/core/session.py @@ -8,10 +8,10 @@ import time import warnings from datetime import datetime, timezone from email.utils import parsedate_to_datetime -from typing import Any, List, Optional, Set, Tuple +from typing import Any from urllib.parse import urlparse -from curl_cffi import Response, Session, exceptions +from curl_cffi.requests import Response, Session, exceptions from unshackle.core.config import config @@ -27,15 +27,16 @@ class MaxRetriesError(exceptions.RequestException): super().__init__(message) self.__cause__ = cause + class CurlSession(Session): def __init__( self, max_retries: int = 10, backoff_factor: float = 0.2, max_backoff: float = 60.0, - status_forcelist: Optional[List[int]] = None, - allowed_methods: Optional[Set[str]] = None, - catch_exceptions: Optional[Tuple[type[Exception], ...]] = None, + status_forcelist: list[int] | None = None, + allowed_methods: set[str] | None = None, + catch_exceptions: tuple[type[Exception], ...] | None = None, **session_kwargs: Any, ): super().__init__(**session_kwargs) @@ -92,8 +93,7 @@ class CurlSession(Session): last_exception = e response = None self.log.warning( - f"{e.__class__.__name__}({urlparse(url).path}). Retrying... " - f"({attempt + 1}/{self.max_retries})" + f"{e.__class__.__name__}({urlparse(url).path}). Retrying... ({attempt + 1}/{self.max_retries})" ) if attempt < self.max_retries: diff --git a/uv.lock b/uv.lock index bd17d09..8b2b799 100644 --- a/uv.lock +++ b/uv.lock @@ -1126,6 +1126,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/5f/af7da8e6f1e42b52f44a24d08b8e4c726207434e2593732d39e7af5e7256/pycryptodomex-3.23.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:14c37aaece158d0ace436f76a7bb19093db3b4deade9797abfc39ec6cd6cc2fe", size = 1806478, upload-time = "2025-05-17T17:23:26.066Z" }, ] +[[package]] +name = "pyexecjs" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ba/8e/aedef81641c8dca6fd0fb7294de5bed9c45f3397d67fddf755c1042c2642/PyExecJS-1.5.1.tar.gz", hash = "sha256:34cc1d070976918183ff7bdc0ad71f8157a891c92708c00c5fbbff7a769f505c", size = 13344, upload-time = "2018-01-18T04:33:55.126Z" } + [[package]] name = "pygments" version = "2.19.2" @@ -1580,6 +1589,7 @@ dependencies = [ { name = "protobuf" }, { name = "pycaption" }, { name = "pycryptodomex" }, + { name = "pyexecjs" }, { name = "pyjwt" }, { name = "pymediainfo" }, { name = "pymp4" }, @@ -1631,6 +1641,7 @@ requires-dist = [ { name = "protobuf", specifier = ">=4.25.3,<5" }, { name = "pycaption", specifier = ">=2.2.6,<3" }, { name = "pycryptodomex", specifier = ">=3.20.0,<4" }, + { name = "pyexecjs", specifier = ">=1.5.1" }, { name = "pyjwt", specifier = ">=2.8.0,<3" }, { name = "pymediainfo", specifier = ">=6.1.0,<7" }, { name = "pymp4", specifier = ">=1.4.0,<2" }, From 087df59fb6a20f8514e806e1311daa250bdd6ca3 Mon Sep 17 00:00:00 2001 From: TPD94 Date: Tue, 21 Oct 2025 21:07:24 -0400 Subject: [PATCH 21/62] Update hls.py --- unshackle/core/manifests/hls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unshackle/core/manifests/hls.py b/unshackle/core/manifests/hls.py index d48d96e..0bb1c9b 100644 --- a/unshackle/core/manifests/hls.py +++ b/unshackle/core/manifests/hls.py @@ -439,7 +439,7 @@ class HLS: elif len(files) != range_len: raise ValueError(f"Missing {range_len - len(files)} segment files for {segment_range}...") - if isinstance(drm, Widevine): + if isinstance(drm, (Widevine, PlayReady)): # with widevine we can merge all segments and decrypt once merge(to=merged_path, via=files, delete=True, include_map_data=True) drm.decrypt(merged_path) From e04399fbce9f14421983d5866e4b38b288ec2edd Mon Sep 17 00:00:00 2001 From: TPD94 Date: Tue, 21 Oct 2025 21:18:36 -0400 Subject: [PATCH 22/62] Update binaries.py Refactor code to search for binaries either in root of binary folder or in a subfolder named after the binary. --- unshackle/core/binaries.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/unshackle/core/binaries.py b/unshackle/core/binaries.py index 878a36f..e3b04aa 100644 --- a/unshackle/core/binaries.py +++ b/unshackle/core/binaries.py @@ -10,23 +10,23 @@ __shaka_platform = {"win32": "win", "darwin": "osx"}.get(sys.platform, sys.platf def find(*names: str) -> Optional[Path]: """Find the path of the first found binary name.""" - # Get the directory containing this file to find the local binaries folder - current_dir = Path(__file__).parent.parent + current_dir = Path(__file__).resolve().parent.parent local_binaries_dir = current_dir / "binaries" - for name in names: - # First check local binaries folder - if local_binaries_dir.exists(): - # On Windows, check for .exe extension first - if sys.platform == "win32": - local_path_exe = local_binaries_dir / f"{name}" / f"{name}.exe" - if local_path_exe.is_file(): - return local_path_exe + ext = ".exe" if sys.platform == "win32" else "" - # Check for exact name match with executable bit on Unix-like systems - local_path = local_binaries_dir / f"{name}" / f"{name}" - if local_path.is_file() and local_path.stat().st_mode & 0o111: # Check if executable - return local_path + for name in names: + if local_binaries_dir.exists(): + candidate_paths = [ + local_binaries_dir / f"{name}{ext}", + local_binaries_dir / name / f"{name}{ext}" + ] + + for path in candidate_paths: + if path.is_file(): + # On Unix-like systems, check if file is executable + if sys.platform == "win32" or (path.stat().st_mode & 0o111): + return path # Fall back to system PATH path = shutil.which(name) From df09998a47ef9b5bebdf3c181b5f696853aa982e Mon Sep 17 00:00:00 2001 From: TPD94 <39639333+TPD94@users.noreply.github.com> Date: Tue, 21 Oct 2025 21:19:55 -0400 Subject: [PATCH 23/62] Update .gitignore --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index e38a05e..26a73b6 100644 --- a/.gitignore +++ b/.gitignore @@ -235,5 +235,3 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ -/unshackle/binaries -/.idea From 98d4bb4333f6dbeeae9590ccad71338e903134e7 Mon Sep 17 00:00:00 2001 From: Andy Date: Wed, 22 Oct 2025 16:48:03 +0000 Subject: [PATCH 24/62] fix(config): support config in user config directory across platforms Fixes #23 --- unshackle/core/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unshackle/core/config.py b/unshackle/core/config.py index 08e620e..6eb7b26 100644 --- a/unshackle/core/config.py +++ b/unshackle/core/config.py @@ -118,8 +118,8 @@ POSSIBLE_CONFIG_PATHS = ( Config._Directories.namespace_dir / Config._Filenames.root_config, # The Parent Folder to the unshackle Namespace Folder (e.g., %appdata%/Python/Python311/site-packages) Config._Directories.namespace_dir.parent / Config._Filenames.root_config, - # The AppDirs User Config Folder (e.g., %localappdata%/unshackle) - Config._Directories.user_configs / Config._Filenames.root_config, + # The AppDirs User Config Folder (e.g., ~/.config/unshackle on Linux, %LOCALAPPDATA%\unshackle on Windows) + Path(Config._Directories.app_dirs.user_config_dir) / Config._Filenames.root_config, ) From 9b5d233c6999beb04eb4a6be903bf531f3a93add Mon Sep 17 00:00:00 2001 From: Andy Date: Wed, 22 Oct 2025 20:46:52 +0000 Subject: [PATCH 25/62] fix(dl): validate HYBRID mode requirements before download Add validation to check that both HDR10 and DV tracks are available when HYBRID mode is requested. This prevents wasted downloads when the hybrid processing would fail due to missing tracks. --- unshackle/commands/dl.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 0436bed..9821674 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -1002,6 +1002,29 @@ class dl: selected_videos.append(match) title.tracks.videos = selected_videos + # validate hybrid mode requirements + if any(r == Video.Range.HYBRID for r in range_): + hdr10_tracks = [v for v in title.tracks.videos if v.range == Video.Range.HDR10] + dv_tracks = [v for v in title.tracks.videos if v.range == Video.Range.DV] + + if not hdr10_tracks and not dv_tracks: + available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) + self.log.error("HYBRID mode requires both HDR10 and DV tracks, but neither is available") + self.log.error( + f"Available ranges: {', '.join(available_ranges) if available_ranges else 'none'}" + ) + sys.exit(1) + elif not hdr10_tracks: + available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) + self.log.error("HYBRID mode requires both HDR10 and DV tracks, but only DV is available") + self.log.error(f"Available ranges: {', '.join(available_ranges)}") + sys.exit(1) + elif not dv_tracks: + available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) + self.log.error("HYBRID mode requires both HDR10 and DV tracks, but only HDR10 is available") + self.log.error(f"Available ranges: {', '.join(available_ranges)}") + sys.exit(1) + # filter subtitle tracks if require_subs: missing_langs = [ From 07574d8d0226dbde8c6596e842bd7a7b8922a705 Mon Sep 17 00:00:00 2001 From: Andy Date: Wed, 22 Oct 2025 20:47:46 +0000 Subject: [PATCH 26/62] refactor(binaries): remove unused mypy import --- unshackle/core/binaries.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/unshackle/core/binaries.py b/unshackle/core/binaries.py index e3b04aa..7dc6109 100644 --- a/unshackle/core/binaries.py +++ b/unshackle/core/binaries.py @@ -3,8 +3,6 @@ import sys from pathlib import Path from typing import Optional -from mypy.types import names - __shaka_platform = {"win32": "win", "darwin": "osx"}.get(sys.platform, sys.platform) From bdd219d90c0404e08cd9d9a65095ede25edf7d98 Mon Sep 17 00:00:00 2001 From: Andy Date: Wed, 22 Oct 2025 21:10:14 +0000 Subject: [PATCH 27/62] chore: update CHANGELOG.md for version 2.0.0 --- CHANGELOG.md | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abe5c9e..dfdbaf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,108 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.0.0] - 2025-10-25 + +### Breaking Changes + +- **REST API Integration**: Core architecture modified to support REST API functionality + - Changes to internal APIs for download management and tracking + - Title and track classes updated with API integration points + - Core component interfaces modified for queue management support +- **Configuration Changes**: New required configuration options for API and enhanced features + - Added `simkl_client_id` now required for Simkl functionality + - Service-specific configuration override structure introduced + - Debug logging configuration options added +- **Forced Subtitles**: Behavior change for forced subtitle inclusion + - Forced subs no longer auto-included, requires explicit `--forced-subs` flag + +### Added + +- **REST API Server**: Complete download management via REST API (early development) + - Implemented download queue management and worker system + - Added OpenAPI/Swagger documentation for easy API exploration + - Included download progress tracking and status endpoints + - API authentication and comprehensive error handling + - Updated core components to support API integration + - Early development work with more changes planned +- **CustomRemoteCDM**: Highly configurable custom CDM API support + - Support for third-party CDM API providers with maximum configurability + - Full configuration through YAML without code changes + - Addresses GitHub issue #26 for flexible CDM integration +- **WindscribeVPN Proxy Provider**: New VPN provider support + - Added WindscribeVPN following NordVPN and SurfsharkVPN patterns + - Fixes GitHub issue #29 +- **Latest Episode Download**: New `--latest-episode` CLI option + - `-le, --latest-episode` flag to download only the most recent episode + - Automatically selects the single most recent episode regardless of season + - Fixes GitHub issue #28 +- **Service-Specific Configuration Overrides**: Per-service fine-tuned control + - Support for per-service configuration overrides in YAML + - Fine-tuned control of downloader and command options per service + - Fixes GitHub issue #13 +- **Comprehensive JSON Debug Logging**: Structured logging for troubleshooting + - Binary toggle via `--debug` flag or `debug: true` in config + - JSON Lines (.jsonl) format for easy parsing and analysis + - Comprehensive logging of all operations (session info, CLI params, CDM details, auth status, title/track metadata, DRM operations, vault queries) + - Configurable key logging via `debug_keys` option with smart redaction + - Error logging for all critical operations + - Removed old text logging system +- **curl_cffi Retry Handler**: Enhanced session reliability + - Added automatic retry mechanism to curl_cffi Session + - Improved download reliability with configurable retries +- **Simkl API Configuration**: New API key support + - Added `simkl_client_id` configuration option + - Simkl now requires client_id from https://simkl.com/settings/developer/ + +### Changed + +- **Binary Search Enhancement**: Improved binary discovery + - Refactored to search for binaries in root of binary folder or subfolder named after the binary + - Better organization of binary dependencies +- **Type Annotations**: Modernized to PEP 604 syntax + - Updated session.py type annotations to use modern Python syntax + - Improved code readability and type checking + +### Fixed + +- **Config Directory Support**: Cross-platform user config directory support + - Fixed config loading to properly support user config directories across all platforms + - Fixes GitHub issue #23 +- **HYBRID Mode Validation**: Pre-download validation for hybrid processing + - Added validation to check both HDR10 and DV tracks are available before download + - Prevents wasted downloads when hybrid processing would fail +- **TMDB/Simkl API Keys**: Graceful handling of missing API keys + - Improved error handling when TMDB or Simkl API keys are not configured + - Better user messaging for API configuration requirements +- **Forced Subtitles Behavior**: Correct forced subtitle filtering + - Fixed forced subtitles being incorrectly included without `--forced-subs` flag + - Forced subs now only included when explicitly requested +- **Font Attachment Constructor**: Fixed ASS/SSA font attachment + - Use keyword arguments for Attachment constructor in font attachment + - Fixes "Invalid URL: No scheme supplied" error + - Fixes GitHub issue #24 +- **Binary Subdirectory Checking**: Enhanced binary location discovery (by @TPD94, PR #19) + - Updated binaries.py to check subdirectories in binaries folders named after the binary + - Improved binary detection and loading +- **HLS Manifest Processing**: Minor HLS parser fix (by @TPD94, PR #19) +- **lxml and pyplayready**: Updated dependencies (by @Sp5rky) + - Updated lxml constraint and pyplayready import path for compatibility + +### Refactored + +- **Import Cleanup**: Removed unused imports + - Removed unused mypy import from binaries.py + - Fixed import ordering in API download_manager and handlers + +### Contributors + +This release includes contributions from: + +- @Sp5rky - REST API server implementation, dependency updates +- @stabbedbybrick - curl_cffi retry handler (PR #31) +- @TPD94 - Binary search enhancements, manifest parser fixes (PR #19) +- @scene (Andy) - Core features, configuration system, bug fixes + ## [1.4.8] - 2025-10-08 ### Added @@ -179,7 +281,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **Matroska Tag Compliance**: Enhanced media container compatibility +- **Matroska Tag Compliance**: Enhanced media container compatibility - Fixed Matroska tag compliance with official specification - **Application Branding**: Cleaned up version display - Removed old devine version reference from banner to avoid developer confusion From 3571c5eb3c634265579afc1df831a3b8f141b9cf Mon Sep 17 00:00:00 2001 From: Andy Date: Thu, 23 Oct 2025 18:11:30 +0000 Subject: [PATCH 28/62] style: apply ruff formatting fixes --- .gitignore | 1 + unshackle/core/binaries.py | 5 +---- unshackle/core/constants.py | 8 +++++++- unshackle/core/session.py | 3 ++- unshackle/core/titles/episode.py | 4 ++-- unshackle/core/titles/movie.py | 4 ++-- unshackle/core/utilities.py | 2 +- unshackle/vaults/SQLite.py | 4 +++- 8 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index a7e3fe3..36a3190 100644 --- a/.gitignore +++ b/.gitignore @@ -218,6 +218,7 @@ cython_debug/ # you could uncomment the following to ignore the entire vscode folder .vscode/ .github/copilot-instructions.md +CLAUDE.md # Ruff stuff: .ruff_cache/ diff --git a/unshackle/core/binaries.py b/unshackle/core/binaries.py index 7dc6109..f846256 100644 --- a/unshackle/core/binaries.py +++ b/unshackle/core/binaries.py @@ -15,10 +15,7 @@ def find(*names: str) -> Optional[Path]: for name in names: if local_binaries_dir.exists(): - candidate_paths = [ - local_binaries_dir / f"{name}{ext}", - local_binaries_dir / name / f"{name}{ext}" - ] + candidate_paths = [local_binaries_dir / f"{name}{ext}", local_binaries_dir / name / f"{name}{ext}"] for path in candidate_paths: if path.is_file(): diff --git a/unshackle/core/constants.py b/unshackle/core/constants.py index 6a14f7d..65c6681 100644 --- a/unshackle/core/constants.py +++ b/unshackle/core/constants.py @@ -8,7 +8,13 @@ DRM_SORT_MAP = ["ClearKey", "Widevine"] LANGUAGE_MAX_DISTANCE = 5 # this is max to be considered "same", e.g., en, en-US, en-AU LANGUAGE_EXACT_DISTANCE = 0 # exact match only, no variants VIDEO_CODEC_MAP = {"AVC": "H.264", "HEVC": "H.265"} -DYNAMIC_RANGE_MAP = {"HDR10": "HDR", "HDR10+": "HDR10P", "Dolby Vision": "DV", "HDR10 / HDR10+": "HDR10P", "HDR10 / HDR10": "HDR"} +DYNAMIC_RANGE_MAP = { + "HDR10": "HDR", + "HDR10+": "HDR10P", + "Dolby Vision": "DV", + "HDR10 / HDR10+": "HDR10P", + "HDR10 / HDR10": "HDR", +} AUDIO_CODEC_MAP = {"E-AC-3": "DDP", "AC-3": "DD"} context_settings = dict( diff --git a/unshackle/core/session.py b/unshackle/core/session.py index bb08ca2..a935752 100644 --- a/unshackle/core/session.py +++ b/unshackle/core/session.py @@ -48,6 +48,7 @@ class CurlSession(Session): self.allowed_methods = allowed_methods or {"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"} self.catch_exceptions = catch_exceptions or ( exceptions.ConnectionError, + exceptions.ProxyError, exceptions.SSLError, exceptions.Timeout, ) @@ -137,7 +138,7 @@ def session(browser: str | None = None, **kwargs) -> CurlSession: - max_backoff: Maximum backoff time (float, default 60.0) - status_forcelist: List of status codes to force retry (list, default [429, 500, 502, 503, 504]) - allowed_methods: List of allowed HTTP methods (set, default {"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"}) - - catch_exceptions: List of exceptions to catch (tuple, default (exceptions.ConnectionError, exceptions.SSLError, exceptions.Timeout)) + - catch_exceptions: List of exceptions to catch (tuple, default (exceptions.ConnectionError, exceptions.ProxyError, exceptions.SSLError, exceptions.Timeout)) Returns: curl_cffi.requests.Session configured with browser impersonation, common headers, diff --git a/unshackle/core/titles/episode.py b/unshackle/core/titles/episode.py index 16ecab6..85ad7ac 100644 --- a/unshackle/core/titles/episode.py +++ b/unshackle/core/titles/episode.py @@ -95,9 +95,9 @@ class Episode(Title): media_info.audio_tracks, key=lambda x: ( float(x.bit_rate) if x.bit_rate else 0, - bool(x.format_additionalfeatures and "JOC" in x.format_additionalfeatures) + bool(x.format_additionalfeatures and "JOC" in x.format_additionalfeatures), ), - reverse=True + reverse=True, ) primary_audio_track = sorted_audio[0] unique_audio_languages = len({x.language.split("-")[0] for x in media_info.audio_tracks if x.language}) diff --git a/unshackle/core/titles/movie.py b/unshackle/core/titles/movie.py index 2e1d8bb..cd153b6 100644 --- a/unshackle/core/titles/movie.py +++ b/unshackle/core/titles/movie.py @@ -58,9 +58,9 @@ class Movie(Title): media_info.audio_tracks, key=lambda x: ( float(x.bit_rate) if x.bit_rate else 0, - bool(x.format_additionalfeatures and "JOC" in x.format_additionalfeatures) + bool(x.format_additionalfeatures and "JOC" in x.format_additionalfeatures), ), - reverse=True + reverse=True, ) primary_audio_track = sorted_audio[0] unique_audio_languages = len({x.language.split("-")[0] for x in media_info.audio_tracks if x.language}) diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index d97e9bd..7a5e10b 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -125,7 +125,7 @@ def is_exact_match(language: Union[str, Language], languages: Sequence[Union[str return closest_match(language, list(map(str, languages)))[1] <= LANGUAGE_EXACT_DISTANCE -def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: # type: ignore +def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: # type: ignore """ Scan a byte array for a wanted MP4/ISOBMFF box, then parse and yield each find. diff --git a/unshackle/vaults/SQLite.py b/unshackle/vaults/SQLite.py index ac89fec..f1922d7 100644 --- a/unshackle/vaults/SQLite.py +++ b/unshackle/vaults/SQLite.py @@ -37,7 +37,9 @@ class SQLite(Vault): if not self.has_table(service_name): continue - cursor.execute(f"SELECT `id`, `key_` FROM `{service_name}` WHERE `kid`=? AND `key_`!=?", (kid, "0" * 32)) + cursor.execute( + f"SELECT `id`, `key_` FROM `{service_name}` WHERE `kid`=? AND `key_`!=?", (kid, "0" * 32) + ) cek = cursor.fetchone() if cek: return cek[1] From ec3e15084613eded8272f06819db6f03789d1c22 Mon Sep 17 00:00:00 2001 From: Andy Date: Fri, 24 Oct 2025 00:53:47 +0000 Subject: [PATCH 29/62] feat(dl): add --audio-description flag to download AD tracks Add support for downloading audio description tracks via the --audio-description/-ad flag. Previously, descriptive audio tracks were always filtered out. Users can now optionally include them. Fixes #33 --- unshackle/commands/dl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 9821674..43e5cd7 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -237,6 +237,7 @@ class dl: @click.option("-ns", "--no-subs", is_flag=True, default=False, help="Do not download subtitle tracks.") @click.option("-na", "--no-audio", is_flag=True, default=False, help="Do not download audio tracks.") @click.option("-nc", "--no-chapters", is_flag=True, default=False, help="Do not download chapters tracks.") + @click.option("-ad", "--audio-description", is_flag=True, default=False, help="Download audio description tracks.") @click.option( "--slow", is_flag=True, @@ -582,6 +583,7 @@ class dl: no_subs: bool, no_audio: bool, no_chapters: bool, + audio_description: bool, slow: bool, list_: bool, list_titles: bool, @@ -1065,7 +1067,8 @@ class dl: # filter audio tracks # might have no audio tracks if part of the video, e.g. transport stream hls if len(title.tracks.audio) > 0: - title.tracks.select_audio(lambda x: not x.descriptive) # exclude descriptive audio + if not audio_description: + title.tracks.select_audio(lambda x: not x.descriptive) # exclude descriptive audio if acodec: title.tracks.select_audio(lambda x: x.codec == acodec) if not title.tracks.audio: From 4787be81903969698111a92967da04097b4fd57f Mon Sep 17 00:00:00 2001 From: Andy Date: Fri, 24 Oct 2025 00:56:28 +0000 Subject: [PATCH 30/62] docs: update CHANGELOG for audio description feature --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfdbaf7..1ded27c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Audio Description Track Support**: Added option to download audio description tracks + - Added `--audio-description/-ad` flag to optionally include descriptive audio tracks + - Previously, audio description tracks were always filtered out + - Users can now choose to download AD tracks when needed + - Fixes GitHub issue #33 - **Config Directory Support**: Cross-platform user config directory support - Fixed config loading to properly support user config directories across all platforms - Fixes GitHub issue #23 From bee2abcf5c50693741ae5a4179fd6df596de51c2 Mon Sep 17 00:00:00 2001 From: Andy Date: Fri, 24 Oct 2025 01:16:01 +0000 Subject: [PATCH 31/62] docs: improve GitHub issue templates for better bug reports and feature requests --- .github/ISSUE_TEMPLATE/bug_report.md | 57 ++++++++++++++++++----- .github/ISSUE_TEMPLATE/feature_request.md | 42 +++++++++++++++-- 2 files changed, 82 insertions(+), 17 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 28ddae8..4721aee 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,10 +1,9 @@ --- name: Bug report about: Create a report to help us improve -title: '' -labels: '' +title: "" +labels: "" assignees: Sp5rky - --- **Describe the bug** @@ -12,21 +11,55 @@ A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: -1. Run command uv run [...] + +1. Run command `uv run unshackle [...]` 2. See error **Expected behavior** A clear and concise description of what you expected to happen. +**System Details** + +- OS: [e.g. Windows 11, Ubuntu 22.04, macOS 14] +- unshackle Version: [e.g. 1.0.1] + +**Dependency Versions** (if relevant) + +- Shaka-packager: [e.g. 2.6.1] +- n_m3u8dl-re: [e.g. 0.3.0-beta] +- aria2c: [e.g. 1.36.0] +- ffmpeg: [e.g. 6.0] +- Other: [e.g. ccextractor, subby] + +**Logs/Error Output** + +
+Click to expand logs + +``` +Paste relevant error messages or stack traces here +``` + +
+ +**Configuration** (if relevant) +Please describe relevant configuration settings (DO NOT paste credentials or API keys): + +- Downloader used: [e.g. requests, aria2c, n_m3u8dl_re] +- Proxy provider: [e.g. NordVPN, none] +- Other relevant config options + **Screenshots** If applicable, add screenshots to help explain your problem. -**Desktop (please complete the following information):** - - OS: [e.g. Windows/Unix] - - Version [e.g. 1.0.1] - - Shaka-packager Version [e.g. 2.6.1] - - n_m3u8dl-re Version [e.g. 0.3.0 beta] - - Any additional software, such as subby/ccextractor/aria2c - **Additional context** -Add any other context about the problem here, if you're reporting issues with services not running or working, please try to expand on where in your service it breaks but don't include service code (unless you have rights to do so.) +Add any other context about the problem here. + +--- + +**⚠️ Important:** + +- **DO NOT include service-specific implementation code** unless you have explicit rights to share it +- **DO NOT share credentials, API keys, WVD files, or authentication tokens** +- For service-specific issues, describe the behavior without revealing proprietary implementation details +- Focus on core framework issues (downloads, DRM, track handling, CLI, configuration, etc.) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index d5714f3..2f3d79e 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,21 +1,53 @@ --- name: Feature request about: Suggest an idea for this project -title: '' -labels: '' +title: "" +labels: "" assignees: Sp5rky - --- +**Feature Category** +What area does this feature request relate to? + +- [ ] Core framework (downloaders, DRM, track handling) +- [ ] CLI/commands (new commands or command improvements) +- [ ] Configuration system +- [ ] Manifest parsing (DASH, HLS, ISM) +- [ ] Output/muxing (naming, metadata, tagging) +- [ ] Proxy system +- [ ] Key vault system +- [ ] Documentation +- [ ] Other (please specify) + **Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +A clear and concise description of what the problem is. +Example: "I'm always frustrated when [...]" **Describe the solution you'd like** A clear and concise description of what you want to happen. **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered. -Other tools like Devine/VT had this function [...] + +**Reference implementations** (if applicable) +Have you seen this feature in other tools? + +- [ ] Vinetrimmer +- [ ] yt-dlp +- [ ] Other: [please specify] + +Please describe how it works there (without sharing proprietary code). + +**Use case / Impact** + +- How would this feature benefit users? +- How often would you use this feature? +- Does this solve a common workflow issue? **Additional context** Add any other context or screenshots about the feature request here. + +--- + +**⚠️ Note:** +This project focuses on the core framework and tooling. Service-specific feature requests should focus on what the framework should support, not specific service implementations. From d0c6a7fa6306c2be5b459e9c424294599065c1b3 Mon Sep 17 00:00:00 2001 From: Andy Date: Sun, 26 Oct 2025 04:19:43 +0000 Subject: [PATCH 32/62] feat(api): add url field to services endpoint response --- unshackle/core/api/routes.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 5445c87..36b458c 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -109,9 +109,14 @@ async def services(request: web.Request) -> web.Response: title_regex: type: string nullable: true + url: + type: string + nullable: true + description: Service URL from short_help help: type: string nullable: true + description: Full service documentation '500': description: Server error """ @@ -120,7 +125,7 @@ async def services(request: web.Request) -> web.Response: services_info = [] for tag in service_tags: - service_data = {"tag": tag, "aliases": [], "geofence": [], "title_regex": None, "help": None} + service_data = {"tag": tag, "aliases": [], "geofence": [], "title_regex": None, "url": None, "help": None} try: service_module = Services.load(tag) @@ -134,6 +139,9 @@ async def services(request: web.Request) -> web.Response: if hasattr(service_module, "TITLE_RE"): service_data["title_regex"] = service_module.TITLE_RE + if hasattr(service_module, "cli") and hasattr(service_module.cli, "short_help"): + service_data["url"] = service_module.cli.short_help + if service_module.__doc__: service_data["help"] = service_module.__doc__.strip() From 5c8eb2107a1b908efe90d84775bcbb6f78b97b6d Mon Sep 17 00:00:00 2001 From: Andy Date: Sun, 26 Oct 2025 04:40:55 +0000 Subject: [PATCH 33/62] feat(api): complete API enhancements for v2.0.0 - Add missing download parameters (latest_episode, exact_lang, audio_description, no_mux) - Expand OpenAPI schema with comprehensive documentation for all 40+ download parameters - Add robust parameter validation with clear error messages - Implement job filtering by status/service and sorting capabilities --- unshackle/core/api/download_manager.py | 4 + unshackle/core/api/handlers.py | 125 ++++++++++++++- unshackle/core/api/routes.py | 205 ++++++++++++++++++++++++- 3 files changed, 328 insertions(+), 6 deletions(-) diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index 4b87c17..e59e083 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -222,12 +222,14 @@ def _perform_download( channels=params.get("channels"), no_atmos=params.get("no_atmos", False), wanted=params.get("wanted", []), + latest_episode=params.get("latest_episode", False), lang=params.get("lang", ["orig"]), v_lang=params.get("v_lang", []), a_lang=params.get("a_lang", []), s_lang=params.get("s_lang", ["all"]), require_subs=params.get("require_subs", []), forced_subs=params.get("forced_subs", False), + exact_lang=params.get("exact_lang", False), sub_format=params.get("sub_format"), video_only=params.get("video_only", False), audio_only=params.get("audio_only", False), @@ -236,6 +238,7 @@ def _perform_download( no_subs=params.get("no_subs", False), no_audio=params.get("no_audio", False), no_chapters=params.get("no_chapters", False), + audio_description=params.get("audio_description", False), slow=params.get("slow", False), list_=False, list_titles=False, @@ -245,6 +248,7 @@ def _perform_download( no_proxy=params.get("no_proxy", False), no_folder=params.get("no_folder", False), no_source=params.get("no_source", False), + no_mux=params.get("no_mux", False), workers=params.get("workers"), downloads=params.get("downloads", 1), best_available=params.get("best_available", False), diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 3b8dd1f..61cee5d 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -558,6 +558,81 @@ async def list_tracks_handler(data: Dict[str, Any]) -> web.Response: return web.json_response({"status": "error", "message": str(e)}, status=500) +def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]: + """ + Validate download parameters and return error message if invalid. + + Returns: + None if valid, error message string if invalid + """ + if "vcodec" in data and data["vcodec"]: + valid_vcodecs = ["H264", "H265", "VP9", "AV1"] + if data["vcodec"].upper() not in valid_vcodecs: + return f"Invalid vcodec: {data['vcodec']}. Must be one of: {', '.join(valid_vcodecs)}" + + if "acodec" in data and data["acodec"]: + valid_acodecs = ["AAC", "AC3", "EAC3", "OPUS", "FLAC", "ALAC", "VORBIS", "DTS"] + if data["acodec"].upper() not in valid_acodecs: + return f"Invalid acodec: {data['acodec']}. Must be one of: {', '.join(valid_acodecs)}" + + if "sub_format" in data and data["sub_format"]: + valid_sub_formats = ["SRT", "VTT", "ASS", "SSA"] + if data["sub_format"].upper() not in valid_sub_formats: + return f"Invalid sub_format: {data['sub_format']}. Must be one of: {', '.join(valid_sub_formats)}" + + if "vbitrate" in data and data["vbitrate"] is not None: + if not isinstance(data["vbitrate"], int) or data["vbitrate"] <= 0: + return "vbitrate must be a positive integer" + + if "abitrate" in data and data["abitrate"] is not None: + if not isinstance(data["abitrate"], int) or data["abitrate"] <= 0: + return "abitrate must be a positive integer" + + if "channels" in data and data["channels"] is not None: + if not isinstance(data["channels"], (int, float)) or data["channels"] <= 0: + return "channels must be a positive number" + + if "workers" in data and data["workers"] is not None: + if not isinstance(data["workers"], int) or data["workers"] <= 0: + return "workers must be a positive integer" + + if "downloads" in data and data["downloads"] is not None: + if not isinstance(data["downloads"], int) or data["downloads"] <= 0: + return "downloads must be a positive integer" + + exclusive_flags = [] + if data.get("video_only"): + exclusive_flags.append("video_only") + if data.get("audio_only"): + exclusive_flags.append("audio_only") + if data.get("subs_only"): + exclusive_flags.append("subs_only") + if data.get("chapters_only"): + exclusive_flags.append("chapters_only") + + if len(exclusive_flags) > 1: + return f"Cannot use multiple exclusive flags: {', '.join(exclusive_flags)}" + + if data.get("no_subs") and data.get("subs_only"): + return "Cannot use both no_subs and subs_only" + if data.get("no_audio") and data.get("audio_only"): + return "Cannot use both no_audio and audio_only" + + if data.get("s_lang") and data.get("require_subs"): + return "Cannot use both s_lang and require_subs" + + if "range" in data and data["range"]: + valid_ranges = ["SDR", "HDR10", "HDR10+", "DV", "HLG"] + if isinstance(data["range"], list): + for r in data["range"]: + if r.upper() not in valid_ranges: + return f"Invalid range value: {r}. Must be one of: {', '.join(valid_ranges)}" + elif data["range"].upper() not in valid_ranges: + return f"Invalid range value: {data['range']}. Must be one of: {', '.join(valid_ranges)}" + + return None + + async def download_handler(data: Dict[str, Any]) -> web.Response: """Handle download request - create and queue a download job.""" from unshackle.core.api.download_manager import get_download_manager @@ -577,6 +652,10 @@ async def download_handler(data: Dict[str, Any]) -> web.Response: {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, status=400 ) + validation_error = validate_download_parameters(data) + if validation_error: + return web.json_response({"status": "error", "message": validation_error}, status=400) + try: # Get download manager and start workers if needed manager = get_download_manager() @@ -596,13 +675,57 @@ async def download_handler(data: Dict[str, Any]) -> web.Response: async def list_download_jobs_handler(data: Dict[str, Any]) -> web.Response: - """Handle list download jobs request.""" + """Handle list download jobs request with optional filtering and sorting.""" from unshackle.core.api.download_manager import get_download_manager try: manager = get_download_manager() jobs = manager.list_jobs() + status_filter = data.get("status") + if status_filter: + jobs = [job for job in jobs if job.status.value == status_filter] + + service_filter = data.get("service") + if service_filter: + jobs = [job for job in jobs if job.service == service_filter] + + sort_by = data.get("sort_by", "created_time") + sort_order = data.get("sort_order", "desc") + + valid_sort_fields = ["created_time", "started_time", "completed_time", "progress", "status", "service"] + if sort_by not in valid_sort_fields: + return web.json_response( + { + "status": "error", + "message": f"Invalid sort_by: {sort_by}. Must be one of: {', '.join(valid_sort_fields)}", + }, + status=400, + ) + + if sort_order not in ["asc", "desc"]: + return web.json_response( + {"status": "error", "message": "Invalid sort_order: must be 'asc' or 'desc'"}, status=400 + ) + + reverse = sort_order == "desc" + + def get_sort_key(job): + """Get the sorting key value, handling None values.""" + value = getattr(job, sort_by, None) + if value is None: + if sort_by in ["created_time", "started_time", "completed_time"]: + from datetime import datetime + + return datetime.min if not reverse else datetime.max + elif sort_by == "progress": + return 0 + elif sort_by in ["status", "service"]: + return "" + return value + + jobs = sorted(jobs, key=get_sort_key, reverse=reverse) + job_list = [job.to_dict(include_full_details=False) for job in jobs] return web.json_response({"jobs": job_list}) diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 36b458c..7b2907b 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -256,9 +256,165 @@ async def download(request: web.Request) -> web.Response: title_id: type: string description: Title identifier + profile: + type: string + description: Profile to use for credentials and cookies + quality: + type: array + items: + type: integer + description: Download resolution(s), defaults to best available + vcodec: + type: string + description: Video codec to download (e.g., H264, H265, VP9, AV1) + acodec: + type: string + description: Audio codec to download (e.g., AAC, AC3, EAC3) + vbitrate: + type: integer + description: Video bitrate in kbps + abitrate: + type: integer + description: Audio bitrate in kbps + range: + type: array + items: + type: string + description: Video color range (SDR, HDR10, DV) + channels: + type: number + description: Audio channels (e.g., 2.0, 5.1, 7.1) + no_atmos: + type: boolean + description: Exclude Dolby Atmos audio tracks + wanted: + type: array + items: + type: string + description: Wanted episodes (e.g., ["S01E01", "S01E02"]) + latest_episode: + type: boolean + description: Download only the single most recent episode + lang: + type: array + items: + type: string + description: Language for video and audio (use 'orig' for original) + v_lang: + type: array + items: + type: string + description: Language for video tracks only + a_lang: + type: array + items: + type: string + description: Language for audio tracks only + s_lang: + type: array + items: + type: string + description: Language for subtitle tracks (default is 'all') + require_subs: + type: array + items: + type: string + description: Required subtitle languages + forced_subs: + type: boolean + description: Include forced subtitle tracks + exact_lang: + type: boolean + description: Use exact language matching (no variants) + sub_format: + type: string + description: Output subtitle format (SRT, VTT, etc.) + video_only: + type: boolean + description: Only download video tracks + audio_only: + type: boolean + description: Only download audio tracks + subs_only: + type: boolean + description: Only download subtitle tracks + chapters_only: + type: boolean + description: Only download chapters + no_subs: + type: boolean + description: Do not download subtitle tracks + no_audio: + type: boolean + description: Do not download audio tracks + no_chapters: + type: boolean + description: Do not download chapters + audio_description: + type: boolean + description: Download audio description tracks + slow: + type: boolean + description: Add 60-120s delay between downloads + skip_dl: + type: boolean + description: Skip downloading, only retrieve decryption keys + export: + type: string + description: Path to export decryption keys as JSON + cdm_only: + type: boolean + description: Only use CDM for key retrieval (true) or only vaults (false) + proxy: + type: string + description: Proxy URI or country code + no_proxy: + type: boolean + description: Force disable all proxy use + tag: + type: string + description: Set the group tag to be used + tmdb_id: + type: integer + description: Use this TMDB ID for tagging + tmdb_name: + type: boolean + description: Rename titles using TMDB name + tmdb_year: + type: boolean + description: Use release year from TMDB + no_folder: + type: boolean + description: Disable folder creation for TV shows + no_source: + type: boolean + description: Disable source tag from output file name + no_mux: + type: boolean + description: Do not mux tracks into a container file + workers: + type: integer + description: Max workers/threads per track download + downloads: + type: integer + description: Amount of tracks to download concurrently + best_available: + type: boolean + description: Continue with best available if requested quality unavailable responses: - '200': - description: Download started + '202': + description: Download job created + content: + application/json: + schema: + type: object + properties: + job_id: + type: string + status: + type: string + created_time: + type: string '400': description: Invalid request """ @@ -272,10 +428,40 @@ async def download(request: web.Request) -> web.Response: async def download_jobs(request: web.Request) -> web.Response: """ - List all download jobs. + List all download jobs with optional filtering and sorting. --- summary: List download jobs - description: Get list of all download jobs with their status + description: Get list of all download jobs with their status, with optional filtering by status/service and sorting + parameters: + - name: status + in: query + required: false + schema: + type: string + enum: [queued, downloading, completed, failed, cancelled] + description: Filter jobs by status + - name: service + in: query + required: false + schema: + type: string + description: Filter jobs by service tag + - name: sort_by + in: query + required: false + schema: + type: string + enum: [created_time, started_time, completed_time, progress, status, service] + default: created_time + description: Field to sort by + - name: sort_order + in: query + required: false + schema: + type: string + enum: [asc, desc] + default: desc + description: Sort order (ascending or descending) responses: '200': description: List of download jobs @@ -301,10 +487,19 @@ async def download_jobs(request: web.Request) -> web.Response: type: string progress: type: number + '400': + description: Invalid query parameters '500': description: Server error """ - return await list_download_jobs_handler({}) + # Extract query parameters + query_params = { + "status": request.query.get("status"), + "service": request.query.get("service"), + "sort_by": request.query.get("sort_by", "created_time"), + "sort_order": request.query.get("sort_order", "desc"), + } + return await list_download_jobs_handler(query_params) async def download_job_detail(request: web.Request) -> web.Response: From 504de2197aa7847bcb5ae86baa1becca1fdad486 Mon Sep 17 00:00:00 2001 From: Andy Date: Tue, 28 Oct 2025 18:49:13 +0000 Subject: [PATCH 34/62] fix(drm): add explicit UTF-8 encoding to mp4decrypt subprocess calls Fixes 'charmap' codec can't decode byte error that occurs on Windows when mp4decrypt outputs non-ASCII characters. Without explicit encoding, --- unshackle/core/drm/playready.py | 2 +- unshackle/core/drm/widevine.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/unshackle/core/drm/playready.py b/unshackle/core/drm/playready.py index a26428a..b1fcea0 100644 --- a/unshackle/core/drm/playready.py +++ b/unshackle/core/drm/playready.py @@ -338,7 +338,7 @@ class PlayReady: ] try: - subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') except subprocess.CalledProcessError as e: error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}" raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg) diff --git a/unshackle/core/drm/widevine.py b/unshackle/core/drm/widevine.py index 6c3d683..7fee1c9 100644 --- a/unshackle/core/drm/widevine.py +++ b/unshackle/core/drm/widevine.py @@ -289,7 +289,7 @@ class Widevine: ] try: - subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') except subprocess.CalledProcessError as e: error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}" raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg) From 351a60625883201e24003db65c54c571bafc3caf Mon Sep 17 00:00:00 2001 From: Andy Date: Thu, 30 Oct 2025 05:16:14 +0000 Subject: [PATCH 35/62] feat(api): add default parameter handling and improved error responses Add default parameter system to API server that matches CLI behavior, eliminating errors from missing optional parameters. --- unshackle/commands/serve.py | 13 +- unshackle/core/api/download_manager.py | 27 ++- unshackle/core/api/download_worker.py | 20 +- unshackle/core/api/errors.py | 322 +++++++++++++++++++++++++ unshackle/core/api/handlers.py | 257 ++++++++++++++++---- unshackle/core/api/routes.py | 285 +++++++++++++++++----- 6 files changed, 814 insertions(+), 110 deletions(-) create mode 100644 unshackle/core/api/errors.py diff --git a/unshackle/commands/serve.py b/unshackle/commands/serve.py index 515cd45..a28d633 100644 --- a/unshackle/commands/serve.py +++ b/unshackle/commands/serve.py @@ -18,7 +18,13 @@ from unshackle.core.constants import context_settings @click.option("--caddy", is_flag=True, default=False, help="Also serve with Caddy.") @click.option("--api-only", is_flag=True, default=False, help="Serve only the REST API, not pywidevine CDM.") @click.option("--no-key", is_flag=True, default=False, help="Disable API key authentication (allows all requests).") -def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool) -> None: +@click.option( + "--debug-api", + is_flag=True, + default=False, + help="Include technical debug information (tracebacks, stderr) in API error responses.", +) +def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool, debug_api: bool) -> None: """ Serve your Local Widevine Devices and REST API for Remote Access. @@ -50,6 +56,9 @@ def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool) -> No api_secret = None log.warning("Running with --no-key: Authentication is DISABLED for all API endpoints!") + if debug_api: + log.warning("Running with --debug-api: Error responses will include technical debug information!") + if caddy: if not binaries.Caddy: raise click.ClickException('Caddy executable "caddy" not found but is required for --caddy.') @@ -73,6 +82,7 @@ def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool) -> No else: app = web.Application(middlewares=[cors_middleware, pywidevine_serve.authentication]) app["config"] = {"users": [api_secret]} + app["debug_api"] = debug_api setup_routes(app) setup_swagger(app) log.info(f"REST API endpoints available at http://{host}:{port}/api/") @@ -101,6 +111,7 @@ def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool) -> No app.on_startup.append(pywidevine_serve._startup) app.on_cleanup.append(pywidevine_serve._cleanup) app.add_routes(pywidevine_serve.routes) + app["debug_api"] = debug_api setup_routes(app) setup_swagger(app) diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index e59e083..2f45d44 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -43,6 +43,9 @@ class DownloadJob: output_files: List[str] = field(default_factory=list) error_message: Optional[str] = None error_details: Optional[str] = None + error_code: Optional[str] = None + error_traceback: Optional[str] = None + worker_stderr: Optional[str] = None # Cancellation support cancel_event: threading.Event = field(default_factory=threading.Event) @@ -67,6 +70,9 @@ class DownloadJob: "output_files": self.output_files, "error_message": self.error_message, "error_details": self.error_details, + "error_code": self.error_code, + "error_traceback": self.error_traceback, + "worker_stderr": self.worker_stderr, } ) @@ -218,7 +224,7 @@ def _perform_download( acodec=params.get("acodec"), vbitrate=params.get("vbitrate"), abitrate=params.get("abitrate"), - range_=params.get("range", []), + range_=params.get("range", ["SDR"]), channels=params.get("channels"), no_atmos=params.get("no_atmos", False), wanted=params.get("wanted", []), @@ -483,9 +489,21 @@ class DownloadQueueManager: job.progress = 100.0 log.info(f"Download completed for job {job.job_id}: {len(output_files)} files") except Exception as e: + import traceback + + from unshackle.core.api.errors import categorize_exception + job.status = JobStatus.FAILED job.error_message = str(e) job.error_details = str(e) + + api_error = categorize_exception( + e, context={"service": job.service, "title_id": job.title_id, "job_id": job.job_id} + ) + job.error_code = api_error.error_code.value + + job.error_traceback = traceback.format_exc() + log.error(f"Download failed for job {job.job_id}: {e}") raise @@ -567,6 +585,7 @@ class DownloadQueueManager: log.debug(f"Worker stdout for job {job.job_id}: {stdout.strip()}") if stderr.strip(): log.warning(f"Worker stderr for job {job.job_id}: {stderr.strip()}") + job.worker_stderr = stderr.strip() result_data: Optional[Dict[str, Any]] = None try: @@ -579,10 +598,16 @@ class DownloadQueueManager: if returncode != 0: message = result_data.get("message") if result_data else "unknown error" + if result_data: + job.error_details = result_data.get("error_details", message) + job.error_code = result_data.get("error_code") raise Exception(f"Worker exited with code {returncode}: {message}") if not result_data or result_data.get("status") != "success": message = result_data.get("message") if result_data else "worker did not report success" + if result_data: + job.error_details = result_data.get("error_details", message) + job.error_code = result_data.get("error_code") raise Exception(f"Worker failure: {message}") return result_data.get("output_files", []) diff --git a/unshackle/core/api/download_worker.py b/unshackle/core/api/download_worker.py index 08810d4..7afca32 100644 --- a/unshackle/core/api/download_worker.py +++ b/unshackle/core/api/download_worker.py @@ -66,10 +66,28 @@ def main(argv: list[str]) -> int: result = {"status": "success", "output_files": output_files} except Exception as exc: # noqa: BLE001 - capture for parent process + from unshackle.core.api.errors import categorize_exception + exit_code = 1 tb = traceback.format_exc() log.error(f"Worker failed with error: {exc}") - result = {"status": "error", "message": str(exc), "traceback": tb} + + api_error = categorize_exception( + exc, + context={ + "service": payload.get("service") if "payload" in locals() else None, + "title_id": payload.get("title_id") if "payload" in locals() else None, + "job_id": payload.get("job_id") if "payload" in locals() else None, + }, + ) + + result = { + "status": "error", + "message": str(exc), + "error_details": api_error.message, + "error_code": api_error.error_code.value, + "traceback": tb, + } finally: try: diff --git a/unshackle/core/api/errors.py b/unshackle/core/api/errors.py new file mode 100644 index 0000000..312ee12 --- /dev/null +++ b/unshackle/core/api/errors.py @@ -0,0 +1,322 @@ +""" +API Error Handling System + +Provides structured error responses with error codes, categorization, +and optional debug information for the unshackle REST API. +""" + +from __future__ import annotations + +import traceback +from datetime import datetime, timezone +from enum import Enum +from typing import Any + +from aiohttp import web + + +class APIErrorCode(str, Enum): + """Standard API error codes for programmatic error handling.""" + + # Client errors (4xx) + INVALID_INPUT = "INVALID_INPUT" # Missing or malformed request data + INVALID_SERVICE = "INVALID_SERVICE" # Unknown service name + INVALID_TITLE_ID = "INVALID_TITLE_ID" # Invalid or malformed title ID + INVALID_PROFILE = "INVALID_PROFILE" # Profile doesn't exist + INVALID_PROXY = "INVALID_PROXY" # Invalid proxy specification + INVALID_LANGUAGE = "INVALID_LANGUAGE" # Invalid language code + INVALID_PARAMETERS = "INVALID_PARAMETERS" # Invalid download parameters + + AUTH_FAILED = "AUTH_FAILED" # Authentication failure (invalid credentials/cookies) + AUTH_REQUIRED = "AUTH_REQUIRED" # Missing authentication + FORBIDDEN = "FORBIDDEN" # Action not allowed + GEOFENCE = "GEOFENCE" # Content not available in region + + NOT_FOUND = "NOT_FOUND" # Resource not found (title, job, etc.) + NO_CONTENT = "NO_CONTENT" # No titles/tracks/episodes found + JOB_NOT_FOUND = "JOB_NOT_FOUND" # Download job doesn't exist + + RATE_LIMITED = "RATE_LIMITED" # Service rate limiting + + # Server errors (5xx) + INTERNAL_ERROR = "INTERNAL_ERROR" # Unexpected server error + SERVICE_ERROR = "SERVICE_ERROR" # Streaming service API error + NETWORK_ERROR = "NETWORK_ERROR" # Network connectivity issue + DRM_ERROR = "DRM_ERROR" # DRM/license acquisition failure + DOWNLOAD_ERROR = "DOWNLOAD_ERROR" # Download process failure + SERVICE_UNAVAILABLE = "SERVICE_UNAVAILABLE" # Service temporarily unavailable + WORKER_ERROR = "WORKER_ERROR" # Download worker process error + + +class APIError(Exception): + """ + Structured API error with error code, message, and details. + + Attributes: + error_code: Standardized error code from APIErrorCode enum + message: User-friendly error message + details: Additional structured error information + retryable: Whether the operation can be retried + http_status: HTTP status code to return (default based on error_code) + """ + + def __init__( + self, + error_code: APIErrorCode, + message: str, + details: dict[str, Any] | None = None, + retryable: bool = False, + http_status: int | None = None, + ): + super().__init__(message) + self.error_code = error_code + self.message = message + self.details = details or {} + self.retryable = retryable + self.http_status = http_status or self._default_http_status(error_code) + + @staticmethod + def _default_http_status(error_code: APIErrorCode) -> int: + """Map error codes to default HTTP status codes.""" + status_map = { + # 400 Bad Request + APIErrorCode.INVALID_INPUT: 400, + APIErrorCode.INVALID_SERVICE: 400, + APIErrorCode.INVALID_TITLE_ID: 400, + APIErrorCode.INVALID_PROFILE: 400, + APIErrorCode.INVALID_PROXY: 400, + APIErrorCode.INVALID_LANGUAGE: 400, + APIErrorCode.INVALID_PARAMETERS: 400, + # 401 Unauthorized + APIErrorCode.AUTH_REQUIRED: 401, + APIErrorCode.AUTH_FAILED: 401, + # 403 Forbidden + APIErrorCode.FORBIDDEN: 403, + APIErrorCode.GEOFENCE: 403, + # 404 Not Found + APIErrorCode.NOT_FOUND: 404, + APIErrorCode.NO_CONTENT: 404, + APIErrorCode.JOB_NOT_FOUND: 404, + # 429 Too Many Requests + APIErrorCode.RATE_LIMITED: 429, + # 500 Internal Server Error + APIErrorCode.INTERNAL_ERROR: 500, + # 502 Bad Gateway + APIErrorCode.SERVICE_ERROR: 502, + APIErrorCode.DRM_ERROR: 502, + # 503 Service Unavailable + APIErrorCode.NETWORK_ERROR: 503, + APIErrorCode.SERVICE_UNAVAILABLE: 503, + APIErrorCode.DOWNLOAD_ERROR: 500, + APIErrorCode.WORKER_ERROR: 500, + } + return status_map.get(error_code, 500) + + +def build_error_response( + error: APIError | Exception, + debug_mode: bool = False, + extra_debug_info: dict[str, Any] | None = None, +) -> web.Response: + """ + Build a structured JSON error response. + + Args: + error: APIError or generic Exception to convert to response + debug_mode: Whether to include technical debug information + extra_debug_info: Additional debug info (stderr, stdout, etc.) + + Returns: + aiohttp JSON response with structured error data + """ + if isinstance(error, APIError): + error_code = error.error_code.value + message = error.message + details = error.details + http_status = error.http_status + retryable = error.retryable + else: + # Generic exception - convert to INTERNAL_ERROR + error_code = APIErrorCode.INTERNAL_ERROR.value + message = str(error) or "An unexpected error occurred" + details = {} + http_status = 500 + retryable = False + + response_data: dict[str, Any] = { + "status": "error", + "error_code": error_code, + "message": message, + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + # Add details if present + if details: + response_data["details"] = details + + # Add retryable hint if specified + if retryable: + response_data["retryable"] = True + + # Add debug information if in debug mode + if debug_mode: + debug_info: dict[str, Any] = { + "exception_type": type(error).__name__, + } + + # Add traceback for debugging + if isinstance(error, Exception): + debug_info["traceback"] = traceback.format_exc() + + # Add any extra debug info provided + if extra_debug_info: + debug_info.update(extra_debug_info) + + response_data["debug_info"] = debug_info + + return web.json_response(response_data, status=http_status) + + +def categorize_exception( + exc: Exception, + context: dict[str, Any] | None = None, +) -> APIError: + """ + Categorize a generic exception into a structured APIError. + + This function attempts to identify the type of error based on the exception + type, message patterns, and optional context information. + + Args: + exc: The exception to categorize + context: Optional context (service name, operation type, etc.) + + Returns: + APIError with appropriate error code and details + """ + context = context or {} + exc_str = str(exc).lower() + exc_type = type(exc).__name__ + + # Authentication errors + if any(keyword in exc_str for keyword in ["auth", "login", "credential", "unauthorized", "forbidden", "token"]): + return APIError( + error_code=APIErrorCode.AUTH_FAILED, + message=f"Authentication failed: {exc}", + details={**context, "reason": "authentication_error"}, + retryable=False, + ) + + # Network errors + if any( + keyword in exc_str + for keyword in [ + "connection", + "timeout", + "network", + "unreachable", + "socket", + "dns", + "resolve", + ] + ) or exc_type in ["ConnectionError", "TimeoutError", "URLError", "SSLError"]: + return APIError( + error_code=APIErrorCode.NETWORK_ERROR, + message=f"Network error occurred: {exc}", + details={**context, "reason": "network_connectivity"}, + retryable=True, + http_status=503, + ) + + # Geofence/region errors + if any(keyword in exc_str for keyword in ["geofence", "region", "not available in", "territory"]): + return APIError( + error_code=APIErrorCode.GEOFENCE, + message=f"Content not available in your region: {exc}", + details={**context, "reason": "geofence_restriction"}, + retryable=False, + ) + + # Not found errors + if any(keyword in exc_str for keyword in ["not found", "404", "does not exist", "invalid id"]): + return APIError( + error_code=APIErrorCode.NOT_FOUND, + message=f"Resource not found: {exc}", + details={**context, "reason": "not_found"}, + retryable=False, + ) + + # Rate limiting + if any(keyword in exc_str for keyword in ["rate limit", "too many requests", "429", "throttle"]): + return APIError( + error_code=APIErrorCode.RATE_LIMITED, + message=f"Rate limit exceeded: {exc}", + details={**context, "reason": "rate_limited"}, + retryable=True, + http_status=429, + ) + + # DRM errors + if any(keyword in exc_str for keyword in ["drm", "license", "widevine", "playready", "decrypt"]): + return APIError( + error_code=APIErrorCode.DRM_ERROR, + message=f"DRM error: {exc}", + details={**context, "reason": "drm_failure"}, + retryable=False, + ) + + # Service unavailable + if any(keyword in exc_str for keyword in ["service unavailable", "503", "maintenance", "temporarily unavailable"]): + return APIError( + error_code=APIErrorCode.SERVICE_UNAVAILABLE, + message=f"Service temporarily unavailable: {exc}", + details={**context, "reason": "service_unavailable"}, + retryable=True, + http_status=503, + ) + + # Validation errors + if any(keyword in exc_str for keyword in ["invalid", "malformed", "validation"]) or exc_type in [ + "ValueError", + "ValidationError", + ]: + return APIError( + error_code=APIErrorCode.INVALID_INPUT, + message=f"Invalid input: {exc}", + details={**context, "reason": "validation_failed"}, + retryable=False, + ) + + # Default to internal error for unknown exceptions + return APIError( + error_code=APIErrorCode.INTERNAL_ERROR, + message=f"An unexpected error occurred: {exc}", + details={**context, "exception_type": exc_type}, + retryable=False, + ) + + +def handle_api_exception( + exc: Exception, + context: dict[str, Any] | None = None, + debug_mode: bool = False, + extra_debug_info: dict[str, Any] | None = None, +) -> web.Response: + """ + Convenience function to categorize an exception and build an error response. + + Args: + exc: The exception to handle + context: Optional context information + debug_mode: Whether to include debug information + extra_debug_info: Additional debug info + + Returns: + Structured JSON error response + """ + if isinstance(exc, APIError): + api_error = exc + else: + api_error = categorize_exception(exc, context) + + return build_error_response(api_error, debug_mode, extra_debug_info) diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 61cee5d..ba94adb 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional from aiohttp import web +from unshackle.core.api.errors import APIError, APIErrorCode, handle_api_exception from unshackle.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_CODEC_MAP from unshackle.core.proxies.basic import Basic from unshackle.core.proxies.hola import Hola @@ -14,6 +15,47 @@ from unshackle.core.tracks import Audio, Subtitle, Video log = logging.getLogger("api") +DEFAULT_DOWNLOAD_PARAMS = { + "profile": None, + "quality": [], + "vcodec": None, + "acodec": None, + "vbitrate": None, + "abitrate": None, + "range": ["SDR"], + "channels": None, + "no_atmos": False, + "wanted": [], + "latest_episode": False, + "lang": ["orig"], + "v_lang": [], + "a_lang": [], + "s_lang": ["all"], + "require_subs": [], + "forced_subs": False, + "exact_lang": False, + "sub_format": None, + "video_only": False, + "audio_only": False, + "subs_only": False, + "chapters_only": False, + "no_subs": False, + "no_audio": False, + "no_chapters": False, + "audio_description": False, + "slow": False, + "skip_dl": False, + "export": None, + "cdm_only": None, + "no_proxy": False, + "no_folder": False, + "no_source": False, + "no_mux": False, + "workers": None, + "downloads": 1, + "best_available": False, +} + def initialize_proxy_providers() -> List[Any]: """Initialize and return available proxy providers.""" @@ -199,22 +241,32 @@ def serialize_subtitle_track(track: Subtitle) -> Dict[str, Any]: } -async def list_titles_handler(data: Dict[str, Any]) -> web.Response: +async def list_titles_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: """Handle list-titles request.""" service_tag = data.get("service") title_id = data.get("title_id") profile = data.get("profile") if not service_tag: - return web.json_response({"status": "error", "message": "Missing required parameter: service"}, status=400) + raise APIError( + APIErrorCode.INVALID_INPUT, + "Missing required parameter: service", + details={"missing_parameter": "service"}, + ) if not title_id: - return web.json_response({"status": "error", "message": "Missing required parameter: title_id"}, status=400) + raise APIError( + APIErrorCode.INVALID_INPUT, + "Missing required parameter: title_id", + details={"missing_parameter": "title_id"}, + ) normalized_service = validate_service(service_tag) if not normalized_service: - return web.json_response( - {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, status=400 + raise APIError( + APIErrorCode.INVALID_SERVICE, + f"Invalid or unavailable service: {service_tag}", + details={"service": service_tag}, ) try: @@ -253,7 +305,11 @@ async def list_titles_handler(data: Dict[str, Any]) -> web.Response: resolved_proxy = resolve_proxy(proxy_param, proxy_providers) proxy_param = resolved_proxy except ValueError as e: - return web.json_response({"status": "error", "message": f"Proxy error: {e}"}, status=400) + raise APIError( + APIErrorCode.INVALID_PROXY, + f"Proxy error: {e}", + details={"proxy": proxy_param, "service": normalized_service}, + ) ctx = click.Context(dummy_service) ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=proxy_providers, profile=profile) @@ -321,27 +377,44 @@ async def list_titles_handler(data: Dict[str, Any]) -> web.Response: return web.json_response({"titles": title_list}) + except APIError: + raise except Exception as e: log.exception("Error listing titles") - return web.json_response({"status": "error", "message": str(e)}, status=500) + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "list_titles", "service": normalized_service, "title_id": title_id}, + debug_mode=debug_mode, + ) -async def list_tracks_handler(data: Dict[str, Any]) -> web.Response: +async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: """Handle list-tracks request.""" service_tag = data.get("service") title_id = data.get("title_id") profile = data.get("profile") if not service_tag: - return web.json_response({"status": "error", "message": "Missing required parameter: service"}, status=400) + raise APIError( + APIErrorCode.INVALID_INPUT, + "Missing required parameter: service", + details={"missing_parameter": "service"}, + ) if not title_id: - return web.json_response({"status": "error", "message": "Missing required parameter: title_id"}, status=400) + raise APIError( + APIErrorCode.INVALID_INPUT, + "Missing required parameter: title_id", + details={"missing_parameter": "title_id"}, + ) normalized_service = validate_service(service_tag) if not normalized_service: - return web.json_response( - {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, status=400 + raise APIError( + APIErrorCode.INVALID_SERVICE, + f"Invalid or unavailable service: {service_tag}", + details={"service": service_tag}, ) try: @@ -380,7 +453,11 @@ async def list_tracks_handler(data: Dict[str, Any]) -> web.Response: resolved_proxy = resolve_proxy(proxy_param, proxy_providers) proxy_param = resolved_proxy except ValueError as e: - return web.json_response({"status": "error", "message": f"Proxy error: {e}"}, status=400) + raise APIError( + APIErrorCode.INVALID_PROXY, + f"Proxy error: {e}", + details={"proxy": proxy_param, "service": normalized_service}, + ) ctx = click.Context(dummy_service) ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=proxy_providers, profile=profile) @@ -457,8 +534,10 @@ async def list_tracks_handler(data: Dict[str, Any]) -> web.Response: wanted = season_range.parse_tokens(wanted_param) log.debug(f"Parsed wanted '{wanted_param}' into {len(wanted)} episodes: {wanted[:10]}...") except Exception as e: - return web.json_response( - {"status": "error", "message": f"Invalid wanted parameter: {e}"}, status=400 + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + f"Invalid wanted parameter: {e}", + details={"wanted": wanted_param, "service": normalized_service}, ) elif season is not None and episode is not None: wanted = [f"{season}x{episode}"] @@ -481,8 +560,14 @@ async def list_tracks_handler(data: Dict[str, Any]) -> web.Response: log.debug(f"Found {len(matching_titles)} matching titles") if not matching_titles: - return web.json_response( - {"status": "error", "message": "No episodes found matching wanted criteria"}, status=404 + raise APIError( + APIErrorCode.NO_CONTENT, + "No episodes found matching wanted criteria", + details={ + "service": normalized_service, + "title_id": title_id, + "wanted": wanted_param or f"{season}x{episode}", + }, ) # If multiple episodes match, return tracks for all episodes @@ -524,12 +609,14 @@ async def list_tracks_handler(data: Dict[str, Any]) -> web.Response: response["unavailable_episodes"] = failed_episodes return web.json_response(response) else: - return web.json_response( - { - "status": "error", - "message": f"No available episodes found. Unavailable: {', '.join(failed_episodes)}", + raise APIError( + APIErrorCode.NO_CONTENT, + f"No available episodes found. Unavailable: {', '.join(failed_episodes)}", + details={ + "service": normalized_service, + "title_id": title_id, + "unavailable_episodes": failed_episodes, }, - status=404, ) else: # Single episode or movie @@ -553,9 +640,16 @@ async def list_tracks_handler(data: Dict[str, Any]) -> web.Response: return web.json_response(response) + except APIError: + raise except Exception as e: log.exception("Error listing tracks") - return web.json_response({"status": "error", "message": str(e)}, status=500) + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "list_tracks", "service": normalized_service, "title_id": title_id}, + debug_mode=debug_mode, + ) def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]: @@ -633,7 +727,7 @@ def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]: return None -async def download_handler(data: Dict[str, Any]) -> web.Response: +async def download_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: """Handle download request - create and queue a download job.""" from unshackle.core.api.download_manager import get_download_manager @@ -641,40 +735,74 @@ async def download_handler(data: Dict[str, Any]) -> web.Response: title_id = data.get("title_id") if not service_tag: - return web.json_response({"status": "error", "message": "Missing required parameter: service"}, status=400) + raise APIError( + APIErrorCode.INVALID_INPUT, + "Missing required parameter: service", + details={"missing_parameter": "service"}, + ) if not title_id: - return web.json_response({"status": "error", "message": "Missing required parameter: title_id"}, status=400) + raise APIError( + APIErrorCode.INVALID_INPUT, + "Missing required parameter: title_id", + details={"missing_parameter": "title_id"}, + ) normalized_service = validate_service(service_tag) if not normalized_service: - return web.json_response( - {"status": "error", "message": f"Invalid or unavailable service: {service_tag}"}, status=400 + raise APIError( + APIErrorCode.INVALID_SERVICE, + f"Invalid or unavailable service: {service_tag}", + details={"service": service_tag}, ) validation_error = validate_download_parameters(data) if validation_error: - return web.json_response({"status": "error", "message": validation_error}, status=400) + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + validation_error, + details={"service": normalized_service, "title_id": title_id}, + ) try: + # Load service module to extract service-specific parameter defaults + service_module = Services.load(normalized_service) + service_specific_defaults = {} + + # Extract default values from the service's click command + if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): + for param in service_module.cli.params: + if hasattr(param, "name") and hasattr(param, "default") and param.default is not None: + # Store service-specific defaults (e.g., drm_system, hydrate_track, profile for NF) + service_specific_defaults[param.name] = param.default + # Get download manager and start workers if needed manager = get_download_manager() await manager.start_workers() # Create download job with filtered parameters (exclude service and title_id as they're already passed) filtered_params = {k: v for k, v in data.items() if k not in ["service", "title_id"]} - job = manager.create_job(normalized_service, title_id, **filtered_params) + # Merge defaults with provided parameters (user params override service defaults, which override global defaults) + params_with_defaults = {**DEFAULT_DOWNLOAD_PARAMS, **service_specific_defaults, **filtered_params} + job = manager.create_job(normalized_service, title_id, **params_with_defaults) return web.json_response( {"job_id": job.job_id, "status": job.status.value, "created_time": job.created_time.isoformat()}, status=202 ) + except APIError: + raise except Exception as e: log.exception("Error creating download job") - return web.json_response({"status": "error", "message": str(e)}, status=500) + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "create_download_job", "service": normalized_service, "title_id": title_id}, + debug_mode=debug_mode, + ) -async def list_download_jobs_handler(data: Dict[str, Any]) -> web.Response: +async def list_download_jobs_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: """Handle list download jobs request with optional filtering and sorting.""" from unshackle.core.api.download_manager import get_download_manager @@ -695,17 +823,17 @@ async def list_download_jobs_handler(data: Dict[str, Any]) -> web.Response: valid_sort_fields = ["created_time", "started_time", "completed_time", "progress", "status", "service"] if sort_by not in valid_sort_fields: - return web.json_response( - { - "status": "error", - "message": f"Invalid sort_by: {sort_by}. Must be one of: {', '.join(valid_sort_fields)}", - }, - status=400, + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + f"Invalid sort_by: {sort_by}. Must be one of: {', '.join(valid_sort_fields)}", + details={"sort_by": sort_by, "valid_values": valid_sort_fields}, ) if sort_order not in ["asc", "desc"]: - return web.json_response( - {"status": "error", "message": "Invalid sort_order: must be 'asc' or 'desc'"}, status=400 + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + "Invalid sort_order: must be 'asc' or 'desc'", + details={"sort_order": sort_order, "valid_values": ["asc", "desc"]}, ) reverse = sort_order == "desc" @@ -730,12 +858,19 @@ async def list_download_jobs_handler(data: Dict[str, Any]) -> web.Response: return web.json_response({"jobs": job_list}) + except APIError: + raise except Exception as e: log.exception("Error listing download jobs") - return web.json_response({"status": "error", "message": str(e)}, status=500) + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "list_download_jobs"}, + debug_mode=debug_mode, + ) -async def get_download_job_handler(job_id: str) -> web.Response: +async def get_download_job_handler(job_id: str, request: Optional[web.Request] = None) -> web.Response: """Handle get specific download job request.""" from unshackle.core.api.download_manager import get_download_manager @@ -744,16 +879,27 @@ async def get_download_job_handler(job_id: str) -> web.Response: job = manager.get_job(job_id) if not job: - return web.json_response({"status": "error", "message": "Job not found"}, status=404) + raise APIError( + APIErrorCode.JOB_NOT_FOUND, + "Job not found", + details={"job_id": job_id}, + ) return web.json_response(job.to_dict(include_full_details=True)) + except APIError: + raise except Exception as e: log.exception(f"Error getting download job {job_id}") - return web.json_response({"status": "error", "message": str(e)}, status=500) + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "get_download_job", "job_id": job_id}, + debug_mode=debug_mode, + ) -async def cancel_download_job_handler(job_id: str) -> web.Response: +async def cancel_download_job_handler(job_id: str, request: Optional[web.Request] = None) -> web.Response: """Handle cancel download job request.""" from unshackle.core.api.download_manager import get_download_manager @@ -761,15 +907,30 @@ async def cancel_download_job_handler(job_id: str) -> web.Response: manager = get_download_manager() if not manager.get_job(job_id): - return web.json_response({"status": "error", "message": "Job not found"}, status=404) + raise APIError( + APIErrorCode.JOB_NOT_FOUND, + "Job not found", + details={"job_id": job_id}, + ) success = manager.cancel_job(job_id) if success: return web.json_response({"status": "success", "message": "Job cancelled"}) else: - return web.json_response({"status": "error", "message": "Job cannot be cancelled"}, status=400) + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + "Job cannot be cancelled (already completed or failed)", + details={"job_id": job_id}, + ) + except APIError: + raise except Exception as e: log.exception(f"Error cancelling download job {job_id}") - return web.json_response({"status": "error", "message": str(e)}, status=500) + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "cancel_download_job", "job_id": job_id}, + debug_mode=debug_mode, + ) diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index 7b2907b..a5202c5 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -1,9 +1,11 @@ import logging +import re from aiohttp import web from aiohttp_swagger3 import SwaggerDocs, SwaggerInfo, SwaggerUiSettings from unshackle.core import __version__ +from unshackle.core.api.errors import APIError, APIErrorCode, build_error_response, handle_api_exception from unshackle.core.api.handlers import (cancel_download_job_handler, download_handler, get_download_job_handler, list_download_jobs_handler, list_titles_handler, list_tracks_handler) from unshackle.core.services import Services @@ -107,7 +109,11 @@ async def services(request: web.Request) -> web.Response: items: type: string title_regex: - type: string + oneOf: + - type: string + - type: array + items: + type: string nullable: true url: type: string @@ -119,6 +125,28 @@ async def services(request: web.Request) -> web.Response: description: Full service documentation '500': description: Server error + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: error + error_code: + type: string + example: INTERNAL_ERROR + message: + type: string + example: An unexpected error occurred + details: + type: object + timestamp: + type: string + format: date-time + debug_info: + type: object + description: Only present when --debug-api flag is enabled """ try: service_tags = Services.get_tags() @@ -137,7 +165,21 @@ async def services(request: web.Request) -> web.Response: service_data["geofence"] = list(service_module.GEOFENCE) if hasattr(service_module, "TITLE_RE"): - service_data["title_regex"] = service_module.TITLE_RE + title_re = service_module.TITLE_RE + # Handle different types of TITLE_RE + if isinstance(title_re, re.Pattern): + service_data["title_regex"] = title_re.pattern + elif isinstance(title_re, str): + service_data["title_regex"] = title_re + elif isinstance(title_re, (list, tuple)): + # Convert list/tuple of patterns to list of strings + patterns = [] + for item in title_re: + if isinstance(item, re.Pattern): + patterns.append(item.pattern) + elif isinstance(item, str): + patterns.append(item) + service_data["title_regex"] = patterns if patterns else None if hasattr(service_module, "cli") and hasattr(service_module.cli, "short_help"): service_data["url"] = service_module.cli.short_help @@ -153,7 +195,8 @@ async def services(request: web.Request) -> web.Response: return web.json_response({"services": services_info}) except Exception as e: log.exception("Error listing services") - return web.json_response({"status": "error", "message": str(e)}, status=500) + debug_mode = request.app.get("debug_api", False) + return handle_api_exception(e, context={"operation": "list_services"}, debug_mode=debug_mode) async def list_titles(request: web.Request) -> web.Response: @@ -182,14 +225,104 @@ async def list_titles(request: web.Request) -> web.Response: '200': description: List of titles '400': - description: Invalid request + description: Invalid request (missing parameters, invalid service) + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: error + error_code: + type: string + example: INVALID_INPUT + message: + type: string + example: Missing required parameter + details: + type: object + timestamp: + type: string + format: date-time + '401': + description: Authentication failed + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: error + error_code: + type: string + example: AUTH_FAILED + message: + type: string + details: + type: object + timestamp: + type: string + format: date-time + '404': + description: Title not found + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: error + error_code: + type: string + example: NOT_FOUND + message: + type: string + details: + type: object + timestamp: + type: string + format: date-time + '500': + description: Server error + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: error + error_code: + type: string + example: INTERNAL_ERROR + message: + type: string + details: + type: object + timestamp: + type: string + format: date-time """ try: data = await request.json() - except Exception: - return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) + except Exception as e: + return build_error_response( + APIError( + APIErrorCode.INVALID_INPUT, + "Invalid JSON request body", + details={"error": str(e)}, + ), + request.app.get("debug_api", False), + ) - return await list_titles_handler(data) + try: + return await list_titles_handler(data, request) + except APIError as e: + debug_mode = request.app.get("debug_api", False) + return build_error_response(e, debug_mode) async def list_tracks(request: web.Request) -> web.Response: @@ -228,10 +361,21 @@ async def list_tracks(request: web.Request) -> web.Response: """ try: data = await request.json() - except Exception: - return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) + except Exception as e: + return build_error_response( + APIError( + APIErrorCode.INVALID_INPUT, + "Invalid JSON request body", + details={"error": str(e)}, + ), + request.app.get("debug_api", False), + ) - return await list_tracks_handler(data) + try: + return await list_tracks_handler(data, request) + except APIError as e: + debug_mode = request.app.get("debug_api", False) + return build_error_response(e, debug_mode) async def download(request: web.Request) -> web.Response: @@ -258,149 +402,149 @@ async def download(request: web.Request) -> web.Response: description: Title identifier profile: type: string - description: Profile to use for credentials and cookies + description: Profile to use for credentials and cookies (default - None) quality: type: array items: type: integer - description: Download resolution(s), defaults to best available + description: Download resolution(s) (default - best available) vcodec: type: string - description: Video codec to download (e.g., H264, H265, VP9, AV1) + description: Video codec to download (e.g., H264, H265, VP9, AV1) (default - None) acodec: type: string - description: Audio codec to download (e.g., AAC, AC3, EAC3) + description: Audio codec to download (e.g., AAC, AC3, EAC3) (default - None) vbitrate: type: integer - description: Video bitrate in kbps + description: Video bitrate in kbps (default - None) abitrate: type: integer - description: Audio bitrate in kbps + description: Audio bitrate in kbps (default - None) range: type: array items: type: string - description: Video color range (SDR, HDR10, DV) + description: Video color range (SDR, HDR10, DV) (default - ["SDR"]) channels: type: number - description: Audio channels (e.g., 2.0, 5.1, 7.1) + description: Audio channels (e.g., 2.0, 5.1, 7.1) (default - None) no_atmos: type: boolean - description: Exclude Dolby Atmos audio tracks + description: Exclude Dolby Atmos audio tracks (default - false) wanted: type: array items: type: string - description: Wanted episodes (e.g., ["S01E01", "S01E02"]) + description: Wanted episodes (e.g., ["S01E01", "S01E02"]) (default - all) latest_episode: type: boolean - description: Download only the single most recent episode + description: Download only the single most recent episode (default - false) lang: type: array items: type: string - description: Language for video and audio (use 'orig' for original) + description: Language for video and audio (use 'orig' for original) (default - ["orig"]) v_lang: type: array items: type: string - description: Language for video tracks only + description: Language for video tracks only (default - []) a_lang: type: array items: type: string - description: Language for audio tracks only + description: Language for audio tracks only (default - []) s_lang: type: array items: type: string - description: Language for subtitle tracks (default is 'all') + description: Language for subtitle tracks (default - ["all"]) require_subs: type: array items: type: string - description: Required subtitle languages + description: Required subtitle languages (default - []) forced_subs: type: boolean - description: Include forced subtitle tracks + description: Include forced subtitle tracks (default - false) exact_lang: type: boolean - description: Use exact language matching (no variants) + description: Use exact language matching (no variants) (default - false) sub_format: type: string - description: Output subtitle format (SRT, VTT, etc.) + description: Output subtitle format (SRT, VTT, etc.) (default - None) video_only: type: boolean - description: Only download video tracks + description: Only download video tracks (default - false) audio_only: type: boolean - description: Only download audio tracks + description: Only download audio tracks (default - false) subs_only: type: boolean - description: Only download subtitle tracks + description: Only download subtitle tracks (default - false) chapters_only: type: boolean - description: Only download chapters + description: Only download chapters (default - false) no_subs: type: boolean - description: Do not download subtitle tracks + description: Do not download subtitle tracks (default - false) no_audio: type: boolean - description: Do not download audio tracks + description: Do not download audio tracks (default - false) no_chapters: type: boolean - description: Do not download chapters + description: Do not download chapters (default - false) audio_description: type: boolean - description: Download audio description tracks + description: Download audio description tracks (default - false) slow: type: boolean - description: Add 60-120s delay between downloads + description: Add 60-120s delay between downloads (default - false) skip_dl: type: boolean - description: Skip downloading, only retrieve decryption keys + description: Skip downloading, only retrieve decryption keys (default - false) export: type: string - description: Path to export decryption keys as JSON + description: Path to export decryption keys as JSON (default - None) cdm_only: type: boolean - description: Only use CDM for key retrieval (true) or only vaults (false) + description: Only use CDM for key retrieval (true) or only vaults (false) (default - None) proxy: type: string - description: Proxy URI or country code + description: Proxy URI or country code (default - None) no_proxy: type: boolean - description: Force disable all proxy use + description: Force disable all proxy use (default - false) tag: type: string - description: Set the group tag to be used + description: Set the group tag to be used (default - None) tmdb_id: type: integer - description: Use this TMDB ID for tagging + description: Use this TMDB ID for tagging (default - None) tmdb_name: type: boolean - description: Rename titles using TMDB name + description: Rename titles using TMDB name (default - false) tmdb_year: type: boolean - description: Use release year from TMDB + description: Use release year from TMDB (default - false) no_folder: type: boolean - description: Disable folder creation for TV shows + description: Disable folder creation for TV shows (default - false) no_source: type: boolean - description: Disable source tag from output file name + description: Disable source tag from output file name (default - false) no_mux: type: boolean - description: Do not mux tracks into a container file + description: Do not mux tracks into a container file (default - false) workers: type: integer - description: Max workers/threads per track download + description: Max workers/threads per track download (default - None) downloads: type: integer - description: Amount of tracks to download concurrently + description: Amount of tracks to download concurrently (default - 1) best_available: type: boolean - description: Continue with best available if requested quality unavailable + description: Continue with best available if requested quality unavailable (default - false) responses: '202': description: Download job created @@ -420,10 +564,21 @@ async def download(request: web.Request) -> web.Response: """ try: data = await request.json() - except Exception: - return web.json_response({"status": "error", "message": "Invalid JSON request body"}, status=400) + except Exception as e: + return build_error_response( + APIError( + APIErrorCode.INVALID_INPUT, + "Invalid JSON request body", + details={"error": str(e)}, + ), + request.app.get("debug_api", False), + ) - return await download_handler(data) + try: + return await download_handler(data, request) + except APIError as e: + debug_mode = request.app.get("debug_api", False) + return build_error_response(e, debug_mode) async def download_jobs(request: web.Request) -> web.Response: @@ -499,7 +654,11 @@ async def download_jobs(request: web.Request) -> web.Response: "sort_by": request.query.get("sort_by", "created_time"), "sort_order": request.query.get("sort_order", "desc"), } - return await list_download_jobs_handler(query_params) + try: + return await list_download_jobs_handler(query_params, request) + except APIError as e: + debug_mode = request.app.get("debug_api", False) + return build_error_response(e, debug_mode) async def download_job_detail(request: web.Request) -> web.Response: @@ -523,7 +682,11 @@ async def download_job_detail(request: web.Request) -> web.Response: description: Server error """ job_id = request.match_info["job_id"] - return await get_download_job_handler(job_id) + try: + return await get_download_job_handler(job_id, request) + except APIError as e: + debug_mode = request.app.get("debug_api", False) + return build_error_response(e, debug_mode) async def cancel_download_job(request: web.Request) -> web.Response: @@ -549,7 +712,11 @@ async def cancel_download_job(request: web.Request) -> web.Response: description: Server error """ job_id = request.match_info["job_id"] - return await cancel_download_job_handler(job_id) + try: + return await cancel_download_job_handler(job_id, request) + except APIError as e: + debug_mode = request.app.get("debug_api", False) + return build_error_response(e, debug_mode) def setup_routes(app: web.Application) -> None: From 6ebdfa88183f864a68155916c30b314cbbed09c8 Mon Sep 17 00:00:00 2001 From: Andy Date: Fri, 31 Oct 2025 14:51:25 +0000 Subject: [PATCH 36/62] fix(subtitle): resolve SDH stripping crash with VTT files Fixes #34 --- unshackle/commands/dl.py | 30 ++++++++++++++++-------------- unshackle/core/tracks/subtitle.py | 31 ++++++++++++++++++++++--------- unshackle/unshackle-example.yaml | 27 +++++++++++++++++++-------- 3 files changed, 57 insertions(+), 31 deletions(-) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 43e5cd7..426b526 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -845,20 +845,22 @@ class dl: # strip SDH subs to non-SDH if no equivalent same-lang non-SDH is available # uses a loose check, e.g, wont strip en-US SDH sub if a non-SDH en-GB is available - for subtitle in title.tracks.subtitles: - if subtitle.sdh and not any( - is_close_match(subtitle.language, [x.language]) - for x in title.tracks.subtitles - if not x.sdh and not x.forced - ): - non_sdh_sub = deepcopy(subtitle) - non_sdh_sub.id += "_stripped" - non_sdh_sub.sdh = False - title.tracks.add(non_sdh_sub) - events.subscribe( - events.Types.TRACK_MULTIPLEX, - lambda track: (track.strip_hearing_impaired()) if track.id == non_sdh_sub.id else None, - ) + # Check if automatic SDH stripping is enabled in config + if config.subtitle.get("strip_sdh", True): + for subtitle in title.tracks.subtitles: + if subtitle.sdh and not any( + is_close_match(subtitle.language, [x.language]) + for x in title.tracks.subtitles + if not x.sdh and not x.forced + ): + non_sdh_sub = deepcopy(subtitle) + non_sdh_sub.id += "_stripped" + non_sdh_sub.sdh = False + title.tracks.add(non_sdh_sub) + events.subscribe( + events.Types.TRACK_MULTIPLEX, + lambda track: (track.strip_hearing_impaired()) if track.id == non_sdh_sub.id else None, + ) with console.status("Sorting tracks by language and bitrate...", spinner="dots"): video_sort_lang = v_lang or lang diff --git a/unshackle/core/tracks/subtitle.py b/unshackle/core/tracks/subtitle.py index e336345..7019142 100644 --- a/unshackle/core/tracks/subtitle.py +++ b/unshackle/core/tracks/subtitle.py @@ -979,20 +979,33 @@ class Subtitle(Track): stdout=subprocess.DEVNULL, ) else: - sub = Subtitles(self.path) + if config.subtitle.get("convert_before_strip", True) and self.codec != Subtitle.Codec.SubRip: + self.path = self.convert(Subtitle.Codec.SubRip) + self.codec = Subtitle.Codec.SubRip + try: - sub.filter(rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=True, rm_author=True) - except ValueError as e: - if "too many values to unpack" in str(e): - # Retry without name removal if the error is due to multiple colons in time references - # This can happen with lines like "at 10:00 and 2:00" - sub = Subtitles(self.path) + sub = Subtitles(self.path) + try: sub.filter( - rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=False, rm_author=True + rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=True, rm_author=True ) + except ValueError as e: + if "too many values to unpack" in str(e): + # Retry without name removal if the error is due to multiple colons in time references + # This can happen with lines like "at 10:00 and 2:00" + sub = Subtitles(self.path) + sub.filter( + rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=False, rm_author=True + ) + else: + raise + sub.save() + except (IOError, OSError) as e: + if "is not valid subtitle file" in str(e): + self.log.warning(f"Failed to strip SDH from {self.path.name}: {e}") + self.log.warning("Continuing without SDH stripping for this subtitle") else: raise - sub.save() def reverse_rtl(self) -> None: """ diff --git a/unshackle/unshackle-example.yaml b/unshackle/unshackle-example.yaml index a56bb77..a2c2408 100644 --- a/unshackle/unshackle-example.yaml +++ b/unshackle/unshackle-example.yaml @@ -1,3 +1,10 @@ +# API key for The Movie Database (TMDB) +tmdb_api_key: "" + +# Client ID for SIMKL API (optional, improves metadata matching) +# Get your free client ID at: https://simkl.com/settings/developer/ +simkl_client_id: "" + # Group or Username to postfix to the end of all download filenames following a dash tag: user_tag @@ -333,22 +340,26 @@ filenames: chapters: "Chapters_{title}_{random}.txt" subtitle: "Subtitle_{id}_{language}.srt" -# API key for The Movie Database (TMDB) -tmdb_api_key: "" - -# Client ID for SIMKL API (optional, improves metadata matching) -# Get your free client ID at: https://simkl.com/settings/developer/ -simkl_client_id: "" - # conversion_method: -# - auto (default): Smart routing - subby for WebVTT/SAMI, standard for others +# - auto (default): Smart routing - subby for WebVTT/SAMI, pycaption for others # - subby: Always use subby with advanced processing # - pycaption: Use only pycaption library (no SubtitleEdit, no subby) # - subtitleedit: Prefer SubtitleEdit when available, fall back to pycaption # - pysubs2: Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP) subtitle: conversion_method: auto + # sdh_method: Method to use for SDH (hearing impaired) stripping + # - auto (default): Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter + # - subby: Use subby library (SRT only) + # - subtitleedit: Use SubtitleEdit tool (Windows only, falls back to subtitle-filter) + # - filter-subs: Use subtitle-filter library directly sdh_method: auto + # strip_sdh: Automatically create stripped (non-SDH) versions of SDH subtitles + # Set to false to disable automatic SDH stripping entirely (default: true) + strip_sdh: true + # convert_before_strip: Auto-convert VTT/other formats to SRT before using subtitle-filter + # This ensures compatibility when subtitle-filter is used as fallback (default: true) + convert_before_strip: true # Configuration for pywidevine's serve functionality serve: From 597a8b791208f93067c8ea2bb85daca48b8539ff Mon Sep 17 00:00:00 2001 From: Andy Date: Sun, 2 Nov 2025 03:19:14 +0000 Subject: [PATCH 37/62] fix(naming): improve HDR detection with comprehensive transfer checks and hybrid DV+HDR10 support HDR10/PQ detection now includes: - PQ (most common) - SMPTE ST 2084 (CICP value 16) - BT.2100 - BT.2020-10 - smpte2084 (lowercase variant) HLG detection now includes: - HLG - Hybrid Log-Gamma - ARIB STD-B67 (CICP value 18) - arib-std-b67 (lowercase variant) Hybrid DV+HDR10 detection: - Now checks full hdr_format field for both "Dolby Vision" AND ("HDR10" OR "SMPTE ST 2086") - Properly generates filenames like "Movie.2160p.DV HDR H.265.mkv" - MediaInfo reports: "Dolby Vision / SMPTE ST 2086, HDR10 compatible" Also adds null safety for transfer characteristics to prevent errors when the field is None. --- unshackle/core/titles/episode.py | 12 +++++++++--- unshackle/core/titles/movie.py | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/unshackle/core/titles/episode.py b/unshackle/core/titles/episode.py index 85ad7ac..6592b60 100644 --- a/unshackle/core/titles/episode.py +++ b/unshackle/core/titles/episode.py @@ -173,20 +173,26 @@ class Episode(Title): if primary_video_track: codec = primary_video_track.format hdr_format = primary_video_track.hdr_format_commercial + hdr_format_full = primary_video_track.hdr_format or "" trc = ( primary_video_track.transfer_characteristics or primary_video_track.transfer_characteristics_original + or "" ) frame_rate = float(primary_video_track.frame_rate) + + # Primary HDR format detection if hdr_format: - if (primary_video_track.hdr_format or "").startswith("Dolby Vision"): + if hdr_format_full.startswith("Dolby Vision"): name += " DV" - if DYNAMIC_RANGE_MAP.get(hdr_format) and DYNAMIC_RANGE_MAP.get(hdr_format) != "DV": + if any(indicator in hdr_format_full for indicator in ["HDR10", "SMPTE ST 2086"]): name += " HDR" else: name += f" {DYNAMIC_RANGE_MAP.get(hdr_format)} " - elif trc and "HLG" in trc: + elif "HLG" in trc or "Hybrid Log-Gamma" in trc or "ARIB STD-B67" in trc or "arib-std-b67" in trc.lower(): name += " HLG" + elif any(indicator in trc for indicator in ["PQ", "SMPTE ST 2084", "BT.2100"]) or "smpte2084" in trc.lower() or "bt.2020-10" in trc.lower(): + name += " HDR" if frame_rate > 30: name += " HFR" name += f" {VIDEO_CODEC_MAP.get(codec, codec)}" diff --git a/unshackle/core/titles/movie.py b/unshackle/core/titles/movie.py index cd153b6..1545b18 100644 --- a/unshackle/core/titles/movie.py +++ b/unshackle/core/titles/movie.py @@ -124,20 +124,26 @@ class Movie(Title): if primary_video_track: codec = primary_video_track.format hdr_format = primary_video_track.hdr_format_commercial + hdr_format_full = primary_video_track.hdr_format or "" trc = ( primary_video_track.transfer_characteristics or primary_video_track.transfer_characteristics_original + or "" ) frame_rate = float(primary_video_track.frame_rate) + + # Primary HDR format detection if hdr_format: - if (primary_video_track.hdr_format or "").startswith("Dolby Vision"): + if hdr_format_full.startswith("Dolby Vision"): name += " DV" - if DYNAMIC_RANGE_MAP.get(hdr_format) and DYNAMIC_RANGE_MAP.get(hdr_format) != "DV": + if any(indicator in hdr_format_full for indicator in ["HDR10", "SMPTE ST 2086"]): name += " HDR" else: name += f" {DYNAMIC_RANGE_MAP.get(hdr_format)} " - elif trc and "HLG" in trc: + elif "HLG" in trc or "Hybrid Log-Gamma" in trc or "ARIB STD-B67" in trc or "arib-std-b67" in trc.lower(): name += " HLG" + elif any(indicator in trc for indicator in ["PQ", "SMPTE ST 2084", "BT.2100"]) or "smpte2084" in trc.lower() or "bt.2020-10" in trc.lower(): + name += " HDR" if frame_rate > 30: name += " HFR" name += f" {VIDEO_CODEC_MAP.get(codec, codec)}" From 27d0ca84a3a0e8bf1d0b408a1328ad5aa78728fa Mon Sep 17 00:00:00 2001 From: Andy Date: Sun, 2 Nov 2025 20:30:06 +0000 Subject: [PATCH 38/62] fix(dash): correct segment count calculation for startNumber=0 Fix off-by-one error in SegmentTemplate segment enumeration when startNumber is 0. Previously, the code would request one extra segment beyond what exists, causing 404 errors on the final segment. The issue was that end_number was calculated as a segment count via math.ceil(), but then used incorrectly with range(start_number, end_number + 1), treating it as both a count and an inclusive endpoint. Changed to explicitly calculate segment_count first, then derive end_number as: start_number + segment_count - 1 Example: - Duration: 3540.996s, segment duration: 4s - Before: segments 0-886 (887 segments) - segment 886 doesn't exist - After: segments 0-885 (886 segments) - correct --- unshackle/core/manifests/dash.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unshackle/core/manifests/dash.py b/unshackle/core/manifests/dash.py index 56fec08..152c274 100644 --- a/unshackle/core/manifests/dash.py +++ b/unshackle/core/manifests/dash.py @@ -384,7 +384,8 @@ class DASH: segment_duration = float(segment_template.get("duration")) or 1 if not end_number: - end_number = math.ceil(period_duration / (segment_duration / segment_timescale)) + segment_count = math.ceil(period_duration / (segment_duration / segment_timescale)) + end_number = start_number + segment_count - 1 for s in range(start_number, end_number + 1): segments.append( From 001f6a0146cab013c91a8b0ff2239c279de24a30 Mon Sep 17 00:00:00 2001 From: Andy Date: Sun, 2 Nov 2025 23:33:24 +0000 Subject: [PATCH 39/62] feat(cache): add TMDB and Simkl metadata caching to title cache --- unshackle/commands/dl.py | 64 +++++++-- unshackle/core/title_cacher.py | 161 +++++++++++++++++++++++ unshackle/core/utils/tags.py | 229 ++++++++++++++++++++++++++++++--- 3 files changed, 422 insertions(+), 32 deletions(-) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 426b526..db808ec 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -51,6 +51,7 @@ from unshackle.core.events import events from unshackle.core.proxies import Basic, Hola, NordVPN, SurfsharkVPN, WindscribeVPN from unshackle.core.service import Service from unshackle.core.services import Services +from unshackle.core.title_cacher import get_account_hash from unshackle.core.titles import Movie, Movies, Series, Song, Title_T from unshackle.core.titles.episode import Episode from unshackle.core.tracks import Audio, Subtitle, Tracks, Video @@ -690,16 +691,49 @@ class dl: level="INFO", operation="get_titles", service=self.service, context={"titles": titles_info} ) - if self.tmdb_year and self.tmdb_id: + title_cacher = service.title_cache if hasattr(service, "title_cache") else None + cache_title_id = None + if hasattr(service, "title"): + cache_title_id = service.title + elif hasattr(service, "title_id"): + cache_title_id = service.title_id + cache_region = service.current_region if hasattr(service, "current_region") else None + cache_account_hash = get_account_hash(service.credential) if hasattr(service, "credential") else None + + if (self.tmdb_year or self.tmdb_name) and self.tmdb_id: sample_title = titles[0] if hasattr(titles, "__getitem__") else titles kind = "tv" if isinstance(sample_title, Episode) else "movie" - tmdb_year_val = tags.get_year(self.tmdb_id, kind) - if tmdb_year_val: - if isinstance(titles, (Series, Movies)): - for t in titles: + + tmdb_year_val = None + tmdb_name_val = None + + if self.tmdb_year: + tmdb_year_val = tags.get_year( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + + if self.tmdb_name: + tmdb_name_val = tags.get_title( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + + if isinstance(titles, (Series, Movies)): + for t in titles: + if tmdb_year_val: t.year = tmdb_year_val - else: + if tmdb_name_val: + if isinstance(t, Episode): + t.title = tmdb_name_val + else: + t.name = tmdb_name_val + else: + if tmdb_year_val: titles.year = tmdb_year_val + if tmdb_name_val: + if isinstance(titles, Episode): + titles.title = tmdb_name_val + else: + titles.name = tmdb_name_val console.print(Padding(Rule(f"[rule.text]{titles.__class__.__name__}: {titles}"), (1, 2))) @@ -729,9 +763,13 @@ class dl: if isinstance(title, Episode) and not self.tmdb_searched: kind = "tv" if self.tmdb_id: - tmdb_title = tags.get_title(self.tmdb_id, kind) + tmdb_title = tags.get_title( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) else: - self.tmdb_id, tmdb_title, self.search_source = tags.search_show_info(title.title, title.year, kind) + self.tmdb_id, tmdb_title, self.search_source = tags.search_show_info( + title.title, title.year, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) if not (self.tmdb_id and tmdb_title and tags.fuzzy_match(tmdb_title, title.title)): self.tmdb_id = None if list_ or list_titles: @@ -747,7 +785,9 @@ class dl: self.tmdb_searched = True if isinstance(title, Movie) and (list_ or list_titles) and not self.tmdb_id: - movie_id, movie_title, _ = tags.search_show_info(title.name, title.year, "movie") + movie_id, movie_title, _ = tags.search_show_info( + title.name, title.year, "movie", title_cacher, cache_title_id, cache_region, cache_account_hash + ) if movie_id: console.print( Padding( @@ -760,11 +800,7 @@ class dl: if self.tmdb_id and getattr(self, "search_source", None) != "simkl": kind = "tv" if isinstance(title, Episode) else "movie" - tags.external_ids(self.tmdb_id, kind) - if self.tmdb_year: - tmdb_year_val = tags.get_year(self.tmdb_id, kind) - if tmdb_year_val: - title.year = tmdb_year_val + tags.external_ids(self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash) if slow and i != 0: delay = random.randint(60, 120) diff --git a/unshackle/core/title_cacher.py b/unshackle/core/title_cacher.py index f3346aa..76ca639 100644 --- a/unshackle/core/title_cacher.py +++ b/unshackle/core/title_cacher.py @@ -180,6 +180,167 @@ class TitleCacher: "hit_rate": f"{hit_rate:.1f}%", } + def get_cached_tmdb( + self, title_id: str, kind: str, region: Optional[str] = None, account_hash: Optional[str] = None + ) -> Optional[dict]: + """ + Get cached TMDB data for a title. + + Args: + title_id: The title identifier + kind: "movie" or "tv" + region: The region/proxy identifier + account_hash: Hash of account credentials + + Returns: + Dict with 'detail' and 'external_ids' if cached and valid, None otherwise + """ + if not config.title_cache_enabled: + return None + + cache_key = self._generate_cache_key(title_id, region, account_hash) + cache = self.cacher.get(cache_key, version=1) + + if not cache or not cache.data: + return None + + tmdb_data = getattr(cache.data, "tmdb_data", None) + if not tmdb_data: + return None + + tmdb_expiration = tmdb_data.get("expires_at") + if not tmdb_expiration or datetime.now() >= tmdb_expiration: + self.log.debug(f"TMDB cache expired for {title_id}") + return None + + if tmdb_data.get("kind") != kind: + self.log.debug(f"TMDB cache kind mismatch for {title_id}: cached {tmdb_data.get('kind')}, requested {kind}") + return None + + self.log.debug(f"TMDB cache hit for {title_id}") + return { + "detail": tmdb_data.get("detail"), + "external_ids": tmdb_data.get("external_ids"), + "fetched_at": tmdb_data.get("fetched_at"), + } + + def cache_tmdb( + self, + title_id: str, + detail_response: dict, + external_ids_response: dict, + kind: str, + region: Optional[str] = None, + account_hash: Optional[str] = None, + ) -> None: + """ + Cache TMDB data for a title. + + Args: + title_id: The title identifier + detail_response: Full TMDB detail API response + external_ids_response: Full TMDB external_ids API response + kind: "movie" or "tv" + region: The region/proxy identifier + account_hash: Hash of account credentials + """ + if not config.title_cache_enabled: + return + + cache_key = self._generate_cache_key(title_id, region, account_hash) + cache = self.cacher.get(cache_key, version=1) + + if not cache or not cache.data: + self.log.debug(f"Cannot cache TMDB data: no title cache exists for {title_id}") + return + + now = datetime.now() + tmdb_data = { + "detail": detail_response, + "external_ids": external_ids_response, + "kind": kind, + "fetched_at": now, + "expires_at": now + timedelta(days=7), # 7-day expiration + } + + cache.data.tmdb_data = tmdb_data + + cache.set(cache.data, expiration=cache.expiration) + self.log.debug(f"Cached TMDB data for {title_id} (kind={kind})") + + def get_cached_simkl( + self, title_id: str, region: Optional[str] = None, account_hash: Optional[str] = None + ) -> Optional[dict]: + """ + Get cached Simkl data for a title. + + Args: + title_id: The title identifier + region: The region/proxy identifier + account_hash: Hash of account credentials + + Returns: + Simkl response dict if cached and valid, None otherwise + """ + if not config.title_cache_enabled: + return None + + cache_key = self._generate_cache_key(title_id, region, account_hash) + cache = self.cacher.get(cache_key, version=1) + + if not cache or not cache.data: + return None + + simkl_data = getattr(cache.data, "simkl_data", None) + if not simkl_data: + return None + + simkl_expiration = simkl_data.get("expires_at") + if not simkl_expiration or datetime.now() >= simkl_expiration: + self.log.debug(f"Simkl cache expired for {title_id}") + return None + + self.log.debug(f"Simkl cache hit for {title_id}") + return simkl_data.get("response") + + def cache_simkl( + self, + title_id: str, + simkl_response: dict, + region: Optional[str] = None, + account_hash: Optional[str] = None, + ) -> None: + """ + Cache Simkl data for a title. + + Args: + title_id: The title identifier + simkl_response: Full Simkl API response + region: The region/proxy identifier + account_hash: Hash of account credentials + """ + if not config.title_cache_enabled: + return + + cache_key = self._generate_cache_key(title_id, region, account_hash) + cache = self.cacher.get(cache_key, version=1) + + if not cache or not cache.data: + self.log.debug(f"Cannot cache Simkl data: no title cache exists for {title_id}") + return + + now = datetime.now() + simkl_data = { + "response": simkl_response, + "fetched_at": now, + "expires_at": now + timedelta(days=7), + } + + cache.data.simkl_data = simkl_data + + cache.set(cache.data, expiration=cache.expiration) + self.log.debug(f"Cached Simkl data for {title_id}") + def get_region_from_proxy(proxy_url: Optional[str]) -> Optional[str]: """ diff --git a/unshackle/core/utils/tags.py b/unshackle/core/utils/tags.py index f9570d0..82a8e95 100644 --- a/unshackle/core/utils/tags.py +++ b/unshackle/core/utils/tags.py @@ -66,8 +66,37 @@ def fuzzy_match(a: str, b: str, threshold: float = 0.8) -> bool: return ratio >= threshold -def search_simkl(title: str, year: Optional[int], kind: str) -> Tuple[Optional[dict], Optional[str], Optional[int]]: +def search_simkl( + title: str, + year: Optional[int], + kind: str, + title_cacher=None, + cache_title_id: Optional[str] = None, + cache_region: Optional[str] = None, + cache_account_hash: Optional[str] = None, +) -> Tuple[Optional[dict], Optional[str], Optional[int]]: """Search Simkl API for show information by filename.""" + + if title_cacher and cache_title_id: + cached_simkl = title_cacher.get_cached_simkl(cache_title_id, cache_region, cache_account_hash) + if cached_simkl: + log.debug("Using cached Simkl data") + if cached_simkl.get("type") == "episode" and "show" in cached_simkl: + show_info = cached_simkl["show"] + show_title = show_info.get("title") + tmdb_id = show_info.get("ids", {}).get("tmdbtv") + if tmdb_id: + tmdb_id = int(tmdb_id) + return cached_simkl, show_title, tmdb_id + elif cached_simkl.get("type") == "movie" and "movie" in cached_simkl: + movie_info = cached_simkl["movie"] + movie_title = movie_info.get("title") + ids = movie_info.get("ids", {}) + tmdb_id = ids.get("tmdb") or ids.get("moviedb") + if tmdb_id: + tmdb_id = int(tmdb_id) + return cached_simkl, movie_title, tmdb_id + log.debug("Searching Simkl for %r (%s, %s)", title, kind, year) client_id = _simkl_client_id() @@ -112,19 +141,23 @@ def search_simkl(title: str, year: Optional[int], kind: str) -> Tuple[Optional[d log.debug("Simkl year mismatch: searched %d, got %d", year, show_year) return None, None, None + if title_cacher and cache_title_id: + try: + title_cacher.cache_simkl(cache_title_id, data, cache_region, cache_account_hash) + except Exception as exc: + log.debug("Failed to cache Simkl data: %s", exc) + tmdb_id = show_info.get("ids", {}).get("tmdbtv") if tmdb_id: tmdb_id = int(tmdb_id) log.debug("Simkl -> %s (TMDB ID %s)", show_title, tmdb_id) return data, show_title, tmdb_id - # Handle movie responses elif data.get("type") == "movie" and "movie" in data: movie_info = data["movie"] movie_title = movie_info.get("title") movie_year = movie_info.get("year") - # Verify title matches and year if provided if not fuzzy_match(movie_title, title): log.debug("Simkl title mismatch: searched %r, got %r", title, movie_title) return None, None, None @@ -132,6 +165,12 @@ def search_simkl(title: str, year: Optional[int], kind: str) -> Tuple[Optional[d log.debug("Simkl year mismatch: searched %d, got %d", year, movie_year) return None, None, None + if title_cacher and cache_title_id: + try: + title_cacher.cache_simkl(cache_title_id, data, cache_region, cache_account_hash) + except Exception as exc: + log.debug("Failed to cache Simkl data: %s", exc) + ids = movie_info.get("ids", {}) tmdb_id = ids.get("tmdb") or ids.get("moviedb") if tmdb_id: @@ -145,18 +184,85 @@ def search_simkl(title: str, year: Optional[int], kind: str) -> Tuple[Optional[d return None, None, None -def search_show_info(title: str, year: Optional[int], kind: str) -> Tuple[Optional[int], Optional[str], Optional[str]]: +def search_show_info( + title: str, + year: Optional[int], + kind: str, + title_cacher=None, + cache_title_id: Optional[str] = None, + cache_region: Optional[str] = None, + cache_account_hash: Optional[str] = None, +) -> Tuple[Optional[int], Optional[str], Optional[str]]: """Search for show information, trying Simkl first, then TMDB fallback. Returns (tmdb_id, title, source).""" - simkl_data, simkl_title, simkl_tmdb_id = search_simkl(title, year, kind) + simkl_data, simkl_title, simkl_tmdb_id = search_simkl( + title, year, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) if simkl_data and simkl_title and fuzzy_match(simkl_title, title): return simkl_tmdb_id, simkl_title, "simkl" - tmdb_id, tmdb_title = search_tmdb(title, year, kind) + tmdb_id, tmdb_title = search_tmdb(title, year, kind, title_cacher, cache_title_id, cache_region, cache_account_hash) return tmdb_id, tmdb_title, "tmdb" -def search_tmdb(title: str, year: Optional[int], kind: str) -> Tuple[Optional[int], Optional[str]]: +def _fetch_tmdb_detail(tmdb_id: int, kind: str) -> Optional[dict]: + """Fetch full TMDB detail response for caching.""" + api_key = _api_key() + if not api_key: + return None + + try: + session = _get_session() + r = session.get( + f"https://api.themoviedb.org/3/{kind}/{tmdb_id}", + params={"api_key": api_key}, + timeout=30, + ) + r.raise_for_status() + return r.json() + except requests.RequestException as exc: + log.debug("Failed to fetch TMDB detail: %s", exc) + return None + + +def _fetch_tmdb_external_ids(tmdb_id: int, kind: str) -> Optional[dict]: + """Fetch full TMDB external_ids response for caching.""" + api_key = _api_key() + if not api_key: + return None + + try: + session = _get_session() + r = session.get( + f"https://api.themoviedb.org/3/{kind}/{tmdb_id}/external_ids", + params={"api_key": api_key}, + timeout=30, + ) + r.raise_for_status() + return r.json() + except requests.RequestException as exc: + log.debug("Failed to fetch TMDB external IDs: %s", exc) + return None + + +def search_tmdb( + title: str, + year: Optional[int], + kind: str, + title_cacher=None, + cache_title_id: Optional[str] = None, + cache_region: Optional[str] = None, + cache_account_hash: Optional[str] = None, +) -> Tuple[Optional[int], Optional[str]]: + if title_cacher and cache_title_id: + cached_tmdb = title_cacher.get_cached_tmdb(cache_title_id, kind, cache_region, cache_account_hash) + if cached_tmdb and cached_tmdb.get("detail"): + detail = cached_tmdb["detail"] + tmdb_id = detail.get("id") + tmdb_title = detail.get("title") or detail.get("name") + log.debug("Using cached TMDB data: %r (ID %s)", tmdb_title, tmdb_id) + return tmdb_id, tmdb_title + api_key = _api_key() if not api_key: return None, None @@ -215,15 +321,41 @@ def search_tmdb(title: str, year: Optional[int], kind: str) -> Tuple[Optional[in ) if best_id is not None: + if title_cacher and cache_title_id: + try: + detail_response = _fetch_tmdb_detail(best_id, kind) + external_ids_response = _fetch_tmdb_external_ids(best_id, kind) + if detail_response and external_ids_response: + title_cacher.cache_tmdb( + cache_title_id, detail_response, external_ids_response, kind, cache_region, cache_account_hash + ) + except Exception as exc: + log.debug("Failed to cache TMDB data: %s", exc) + return best_id, best_title first = results[0] return first.get("id"), first.get("title") or first.get("name") -def get_title(tmdb_id: int, kind: str) -> Optional[str]: +def get_title( + tmdb_id: int, + kind: str, + title_cacher=None, + cache_title_id: Optional[str] = None, + cache_region: Optional[str] = None, + cache_account_hash: Optional[str] = None, +) -> Optional[str]: """Fetch the name/title of a TMDB entry by ID.""" + if title_cacher and cache_title_id: + cached_tmdb = title_cacher.get_cached_tmdb(cache_title_id, kind, cache_region, cache_account_hash) + if cached_tmdb and cached_tmdb.get("detail"): + detail = cached_tmdb["detail"] + tmdb_title = detail.get("title") or detail.get("name") + log.debug("Using cached TMDB title: %r", tmdb_title) + return tmdb_title + api_key = _api_key() if not api_key: return None @@ -236,17 +368,44 @@ def get_title(tmdb_id: int, kind: str) -> Optional[str]: timeout=30, ) r.raise_for_status() + js = r.json() + + if title_cacher and cache_title_id: + try: + external_ids_response = _fetch_tmdb_external_ids(tmdb_id, kind) + if external_ids_response: + title_cacher.cache_tmdb( + cache_title_id, js, external_ids_response, kind, cache_region, cache_account_hash + ) + except Exception as exc: + log.debug("Failed to cache TMDB data: %s", exc) + + return js.get("title") or js.get("name") except requests.RequestException as exc: log.debug("Failed to fetch TMDB title: %s", exc) return None - js = r.json() - return js.get("title") or js.get("name") - -def get_year(tmdb_id: int, kind: str) -> Optional[int]: +def get_year( + tmdb_id: int, + kind: str, + title_cacher=None, + cache_title_id: Optional[str] = None, + cache_region: Optional[str] = None, + cache_account_hash: Optional[str] = None, +) -> Optional[int]: """Fetch the release year of a TMDB entry by ID.""" + if title_cacher and cache_title_id: + cached_tmdb = title_cacher.get_cached_tmdb(cache_title_id, kind, cache_region, cache_account_hash) + if cached_tmdb and cached_tmdb.get("detail"): + detail = cached_tmdb["detail"] + date = detail.get("release_date") or detail.get("first_air_date") + if date and len(date) >= 4 and date[:4].isdigit(): + year = int(date[:4]) + log.debug("Using cached TMDB year: %d", year) + return year + api_key = _api_key() if not api_key: return None @@ -259,18 +418,41 @@ def get_year(tmdb_id: int, kind: str) -> Optional[int]: timeout=30, ) r.raise_for_status() + js = r.json() + + if title_cacher and cache_title_id: + try: + external_ids_response = _fetch_tmdb_external_ids(tmdb_id, kind) + if external_ids_response: + title_cacher.cache_tmdb( + cache_title_id, js, external_ids_response, kind, cache_region, cache_account_hash + ) + except Exception as exc: + log.debug("Failed to cache TMDB data: %s", exc) + + date = js.get("release_date") or js.get("first_air_date") + if date and len(date) >= 4 and date[:4].isdigit(): + return int(date[:4]) + return None except requests.RequestException as exc: log.debug("Failed to fetch TMDB year: %s", exc) return None - js = r.json() - date = js.get("release_date") or js.get("first_air_date") - if date and len(date) >= 4 and date[:4].isdigit(): - return int(date[:4]) - return None +def external_ids( + tmdb_id: int, + kind: str, + title_cacher=None, + cache_title_id: Optional[str] = None, + cache_region: Optional[str] = None, + cache_account_hash: Optional[str] = None, +) -> dict: + if title_cacher and cache_title_id: + cached_tmdb = title_cacher.get_cached_tmdb(cache_title_id, kind, cache_region, cache_account_hash) + if cached_tmdb and cached_tmdb.get("external_ids"): + log.debug("Using cached TMDB external IDs") + return cached_tmdb["external_ids"] -def external_ids(tmdb_id: int, kind: str) -> dict: api_key = _api_key() if not api_key: return {} @@ -287,6 +469,17 @@ def external_ids(tmdb_id: int, kind: str) -> dict: r.raise_for_status() js = r.json() log.debug("External IDs response: %s", js) + + if title_cacher and cache_title_id: + try: + detail_response = _fetch_tmdb_detail(tmdb_id, kind) + if detail_response: + title_cacher.cache_tmdb( + cache_title_id, detail_response, js, kind, cache_region, cache_account_hash + ) + except Exception as exc: + log.debug("Failed to cache TMDB data: %s", exc) + return js except requests.RequestException as exc: log.warning("Failed to fetch external IDs for %s %s: %s", kind, tmdb_id, exc) From 565b0e0ea799c84494506da33a720c8e0b451841 Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 3 Nov 2025 01:15:49 +0000 Subject: [PATCH 40/62] feat(session): add custom fingerprint and preset support Add support for custom TLS/HTTP fingerprints to session() function, enabling services to impersonate Android/OkHttp clients instead of just browsers. --- unshackle/core/session.py | 108 +++++++++++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 20 deletions(-) diff --git a/unshackle/core/session.py b/unshackle/core/session.py index a935752..67e0e12 100644 --- a/unshackle/core/session.py +++ b/unshackle/core/session.py @@ -21,6 +21,20 @@ warnings.filterwarnings( "ignore", message="Make sure you are using https over https proxy.*", category=RuntimeWarning, module="curl_cffi.*" ) +FINGERPRINT_PRESETS = { + "okhttp4": { + "ja3": ( + "771," # TLS 1.2 + "4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53," # Ciphers + "0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21," # Extensions + "29-23-24," # Named groups (x25519, secp256r1, secp384r1) + "0" # EC point formats + ), + "akamai": "1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p", + "description": "OkHttp 4.x on Android (BoringSSL TLS stack)", + }, +} + class MaxRetriesError(exceptions.RequestException): def __init__(self, message, cause=None): @@ -107,18 +121,34 @@ class CurlSession(Session): raise MaxRetriesError(f"Max retries exceeded for {method} {url}", cause=last_exception) -def session(browser: str | None = None, **kwargs) -> CurlSession: +def session( + browser: str | None = None, + ja3: str | None = None, + akamai: str | None = None, + extra_fp: dict | None = None, + **kwargs, +) -> CurlSession: """ - Create a curl_cffi session that impersonates a browser. + Create a curl_cffi session that impersonates a browser or custom TLS/HTTP fingerprint. This is a full replacement for requests.Session with browser impersonation and anti-bot capabilities. The session uses curl-impersonate under the hood to mimic real browser behavior. Args: - browser: Browser to impersonate (e.g. "chrome124", "firefox", "safari"). + browser: Browser to impersonate (e.g. "chrome124", "firefox", "safari") OR + fingerprint preset name (e.g. "okhttp4"). Uses the configured default from curl_impersonate.browser if not specified. - See https://github.com/lexiforest/curl_cffi#sessions for available options. + Available presets: okhttp4 + See https://github.com/lexiforest/curl_cffi#sessions for browser options. + ja3: Custom JA3 TLS fingerprint string (format: "SSLVersion,Ciphers,Extensions,Curves,PointFormats"). + When provided, curl_cffi will use this exact TLS fingerprint instead of the browser's default. + See https://curl-cffi.readthedocs.io/en/latest/impersonate/customize.html + akamai: Custom Akamai HTTP/2 fingerprint string (format: "SETTINGS|WINDOW_UPDATE|PRIORITY|PSEUDO_HEADERS"). + When provided, curl_cffi will use this exact HTTP/2 fingerprint instead of the browser's default. + See https://curl-cffi.readthedocs.io/en/latest/impersonate/customize.html + extra_fp: Additional fingerprint parameters dict for advanced customization. + See https://curl-cffi.readthedocs.io/en/latest/impersonate/customize.html **kwargs: Additional arguments passed to CurlSession constructor: - headers: Additional headers (dict) - cookies: Cookie jar or dict @@ -129,8 +159,6 @@ def session(browser: str | None = None, **kwargs) -> CurlSession: - allow_redirects: Follow redirects (bool, default True) - max_redirects: Maximum redirect count (int) - cert: Client certificate (str or tuple) - - ja3: JA3 fingerprint (str) - - akamai: Akamai fingerprint (str) Extra arguments for retry handler: - max_retries: Maximum number of retries (int, default 10) @@ -141,30 +169,70 @@ def session(browser: str | None = None, **kwargs) -> CurlSession: - catch_exceptions: List of exceptions to catch (tuple, default (exceptions.ConnectionError, exceptions.ProxyError, exceptions.SSLError, exceptions.Timeout)) Returns: - curl_cffi.requests.Session configured with browser impersonation, common headers, - and equivalent retry behavior to requests.Session. + curl_cffi.requests.Session configured with browser impersonation or custom fingerprints, + common headers, and equivalent retry behavior to requests.Session. - Example: - from unshackle.core.session import session as CurlSession + Examples: + # Standard browser impersonation + from unshackle.core.session import session class MyService(Service): @staticmethod - def get_session() -> CurlSession: - session = CurlSession( - impersonate="chrome", - ja3="...", - akamai="...", + def get_session(): + return session() # Uses config default browser + + # Use OkHttp 4.x preset for Android TV + class AndroidService(Service): + @staticmethod + def get_session(): + return session("okhttp4") + + # Custom fingerprint (manual) + class CustomService(Service): + @staticmethod + def get_session(): + return session( + ja3="771,4865-4866-4867-49195...", + akamai="1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p", + ) + + # With retry configuration + class MyService(Service): + @staticmethod + def get_session(): + return session( + "okhttp4", max_retries=5, status_forcelist=[429, 500], allowed_methods={"GET", "HEAD", "OPTIONS"}, ) - return session # Uses config default browser """ - session_config = { - "impersonate": browser or config.curl_impersonate.get("browser", "chrome"), - **kwargs, - } + if browser and browser in FINGERPRINT_PRESETS: + preset = FINGERPRINT_PRESETS[browser] + if ja3 is None: + ja3 = preset.get("ja3") + if akamai is None: + akamai = preset.get("akamai") + if extra_fp is None: + extra_fp = preset.get("extra_fp") + browser = None + + if browser is None and ja3 is None and akamai is None: + browser = config.curl_impersonate.get("browser", "chrome") + + session_config = {} + if browser: + session_config["impersonate"] = browser + + if ja3: + session_config["ja3"] = ja3 + if akamai: + session_config["akamai"] = akamai + if extra_fp: + session_config["extra_fp"] = extra_fp + + session_config.update(kwargs) session_obj = CurlSession(**session_config) session_obj.headers.update(config.headers) From f1fe940708ab3f73b5850557f4fa7f0319dd336e Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 3 Nov 2025 03:16:54 +0000 Subject: [PATCH 41/62] fix(session): update OkHttp fingerprint presets --- unshackle/core/session.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/unshackle/core/session.py b/unshackle/core/session.py index 67e0e12..298b6c6 100644 --- a/unshackle/core/session.py +++ b/unshackle/core/session.py @@ -25,13 +25,24 @@ FINGERPRINT_PRESETS = { "okhttp4": { "ja3": ( "771," # TLS 1.2 - "4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53," # Ciphers - "0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21," # Extensions + "4865-4866-4867-49195-49196-52393-49199-49200-52392-49171-49172-156-157-47-53," # Ciphers + "0-23-65281-10-11-35-16-5-13-51-45-43-21," # Extensions "29-23-24," # Named groups (x25519, secp256r1, secp384r1) "0" # EC point formats ), - "akamai": "1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p", - "description": "OkHttp 4.x on Android (BoringSSL TLS stack)", + "akamai": "4:16777216|16711681|0|m,p,a,s", + "description": "OkHttp 3.x/4.x (BoringSSL TLS stack)", + }, + "okhttp5": { + "ja3": ( + "771," # TLS 1.2 + "4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53," # Ciphers + "0-23-65281-10-11-35-16-5-13-51-45-43-21," # Extensions + "29-23-24," # Named groups (x25519, secp256r1, secp384r1) + "0" # EC point formats + ), + "akamai": "4:16777216|16711681|0|m,p,a,s", + "description": "OkHttp 5.x (BoringSSL TLS stack)", }, } From de48a98e92dcd107789e159a63f49bc15576c92a Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 3 Nov 2025 03:27:36 +0000 Subject: [PATCH 42/62] docs(changelog): complete v2.0.0 release documentation --- CHANGELOG.md | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ded27c..006f6b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [2.0.0] - 2025-10-25 +## [2.0.0] - Unreleased ### Breaking Changes @@ -57,6 +57,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Simkl API Configuration**: New API key support - Added `simkl_client_id` configuration option - Simkl now requires client_id from https://simkl.com/settings/developer/ +- **Custom Session Fingerprints**: Enhanced browser impersonation capabilities + - Added custom fingerprint and preset support for better service compatibility + - Configurable fingerprint presets for different device types + - Improved success rate with services using advanced bot detection +- **TMDB and Simkl Metadata Caching**: Enhanced title cache system + - Added metadata caching to title cache to reduce API calls + - Caches movie/show metadata alongside title information + - Improves performance for repeated title lookups and reduces API rate limiting +- **API Enhancements**: Improved REST API functionality + - Added default parameter handling for better request processing + - Added URL field to services endpoint response for easier service identification + - Complete API enhancements for production readiness + - Improved error responses with better detail and debugging information ### Changed @@ -96,6 +109,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **HLS Manifest Processing**: Minor HLS parser fix (by @TPD94, PR #19) - **lxml and pyplayready**: Updated dependencies (by @Sp5rky) - Updated lxml constraint and pyplayready import path for compatibility +- **DASH Segment Calculation**: Corrected segment handling + - Fixed segment count calculation for DASH manifests with startNumber=0 + - Ensures accurate segment processing for all DASH manifest configurations + - Prevents off-by-one errors in segment downloads +- **HDR Detection and Naming**: Comprehensive HDR format support + - Improved HDR detection with comprehensive transfer characteristics checks + - Added hybrid DV+HDR10 support for accurate file naming + - Better identification of HDR formats across different streaming services + - More accurate HDR/DV detection in filename generation +- **Subtitle Processing**: VTT subtitle handling improvements + - Resolved SDH (Subtitles for Deaf and Hard of hearing) stripping crash when processing VTT files + - More robust subtitle processing pipeline with better error handling + - Fixes crashes when filtering specific VTT subtitle formats +- **DRM Processing**: Enhanced encoding handling + - Added explicit UTF-8 encoding to mp4decrypt subprocess calls + - Prevents encoding issues on systems with non-UTF-8 default encodings + - Improves cross-platform compatibility for Windows and some Linux configurations +- **Session Fingerprints**: Updated OkHttp presets + - Updated OkHttp fingerprint presets for better Android TV compatibility + - Improved success rate with services using fingerprint-based detection + +### Documentation + +- **GitHub Issue Templates**: Enhanced issue reporting + - Improved bug report template with better structure and required fields + - Enhanced feature request template for clearer specifications + - Added helpful guidance for contributors to provide complete information ### Refactored From f979e94235647803cab277f292170fb1fbedc26c Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 3 Nov 2025 05:32:57 +0000 Subject: [PATCH 43/62] fix(session): remove padding extension from OkHttp JA3 fingerprints Remove extension 21 (TLS padding) from okhttp4 and okhttp5 JA3 strings to resolve SSL/TLS handshake failures. --- unshackle/core/session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unshackle/core/session.py b/unshackle/core/session.py index 298b6c6..dd5dc5b 100644 --- a/unshackle/core/session.py +++ b/unshackle/core/session.py @@ -26,7 +26,7 @@ FINGERPRINT_PRESETS = { "ja3": ( "771," # TLS 1.2 "4865-4866-4867-49195-49196-52393-49199-49200-52392-49171-49172-156-157-47-53," # Ciphers - "0-23-65281-10-11-35-16-5-13-51-45-43-21," # Extensions + "0-23-65281-10-11-35-16-5-13-51-45-43," # Extensions "29-23-24," # Named groups (x25519, secp256r1, secp384r1) "0" # EC point formats ), @@ -37,7 +37,7 @@ FINGERPRINT_PRESETS = { "ja3": ( "771," # TLS 1.2 "4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53," # Ciphers - "0-23-65281-10-11-35-16-5-13-51-45-43-21," # Extensions + "0-23-65281-10-11-35-16-5-13-51-45-43," # Extensions "29-23-24," # Named groups (x25519, secp256r1, secp384r1) "0" # EC point formats ), From f00790f31b7e777a4e1c23df4cefeb91cd4c702b Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 3 Nov 2025 16:56:58 +0000 Subject: [PATCH 44/62] feat: add service-specific configuration overrides Implement comprehensive per-service config override system that allows any configuration section (dl, n_m3u8dl_re, aria2c, subtitle, etc.) to be customized on a per-service basis. Fixes #13 --- unshackle/commands/dl.py | 77 +++++++++++++++++++++++--------- unshackle/unshackle-example.yaml | 16 ++++++- 2 files changed, 70 insertions(+), 23 deletions(-) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index db808ec..d546ac2 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -318,6 +318,38 @@ class dl: self.log = logging.getLogger("download") self.service = Services.get_tag(ctx.invoked_subcommand) + service_dl_config = config.services.get(self.service, {}).get("dl", {}) + if service_dl_config: + param_types = {param.name: param.type for param in ctx.command.params if param.name} + + for param_name, service_value in service_dl_config.items(): + if param_name not in ctx.params: + continue + + current_value = ctx.params[param_name] + global_default = config.dl.get(param_name) + param_type = param_types.get(param_name) + + try: + if param_type and global_default is not None: + global_default = param_type.convert(global_default, None, ctx) + except Exception as e: + self.log.debug(f"Failed to convert global default for '{param_name}': {e}") + + if current_value == global_default or (current_value is None and global_default is None): + try: + converted_value = service_value + if param_type and service_value is not None: + converted_value = param_type.convert(service_value, None, ctx) + + ctx.params[param_name] = converted_value + self.log.debug(f"Applied service-specific '{param_name}' override: {converted_value}") + except Exception as e: + self.log.warning( + f"Failed to apply service-specific '{param_name}' override: {e}. " + f"Check that the value '{service_value}' is valid for this parameter." + ) + self.profile = profile self.tmdb_id = tmdb_id self.tmdb_name = tmdb_name @@ -383,31 +415,34 @@ class dl: config.decryption = config.decryption_map.get(self.service, config.decryption) service_config = config.services.get(self.service, {}) + if service_config: + reserved_keys = { + "profiles", + "api_key", + "certificate", + "api_endpoint", + "region", + "device", + "endpoints", + "client", + "dl", + } - reserved_keys = { - "profiles", - "api_key", - "certificate", - "api_endpoint", - "region", - "device", - "endpoints", - "client", - } + for config_key, override_value in service_config.items(): + if config_key in reserved_keys or not isinstance(override_value, dict): + continue - for config_key, override_value in service_config.items(): - if config_key in reserved_keys: - continue + if hasattr(config, config_key): + current_config = getattr(config, config_key, {}) - if isinstance(override_value, dict) and hasattr(config, config_key): - current_config = getattr(config, config_key, {}) - if isinstance(current_config, dict): - merged_config = {**current_config, **override_value} - setattr(config, config_key, merged_config) + if isinstance(current_config, dict): + merged_config = deepcopy(current_config) + merge_dict(override_value, merged_config) + setattr(config, config_key, merged_config) - self.log.debug( - f"Applied service-specific '{config_key}' overrides for {self.service}: {override_value}" - ) + self.log.debug( + f"Applied service-specific '{config_key}' overrides for {self.service}: {override_value}" + ) with console.status("Loading Key Vaults...", spinner="dots"): self.vaults = Vaults(self.service) diff --git a/unshackle/unshackle-example.yaml b/unshackle/unshackle-example.yaml index a2c2408..e5e1b68 100644 --- a/unshackle/unshackle-example.yaml +++ b/unshackle/unshackle-example.yaml @@ -450,12 +450,24 @@ services: region: "GB" api_endpoint: "https://api.uk.service.com" + # Example: Rate-limited service + RATE_LIMITED_SERVICE: + dl: + downloads: 2 # Limit concurrent downloads + workers: 4 # Reduce workers to avoid rate limits + n_m3u8dl_re: + thread_count: 4 # Very low thread count + retry_count: 20 # More retries for flaky service + aria2c: + max_concurrent_downloads: 1 # Download tracks one at a time + max_connection_per_server: 1 # Single connection only + # Notes on service-specific overrides: # - Overrides are merged with global config, not replaced # - Only specified keys are overridden, others use global defaults # - Reserved keys (profiles, api_key, certificate, etc.) are NOT treated as overrides - # - Any dict-type config option can be overridden (dl, aria2c, n_m3u8dl_re, etc.) - # - Use --debug flag to see which overrides are applied during downloads + # - Any dict-type config option can be overridden (dl, aria2c, n_m3u8dl_re, subtitle, etc.) + # - CLI arguments always take priority over service-specific config # External proxy provider services proxy_providers: From 8b0b3045e3a932ab7245aad0cdd1a7b905ff6e13 Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 3 Nov 2025 20:23:45 +0000 Subject: [PATCH 45/62] feat(fonts): add Linux font support for ASS/SSA subtitles Implements cross-platform font discovery and intelligent fallback system for ASS/SSA subtitle rendering on Linux/macOS systems. Windows support has not been tested --- pyproject.toml | 2 + unshackle/commands/dl.py | 153 ++++++++++++++++--- unshackle/core/utilities.py | 286 ++++++++++++++++++++++++++++++++++-- uv.lock | 37 ++++- 4 files changed, 441 insertions(+), 37 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cd1552d..be4c2db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "click>=8.1.8,<9", "construct>=2.8.8,<3", "crccheck>=1.3.0,<2", + "fonttools>=4.0.0,<5", "jsonpickle>=3.0.4,<4", "langcodes>=3.4.0,<4", "lxml>=5.2.1,<7", @@ -60,6 +61,7 @@ dependencies = [ "subby", "aiohttp-swagger3>=0.9.0,<1", "pysubs2>=1.7.0,<2", + "PyExecJS>=1.5.1,<2", ] [project.urls] diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index d546ac2..0a0e23d 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -57,8 +57,8 @@ from unshackle.core.titles.episode import Episode from unshackle.core.tracks import Audio, Subtitle, Tracks, Video from unshackle.core.tracks.attachment import Attachment from unshackle.core.tracks.hybrid import Hybrid -from unshackle.core.utilities import (get_debug_logger, get_system_fonts, init_debug_logger, is_close_match, - time_elapsed_since) +from unshackle.core.utilities import (find_font_with_fallbacks, get_debug_logger, get_system_fonts, init_debug_logger, + is_close_match, suggest_font_packages, time_elapsed_since) from unshackle.core.utils import tags from unshackle.core.utils.click_types import (LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE, ContextData, MultipleChoice, SubtitleCodecChoice, VideoCodecChoice) @@ -69,7 +69,7 @@ from unshackle.core.vaults import Vaults class dl: @staticmethod - def _truncate_pssh_for_display(pssh_string: str, drm_type: str) -> str: + def truncate_pssh_for_display(pssh_string: str, drm_type: str) -> str: """Truncate PSSH string for display when not in debug mode.""" if logging.root.level == logging.DEBUG or not pssh_string: return pssh_string @@ -80,6 +80,115 @@ class dl: return pssh_string[: max_width - 3] + "..." + def find_custom_font(self, font_name: str) -> Optional[Path]: + """ + Find font in custom fonts directory. + + Args: + font_name: Font family name to find + + Returns: + Path to font file, or None if not found + """ + family_dir = Path(config.directories.fonts, font_name) + if family_dir.exists(): + fonts = list(family_dir.glob("*.*tf")) + return fonts[0] if fonts else None + return None + + def prepare_temp_font( + self, + font_name: str, + matched_font: Path, + system_fonts: dict[str, Path], + temp_font_files: list[Path] + ) -> Path: + """ + Copy system font to temp and log if using fallback. + + Args: + font_name: Requested font name + matched_font: Path to matched system font + system_fonts: Dictionary of available system fonts + temp_font_files: List to track temp files for cleanup + + Returns: + Path to temp font file + """ + # Find the matched name for logging + matched_name = next( + (name for name, path in system_fonts.items() if path == matched_font), + None + ) + + if matched_name and matched_name.lower() != font_name.lower(): + self.log.info(f"Using '{matched_name}' as fallback for '{font_name}'") + + # Create unique temp file path + safe_name = font_name.replace(" ", "_").replace("/", "_") + temp_path = config.directories.temp / f"font_{safe_name}{matched_font.suffix}" + + # Copy if not already exists + if not temp_path.exists(): + shutil.copy2(matched_font, temp_path) + temp_font_files.append(temp_path) + + return temp_path + + def _attach_subtitle_fonts( + self, + font_names: list[str], + title: Title_T, + temp_font_files: list[Path] + ) -> tuple[int, list[str]]: + """ + Attach fonts for subtitle rendering. + + Args: + font_names: List of font names requested by subtitles + title: Title object to attach fonts to + temp_font_files: List to track temp files for cleanup + + Returns: + Tuple of (fonts_attached_count, missing_fonts_list) + """ + system_fonts = get_system_fonts() + self.log.info(f"Discovered {len(system_fonts)} system font families") + + font_count = 0 + missing_fonts = [] + + for font_name in set(font_names): + # Try custom fonts first + if custom_font := self.find_custom_font(font_name): + title.tracks.add(Attachment(path=custom_font, name=f"{font_name} ({custom_font.stem})")) + font_count += 1 + continue + + # Try system fonts with fallback + if system_font := find_font_with_fallbacks(font_name, system_fonts): + temp_path = self.prepare_temp_font(font_name, system_font, system_fonts, temp_font_files) + title.tracks.add(Attachment(path=temp_path, name=f"{font_name} ({system_font.stem})")) + font_count += 1 + else: + self.log.warning(f"Subtitle uses font '{font_name}' but it could not be found") + missing_fonts.append(font_name) + + return font_count, missing_fonts + + def _suggest_missing_fonts(self, missing_fonts: list[str]) -> None: + """ + Show package installation suggestions for missing fonts. + + Args: + missing_fonts: List of font names that couldn't be found + """ + if suggestions := suggest_font_packages(missing_fonts): + self.log.info("Install font packages to improve subtitle rendering:") + for package_cmd, fonts in suggestions.items(): + self.log.info(f" $ sudo apt install {package_cmd}") + self.log.info(f" → Provides: {', '.join(fonts)}") + @click.command( short_help="Download, Decrypt, and Mux tracks for titles from a Service.", cls=Services, @@ -794,6 +903,7 @@ class dl: continue console.print(Padding(Rule(f"[rule.text]{title}"), (1, 2))) + temp_font_files = [] if isinstance(title, Episode) and not self.tmdb_searched: kind = "tv" @@ -1435,26 +1545,16 @@ class dl: if line.startswith("Style: "): font_names.append(line.removesuffix("Style: ").split(",")[1]) - font_count = 0 - system_fonts = get_system_fonts() - for font_name in set(font_names): - family_dir = Path(config.directories.fonts, font_name) - fonts_from_system = [file for name, file in system_fonts.items() if name.startswith(font_name)] - if family_dir.exists(): - fonts = family_dir.glob("*.*tf") - for font in fonts: - title.tracks.add(Attachment(path=font, name=f"{font_name} ({font.stem})")) - font_count += 1 - elif fonts_from_system: - for font in fonts_from_system: - title.tracks.add(Attachment(path=font, name=f"{font_name} ({font.stem})")) - font_count += 1 - else: - self.log.warning(f"Subtitle uses font [text2]{font_name}[/] but it could not be found...") + font_count, missing_fonts = self._attach_subtitle_fonts( + font_names, title, temp_font_files + ) if font_count: self.log.info(f"Attached {font_count} fonts for the Subtitles") + if missing_fonts and sys.platform != "win32": + self._suggest_missing_fonts(missing_fonts) + # Handle DRM decryption BEFORE repacking (must decrypt first!) service_name = service.__class__.__name__.upper() decryption_method = config.decryption_map.get(service_name, config.decryption) @@ -1608,8 +1708,17 @@ class dl: video_track.delete() for track in title.tracks: track.delete() + + # Clear temp font attachment paths and delete other attachments for attachment in title.tracks.attachments: - attachment.delete() + if attachment.path and attachment.path in temp_font_files: + attachment.path = None + else: + attachment.delete() + + # Clean up temp fonts + for temp_path in temp_font_files: + temp_path.unlink(missing_ok=True) else: # dont mux @@ -1752,7 +1861,7 @@ class dl: ) with self.DRM_TABLE_LOCK: - pssh_display = self._truncate_pssh_for_display(drm.pssh.dumps(), "Widevine") + pssh_display = self.truncate_pssh_for_display(drm.pssh.dumps(), "Widevine") cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="fold")) pre_existing_tree = next( (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None @@ -1921,7 +2030,7 @@ class dl: ) with self.DRM_TABLE_LOCK: - pssh_display = self._truncate_pssh_for_display(drm.pssh_b64 or "", "PlayReady") + pssh_display = self.truncate_pssh_for_display(drm.pssh_b64 or "", "PlayReady") cek_tree = Tree( Text.assemble( ("PlayReady", "cyan"), diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index 7a5e10b..a09f41c 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -21,6 +21,7 @@ from uuid import uuid4 import chardet import requests from construct import ValidationError +from fontTools import ttLib from langcodes import Language, closest_match from pymp4.parser import Box from unidecode import unidecode @@ -29,6 +30,30 @@ from unshackle.core.cacher import Cacher from unshackle.core.config import config from unshackle.core.constants import LANGUAGE_EXACT_DISTANCE, LANGUAGE_MAX_DISTANCE +""" +Utility functions for the unshackle media archival tool. + +This module provides various utility functions including: +- Font discovery and fallback system for subtitle rendering +- Cross-platform system font scanning with Windows → Linux font family mapping +- Log file management and rotation +- IP geolocation with caching and provider rotation +- Language matching utilities +- MP4/ISOBMFF box parsing +- File sanitization and path handling +- Structured JSON debug logging + +Font System: + The font subsystem enables cross-platform font discovery for ASS/SSA subtitles. + On Linux, it scans standard font directories and maps Windows font names (Arial, + Times New Roman) to their Linux equivalents (Liberation Sans, Liberation Serif). + +Main Font Functions: + - get_system_fonts(): Discover installed fonts across platforms + - find_font_with_fallbacks(): Match fonts with intelligent fallback strategies + - suggest_font_packages(): Recommend packages to install for missing fonts +""" + def rotate_log_file(log_path: Path, keep: int = 20) -> Path: """ @@ -428,21 +453,254 @@ def get_extension(value: Union[str, Path, ParseResult]) -> Optional[str]: return ext -def get_system_fonts() -> dict[str, Path]: - if sys.platform == "win32": - import winreg +def extract_font_family(font_path: Path) -> Optional[str]: + """ + Extract font family name from TTF/OTF file using fontTools. - with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as reg: - key = winreg.OpenKey(reg, r"SOFTWARE\Microsoft\Windows NT\CurrentVersion\Fonts", 0, winreg.KEY_READ) - total_fonts = winreg.QueryInfoKey(key)[1] - return { - name.replace(" (TrueType)", ""): Path(r"C:\Windows\Fonts", filename) - for n in range(0, total_fonts) - for name, filename, _ in [winreg.EnumValue(key, n)] - } - else: - # TODO: Get System Fonts for Linux and mac OS - return {} + Args: + font_path: Path to the font file + + Returns: + Font family name if successfully extracted, None otherwise + """ + try: + font = ttLib.TTFont(font_path) + name_table = font["name"] + + # Try to get family name (nameID 1) for Windows platform (platformID 3) + # This matches the naming convention used in Windows registry + for record in name_table.names: + if record.nameID == 1 and record.platformID == 3: + return record.toUnicode() + + # Fallback to other platforms if Windows name not found + for record in name_table.names: + if record.nameID == 1: + return record.toUnicode() + + except Exception: + # Silently ignore font parsing errors (corrupted fonts, etc.) + pass + + return None + + +def _get_windows_fonts() -> dict[str, Path]: + """ + Get fonts from Windows registry. + + Returns: + Dictionary mapping font family names to their file paths + """ + import winreg + + with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as reg: + key = winreg.OpenKey(reg, r"SOFTWARE\Microsoft\Windows NT\CurrentVersion\Fonts", 0, winreg.KEY_READ) + total_fonts = winreg.QueryInfoKey(key)[1] + return { + name.replace(" (TrueType)", ""): Path(r"C:\Windows\Fonts", filename) + for n in range(0, total_fonts) + for name, filename, _ in [winreg.EnumValue(key, n)] + } + + +def _scan_font_directory(font_dir: Path, fonts: dict[str, Path], log: logging.Logger) -> None: + """ + Scan a single directory for fonts. + + Args: + font_dir: Directory to scan + fonts: Dictionary to populate with found fonts + log: Logger instance for error reporting + """ + font_files = list(font_dir.rglob("*.ttf")) + list(font_dir.rglob("*.otf")) + + for font_file in font_files: + try: + if family_name := extract_font_family(font_file): + if family_name not in fonts: + fonts[family_name] = font_file + except Exception as e: + log.debug(f"Failed to process {font_file}: {e}") + + +def _get_unix_fonts() -> dict[str, Path]: + """ + Get fonts from Linux/macOS standard directories. + + Returns: + Dictionary mapping font family names to their file paths + """ + log = logging.getLogger("get_system_fonts") + fonts = {} + + font_dirs = [ + Path("/usr/share/fonts"), + Path("/usr/local/share/fonts"), + Path.home() / ".fonts", + Path.home() / ".local/share/fonts", + ] + + for font_dir in font_dirs: + if not font_dir.exists(): + continue + + try: + _scan_font_directory(font_dir, fonts, log) + except Exception as e: + log.warning(f"Failed to scan {font_dir}: {e}") + + log.debug(f"Discovered {len(fonts)} system font families") + return fonts + + +def get_system_fonts() -> dict[str, Path]: + """ + Get system fonts as a mapping of font family names to font file paths. + + On Windows: Uses registry to get font display names + On Linux/macOS: Scans standard font directories and extracts family names using fontTools + + Returns: + Dictionary mapping font family names to their file paths + """ + if sys.platform == "win32": + return _get_windows_fonts() + return _get_unix_fonts() + + +# Common Windows font names mapped to their Linux equivalents +# Ordered by preference (first match is used) +FONT_ALIASES = { + "Arial": ["Liberation Sans", "DejaVu Sans", "Nimbus Sans", "FreeSans"], + "Arial Black": ["Liberation Sans", "DejaVu Sans", "Nimbus Sans"], + "Arial Bold": ["Liberation Sans", "DejaVu Sans"], + "Arial Unicode MS": ["DejaVu Sans", "Noto Sans", "FreeSans"], + "Times New Roman": ["Liberation Serif", "DejaVu Serif", "Nimbus Roman", "FreeSerif"], + "Courier New": ["Liberation Mono", "DejaVu Sans Mono", "Nimbus Mono PS", "FreeMono"], + "Comic Sans MS": ["Comic Neue", "Comic Relief", "DejaVu Sans"], + "Georgia": ["Gelasio", "DejaVu Serif", "Liberation Serif"], + "Impact": ["Impact", "Anton", "Liberation Sans"], + "Trebuchet MS": ["Ubuntu", "DejaVu Sans", "Liberation Sans"], + "Verdana": ["DejaVu Sans", "Bitstream Vera Sans", "Liberation Sans"], + "Tahoma": ["DejaVu Sans", "Liberation Sans"], + "Adobe Arabic": ["Noto Sans Arabic", "DejaVu Sans"], + "Noto Sans Thai": ["Noto Sans Thai", "Noto Sans"], +} + + +def find_case_insensitive(font_name: str, fonts: dict[str, Path]) -> Optional[Path]: + """ + Find font by case-insensitive name match. + + Args: + font_name: Font family name to find + fonts: Dictionary of available fonts + + Returns: + Path to matched font, or None if not found + """ + font_lower = font_name.lower() + for name, path in fonts.items(): + if name.lower() == font_lower: + return path + return None + + +def find_font_with_fallbacks(font_name: str, system_fonts: dict[str, Path]) -> Optional[Path]: + """ + Find a font by name with intelligent fallback matching. + + Tries multiple strategies in order: + 1. Exact match (case-sensitive) + 2. Case-insensitive match + 3. Alias lookup (Windows → Linux font equivalents) + 4. Partial/prefix match + + Args: + font_name: The requested font family name (e.g., "Arial", "Times New Roman") + system_fonts: Dictionary of available fonts (family name → path) + + Returns: + Path to the matched font file, or None if no match found + """ + if not system_fonts: + return None + + # Strategy 1: Exact match (case-sensitive) + if font_name in system_fonts: + return system_fonts[font_name] + + # Strategy 2: Case-insensitive match + if result := find_case_insensitive(font_name, system_fonts): + return result + + # Strategy 3: Alias lookup + if font_name in FONT_ALIASES: + for alias in FONT_ALIASES[font_name]: + # Try exact match for alias + if alias in system_fonts: + return system_fonts[alias] + # Try case-insensitive match for alias + if result := find_case_insensitive(alias, system_fonts): + return result + + # Strategy 4: Partial/prefix match as last resort + font_name_lower = font_name.lower() + for name, path in system_fonts.items(): + if name.lower().startswith(font_name_lower): + return path + + return None + + +# Mapping of font families to system packages that provide them +FONT_PACKAGES = { + "liberation": { + "debian": "fonts-liberation fonts-liberation2", + "fonts": ["Liberation Sans", "Liberation Serif", "Liberation Mono"], + }, + "dejavu": { + "debian": "fonts-dejavu fonts-dejavu-core fonts-dejavu-extra", + "fonts": ["DejaVu Sans", "DejaVu Serif", "DejaVu Sans Mono"], + }, + "noto": { + "debian": "fonts-noto fonts-noto-core", + "fonts": ["Noto Sans", "Noto Serif", "Noto Sans Mono", "Noto Sans Arabic", "Noto Sans Thai"], + }, + "ubuntu": { + "debian": "fonts-ubuntu", + "fonts": ["Ubuntu", "Ubuntu Mono"], + }, +} + + +def suggest_font_packages(missing_fonts: list[str]) -> dict[str, list[str]]: + """ + Suggest system packages to install for missing fonts. + + Args: + missing_fonts: List of font family names that couldn't be found + + Returns: + Dictionary mapping package names to lists of fonts they would provide + """ + suggestions = {} + + # Check which fonts from aliases would help + needed_aliases = set() + for font in missing_fonts: + if font in FONT_ALIASES: + needed_aliases.update(FONT_ALIASES[font]) + + # Map needed aliases to packages + for package_name, package_info in FONT_PACKAGES.items(): + provided_fonts = package_info["fonts"] + matching_fonts = [f for f in provided_fonts if f in needed_aliases] + if matching_fonts: + suggestions[package_info["debian"]] = matching_fonts + + return suggestions class FPS(ast.NodeVisitor): diff --git a/uv.lock b/uv.lock index 8b2b799..8fe734f 100644 --- a/uv.lock +++ b/uv.lock @@ -502,6 +502,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" }, ] +[[package]] +name = "fonttools" +version = "4.60.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/42/97a13e47a1e51a5a7142475bbcf5107fe3a68fc34aef331c897d5fb98ad0/fonttools-4.60.1.tar.gz", hash = "sha256:ef00af0439ebfee806b25f24c8f92109157ff3fac5731dc7867957812e87b8d9", size = 3559823, upload-time = "2025-09-29T21:13:27.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/70/03e9d89a053caff6ae46053890eba8e4a5665a7c5638279ed4492e6d4b8b/fonttools-4.60.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:9a52f254ce051e196b8fe2af4634c2d2f02c981756c6464dc192f1b6050b4e28", size = 2810747, upload-time = "2025-09-29T21:10:59.653Z" }, + { url = "https://files.pythonhosted.org/packages/6f/41/449ad5aff9670ab0df0f61ee593906b67a36d7e0b4d0cd7fa41ac0325bf5/fonttools-4.60.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c7420a2696a44650120cdd269a5d2e56a477e2bfa9d95e86229059beb1c19e15", size = 2346909, upload-time = "2025-09-29T21:11:02.882Z" }, + { url = "https://files.pythonhosted.org/packages/9a/18/e5970aa96c8fad1cb19a9479cc3b7602c0c98d250fcdc06a5da994309c50/fonttools-4.60.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee0c0b3b35b34f782afc673d503167157094a16f442ace7c6c5e0ca80b08f50c", size = 4864572, upload-time = "2025-09-29T21:11:05.096Z" }, + { url = "https://files.pythonhosted.org/packages/ce/20/9b2b4051b6ec6689480787d506b5003f72648f50972a92d04527a456192c/fonttools-4.60.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:282dafa55f9659e8999110bd8ed422ebe1c8aecd0dc396550b038e6c9a08b8ea", size = 4794635, upload-time = "2025-09-29T21:11:08.651Z" }, + { url = "https://files.pythonhosted.org/packages/10/52/c791f57347c1be98f8345e3dca4ac483eb97666dd7c47f3059aeffab8b59/fonttools-4.60.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4ba4bd646e86de16160f0fb72e31c3b9b7d0721c3e5b26b9fa2fc931dfdb2652", size = 4843878, upload-time = "2025-09-29T21:11:10.893Z" }, + { url = "https://files.pythonhosted.org/packages/69/e9/35c24a8d01644cee8c090a22fad34d5b61d1e0a8ecbc9945ad785ebf2e9e/fonttools-4.60.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0b0835ed15dd5b40d726bb61c846a688f5b4ce2208ec68779bc81860adb5851a", size = 4954555, upload-time = "2025-09-29T21:11:13.24Z" }, + { url = "https://files.pythonhosted.org/packages/f7/86/fb1e994971be4bdfe3a307de6373ef69a9df83fb66e3faa9c8114893d4cc/fonttools-4.60.1-cp310-cp310-win32.whl", hash = "sha256:1525796c3ffe27bb6268ed2a1bb0dcf214d561dfaf04728abf01489eb5339dce", size = 2232019, upload-time = "2025-09-29T21:11:15.73Z" }, + { url = "https://files.pythonhosted.org/packages/40/84/62a19e2bd56f0e9fb347486a5b26376bade4bf6bbba64dda2c103bd08c94/fonttools-4.60.1-cp310-cp310-win_amd64.whl", hash = "sha256:268ecda8ca6cb5c4f044b1fb9b3b376e8cd1b361cef275082429dc4174907038", size = 2276803, upload-time = "2025-09-29T21:11:18.152Z" }, + { url = "https://files.pythonhosted.org/packages/ea/85/639aa9bface1537e0fb0f643690672dde0695a5bbbc90736bc571b0b1941/fonttools-4.60.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7b4c32e232a71f63a5d00259ca3d88345ce2a43295bb049d21061f338124246f", size = 2831872, upload-time = "2025-09-29T21:11:20.329Z" }, + { url = "https://files.pythonhosted.org/packages/6b/47/3c63158459c95093be9618794acb1067b3f4d30dcc5c3e8114b70e67a092/fonttools-4.60.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3630e86c484263eaac71d117085d509cbcf7b18f677906824e4bace598fb70d2", size = 2356990, upload-time = "2025-09-29T21:11:22.754Z" }, + { url = "https://files.pythonhosted.org/packages/94/dd/1934b537c86fcf99f9761823f1fc37a98fbd54568e8e613f29a90fed95a9/fonttools-4.60.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5c1015318e4fec75dd4943ad5f6a206d9727adf97410d58b7e32ab644a807914", size = 5042189, upload-time = "2025-09-29T21:11:25.061Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d2/9f4e4c4374dd1daa8367784e1bd910f18ba886db1d6b825b12edf6db3edc/fonttools-4.60.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e6c58beb17380f7c2ea181ea11e7db8c0ceb474c9dd45f48e71e2cb577d146a1", size = 4978683, upload-time = "2025-09-29T21:11:27.693Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c4/0fb2dfd1ecbe9a07954cc13414713ed1eab17b1c0214ef07fc93df234a47/fonttools-4.60.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ec3681a0cb34c255d76dd9d865a55f260164adb9fa02628415cdc2d43ee2c05d", size = 5021372, upload-time = "2025-09-29T21:11:30.257Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d5/495fc7ae2fab20223cc87179a8f50f40f9a6f821f271ba8301ae12bb580f/fonttools-4.60.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f4b5c37a5f40e4d733d3bbaaef082149bee5a5ea3156a785ff64d949bd1353fa", size = 5132562, upload-time = "2025-09-29T21:11:32.737Z" }, + { url = "https://files.pythonhosted.org/packages/bc/fa/021dab618526323c744e0206b3f5c8596a2e7ae9aa38db5948a131123e83/fonttools-4.60.1-cp311-cp311-win32.whl", hash = "sha256:398447f3d8c0c786cbf1209711e79080a40761eb44b27cdafffb48f52bcec258", size = 2230288, upload-time = "2025-09-29T21:11:35.015Z" }, + { url = "https://files.pythonhosted.org/packages/bb/78/0e1a6d22b427579ea5c8273e1c07def2f325b977faaf60bb7ddc01456cb1/fonttools-4.60.1-cp311-cp311-win_amd64.whl", hash = "sha256:d066ea419f719ed87bc2c99a4a4bfd77c2e5949cb724588b9dd58f3fd90b92bf", size = 2278184, upload-time = "2025-09-29T21:11:37.434Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f7/a10b101b7a6f8836a5adb47f2791f2075d044a6ca123f35985c42edc82d8/fonttools-4.60.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7b0c6d57ab00dae9529f3faf187f2254ea0aa1e04215cf2f1a8ec277c96661bc", size = 2832953, upload-time = "2025-09-29T21:11:39.616Z" }, + { url = "https://files.pythonhosted.org/packages/ed/fe/7bd094b59c926acf2304d2151354ddbeb74b94812f3dc943c231db09cb41/fonttools-4.60.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:839565cbf14645952d933853e8ade66a463684ed6ed6c9345d0faf1f0e868877", size = 2352706, upload-time = "2025-09-29T21:11:41.826Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ca/4bb48a26ed95a1e7eba175535fe5805887682140ee0a0d10a88e1de84208/fonttools-4.60.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8177ec9676ea6e1793c8a084a90b65a9f778771998eb919d05db6d4b1c0b114c", size = 4923716, upload-time = "2025-09-29T21:11:43.893Z" }, + { url = "https://files.pythonhosted.org/packages/b8/9f/2cb82999f686c1d1ddf06f6ae1a9117a880adbec113611cc9d22b2fdd465/fonttools-4.60.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:996a4d1834524adbb423385d5a629b868ef9d774670856c63c9a0408a3063401", size = 4968175, upload-time = "2025-09-29T21:11:46.439Z" }, + { url = "https://files.pythonhosted.org/packages/18/79/be569699e37d166b78e6218f2cde8c550204f2505038cdd83b42edc469b9/fonttools-4.60.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a46b2f450bc79e06ef3b6394f0c68660529ed51692606ad7f953fc2e448bc903", size = 4911031, upload-time = "2025-09-29T21:11:48.977Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9f/89411cc116effaec5260ad519162f64f9c150e5522a27cbb05eb62d0c05b/fonttools-4.60.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6ec722ee589e89a89f5b7574f5c45604030aa6ae24cb2c751e2707193b466fed", size = 5062966, upload-time = "2025-09-29T21:11:54.344Z" }, + { url = "https://files.pythonhosted.org/packages/62/a1/f888221934b5731d46cb9991c7a71f30cb1f97c0ef5fcf37f8da8fce6c8e/fonttools-4.60.1-cp312-cp312-win32.whl", hash = "sha256:b2cf105cee600d2de04ca3cfa1f74f1127f8455b71dbad02b9da6ec266e116d6", size = 2218750, upload-time = "2025-09-29T21:11:56.601Z" }, + { url = "https://files.pythonhosted.org/packages/88/8f/a55b5550cd33cd1028601df41acd057d4be20efa5c958f417b0c0613924d/fonttools-4.60.1-cp312-cp312-win_amd64.whl", hash = "sha256:992775c9fbe2cf794786fa0ffca7f09f564ba3499b8fe9f2f80bd7197db60383", size = 2267026, upload-time = "2025-09-29T21:11:58.852Z" }, + { url = "https://files.pythonhosted.org/packages/c7/93/0dd45cd283c32dea1545151d8c3637b4b8c53cdb3a625aeb2885b184d74d/fonttools-4.60.1-py3-none-any.whl", hash = "sha256:906306ac7afe2156fcf0042173d6ebbb05416af70f6b370967b47f8f00103bbb", size = 1143175, upload-time = "2025-09-29T21:13:24.134Z" }, +] + [[package]] name = "frozenlist" version = "1.7.0" @@ -1581,6 +1614,7 @@ dependencies = [ { name = "crccheck" }, { name = "cryptography" }, { name = "curl-cffi" }, + { name = "fonttools" }, { name = "httpx" }, { name = "jsonpickle" }, { name = "langcodes" }, @@ -1633,6 +1667,7 @@ requires-dist = [ { name = "crccheck", specifier = ">=1.3.0,<2" }, { name = "cryptography", specifier = ">=45.0.0" }, { name = "curl-cffi", specifier = ">=0.7.0b4,<0.8" }, + { name = "fonttools", specifier = ">=4.0.0,<5" }, { name = "httpx", specifier = ">=0.28.1,<0.29" }, { name = "jsonpickle", specifier = ">=3.0.4,<4" }, { name = "langcodes", specifier = ">=3.4.0,<4" }, @@ -1641,7 +1676,7 @@ requires-dist = [ { name = "protobuf", specifier = ">=4.25.3,<5" }, { name = "pycaption", specifier = ">=2.2.6,<3" }, { name = "pycryptodomex", specifier = ">=3.20.0,<4" }, - { name = "pyexecjs", specifier = ">=1.5.1" }, + { name = "pyexecjs", specifier = ">=1.5.1,<2" }, { name = "pyjwt", specifier = ">=2.8.0,<3" }, { name = "pymediainfo", specifier = ">=6.1.0,<7" }, { name = "pymp4", specifier = ">=1.4.0,<2" }, From 8a46655d21b14f765eab84c397b0b37f30f13c00 Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 3 Nov 2025 23:01:31 +0000 Subject: [PATCH 46/62] feat(subtitle): preserve original formatting when no conversion requested Add preserve_formatting config option to prevent automatic subtitle processing that strips formatting tags and styling. When enabled (default: true), WebVTT files skip pycaption read/write cycle to preserve tags like , , positioning, and other formatting. --- unshackle/core/tracks/subtitle.py | 28 ++++++++++++++++------------ unshackle/unshackle-example.yaml | 4 ++++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/unshackle/core/tracks/subtitle.py b/unshackle/core/tracks/subtitle.py index 7019142..e807bff 100644 --- a/unshackle/core/tracks/subtitle.py +++ b/unshackle/core/tracks/subtitle.py @@ -239,25 +239,29 @@ class Subtitle(Track): # Sanitize WebVTT timestamps before parsing text = Subtitle.sanitize_webvtt_timestamps(text) + preserve_formatting = config.subtitle.get("preserve_formatting", True) - try: - caption_set = pycaption.WebVTTReader().read(text) - Subtitle.merge_same_cues(caption_set) - Subtitle.filter_unwanted_cues(caption_set) - subtitle_text = pycaption.WebVTTWriter().write(caption_set) - self.path.write_text(subtitle_text, encoding="utf8") - except pycaption.exceptions.CaptionReadSyntaxError: - # If first attempt fails, try more aggressive sanitization - text = Subtitle.sanitize_webvtt(text) + if preserve_formatting: + self.path.write_text(text, encoding="utf8") + else: try: caption_set = pycaption.WebVTTReader().read(text) Subtitle.merge_same_cues(caption_set) Subtitle.filter_unwanted_cues(caption_set) subtitle_text = pycaption.WebVTTWriter().write(caption_set) self.path.write_text(subtitle_text, encoding="utf8") - except Exception: - # Keep the sanitized version even if parsing failed - self.path.write_text(text, encoding="utf8") + except pycaption.exceptions.CaptionReadSyntaxError: + # If first attempt fails, try more aggressive sanitization + text = Subtitle.sanitize_webvtt(text) + try: + caption_set = pycaption.WebVTTReader().read(text) + Subtitle.merge_same_cues(caption_set) + Subtitle.filter_unwanted_cues(caption_set) + subtitle_text = pycaption.WebVTTWriter().write(caption_set) + self.path.write_text(subtitle_text, encoding="utf8") + except Exception: + # Keep the sanitized version even if parsing failed + self.path.write_text(text, encoding="utf8") @staticmethod def sanitize_webvtt_timestamps(text: str) -> str: diff --git a/unshackle/unshackle-example.yaml b/unshackle/unshackle-example.yaml index e5e1b68..36e7c2c 100644 --- a/unshackle/unshackle-example.yaml +++ b/unshackle/unshackle-example.yaml @@ -360,6 +360,10 @@ subtitle: # convert_before_strip: Auto-convert VTT/other formats to SRT before using subtitle-filter # This ensures compatibility when subtitle-filter is used as fallback (default: true) convert_before_strip: true + # preserve_formatting: Preserve original subtitle formatting (tags, positioning, styling) + # When true, skips pycaption processing for WebVTT files to keep tags like , , positioning intact + # Combined with no sub_format setting, ensures subtitles remain in their original format (default: true) + preserve_formatting: true # Configuration for pywidevine's serve functionality serve: From 0c3a6c47f2f1799d994d9cc02b2474fa0da8efb5 Mon Sep 17 00:00:00 2001 From: Andy Date: Thu, 6 Nov 2025 07:05:44 +0000 Subject: [PATCH 47/62] fix(dl): prevent vault loading when --cdm-only flag is set The --cdm-only flag was only preventing vault queries during DRM operations but vaults were still being loaded --- unshackle/commands/dl.py | 93 +++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 40 deletions(-) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 0a0e23d..ec7bf25 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -553,53 +553,66 @@ class dl: f"Applied service-specific '{config_key}' overrides for {self.service}: {override_value}" ) - with console.status("Loading Key Vaults...", spinner="dots"): + cdm_only = ctx.params.get("cdm_only") + + if cdm_only: self.vaults = Vaults(self.service) - total_vaults = len(config.key_vaults) - failed_vaults = [] + self.log.info("CDM-only mode: Skipping vault loading") + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="vault_loading_skipped", + service=self.service, + context={"reason": "cdm_only flag set"}, + ) + else: + with console.status("Loading Key Vaults...", spinner="dots"): + self.vaults = Vaults(self.service) + total_vaults = len(config.key_vaults) + failed_vaults = [] - for vault in config.key_vaults: - vault_type = vault["type"] - vault_name = vault.get("name", vault_type) - vault_copy = vault.copy() - del vault_copy["type"] + for vault in config.key_vaults: + vault_type = vault["type"] + vault_name = vault.get("name", vault_type) + vault_copy = vault.copy() + del vault_copy["type"] - if vault_type.lower() == "api" and "decrypt_labs" in vault_name.lower(): - if "token" not in vault_copy or not vault_copy["token"]: - if config.decrypt_labs_api_key: - vault_copy["token"] = config.decrypt_labs_api_key - else: - self.log.warning( - f"No token provided for DecryptLabs vault '{vault_name}' and no global " - "decrypt_labs_api_key configured" - ) + if vault_type.lower() == "api" and "decrypt_labs" in vault_name.lower(): + if "token" not in vault_copy or not vault_copy["token"]: + if config.decrypt_labs_api_key: + vault_copy["token"] = config.decrypt_labs_api_key + else: + self.log.warning( + f"No token provided for DecryptLabs vault '{vault_name}' and no global " + "decrypt_labs_api_key configured" + ) - if vault_type.lower() == "sqlite": - try: - self.vaults.load_critical(vault_type, **vault_copy) - self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})") - except Exception as e: - self.log.error(f"vault failure: {vault_name} ({vault_type}) - {e}") - raise - else: - # Other vaults (MySQL, HTTP, API) - soft fail - if not self.vaults.load(vault_type, **vault_copy): - failed_vaults.append(vault_name) - self.log.debug(f"Failed to load vault: {vault_name} ({vault_type})") + if vault_type.lower() == "sqlite": + try: + self.vaults.load_critical(vault_type, **vault_copy) + self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})") + except Exception as e: + self.log.error(f"vault failure: {vault_name} ({vault_type}) - {e}") + raise else: - self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})") + # Other vaults (MySQL, HTTP, API) - soft fail + if not self.vaults.load(vault_type, **vault_copy): + failed_vaults.append(vault_name) + self.log.debug(f"Failed to load vault: {vault_name} ({vault_type})") + else: + self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})") - loaded_count = len(self.vaults) - if failed_vaults: - self.log.warning(f"Failed to load {len(failed_vaults)} vault(s): {', '.join(failed_vaults)}") - self.log.info(f"Loaded {loaded_count}/{total_vaults} Vaults") + loaded_count = len(self.vaults) + if failed_vaults: + self.log.warning(f"Failed to load {len(failed_vaults)} vault(s): {', '.join(failed_vaults)}") + self.log.info(f"Loaded {loaded_count}/{total_vaults} Vaults") - # Debug: Show detailed vault status - if loaded_count > 0: - vault_names = [vault.name for vault in self.vaults] - self.log.debug(f"Active vaults: {', '.join(vault_names)}") - else: - self.log.debug("No vaults are currently active") + # Debug: Show detailed vault status + if loaded_count > 0: + vault_names = [vault.name for vault in self.vaults] + self.log.debug(f"Active vaults: {', '.join(vault_names)}") + else: + self.log.debug("No vaults are currently active") with console.status("Loading DRM CDM...", spinner="dots"): try: From cc7263884f442210ae4c3881d28f758f9a4597ff Mon Sep 17 00:00:00 2001 From: Andy Date: Sat, 8 Nov 2025 03:00:19 +0000 Subject: [PATCH 48/62] fix(cdm): resolve session key handling for partial cached keys When decrypt-labs returns cached keys that don't cover all required KIDs, the CDM now properly stores them in session["cached_keys"] instead of session["keys"]. This allows parse_license() to correctly combine vault_keys + cached_keys + license_keys, fixing downloads that previously failed when mixing cached and fresh licenses. --- unshackle/core/cdm/decrypt_labs_remote_cdm.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/unshackle/core/cdm/decrypt_labs_remote_cdm.py b/unshackle/core/cdm/decrypt_labs_remote_cdm.py index 8645b4e..3806107 100644 --- a/unshackle/core/cdm/decrypt_labs_remote_cdm.py +++ b/unshackle/core/cdm/decrypt_labs_remote_cdm.py @@ -474,7 +474,6 @@ class DecryptLabsRemoteCDM: if "vault_keys" in session: all_available_keys.extend(session["vault_keys"]) - session["keys"] = all_available_keys session["tried_cache"] = True if self._required_kids: @@ -505,10 +504,7 @@ class DecryptLabsRemoteCDM: license_request_data = request_data.copy() license_request_data["get_cached_keys_if_exists"] = False - session["decrypt_labs_session_id"] = None - session["challenge"] = None - session["tried_cache"] = False - + # Make license request for missing keys response = self._http_session.post( f"{self.host}/get-request", json=license_request_data, timeout=30 ) @@ -522,8 +518,12 @@ class DecryptLabsRemoteCDM: return b"" else: + # All required keys are available from cache + session["keys"] = all_available_keys return b"" else: + # No required KIDs specified - return cached keys + session["keys"] = all_available_keys return b"" if message_type == "license-request" or "challenge" in data: @@ -572,7 +572,9 @@ class DecryptLabsRemoteCDM: session = self._sessions[session_id] - if session["keys"] and not (self.is_playready and "cached_keys" in session): + # Skip parsing if we already have final keys (no cached keys to combine) + # If cached_keys exist (Widevine or PlayReady), we need to combine them with license keys + if session["keys"] and "cached_keys" not in session: return if not session.get("challenge") or not session.get("decrypt_labs_session_id"): From 11bcca9632a1990e098ad41d13190dfabf3a79ed Mon Sep 17 00:00:00 2001 From: Andy Date: Sat, 8 Nov 2025 03:02:17 +0000 Subject: [PATCH 49/62] fix(cdm): apply session key fix to custom_remote_cdm Apply the same partial cached keys fix from decrypt_labs_remote_cdm to custom_remote_cdm. When cached keys don't cover all required KIDs, store them in session["cached_keys"] instead of session["keys"] to allow parse_license() to properly combine vault_keys + cached_keys + license_keys. --- unshackle/core/cdm/custom_remote_cdm.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/unshackle/core/cdm/custom_remote_cdm.py b/unshackle/core/cdm/custom_remote_cdm.py index 5ae6a17..cd4c559 100644 --- a/unshackle/core/cdm/custom_remote_cdm.py +++ b/unshackle/core/cdm/custom_remote_cdm.py @@ -891,7 +891,6 @@ class CustomRemoteCDM: if "vault_keys" in session: all_available_keys.extend(session["vault_keys"]) - session["keys"] = all_available_keys session["tried_cache"] = True # Check if we have all required keys @@ -904,11 +903,18 @@ class CustomRemoteCDM: required_kids = set(self._required_kids) missing_kids = required_kids - available_kids - if not missing_kids: + if missing_kids: + # Store cached keys separately - don't populate session["keys"] yet + # This allows parse_license() to properly combine cached + license keys + session["cached_keys"] = cached_keys + else: + # All required keys are available from cache + session["keys"] = all_available_keys return b"" - - # Store cached keys for later combination - session["cached_keys"] = cached_keys + else: + # No required KIDs specified - return cached keys + session["keys"] = all_available_keys + return b"" # Handle license request response or fetch license if keys missing challenge = parsed_response.get("challenge") @@ -952,8 +958,9 @@ class CustomRemoteCDM: session = self._sessions[session_id] - # If we already have keys and no cached keys to combine, skip - if session["keys"] and not session.get("cached_keys"): + # Skip parsing if we already have final keys (no cached keys to combine) + # If cached_keys exist (Widevine or PlayReady), we need to combine them with license keys + if session["keys"] and "cached_keys" not in session: return # Ensure we have a challenge and session ID From 90e4030a8811f1051402e4c10fcfd4b489bf83c7 Mon Sep 17 00:00:00 2001 From: Andy Date: Sat, 8 Nov 2025 06:04:37 +0000 Subject: [PATCH 50/62] fix(n_m3u8dl_re): read lang attribute from DASH manifests correctly The track_selection function was using findall() to search for lang child elements, but in DASH manifests lang is an XML attribute on AdaptationSet. This caused language selection to fail for region-specific codes like es-419. --- unshackle/core/downloaders/n_m3u8dl_re.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/unshackle/core/downloaders/n_m3u8dl_re.py b/unshackle/core/downloaders/n_m3u8dl_re.py index d183111..110a977 100644 --- a/unshackle/core/downloaders/n_m3u8dl_re.py +++ b/unshackle/core/downloaders/n_m3u8dl_re.py @@ -57,7 +57,11 @@ def track_selection(track: object) -> list[str]: if track_type == "Audio": codecs = AUDIO_CODEC_MAP.get(codec) - langs = adaptation_set.findall("lang") + representation.findall("lang") + langs = [] + if adaptation_set.get("lang"): + langs.append(adaptation_set.get("lang")) + if representation is not None and representation.get("lang"): + langs.append(representation.get("lang")) track_ids = list( set( v From 9ed5133c4c8d57ee1880956b1a87b093a0b64a6a Mon Sep 17 00:00:00 2001 From: stabbedbybrick <125766685+stabbedbybrick@users.noreply.github.com> Date: Sat, 8 Nov 2025 21:57:52 +0100 Subject: [PATCH 51/62] N_m3u8DL-RE: Improve track selection, add download arguments and option to load manifest from file (#38) * feat: Add 'from_file', 'downloader_args' to Track * feat: Add loading HLS playlist from file * refactor: Improve track selection, args for n_m3u8dl_re --- unshackle/core/downloaders/n_m3u8dl_re.py | 501 +++++++++++++--------- unshackle/core/manifests/hls.py | 21 +- unshackle/core/tracks/track.py | 22 +- 3 files changed, 319 insertions(+), 225 deletions(-) diff --git a/unshackle/core/downloaders/n_m3u8dl_re.py b/unshackle/core/downloaders/n_m3u8dl_re.py index 110a977..52ea3be 100644 --- a/unshackle/core/downloaders/n_m3u8dl_re.py +++ b/unshackle/core/downloaders/n_m3u8dl_re.py @@ -1,12 +1,10 @@ -import logging import os import re import subprocess import warnings from http.cookiejar import CookieJar -from itertools import chain from pathlib import Path -from typing import Any, Generator, MutableMapping, Optional, Union +from typing import Any, Generator, MutableMapping import requests from requests.cookies import cookiejar_from_dict, get_cookie_header @@ -16,255 +14,325 @@ from unshackle.core.config import config from unshackle.core.console import console from unshackle.core.constants import DOWNLOAD_CANCELLED +PERCENT_RE = re.compile(r"(\d+\.\d+%)") +SPEED_RE = re.compile(r"(\d+\.\d+(?:MB|KB)ps)") +SIZE_RE = re.compile(r"(\d+\.\d+(?:MB|GB|KB)/\d+\.\d+(?:MB|GB|KB))") +WARN_RE = re.compile(r"(WARN : Response.*|WARN : One or more errors occurred.*)") +ERROR_RE = re.compile(r"(ERROR.*)") + +DECRYPTION_ENGINE = { + "shaka": "SHAKA_PACKAGER", + "mp4decrypt": "MP4DECRYPT", +} + # Ignore FutureWarnings warnings.simplefilter(action="ignore", category=FutureWarning) -AUDIO_CODEC_MAP = {"AAC": "mp4a", "AC3": "ac-3", "EC3": "ec-3"} -VIDEO_CODEC_MAP = {"AVC": "avc", "HEVC": "hvc", "DV": "dvh", "HLG": "hev"} + +def get_track_selection_args(track: Any) -> list[str]: + """ + Generates track selection arguments for N_m3u8dl_RE. + + Args: + track: A track object with attributes like descriptor, data, and class name. + + Returns: + A list of strings for track selection. + + Raises: + ValueError: If the manifest type is unsupported or track selection fails. + """ + descriptor = track.descriptor.name + track_type = track.__class__.__name__ + + def _create_args(flag: str, parts: list[str], type_str: str, extra_args: list[str] | None = None) -> list[str]: + if not parts: + raise ValueError(f"[N_m3u8DL-RE]: Unable to select {type_str} track from {descriptor} manifest") + + final_args = [flag, ":".join(parts)] + if extra_args: + final_args.extend(extra_args) + + return final_args + + match descriptor: + case "HLS": + # HLS playlists are direct inputs; no selection arguments needed. + return [] + + case "DASH": + representation = track.data.get("dash", {}).get("representation", {}) + adaptation_set = track.data.get("dash", {}).get("adaptation_set", {}) + parts = [] + + if track_type == "Audio": + if track_id := representation.get("id") or adaptation_set.get("audioTrackId"): + parts.append(rf'"id=\b{track_id}\b"') + else: + if codecs := representation.get("codecs"): + parts.append(f"codecs={codecs}") + if lang := representation.get("lang") or adaptation_set.get("lang"): + parts.append(f"lang={lang}") + if bw := representation.get("bandwidth"): + bitrate = int(bw) // 1000 + parts.append(f"bwMin={bitrate}:bwMax={bitrate + 5}") + if roles := representation.findall("Role") + adaptation_set.findall("Role"): + if role := next((r.get("value") for r in roles if r.get("value", "").lower() == "main"), None): + parts.append(f"role={role}") + return _create_args("-sa", parts, "audio") + + if track_type == "Video": + if track_id := representation.get("id"): + parts.append(rf'"id=\b{track_id}\b"') + else: + if width := representation.get("width"): + parts.append(f"res={width}*") + if codecs := representation.get("codecs"): + parts.append(f"codecs={codecs}") + if bw := representation.get("bandwidth"): + bitrate = int(bw) // 1000 + parts.append(f"bwMin={bitrate}:bwMax={bitrate + 5}") + return _create_args("-sv", parts, "video") + + if track_type == "Subtitle": + if track_id := representation.get("id"): + parts.append(rf'"id=\b{track_id}\b"') + else: + if lang := representation.get("lang"): + parts.append(f"lang={lang}") + return _create_args("-ss", parts, "subtitle", extra_args=["--auto-subtitle-fix", "false"]) + + case "ISM": + quality_level = track.data.get("ism", {}).get("quality_level", {}) + stream_index = track.data.get("ism", {}).get("stream_index", {}) + parts = [] + + if track_type == "Audio": + if name := stream_index.get("Name") or quality_level.get("Index"): + parts.append(rf'"id=\b{name}\b"') + else: + if codecs := quality_level.get("FourCC"): + parts.append(f"codecs={codecs}") + if lang := stream_index.get("Language"): + parts.append(f"lang={lang}") + if br := quality_level.get("Bitrate"): + bitrate = int(br) // 1000 + parts.append(f"bwMin={bitrate}:bwMax={bitrate + 5}") + return _create_args("-sa", parts, "audio") + + if track_type == "Video": + if name := stream_index.get("Name") or quality_level.get("Index"): + parts.append(rf'"id=\b{name}\b"') + else: + if width := quality_level.get("MaxWidth"): + parts.append(f"res={width}*") + if codecs := quality_level.get("FourCC"): + parts.append(f"codecs={codecs}") + if br := quality_level.get("Bitrate"): + bitrate = int(br) // 1000 + parts.append(f"bwMin={bitrate}:bwMax={bitrate + 5}") + return _create_args("-sv", parts, "video") + + # I've yet to encounter a subtitle track in ISM manifests, so this is mostly theoretical. + if track_type == "Subtitle": + if name := stream_index.get("Name") or quality_level.get("Index"): + parts.append(rf'"id=\b{name}\b"') + else: + if lang := stream_index.get("Language"): + parts.append(f"lang={lang}") + return _create_args("-ss", parts, "subtitle", extra_args=["--auto-subtitle-fix", "false"]) + + raise ValueError(f"[N_m3u8DL-RE]: Unsupported manifest type: {descriptor}") -def track_selection(track: object) -> list[str]: - """Return the N_m3u8DL-RE stream selection arguments for a track.""" +def build_download_args( + track_url: str, + filename: str, + output_dir: Path, + thread_count: int, + retry_count: int, + track_from_file: Path | None, + custom_args: dict[str, Any] | None, + headers: dict[str, Any] | None, + cookies: CookieJar | None, + proxy: str | None, + content_keys: dict[str, str] | None, + ad_keyword: str | None, + skip_merge: bool | None = False, +) -> list[str]: + """Constructs the CLI arguments for N_m3u8DL-RE.""" - if "dash" in track.data: - adaptation_set = track.data["dash"]["adaptation_set"] - representation = track.data["dash"]["representation"] + # Default arguments + args = { + "--save-name": filename, + "--save-dir": output_dir, + "--tmp-dir": output_dir, + "--thread-count": thread_count, + "--download-retry-count": retry_count, + "--write-meta-json": False, + "--no-log": True, + } + if proxy: + args["--custom-proxy"] = proxy + if skip_merge: + args["--skip-merge"] = skip_merge + if ad_keyword: + args["--ad-keyword"] = ad_keyword + if content_keys: + args["--key"] = next((f"{kid.hex}:{key.lower()}" for kid, key in content_keys.items()), None) + args["--decryption-engine"] = DECRYPTION_ENGINE.get(config.decryption.lower()) or "SHAKA_PACKAGER" + if custom_args: + args.update(custom_args) - track_type = track.__class__.__name__ - codec = track.codec.name - bitrate = track.bitrate // 1000 - language = track.language - width = track.width if track_type == "Video" else None - height = track.height if track_type == "Video" else None - range = track.range.name if track_type == "Video" else None + command = [track_from_file or track_url] + for flag, value in args.items(): + if value is True: + command.append(flag) + elif value is False: + command.extend([flag, "false"]) + elif value is not False and value is not None: + command.extend([flag, str(value)]) - elif "ism" in track.data: - stream_index = track.data["ism"]["stream_index"] - quality_level = track.data["ism"]["quality_level"] + if headers: + for key, value in headers.items(): + if key.lower() not in ("accept-encoding", "cookie"): + command.extend(["--header", f"{key}: {value}"]) - track_type = track.__class__.__name__ - codec = track.codec.name - bitrate = track.bitrate // 1000 - language = track.language - width = track.width if track_type == "Video" else None - height = track.height if track_type == "Video" else None - range = track.range.name if track_type == "Video" else None - adaptation_set = stream_index - representation = quality_level + if cookies: + req = requests.Request(method="GET", url=track_url) + cookie_header = get_cookie_header(cookies, req) + command.extend(["--header", f"Cookie: {cookie_header}"]) - else: - return [] - - if track_type == "Audio": - codecs = AUDIO_CODEC_MAP.get(codec) - langs = [] - if adaptation_set.get("lang"): - langs.append(adaptation_set.get("lang")) - if representation is not None and representation.get("lang"): - langs.append(representation.get("lang")) - track_ids = list( - set( - v - for x in chain(adaptation_set, representation) - for v in (x.get("audioTrackId"), x.get("id")) - if v is not None - ) - ) - roles = adaptation_set.findall("Role") + representation.findall("Role") - role = ":role=main" if next((i for i in roles if i.get("value").lower() == "main"), None) else "" - bandwidth = f"bwMin={bitrate}:bwMax={bitrate + 5}" - - if langs: - track_selection = ["-sa", f"lang={language}:codecs={codecs}:{bandwidth}{role}"] - elif len(track_ids) == 1: - track_selection = ["-sa", f"id={track_ids[0]}"] - else: - track_selection = ["-sa", f"for=best{role}"] - return track_selection - - if track_type == "Video": - # adjust codec based on range - codec_adjustments = {("HEVC", "DV"): "DV", ("HEVC", "HLG"): "HLG"} - codec = codec_adjustments.get((codec, range), codec) - codecs = VIDEO_CODEC_MAP.get(codec) - - bandwidth = f"bwMin={bitrate}:bwMax={bitrate + 5}" - if width and height: - resolution = f"{width}x{height}" - elif width: - resolution = f"{width}*" - else: - resolution = "for=best" - if resolution.startswith("for="): - track_selection = ["-sv", resolution] - track_selection.append(f"codecs={codecs}:{bandwidth}") - else: - track_selection = ["-sv", f"res={resolution}:codecs={codecs}:{bandwidth}"] - return track_selection + return command def download( - urls: Union[str, dict[str, Any], list[str], list[dict[str, Any]]], - track: object, + urls: str | dict[str, Any] | list[str | dict[str, Any]], + track: Any, output_dir: Path, filename: str, - headers: Optional[MutableMapping[str, Union[str, bytes]]] = None, - cookies: Optional[Union[MutableMapping[str, str], CookieJar]] = None, - proxy: Optional[str] = None, - max_workers: Optional[int] = None, - content_keys: Optional[dict[str, Any]] = None, + headers: MutableMapping[str, str | bytes] | None, + cookies: MutableMapping[str, str] | CookieJar | None, + proxy: str | None, + max_workers: int | None, + content_keys: dict[str, Any] | None, + skip_merge: bool | None = False, ) -> Generator[dict[str, Any], None, None]: if not urls: raise ValueError("urls must be provided and not empty") - elif not isinstance(urls, (str, dict, list)): - raise TypeError(f"Expected urls to be {str} or {dict} or a list of one of them, not {type(urls)}") - - if not output_dir: - raise ValueError("output_dir must be provided") - elif not isinstance(output_dir, Path): - raise TypeError(f"Expected output_dir to be {Path}, not {type(output_dir)}") - - if not filename: - raise ValueError("filename must be provided") - elif not isinstance(filename, str): - raise TypeError(f"Expected filename to be {str}, not {type(filename)}") - + if not isinstance(urls, (str, dict, list)): + raise TypeError(f"Expected urls to be str, dict, or list, not {type(urls)}") + if not isinstance(output_dir, Path): + raise TypeError(f"Expected output_dir to be Path, not {type(output_dir)}") + if not isinstance(filename, str) or not filename: + raise ValueError("filename must be a non-empty string") if not isinstance(headers, (MutableMapping, type(None))): - raise TypeError(f"Expected headers to be {MutableMapping}, not {type(headers)}") - + raise TypeError(f"Expected headers to be a mapping or None, not {type(headers)}") if not isinstance(cookies, (MutableMapping, CookieJar, type(None))): - raise TypeError(f"Expected cookies to be {MutableMapping} or {CookieJar}, not {type(cookies)}") - + raise TypeError(f"Expected cookies to be a mapping, CookieJar, or None, not {type(cookies)}") if not isinstance(proxy, (str, type(None))): - raise TypeError(f"Expected proxy to be {str}, not {type(proxy)}") - - if not max_workers: - max_workers = min(32, (os.cpu_count() or 1) + 4) - elif not isinstance(max_workers, int): - raise TypeError(f"Expected max_workers to be {int}, not {type(max_workers)}") - - if not isinstance(urls, list): - urls = [urls] - - if not binaries.N_m3u8DL_RE: - raise EnvironmentError("N_m3u8DL-RE executable not found...") + raise TypeError(f"Expected proxy to be a str or None, not {type(proxy)}") + if not isinstance(max_workers, (int, type(None))): + raise TypeError(f"Expected max_workers to be an int or None, not {type(max_workers)}") + if not isinstance(content_keys, (dict, type(None))): + raise TypeError(f"Expected content_keys to be a dict or None, not {type(content_keys)}") + if not isinstance(skip_merge, (bool, type(None))): + raise TypeError(f"Expected skip_merge to be a bool or None, not {type(skip_merge)}") if cookies and not isinstance(cookies, CookieJar): cookies = cookiejar_from_dict(cookies) - track_type = track.__class__.__name__ - thread_count = str(config.n_m3u8dl_re.get("thread_count", max_workers)) - retry_count = str(config.n_m3u8dl_re.get("retry_count", max_workers)) + if not binaries.N_m3u8DL_RE: + raise EnvironmentError("N_m3u8DL-RE executable not found...") + + effective_max_workers = max_workers or min(32, (os.cpu_count() or 1) + 4) + + if proxy and not config.n_m3u8dl_re.get("use_proxy", True): + proxy = None + + thread_count = config.n_m3u8dl_re.get("thread_count", effective_max_workers) + retry_count = config.n_m3u8dl_re.get("retry_count", 10) ad_keyword = config.n_m3u8dl_re.get("ad_keyword") - arguments = [ - track.url, - "--save-dir", - output_dir, - "--tmp-dir", - output_dir, - "--thread-count", - thread_count, - "--download-retry-count", - retry_count, - "--no-log", - "--write-meta-json", - "false", - ] + arguments = build_download_args( + track_url=track.url, + track_from_file=track.from_file, + filename=filename, + output_dir=output_dir, + thread_count=thread_count, + retry_count=retry_count, + custom_args=track.downloader_args, + headers=headers, + cookies=cookies, + proxy=proxy, + content_keys=content_keys, + skip_merge=skip_merge, + ad_keyword=ad_keyword, + ) + arguments.extend(get_track_selection_args(track)) - for header, value in (headers or {}).items(): - if header.lower() in ("accept-encoding", "cookie"): - continue - arguments.extend(["--header", f"{header}: {value}"]) - - if cookies: - cookie_header = get_cookie_header(cookies, requests.Request(url=track.url)) - if cookie_header: - arguments.extend(["--header", f"Cookie: {cookie_header}"]) - - if proxy: - arguments.extend(["--custom-proxy", proxy]) - - if content_keys: - for kid, key in content_keys.items(): - keys = f"{kid.hex}:{key.lower()}" - arguments.extend(["--key", keys]) - arguments.extend(["--use-shaka-packager"]) - - if ad_keyword: - arguments.extend(["--ad-keyword", ad_keyword]) - - if track.descriptor.name == "URL": - error = f"[N_m3u8DL-RE]: {track.descriptor} is currently not supported" - raise ValueError(error) - elif track.descriptor.name == "DASH": - arguments.extend(track_selection(track)) - - # TODO: improve this nonsense - percent_re = re.compile(r"(\d+\.\d+%)") - speed_re = re.compile(r"(? Generator[dict[str, Any], None, None]: """ Download files using N_m3u8DL-RE. @@ -279,28 +347,33 @@ def n_m3u8dl_re( The data is in the same format accepted by rich's progress.update() function. Parameters: - urls: Web URL(s) to file(s) to download. You can use a dictionary with the key - "url" for the URI, and other keys for extra arguments to use per-URL. + urls: Web URL(s) to file(s) to download. NOTE: This parameter is ignored for now. track: The track to download. Used to get track attributes for the selection process. Note that Track.Descriptor.URL is not supported by N_m3u8DL-RE. output_dir: The folder to save the file into. If the save path's directory does not exist then it will be made automatically. - filename: The filename or filename template to use for each file. The variables - you can use are `i` for the URL index and `ext` for the URL extension. - headers: A mapping of HTTP Header Key/Values to use for the download. - cookies: A mapping of Cookie Key/Values or a Cookie Jar to use for the download. + filename: The filename or filename template to use for each file. + headers: A mapping of HTTP Header Key/Values to use for all downloads. + cookies: A mapping of Cookie Key/Values or a Cookie Jar to use for all downloads. + proxy: A proxy to use for all downloads. max_workers: The maximum amount of threads to use for downloads. Defaults to min(32,(cpu_count+4)). Can be set in config with --thread-count option. content_keys: The content keys to use for decryption. + skip_merge: Whether to skip merging the downloaded chunks. """ - track_type = track.__class__.__name__ - log = logging.getLogger("N_m3u8DL-RE") - if proxy and not config.n_m3u8dl_re.get("use_proxy", True): - log.warning(f"{track_type}: Ignoring proxy as N_m3u8DL-RE is set to use_proxy=False") - proxy = None - - yield from download(urls, track, output_dir, filename, headers, cookies, proxy, max_workers, content_keys) + yield from download( + urls=urls, + track=track, + output_dir=output_dir, + filename=filename, + headers=headers, + cookies=cookies, + proxy=proxy, + max_workers=max_workers, + content_keys=content_keys, + skip_merge=skip_merge, + ) __all__ = ("n_m3u8dl_re",) diff --git a/unshackle/core/manifests/hls.py b/unshackle/core/manifests/hls.py index 0bb1c9b..46ceaa4 100644 --- a/unshackle/core/manifests/hls.py +++ b/unshackle/core/manifests/hls.py @@ -249,17 +249,20 @@ class HLS: log = logging.getLogger("HLS") - # Get the playlist text and handle both session types - response = session.get(track.url) - if isinstance(response, requests.Response): - if not response.ok: - log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}") - sys.exit(1) - playlist_text = response.text + if track.from_file: + master = m3u8.load(str(track.from_file)) else: - raise TypeError(f"Expected response to be a requests.Response or curl_cffi.Response, not {type(response)}") + # Get the playlist text and handle both session types + response = session.get(track.url) + if isinstance(response, requests.Response): + if not response.ok: + log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}") + sys.exit(1) + playlist_text = response.text + else: + raise TypeError(f"Expected response to be a requests.Response or curl_cffi.Response, not {type(response)}") - master = m3u8.loads(playlist_text, uri=track.url) + master = m3u8.loads(playlist_text, uri=track.url) if not master.segments: log.error("Track's HLS playlist has no segments, expecting an invariant M3U8 playlist.") diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 12c7af0..72b8e60 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -25,7 +25,7 @@ from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY from unshackle.core.downloaders import aria2c, curl_impersonate, n_m3u8dl_re, requests from unshackle.core.drm import DRM_T, PlayReady, Widevine from unshackle.core.events import events -from unshackle.core.utilities import get_boxes, try_ensure_utf8 +from unshackle.core.utilities import get_boxes, try_ensure_utf8, get_extension from unshackle.core.utils.subprocess import ffprobe @@ -47,6 +47,8 @@ class Track: drm: Optional[Iterable[DRM_T]] = None, edition: Optional[str] = None, downloader: Optional[Callable] = None, + downloader_args: Optional[dict] = None, + from_file: Optional[Path] = None, data: Optional[Union[dict, defaultdict]] = None, id_: Optional[str] = None, extra: Optional[Any] = None, @@ -69,6 +71,10 @@ class Track: raise TypeError(f"Expected edition to be a {str}, not {type(edition)}") if not isinstance(downloader, (Callable, type(None))): raise TypeError(f"Expected downloader to be a {Callable}, not {type(downloader)}") + if not isinstance(downloader_args, (dict, type(None))): + raise TypeError(f"Expected downloader_args to be a {dict}, not {type(downloader_args)}") + if not isinstance(from_file, (Path, type(None))): + raise TypeError(f"Expected from_file to be a {Path}, not {type(from_file)}") if not isinstance(data, (dict, defaultdict, type(None))): raise TypeError(f"Expected data to be a {dict} or {defaultdict}, not {type(data)}") @@ -100,6 +106,8 @@ class Track: self.drm = drm self.edition: str = edition self.downloader = downloader + self.downloader_args = downloader_args + self.from_file = from_file self._data: defaultdict[Any, Any] = defaultdict(dict) self.data = data or {} self.extra: Any = extra or {} # allow anything for extra, but default to a dict @@ -203,7 +211,17 @@ class Track: save_path = config.directories.temp / f"{track_type}_{self.id}.mp4" if track_type == "Subtitle": save_path = save_path.with_suffix(f".{self.codec.extension}") - if self.downloader.__name__ == "n_m3u8dl_re": + # n_m3u8dl_re doesn't support directly downloading subtitles + if self.downloader.__name__ == "n_m3u8dl_re" and get_extension(self.url) in { + ".srt", + ".vtt", + ".ttml", + ".ssa", + ".ass", + ".stpp", + ".wvtt", + ".xml", + }: self.downloader = requests if self.descriptor != self.Descriptor.URL: From 5d20bf9d528d695fbc7e90070336052de0bb63d6 Mon Sep 17 00:00:00 2001 From: Andy Date: Sat, 8 Nov 2025 22:49:23 +0000 Subject: [PATCH 52/62] fix(subtitles): fix closure bug preventing SDH subtitle stripping Fixed a Python late binding closure issue in the SDH subtitle duplication logic that prevented strip_hearing_impaired() from being called correctly. --- unshackle/commands/dl.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index ec7bf25..aa8f4ba 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -135,7 +135,7 @@ class dl: return temp_path - def _attach_subtitle_fonts( + def attach_subtitle_fonts( self, font_names: list[str], title: Title_T, @@ -153,7 +153,6 @@ class dl: Tuple of (fonts_attached_count, missing_fonts_list) """ system_fonts = get_system_fonts() - self.log.info(f"Discovered {len(system_fonts)} system font families") font_count = 0 missing_fonts = [] @@ -176,7 +175,7 @@ class dl: return font_count, missing_fonts - def _suggest_missing_fonts(self, missing_fonts: list[str]) -> None: + def suggest_missing_fonts(self, missing_fonts: list[str]) -> None: """ Show package installation suggestions for missing fonts. @@ -1053,7 +1052,7 @@ class dl: title.tracks.add(non_sdh_sub) events.subscribe( events.Types.TRACK_MULTIPLEX, - lambda track: (track.strip_hearing_impaired()) if track.id == non_sdh_sub.id else None, + lambda track, sub_id=non_sdh_sub.id: (track.strip_hearing_impaired()) if track.id == sub_id else None, ) with console.status("Sorting tracks by language and bitrate...", spinner="dots"): @@ -1558,7 +1557,7 @@ class dl: if line.startswith("Style: "): font_names.append(line.removesuffix("Style: ").split(",")[1]) - font_count, missing_fonts = self._attach_subtitle_fonts( + font_count, missing_fonts = self.attach_subtitle_fonts( font_names, title, temp_font_files ) @@ -1566,7 +1565,7 @@ class dl: self.log.info(f"Attached {font_count} fonts for the Subtitles") if missing_fonts and sys.platform != "win32": - self._suggest_missing_fonts(missing_fonts) + self.suggest_missing_fonts(missing_fonts) # Handle DRM decryption BEFORE repacking (must decrypt first!) service_name = service.__class__.__name__.upper() From 55db8da1256988310ce1eb6caf19b7c09841b3fb Mon Sep 17 00:00:00 2001 From: Andy Date: Sat, 8 Nov 2025 22:53:47 +0000 Subject: [PATCH 53/62] refactor: remove unnecessary underscore prefixes from function names --- unshackle/commands/kv.py | 16 ++++++++-------- unshackle/core/cacher.py | 14 +++++++------- unshackle/core/session.py | 4 ++-- unshackle/core/tracks/track.py | 2 +- unshackle/core/update_checker.py | 16 ++++++++-------- unshackle/core/utilities.py | 32 +++++++++++++++----------------- unshackle/core/utils/webvtt.py | 22 +++++++++++++++++++++- 7 files changed, 62 insertions(+), 44 deletions(-) diff --git a/unshackle/commands/kv.py b/unshackle/commands/kv.py index 035f7f7..28c870d 100644 --- a/unshackle/commands/kv.py +++ b/unshackle/commands/kv.py @@ -12,7 +12,7 @@ from unshackle.core.vault import Vault from unshackle.core.vaults import Vaults -def _load_vaults(vault_names: list[str]) -> Vaults: +def load_vaults(vault_names: list[str]) -> Vaults: """Load and validate vaults by name.""" vaults = Vaults() for vault_name in vault_names: @@ -30,7 +30,7 @@ def _load_vaults(vault_names: list[str]) -> Vaults: return vaults -def _process_service_keys(from_vault: Vault, service: str, log: logging.Logger) -> dict[str, str]: +def process_service_keys(from_vault: Vault, service: str, log: logging.Logger) -> dict[str, str]: """Get and validate keys from a vault for a specific service.""" content_keys = list(from_vault.get_keys(service)) @@ -41,9 +41,9 @@ def _process_service_keys(from_vault: Vault, service: str, log: logging.Logger) return {kid: key for kid, key in content_keys if kid not in bad_keys} -def _copy_service_data(to_vault: Vault, from_vault: Vault, service: str, log: logging.Logger) -> int: +def copy_service_data(to_vault: Vault, from_vault: Vault, service: str, log: logging.Logger) -> int: """Copy data for a single service between vaults.""" - content_keys = _process_service_keys(from_vault, service, log) + content_keys = process_service_keys(from_vault, service, log) total_count = len(content_keys) if total_count == 0: @@ -95,7 +95,7 @@ def copy(to_vault_name: str, from_vault_names: list[str], service: Optional[str] log = logging.getLogger("kv") all_vault_names = [to_vault_name] + list(from_vault_names) - vaults = _load_vaults(all_vault_names) + vaults = load_vaults(all_vault_names) to_vault = vaults.vaults[0] from_vaults = vaults.vaults[1:] @@ -112,7 +112,7 @@ def copy(to_vault_name: str, from_vault_names: list[str], service: Optional[str] services_to_copy = [service] if service else from_vault.get_services() for service_tag in services_to_copy: - added = _copy_service_data(to_vault, from_vault, service_tag, log) + added = copy_service_data(to_vault, from_vault, service_tag, log) total_added += added if total_added > 0: @@ -164,7 +164,7 @@ def add(file: Path, service: str, vaults: list[str]) -> None: log = logging.getLogger("kv") service = Services.get_tag(service) - vaults_ = _load_vaults(list(vaults)) + vaults_ = load_vaults(list(vaults)) data = file.read_text(encoding="utf8") kid_keys: dict[str, str] = {} @@ -194,7 +194,7 @@ def prepare(vaults: list[str]) -> None: """Create Service Tables on Vaults if not yet created.""" log = logging.getLogger("kv") - vaults_ = _load_vaults(vaults) + vaults_ = load_vaults(vaults) for vault in vaults_: if hasattr(vault, "has_table") and hasattr(vault, "create_table"): diff --git a/unshackle/core/cacher.py b/unshackle/core/cacher.py index ba0c6a8..28cee47 100644 --- a/unshackle/core/cacher.py +++ b/unshackle/core/cacher.py @@ -91,7 +91,7 @@ class Cacher: except jwt.DecodeError: pass - self.expiration = self._resolve_datetime(expiration) if expiration else None + self.expiration = self.resolve_datetime(expiration) if expiration else None payload = {"data": self.data, "expiration": self.expiration, "version": self.version} payload["crc32"] = zlib.crc32(jsonpickle.dumps(payload).encode("utf8")) @@ -109,7 +109,7 @@ class Cacher: return self.path.stat() @staticmethod - def _resolve_datetime(timestamp: EXP_T) -> datetime: + def resolve_datetime(timestamp: EXP_T) -> datetime: """ Resolve multiple formats of a Datetime or Timestamp to an absolute Datetime. @@ -118,15 +118,15 @@ class Cacher: datetime.datetime(2022, 6, 27, 9, 49, 13, 657208) >>> iso8601 = now.isoformat() '2022-06-27T09:49:13.657208' - >>> Cacher._resolve_datetime(iso8601) + >>> Cacher.resolve_datetime(iso8601) datetime.datetime(2022, 6, 27, 9, 49, 13, 657208) - >>> Cacher._resolve_datetime(iso8601 + "Z") + >>> Cacher.resolve_datetime(iso8601 + "Z") datetime.datetime(2022, 6, 27, 9, 49, 13, 657208) - >>> Cacher._resolve_datetime(3600) + >>> Cacher.resolve_datetime(3600) datetime.datetime(2022, 6, 27, 10, 52, 50, 657208) - >>> Cacher._resolve_datetime('3600') + >>> Cacher.resolve_datetime('3600') datetime.datetime(2022, 6, 27, 10, 52, 51, 657208) - >>> Cacher._resolve_datetime(7800.113) + >>> Cacher.resolve_datetime(7800.113) datetime.datetime(2022, 6, 27, 11, 59, 13, 770208) In the int/float examples you may notice that it did not return now + 3600 seconds diff --git a/unshackle/core/session.py b/unshackle/core/session.py index dd5dc5b..3a4f704 100644 --- a/unshackle/core/session.py +++ b/unshackle/core/session.py @@ -79,7 +79,7 @@ class CurlSession(Session): ) self.log = logging.getLogger(self.__class__.__name__) - def _get_sleep_time(self, response: Response | None, attempt: int) -> float | None: + def get_sleep_time(self, response: Response | None, attempt: int) -> float | None: if response: retry_after = response.headers.get("Retry-After") if retry_after: @@ -123,7 +123,7 @@ class CurlSession(Session): ) if attempt < self.max_retries: - if sleep_duration := self._get_sleep_time(response, attempt + 1): + if sleep_duration := self.get_sleep_time(response, attempt + 1): if sleep_duration > 0: time.sleep(sleep_duration) else: diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 72b8e60..6cf12c9 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -25,7 +25,7 @@ from unshackle.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY from unshackle.core.downloaders import aria2c, curl_impersonate, n_m3u8dl_re, requests from unshackle.core.drm import DRM_T, PlayReady, Widevine from unshackle.core.events import events -from unshackle.core.utilities import get_boxes, try_ensure_utf8, get_extension +from unshackle.core.utilities import get_boxes, get_extension, try_ensure_utf8 from unshackle.core.utils.subprocess import ffprobe diff --git a/unshackle/core/update_checker.py b/unshackle/core/update_checker.py index 5ca6502..8d601fc 100644 --- a/unshackle/core/update_checker.py +++ b/unshackle/core/update_checker.py @@ -28,21 +28,21 @@ class UpdateChecker: DEFAULT_CHECK_INTERVAL = 24 * 60 * 60 @classmethod - def _get_cache_file(cls) -> Path: + def get_cache_file(cls) -> Path: """Get the path to the update check cache file.""" from unshackle.core.config import config return config.directories.cache / "update_check.json" @classmethod - def _load_cache_data(cls) -> dict: + def load_cache_data(cls) -> dict: """ Load cache data from file. Returns: Cache data dictionary or empty dict if loading fails """ - cache_file = cls._get_cache_file() + cache_file = cls.get_cache_file() if not cache_file.exists(): return {} @@ -54,7 +54,7 @@ class UpdateChecker: return {} @staticmethod - def _parse_version(version_string: str) -> str: + def parse_version(version_string: str) -> str: """ Parse and normalize version string by removing 'v' prefix. @@ -107,7 +107,7 @@ class UpdateChecker: return None data = response.json() - latest_version = cls._parse_version(data.get("tag_name", "")) + latest_version = cls.parse_version(data.get("tag_name", "")) return latest_version if cls._is_valid_version(latest_version) else None @@ -125,7 +125,7 @@ class UpdateChecker: Returns: True if we should check for updates, False otherwise """ - cache_data = cls._load_cache_data() + cache_data = cls.load_cache_data() if not cache_data: return True @@ -144,7 +144,7 @@ class UpdateChecker: latest_version: The latest version found, if any current_version: The current version being used """ - cache_file = cls._get_cache_file() + cache_file = cls.get_cache_file() try: cache_file.parent.mkdir(parents=True, exist_ok=True) @@ -231,7 +231,7 @@ class UpdateChecker: Returns: The latest version string if an update is available from cache, None otherwise """ - cache_data = cls._load_cache_data() + cache_data = cls.load_cache_data() if not cache_data: return None diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index a09f41c..dae0dc6 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -485,7 +485,7 @@ def extract_font_family(font_path: Path) -> Optional[str]: return None -def _get_windows_fonts() -> dict[str, Path]: +def get_windows_fonts() -> dict[str, Path]: """ Get fonts from Windows registry. @@ -504,7 +504,7 @@ def _get_windows_fonts() -> dict[str, Path]: } -def _scan_font_directory(font_dir: Path, fonts: dict[str, Path], log: logging.Logger) -> None: +def scan_font_directory(font_dir: Path, fonts: dict[str, Path], log: logging.Logger) -> None: """ Scan a single directory for fonts. @@ -524,7 +524,7 @@ def _scan_font_directory(font_dir: Path, fonts: dict[str, Path], log: logging.Lo log.debug(f"Failed to process {font_file}: {e}") -def _get_unix_fonts() -> dict[str, Path]: +def get_unix_fonts() -> dict[str, Path]: """ Get fonts from Linux/macOS standard directories. @@ -546,11 +546,9 @@ def _get_unix_fonts() -> dict[str, Path]: continue try: - _scan_font_directory(font_dir, fonts, log) + scan_font_directory(font_dir, fonts, log) except Exception as e: log.warning(f"Failed to scan {font_dir}: {e}") - - log.debug(f"Discovered {len(fonts)} system font families") return fonts @@ -565,8 +563,8 @@ def get_system_fonts() -> dict[str, Path]: Dictionary mapping font family names to their file paths """ if sys.platform == "win32": - return _get_windows_fonts() - return _get_unix_fonts() + return get_windows_fonts() + return get_unix_fonts() # Common Windows font names mapped to their Linux equivalents @@ -754,9 +752,9 @@ class DebugLogger: if self.enabled: self.log_path.parent.mkdir(parents=True, exist_ok=True) self.file_handle = open(self.log_path, "a", encoding="utf-8") - self._log_session_start() + self.log_session_start() - def _log_session_start(self): + def log_session_start(self): """Log the start of a new session with environment information.""" import platform @@ -821,11 +819,11 @@ class DebugLogger: if service: entry["service"] = service if context: - entry["context"] = self._sanitize_data(context) + entry["context"] = self.sanitize_data(context) if request: - entry["request"] = self._sanitize_data(request) + entry["request"] = self.sanitize_data(request) if response: - entry["response"] = self._sanitize_data(response) + entry["response"] = self.sanitize_data(response) if duration_ms is not None: entry["duration_ms"] = duration_ms if success is not None: @@ -840,7 +838,7 @@ class DebugLogger: for key, value in kwargs.items(): if key not in entry: - entry[key] = self._sanitize_data(value) + entry[key] = self.sanitize_data(value) try: self.file_handle.write(json.dumps(entry, default=str) + "\n") @@ -848,7 +846,7 @@ class DebugLogger: except Exception as e: print(f"Failed to write debug log: {e}", file=sys.stderr) - def _sanitize_data(self, data: Any) -> Any: + def sanitize_data(self, data: Any) -> Any: """ Sanitize data for JSON serialization. Handles complex objects and removes sensitive information. @@ -860,7 +858,7 @@ class DebugLogger: return data if isinstance(data, (list, tuple)): - return [self._sanitize_data(item) for item in data] + return [self.sanitize_data(item) for item in data] if isinstance(data, dict): sanitized = {} @@ -883,7 +881,7 @@ class DebugLogger: if should_redact: sanitized[key] = "[REDACTED]" else: - sanitized[key] = self._sanitize_data(value) + sanitized[key] = self.sanitize_data(value) return sanitized if isinstance(data, bytes): diff --git a/unshackle/core/utils/webvtt.py b/unshackle/core/utils/webvtt.py index 76a8a36..9379fc6 100644 --- a/unshackle/core/utils/webvtt.py +++ b/unshackle/core/utils/webvtt.py @@ -3,8 +3,11 @@ import sys import typing from typing import Optional +import pysubs2 from pycaption import Caption, CaptionList, CaptionNode, CaptionReadError, WebVTTReader, WebVTTWriter +from unshackle.core.config import config + class CaptionListExt(CaptionList): @typing.no_type_check @@ -142,7 +145,24 @@ def merge_segmented_webvtt(vtt_raw: str, segment_durations: Optional[list[int]] """ MPEG_TIMESCALE = 90_000 - vtt = WebVTTReaderExt().read(vtt_raw) + # Check config for conversion method preference + conversion_method = config.subtitle.get("conversion_method", "auto") + use_pysubs2 = conversion_method in ("pysubs2", "auto") + + if use_pysubs2: + # Try using pysubs2 first for more lenient parsing + try: + # Use pysubs2 to parse and normalize the VTT + subs = pysubs2.SSAFile.from_string(vtt_raw) + # Convert back to WebVTT string for pycaption processing + normalized_vtt = subs.to_string("vtt") + vtt = WebVTTReaderExt().read(normalized_vtt) + except Exception: + # Fall back to direct pycaption parsing + vtt = WebVTTReaderExt().read(vtt_raw) + else: + # Use pycaption directly + vtt = WebVTTReaderExt().read(vtt_raw) for lang in vtt.get_languages(): prev_caption = None duplicate_index: list[int] = [] From 59d35da6d05e6168f14f76a3a800d506b02a4d86 Mon Sep 17 00:00:00 2001 From: Andy Date: Sun, 9 Nov 2025 17:47:51 +0000 Subject: [PATCH 54/62] chore(deps): update requests to >=2.32.5 --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index be4c2db..cf22736 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dependencies = [ "pymysql>=1.1.0,<2", "pywidevine[serve]>=1.8.0,<2", "PyYAML>=6.0.1,<7", - "requests[socks]>=2.31.0,<3", + "requests[socks]>=2.32.5,<3", "rich>=13.7.1,<14", "rlaphoenix.m3u8>=3.4.0,<4", "ruamel.yaml>=0.18.6,<0.19", diff --git a/uv.lock b/uv.lock index 8fe734f..3928ea5 100644 --- a/uv.lock +++ b/uv.lock @@ -1316,7 +1316,7 @@ wheels = [ [[package]] name = "requests" -version = "2.32.4" +version = "2.32.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -1324,9 +1324,9 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" }, + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] [package.optional-dependencies] @@ -1685,7 +1685,7 @@ requires-dist = [ { name = "pysubs2", specifier = ">=1.7.0,<2" }, { name = "pywidevine", extras = ["serve"], specifier = ">=1.8.0,<2" }, { name = "pyyaml", specifier = ">=6.0.1,<7" }, - { name = "requests", extras = ["socks"], specifier = ">=2.31.0,<3" }, + { name = "requests", extras = ["socks"], specifier = ">=2.32.5,<3" }, { name = "rich", specifier = ">=13.7.1,<14" }, { name = "rlaphoenix-m3u8", specifier = ">=3.4.0,<4" }, { name = "ruamel-yaml", specifier = ">=0.18.6,<0.19" }, From 87ff66f8fe8c8f42980fbf758641e380282870c8 Mon Sep 17 00:00:00 2001 From: Andy Date: Sun, 9 Nov 2025 21:27:19 +0000 Subject: [PATCH 55/62] fix: ensure subtitles use requests downloader instead of n_m3u8dl_re if Descriptor.URL PR #38 refactored n_m3u8dl_re track selection to support DASH/ISM subtitle tracks, but this broke some subtitle downloads. Services that use direct URL downloads (Descriptor.URL) for subtitles, which n_m3u8dl_re does not support. --- unshackle/core/downloaders/n_m3u8dl_re.py | 6 ++++++ unshackle/core/tracks/track.py | 26 +++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/unshackle/core/downloaders/n_m3u8dl_re.py b/unshackle/core/downloaders/n_m3u8dl_re.py index 52ea3be..7472c59 100644 --- a/unshackle/core/downloaders/n_m3u8dl_re.py +++ b/unshackle/core/downloaders/n_m3u8dl_re.py @@ -142,6 +142,12 @@ def get_track_selection_args(track: Any) -> list[str]: parts.append(f"lang={lang}") return _create_args("-ss", parts, "subtitle", extra_args=["--auto-subtitle-fix", "false"]) + case "URL": + raise ValueError( + f"[N_m3u8DL-RE]: Direct URL downloads are not supported for {track_type} tracks. " + f"The track should use a different downloader (e.g., 'requests', 'aria2c')." + ) + raise ValueError(f"[N_m3u8DL-RE]: Unsupported manifest type: {descriptor}") diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 6cf12c9..0b1a38f 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -211,17 +211,21 @@ class Track: save_path = config.directories.temp / f"{track_type}_{self.id}.mp4" if track_type == "Subtitle": save_path = save_path.with_suffix(f".{self.codec.extension}") - # n_m3u8dl_re doesn't support directly downloading subtitles - if self.downloader.__name__ == "n_m3u8dl_re" and get_extension(self.url) in { - ".srt", - ".vtt", - ".ttml", - ".ssa", - ".ass", - ".stpp", - ".wvtt", - ".xml", - }: + # n_m3u8dl_re doesn't support directly downloading subtitles from URLs + # or when the subtitle has a direct file extension + if self.downloader.__name__ == "n_m3u8dl_re" and ( + self.descriptor == self.Descriptor.URL + or get_extension(self.url) in { + ".srt", + ".vtt", + ".ttml", + ".ssa", + ".ass", + ".stpp", + ".wvtt", + ".xml", + } + ): self.downloader = requests if self.descriptor != self.Descriptor.URL: From eef06fb9865691145cb76dee3159dc41d500b167 Mon Sep 17 00:00:00 2001 From: Andy Date: Sun, 9 Nov 2025 23:19:12 +0000 Subject: [PATCH 56/62] fix: suppress verbose fontTools logging when scanning system fonts --- unshackle/core/utilities.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index dae0dc6..4892c24 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -463,8 +463,18 @@ def extract_font_family(font_path: Path) -> Optional[str]: Returns: Font family name if successfully extracted, None otherwise """ + # Suppress verbose fontTools logging during font table parsing + import io + + logging.getLogger("fontTools").setLevel(logging.ERROR) + logging.getLogger("fontTools.ttLib").setLevel(logging.ERROR) + logging.getLogger("fontTools.ttLib.tables").setLevel(logging.ERROR) + logging.getLogger("fontTools.ttLib.tables._n_a_m_e").setLevel(logging.ERROR) + stderr_backup = sys.stderr + sys.stderr = io.StringIO() + try: - font = ttLib.TTFont(font_path) + font = ttLib.TTFont(font_path, lazy=True) name_table = font["name"] # Try to get family name (nameID 1) for Windows platform (platformID 3) @@ -479,8 +489,9 @@ def extract_font_family(font_path: Path) -> Optional[str]: return record.toUnicode() except Exception: - # Silently ignore font parsing errors (corrupted fonts, etc.) pass + finally: + sys.stderr = stderr_backup return None From 240c70a2aa03138612822eb81336bdcb9708b778 Mon Sep 17 00:00:00 2001 From: Andy Date: Sun, 9 Nov 2025 23:30:33 +0000 Subject: [PATCH 57/62] fix(tags): skip metadata lookup when API keys not configured --- unshackle/core/utils/tags.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/unshackle/core/utils/tags.py b/unshackle/core/utils/tags.py index 82a8e95..98d7980 100644 --- a/unshackle/core/utils/tags.py +++ b/unshackle/core/utils/tags.py @@ -486,7 +486,7 @@ def external_ids( return {} -def _apply_tags(path: Path, tags: dict[str, str]) -> None: +def apply_tags(path: Path, tags: dict[str, str]) -> None: if not tags: return if not binaries.Mkvpropedit: @@ -537,7 +537,7 @@ def tag_file(path: Path, title: Title, tmdb_id: Optional[int] | None = None) -> name = title.title year = title.year else: - _apply_tags(path, custom_tags) + apply_tags(path, custom_tags) return if config.tag_imdb_tmdb: @@ -547,6 +547,8 @@ def tag_file(path: Path, title: Title, tmdb_id: Optional[int] | None = None) -> if not api_key and not simkl_client: log.debug("No TMDB API key or Simkl client ID configured; skipping IMDB/TMDB tag lookup") + apply_tags(path, custom_tags) + return else: # If tmdb_id is provided (via --tmdb), skip Simkl and use TMDB directly if tmdb_id is not None: @@ -637,7 +639,7 @@ def tag_file(path: Path, title: Title, tmdb_id: Optional[int] | None = None) -> **custom_tags, **standard_tags, } - _apply_tags(path, merged_tags) + apply_tags(path, merged_tags) __all__ = [ From 1ebb62ee91bcb5c863caa41fd244ea71a3298c45 Mon Sep 17 00:00:00 2001 From: Andy Date: Sun, 9 Nov 2025 23:46:31 +0000 Subject: [PATCH 58/62] refactor(tags): remove environment variable fallbacks for API keys --- unshackle/core/utils/tags.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/unshackle/core/utils/tags.py b/unshackle/core/utils/tags.py index 98d7980..5fad48c 100644 --- a/unshackle/core/utils/tags.py +++ b/unshackle/core/utils/tags.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -import os import re import subprocess import tempfile @@ -44,11 +43,11 @@ def _get_session() -> requests.Session: def _api_key() -> Optional[str]: - return config.tmdb_api_key or os.getenv("TMDB_API_KEY") + return config.tmdb_api_key def _simkl_client_id() -> Optional[str]: - return config.simkl_client_id or os.getenv("SIMKL_CLIENT_ID") + return config.simkl_client_id def _clean(s: str) -> str: @@ -474,9 +473,7 @@ def external_ids( try: detail_response = _fetch_tmdb_detail(tmdb_id, kind) if detail_response: - title_cacher.cache_tmdb( - cache_title_id, detail_response, js, kind, cache_region, cache_account_hash - ) + title_cacher.cache_tmdb(cache_title_id, detail_response, js, kind, cache_region, cache_account_hash) except Exception as exc: log.debug("Failed to cache TMDB data: %s", exc) From 9488a40f51685c0c63aecc63f9bdf7e62d838813 Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 10 Nov 2025 22:12:15 +0000 Subject: [PATCH 59/62] feat(dl): add --no-video flag to skip video track downloads Add new -nv/--no-video CLI flag that allows users to download audio, subtitles, attachments, and chapters without downloading video tracks. Fixes #39 --- unshackle/commands/dl.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index aa8f4ba..1830805 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -346,6 +346,7 @@ class dl: @click.option("-ns", "--no-subs", is_flag=True, default=False, help="Do not download subtitle tracks.") @click.option("-na", "--no-audio", is_flag=True, default=False, help="Do not download audio tracks.") @click.option("-nc", "--no-chapters", is_flag=True, default=False, help="Do not download chapters tracks.") + @click.option("-nv", "--no-video", is_flag=True, default=False, help="Do not download video tracks.") @click.option("-ad", "--audio-description", is_flag=True, default=False, help="Download audio description tracks.") @click.option( "--slow", @@ -740,6 +741,7 @@ class dl: no_subs: bool, no_audio: bool, no_chapters: bool, + no_video: bool, audio_description: bool, slow: bool, list_: bool, @@ -982,6 +984,11 @@ class dl: s_lang = None title.tracks.subtitles = [] + if no_video: + console.log("Skipped video as --no-video was used...") + v_lang = None + title.tracks.videos = [] + with console.status("Getting tracks...", spinner="dots"): try: title.tracks.add(service.get_tracks(title), warn_only=True) @@ -1322,7 +1329,7 @@ class dl: self.log.error(f"There's no {processed_lang} Audio Track, cannot continue...") sys.exit(1) - if video_only or audio_only or subs_only or chapters_only or no_subs or no_audio or no_chapters: + if video_only or audio_only or subs_only or chapters_only or no_subs or no_audio or no_chapters or no_video: keep_videos = False keep_audio = False keep_subtitles = False @@ -1349,6 +1356,8 @@ class dl: keep_audio = False if no_chapters: keep_chapters = False + if no_video: + keep_videos = False kept_tracks = [] if keep_videos: @@ -1505,6 +1514,7 @@ class dl: and not no_subs and not (hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES) and not video_only + and not no_video and len(title.tracks.videos) > video_track_n and any( x.get("codec_name", "").startswith("eia_") From 7883ff56c64983ff0183ac6a7ede6120d8402d5f Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 10 Nov 2025 22:17:20 +0000 Subject: [PATCH 60/62] docs(changelog): add --no-video flag and PR #38 credit --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 006f6b6..9483819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `-le, --latest-episode` flag to download only the most recent episode - Automatically selects the single most recent episode regardless of season - Fixes GitHub issue #28 +- **Video Track Exclusion**: New `--no-video` CLI option + - `-nv, --no-video` flag to skip downloading video tracks + - Allows downloading only audio, subtitles, attachments, and chapters + - Useful for audio-only or subtitle extraction workflows + - Fixes GitHub issue #39 - **Service-Specific Configuration Overrides**: Per-service fine-tuned control - Support for per-service configuration overrides in YAML - Fine-tuned control of downloader and command options per service @@ -149,6 +154,7 @@ This release includes contributions from: - @Sp5rky - REST API server implementation, dependency updates - @stabbedbybrick - curl_cffi retry handler (PR #31) +- @stabbedbybrick - n_m3u8dl-re refactor (PR #38) - @TPD94 - Binary search enhancements, manifest parser fixes (PR #19) - @scene (Andy) - Core features, configuration system, bug fixes From c1e7fcab019f6b15fdbbad80e194dfb3309d7cf4 Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 10 Nov 2025 22:29:44 +0000 Subject: [PATCH 61/62] docs(changelog): set release date for version 2.0.0 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9483819..59d0d67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [2.0.0] - Unreleased +## [2.0.0] - 2025-11-10 ### Breaking Changes From 76d73355f78c0dc326734606c4a7e78d90a722be Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 10 Nov 2025 22:31:15 +0000 Subject: [PATCH 62/62] docs(readme): remove dev branch warning for main merge --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index a5bd916..a9b78c4 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,6 @@

-> [!WARNING] -> **Development Branch**: This is the `dev` branch containing bleeding-edge features and experimental changes. Use for testing only. For stable releases, use the [`main`](https://github.com/unshackle-dl/unshackle/tree/main) branch. - ## What is unshackle? unshackle is a fork of [Devine](https://github.com/devine-dl/devine/), a powerful archival tool for downloading movies, TV shows, and music from streaming services. Built with a focus on modularity and extensibility, it provides a robust framework for content acquisition with support for DRM-protected content.