From cbcc42d831ef82fb178fe84dac0ab3f0a5f72381 Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Tue, 22 Sep 2026 21:39:33 -0400 Subject: [PATCH] Extract podcast metadata from tags and sidecar files --- .gitea/workflows/ci.yml | 20 +++ .gitea/workflows/release.yml | 7 + MANIFEST.in | 4 +- README.md | 38 +++++ mopidy_webhooks/__init__.py | 2 + mopidy_webhooks/ext.conf | 2 + mopidy_webhooks/frontend.py | 130 ++++++++++++-- mopidy_webhooks/metadata.py | 309 +++++++++++++++++++++++++++++++++ setup.cfg | 4 +- tests/data/silence.mp3 | Bin 0 -> 2411 bytes tests/test_extension.py | 11 +- tests/test_frontend.py | 321 +++++++++++++++++++++-------------- tests/test_metadata.py | 186 ++++++++++++++++++++ tox.ini | 4 +- 14 files changed, 884 insertions(+), 154 deletions(-) create mode 100644 .gitea/workflows/ci.yml create mode 100644 mopidy_webhooks/metadata.py create mode 100644 tests/data/silence.mp3 create mode 100644 tests/test_metadata.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..8c918b3 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,20 @@ +name: CI + +on: + push: + branches: ["**"] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + container: ghcr.io/mopidy/ci:7 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install package and test dependencies + run: python -m pip install -e ".[test]" + + - name: Run tests + run: python -m pytest diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 32c2174..c673d70 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -30,6 +30,13 @@ jobs: - name: Build sdist and wheel run: python -m build + - name: Build release zip + run: | + set -euo pipefail + VERSION="${{ gitea.ref_name }}" + cd dist + zip "Mopidy-Webhooks-${VERSION}.zip" *.tar.gz + - name: Publish to Gitea release env: GITEA_TOKEN: ${{ secrets.GITEA }} diff --git a/MANIFEST.in b/MANIFEST.in index 9d9fd9b..d778be6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,11 +1,13 @@ include *.py -include *.rst +include *.md include .mailmap include LICENSE include MANIFEST.in include pyproject.toml +include setup.cfg include tox.ini +recursive-include .gitea * recursive-include .github * include mopidy_*/ext.conf diff --git a/README.md b/README.md index 7ff7ebd..3818d48 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ them with a comma. Tokens match the listing of urls:: enabled = true urls = https://example.com/api/receiver/,http://127.0.0.1:8000/webhook/ tokens = ,2349080989234089 + media_dirs = /var/lib/mopidy/media + podcast_dirs = /var/lib/mopidy/media/podcasts The following configuration values are available: @@ -31,6 +33,42 @@ The following configuration values are available: - ``webhooks/urls``: Comma-separated list of endpoints to send play data to - ``webhooks/tokens``: Comma-separated list of tokens to send in the Authorization header for each URL +- ``webhooks/media_dirs``: Comma-separated list of local media roots used to + resolve ``local:track:``/``local:podcast:`` URIs to files so their tags can + be read. Defaults to ``/var/lib/mopidy/media``. +- ``webhooks/podcast_dirs``: Comma-separated list of directories that should + always be treated as podcasts, even when the URI does not mention it. + + +Podcast metadata +================ + +When a track is detected as a podcast episode, the webhook payload is enriched +with metadata read from the audio file itself, so the receiver no longer has to +guess the podcast, episode number or publication date from the file name. + +Sources, in order of preference: + +1. Audio tags (ID3, Vorbis, MP4) read with `mutagen + `_: title, artist, album, track number, + date, genre, comment/description, publisher, language, copyright and + website. +2. A sidecar JSON file written next to the episode (``.json``) or in + a ``metadata/`` directory (``metadata/.json``), as produced by + common podcatchers. This provides ``feed_url``, ``guid``, ``episode_url``, + ``duration``, ``image`` and a fallback ``description``. + +Podcast episodes are detected when the URI contains ``podcast``, when the file +lives under one of the ``webhooks/podcast_dirs``, or when the file has a +sidecar JSON with a feed URL or guid. + +In addition to the regular fields, podcast payloads include: + +- ``media_type``: ``"podcast"`` or ``"track"`` +- ``podcast_name``, ``podcast_producer``, ``podcast_description`` +- ``podcast_feed_url``, ``podcast_site_link`` +- ``episode_num``, ``pub_date`` (``YYYY-MM-DD``) +- ``episode_description``, ``episode_url``, ``episode_guid`` Project resources diff --git a/mopidy_webhooks/__init__.py b/mopidy_webhooks/__init__.py index a7b4aa5..ccbae27 100644 --- a/mopidy_webhooks/__init__.py +++ b/mopidy_webhooks/__init__.py @@ -20,6 +20,8 @@ class Extension(ext.Extension): schema = super().get_config_schema() schema["urls"] = config.String() schema["tokens"] = config.Secret(optional=True) + schema["media_dirs"] = config.String(optional=True) + schema["podcast_dirs"] = config.String(optional=True) return schema def setup(self, registry): diff --git a/mopidy_webhooks/ext.conf b/mopidy_webhooks/ext.conf index f1821ee..f225d3c 100644 --- a/mopidy_webhooks/ext.conf +++ b/mopidy_webhooks/ext.conf @@ -2,3 +2,5 @@ enabled = true urls = "" tokens = "" +media_dirs = /var/lib/mopidy/media +podcast_dirs = "" diff --git a/mopidy_webhooks/frontend.py b/mopidy_webhooks/frontend.py index 7163550..52b9944 100644 --- a/mopidy_webhooks/frontend.py +++ b/mopidy_webhooks/frontend.py @@ -1,12 +1,15 @@ import logging +import os import time import json -from typing import Optional +from typing import Any, Dict, Optional import pykka import requests from mopidy.core import CoreListener +from . import metadata as metadata_lib + logger = logging.getLogger(__name__) @@ -15,14 +18,84 @@ class WebhooksFrontend(pykka.ThreadingActor, CoreListener): super().__init__() self.config = config self.webhook_urls = [] + self.webhook_tokens = [] + self.media_dirs = [] + self.podcast_dirs = [] + self._metadata_cache: Dict[str, Dict[str, Any]] = {} self.last_start_time = None def on_start(self): self.webhook_urls = self.config["webhooks"]["urls"].split(",") self.webhook_tokens = self.config["webhooks"]["tokens"].split(",") + self.media_dirs = metadata_lib.parse_dirs( + self.config["webhooks"].get("media_dirs", "") + ) + self.podcast_dirs = metadata_lib.parse_dirs( + self.config["webhooks"].get("podcast_dirs", "") + ) logger.info(f"Parsing webhook URLs and tokens: {self.webhook_urls}") - def _build_post_data(self, track, time_position: Optional[int]=None) -> dict: + def _get_track_metadata(self, track) -> tuple: + """Return ``(metadata, path)`` for a local track, or empty values.""" + path = metadata_lib.resolve_media_path(track.uri, self.media_dirs) + if not path: + return {}, None + + try: + cache_key = f"{path}:{int(os.path.getmtime(path))}" + except OSError: + cache_key = path + + if len(self._metadata_cache) > 512: + self._metadata_cache.clear() + + if cache_key not in self._metadata_cache: + self._metadata_cache[cache_key] = metadata_lib.read_metadata(path) + return self._metadata_cache[cache_key], path + + def _build_podcast_data( + self, track, metadata: Dict[str, Any] + ) -> Dict[str, Any]: + artists = ", ".join(sorted([a.name for a in track.artists])) + feed_url, guid = metadata_lib.split_podcast_uri(track.uri) + feed_url = metadata.get("feed_url") or feed_url + guid = metadata.get("guid") or guid + album_name = track.album.name if track.album else "" + + episode_num = metadata.get("track_number") or track.track_no or 0 + try: + episode_num = int(str(episode_num).split("/")[0].strip()) + except (TypeError, ValueError): + episode_num = 0 + + description = ( + metadata.get("description") or metadata.get("comment") or "" + ) + + try: + duration_seconds = int(float(metadata.get("duration") or 0)) + except (TypeError, ValueError): + duration_seconds = 0 + if not duration_seconds: + duration_seconds = track.length and track.length // 1000 or 0 + + return { + "podcast_name": metadata.get("album") or album_name, + "podcast_producer": metadata.get("artist") or artists, + "podcast_description": description, + "podcast_feed_url": feed_url or "", + "podcast_site_link": metadata.get("website") or "", + "episode_num": episode_num, + "pub_date": metadata_lib.normalize_date(metadata.get("date")), + "episode_description": metadata.get("comment") or description, + "episode_url": metadata.get("episode_url") or "", + "episode_guid": guid or "", + "duration_seconds": duration_seconds, + } + + def _build_post_data( + self, track, time_position: Optional[int] = None + ) -> dict: artists = ", ".join(sorted([a.name for a in track.artists])) artists_list = [a for a in track.artists] try: @@ -30,25 +103,47 @@ class WebhooksFrontend(pykka.ThreadingActor, CoreListener): except IndexError: musicbrainz_artist_id = None duration = track.length and track.length // 1000 or 0 - album_name = "" - if track.album: - album_name = getattr(track.album, "name") + album_name = track.album.name if track.album else "" - return { - "name": track.name, + metadata, path = self._get_track_metadata(track) + podcast = metadata_lib.is_podcast( + track.uri, + path=path, + metadata=metadata, + podcast_dirs=self.podcast_dirs, + ) + + name = track.name + track_number = track.track_no + if podcast: + name = metadata.get("title") or track.name + artists = metadata.get("artist") or artists + album_name = metadata.get("album") or album_name + track_number = metadata.get("track_number") or track.track_no + + post_data = { + "name": name, "artist": artists, "album": album_name, - "track_number": track.track_no, + "track_number": track_number, "run_time_ticks": track.length, "run_time": str(duration), "playback_time_ticks": time_position, "musicbrainz_track_id": track.musicbrainz_id if track.album else "", - "musicbrainz_album_id": track.album.musicbrainz_id if track.album else "", + "musicbrainz_album_id": ( + track.album.musicbrainz_id if track.album else "" + ), "musicbrainz_artist_id": musicbrainz_artist_id, "mopidy_uri": track.uri, "primary_artist_mopidy_uri": artists_list[0].uri, + "media_type": "podcast" if podcast else "track", } + if podcast: + post_data.update(self._build_podcast_data(track, metadata)) + + return post_data + def _post_update_to_webhooks(self, post_data: dict, status: str): post_data["status"] = status @@ -88,7 +183,9 @@ class WebhooksFrontend(pykka.ThreadingActor, CoreListener): duration = track.length and track.length // 1000 or 0 time_position_sec = time_position // 1000 - post_data = self._build_post_data(tl_track.track, time_position=time_position) + post_data = self._build_post_data( + tl_track.track, time_position=time_position + ) if time_position_sec < duration // 2 and time_position_sec < 240: logger.debug( @@ -108,15 +205,14 @@ class WebhooksFrontend(pykka.ThreadingActor, CoreListener): track = tl_track.track artists = ", ".join(sorted([a.name for a in track.artists])) duration = track.length and track.length // 1000 or 0 - time_position_sec = time_position // 1000 - post_data = self._build_post_data(tl_track.track, time_position=time_position) + post_data = self._build_post_data( + tl_track.track, time_position=time_position + ) if self.last_start_time is None: self.last_start_time = int(time.time()) - duration - logger.info( - f"Scrobbling paused via webhooks: {artists} - {track.name}" - ) + logger.info(f"Scrobbling paused via webhooks: {artists} - {track.name}") self._post_update_to_webhooks(post_data, "paused") @@ -126,7 +222,9 @@ class WebhooksFrontend(pykka.ThreadingActor, CoreListener): artists = ", ".join(sorted([a.name for a in track.artists])) self.last_start_time = int(time.time()) logger.debug(f"Now resuming track: {artists} - {track.name}") - post_data = self._build_post_data(tl_track.track, time_position=time_position) + post_data = self._build_post_data( + tl_track.track, time_position=time_position + ) # Build post data to send to urls if not self.webhook_urls: diff --git a/mopidy_webhooks/metadata.py b/mopidy_webhooks/metadata.py new file mode 100644 index 0000000..fde2980 --- /dev/null +++ b/mopidy_webhooks/metadata.py @@ -0,0 +1,309 @@ +import json +import logging +import os +import re +from typing import Any, Dict, Iterable, List, Optional +from urllib.parse import unquote, urlparse +from urllib.request import url2pathname + +try: + import mutagen +except ImportError: # pragma: no cover + mutagen = None + +logger = logging.getLogger(__name__) + +PODCAST_URI_PREFIX = "podcast+" + +_LOCAL_URI_TYPES = frozenset( + {"album", "artist", "directory", "image", "podcast", "track"} +) + +_SIMPLE_TAG_KEYS = { + "title": ("title",), + "artist": ("artist", "albumartist"), + "album": ("album",), + "track_number": ("tracknumber",), + "date": ("date", "year"), + "genre": ("genre",), + "publisher": ("publisher", "organization"), + "language": ("language",), + "copyright": ("copyright",), +} + +# Fields that are common in podcast files but not exposed by mutagen's +# "easy" interface (notably the ID3 COMM frame holding the description). +_RAW_TAG_KEYS = { + "comment": ( + "comment", + "COMM::eng", + "COMM", + "description", + "desc", + "\xa9cmt", + ), + "website": ("website", "WXXX", "WOAR", "url", "\xa9url"), +} + +_SIDECAR_KEYS = ( + "feed_url", + "guid", + "episode_url", + "description", + "duration", + "image", +) + +_DATE_RE = re.compile(r"(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?") + + +def parse_dirs(value: Any) -> List[str]: + """Split a comma-separated config value into a list of directories.""" + if not value: + return [] + if isinstance(value, (list, tuple)): + values = value + else: + values = str(value).split(",") + return [ + str(directory).strip() for directory in values if str(directory).strip() + ] + + +def split_podcast_uri(uri: Optional[str]) -> tuple: + """Split a ``podcast+#`` URI into its feed and guid parts.""" + if not uri or not uri.startswith(PODCAST_URI_PREFIX): + return None, None + feed_url, _, guid = uri[len(PODCAST_URI_PREFIX) :].partition("#") + return feed_url or None, guid or None + + +def _local_relative_path(uri: str) -> Optional[str]: + rest = uri.split(":", 1)[1] if ":" in uri else uri + if ":" in rest: + kind, relative = rest.split(":", 1) + if kind in _LOCAL_URI_TYPES: + return relative + return rest + + +def resolve_media_path( + uri: Optional[str], media_dirs: Optional[Iterable[str]] = None +) -> Optional[str]: + """Resolve a Mopidy track URI to a local filesystem path if possible.""" + if not uri: + return None + + parsed = urlparse(uri) + if parsed.scheme == "file": + path = url2pathname(parsed.path) + return path or None + + if parsed.scheme != "local": + return None + + relative = _local_relative_path(uri) + if not relative: + return None + + relative = unquote(relative) + if os.path.isabs(relative): + return relative + + directories = list(media_dirs or []) + for media_dir in directories: + candidate = os.path.join(media_dir, relative) + if os.path.exists(candidate): + return candidate + + if directories: + return os.path.join(directories[0], relative) + return None + + +def is_under(path: Optional[str], directories: Iterable[str]) -> bool: + """Return True when ``path`` lives under one of ``directories``.""" + if not path: + return False + real_path = os.path.realpath(path) + for directory in directories: + real_dir = os.path.realpath(directory) + if real_path == real_dir or real_path.startswith(real_dir + os.sep): + return True + return False + + +def _frame_text(frame: Any) -> Optional[str]: + text = getattr(frame, "text", None) + if text: + value = text[0] if isinstance(text, (list, tuple)) else text + return str(value).strip() or None + if isinstance(frame, (list, tuple)): + return str(frame[0]).strip() or None + return str(frame).strip() or None + + +def _first_raw_tag_value(path: str, keys: Iterable[str]) -> Optional[str]: + if mutagen is None: + return None + try: + audio = mutagen.File(path) + except Exception: # pragma: no cover - defensive against bad files + return None + tags = getattr(audio, "tags", None) + if not tags: + return None + for key in keys: + frame = None + try: + frame = tags.get(key) + except Exception: + frame = None + if frame is None and hasattr(tags, "getall"): + try: + frames = tags.getall(key) + except Exception: + frames = [] + frame = frames[0] if frames else None + if frame is None: + continue + value = _frame_text(frame) + if value: + return value + return None + + +def _first_tag_value(audio: Any, keys: Iterable[str]) -> Optional[str]: + for key in keys: + try: + value = audio.get(key) + except Exception: # pragma: no cover - defensive against bad tags + continue + if not value: + continue + if isinstance(value, (list, tuple)): + value = value[0] + value = str(value).strip() + if value: + return value + return None + + +def read_tags(path: Optional[str]) -> Dict[str, str]: + """Read normalized tags from an audio file with mutagen.""" + if mutagen is None or not path or not os.path.isfile(path): + return {} + + try: + audio = mutagen.File(path, easy=True) + except Exception: + logger.warning("Could not read audio tags from %s", path, exc_info=True) + return {} + + if audio is None: + return {} + + metadata: Dict[str, str] = {} + for field, keys in _SIMPLE_TAG_KEYS.items(): + value = _first_tag_value(audio, keys) + if value: + metadata[field] = value + + for field, keys in _RAW_TAG_KEYS.items(): + if metadata.get(field): + continue + value = _first_raw_tag_value(path, keys) + if value: + metadata[field] = value + + track_number = metadata.get("track_number") + if track_number: + metadata["track_number"] = track_number.split("/")[0].strip() + + return metadata + + +def sidecar_paths(path: str) -> List[str]: + """Return the candidate sidecar JSON paths for an audio file.""" + stem, _ = os.path.splitext(path) + return [ + stem + ".json", + os.path.join( + os.path.dirname(path), "metadata", os.path.basename(stem) + ".json" + ), + ] + + +def read_sidecar_json(path: str) -> Dict[str, Any]: + """Read a podcatcher/yt-dlp style sidecar JSON file if present.""" + for candidate in sidecar_paths(path): + if not os.path.isfile(candidate): + continue + try: + with open(candidate, encoding="utf-8") as sidecar: + data = json.load(sidecar) + except (OSError, ValueError): + logger.warning("Could not read sidecar metadata from %s", candidate) + continue + if isinstance(data, dict): + return data + return {} + + +def normalize_date(value: Any) -> str: + """Normalize a tag/sidecar date to a ``YYYY-MM-DD`` string when possible.""" + if not value: + return "" + if hasattr(value, "strftime"): + return value.strftime("%Y-%m-%d") + match = _DATE_RE.search(str(value)) + if not match or not match.group(2): + return "" + year, month, day = match.groups() + return "-".join((year, month, day or "01")) + + +def read_metadata(path: Optional[str]) -> Dict[str, Any]: + """Read tags and sidecar metadata for a local media file. + + Audio tags win for title/artist/album/genre, while the sidecar JSON is the + source for podcast specific fields such as the feed URL and episode guid. + """ + if not path: + return {} + + metadata: Dict[str, Any] = {} + sidecar = read_sidecar_json(path) + for key in _SIDECAR_KEYS: + value = sidecar.get(key) + if value: + metadata[key] = value + for field in ("title", "artist", "album", "date"): + value = sidecar.get(field) + if value: + metadata.setdefault(field, str(value)) + + metadata.update( + {key: value for key, value in read_tags(path).items() if value} + ) + + if sidecar.get("date"): + metadata["date"] = str(sidecar["date"]) + + return metadata + + +def is_podcast( + uri: Optional[str], + path: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + podcast_dirs: Optional[Iterable[str]] = None, +) -> bool: + """Best effort detection of podcast episodes.""" + uri = uri or "" + if PODCAST_URI_PREFIX in uri or "podcast" in uri.lower(): + return True + if path and is_under(path, podcast_dirs or []): + return True + metadata = metadata or {} + return bool(metadata.get("feed_url") or metadata.get("guid")) diff --git a/setup.cfg b/setup.cfg index 6a82554..0812eea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,7 +7,8 @@ author_email = colin@unbl.ink license = Apache License, Version 2.0 license_file = LICENSE description = Mopidy extension for sending playback data to webhook urls -long_description = file: README.rst +long_description = file: README.md +long_description_content_type = text/markdown classifiers = Environment :: No Input/Output (Daemon) Intended Audience :: End Users/Desktop @@ -28,6 +29,7 @@ python_requires = >= 3.7 install_requires = Mopidy >= 3.0.0 Pykka >= 2.0.1 + mutagen >= 1.45 setuptools diff --git a/tests/data/silence.mp3 b/tests/data/silence.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..a90485e511dbe33d5fc1369d05921f23c1f5b1d2 GIT binary patch literal 2411 zcmeZtF=k-^0p*b3U{@f`&%nU!lUSB!W~65bLE`;fsmzED?0rfG6 zGB9x3F~|Xd76_PvfCC7413@?lB!NIK2vh(;3kdXsz$_403IrQLU=Ii!2Z2jKa1R7t zg1{FL_zwgin}Lo@20GHn0>ps<6*f5tlhZEs$k!)e_W$1kM;Mq7Ffc9waupaDOd1#% zSbQ9PU5!EBQ_x6CElE@`)(j;BpqK)(N3$8t9>kv%nZ^3O6l`R6>H z{Bw;?{u#9gC7