Added jimaku provider (#2505)

Co-authored-by: Danny <[email protected]>

bazarr/app/config.py

@@ -300,6 +300,12 @@ def check_parser_binary(value):
 
     # analytics section
     Validator('analytics.enabled', must_exist=True, default=True, is_type_of=bool),
+
+    # jimaku section
+    Validator('jimaku.api_key', must_exist=True, default='', is_type_of=str),
+    Validator('jimaku.enable_name_search_fallback', must_exist=True, default=True, is_type_of=bool),
+    Validator('jimaku.enable_archives_download', must_exist=True, default=False, is_type_of=bool),
+    Validator('jimaku.enable_ai_subs', must_exist=True, default=False, is_type_of=bool),
 
     # titlovi section
     Validator('titlovi.username', must_exist=True, default='', is_type_of=str, cast=str),

bazarr/app/get_providers.py

@@ -285,6 +285,12 @@ def get_providers_auth():
             'username': settings.titlovi.username,
             'password': settings.titlovi.password,
         },
+        'jimaku': {
+            'api_key': settings.jimaku.api_key,
+            'enable_name_search_fallback': settings.jimaku.enable_name_search_fallback,
+            'enable_archives_download': settings.jimaku.enable_archives_download,
+            'enable_ai_subs': settings.jimaku.enable_ai_subs,
+        },
         'ktuvit': {
             'email': settings.ktuvit.email,
             'hashed_password': settings.ktuvit.hashed_password,

bazarr/subtitles/refiners/__init__.py

@@ -4,10 +4,12 @@
 from .database import refine_from_db
 from .arr_history import refine_from_arr_history
 from .anidb import refine_from_anidb
+from .anilist import refine_from_anilist
 
 registered = {
     "database": refine_from_db,
     "ffprobe": refine_from_ffprobe,
     "arr_history": refine_from_arr_history,
     "anidb": refine_from_anidb,
+    "anilist": refine_from_anilist, # Must run AFTER AniDB
 }

bazarr/subtitles/refiners/anidb.py

@@ -20,7 +20,10 @@
     except ImportError:
         import xml.etree.ElementTree as etree
 
-refined_providers = {'animetosho'}
+refined_providers = {'animetosho', 'jimaku'}
+providers_requiring_anidb_api = {'animetosho'}
+
+logger = logging.getLogger(__name__)
 
 api_url = 'http://api.anidb.net:9001/httpapi'
 
@@ -40,6 +43,10 @@ def __init__(self, api_client_key=None, api_client_ver=1, session=None):
     @property
     def is_throttled(self):
         return self.cache and self.cache.get('is_throttled')
+
+    @property
+    def has_api_credentials(self):
+        return self.api_client_key != '' and self.api_client_key is not None
 
     @property
     def daily_api_request_count(self):
@@ -62,7 +69,9 @@ def get_series_mappings(self):
         return r.content
 
     @region.cache_on_arguments(expiration_time=timedelta(days=1).total_seconds())
-    def get_series_id(self, mappings, tvdb_series_season, tvdb_series_id, episode):
+    def get_show_information(self, tvdb_series_id, tvdb_series_season, episode):
+        mappings = etree.fromstring(self.get_series_mappings())
+
         # Enrich the collection of anime with the episode offset
         animes = [
             self.AnimeInfo(anime, int(anime.attrib.get('episodeoffset', 0)))
@@ -71,49 +80,60 @@ def get_series_id(self, mappings, tvdb_series_season, tvdb_series_id, episode):
             )
         ]
 
+        is_special_entry = False
         if not animes:
-            return None, None
+            # Some entries will store TVDB seasons in a nested mapping list, identifiable by the value 'a' as the season
+            special_entries = mappings.findall(
+                f".//anime[@tvdbid='{tvdb_series_id}'][@defaulttvdbseason='a']"
+            )
 
-        # Sort the anime by offset in ascending order
-        animes.sort(key=lambda a: a.episode_offset)
+            if not special_entries:
+                return None, None, None
 
-        # Different from Tvdb, Anidb have different ids for the Parts of a season
-        anidb_id = None
-        offset = 0
+            is_special_entry = True
+            for special_entry in special_entries:
+                mapping_list = special_entry.findall(f".//mapping[@tvdbseason='{tvdb_series_season}']")
+                if len(mapping_list) > 0:
+                    anidb_id = int(special_entry.attrib.get('anidbid'))
+                    offset = int(mapping_list[0].attrib.get('offset', 0))
 
-        for index, anime_info in enumerate(animes):
-            anime, episode_offset = anime_info
+        if not is_special_entry:
+            # Sort the anime by offset in ascending order
+            animes.sort(key=lambda a: a.episode_offset)
 
-            mapping_list = anime.find('mapping-list')
+            # Different from Tvdb, Anidb have different ids for the Parts of a season
+            anidb_id = None
+            offset = 0
 
-            # Handle mapping list for Specials
-            if mapping_list:
-                for mapping in mapping_list.findall("mapping"):
-                    # Mapping values are usually like ;1-1;2-1;3-1;
-                    for episode_ref in mapping.text.split(';'):
-                        if not episode_ref:
-                            continue
+            for index, anime_info in enumerate(animes):
+                anime, episode_offset = anime_info
+
+                mapping_list = anime.find('mapping-list')
 
-                        anidb_episode, tvdb_episode = map(int, episode_ref.split('-'))
-                        if tvdb_episode == episode:
-                            anidb_id = int(anime.attrib.get('anidbid'))
+                # Handle mapping list for Specials
+                if mapping_list:
+                    for mapping in mapping_list.findall("mapping"):
+                        # Mapping values are usually like ;1-1;2-1;3-1;
+                        for episode_ref in mapping.text.split(';'):
+                            if not episode_ref:
+                                continue
 
-                            return anidb_id, anidb_episode
+                            anidb_episode, tvdb_episode = map(int, episode_ref.split('-'))
+                            if tvdb_episode == episode:
+                                anidb_id = int(anime.attrib.get('anidbid'))
 
-            if episode > episode_offset:
-                anidb_id = int(anime.attrib.get('anidbid'))
-                offset = episode_offset
+                                return anidb_id, anidb_episode, 0
 
-        return anidb_id, episode - offset
+                if episode > episode_offset:
+                    anidb_id = int(anime.attrib.get('anidbid'))
+                    offset = episode_offset
 
-    @region.cache_on_arguments(expiration_time=timedelta(days=1).total_seconds())
-    def get_series_episodes_ids(self, tvdb_series_id, season, episode):
-        mappings = etree.fromstring(self.get_series_mappings())
-
-        series_id, episode_no = self.get_series_id(mappings, season, tvdb_series_id, episode)
+        return anidb_id, episode - offset, offset
 
+    @region.cache_on_arguments(expiration_time=timedelta(days=1).total_seconds())
+    def get_episode_ids(self, series_id, episode_no):
         if not series_id:
-            return None, None
+            return None
 
         episodes = etree.fromstring(self.get_episodes(series_id))
 
@@ -177,7 +197,7 @@ def mark_as_throttled():
 
 def refine_from_anidb(path, video):
     if not isinstance(video, Episode) or not video.series_tvdb_id:
-        logging.debug(f'Video is not an Anime TV series, skipping refinement for {video}')
+        logger.debug(f'Video is not an Anime TV series, skipping refinement for {video}')
 
         return
 
@@ -190,27 +210,35 @@ def refine_anidb_ids(video):
 
     season = video.season if video.season else 0
 
-    if anidb_client.is_throttled:
-        logging.warning(f'API daily limit reached. Skipping refinement for {video.series}')
-
-        return video
-
-    try:
-        anidb_series_id, anidb_episode_id = anidb_client.get_series_episodes_ids(
-            video.series_tvdb_id,
-            season, video.episode,
-        )
-    except TooManyRequests:
-        logging.error(f'API daily limit reached while refining {video.series}')
-
-        anidb_client.mark_as_throttled()
-
-        return video
-
-    if not anidb_episode_id:
-        logging.error(f'Could not find anime series {video.series}')
-
+    anidb_series_id, anidb_episode_no, anidb_season_episode_offset = anidb_client.get_show_information(
+        video.series_tvdb_id,
+        season,
+        video.episode,
+    )
+
+    if not anidb_series_id:
+        logger.error(f'Could not find anime series {video.series}')
         return video
+
+    anidb_episode_id = None
+    if anidb_client.has_api_credentials:
+        if anidb_client.is_throttled:
+            logger.warning(f'API daily limit reached. Skipping episode ID refinement for {video.series}')
+        else:
+            try:
+                anidb_episode_id = anidb_client.get_episode_ids(
+                    anidb_series_id,
+                    anidb_episode_no
+                )
+            except TooManyRequests:
+                logger.error(f'API daily limit reached while refining {video.series}')
+                anidb_client.mark_as_throttled()
+    else:
+        intersect = providers_requiring_anidb_api.intersection(settings.general.enabled_providers)
+        if len(intersect) >= 1:
+            logger.warn(f'AniDB API credentials are not fully set up, the following providers may not work: {intersect}')
 
     video.series_anidb_id = anidb_series_id
     video.series_anidb_episode_id = anidb_episode_id
+    video.series_anidb_episode_no = anidb_episode_no
+    video.series_anidb_season_episode_offset = anidb_season_episode_offset

bazarr/subtitles/refiners/anilist.py

@@ -0,0 +1,77 @@
+# coding=utf-8
+# fmt: off
+
+import logging
+import time
+import requests
+from collections import namedtuple
+from datetime import timedelta
+
+from app.config import settings
+from subliminal import Episode, region, __short_version__
+
+logger = logging.getLogger(__name__)
+refined_providers = {'jimaku'}
+
+class AniListClient(object):    
+    def __init__(self, session=None, timeout=10):
+        self.session = session or requests.Session()
+        self.session.timeout = timeout
+        self.session.headers['Content-Type'] = 'application/json'
+        self.session.headers['User-Agent'] = 'Subliminal/%s' % __short_version__
+
+    @region.cache_on_arguments(expiration_time=timedelta(days=1).total_seconds())
+    def get_series_mappings(self):
+        r = self.session.get(
+            'https://raw.githubusercontent.com/Fribb/anime-lists/master/anime-list-mini.json'
+        )
+
+        r.raise_for_status()
+        return r.json()
+
+    def get_series_id(self, candidate_id_name, candidate_id_value):
+        anime_list = self.get_series_mappings()
+
+        tag_map = {
+            "series_anidb_id": "anidb_id",
+            "imdb_id": "imdb_id"
+        }
+        mapped_tag = tag_map.get(candidate_id_name, candidate_id_name)        
+
+        obj = [obj for obj in anime_list if mapped_tag in obj and str(obj[mapped_tag]) == str(candidate_id_value)]
+        logger.debug(f"Based on '{mapped_tag}': '{candidate_id_value}', anime-list matched: {obj}")
+
+        if len(obj) > 0:
+            return obj[0]["anilist_id"]
+        else:
+            logger.debug(f"Could not find corresponding AniList ID with '{mapped_tag}': {candidate_id_value}")
+            return None
+
+def refine_from_anilist(path, video):
+    # Safety checks
+    if isinstance(video, Episode):
+        if not video.series_anidb_id:
+            logger.error(f"Will not refine '{video.series}' as it does not have an AniDB ID.")
+            return
+
+    if refined_providers.intersection(settings.general.enabled_providers) and video.anilist_id is None:
+        refine_anilist_ids(video)
+
+def refine_anilist_ids(video):
+    anilist_client = AniListClient()
+
+    if isinstance(video, Episode):
+        candidate_id_name = "series_anidb_id"
+    else:
+        candidate_id_name = "imdb_id"
+
+    candidate_id_value = getattr(video, candidate_id_name, None)
+    if not candidate_id_value:
+        logger.error(f"Found no value for property {candidate_id_name} of video.")
+        return video
+
+    anilist_id = anilist_client.get_series_id(candidate_id_name, candidate_id_value)
+    if not anilist_id:
+        return video
+
+    video.anilist_id = anilist_id

custom_libs/subliminal/video.py

@@ -130,7 +130,8 @@ class Episode(Video):
     """
     def __init__(self, name, series, season, episode, title=None, year=None, original_series=True, tvdb_id=None,
                  series_tvdb_id=None, series_imdb_id=None, alternative_series=None, series_anidb_id=None,
-                 series_anidb_episode_id=None, **kwargs):
+                 series_anidb_episode_id=None, series_anidb_season_episode_offset=None,
+                 anilist_id=None, **kwargs):
         super(Episode, self).__init__(name, **kwargs)
 
         #: Series of the episode
@@ -163,8 +164,11 @@ def __init__(self, name, series, season, episode, title=None, year=None, origina
         #: Alternative names of the series
         self.alternative_series = alternative_series or []
 
+        #: Anime specific information
         self.series_anidb_episode_id = series_anidb_episode_id
         self.series_anidb_id = series_anidb_id
+        self.series_anidb_season_episode_offset = series_anidb_season_episode_offset
+        self.anilist_id = anilist_id
 
     @classmethod
     def fromguess(cls, name, guess):
@@ -207,10 +211,11 @@ class Movie(Video):
     :param str title: title of the movie.
     :param int year: year of the movie.
     :param list alternative_titles: alternative titles of the movie
+    :param int anilist_id: AniList ID of movie (if Anime)
     :param \*\*kwargs: additional parameters for the :class:`Video` constructor.
 
     """
-    def __init__(self, name, title, year=None, alternative_titles=None, **kwargs):
+    def __init__(self, name, title, year=None, alternative_titles=None, anilist_id=None, **kwargs):
         super(Movie, self).__init__(name, **kwargs)
 
         #: Title of the movie
@@ -221,6 +226,9 @@ def __init__(self, name, title, year=None, alternative_titles=None, **kwargs):
 
         #: Alternative titles of the movie
         self.alternative_titles = alternative_titles or []
+
+        #: AniList ID of the movie
+        self.anilist_id = anilist_id
 
     @classmethod
     def fromguess(cls, name, guess):