From 7754e918bfbea49b91bdf3d51b0ffd93aaf1d883 Mon Sep 17 00:00:00 2001 From: Baptiste Fouques Date: Thu, 28 Nov 2024 11:21:56 +0100 Subject: [PATCH] Robustify + speedup through http request caching and multithreading --- plugin.audio.radiofrance/addon.xml | 14 +- plugin.audio.radiofrance/default.py | 258 +++++------- plugin.audio.radiofrance/interface.py | 113 ++++++ plugin.audio.radiofrance/utils.py | 555 ++++++++++++++------------ 4 files changed, 534 insertions(+), 406 deletions(-) create mode 100644 plugin.audio.radiofrance/interface.py diff --git a/plugin.audio.radiofrance/addon.xml b/plugin.audio.radiofrance/addon.xml index 200457817..f09a74076 100644 --- a/plugin.audio.radiofrance/addon.xml +++ b/plugin.audio.radiofrance/addon.xml @@ -1,5 +1,5 @@ - + @@ -15,6 +15,18 @@ https://github.com/bateast/plugin.audio.radiofrance all + v1.1.4 (2024-11) + - minor fix on slider element + v1.1.3 (2024-10) + - robustify against radiofrance pages + v1.1.2 (2024-10) + - refactor + - speed-up by caching http responses + v1.1.1 (2024-09) + - robustify against json parsing errors + v1.1.0 (2024-08) + - Speedup by using multithreading pool + - [backend] rework to move gui element creation to Class based structure v1.0.0 (2024-08) - Open to live station and RadioFrance landing page - Add translation diff --git a/plugin.audio.radiofrance/default.py b/plugin.audio.radiofrance/default.py index 99695c4d1..5af70fc35 100644 --- a/plugin.audio.radiofrance/default.py +++ b/plugin.audio.radiofrance/default.py @@ -1,163 +1,116 @@ +import os +import datetime import json + import sys import requests +from pathlib import Path from urllib.parse import parse_qs -from enum import Enum +from concurrent.futures import ThreadPoolExecutor +import itertools -# http://mirrors.kodi.tv/docs/python-docs/ -# http://www.crummy.com/software/BeautifulSoup/bs4/doc/ -from urllib.parse import urlencode, quote_plus -from ast import literal_eval import xbmc +import xbmcaddon +import xbmcvfs import xbmcgui import xbmcplugin from utils import * +from interface import * DEFAULT_MANIFESTATION = 0 RADIOFRANCE_PAGE = "https://www.radiofrance.fr" +CACHE_DIR = xbmcvfs.translatePath(xbmcaddon.Addon().getAddonInfo('profile')) +CACHE_FILE = os.path.join(CACHE_DIR, 'radiofrance_cache.json') +CACHE_TIMEOUT = datetime.timedelta(seconds=300) + +# Function to save cache to a file +def save_cache(data): + Path(CACHE_DIR).mkdir(parents=True, exist_ok=True) + with open(CACHE_FILE, 'w') as f: + xbmc.log("Caching to :" + CACHE_FILE, xbmc.LOGINFO) + json.dump(data, f) + +# Function to load cache from a file +def load_cache(): + if os.path.exists(CACHE_FILE): + with open(CACHE_FILE, 'r') as f: + xbmc.log("Loading cach from :" + CACHE_FILE, xbmc.LOGINFO) + try: + data = json.load(f) + except: + return {} + return data + return {} def build_lists(data, args, url): - xbmc.log(str(args), xbmc.LOGINFO) - - def add_search(): - new_args = {k: v[0] for (k, v) in list(args.items())} - new_args["mode"] = "search" - li = xbmcgui.ListItem(label=localize(30100)) - li.setIsFolder(True) - new_url = build_url(new_args) - highlight_list.append((new_url, li, True)) - - def add_podcasts(): - new_args = {k: v[0] for (k, v) in list(args.items())} - new_args["mode"] = "podcasts" - li = xbmcgui.ListItem(label=localize(30104)) - li.setIsFolder(True) - new_url = build_url(new_args) - highlight_list.append((new_url, li, True)) - - def add_pages(item): - new_args = {k: v[0] for (k, v) in list(args.items())} - (num, last) = item.pages - if 1 < num: - new_args["page"] = num - 1 - li = xbmcgui.ListItem(label=localize(30101)) - li.setIsFolder(True) - new_url = build_url(new_args) - highlight_list.append((new_url, li, True)) - if num < last: - new_args["page"] = num + 1 - li = xbmcgui.ListItem(label=localize(30102)) - li.setIsFolder(True) - new_url = build_url(new_args) - highlight_list.append((new_url, li, True)) - - def add(item, index): - new_args = {} - # Create kodi element - if item.is_folder(): - if item.path is not None: - li = xbmcgui.ListItem(label=item.title) - li.setArt({"thumb": item.image, "icon": item.icon}) - li.setIsFolder(True) - new_args = {"title": item.title} - new_args["url"] = item.path - new_args["mode"] = "url" - builded_url = build_url(new_args) - highlight_list.append((builded_url, li, True)) - - xbmc.log( - str(new_args), - xbmc.LOGINFO, - ) - if 1 == len(item.subs): - add(create_item(item.subs[0]), index) - elif 1 < len(item.subs): - li = xbmcgui.ListItem(label="⭐ " + item.title if item.title is not None else "") - li.setArt({"thumb": item.image, "icon": item.icon}) - li.setIsFolder(True) - new_args = {"title": "⭐ " + item.title if item.title is not None else ""} - new_args["url"] = url - new_args["index"] = index - new_args["mode"] = "index" - builded_url = build_url(new_args) - highlight_list.append((builded_url, li, True)) - - xbmc.log( - str(new_args), - xbmc.LOGINFO, - ) - - else: - # Playable element - li = xbmcgui.ListItem(label=item.title) - li.setArt({"thumb": item.image, "icon": item.icon}) - new_args = {"title": item.title} - li.setIsFolder(False) - tag = li.getMusicInfoTag(offscreen=True) - tag.setMediaType("audio") - tag.setTitle(item.title) - tag.setURL(item.path) - tag.setGenres([item.genre if item.model == Model['Brand'] else "podcast"]) - tag.setArtist(item.artists) - tag.setDuration(item.duration if item.duration is not None else 0) - tag.setReleaseDate(item.release) - li.setProperty("IsPlayable", "true") - if item.path is not None: - new_args["url"] = item.path - new_args["mode"] = ( - "brand" if item.model == Model["Brand"] else "stream" - ) - - builded_url = build_url(new_args) - song_list.append((builded_url, li, False)) - - xbmc.log( - str(new_args), - xbmc.LOGINFO, - ) - - highlight_list = [] - song_list = [] + gui_elements_list = [] mode = args.get("mode", [None])[0] if mode is None: - add_search() - add_podcasts() + Search(args).add(gui_elements_list) + Podcasts(args).add(gui_elements_list) item = create_item_from_page(data) + context = data.get('context', {}) + if mode == "index": element_index = int(args.get("index", [None])[0]) - items_list = create_item(item.subs[element_index]).elements + items_list = create_item(0, item.subs[element_index], context).subs else: items_list = item.subs - add_pages(item) - index = 0 - for data in items_list: - sub_item = create_item(data) - xbmc.log(str(sub_item), xbmc.LOGINFO) - add(sub_item, index) - index += 1 + Pages(item, args).add(gui_elements_list) + + context = data.get('context', {}) + with ThreadPoolExecutor() as executor: + elements_lists = list(executor.map(lambda idx, data: add_with_index(idx, data, args, context), range(len(items_list)), items_list)) + + gui_elements_list.extend(itertools.chain.from_iterable(elements_lists)) xbmcplugin.setContent(addon_handle, "episodes") - xbmcplugin.addDirectoryItems(addon_handle, highlight_list, len(highlight_list)) - xbmcplugin.addDirectoryItems(addon_handle, song_list, len(song_list)) + xbmcplugin.addDirectoryItems(addon_handle, gui_elements_list, len(gui_elements_list)) xbmcplugin.endOfDirectory(addon_handle) +def add_with_index(index, data, args, context): + item = create_item(index, data, context) + if not isinstance(item, Item): + _, data, exception = item + xbmc.log(f"Error: {exception} on {data}", xbmc.LOGERROR) + return [] -def brand(args): + xbmc.log(str(item), xbmc.LOGINFO) + elements_list = [] url = args.get("url", [""])[0] - xbmc.log("[Play Brand]: " + url, xbmc.LOGINFO) + if len(item.subs) <= 2: + sub_list = list(map(lambda idx, data: create_item(idx, data, context), range(len(item.subs)), item.subs)) + + for sub_item in sub_list: + if sub_item.is_folder(): + elements_list.append(Folder(sub_item, args).construct()) + else: + elements_list.append(Playable(sub_item, args).construct()) + elif len(item.subs) > 1: + elements_list.append(Indexed(item, url, index, args).construct()) + + if item.is_folder() and item.path is not None: + elements_list.append(Folder(item, args).construct()) + elif not item.is_folder(): + elements_list.append(Playable(item, args).construct()) + + return elements_list + +def brand(args): + url = args.get("url", [""])[0] + xbmc.log(f"[Play Brand]: {url}", xbmc.LOGINFO) play(url) def play(url): play_item = xbmcgui.ListItem(path=url) xbmcplugin.setResolvedUrl(addon_handle, True, listitem=play_item) - def search(args): def GUIEditExportName(name): kb = xbmc.Keyboard("Odyssées", localize(30103)) @@ -167,64 +120,63 @@ def GUIEditExportName(name): query = kb.getText() return query - new_args = {k: v[0] for (k, v) in list(args.items())} + new_args = {k: v[0] for k, v in args.items()} new_args["mode"] = "page" value = GUIEditExportName("Odyssées") if value is None: return - new_args["url"] = RADIOFRANCE_PAGE + "/recherche" - new_args = {k: [v] for (k, v) in list(new_args.items())} + new_args["url"] = f"{RADIOFRANCE_PAGE}/recherche" + new_args = {k: [v] for k, v in new_args.items()} build_url(new_args) - get_and_build_lists(new_args, url_args="?term=" + value + "&") - + get_and_build_lists(new_args, url_args=f"?term={value}&") def get_and_build_lists(args, url_args="?"): - xbmc.log( - "".join(["Get and build: " + str(args) + "(url args: " + url_args + ")"]), - xbmc.LOGINFO, - ) + + cache = load_cache() + + xbmc.log(f"Get and build: {args} (url args: {url_args})", xbmc.LOGINFO) url = args.get("url", [RADIOFRANCE_PAGE])[0] - page = requests.get(url + "/__data.json" + url_args).text - content = expand_json(page) - build_lists(content, args, url) + now = datetime.datetime.now() + if url + url_args in cache and now - datetime.datetime.fromisoformat(cache[url + url_args]['datetime']) < CACHE_TIMEOUT: + xbmc.log(f"Using cached data for url: {url + url_args}", xbmc.LOGINFO) + data = cache[url + url_args]['data'] + else: + page = requests.get(f"{url}/__data.json{url_args}").text + data = expand_json(page) + cache[url + url_args] = {'datetime': datetime.datetime.now().isoformat(), 'data': data} + save_cache(cache) + build_lists(data, args, url) def main(): args = parse_qs(sys.argv[2][1:]) - mode = args.get("mode", None) + mode = args.get("mode", [None])[0] - xbmc.log( - "".join( - ["mode: ", str("" if mode is None else mode[0]), ", args: ", str(args)] - ), - xbmc.LOGINFO, - ) + xbmc.log(f"Mode: {mode}, Args: {args}", xbmc.LOGINFO) - # initial launch of add-on + # Initial launch of add-on url = "" url_args = "?" - url_args += "recent=false&" - if "page" in args and 1 < int(args.get("page", ["1"])[0]): - url_args += "p=" + str(args.get("page", ["1"])[0]) - if mode is not None and mode[0] == "stream": - play(args("url")) - elif mode is not None and mode[0] == "search": + # url_args += "recent=false&" + if "page" in args and int(args.get("page", ["1"])[0]) > 1: + url_args += f"&p={args.get('page', ['1'])[0]}" + if mode == "stream": + play(args["url"][0]) + elif mode == "search": search(args) - elif mode is not None and mode[0] == "brand": + elif mode == "brand": brand(args) else: - if mode is not None and mode[0] == "podcasts": + if mode == "podcasts": args["url"][0] += "/podcasts" - elif mode is None: + elif not mode: url = RADIOFRANCE_PAGE - args["url"] = [] - args["url"].append(url) + args["url"] = [url] # New page get_and_build_lists(args, url_args) - if __name__ == "__main__": addon_handle = int(sys.argv[1]) main() diff --git a/plugin.audio.radiofrance/interface.py b/plugin.audio.radiofrance/interface.py new file mode 100644 index 000000000..f78758a5c --- /dev/null +++ b/plugin.audio.radiofrance/interface.py @@ -0,0 +1,113 @@ +from utils import * +import xbmcgui + +class Element: + def __init__(self, title="", args={}): + self.is_folder = False + self.title = title + self.mode = "url" + self.art = None + self.args = {k: v[0] for k, v in args.items()} + + def set_mode(self, mode): + self.mode = mode + + def is_folder(self): + return self.is_folder + + def construct(self): + li = xbmcgui.ListItem(self.title) + li.setIsFolder(self.is_folder) + li.setArt(self.art) + self.args['mode'] = self.mode + new_url = build_url(self.args) + return (new_url, li, True) + + def add(self, kodi_list): + kodi_tuple = self.construct() + kodi_list.append(kodi_tuple) + +class Search(Element): + def __init__(self, args): + super().__init__(localize(30100), args) + self.is_folder = True + self.mode = "search" + +class Podcasts(Element): + def __init__(self, args): + super().__init__(localize(30104), args) + self.is_folder = True + self.mode = "podcasts" + +class Page(Element): + def __init__(self, num, plus, args): + super().__init__(localize(30101) if plus == -1 else localize(30102), args) + self.is_folder = True + self.args['page'] = num + plus + +class Pages(Element): + def __init__(self, item, args): + self.is_folder = True + num, last = item.pages + self.pages = [] + if num > 1: + self.pages.append(Page(num, -1, args)) + if num < last: + self.pages.append(Page(num, +1, args)) + + def add(self, kodi_list): + for page in self.pages: + page.add(kodi_list) + +class Folder(Element): + def __init__(self, item, args): + super().__init__(item.title, args) + self.is_folder = True + self.art = {'thumb': item.image, 'icon': item.icon} + self.args = { + 'title': item.title, + 'url': item.path, + 'mode': "url", + } + +class Indexed(Element): + def __init__(self, item, url, index, args): + super().__init__("⭐ " + item.title if item.title is not None else "", args) + self.is_folder = True + self.art = {'thumb': item.image, 'icon': item.icon} + self.mode = "index" + self.args = { + 'title': item.title, + 'url': url, + 'index': index, + } + +class Playable(Element): + def __init__(self, item, args): + super().__init__(item.title, args) + self.art = {'thumb': item.image, 'icon': item.icon} + self.is_folder = False + self.genre = item.genre if item.model == Model['BRAND'] else "podcast" + self.artists = item.artists + self.duration = item.duration if item.duration is not None else 0 + self.release = item.release + self.args['url'] = item.path + self.path = item.path + self.mode = "brand" if item.model == Model["BRAND"] else "stream" + + def construct(self): + (url, li, _) = super().construct() + li.setProperty("IsPlayable", "true") + + tag = li.getMusicInfoTag(offscreen=True) + tag.setMediaType("audio") + tag.setTitle(self.title) + tag.setURL(self.path) + tag.setGenres([self.genre]) + tag.setArtist(self.artists) + tag.setDuration(self.duration) + tag.setReleaseDate(self.release) + + return url, li, False + + diff --git a/plugin.audio.radiofrance/utils.py b/plugin.audio.radiofrance/utils.py index 0880c006e..9a2500637 100644 --- a/plugin.audio.radiofrance/utils.py +++ b/plugin.audio.radiofrance/utils.py @@ -1,4 +1,4 @@ -#!/bin/python3 +#!/usr/bin/python3 from urllib.parse import urlencode, quote_plus import json @@ -6,31 +6,54 @@ import requests from enum import Enum from time import localtime, strftime +from concurrent.futures import ThreadPoolExecutor +import itertools -RADIOFRANCE_PAGE = "https://www.radiofrance.fr/" -BRAND_EXTENSION = "/api/live/webradios/" +RADIOFRANCE_PAGE = "https://www.radiofrance.fr" +BRAND_EXTENSION = "/api/live/webradios" +NOT_ITEM_FORMAT = ["TYPED_ELEMENT_NEWSLETTER_SUBSCRIPTION", "TYPED_ELEMENT_AUTOPROMO_IMMERSIVE"] class Model(Enum): - Other = 0 - Theme = 1 - Concept = 2 - Highlight = 3 - HighlightElement = 4 - Expression = 5 - ManifestationAudio = 6 - EmbedImage = 7 - PageTemplate = 8 - Brand = 9 - Tag = 10 - Search = 11 - Article = 12 - Event = 13 - Slug = 14 - Station = 15 - + OTHER = 0 + THEME = 1 + CONCEPT = 2 + HIGHLIGHT = 3 + HIGHLIGHTELEMENT = 4 + EXPRESSION = 5 + MANIFESTATIONAUDIO = 6 + EMBEDIMAGE = 7 + PAGETEMPLATE = 8 + BRAND = 9 + TAG = 10 + SEARCH = 11 + ARTICLE = 12 + EVENT = 13 + SLUG = 14 + STATION = 15 + STATIONPAGE = 16 + GRID = 17 + PROGRAM = 18 + SLIDER = 19 + +class Format(Enum): + SLIDER_CHAINE = 1 + +def fetch_data(url): + response = requests.get(url) + response.raise_for_status() + return response.json() + +def get_key_src(key, data): + if data is None: + return None + value = data.get(key, {}) + if value is not None and 'src' in value: + return value['src'] + return None def create_item_from_page(data): + context = data.get('context', {}) if "model" not in data: if "content" in data: @@ -40,187 +63,201 @@ def create_item_from_page(data): elif "podcastsData" in data: data = data["podcastsData"] - item = create_item(data) + item = create_item(0, data, context) + if not isinstance(item, Item): + _, data, e = item + print(json.dumps(data)) + raise(e) + index = 0 while len(item.subs) == 1: - item = create_item(item.subs[0]) + new_item = create_item(index, item.subs[0], context) + if isinstance(new_item, Item): + item = new_item + else: + break + index += 1 return item - -def create_item(data): - match_list = { - Model.Other.name: Other, - Model.Brand.name: Brand, - Model.Theme.name: Theme, - Model.Concept.name: Concept, - Model.Highlight.name: Highlight, - Model.HighlightElement.name: HighlightElement, - Model.Expression.name: Expression, - Model.ManifestationAudio.name: ManifestationAudio, - Model.EmbedImage.name: EmbedImage, - Model.PageTemplate.name: PageTemplate, - Model.Tag.name: Tag, - Model.Article.name: Article, - Model.Event.name: Event, - Model.Slug.name: Slug, - Model.Station.name: Station, +def create_item(index, data, context = {}): + model_map = { + Model.OTHER.name: Other, + Model.BRAND.name: Brand, + Model.THEME.name: Theme, + Model.CONCEPT.name: Concept, + Model.HIGHLIGHT.name: Highlight, + Model.HIGHLIGHTELEMENT.name: HighlightElement, + Model.EXPRESSION.name: Expression, + Model.MANIFESTATIONAUDIO.name: ManifestationAudio, + Model.EMBEDIMAGE.name: EmbedImage, + Model.PAGETEMPLATE.name: PageTemplate, + Model.TAG.name: Tag, + Model.ARTICLE.name: Article, + Model.EVENT.name: Event, + Model.SLUG.name: Slug, + Model.STATIONPAGE.name: StationPage, + Model.STATION.name: Station, + Model.GRID.name: Grid, + Model.PROGRAM.name: Program + } + format_map = { + Format.SLIDER_CHAINE.name : Slider } - if 'model' in data: - item = match_list[data['model']](data) - elif 'stationName' in data: - item = Station(data) - elif 'items' in data and 'concepts' in data['items'] and 'expressions_articles' in data['items']: - item = Search(data) - elif 'slug' in data: - item = match_list['Slug'](data) - elif 'brand' in data: - item = match_list['Brand'](data) - else: - item = match_list['Other'](data) - - # Remove singletons - if item.path is None and len(item.subs) == 1: - item = create_item(item.subs[0]) - - item.elements = item.subs - while len(item.elements) == 1 and item.elements[0] is not None: - item.elements = create_item(item.elements[0]).elements - + try: + if data.get('model', "").upper() in model_map: + model_class = model_map[data['model'].upper()] + item = model_class(data, index, context) + elif data.get('format', "").upper() in format_map: + format_class = format_map[data['format'].upper()] + item = format_class(data, index, context) + elif 'stationName' in data: + item = StationDir(data, index, context) + elif 'items' in data and 'concepts' in data['items'] and 'expressions_articles' in data['items']: + item = Search(data, index, context) + elif 'slug' in data: + item = model_map['SLUG'](data, index, context) + elif 'brand' in data and not data.get('format', "") in NOT_ITEM_FORMAT : + item = model_map['BRAND'](data, index, context) + elif 'grid' in data: + item = model_map['GRID'](data, index, context) + elif 'concept' in data and 'expression' in data: + item = model_map['PROGRAM'](data, index, context) + else: + item = model_map['OTHER'](data, index, context) + except Exception as e: + return (None, data, e) + + item.index = index + item.clean_subs() + item.remove_singletons() return item - class Item: - def __init__(self, data): - self.id = data['id'] if 'id' in data else "x" * 8 - - # Model - self.model = Model[data['model']] if "model" in data else Model['Other'] - # Path - self.path = podcast_url(data['path'] if "path" in data else None) - - # Sub elements + def __init__(self, data, index, context = {}): + self.id = data.get('id', "x" * 8) + self.context = context + self.model = Model[data.get('model', "Other").upper()] + url_station = "" + if data.get('format', None) == "TYPED_ELEMENT_AUTOPROMO" and context.get('station', None) is not None: + url_station = "/" + context.get('station') + if isinstance(data.get('link', None), dict) and data['link'].get('type', "") != "mail": + link = data.get('link') + self.path = podcast_url(link.get('path',link.get('url',"")), url_station) + else: + self.path = podcast_url(data.get('path', data.get('href', None))) self.subs = [] - self.elements = [] - - # Image - self.image = ( - data['visual']['src'] - if "visual" in data - and data['visual'] is not None - and "src" in data['visual'] - else None - ) - self.icon = ( - data['squaredVisual']['src'] - if "squaredVisual" in data and data['squaredVisual'] is not None - else None - ) - - # Other pages (tuple (x,n): current page x over n) + try: + self.image = get_key_src('visual', data) + self.icon = get_key_src('squaredVisual', data) + except Exception as e: + print(json.dumps(data), e) + raise (e) self.pages = (1, 1) - if "pagination" in data: - self.pages = ( - data['pagination']['pageNumber'], - data['pagination']['lastPage'] if "lastPage" in data['pagination'] else data['pagination']['pageNumber'], - ) - - # Title - self.title = ( - str(data['title']) - if "title" in data and data['title'] is not None - else None - ) + pagination = data.get('pagination', {'pageNumber': 1}) + self.pages = (pagination['pageNumber'], pagination.get('lastPage', pagination['pageNumber'])) + self.title = str(data.get('title', "")) + self.index = index + + def clean_subs(self) : + self.subs = list(filter(lambda i : i is not None, self.subs)) + + def remove_singletons(self): + if self.path is None and len(self.subs) == 1 : + new_item = create_item(self.index, self.subs[0], self.context) + if not isinstance(new_item, Item): + _, data, e = new_item + print(json.dumps(data)) + raise(e) + self = new_item + while len(self.subs) == 1 and self.subs[0] is not None: + sub_item = create_item(self.index, self.subs[0], self.context) + if not isinstance(sub_item, Item): + _, data, e = sub_item + print(json.dumps(data)) + raise(e) + self.subs = sub_item.subs if isinstance(sub_item, Item) else [] + self.index += 1 def __str__(self): - return ( - str(self.pages) - if self.pages != (1, 1) - else "" - + str(self.title) - + " [" - + str(self.model) - + "]" - + " [" - + str(len(self.elements)) - + "] (" - + str(self.path) - + ")" - + " — " - + str(self.id[:8]) - ) + return (f"{self.pages}{''.join([f'{self.index}. {self.title} [{self.model}] [{len(self.subs)}] ({self.path}) — {self.id[:8]}'])}") def is_folder(self): - return self.model in [ - Model['Theme'], - Model['Concept'], - Model['Highlight'], - Model['HighlightElement'], - Model['PageTemplate'], - Model['Tag'], - Model['Article'], - Model['Slug'], - Model['Other'], - ] + return self.model in [Model.THEME, Model.CONCEPT, Model.HIGHLIGHT, Model.HIGHLIGHTELEMENT, Model.PAGETEMPLATE, Model.TAG, Model.ARTICLE, Model.SLUG, Model.STATIONPAGE, Model.GRID, Model.SLIDER, Model.OTHER] def is_image(self): - return self.model in [Model['EmbedImage']] + return self.model in [Model.EMBED_IMAGE] def is_audio(self): return not self.is_folder() and not self.is_image() - class Event(Item): - def __init__(self, data): - super().__init__(data) + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) self.path = podcast_url(data['href']) +class StationDir(Item): + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + self.model = Model.STATIONPAGE + self.title = data['stationName'] + self.subs += [dict(data, **{'model': "Station"}), dict(data, **{'model': "StationPage"})] class Station(Item): - def __init__(self, data): - super().__init__(data) - self.model = Model['Station'] - self.title = data['stationName'] + ": " + data['now']['secondLine']['title'] + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + self.model = Model.STATION + self.title = f"{data['stationName']}: {data['now']['secondLine']['title']}" self.artists = data['stationName'] self.duration = None self.release = None - self.subs = [] self.path = data['now']['media']['sources'][0]['url'] if 0 < len(data['now']['media']['sources']) else None +class StationPage(Item): + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + self.model = Model.STATIONPAGE + self.title = data['stationName'] + self.path = podcast_url(data['stationName']) + +class Grid(Item): + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + self.title = data['metadata']['seo']['title'] + self.model = Model.GRID + self.subs = data['grid']['steps'] + +class Program(Item): + def __init__(self, data, index, context = {}): + super().__init__(data['concept'], index, context) + self.model = Model.PROGRAM + if 'expression' in data and data['expression'] is not None: + self.subs += [data['expression'] | {'model': "Expression"}] class Tag(Item): - def __init__(self, data): - super().__init__(data) + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) self.path = podcast_url(data['path']) - self.subs = data['documents']['items'] if 'documents' in data else [] - if 'documents' in data and 'pagination' in data['documents']: - self.pages = ( - data['documents']['pagination']['pageNumber'], - data['documents']['pagination']['lastPage'] if "lastPage" in data['documents']['pagination'] else data['documents']['pagination']['pageNumber'], - ) - + self.subs = data.get('documents', {'items': []})['items'] + document = data.get('documents', {}) + pagination = document.get('pagination', {'pageNumber': 1}) + self.pages = (pagination['pageNumber'], pagination.get('lastPage', pagination['pageNumber'])) class Search(Item): - def __init__(self, data): - super().__init__(data) + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) self.subs = data['items']['concepts']['contents'] + data['items']['expressions_articles']['contents'] - class Article(Item): - def __init__(self, data): - super().__init__(data) - + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) class Other(Item): - def __init__(self, data): - super().__init__(data) - - if 'link' in data and isinstance(data['link'], str) and data['link'] != "": - self.path = podcast_url(data['link']) - + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) self.subs = [] if "items" in data: if isinstance(data['items'], dict): - for k in ['concepts", "personalities", "expressions_articles']: + for k in ['concepts', 'personalities', 'expressions_articles']: if k in data['items']: self.subs += data['items'][k]['contents'] elif isinstance(data['items'], list): @@ -228,52 +265,42 @@ def __init__(self, data): else: self.subs = data['items'] if "items" in data else [] - class PageTemplate(Item): - def __init__(self, data): - super().__init__(data) - if data['model'] == Model.PageTemplate.name: - self.subs = [data['layout']] if "layout" in data else [] - + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + self.path = podcast_url(data.get('slug', self.path)) + self.title = data.get('label', self.title) + if data['model'].upper() == Model.PAGETEMPLATE.name: + self.subs = [data.get('layout', None)] class ManifestationAudio(Item): - def __init__(self, data): - super().__init__(data) - if data['model'] == Model.ManifestationAudio.name: + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + if data['model'].upper() == Model.MANIFESTATIONAUDIO.name: self.path = podcast_url(data['url']) self.duration = int(data['duration']) self.release = strftime("%d-%m.%y", localtime(data['created'])) - class Concept(Item): - def __init__(self, data): - super().__init__(data) - if data['model'] == Model.Concept.name: - if "expressions" in data: - self.subs = data['expressions']['items'] - self.pages = ( - data['expressions']['pageNumber'], - data['expressions']['lastPage'] if 'lastPage' in data['expressions'] else data['expressions']['pageNumber'], - ) - elif "promoEpisode" in data: - self.subs = data['promoEpisode']['items'] - + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + self.model = Model.CONCEPT + expressions = data.get('expressions', {'pageNumber': 1}) + self.subs = expressions.get('items', data.get('promoEpisode', {'items': []})['items']) + self.pages = (expressions['pageNumber'], expressions.get('lastPage', expressions['pageNumber'])) class Highlight(Item): - def __init__(self, data): - super().__init__(data) - if data['model'] == Model.Highlight.name: - self.subs = data['elements'] - - # Update title if necessary + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + if data['model'].upper() == Model.HIGHLIGHT.name: + self.subs = data.get('highlights') if self.title is None and len(self.subs) == 1: self.title = self.subs[0]['title'] - class HighlightElement(Item): - def __init__(self, data): - super().__init__(data) - if data['model'] == Model.HighlightElement.name: + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + if data['model'].upper() == Model.HIGHLIGHTELEMENT.name: if 0 < len(data['links']): url = data['links'][0]['path'] if data['links'][0]['type'] == "path": @@ -281,28 +308,22 @@ def __init__(self, data): self.path = podcast_url(url, local_link) else: self.path = podcast_url(url) - self.subs = data['contents'] - self.image = ( - data['mainImage']['src'] if data['mainImage'] is not None else None - ) - - # Update title if necessary + self.subs = data.get('contents',[]) + self.image = data['mainImage']['src'] if data['mainImage'] is not None else None if self.title is None and len(self.subs) == 1: self.title = self.subs[0]['title'] - class Brand(Item): - def __init__(self, data): - super().__init__(data) - brand = data['slug'] if 'slug' in data else data['brand'] + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + brand = data.get('slug', data.get('brand', "franceinter")) url = podcast_url(brand.split("_")[0] + BRAND_EXTENSION + brand) - page = requests.get(url).text - data = json.loads(page) + data = fetch_data(url) - self.model = Model['Brand'] + self.model = Model.BRAND self.station = data['stationName'] self.now = data['now']['firstLine']['title'] - self.title = self.now + " (" + self.station + ")" + self.title = f"{self.now} ({self.station})" self.artists = data['now']['secondLine']['title'] self.duration = 0 try: @@ -311,7 +332,7 @@ def __init__(self, data): self.release = None self.genre = data['now']['thirdLine']['title'] self.image = None - for key in ['mainImage", "visual']: + for key in ['mainImage', 'visual']: if key in data and data[key] is not None and "src" in data[key]: self.image = data[key]['src'] self.icon = None @@ -321,43 +342,43 @@ def __init__(self, data): self.path = data['now']['media']['sources'][0]['url'] class Slug(Item): - def __init__(self, data): - super().__init__(data) - self.model = Model['Slug'] + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + self.model = Model.SLUG name = data['slug'] self.path = podcast_url(name) - self.title = data['brand'] if 'brand' in data else name - + self.title = data.get('brand', name) class Expression(Item): - def __init__(self, data): - super().__init__(data) - if data['model'] == Model.Expression.name: - self.artists = ", ".join([g['name'] for g in (data['guest'] if "guest" in data else [])]) - self.release = strftime("%d-%m.%y", localtime(data['publishedDate'])) if "publishedDate" in data else "" - self.duration = 0 - manifestations_audio = list([d for d in data['manifestations'] if d['model'] == "ManifestationAudio"]) - if 0 < len(manifestations_audio): - manifestation = create_item( - next(filter(lambda d: d['principal'], manifestations_audio), data['manifestations'][0]) - ) - self.duration = manifestation.duration - self.path = podcast_url(manifestation.path) - + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + self.model = Model.EXPRESSION + self.artists = ", ".join([g['name'] for g in (data['guest'] if "guest" in data else [])]) + self.release = strftime("%d-%m.%y", localtime(data['publishedDate'])) if "publishedDate" in data else "" + self.duration = 0 + manifestations_audio = list([d for d in data.get('manifestations', []) if d['model'] == "ManifestationAudio"]) + if 0 < len(manifestations_audio): + manifestation = create_item(self.index, next(filter(lambda d: d['principal'], manifestations_audio), data['manifestations'][0]), self.context) + self.duration = manifestation.duration + self.path = podcast_url(manifestation.path) + +class Slider(Item): + def __init__(self, data, index, context = {}): + super().__init__(data, index, context) + self.model = model.SLIDER + self.subs = data.get('items', []) class Theme(Item): - None - + pass class EmbedImage(Item): - None - + pass class BrandPage: - def __init__(self, data): + def __init__(self, data, index): self.title = data['stationName'] self.image = None - for key in ['mainImage", "visual']: + for key in ['mainImage', 'visual']: if key in data and data[key] is not None and "src" in data[key]: self.image = data[key]['src'] self.icon = None @@ -366,9 +387,7 @@ def __init__(self, data): self.icon = data[key]['src'] self.path = data['now']['media']['sources'][0]['url'] - def expand_json(data): - parsed = json.loads(data)['nodes'][-1]['data'] def expand_element(e): if isinstance(e, dict): @@ -392,51 +411,83 @@ def expand_tuple(element): def expand_dict(element): return {k: expand_element(v) for k, v in list(element.items())} - return expand_element(parsed[0]) + nodes = json.loads(data)['nodes'] + expanded = {} + for node in nodes[::-1]: + if 'data' in node: + parsed = node['data'] + expanded = expand_element(parsed[0]) + if expanded.get('content', expanded.get('metadata', None)) is not None : + break + else: + print(json.dumps(expanded)) + return expanded def podcast_url(url, local=""): if url is None: return None - print(url) - return RADIOFRANCE_PAGE + local + "/" + url if url[:8] != "https://" else "" + url + return (RADIOFRANCE_PAGE \ + + local \ + + ("/" if 0 < len(url) and url[0] != "/" else "" ) \ + + url) \ + if url[:8] != "https://" else url - -# From plugin.video.orange.fr by f-lawe (https://github.com/f-lawe/plugin.video.orange.fr/) def localize(string_id: int, **kwargs) -> str: - """Return the translated string from the .po language files, optionally translating variables.""" import xbmcaddon - ADDON = xbmcaddon.Addon() if not isinstance(string_id, int) and not string_id.isdecimal(): return string_id return ADDON.getLocalizedString(string_id) - def build_url(query): base_url = sys.argv[0] url = base_url + "?" + urlencode(query, quote_via=quote_plus) return url +def combine(l): + while True: + try: + yield [next(a) for a in l] + except StopIteration: + break if __name__ == "__main__": data = sys.stdin.read() data = expand_json(data) # print(json.dumps(data)) - # exit() + # exit(0) + + def repeat(item): + while True: + yield item + + def display(item): + if isinstance(item, Item): + if len(item.subs) != 0 or (item.path is not None and item.path != ""): + print(item) + if len(item.subs) == 1: + display(create_item(0, item.subs[0], context)) + else: + (_, data, e) = item + print(f"Error : {e} on {json.dumps(data)}") + raise e item = create_item_from_page(data) - print(str(item)) - + context = data.get('context', {}) subs = item.subs - while 1 < len(sys.argv): index = int(sys.argv.pop()) - print("Using index: " + str(index)) - subs = create_item(subs[index]).elements - - for data in subs: - sub_item = create_item(data) - if len(sub_item.subs) == 0 and (sub_item.path is None or sub_item.path == ""): - continue - print(str(sub_item)) + print(f"Using index: {index}") + sub_item = create_item(0, subs[index], context) + if not isinstance(sub_item, Item): + (_, data, e) = sub_item + print(f"Error : {e} on {json.dumps(data)}") + raise e + subs = sub_item.subs + + # print(json.dumps(subs)) + display(item) + with ThreadPoolExecutor() as p: + sub_items = list(p.map(create_item, itertools.count(), iter(subs), repeat(context))) + list(map(display, sub_items))