From 4e8fd693a6fbce3ab25472a92b03d7cd73769ef2 Mon Sep 17 00:00:00 2001 From: DYefremov Date: Mon, 2 Nov 2020 21:55:34 +0300 Subject: [PATCH] added web import for services (#16) --- app/tools/satellites.py | 283 ++++++++++++++++++++++-- app/ui/main_app_window.py | 29 ++- app/ui/main_window.glade | 28 ++- app/ui/satellites_dialog.glade | 390 ++++++++++++++++++++++++++------- app/ui/satellites_dialog.py | 389 +++++++++++++++++++++++++------- 5 files changed, 929 insertions(+), 190 deletions(-) diff --git a/app/tools/satellites.py b/app/tools/satellites.py index f0ca2475..b2098ce4 100644 --- a/app/tools/satellites.py +++ b/app/tools/satellites.py @@ -1,15 +1,20 @@ -""" Module for download satellites from internet ("flysat.com") - for replace or update current satellites.xml file. +""" Module for downloading satellites, transponders ans services from the web. + + Sources: www.flysat.com, www.lyngsat.com. + Replaces or updates the current satellites.xml file. """ import re - -import requests from enum import Enum from html.parser import HTMLParser +import requests + from app.commons import log from app.eparser import Satellite, Transponder, is_transponder_valid -from app.eparser.ecommons import PLS_MODE +from app.eparser.ecommons import (PLS_MODE, get_key_by_value, FEC, SYSTEM, POLARIZATION, MODULATION, SERVICE_TYPE, + Service, CAS) + +_HEADERS = {"User-Agent": "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/69.0"} class SatelliteSource(Enum): @@ -22,11 +27,55 @@ class SatelliteSource(Enum): return src.value +class Cell: + """ Cell representation for table parsers. """ + __slots__ = ["_text", "_url", "_img"] + + def __init__(self, text=None, link=None, img=None): + self._text = text + self._url = link + self._img = img + + def __repr__(self): + return "Cell({}, {}, {})".format(self._text, self._url, self._img) + + def __str__(self): + return "".format(self._text, self._url, self._img) + + def __iter__(self): + return (x for x in (self._text, self._url, self._img)) + + def __len__(self): + return 3 + + @property + def text(self): + return self._text + + @text.setter + def text(self, value): + self._text = value + + @property + def url(self): + return self._url + + @url.setter + def url(self, value): + self._url = value + + @property + def img(self): + return self._img + + @img.setter + def img(self, value): + self._img = value + + class SatellitesParser(HTMLParser): """ Parser for satellite html page. """ - _HEADERS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:45.0) Gecko/20100101 Firefox/59.02"} - def __init__(self, source=SatelliteSource.FLYSAT, entities=False, separator=' '): HTMLParser.__init__(self) @@ -42,9 +91,9 @@ class SatellitesParser(HTMLParser): self._source = source def handle_starttag(self, tag, attrs): - if tag == 'td': + if tag == "td": self._is_td = True - if tag == 'tr': + if tag == "tr": self._is_th = True if tag == "a": self._current_row.append(attrs[0][1]) @@ -55,16 +104,16 @@ class SatellitesParser(HTMLParser): self._current_cell.append(data.strip()) def handle_endtag(self, tag): - if tag == 'td': + if tag == "td": self._is_td = False - elif tag == 'tr': + elif tag == "tr": self._is_th = False - if tag in ('td', 'th'): + if tag in ("td", "th"): final_cell = self._separator.join(self._current_cell).strip() self._current_row.append(final_cell) self._current_cell = [] - elif tag == 'tr': + elif tag == "tr": row = self._current_row self._rows.append(row) self._current_row = [] @@ -80,7 +129,7 @@ class SatellitesParser(HTMLParser): for src in SatelliteSource.get_sources(self._source): try: - request = requests.get(url=src, headers=self._HEADERS) + request = requests.get(url=src, headers=_HEADERS) except requests.exceptions.ConnectionError as e: log(repr(e)) return [] @@ -98,17 +147,24 @@ class SatellitesParser(HTMLParser): return list(map(get_sat, filter(lambda x: all(x) and len(x) == 5, self._rows))) elif self._source is SatelliteSource.LYNGSAT: - extra_pattern = re.compile("^https://www\.lyngsat\.com/[\w-]+\.html") + extra_pattern = re.compile(r"^https://www\.lyngsat\.com/[\w-]+\.html") base_url = "https://www.lyngsat.com/" sats = [] + names = set() current_pos = "0" for row in filter(lambda x: len(x) in (5, 7, 8), self._rows): r_len = len(row) if r_len == 7: current_pos = self.parse_position(row[2]) name = row[1].rsplit("/")[-1].rstrip(".html").replace("-", " ") - sats.append((name, current_pos, row[5], base_url + row[1], False)) # [all in one] satellites - sats.append((row[4], current_pos, row[5], base_url + row[3], False)) + if name not in names: + # [all in one] satellites + sats.append((name, current_pos, row[5], base_url + row[1], False)) + names.add(name) + name = row[4] + if name not in names: + sats.append((name, current_pos, row[5], base_url + row[3], False)) + names.add(name) if r_len == 8: # for a very limited number of satellites data = list(filter(None, row)) urls = set() @@ -146,7 +202,7 @@ class SatellitesParser(HTMLParser): """ Getting transponders(sorted by frequency). """ self._rows.clear() url = "https://www.flysat.com/" + sat_url if self._source is SatelliteSource.FLYSAT else sat_url - request = requests.get(url=url, headers=self._HEADERS) + request = requests.get(url=url, headers=_HEADERS) reason = request.reason trs = [] if reason == "OK": @@ -247,5 +303,196 @@ class SatellitesParser(HTMLParser): trs.append(tr) +class ServicesParser(HTMLParser): + """ Services parser for LYNGSAT source. """ + + def __init__(self, source=SatelliteSource.LYNGSAT, entities=False, separator=' '): + + HTMLParser.__init__(self) + + self._S_TYPES = {"": "2", "MPEG-2 SD": "1", "MPEG-4 SD": "22", "MPEG-4 HD": "25", "HEVC UHD": "31"} + self._TR_PAT = re.compile(r"(DVB-S[2]?)/?(.*PSK)?\s+SR\s+(\d+)\s+FEC\s+(\d/\d)\s+ONID/TID:\s+(\d+)/(\d+)\s+.*") + self._PTR_PAT = re.compile(r".*?(\d+\.\d°[EW]):\s+(\d+)\s+([RLHV]).*") + self._TR = "s {}000:{}000:{}:{}:{}:{}:{}:{}" + self._S2_TR = "{}:{}:{}:{}" + + self._parse_html_entities = entities + self._separator = separator + self._is_td = False + self._is_th = False + self._current_row = [] + self._current_cell_text = [] + self._current_cell = Cell() + self._rows = [] + self._source = source + + def handle_starttag(self, tag, attrs): + if tag == "td": + self._is_td = True + elif tag == "tr": + self._is_th = True + elif tag == "a" and not self._current_cell.url: + self._current_cell.url = attrs[0][1] + elif tag == "img": + img_link = attrs[0][1] + if img_link.startswith("/logo/"): + self._current_cell.img = img_link + + def handle_data(self, data): + """ Save content to a cell """ + if self._is_td or self._is_th: + self._current_cell_text.append(data.strip()) + + def handle_endtag(self, tag): + if tag == "td": + self._is_td = False + elif tag == "tr": + self._is_th = False + + if tag in ("td", "th"): + final_cell = self._separator.join(self._current_cell_text).strip() + self._current_cell.text = final_cell + self._current_row.append(self._current_cell) + self._current_cell_text = [] + self._current_cell = Cell() + elif tag == "tr": + row = self._current_row + self._rows.append(row) + self._current_row = [] + + def error(self, message): + log("ServicesParser error: {}".format(message)) + + def init_data(self, url): + """ Initializes data for the given URL. """ + if self._source is not SatelliteSource.LYNGSAT: + raise ValueError("Unsupported source: {}!".format(self._source.name)) + + self._rows.clear() + request = requests.get(url=url, headers=_HEADERS) + reason = request.reason + + if reason == "OK": + self.feed(request.text) + else: + raise ValueError(reason) + + def get_transponders_links(self, sat_url): + """ Returns transponder links. """ + try: + self.init_data(sat_url) + except ValueError as e: + log(e) + else: + url = "https://www.lyngsat.com/muxes/" + return [row[1] for row in + filter(lambda x: x and len(x) > 8 and x[1].url and x[1].url.startswith(url), self._rows)] + return [] + + def get_transponder_services(self, tr_url, sat_position=None, use_pids=False): + """ Returns services for given transponder. + + @param tr_url: transponder URL. + @param sat_position: custom satellite position. Sometimes required to adjust the namespace. + @param use_pids: if possible use additional pids [video, audio]. + """ + services = [] + try: + self.init_data(tr_url) + except ValueError as e: + log(e) + else: + pos, freq, sr, fec, pol, namespace, tid, nid = sat_position or 0, 0, 0, 0, 0, 0, 0, 0 + sys = "DVB-S" + tr_found = False + pos_found = False + tr = None + # Transponder + for r in filter(lambda x: x and len(x) == 2, self._rows): + if not pos_found: + pos_tr = re.match(self._PTR_PAT, r[1].text) + if pos_tr: + if not sat_position: + pos = int(SatellitesParser.get_position( + "".join(c for c in pos_tr.group(1) if c.isdigit() or c.isalpha()))) + freq = int(pos_tr.group(2)) + pol = get_key_by_value(POLARIZATION, pos_tr.group(3)) + pos_found = True + + if pos_found and not tr_found: + td = re.match(self._TR_PAT, r[1].text) or re.match(self._TR_PAT, r[0].text) + if td: + sys, mod, sr, _fec, nid, tid = td.group(1), td.group(2), td.group(3), td.group(4), td.group( + 5), td.group(6) + neg_pos = False # POS = W + # For negative (West) positions: 3600 - numeric position value!!! + namespace = "{:04x}0000".format(3600 - pos if neg_pos else pos) + inv = 2 # Default + fec = get_key_by_value(FEC, _fec) + sys = get_key_by_value(SYSTEM, sys) + tr_flag = 1 + mod = get_key_by_value(MODULATION, mod) + roll_off = 0 # 35% DVB-S2/DVB-S (default) + pilot = 2 # Auto + s2_flags = "" if sys == "DVB-S" else self._S2_TR.format(tr_flag, mod or 0, roll_off, pilot) + nid, tid = int(nid), int(tid) + tr = self._TR.format(freq, sr, pol, fec, pos, inv, sys, s2_flags) + tr_found = True + + if not tr: + msg = "ServicesParser error [get transponder services]: {}" + er = "Transponder [{}] not found or its type [T2-MI, etc] not supported yet.".format(freq) + log(msg.format(er)) + return [] + + # Services + for r in filter(lambda x: x and len(x) == 12 and (x[0].text.isdigit()), self._rows): + sid, name, cas, pkg, s_type, v_pid, a_pid = r[0].text, r[2].text, r[4].text, r[5].text, r[ + 6].text.strip(), r[7].text, r[8].text.split() + + try: + s_type = self._S_TYPES.get(s_type, "3") # 3 = Data + _s_type = SERVICE_TYPE.get(s_type, SERVICE_TYPE.get("3")) # str repr + sid = int(sid) + data_id = "{:04x}:{}:{:04x}:{:04x}:{}:0:0".format(sid, namespace, tid, nid, s_type) + fav_id = "{}:{}:{}:{}".format(sid, tid, nid, namespace) + picon_id = "1_0_{:X}_{}_{}_{}_{}_0_0_0.png".format(int(s_type), sid, tid, nid, namespace) + # Flags. + flags = "p:{}".format(pkg) + cas = ",".join(get_key_by_value(CAS, c) or "C:0000" for c in cas.split()) if cas else None + if use_pids: + v_pid = "c:00{:04x}".format(int(v_pid)) if v_pid else None + a_pid = ",".join(["c:01{:04x}".format(int(p)) for p in a_pid]) if a_pid else None + flags = ",".join(filter(None, (flags, v_pid, a_pid, cas))) + else: + flags = ",".join(filter(None, (flags, cas))) + + srv = Service(flags_cas=flags, + transponder_type="s", + coded=None, + service=name, + locked=None, + hide=None, + package=pkg, + service_type=_s_type, + picon=r[1].img, + picon_id=picon_id, + ssid=sid, + freq=freq, + rate=sr, + pol=pol, + fec=fec, + system=sys, + pos=pos, + data_id=data_id, + fav_id=fav_id, + transponder=tr) + services.append(srv) + except ValueError as e: + log("ServicesParser error [get transponder services]: {}".format(e)) + + return services + + if __name__ == "__main__": pass diff --git a/app/ui/main_app_window.py b/app/ui/main_app_window.py index fc39abec..370fcd7f 100644 --- a/app/ui/main_app_window.py +++ b/app/ui/main_app_window.py @@ -31,7 +31,7 @@ from .main_helper import (insert_marker, move_items, rename, ViewTarget, set_fla remove_picon, is_only_one_item_selected, gen_bouquets, BqGenType, get_iptv_url, append_picons, get_selection, get_model_data, remove_all_unused_picons, get_picon_pixbuf, get_base_itrs) from .picons_manager import PiconsDialog -from .satellites_dialog import show_satellites_dialog +from .satellites_dialog import show_satellites_dialog, ServicesUpdateDialog from .search import SearchProvider from .service_details_dialog import ServiceDetailsDialog, Action from .settings_dialog import show_settings_dialog @@ -355,6 +355,7 @@ class Application(Gtk.Application): set_action("on_download", self.on_download) set_action("on_data_open", self.on_data_open) set_action("on_archive_open", self.on_archive_open) + set_action("on_import_from_web", self.on_import_from_web) # Search, Filter search_action = Gio.SimpleAction.new_stateful("search", None, GLib.Variant.new_boolean(False)) search_action.connect("change-state", self.on_search_toggled) @@ -1666,6 +1667,8 @@ class Application(Gtk.Application): self._bouquets_model.append(None, ["Providers", None, None, BqType.BOUQUET.value]) self._bouquets_model.append(None, ["FAV", None, None, BqType.TV.value]) self._bouquets_model.append(None, ["WEBTV", None, None, BqType.WEBTV.value]) + + self._data_hash = self.get_data_hash() yield True def on_services_selection(self, model, path, column): @@ -2178,6 +2181,30 @@ class Application(Gtk.Application): callback() self._wait_dialog.hide() + def on_import_from_web(self, action, value=None): + if self._s_type is not SettingsType.ENIGMA_2: + self.show_error_dialog("Not allowed in this context!") + return + ServicesUpdateDialog(self._main_window, self._settings, self.on_import_data_from_web).show() + + @run_idle + def on_import_data_from_web(self, services): + msg = "Combine with the current data?" + if len(self._services_model) > 0 and show_dialog(DialogType.QUESTION, self._main_window, + msg) == Gtk.ResponseType.OK: + gen = self.append_imported_data([], services) + GLib.idle_add(lambda: next(gen, False), priority=GLib.PRIORITY_LOW) + else: + gen = self.import_data_from_web(services) + GLib.idle_add(lambda: next(gen, False), priority=GLib.PRIORITY_LOW) + + def import_data_from_web(self, services): + self._wait_dialog.show() + if self._app_info_box.get_visible(): + yield from self.create_new_configuration(self._s_type) + yield from self.append_services(services) + self._wait_dialog.hide() + # ***************** Backup ********************# def on_backup_tool_show(self, action, value=None): diff --git a/app/ui/main_window.glade b/app/ui/main_window.glade index e64ce903..2358782c 100644 --- a/app/ui/main_window.glade +++ b/app/ui/main_window.glade @@ -434,6 +434,20 @@ Author: Dmitriy Yefremov 0 + + + True + True + True + app.on_import_from_web + Import from Web + + + False + True + 1 + + True @@ -445,7 +459,7 @@ Author: Dmitriy Yefremov False True - 1 + 2 @@ -456,7 +470,7 @@ Author: Dmitriy Yefremov False True - 2 + 3 @@ -470,7 +484,7 @@ Author: Dmitriy Yefremov False True - 3 + 4 @@ -484,7 +498,7 @@ Author: Dmitriy Yefremov False True - 4 + 5 @@ -3358,8 +3372,6 @@ Author: Dmitriy Yefremov False True Screenshot - center - center screenshots_menu @@ -3394,6 +3406,10 @@ Author: Dmitriy Yefremov True False + 5 + 5 + 5 + 5 vertical diff --git a/app/ui/satellites_dialog.glade b/app/ui/satellites_dialog.glade index 15d10ff7..1e83d9cf 100644 --- a/app/ui/satellites_dialog.glade +++ b/app/ui/satellites_dialog.glade @@ -1328,7 +1328,35 @@ Author: Dmitriy Yefremov + + + + + + + + + + + + + + + + + + + + + + + + + + + 480 + 400 False True center-on-parent @@ -1338,7 +1366,7 @@ Author: Dmitriy Yefremov center - + True False Satellites update @@ -1398,6 +1426,7 @@ Author: Dmitriy Yefremov gtk-refresh + False @@ -1412,8 +1441,8 @@ Author: Dmitriy Yefremov True False - - True + + False True True Cancel @@ -1434,13 +1463,14 @@ Author: Dmitriy Yefremov - + True + False True True Receive True - + True @@ -1604,7 +1634,7 @@ Author: Dmitriy Yefremov False True - 1 + 0 @@ -1727,97 +1757,290 @@ Author: Dmitriy Yefremov False True - 2 + 1 - - 480 - 320 + True True - 2 - in + vertical + True - + True True - update_sat_list_model_sort - - - - - multiple - - + 2 + in - - Satellite - True - True - 0 - - - - 0 - - - - - - - Position - True - 1 - - - - 1 - - - - - - - Type - True - 2 - - - - 2 - - - - - - - False - Url - - - - 3 - - - - - - - Selected - True - 4 - - - + + True + True + update_sat_list_model_sort + True + + + + + multiple + + + + + Satellite + True + 0.5 + True + 0 + + + 0.01 + + + 0 + + + + + + + Position + 0.5 + True + 1 + + + 0.49 + + + 1 + + + + + + + Type + 0.5 + True + 2 + + + 0.49 + + + 2 + + + + + + + False + Url + + + + 3 + + + + + + + Selected + True + 4 + + + + + + 4 + + - - 4 - + + True + True + + + + + True + True + + + True + True + in + + + True + True + update_transponder_store + True + + + + + + Transponder + True + 0.5 + + + 0.01 + + + 0 + + + + + + + False + link + + + + 1 + + + + + + + Selected + + + + + + 2 + + + + + + + + + True + True + + + + + True + True + in + + + True + True + update_service_store + + + + + + Service + True + 0.5 + + + 2 + + + 0 + + + + + 0.01 + + + 1 + + + + + + + Package + True + 0.5 + + + end + + + 2 + + + + + + + Type + True + 0.5 + + + 0.49 + + + 3 + + + + + + + SID + True + 0.5 + + + 0.49 + + + 4 + + + + + + + CAS + True + 0.5 + + + 0.49 + + + + + + + + + True + True + + + + + True + True + @@ -1834,7 +2057,6 @@ Author: Dmitriy Yefremov 1 2 2 - True 120 diff --git a/app/ui/satellites_dialog.py b/app/ui/satellites_dialog.py index 5c9fa190..0a4d8bd2 100644 --- a/app/ui/satellites_dialog.py +++ b/app/ui/satellites_dialog.py @@ -1,14 +1,14 @@ +import concurrent.futures import re import time -import concurrent.futures from math import fabs from gi.repository import GLib -from app.commons import run_idle, run_task +from app.commons import run_idle, run_task, log from app.eparser import get_satellites, write_satellites, Satellite, Transponder from app.eparser.ecommons import PLS_MODE, get_key_by_value -from app.tools.satellites import SatellitesParser, SatelliteSource +from app.tools.satellites import SatellitesParser, SatelliteSource, ServicesParser from .dialogs import show_dialog, DialogType, get_dialogs_string, get_chooser_dialog from .main_helper import move_items, scroll_to, append_text_to_tview, get_base_model, on_popup_menu from .search import SearchProvider @@ -279,7 +279,7 @@ class SatellitesDialog: @run_idle def on_update(self, item): - SatellitesUpdateDialog(self._window, self._sat_view.get_model()).show() + SatellitesUpdateDialog(self._window, self._settings, self._sat_view.get_model()).show() @staticmethod def parse_data(model, path, itr, sats): @@ -326,7 +326,7 @@ class TransponderDialog: self._pls_code_entry = builder.get_object("pls_code_entry") self._is_id_entry = builder.get_object("is_id_entry") # pattern for frequency and rate entries (only digits) - self._pattern = re.compile("\D") + self._pattern = re.compile(r"\D") # style self._style_provider = Gtk.CssProvider() self._style_provider.load_from_path(UI_RESOURCES_PATH + "style.css") @@ -428,16 +428,17 @@ class SatelliteDialog: return Satellite(name=name, flags="0", position=pos, transponders=None) -# ***************** Satellite update dialog *******************# +# ********************** Update dialogs ************************ # -class SatellitesUpdateDialog: - """ Dialog for update satellites over internet """ +class UpdateDialog: + """ Base dialog for update satellites, transponders and services from the web.""" - def __init__(self, transient, main_model): + def __init__(self, transient, settings, title=None): handlers = {"on_update_satellites_list": self.on_update_satellites_list, - "on_receive_satellites_list": self.on_receive_satellites_list, + "on_receive_data": self.on_receive_data, "on_cancel_receive": self.on_cancel_receive, - "on_selected_toggled": self.on_selected_toggled, + "on_satellite_toggled": self.on_satellite_toggled, + "on_transponder_toggled": self.on_transponder_toggled, "on_info_bar_close": self.on_info_bar_close, "on_filter_toggled": self.on_filter_toggled, "on_find_toggled": self.on_find_toggled, @@ -450,26 +451,36 @@ class SatellitesUpdateDialog: "on_search_up": self.on_search_up, "on_quit": self.on_quit} + self._settings = settings + self._download_task = False + self._parser = None + self._size_name = "{}_window_size".format("_".join(re.findall("[A-Z][^A-Z]*", self.__class__.__name__))).lower() + builder = Gtk.Builder() builder.set_translation_domain(TEXT_DOMAIN) builder.add_objects_from_file(UI_RESOURCES_PATH + "satellites_dialog.glade", ("satellites_update_window", "update_source_store", "update_sat_list_store", "update_sat_list_model_filter", "update_sat_list_model_sort", "side_store", "pos_adjustment", "pos_adjustment2", "satellites_update_popup_menu", - "remove_selection_image")) + "remove_selection_image", "update_transponder_store", "update_service_store")) builder.connect_signals(handlers) self._window = builder.get_object("satellites_update_window") self._window.set_transient_for(transient) - self._main_model = main_model - # self._dialog.get_content_area().set_border_width(0) + if title: + self._window.set_title(title) + + self._transponder_paned = builder.get_object("sat_update_tr_paned") self._sat_view = builder.get_object("sat_update_tree_view") + self._transponder_view = builder.get_object("sat_update_tr_view") + self._service_view = builder.get_object("sat_update_srv_view") self._source_box = builder.get_object("source_combo_box") self._sat_update_expander = builder.get_object("sat_update_expander") self._text_view = builder.get_object("text_view") - self._receive_button = builder.get_object("receive_sat_list_tool_button") + self._receive_button = builder.get_object("receive_data_button") self._sat_update_info_bar = builder.get_object("sat_update_info_bar") self._info_bar_message_label = builder.get_object("info_bar_message_label") + self._receive_button.bind_property("visible", builder.get_object("cancel_data_button"), "visible", 4) # Filter self._filter_bar = builder.get_object("sat_update_filter_bar") self._from_pos_button = builder.get_object("from_pos_button") @@ -485,21 +496,31 @@ class SatellitesUpdateDialog: builder.get_object("sat_update_search_down_button"), builder.get_object("sat_update_search_up_button")) - self._download_task = False - self._parser = None + window_size = self._settings.get(self._size_name) + if window_size: + self._window.resize(*window_size) def show(self): self._window.show() + @property + def is_download(self): + return self._download_task + + @is_download.setter + def is_download(self, value): + self._download_task = value + self._receive_button.set_visible(not value) + @run_idle def on_update_satellites_list(self, item): - if self._download_task: + if self.is_download: show_dialog(DialogType.ERROR, self._window, "The task is already running!") return model = get_base_model(self._sat_view.get_model()) model.clear() - self._download_task = True + self.is_download = True src = self._source_box.get_active() if not self._parser: self._parser = SatellitesParser() @@ -511,7 +532,7 @@ class SatellitesUpdateDialog: sats = self._parser.get_satellites_list(SatelliteSource.FLYSAT if src == 0 else SatelliteSource.LYNGSAT) if sats: callback(sats) - self._download_task = False + self.is_download = False @run_idle def append_satellites(self, sats): @@ -520,70 +541,16 @@ class SatellitesUpdateDialog: model.append(sat) @run_idle - def on_receive_satellites_list(self, item): - if self._download_task: + def on_receive_data(self, item): + if self.is_download: show_dialog(DialogType.ERROR, self._window, "The task is already running!") return - self.receive_satellites() - - @run_task - def receive_satellites(self): - self._download_task = True - self.update_expander() - model = self._sat_view.get_model() - start = time.time() - - with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor: - text = "Processing: {}\n" - sats = [] - appender = self.append_output() - next(appender) - futures = {executor.submit(self._parser.get_satellite, sat[:-1]): sat for sat in [r for r in model if r[4]]} - for future in concurrent.futures.as_completed(futures): - if not self._download_task: - self._download_task = True - executor.shutdown() - appender.send("\nCanceled\n") - appender.close() - self._download_task = False - return - data = future.result() - appender.send(text.format(data[0])) - sats.append(data) - - appender.send("-" * 75 + "\n") - appender.send("Consumed : {:0.0f}s, {} satellites received.".format(start - time.time(), len(sats))) - appender.close() - - sats = {s[2]: s for s in sats} # key = position, v = satellite - - for row in self._main_model: - pos = row[-1] - if pos in sats: - sat = sats.pop(pos) - itr = row.iter - self.update_satellite(itr, row, sat) - - for sat in sats.values(): - append_satellite(self._main_model, sat) - - self._download_task = False @run_idle def update_expander(self): self._sat_update_expander.set_expanded(True) self._text_view.get_buffer().set_text("", 0) - @run_idle - def update_satellite(self, itr, row, sat): - if self._main_model.iter_has_child(itr): - children = row.iterchildren() - for ch in children: - self._main_model.remove(ch.iter) - - for tr in sat[3]: - self._main_model.append(itr, ["Transponder:", *tr, None, None]) - def append_output(self): @run_idle def append(t): @@ -596,11 +563,15 @@ class SatellitesUpdateDialog: def on_cancel_receive(self, item=None): self._download_task = False - def on_selected_toggled(self, toggle, path): + def on_satellite_toggled(self, toggle, path): model = self._sat_view.get_model() self.update_state(model, path, not toggle.get_active()) self.update_receive_button_state(self._filter_model) + def on_transponder_toggled(self, toggle, path): + model = self._transponder_view.get_model() + model.set_value(model.get_iter(path), 2, not toggle.get_active()) + @run_idle def update_receive_button_state(self, model): self._receive_button.set_sensitive((any(r[4] for r in model))) @@ -625,7 +596,7 @@ class SatellitesUpdateDialog: self._filter_positions = self.get_positions() self._filter_model.refilter() - def filter_function(self, model, iter, data): + def filter_function(self, model, itr, data): if self._filter_model is None or self._filter_model == "None": return True @@ -636,7 +607,7 @@ class SatellitesUpdateDialog: if from_pos > to_pos: from_pos, to_pos = to_pos, from_pos - return from_pos <= float(self._parser.get_position(model.get(iter, 1)[0])) <= to_pos + return from_pos <= float(self._parser.get_position(model.get(itr, 1)[0])) <= to_pos def get_positions(self): from_pos = round(self._from_pos_button.get_value(), 1) * (-1 if self._filter_from_combo_box.get_active() else 1) @@ -669,10 +640,266 @@ class SatellitesUpdateDialog: self._filter_model.get_model().set_value(itr, 4, select) def on_quit(self, window, event): - self._download_task = False + self._settings.add(self._size_name, window.get_size()) + self.is_download = False -# ***************** Commons *******************# +class SatellitesUpdateDialog(UpdateDialog): + """ Dialog for update satellites from the web. """ + + def __init__(self, transient, settings, main_model): + super().__init__(transient=transient, settings=settings) + + self._main_model = main_model + + @run_idle + def on_receive_data(self, item): + if self.is_download: + show_dialog(DialogType.ERROR, self._window, "The task is already running!") + return + + self.receive_satellites() + + @run_task + def receive_satellites(self): + self.is_download = True + self.update_expander() + model = self._sat_view.get_model() + start = time.time() + + with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor: + text = "Processing: {}\n" + sats = [] + appender = self.append_output() + next(appender) + futures = {executor.submit(self._parser.get_satellite, sat[:-1]): sat for sat in [r for r in model if r[4]]} + for future in concurrent.futures.as_completed(futures): + if not self.is_download: + self.is_download = True + executor.shutdown() + appender.send("\nCanceled\n") + appender.close() + self.is_download = False + return + data = future.result() + appender.send(text.format(data[0])) + sats.append(data) + + appender.send("-" * 75 + "\n") + appender.send("Consumed: {:0.0f}s, {} satellites received.".format(time.time() - start, len(sats))) + appender.close() + + sats = {s[2]: s for s in sats} # key = position, v = satellite + + for row in self._main_model: + pos = row[-1] + if pos in sats: + sat = sats.pop(pos) + itr = row.iter + self.update_satellite(itr, row, sat) + + for sat in sats.values(): + append_satellite(self._main_model, sat) + + self.is_download = False + + @run_idle + def update_satellite(self, itr, row, sat): + if self._main_model.iter_has_child(itr): + children = row.iterchildren() + for ch in children: + self._main_model.remove(ch.iter) + + for tr in sat[3]: + self._main_model.append(itr, ["Transponder:", *tr, None, None]) + + +class ServicesUpdateDialog(UpdateDialog): + """ Dialog for updating services from the web. """ + + def __init__(self, transient, settings, callback): + super().__init__(transient=transient, settings=settings, title="Services update") + + self._callback = callback + self._satellite_paths = {} + self._transponders = {} + self._services = {} + self._selected_transponders = set() + self._services_parser = ServicesParser(source=SatelliteSource.LYNGSAT) + + self._transponder_paned.set_visible(True) + s_model = self._source_box.get_model() + s_model.remove(s_model.get_iter_first()) + self._source_box.set_active(0) + self._sat_view.connect("row-activated", self.on_activate_satellite) + self._transponder_view.connect("row-activated", self.on_activate_transponder) + + @run_idle + def on_receive_data(self, item): + if self.is_download: + show_dialog(DialogType.ERROR, self._window, "The task is already running!") + return + + self.receive_services() + + @run_task + def receive_services(self): + self.is_download = True + self.update_expander() + model = self._sat_view.get_model() + appender = self.append_output() + next(appender) + + start = time.time() + non_cached_sats = [] + sat_names = {} + t_names = {} + t_urls = [] + services = [] + + for sat, url in (r[0, 3] for r in model if r[-1]): + if not self.is_download: + appender.send("\nCanceled\n") + return + + trs = self._transponders.get(url, None) + if trs: + for t in filter(lambda tp: tp.url in self._selected_transponders, trs): + t_urls.append(t.url) + t_names[t.url] = t.text + else: + non_cached_sats.append(url) + sat_names[url] = sat + + if non_cached_sats: + with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor: + futures = {executor.submit(self._services_parser.get_transponders_links, u): u for u in non_cached_sats} + for future in concurrent.futures.as_completed(futures): + if not self.is_download: + appender.send("\nCanceled.\n") + self.is_download = False + return + + appender.send("Getting transponders for: {}.\n".format(sat_names.get(futures[future]))) + for t in future.result(): + t_urls.append(t.url) + t_names[t.url] = t.text + + appender.send("-" * 75 + "\n") + appender.send("{} transponders received.\n\n".format(len(t_urls))) + + non_cached_ts = [] + for tr in t_urls: + srvs = self._services.get(tr) + services.extend(srvs) if srvs else non_cached_ts.append(tr) + + if non_cached_ts: + with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor: + futures = {executor.submit(self._services_parser.get_transponder_services, u): u for u in non_cached_ts} + for future in concurrent.futures.as_completed(futures): + if not self.is_download: + appender.send("\nCanceled.\n") + self.is_download = False + return + + appender.send("Getting services for: {}.\n".format(t_names.get(futures[future], ""))) + list(map(services.append, future.result())) + + appender.send("-" * 75 + "\n") + appender.send("Consumed: {:0.0f}s, {} services received.".format(time.time() - start, len(services))) + + try: + from app.eparser.enigma.lamedb import get_services_lines, get_services_list + # Used for double checking! + srvs = get_services_list("".join(get_services_lines(services))) + except ValueError as e: + log("ServicesUpdateDialog [on receive data] error: {}".format(e)) + else: + self._callback(srvs) + + self.is_download = False + + @run_task + def get_sat_list(self, src, callback): + sats = self._parser.get_satellites_list(SatelliteSource.LYNGSAT) + if sats: + callback(sats) + self.is_download = False + + def on_satellite_toggled(self, toggle, path): + model = self._sat_view.get_model() + self.update_state(model, path, not toggle.get_active()) + self.update_receive_button_state(self._filter_model) + + url = model.get_value(model.get_iter(path), 3) + selected = toggle.get_active() + transponders = self._transponders.get(url, None) + + if transponders: + for t in transponders: + self._selected_transponders.add(t.url) if selected else self._selected_transponders.discard(t.url) + + def on_transponder_toggled(self, toggle, path): + model = self._transponder_view.get_model() + itr = model.get_iter(path) + active = not toggle.get_active() + model.set_value(itr, 2, active) + url = model.get_value(itr, 1) + self._selected_transponders.add(url) if active else self._selected_transponders.discard(url) + + s_path = self._satellite_paths.get(url) + if s_path: + sat_model = self._sat_view.get_model() + if active: + self.update_state(sat_model, s_path, active) + else: + self.update_state(sat_model, s_path, any((r[-1] for r in model))) + self.update_receive_button_state(self._filter_model) + + @run_task + def on_activate_satellite(self, view, path, column): + url, selected = view.get_model()[path][3, 4] + transponders = self._transponders.get(url, None) + if transponders is None: + GLib.idle_add(view.set_sensitive, False) + transponders = self._services_parser.get_transponders_links(url) + self._transponders[url] = transponders + + for t in transponders: + t_url = t.url + self._satellite_paths[t_url] = path + self._selected_transponders.add(t_url) if selected else self._selected_transponders.discard(t_url) + + self.append_transponders(self._transponder_view.get_model(), transponders) + + @run_idle + def append_transponders(self, model, trs_list): + model.clear() + list(map(model.append, [(t.text, t.url, t.url in self._selected_transponders) for t in trs_list])) + self._sat_view.set_sensitive(True) + + @run_task + def on_activate_transponder(self, view, path, column): + url = view.get_model()[path][1] + services = self._services.get(url, None) + if services is None: + GLib.idle_add(view.set_sensitive, False) + services = self._services_parser.get_transponder_services(url) + self._services[url] = services + + self.append_services(self._service_view.get_model(), services) + + @run_idle + def append_services(self, model, srv_list): + model.clear() + for s in srv_list: + model.append((None, s.service, s.package, s.service_type, str(s.ssid), None)) + + self._transponder_view.set_sensitive(True) + + +# ************************* Commons ************************* # + @run_idle def append_satellite(model, sat):