From 7922f368b5c106f59cbb6eb2454dcf312a3c6579 Mon Sep 17 00:00:00 2001 From: DYefremov Date: Wed, 10 Feb 2021 23:21:30 +0300 Subject: [PATCH] refactoring of picons downloading --- app/tools/picons.py | 165 ++++++++++++++++++++++++------------ app/ui/picons_manager.glade | 2 + app/ui/picons_manager.py | 145 ++++++++++++++----------------- 3 files changed, 177 insertions(+), 135 deletions(-) diff --git a/app/tools/picons.py b/app/tools/picons.py index 54d9d175..41b9bd0b 100644 --- a/app/tools/picons.py +++ b/app/tools/picons.py @@ -5,8 +5,11 @@ import shutil from collections import namedtuple from html.parser import HTMLParser +import requests + from app.commons import run_task, log from app.settings import SettingsType +from .satellites import _HEADERS _ENIGMA2_PICON_KEY = "{:X}:{:X}:{}" _NEUTRINO_PICON_KEY = "{:x}{:04x}{:04x}.png" @@ -17,6 +20,7 @@ Picon = namedtuple("Picon", ["ref", "ssid", "v_pid"]) class PiconsParser(HTMLParser): """ Parser for package html page. (https://www.lyngsat.com/packages/*provider-name*.html) """ + _BASE_URL = "https://www.lyngsat.com" def __init__(self, entities=False, separator=' ', single=None): @@ -58,14 +62,14 @@ class PiconsParser(HTMLParser): row = self._current_row ln = len(row) - if self._single and ln == 4 and row[0].startswith("../../logo/"): - self.picons.append(Picon(row[0].strip("../"), "0", "0")) + if self._single and ln == 4 and row[0].startswith("/logo/"): + self.picons.append(Picon(row[0].strip(), "0", "0")) else: if 9 < ln < 13: url = None - if row[0].startswith("../logo/"): + if row[0].startswith("/logo/"): url = row[0] - elif row[1].startswith("../logo/"): + elif row[1].startswith("/logo/"): url = row[1] ssid = row[-4] @@ -78,37 +82,44 @@ class PiconsParser(HTMLParser): pass @staticmethod - def parse(open_path, picons_path, tmp_path, provider, picon_ids, s_type=SettingsType.ENIGMA_2): - if not os.path.isfile(open_path): - log("PiconsParser error [parse]. No such file or directory: {}".format(open_path)) + def parse(provider, picons_path, picon_ids, s_type=SettingsType.ENIGMA_2): + """ Returns tuple(url, picon file name) list. """ + req = requests.get(provider.url, timeout=5) + if req.status_code == 200: + logo_data = req.text + else: + log("Provider picons downloading error: {} {}".format(provider.url, req.reason)) return - with open(open_path, encoding="utf-8", errors="replace") as f: - on_id, pos, ssid, single = provider.on_id, provider.pos, provider.ssid, provider.single - neg_pos = pos.endswith("W") - pos = int("".join(c for c in pos if c.isdigit())) - # For negative (West) positions 3600 - numeric position value!!! - if neg_pos: - pos = 3600 - pos - parser = PiconsParser(single=single) - parser.reset() - parser.feed(f.read()) - picons = parser.picons - if picons: - os.makedirs(picons_path, exist_ok=True) - for p in picons: - try: - if single: - on_id, freq = on_id.strip().split("::") - namespace = "{:X}{:X}".format(int(pos), int(freq)) - else: - namespace = "{:X}0000".format(int(pos)) - name = PiconsParser.format(ssid if single else p.ssid, on_id, namespace, picon_ids, s_type) - p_name = picons_path + (name if name else os.path.basename(p.ref)) - shutil.copyfile(tmp_path + "www.lyngsat.com/" + p.ref.lstrip("."), p_name) - except (TypeError, ValueError) as e: - msg = "Picons format parse error: {}".format(p) + "\n" + str(e) - log(msg) + on_id, pos, ssid, single = provider.on_id, provider.pos, provider.ssid, provider.single + neg_pos = pos.endswith("W") + pos = int("".join(c for c in pos if c.isdigit())) + # For negative (West) positions 3600 - numeric position value!!! + if neg_pos: + pos = 3600 - pos + + parser = PiconsParser(single=provider.single) + parser.reset() + parser.feed(logo_data) + picons = parser.picons + picons_data = [] + + if picons: + for p in picons: + try: + if single: + on_id, freq = on_id.strip().split("::") + namespace = "{:X}{:X}".format(int(pos), int(freq)) + else: + namespace = "{:X}0000".format(int(pos)) + name = PiconsParser.format(ssid if single else p.ssid, on_id, namespace, picon_ids, s_type) + p_name = picons_path + (name if name else os.path.basename(p.ref)) + picons_data.append(("{}{}".format(PiconsParser._BASE_URL, p.ref), p_name)) + except (TypeError, ValueError) as e: + msg = "Picons format parse error: {}".format(p) + "\n" + str(e) + log(msg) + + return picons_data @staticmethod def format(ssid, on_id, namespace, picon_ids, s_type): @@ -127,7 +138,8 @@ class ProviderParser(HTMLParser): _POSITION_PATTERN = re.compile("at\s\d+\..*(?:E|W)']") _ONID_TID_PATTERN = re.compile("^\d+-\d+.*") _TRANSPONDER_FREQUENCY_PATTERN = re.compile("^\d+ [HVLR]+") - _DOMAINS = {"/tvchannels/", "/radiochannels/", "/packages/"} + _DOMAINS = {"/tvchannels/", "/radiochannels/", "/packages/", "/logo/"} + _BASE_URL = "https://www.lyngsat.com" def __init__(self, entities=False, separator=' '): @@ -155,7 +167,7 @@ class ProviderParser(HTMLParser): if tag == 'tr': self._is_th = True if tag == "img": - if attrs[0][1].startswith("logo/"): + if attrs[0][1].startswith("/logo/"): self._current_row.append(attrs[0][1]) if tag == "a": url = attrs[0][1] @@ -187,41 +199,47 @@ class ProviderParser(HTMLParser): self._current_row.append(final_cell) self._current_cell = [] elif tag == 'tr': - r = self._current_row + row = self._current_row # Satellite position if not self._positon: - pos = re.findall(self._POSITION_PATTERN, str(r)) + pos = re.findall(self._POSITION_PATTERN, str(row)) if pos: self._positon = "".join(c for c in str(pos) if c.isdigit() or c in ".EW") - len_row = len(r) + len_row = len(row) if len_row > 2: - m = self._TRANSPONDER_FREQUENCY_PATTERN.match(r[1]) + m = self._TRANSPONDER_FREQUENCY_PATTERN.match(row[1]) if m: self._freq = m.group().split()[0] - if len_row == 12: + if len_row == 14: # Providers - name = r[5] + name = row[6] self._prv_names.add(name) - m = self._ONID_TID_PATTERN.match(str(r[-2])) + m = self._ONID_TID_PATTERN.match(str(row[9])) if m: on_id, tid = m.group().split("-") if on_id not in self._ids: - r[-2] = on_id + row[-2] = on_id self._ids.add(on_id) - r[0] = self._positon + row[0] = self._positon if name + on_id not in self._prv_names: self._prv_names.add(name + on_id) - self.rows.append(Provider(logo=r[2], name=name, pos=self._positon, url=r[6], on_id=on_id, + logo_data = None + req = requests.get(self._BASE_URL + row[3], timeout=5) + if req.status_code == 200: + logo_data = req.content + else: + log("Downloading provider logo error: {}".format(req.reason)) + self.rows.append(Provider(logo=logo_data, name=name, pos=self._positon, url=row[5], on_id=on_id, ssid=None, single=False, selected=True)) - elif 6 < len_row < 10: + elif 6 < len_row < 14: # Single services name, url, ssid = None, None, None - if r[0].startswith("http"): - name, url, ssid = r[1], r[0], r[4] - elif r[1].startswith("http"): - name, url, ssid = r[2], r[1], r[5] + if row[0].startswith("http"): + name, url, ssid = row[1], row[0], row[0] + elif row[1].startswith("http"): + name, url, ssid = row[2], row[1], row[0] if name and url: on_id = "{}::{}".format(self._on_id if self._on_id else "1", self._freq) @@ -237,14 +255,51 @@ class ProviderParser(HTMLParser): super().reset() -def parse_providers(open_path): +def parse_providers(url): + """ Returns a list of providers sorted by logo [single channels after providers]. """ parser = ProviderParser() - parser.reset() - with open(open_path, encoding="utf-8", errors="replace") as f: - parser.feed(f.read()) + request = requests.get(url=url, headers=_HEADERS) + if request.status_code == 200: + parser.feed(request.text) + else: + log("Parse providers error [{}]: {}".format(url, request.reason)) - return parser.rows + def srt(p): + if p.logo is None: + return 1 + return 0 + + providers = parser.rows + providers.sort(key=srt) + + return providers + + +def download_picon(src_url, dest_path, callback): + """ Downloads and saves the picon to file. """ + err_msg = "Picon download error: {} [{}]" + timeout = (3, 5) # connect and read timeouts + + if callback: + callback("Downloading: {}.\n".format(os.path.basename(dest_path))) + + req = requests.get(src_url, timeout=timeout, stream=True) + if req.status_code != 200: + err_msg = err_msg.format(src_url, req.reason) + log(err_msg) + if callback: + callback(err_msg + "\n") + else: + try: + with open(dest_path, "wb") as f: + for chunk in req: + f.write(chunk) + except OSError as e: + err_msg = "Saving picon [{}] error: {}".format(dest_path, e) + log(err_msg) + if callback: + callback(err_msg + "\n") @run_task diff --git a/app/ui/picons_manager.glade b/app/ui/picons_manager.glade index 46d6cc32..d18378a0 100644 --- a/app/ui/picons_manager.glade +++ b/app/ui/picons_manager.glade @@ -1704,6 +1704,8 @@ Author: Dmitriy Yefremov True False word-char + 5 + 5 True diff --git a/app/ui/picons_manager.py b/app/ui/picons_manager.py index de6394dd..cc418367 100644 --- a/app/ui/picons_manager.py +++ b/app/ui/picons_manager.py @@ -1,17 +1,16 @@ import os import re import shutil -import subprocess import tempfile from pathlib import Path from urllib.parse import urlparse, unquote -from gi.repository import GLib, GdkPixbuf +from gi.repository import GLib, GdkPixbuf, Gio from app.commons import run_idle, run_task, run_with_delay from app.connections import upload_data, DownloadType, download_data, remove_picons from app.settings import SettingsType, Settings -from app.tools.picons import PiconsParser, parse_providers, Provider, convert_to +from app.tools.picons import PiconsParser, parse_providers, Provider, convert_to, download_picon from app.tools.satellites import SatellitesParser, SatelliteSource from .dialogs import show_dialog, DialogType, get_message from .main_helper import (update_entry_data, append_text_to_tview, scroll_to, on_popup_menu, get_base_model, set_picon, @@ -30,6 +29,7 @@ class PiconsDialog: self._POS_PATTERN = re.compile(r"^\d+\.\d+[EW]?$") self._current_process = None self._terminate = False + self._is_downloading = False self._filter_binding = None self._services = None self._current_picon_info = None @@ -480,7 +480,7 @@ class PiconsDialog: try: for sat in sats: pos = sat[1] - name, pos = "{} ({})".format(sat[0], pos), "{}{}".format("-" if pos[-1] == "W" else "", pos[:-1]) + name = "{} ({})".format(sat[0], pos) if not self._terminate and model: if pos in self._sat_positions: @@ -492,49 +492,28 @@ class PiconsDialog: model = view.get_model() self._url_entry.set_text(model.get(model.get_iter(path), 1)[0]) - @run_idle def on_load_providers(self, item): - self._expander.set_expanded(True) self.on_info_bar_close() - self._cancel_button.show() - url = self._url_entry.get_text() - - try: - self._current_process = subprocess.Popen(["wget", "-pkP", self._TMP_DIR, url], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - except FileNotFoundError as e: - self._cancel_button.hide() - self.show_info_message(str(e), Gtk.MessageType.ERROR) - else: - GLib.io_add_watch(self._current_process.stderr, GLib.IO_IN, self.write_to_buffer) - model = self._providers_view.get_model() - model.clear() - self.append_providers(url, model) + model = self._providers_view.get_model() + model.clear() + self.get_providers(model) @run_task - def append_providers(self, url, model): - self._current_process.wait() - try: - self._terminate = False - providers = parse_providers(self._TMP_DIR + url[url.find("w"):]) - except FileNotFoundError: - pass # NOP - else: - if providers: - for p in providers: - if self._terminate: - return - model.append((self.get_pixbuf(p[0]) if p[0] else TV_ICON, *p[1:])) - self.update_receive_button_state() - finally: - GLib.idle_add(self._cancel_button.hide) - self._terminate = False + def get_providers(self, model): + providers = parse_providers(self._url_entry.get_text()) + if providers: + self.append_providers(providers, model) - def get_pixbuf(self, img_url): - return GdkPixbuf.Pixbuf.new_from_file_at_scale(filename=self._TMP_DIR + "www.lyngsat.com/" + img_url, - width=48, height=48, preserve_aspect_ratio=True) + @run_idle + def append_providers(self, providers, model): + for p in providers: + model.append((self.get_pixbuf(p[0]) if p[0] else TV_ICON, *p[1:])) + self.update_receive_button_state() + + def get_pixbuf(self, img_data): + if img_data: + f = Gio.MemoryInputStream.new_from_data(img_data) + return GdkPixbuf.Pixbuf.new_from_stream_at_scale(f, 48, 32, True, None) def on_receive(self, item): self._cancel_button.show() @@ -542,12 +521,12 @@ class PiconsDialog: @run_task def start_download(self): - if self._current_process.poll() is None: + if self._is_downloading: self.show_dialog("The task is already running!", DialogType.ERROR) return - self._terminate = False - self._expander.set_expanded(True) + self._is_downloading = True + GLib.idle_add(self._expander.set_expanded, True) providers = self.get_selected_providers() for prv in providers: @@ -558,38 +537,47 @@ class PiconsDialog: return try: - for prv in providers: - if self._terminate: - return - self.process_provider(Provider(*prv)) + picons_path = self._picons_dir_entry.get_text() + os.makedirs(os.path.dirname(picons_path), exist_ok=True) + picons = [] + + self.show_info_message(get_message("Please, wait..."), Gtk.MessageType.INFO) + + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + # Getting links to picons. + futures = {executor.submit(self.process_provider, Provider(*p), picons_path): p for p in providers} + for future in concurrent.futures.as_completed(futures): + if not self._is_downloading: + executor.shutdown() + return + + picons.extend(future.result()) + # Getting picon images. + futures = {executor.submit(download_picon, *pic, self.append_output): pic for pic in picons} + done, not_done = concurrent.futures.wait(futures, timeout=0) + while self._is_downloading and not_done: + done, not_done = concurrent.futures.wait(not_done, timeout=5) + + for future in not_done: + future.cancel() + concurrent.futures.wait(not_done) + + if not self._is_downloading: + return if not self._resize_no_radio_button.get_active(): - self.resize(self._picons_dir_entry.get_text()) + self.resize(picons_path) else: self.show_info_message(get_message("Done!"), Gtk.MessageType.INFO) finally: GLib.idle_add(self._cancel_button.hide) - self._terminate = False + self._is_downloading = False - def process_provider(self, prv): - url = prv.url - self.show_info_message(get_message("Please, wait..."), Gtk.MessageType.INFO) - self._current_process = subprocess.Popen(["wget", "-pkP", self._TMP_DIR, url], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - GLib.io_add_watch(self._current_process.stderr, GLib.IO_IN, self.write_to_buffer) - self._current_process.wait() - path = self._TMP_DIR + (url[url.find("//") + 2:] if prv.single else self._BASE_URL + url[url.rfind("/") + 1:]) - PiconsParser.parse(path, self._picons_dir_entry.get_text(), - self._TMP_DIR, prv, self._picon_ids, self.get_picons_format()) - - def write_to_buffer(self, fd, condition): - if condition == GLib.IO_IN: - char = fd.read(1) - self.append_output(char) - return True - return False + def process_provider(self, prv, picons_path): + self.append_output("Getting links to picons for: {}.\n".format(prv.name)) + return PiconsParser.parse(prv, picons_path, self._picon_ids, self.get_picons_format()) @run_idle def append_output(self, char): @@ -615,7 +603,7 @@ class PiconsDialog: self.show_info_message(get_message("Done!"), Gtk.MessageType.INFO) def on_cancel(self, item=None): - if self.is_task_running() and show_dialog(DialogType.QUESTION, self._dialog) == Gtk.ResponseType.CANCEL: + if self._is_downloading and show_dialog(DialogType.QUESTION, self._dialog) == Gtk.ResponseType.CANCEL: return True self.terminate_task() @@ -623,16 +611,15 @@ class PiconsDialog: @run_task def terminate_task(self): self._terminate = True - - if self._current_process: - self._current_process.terminate() - self.show_info_message(get_message("The task is canceled!"), Gtk.MessageType.WARNING) + self._is_downloading = False + self.show_info_message(get_message("The task is canceled!"), Gtk.MessageType.WARNING) def on_close(self, window, event): if self.on_cancel(): return True self._terminate = True + self._is_downloading = False self.save_window_size(window) self.clean_data() self._app.update_picons() @@ -667,9 +654,10 @@ class PiconsDialog: @run_idle def show_info_message(self, text, message_type): - self._info_bar.set_visible(True) - self._info_bar.set_message_type(message_type) + self._info_bar.set_visible(False) self._message_label.set_text(get_message(text)) + self._info_bar.set_message_type(message_type) + self._info_bar.set_visible(True) def on_picons_dir_open(self, entry, icon, event_button): update_entry_data(entry, self._dialog, settings=self._settings) @@ -834,9 +822,6 @@ class PiconsDialog: return picon_format - def is_task_running(self): - return self._current_process and self._current_process.poll() is None - if __name__ == "__main__": pass