refactoring of picons downloading

app/tools/picons.py

@@ -5,8 +5,11 @@ import shutil
 from collections import namedtuple
 from html.parser import HTMLParser
 
+import requests
+
 from app.commons import run_task, log
 from app.settings import SettingsType
+from .satellites import _HEADERS
 
 _ENIGMA2_PICON_KEY = "{:X}:{:X}:{}"
 _NEUTRINO_PICON_KEY = "{:x}{:04x}{:04x}.png"
@@ -17,6 +20,7 @@ Picon = namedtuple("Picon", ["ref", "ssid", "v_pid"])
 
 class PiconsParser(HTMLParser):
     """ Parser for package html page. (https://www.lyngsat.com/packages/*provider-name*.html) """
+    _BASE_URL = "https://www.lyngsat.com"
 
     def __init__(self, entities=False, separator=' ', single=None):
 
@@ -58,14 +62,14 @@ class PiconsParser(HTMLParser):
             row = self._current_row
             ln = len(row)
 
-            if self._single and ln == 4 and row[0].startswith("../../logo/"):
-                self.picons.append(Picon(row[0].strip("../"), "0", "0"))
+            if self._single and ln == 4 and row[0].startswith("/logo/"):
+                self.picons.append(Picon(row[0].strip(), "0", "0"))
             else:
                 if 9 < ln < 13:
                     url = None
-                    if row[0].startswith("../logo/"):
+                    if row[0].startswith("/logo/"):
                         url = row[0]
-                    elif row[1].startswith("../logo/"):
+                    elif row[1].startswith("/logo/"):
                         url = row[1]
 
                     ssid = row[-4]
@@ -78,37 +82,44 @@ class PiconsParser(HTMLParser):
         pass
 
     @staticmethod
-    def parse(open_path, picons_path, tmp_path, provider, picon_ids, s_type=SettingsType.ENIGMA_2):
-        if not os.path.isfile(open_path):
-            log("PiconsParser error [parse]. No such file or directory: {}".format(open_path))
+    def parse(provider, picons_path, picon_ids, s_type=SettingsType.ENIGMA_2):
+        """ Returns tuple(url, picon file name) list. """
+        req = requests.get(provider.url, timeout=5)
+        if req.status_code == 200:
+            logo_data = req.text
+        else:
+            log("Provider picons downloading error: {} {}".format(provider.url, req.reason))
             return
 
-        with open(open_path, encoding="utf-8", errors="replace") as f:
-            on_id, pos, ssid, single = provider.on_id, provider.pos, provider.ssid, provider.single
-            neg_pos = pos.endswith("W")
-            pos = int("".join(c for c in pos if c.isdigit()))
-            # For negative (West) positions 3600 - numeric position value!!!
-            if neg_pos:
-                pos = 3600 - pos
-            parser = PiconsParser(single=single)
-            parser.reset()
-            parser.feed(f.read())
-            picons = parser.picons
-            if picons:
-                os.makedirs(picons_path, exist_ok=True)
-                for p in picons:
-                    try:
-                        if single:
-                            on_id, freq = on_id.strip().split("::")
-                            namespace = "{:X}{:X}".format(int(pos), int(freq))
-                        else:
-                            namespace = "{:X}0000".format(int(pos))
-                        name = PiconsParser.format(ssid if single else p.ssid, on_id, namespace, picon_ids, s_type)
-                        p_name = picons_path + (name if name else os.path.basename(p.ref))
-                        shutil.copyfile(tmp_path + "www.lyngsat.com/" + p.ref.lstrip("."), p_name)
-                    except (TypeError, ValueError) as e:
-                        msg = "Picons format parse error: {}".format(p) + "\n" + str(e)
-                        log(msg)
+        on_id, pos, ssid, single = provider.on_id, provider.pos, provider.ssid, provider.single
+        neg_pos = pos.endswith("W")
+        pos = int("".join(c for c in pos if c.isdigit()))
+        # For negative (West) positions 3600 - numeric position value!!!
+        if neg_pos:
+            pos = 3600 - pos
+
+        parser = PiconsParser(single=provider.single)
+        parser.reset()
+        parser.feed(logo_data)
+        picons = parser.picons
+        picons_data = []
+
+        if picons:
+            for p in picons:
+                try:
+                    if single:
+                        on_id, freq = on_id.strip().split("::")
+                        namespace = "{:X}{:X}".format(int(pos), int(freq))
+                    else:
+                        namespace = "{:X}0000".format(int(pos))
+                    name = PiconsParser.format(ssid if single else p.ssid, on_id, namespace, picon_ids, s_type)
+                    p_name = picons_path + (name if name else os.path.basename(p.ref))
+                    picons_data.append(("{}{}".format(PiconsParser._BASE_URL, p.ref), p_name))
+                except (TypeError, ValueError) as e:
+                    msg = "Picons format parse error: {}".format(p) + "\n" + str(e)
+                    log(msg)
+
+        return picons_data
 
     @staticmethod
     def format(ssid, on_id, namespace, picon_ids, s_type):
@@ -127,7 +138,8 @@ class ProviderParser(HTMLParser):
     _POSITION_PATTERN = re.compile("at\s\d+\..*(?:E|W)']")
     _ONID_TID_PATTERN = re.compile("^\d+-\d+.*")
     _TRANSPONDER_FREQUENCY_PATTERN = re.compile("^\d+ [HVLR]+")
-    _DOMAINS = {"/tvchannels/", "/radiochannels/", "/packages/"}
+    _DOMAINS = {"/tvchannels/", "/radiochannels/", "/packages/", "/logo/"}
+    _BASE_URL = "https://www.lyngsat.com"
 
     def __init__(self, entities=False, separator=' '):
 
@@ -155,7 +167,7 @@ class ProviderParser(HTMLParser):
         if tag == 'tr':
             self._is_th = True
         if tag == "img":
-            if attrs[0][1].startswith("logo/"):
+            if attrs[0][1].startswith("/logo/"):
                 self._current_row.append(attrs[0][1])
         if tag == "a":
             url = attrs[0][1]
@@ -187,41 +199,47 @@ class ProviderParser(HTMLParser):
             self._current_row.append(final_cell)
             self._current_cell = []
         elif tag == 'tr':
-            r = self._current_row
+            row = self._current_row
             # Satellite position
             if not self._positon:
-                pos = re.findall(self._POSITION_PATTERN, str(r))
+                pos = re.findall(self._POSITION_PATTERN, str(row))
                 if pos:
                     self._positon = "".join(c for c in str(pos) if c.isdigit() or c in ".EW")
 
-            len_row = len(r)
+            len_row = len(row)
             if len_row > 2:
-                m = self._TRANSPONDER_FREQUENCY_PATTERN.match(r[1])
+                m = self._TRANSPONDER_FREQUENCY_PATTERN.match(row[1])
                 if m:
                     self._freq = m.group().split()[0]
 
-            if len_row == 12:
+            if len_row == 14:
                 # Providers
-                name = r[5]
+                name = row[6]
                 self._prv_names.add(name)
-                m = self._ONID_TID_PATTERN.match(str(r[-2]))
+                m = self._ONID_TID_PATTERN.match(str(row[9]))
                 if m:
                     on_id, tid = m.group().split("-")
                     if on_id not in self._ids:
-                        r[-2] = on_id
+                        row[-2] = on_id
                         self._ids.add(on_id)
-                        r[0] = self._positon
+                        row[0] = self._positon
                     if name + on_id not in self._prv_names:
                         self._prv_names.add(name + on_id)
-                        self.rows.append(Provider(logo=r[2], name=name, pos=self._positon, url=r[6], on_id=on_id,
+                        logo_data = None
+                        req = requests.get(self._BASE_URL + row[3], timeout=5)
+                        if req.status_code == 200:
+                            logo_data = req.content
+                        else:
+                            log("Downloading provider logo error: {}".format(req.reason))
+                        self.rows.append(Provider(logo=logo_data, name=name, pos=self._positon, url=row[5], on_id=on_id,
                                                   ssid=None, single=False, selected=True))
-            elif 6 < len_row < 10:
+            elif 6 < len_row < 14:
                 # Single services
                 name, url, ssid = None, None, None
-                if r[0].startswith("http"):
-                    name, url, ssid = r[1], r[0], r[4]
-                elif r[1].startswith("http"):
-                    name, url, ssid = r[2], r[1], r[5]
+                if row[0].startswith("http"):
+                    name, url, ssid = row[1], row[0], row[0]
+                elif row[1].startswith("http"):
+                    name, url, ssid = row[2], row[1], row[0]
 
                 if name and url:
                     on_id = "{}::{}".format(self._on_id if self._on_id else "1", self._freq)
@@ -237,14 +255,51 @@ class ProviderParser(HTMLParser):
         super().reset()
 
 
-def parse_providers(open_path):
+def parse_providers(url):
+    """ Returns a list of providers sorted by logo [single channels after providers]. """
     parser = ProviderParser()
-    parser.reset()
 
-    with open(open_path, encoding="utf-8", errors="replace") as f:
-        parser.feed(f.read())
+    request = requests.get(url=url, headers=_HEADERS)
+    if request.status_code == 200:
+        parser.feed(request.text)
+    else:
+        log("Parse providers error [{}]: {}".format(url, request.reason))
 
-        return parser.rows
+    def srt(p):
+        if p.logo is None:
+            return 1
+        return 0
+
+    providers = parser.rows
+    providers.sort(key=srt)
+
+    return providers
+
+
+def download_picon(src_url, dest_path, callback):
+    """ Downloads and saves the picon to file.  """
+    err_msg = "Picon download error: {}  [{}]"
+    timeout = (3, 5)  # connect and read timeouts
+
+    if callback:
+        callback("Downloading: {}.\n".format(os.path.basename(dest_path)))
+
+    req = requests.get(src_url, timeout=timeout, stream=True)
+    if req.status_code != 200:
+        err_msg = err_msg.format(src_url, req.reason)
+        log(err_msg)
+        if callback:
+            callback(err_msg + "\n")
+    else:
+        try:
+            with open(dest_path, "wb") as f:
+                for chunk in req:
+                    f.write(chunk)
+        except OSError as e:
+            err_msg = "Saving picon [{}] error: {}".format(dest_path, e)
+            log(err_msg)
+            if callback:
+                callback(err_msg + "\n")
 
 
 @run_task

app/ui/picons_manager.glade

@@ -1704,6 +1704,8 @@ Author: Dmitriy Yefremov
                     <property name="can_focus">True</property>
                     <property name="editable">False</property>
                     <property name="wrap_mode">word-char</property>
+                    <property name="left_margin">5</property>
+                    <property name="right_margin">5</property>
                     <property name="overwrite">True</property>
                   </object>
                 </child>

app/ui/picons_manager.py

@@ -1,17 +1,16 @@
 import os
 import re
 import shutil
-import subprocess
 import tempfile
 from pathlib import Path
 from urllib.parse import urlparse, unquote
 
-from gi.repository import GLib, GdkPixbuf
+from gi.repository import GLib, GdkPixbuf, Gio
 
 from app.commons import run_idle, run_task, run_with_delay
 from app.connections import upload_data, DownloadType, download_data, remove_picons
 from app.settings import SettingsType, Settings
-from app.tools.picons import PiconsParser, parse_providers, Provider, convert_to
+from app.tools.picons import PiconsParser, parse_providers, Provider, convert_to, download_picon
 from app.tools.satellites import SatellitesParser, SatelliteSource
 from .dialogs import show_dialog, DialogType, get_message
 from .main_helper import (update_entry_data, append_text_to_tview, scroll_to, on_popup_menu, get_base_model, set_picon,
@@ -30,6 +29,7 @@ class PiconsDialog:
         self._POS_PATTERN = re.compile(r"^\d+\.\d+[EW]?$")
         self._current_process = None
         self._terminate = False
+        self._is_downloading = False
         self._filter_binding = None
         self._services = None
         self._current_picon_info = None
@@ -480,7 +480,7 @@ class PiconsDialog:
         try:
             for sat in sats:
                 pos = sat[1]
-                name, pos = "{} ({})".format(sat[0], pos), "{}{}".format("-" if pos[-1] == "W" else "", pos[:-1])
+                name = "{} ({})".format(sat[0], pos)
 
                 if not self._terminate and model:
                     if pos in self._sat_positions:
@@ -492,49 +492,28 @@ class PiconsDialog:
         model = view.get_model()
         self._url_entry.set_text(model.get(model.get_iter(path), 1)[0])
 
-    @run_idle
     def on_load_providers(self, item):
-        self._expander.set_expanded(True)
         self.on_info_bar_close()
-        self._cancel_button.show()
-        url = self._url_entry.get_text()
-
-        try:
-            self._current_process = subprocess.Popen(["wget", "-pkP", self._TMP_DIR, url],
-                                                     stdout=subprocess.PIPE,
-                                                     stderr=subprocess.PIPE,
-                                                     universal_newlines=True)
-        except FileNotFoundError as e:
-            self._cancel_button.hide()
-            self.show_info_message(str(e), Gtk.MessageType.ERROR)
-        else:
-            GLib.io_add_watch(self._current_process.stderr, GLib.IO_IN, self.write_to_buffer)
-            model = self._providers_view.get_model()
-            model.clear()
-            self.append_providers(url, model)
+        model = self._providers_view.get_model()
+        model.clear()
+        self.get_providers(model)
 
     @run_task
-    def append_providers(self, url, model):
-        self._current_process.wait()
-        try:
-            self._terminate = False
-            providers = parse_providers(self._TMP_DIR + url[url.find("w"):])
-        except FileNotFoundError:
-            pass  # NOP
-        else:
-            if providers:
-                for p in providers:
-                    if self._terminate:
-                        return
-                    model.append((self.get_pixbuf(p[0]) if p[0] else TV_ICON, *p[1:]))
-            self.update_receive_button_state()
-        finally:
-            GLib.idle_add(self._cancel_button.hide)
-            self._terminate = False
+    def get_providers(self, model):
+        providers = parse_providers(self._url_entry.get_text())
+        if providers:
+            self.append_providers(providers, model)
 
-    def get_pixbuf(self, img_url):
-        return GdkPixbuf.Pixbuf.new_from_file_at_scale(filename=self._TMP_DIR + "www.lyngsat.com/" + img_url,
-                                                       width=48, height=48, preserve_aspect_ratio=True)
+    @run_idle
+    def append_providers(self, providers, model):
+        for p in providers:
+            model.append((self.get_pixbuf(p[0]) if p[0] else TV_ICON, *p[1:]))
+        self.update_receive_button_state()
+
+    def get_pixbuf(self, img_data):
+        if img_data:
+            f = Gio.MemoryInputStream.new_from_data(img_data)
+            return GdkPixbuf.Pixbuf.new_from_stream_at_scale(f, 48, 32, True, None)
 
     def on_receive(self, item):
         self._cancel_button.show()
@@ -542,12 +521,12 @@ class PiconsDialog:
 
     @run_task
     def start_download(self):
-        if self._current_process.poll() is None:
+        if self._is_downloading:
             self.show_dialog("The task is already running!", DialogType.ERROR)
             return
 
-        self._terminate = False
-        self._expander.set_expanded(True)
+        self._is_downloading = True
+        GLib.idle_add(self._expander.set_expanded, True)
 
         providers = self.get_selected_providers()
         for prv in providers:
@@ -558,38 +537,47 @@ class PiconsDialog:
                 return
 
         try:
-            for prv in providers:
-                if self._terminate:
-                    return
-                self.process_provider(Provider(*prv))
+            picons_path = self._picons_dir_entry.get_text()
+            os.makedirs(os.path.dirname(picons_path), exist_ok=True)
+            picons = []
+
+            self.show_info_message(get_message("Please, wait..."), Gtk.MessageType.INFO)
+
+            import concurrent.futures
+
+            with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+                # Getting links to picons.
+                futures = {executor.submit(self.process_provider, Provider(*p), picons_path): p for p in providers}
+                for future in concurrent.futures.as_completed(futures):
+                    if not self._is_downloading:
+                        executor.shutdown()
+                        return
+
+                    picons.extend(future.result())
+                # Getting picon images.
+                futures = {executor.submit(download_picon, *pic, self.append_output): pic for pic in picons}
+                done, not_done = concurrent.futures.wait(futures, timeout=0)
+                while self._is_downloading and not_done:
+                    done, not_done = concurrent.futures.wait(not_done, timeout=5)
+
+                for future in not_done:
+                    future.cancel()
+                concurrent.futures.wait(not_done)
+
+            if not self._is_downloading:
+                return
 
             if not self._resize_no_radio_button.get_active():
-                self.resize(self._picons_dir_entry.get_text())
+                self.resize(picons_path)
             else:
                 self.show_info_message(get_message("Done!"), Gtk.MessageType.INFO)
         finally:
             GLib.idle_add(self._cancel_button.hide)
-            self._terminate = False
+            self._is_downloading = False
 
-    def process_provider(self, prv):
-        url = prv.url
-        self.show_info_message(get_message("Please, wait..."), Gtk.MessageType.INFO)
-        self._current_process = subprocess.Popen(["wget", "-pkP", self._TMP_DIR, url],
-                                                 stdout=subprocess.PIPE,
-                                                 stderr=subprocess.PIPE,
-                                                 universal_newlines=True)
-        GLib.io_add_watch(self._current_process.stderr, GLib.IO_IN, self.write_to_buffer)
-        self._current_process.wait()
-        path = self._TMP_DIR + (url[url.find("//") + 2:] if prv.single else self._BASE_URL + url[url.rfind("/") + 1:])
-        PiconsParser.parse(path, self._picons_dir_entry.get_text(),
-                           self._TMP_DIR, prv, self._picon_ids, self.get_picons_format())
-
-    def write_to_buffer(self, fd, condition):
-        if condition == GLib.IO_IN:
-            char = fd.read(1)
-            self.append_output(char)
-            return True
-        return False
+    def process_provider(self, prv, picons_path):
+        self.append_output("Getting links to picons for: {}.\n".format(prv.name))
+        return PiconsParser.parse(prv, picons_path, self._picon_ids, self.get_picons_format())
 
     @run_idle
     def append_output(self, char):
@@ -615,7 +603,7 @@ class PiconsDialog:
             self.show_info_message(get_message("Done!"), Gtk.MessageType.INFO)
 
     def on_cancel(self, item=None):
-        if self.is_task_running() and show_dialog(DialogType.QUESTION, self._dialog) == Gtk.ResponseType.CANCEL:
+        if self._is_downloading and show_dialog(DialogType.QUESTION, self._dialog) == Gtk.ResponseType.CANCEL:
             return True
 
         self.terminate_task()
@@ -623,16 +611,15 @@ class PiconsDialog:
     @run_task
     def terminate_task(self):
         self._terminate = True
-
-        if self._current_process:
-            self._current_process.terminate()
-            self.show_info_message(get_message("The task is canceled!"), Gtk.MessageType.WARNING)
+        self._is_downloading = False
+        self.show_info_message(get_message("The task is canceled!"), Gtk.MessageType.WARNING)
 
     def on_close(self, window, event):
         if self.on_cancel():
             return True
 
         self._terminate = True
+        self._is_downloading = False
         self.save_window_size(window)
         self.clean_data()
         self._app.update_picons()
@@ -667,9 +654,10 @@ class PiconsDialog:
 
     @run_idle
     def show_info_message(self, text, message_type):
-        self._info_bar.set_visible(True)
-        self._info_bar.set_message_type(message_type)
+        self._info_bar.set_visible(False)
         self._message_label.set_text(get_message(text))
+        self._info_bar.set_message_type(message_type)
+        self._info_bar.set_visible(True)
 
     def on_picons_dir_open(self, entry, icon, event_button):
         update_entry_data(entry, self._dialog, settings=self._settings)
@@ -834,9 +822,6 @@ class PiconsDialog:
 
         return picon_format
 
-    def is_task_running(self):
-        return self._current_process and self._current_process.poll() is None
-
 
 if __name__ == "__main__":
     pass