diff --git a/app/tools/picons.py b/app/tools/picons.py
index 54d9d175..41b9bd0b 100644
--- a/app/tools/picons.py
+++ b/app/tools/picons.py
@@ -5,8 +5,11 @@ import shutil
from collections import namedtuple
from html.parser import HTMLParser
+import requests
+
from app.commons import run_task, log
from app.settings import SettingsType
+from .satellites import _HEADERS
_ENIGMA2_PICON_KEY = "{:X}:{:X}:{}"
_NEUTRINO_PICON_KEY = "{:x}{:04x}{:04x}.png"
@@ -17,6 +20,7 @@ Picon = namedtuple("Picon", ["ref", "ssid", "v_pid"])
class PiconsParser(HTMLParser):
""" Parser for package html page. (https://www.lyngsat.com/packages/*provider-name*.html) """
+ _BASE_URL = "https://www.lyngsat.com"
def __init__(self, entities=False, separator=' ', single=None):
@@ -58,14 +62,14 @@ class PiconsParser(HTMLParser):
row = self._current_row
ln = len(row)
- if self._single and ln == 4 and row[0].startswith("../../logo/"):
- self.picons.append(Picon(row[0].strip("../"), "0", "0"))
+ if self._single and ln == 4 and row[0].startswith("/logo/"):
+ self.picons.append(Picon(row[0].strip(), "0", "0"))
else:
if 9 < ln < 13:
url = None
- if row[0].startswith("../logo/"):
+ if row[0].startswith("/logo/"):
url = row[0]
- elif row[1].startswith("../logo/"):
+ elif row[1].startswith("/logo/"):
url = row[1]
ssid = row[-4]
@@ -78,37 +82,44 @@ class PiconsParser(HTMLParser):
pass
@staticmethod
- def parse(open_path, picons_path, tmp_path, provider, picon_ids, s_type=SettingsType.ENIGMA_2):
- if not os.path.isfile(open_path):
- log("PiconsParser error [parse]. No such file or directory: {}".format(open_path))
+ def parse(provider, picons_path, picon_ids, s_type=SettingsType.ENIGMA_2):
+ """ Returns tuple(url, picon file name) list. """
+ req = requests.get(provider.url, timeout=5)
+ if req.status_code == 200:
+ logo_data = req.text
+ else:
+ log("Provider picons downloading error: {} {}".format(provider.url, req.reason))
return
- with open(open_path, encoding="utf-8", errors="replace") as f:
- on_id, pos, ssid, single = provider.on_id, provider.pos, provider.ssid, provider.single
- neg_pos = pos.endswith("W")
- pos = int("".join(c for c in pos if c.isdigit()))
- # For negative (West) positions 3600 - numeric position value!!!
- if neg_pos:
- pos = 3600 - pos
- parser = PiconsParser(single=single)
- parser.reset()
- parser.feed(f.read())
- picons = parser.picons
- if picons:
- os.makedirs(picons_path, exist_ok=True)
- for p in picons:
- try:
- if single:
- on_id, freq = on_id.strip().split("::")
- namespace = "{:X}{:X}".format(int(pos), int(freq))
- else:
- namespace = "{:X}0000".format(int(pos))
- name = PiconsParser.format(ssid if single else p.ssid, on_id, namespace, picon_ids, s_type)
- p_name = picons_path + (name if name else os.path.basename(p.ref))
- shutil.copyfile(tmp_path + "www.lyngsat.com/" + p.ref.lstrip("."), p_name)
- except (TypeError, ValueError) as e:
- msg = "Picons format parse error: {}".format(p) + "\n" + str(e)
- log(msg)
+ on_id, pos, ssid, single = provider.on_id, provider.pos, provider.ssid, provider.single
+ neg_pos = pos.endswith("W")
+ pos = int("".join(c for c in pos if c.isdigit()))
+ # For negative (West) positions 3600 - numeric position value!!!
+ if neg_pos:
+ pos = 3600 - pos
+
+ parser = PiconsParser(single=provider.single)
+ parser.reset()
+ parser.feed(logo_data)
+ picons = parser.picons
+ picons_data = []
+
+ if picons:
+ for p in picons:
+ try:
+ if single:
+ on_id, freq = on_id.strip().split("::")
+ namespace = "{:X}{:X}".format(int(pos), int(freq))
+ else:
+ namespace = "{:X}0000".format(int(pos))
+ name = PiconsParser.format(ssid if single else p.ssid, on_id, namespace, picon_ids, s_type)
+ p_name = picons_path + (name if name else os.path.basename(p.ref))
+ picons_data.append(("{}{}".format(PiconsParser._BASE_URL, p.ref), p_name))
+ except (TypeError, ValueError) as e:
+ msg = "Picons format parse error: {}".format(p) + "\n" + str(e)
+ log(msg)
+
+ return picons_data
@staticmethod
def format(ssid, on_id, namespace, picon_ids, s_type):
@@ -127,7 +138,8 @@ class ProviderParser(HTMLParser):
_POSITION_PATTERN = re.compile("at\s\d+\..*(?:E|W)']")
_ONID_TID_PATTERN = re.compile("^\d+-\d+.*")
_TRANSPONDER_FREQUENCY_PATTERN = re.compile("^\d+ [HVLR]+")
- _DOMAINS = {"/tvchannels/", "/radiochannels/", "/packages/"}
+ _DOMAINS = {"/tvchannels/", "/radiochannels/", "/packages/", "/logo/"}
+ _BASE_URL = "https://www.lyngsat.com"
def __init__(self, entities=False, separator=' '):
@@ -155,7 +167,7 @@ class ProviderParser(HTMLParser):
if tag == 'tr':
self._is_th = True
if tag == "img":
- if attrs[0][1].startswith("logo/"):
+ if attrs[0][1].startswith("/logo/"):
self._current_row.append(attrs[0][1])
if tag == "a":
url = attrs[0][1]
@@ -187,41 +199,47 @@ class ProviderParser(HTMLParser):
self._current_row.append(final_cell)
self._current_cell = []
elif tag == 'tr':
- r = self._current_row
+ row = self._current_row
# Satellite position
if not self._positon:
- pos = re.findall(self._POSITION_PATTERN, str(r))
+ pos = re.findall(self._POSITION_PATTERN, str(row))
if pos:
self._positon = "".join(c for c in str(pos) if c.isdigit() or c in ".EW")
- len_row = len(r)
+ len_row = len(row)
if len_row > 2:
- m = self._TRANSPONDER_FREQUENCY_PATTERN.match(r[1])
+ m = self._TRANSPONDER_FREQUENCY_PATTERN.match(row[1])
if m:
self._freq = m.group().split()[0]
- if len_row == 12:
+ if len_row == 14:
# Providers
- name = r[5]
+ name = row[6]
self._prv_names.add(name)
- m = self._ONID_TID_PATTERN.match(str(r[-2]))
+ m = self._ONID_TID_PATTERN.match(str(row[9]))
if m:
on_id, tid = m.group().split("-")
if on_id not in self._ids:
- r[-2] = on_id
+ row[-2] = on_id
self._ids.add(on_id)
- r[0] = self._positon
+ row[0] = self._positon
if name + on_id not in self._prv_names:
self._prv_names.add(name + on_id)
- self.rows.append(Provider(logo=r[2], name=name, pos=self._positon, url=r[6], on_id=on_id,
+ logo_data = None
+ req = requests.get(self._BASE_URL + row[3], timeout=5)
+ if req.status_code == 200:
+ logo_data = req.content
+ else:
+ log("Downloading provider logo error: {}".format(req.reason))
+ self.rows.append(Provider(logo=logo_data, name=name, pos=self._positon, url=row[5], on_id=on_id,
ssid=None, single=False, selected=True))
- elif 6 < len_row < 10:
+ elif 6 < len_row < 14:
# Single services
name, url, ssid = None, None, None
- if r[0].startswith("http"):
- name, url, ssid = r[1], r[0], r[4]
- elif r[1].startswith("http"):
- name, url, ssid = r[2], r[1], r[5]
+ if row[0].startswith("http"):
+ name, url, ssid = row[1], row[0], row[0]
+ elif row[1].startswith("http"):
+ name, url, ssid = row[2], row[1], row[0]
if name and url:
on_id = "{}::{}".format(self._on_id if self._on_id else "1", self._freq)
@@ -237,14 +255,51 @@ class ProviderParser(HTMLParser):
super().reset()
-def parse_providers(open_path):
+def parse_providers(url):
+ """ Returns a list of providers sorted by logo [single channels after providers]. """
parser = ProviderParser()
- parser.reset()
- with open(open_path, encoding="utf-8", errors="replace") as f:
- parser.feed(f.read())
+ request = requests.get(url=url, headers=_HEADERS)
+ if request.status_code == 200:
+ parser.feed(request.text)
+ else:
+ log("Parse providers error [{}]: {}".format(url, request.reason))
- return parser.rows
+ def srt(p):
+ if p.logo is None:
+ return 1
+ return 0
+
+ providers = parser.rows
+ providers.sort(key=srt)
+
+ return providers
+
+
+def download_picon(src_url, dest_path, callback):
+ """ Downloads and saves the picon to file. """
+ err_msg = "Picon download error: {} [{}]"
+ timeout = (3, 5) # connect and read timeouts
+
+ if callback:
+ callback("Downloading: {}.\n".format(os.path.basename(dest_path)))
+
+ req = requests.get(src_url, timeout=timeout, stream=True)
+ if req.status_code != 200:
+ err_msg = err_msg.format(src_url, req.reason)
+ log(err_msg)
+ if callback:
+ callback(err_msg + "\n")
+ else:
+ try:
+ with open(dest_path, "wb") as f:
+ for chunk in req:
+ f.write(chunk)
+ except OSError as e:
+ err_msg = "Saving picon [{}] error: {}".format(dest_path, e)
+ log(err_msg)
+ if callback:
+ callback(err_msg + "\n")
@run_task
diff --git a/app/ui/picons_manager.glade b/app/ui/picons_manager.glade
index 46d6cc32..d18378a0 100644
--- a/app/ui/picons_manager.glade
+++ b/app/ui/picons_manager.glade
@@ -1704,6 +1704,8 @@ Author: Dmitriy Yefremov
True
False
word-char
+ 5
+ 5
True
diff --git a/app/ui/picons_manager.py b/app/ui/picons_manager.py
index de6394dd..cc418367 100644
--- a/app/ui/picons_manager.py
+++ b/app/ui/picons_manager.py
@@ -1,17 +1,16 @@
import os
import re
import shutil
-import subprocess
import tempfile
from pathlib import Path
from urllib.parse import urlparse, unquote
-from gi.repository import GLib, GdkPixbuf
+from gi.repository import GLib, GdkPixbuf, Gio
from app.commons import run_idle, run_task, run_with_delay
from app.connections import upload_data, DownloadType, download_data, remove_picons
from app.settings import SettingsType, Settings
-from app.tools.picons import PiconsParser, parse_providers, Provider, convert_to
+from app.tools.picons import PiconsParser, parse_providers, Provider, convert_to, download_picon
from app.tools.satellites import SatellitesParser, SatelliteSource
from .dialogs import show_dialog, DialogType, get_message
from .main_helper import (update_entry_data, append_text_to_tview, scroll_to, on_popup_menu, get_base_model, set_picon,
@@ -30,6 +29,7 @@ class PiconsDialog:
self._POS_PATTERN = re.compile(r"^\d+\.\d+[EW]?$")
self._current_process = None
self._terminate = False
+ self._is_downloading = False
self._filter_binding = None
self._services = None
self._current_picon_info = None
@@ -480,7 +480,7 @@ class PiconsDialog:
try:
for sat in sats:
pos = sat[1]
- name, pos = "{} ({})".format(sat[0], pos), "{}{}".format("-" if pos[-1] == "W" else "", pos[:-1])
+ name = "{} ({})".format(sat[0], pos)
if not self._terminate and model:
if pos in self._sat_positions:
@@ -492,49 +492,28 @@ class PiconsDialog:
model = view.get_model()
self._url_entry.set_text(model.get(model.get_iter(path), 1)[0])
- @run_idle
def on_load_providers(self, item):
- self._expander.set_expanded(True)
self.on_info_bar_close()
- self._cancel_button.show()
- url = self._url_entry.get_text()
-
- try:
- self._current_process = subprocess.Popen(["wget", "-pkP", self._TMP_DIR, url],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- universal_newlines=True)
- except FileNotFoundError as e:
- self._cancel_button.hide()
- self.show_info_message(str(e), Gtk.MessageType.ERROR)
- else:
- GLib.io_add_watch(self._current_process.stderr, GLib.IO_IN, self.write_to_buffer)
- model = self._providers_view.get_model()
- model.clear()
- self.append_providers(url, model)
+ model = self._providers_view.get_model()
+ model.clear()
+ self.get_providers(model)
@run_task
- def append_providers(self, url, model):
- self._current_process.wait()
- try:
- self._terminate = False
- providers = parse_providers(self._TMP_DIR + url[url.find("w"):])
- except FileNotFoundError:
- pass # NOP
- else:
- if providers:
- for p in providers:
- if self._terminate:
- return
- model.append((self.get_pixbuf(p[0]) if p[0] else TV_ICON, *p[1:]))
- self.update_receive_button_state()
- finally:
- GLib.idle_add(self._cancel_button.hide)
- self._terminate = False
+ def get_providers(self, model):
+ providers = parse_providers(self._url_entry.get_text())
+ if providers:
+ self.append_providers(providers, model)
- def get_pixbuf(self, img_url):
- return GdkPixbuf.Pixbuf.new_from_file_at_scale(filename=self._TMP_DIR + "www.lyngsat.com/" + img_url,
- width=48, height=48, preserve_aspect_ratio=True)
+ @run_idle
+ def append_providers(self, providers, model):
+ for p in providers:
+ model.append((self.get_pixbuf(p[0]) if p[0] else TV_ICON, *p[1:]))
+ self.update_receive_button_state()
+
+ def get_pixbuf(self, img_data):
+ if img_data:
+ f = Gio.MemoryInputStream.new_from_data(img_data)
+ return GdkPixbuf.Pixbuf.new_from_stream_at_scale(f, 48, 32, True, None)
def on_receive(self, item):
self._cancel_button.show()
@@ -542,12 +521,12 @@ class PiconsDialog:
@run_task
def start_download(self):
- if self._current_process.poll() is None:
+ if self._is_downloading:
self.show_dialog("The task is already running!", DialogType.ERROR)
return
- self._terminate = False
- self._expander.set_expanded(True)
+ self._is_downloading = True
+ GLib.idle_add(self._expander.set_expanded, True)
providers = self.get_selected_providers()
for prv in providers:
@@ -558,38 +537,47 @@ class PiconsDialog:
return
try:
- for prv in providers:
- if self._terminate:
- return
- self.process_provider(Provider(*prv))
+ picons_path = self._picons_dir_entry.get_text()
+ os.makedirs(os.path.dirname(picons_path), exist_ok=True)
+ picons = []
+
+ self.show_info_message(get_message("Please, wait..."), Gtk.MessageType.INFO)
+
+ import concurrent.futures
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+ # Getting links to picons.
+ futures = {executor.submit(self.process_provider, Provider(*p), picons_path): p for p in providers}
+ for future in concurrent.futures.as_completed(futures):
+ if not self._is_downloading:
+ executor.shutdown()
+ return
+
+ picons.extend(future.result())
+ # Getting picon images.
+ futures = {executor.submit(download_picon, *pic, self.append_output): pic for pic in picons}
+ done, not_done = concurrent.futures.wait(futures, timeout=0)
+ while self._is_downloading and not_done:
+ done, not_done = concurrent.futures.wait(not_done, timeout=5)
+
+ for future in not_done:
+ future.cancel()
+ concurrent.futures.wait(not_done)
+
+ if not self._is_downloading:
+ return
if not self._resize_no_radio_button.get_active():
- self.resize(self._picons_dir_entry.get_text())
+ self.resize(picons_path)
else:
self.show_info_message(get_message("Done!"), Gtk.MessageType.INFO)
finally:
GLib.idle_add(self._cancel_button.hide)
- self._terminate = False
+ self._is_downloading = False
- def process_provider(self, prv):
- url = prv.url
- self.show_info_message(get_message("Please, wait..."), Gtk.MessageType.INFO)
- self._current_process = subprocess.Popen(["wget", "-pkP", self._TMP_DIR, url],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- universal_newlines=True)
- GLib.io_add_watch(self._current_process.stderr, GLib.IO_IN, self.write_to_buffer)
- self._current_process.wait()
- path = self._TMP_DIR + (url[url.find("//") + 2:] if prv.single else self._BASE_URL + url[url.rfind("/") + 1:])
- PiconsParser.parse(path, self._picons_dir_entry.get_text(),
- self._TMP_DIR, prv, self._picon_ids, self.get_picons_format())
-
- def write_to_buffer(self, fd, condition):
- if condition == GLib.IO_IN:
- char = fd.read(1)
- self.append_output(char)
- return True
- return False
+ def process_provider(self, prv, picons_path):
+ self.append_output("Getting links to picons for: {}.\n".format(prv.name))
+ return PiconsParser.parse(prv, picons_path, self._picon_ids, self.get_picons_format())
@run_idle
def append_output(self, char):
@@ -615,7 +603,7 @@ class PiconsDialog:
self.show_info_message(get_message("Done!"), Gtk.MessageType.INFO)
def on_cancel(self, item=None):
- if self.is_task_running() and show_dialog(DialogType.QUESTION, self._dialog) == Gtk.ResponseType.CANCEL:
+ if self._is_downloading and show_dialog(DialogType.QUESTION, self._dialog) == Gtk.ResponseType.CANCEL:
return True
self.terminate_task()
@@ -623,16 +611,15 @@ class PiconsDialog:
@run_task
def terminate_task(self):
self._terminate = True
-
- if self._current_process:
- self._current_process.terminate()
- self.show_info_message(get_message("The task is canceled!"), Gtk.MessageType.WARNING)
+ self._is_downloading = False
+ self.show_info_message(get_message("The task is canceled!"), Gtk.MessageType.WARNING)
def on_close(self, window, event):
if self.on_cancel():
return True
self._terminate = True
+ self._is_downloading = False
self.save_window_size(window)
self.clean_data()
self._app.update_picons()
@@ -667,9 +654,10 @@ class PiconsDialog:
@run_idle
def show_info_message(self, text, message_type):
- self._info_bar.set_visible(True)
- self._info_bar.set_message_type(message_type)
+ self._info_bar.set_visible(False)
self._message_label.set_text(get_message(text))
+ self._info_bar.set_message_type(message_type)
+ self._info_bar.set_visible(True)
def on_picons_dir_open(self, entry, icon, event_button):
update_entry_data(entry, self._dialog, settings=self._settings)
@@ -834,9 +822,6 @@ class PiconsDialog:
return picon_format
- def is_task_running(self):
- return self._current_process and self._current_process.poll() is None
-
if __name__ == "__main__":
pass