From ca1e823bf19ec2cf6d6940933a1c5cebb6eceb62 Mon Sep 17 00:00:00 2001 From: DYefremov Date: Sun, 21 Feb 2021 08:44:33 +0300 Subject: [PATCH] improved satellites import [added KingOfSat support] --- app/eparser/satxml.py | 7 ++- app/tools/satellites.py | 86 +++++++++++++++++++++------------- app/ui/satellites_dialog.glade | 69 +++++++-------------------- app/ui/satellites_dialog.py | 31 ++++++++---- 4 files changed, 96 insertions(+), 97 deletions(-) diff --git a/app/eparser/satxml.py b/app/eparser/satxml.py index 360fbab7..e4c78384 100644 --- a/app/eparser/satxml.py +++ b/app/eparser/satxml.py @@ -53,9 +53,9 @@ def write_satellites(satellites, data_path): transponder_child.setAttribute("frequency", tr.frequency) transponder_child.setAttribute("symbol_rate", tr.symbol_rate) transponder_child.setAttribute("polarization", get_key_by_value(POLARIZATION, tr.polarization)) - transponder_child.setAttribute("fec_inner", get_key_by_value(FEC, tr.fec_inner)) - transponder_child.setAttribute("system", get_key_by_value(SYSTEM, tr.system)) - transponder_child.setAttribute("modulation", get_key_by_value(MODULATION, tr.modulation)) + transponder_child.setAttribute("fec_inner", get_key_by_value(FEC, tr.fec_inner) or "0") + transponder_child.setAttribute("system", get_key_by_value(SYSTEM, tr.system) or "0") + transponder_child.setAttribute("modulation", get_key_by_value(MODULATION, tr.modulation) or "0") if tr.pls_mode: transponder_child.setAttribute("pls_mode", tr.pls_mode) if tr.pls_code: @@ -90,7 +90,6 @@ def parse_transponders(elem, sat_name): atr["is_id"].value if "is_id" in atr else None) except Exception as e: message = "Error: can't parse transponder for '{}' satellite! {}".format(sat_name, repr(e)) - print(message) log(message) else: transponders.append(tr) diff --git a/app/tools/satellites.py b/app/tools/satellites.py index f2931b7a..c0df73f6 100644 --- a/app/tools/satellites.py +++ b/app/tools/satellites.py @@ -14,13 +14,14 @@ from app.eparser import Satellite, Transponder, is_transponder_valid from app.eparser.ecommons import (PLS_MODE, get_key_by_value, FEC, SYSTEM, POLARIZATION, MODULATION, SERVICE_TYPE, Service, CAS) -_HEADERS = {"User-Agent": "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/69.0"} +_HEADERS = {"User-Agent": "Mozilla/5.0 (Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0"} class SatelliteSource(Enum): FLYSAT = ("https://www.flysat.com/satlist.php",) LYNGSAT = ("https://www.lyngsat.com/asia.html", "https://www.lyngsat.com/europe.html", "https://www.lyngsat.com/atlantic.html", "https://www.lyngsat.com/america.html") + KINGOFSAT = ("https://en.kingofsat.net/satellites.php",) @staticmethod def get_sources(src): @@ -96,7 +97,9 @@ class SatellitesParser(HTMLParser): if tag == "tr": self._is_th = True if tag == "a": - self._current_row.append(attrs[0][1]) + for atr in attrs: + if atr[0] == "href": + self._current_row.append(atr[1]) def handle_data(self, data): """ Save content to a cell """ @@ -182,6 +185,11 @@ class SatellitesParser(HTMLParser): elif r_len == 5: sats.append((row[2], current_pos, row[3], base_url + row[1], False)) return sats + elif source is SatelliteSource.KINGOFSAT: + def get_sat(r): + return r[3], self.parse_position(r[1]), None, r[0], False + + return list(map(get_sat, filter(lambda x: len(x) == 17, self._rows))) def get_satellite(self, sat): pos = sat[1] @@ -201,18 +209,29 @@ class SatellitesParser(HTMLParser): def get_transponders(self, sat_url): """ Getting transponders(sorted by frequency). """ self._rows.clear() - url = "https://www.flysat.com/" + sat_url if self._source is SatelliteSource.FLYSAT else sat_url - request = requests.get(url=url, headers=_HEADERS) - trs = [] - if request.status_code == 200: - self.feed(request.text) - if self._source is SatelliteSource.FLYSAT: - self.get_transponders_for_fly_sat(trs) - elif self._source is SatelliteSource.LYNGSAT: - self.get_transponders_for_lyng_sat(trs) + + url = sat_url + if self._source is SatelliteSource.FLYSAT: + url = "https://www.flysat.com/" + sat_url + elif self._source is SatelliteSource.KINGOFSAT: + url = "https://en.kingofsat.net/" + sat_url + + try: + request = requests.get(url=url, headers=_HEADERS) + except requests.exceptions.ConnectionError as e: + log("Getting transponders error: {}".format(e)) else: - log("SatellitesParser [get transponders] error: {} {}".format(url, request.reason)) + if request.status_code == 200: + self.feed(request.text) + if self._source is SatelliteSource.FLYSAT: + self.get_transponders_for_fly_sat(trs) + elif self._source is SatelliteSource.LYNGSAT: + self.get_transponders_for_lyng_sat(trs) + elif self._source is SatelliteSource.KINGOFSAT: + self.get_transponders_for_king_of_sat(trs) + else: + log("SatellitesParser [get transponders] error: {} {}".format(url, request.reason)) return sorted(trs, key=lambda x: int(x.frequency)) @@ -296,6 +315,26 @@ class SatellitesParser(HTMLParser): if is_transponder_valid(tr): trs.append(tr) + def get_transponders_for_king_of_sat(self, trs): + """ Getting transponders for KingOfSat source. + + Since the *.ini file contains incomplete information, it is not used. + """ + zeros = "000" + pos_pat = re.compile(r".*?(\d+\.\d°[EW]).*") + pat = re.compile( + r"(\d+).00\s+([RLHV])\s+(DVB-S[2]?)\s+(?:T2-MI, PLP (\d+)\s+)?(.*PSK).*?(?:Stream\s+(\d+))?\s+(\d+)\s+(\d+/\d+)$") + + for row in filter(lambda r: len(r) == 16 and pos_pat.match(r[0]), self._rows): + res = pat.search(" ".join((row[0], row[2], row[3], row[8], row[9], row[10]))) + if res: + freq, sr, pol, fec, sys = res.group(1), res.group(7), res.group(2), res.group(8), res.group(3) + mod, pls_id, pls_code = res.group(5), res.group(4), res.group(6) + + tr = Transponder(freq + zeros, sr + zeros, pol, fec, sys, mod, None, pls_code, pls_id) + if is_transponder_valid(tr): + trs.append(tr) + class ServicesParser(HTMLParser): """ Services parser for LYNGSAT source. """ @@ -470,27 +509,8 @@ class ServicesParser(HTMLParser): else: flags = ",".join(filter(None, (flags, cas))) - srv = Service(flags_cas=flags, - transponder_type="s", - coded=None, - service=name, - locked=None, - hide=None, - package=pkg, - service_type=_s_type, - picon=r[1].img, - picon_id=picon_id, - ssid=sid, - freq=freq, - rate=sr, - pol=pol, - fec=fec, - system=sys, - pos=pos, - data_id=data_id, - fav_id=fav_id, - transponder=tr) - services.append(srv) + services.append(Service(flags, "s", None, name, None, None, pkg, _s_type, r[1].img, picon_id, + sid, freq, sr, pol, fec, sys, pos, data_id, fav_id, tr)) except ValueError as e: log("ServicesParser error [get transponder services]: {}".format(e)) diff --git a/app/ui/satellites_dialog.glade b/app/ui/satellites_dialog.glade index e8ca8218..48aa7efe 100644 --- a/app/ui/satellites_dialog.glade +++ b/app/ui/satellites_dialog.glade @@ -25,33 +25,6 @@ THE SOFTWARE. Author: Dmitriy Yefremov ---> - @@ -577,11 +550,14 @@ Author: Dmitriy Yefremov True - 20 + 250 Satellite True + 0.5 - + + end + 0 @@ -594,6 +570,7 @@ Author: Dmitriy Yefremov 20 Freq True + 0.5 @@ -608,6 +585,7 @@ Author: Dmitriy Yefremov 20 Rate True + 0.5 @@ -622,6 +600,7 @@ Author: Dmitriy Yefremov 20 Pol True + 0.5 @@ -636,6 +615,7 @@ Author: Dmitriy Yefremov 20 FEC True + 0.5 @@ -650,6 +630,7 @@ Author: Dmitriy Yefremov 20 System True + 0.5 @@ -664,6 +645,7 @@ Author: Dmitriy Yefremov 20 Mod True + 0.5 @@ -1392,20 +1374,6 @@ Author: Dmitriy Yefremov update_sat_list_model_filter - - - - - - - - FlySat - - - LyngSat - - - @@ -1569,18 +1537,15 @@ Author: Dmitriy Yefremov center expand - + True False - center - update_source_store 0 - - - - 0 - - + + FlySat + LyngSat + KingOfSat + False diff --git a/app/ui/satellites_dialog.py b/app/ui/satellites_dialog.py index d190d41e..965907af 100644 --- a/app/ui/satellites_dialog.py +++ b/app/ui/satellites_dialog.py @@ -254,13 +254,22 @@ class SatellitesDialog: return paths - @staticmethod - def on_remove(view): + @run_idle + def on_remove(self, view): + """ Removal of selected satellites and transponders. + + The satellites are removed first! Then transponders. + """ selection = view.get_selection() model, paths = selection.get_selected_rows() - - for itr in [model.get_iter(path) for path in paths]: - model.remove(itr) + itrs = [model.get_iter(path) for path in paths] + satellites = list(filter(model.iter_has_child, itrs)) + if len(satellites): + # Removing selected satellites. + list(map(model.remove, satellites)) + else: + # Removing selected transponders. + list(map(model.remove, itrs)) @run_idle def on_save(self, view): @@ -532,7 +541,13 @@ class UpdateDialog: @run_task def get_sat_list(self, src, callback): - sats = self._parser.get_satellites_list(SatelliteSource.FLYSAT if src == 0 else SatelliteSource.LYNGSAT) + sat_src = SatelliteSource.FLYSAT + if src == 1: + sat_src = SatelliteSource.LYNGSAT + elif src == 2: + sat_src = SatelliteSource.KINGOFSAT + + sats = self._parser.get_satellites_list(sat_src) if sats: callback(sats) self.is_download = False @@ -731,8 +746,8 @@ class ServicesUpdateDialog(UpdateDialog): self._services_parser = ServicesParser(source=SatelliteSource.LYNGSAT) self._transponder_paned.set_visible(True) - s_model = self._source_box.get_model() - s_model.remove(s_model.get_iter_first()) + self._source_box.remove(0) + self._source_box.remove(1) self._source_box.set_active(0) # Transponder view popup menu tr_popup_menu = Gtk.Menu()