added basic youtube-dl support

This commit is contained in:
DYefremov
2020-06-10 11:10:41 +03:00
parent b02eb37f1c
commit c0c2ddef34
4 changed files with 240 additions and 68 deletions

View File

@@ -1,11 +1,15 @@
""" Module for working with YouTube service """
import gzip
import json
import os
import re
import urllib
import shutil
import sys
from html.parser import HTMLParser
from json import JSONDecodeError
from urllib.request import Request
from urllib.error import URLError
from urllib.parse import unquote
from urllib.request import Request, urlopen, urlretrieve
from app.commons import log
@@ -23,6 +27,8 @@ Quality = {137: "1080p", 136: "720p", 135: "480p", 134: "360p",
class YouTube:
""" Helper class for working with YouTube service. """
_VIDEO_INFO_LINK = "https://youtube.com/get_video_info?video_id={}&hl=en"
@staticmethod
def is_yt_video_link(url):
return re.match(_YT_VIDEO_PATTERN, url)
@@ -47,11 +53,11 @@ class YouTube:
returns tuple from the video links dict and title
"""
req = Request("https://youtube.com/get_video_info?video_id={}&hl=en".format(video_id), headers=_HEADERS)
req = Request(YouTube._VIDEO_INFO_LINK.format(video_id), headers=_HEADERS)
with urllib.request.urlopen(req, timeout=2) as resp:
data = urllib.request.unquote(gzip.decompress(resp.read()).decode("utf-8")).split("&")
out = {k: v for k, sep, v in (str(d).partition("=") for d in map(urllib.request.unquote, data))}
with urlopen(req, timeout=2) as resp:
data = unquote(gzip.decompress(resp.read()).decode("utf-8")).split("&")
out = {k: v for k, sep, v in (str(d).partition("=") for d in map(unquote, data))}
player_resp = out.get("player_response", None)
if player_resp:
@@ -76,7 +82,7 @@ class YouTube:
if stream_map:
s_map = {k: v for k, sep, v in (str(d).partition("=") for d in stream_map.split("&"))}
url, title = s_map.get("url", None), out.get("title", None)
url, title = urllib.request.unquote(url) if url else "", title.replace("+", " ") if title else ""
url, title = unquote(url) if url else "", title.replace("+", " ") if title else ""
if url and title:
return {Quality[0]: url}, title.replace("+", " ")
@@ -145,13 +151,147 @@ class PlayListParser(HTMLParser):
"""
request = Request("https://www.youtube.com/playlist?list={}&hl=en".format(play_list_id), headers=_HEADERS)
with urllib.request.urlopen(request, timeout=2) as resp:
with urlopen(request, timeout=2) as resp:
data = gzip.decompress(resp.read()).decode("utf-8")
parser = PlayListParser()
parser.feed(data)
return parser.header, parser.playlist
class YouTubeDL:
""" Utility class [experimental] for working with youtube-dl.
[https://github.com/ytdl-org/youtube-dl]
"""
_DL_INSTANCE = None
_DownloadError = None
_LATEST_RELEASE_URL = "https://api.github.com/repos/ytdl-org/youtube-dl/releases/latest"
_OPTIONS = {"noplaylist": True, # Single video instead of a playlist [ignoring playlist in URL].
"quiet": True, # Do not print messages to stdout.
"simulate": True} # Do not download the video files.
VIDEO_LINK = "https://www.youtube.com/watch?v={}"
class YouTubeDLException(Exception):
pass
def __init__(self, settings, callback):
self._path = settings.default_data_path + "tools/"
self._update = settings.enable_yt_dl_update
self._supported = {"22", "18"}
self._dl = None
self._callback = callback
self._download_exception = None
self._is_update_process = False
@classmethod
def get_instance(cls, settings, callback=print):
if not cls._DL_INSTANCE:
cls._DL_INSTANCE = YouTubeDL(settings, callback)
return cls._DL_INSTANCE
def init(self):
if not os.path.isfile(self._path + "youtube_dl/version.py"):
self.get_latest_release()
if self._path not in sys.path:
sys.path.append(self._path)
self.init_dl()
def init_dl(self):
try:
import youtube_dl
except ModuleNotFoundError as e:
log("YouTubeDLHelper error: {}".format(str(e)))
raise self.YouTubeDLException(e)
else:
if self._update:
if hasattr(youtube_dl.version, "__version__"):
l_ver = self.get_last_release_id()
cur_ver = youtube_dl.version.__version__
if youtube_dl.version.__version__ < l_ver:
msg = "youtube-dl has new release! Current: {}. Last: {}.".format(cur_ver, l_ver)
log(msg)
self._callback(msg, False)
self.get_latest_release()
self._DownloadError = youtube_dl.utils.DownloadError
self._dl = youtube_dl.YoutubeDL(self._OPTIONS)
log("youtube-dl initialized...")
def get_last_release_id(self):
""" Getting last release id. """
url = "https://api.github.com/repos/ytdl-org/youtube-dl/releases/latest"
with urlopen(url, timeout=10) as resp:
return json.load(resp).get("tag_name", "0")
def get_latest_release(self):
try:
self._is_update_process = True
log("Getting the last youtube-dl release...")
with urlopen(YouTubeDL._LATEST_RELEASE_URL, timeout=10) as resp:
r = json.load(resp)
zip_url = r.get("zipball_url", None)
if zip_url:
zip_file = self._path + "yt.zip"
os.makedirs(os.path.dirname(self._path), exist_ok=True)
f_name, headers = urlretrieve(zip_url, filename=zip_file)
import zipfile
with zipfile.ZipFile(f_name) as arch:
if os.path.isdir(self._path):
shutil.rmtree(self._path)
else:
os.makedirs(os.path.dirname(self._path), exist_ok=True)
for info in arch.infolist():
pref, sep, f = info.filename.partition("/youtube_dl/")
if sep:
arch.extract(info.filename)
shutil.move(info.filename, "{}{}{}".format(self._path, sep, f))
shutil.rmtree(pref)
msg = "Getting the last youtube-dl release is done! Please restart."
log(msg)
self._callback(msg, False)
return True
except URLError as e:
log("YouTubeDLHelper error: {}".format(e))
raise self.YouTubeDLException(e)
finally:
self._is_update_process = False
def get_yt_link(self, url, skip_errors=False):
""" Returns tuple from the video links [dict] and title. """
if not self._dl:
self.init()
if self._is_update_process:
self._callback("Update process. Please wait.", False)
return {}, ""
try:
info = self._dl.extract_info(url, download=False)
except URLError as e:
log(str(e))
raise self.YouTubeDLException(e)
except self._DownloadError as e:
log(str(e))
if not skip_errors:
raise self.YouTubeDLException(e)
else:
fmts = info.get("formats", None)
if fmts:
return {Quality.get(int(fm["format_id"])): fm.get("url", "") for fm in fmts if
fm.get("format_id", "") in self._supported}, info.get("title", "")
return {}, info.get("title", "")
def flat(key, d):
for k, v in d.items():
if k == key:

View File

@@ -3,7 +3,7 @@
The MIT License (MIT)
Copyright (c) 2018-2019 Dmitriy Yefremov
Copyright (c) 2018-2020 Dmitriy Yefremov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -31,7 +31,7 @@ Author: Dmitriy Yefremov
<!-- interface-license-type mit -->
<!-- interface-name DemonEditor -->
<!-- interface-description Enigma2 channel and satellites list editor for GNU/Linux. -->
<!-- interface-copyright 2018-2019 Dmitriy Yefremov -->
<!-- interface-copyright 2018-2020 Dmitriy Yefremov -->
<!-- interface-authors Dmitriy Yefremov -->
<object class="GtkImage" id="remove_selection_image">
<property name="visible">True</property>
@@ -77,9 +77,6 @@ Author: Dmitriy Yefremov
<property name="decorated">False</property>
<property name="gravity">center</property>
<signal name="response" handler="on_response" swapped="no"/>
<child type="titlebar">
<placeholder/>
</child>
<child internal-child="vbox">
<object class="GtkBox" id="search_unavailable_dialog_box">
<property name="can_focus">False</property>
@@ -251,6 +248,9 @@ Author: Dmitriy Yefremov
<row>
<col id="0">none-REC2</col>
</row>
<row>
<col id="0">eServiceUri</col>
</row>
</data>
</object>
<object class="GtkDialog" id="iptv_dialog">
@@ -293,9 +293,6 @@ Author: Dmitriy Yefremov
<signal name="clicked" handler="on_save" swapped="no"/>
</object>
</child>
<child type="titlebar">
<placeholder/>
</child>
<child internal-child="vbox">
<object class="GtkBox" id="iptv_dialog_box">
<property name="can_focus">False</property>

View File

@@ -2,7 +2,7 @@ import concurrent.futures
import re
import urllib
from urllib.error import HTTPError
from urllib.parse import urlparse
from urllib.parse import urlparse, unquote, quote
from urllib.request import Request, urlopen
from gi.repository import GLib
@@ -11,11 +11,11 @@ from app.commons import run_idle, run_task
from app.eparser.ecommons import BqServiceType, Service
from app.eparser.iptv import NEUTRINO_FAV_ID_FORMAT, StreamType, ENIGMA2_FAV_ID_FORMAT, get_fav_id, MARKER_FORMAT
from app.settings import SettingsType
from app.tools.yt import YouTube, PlayListParser
from app.tools.yt import YouTube, PlayListParser, YouTubeDL
from .dialogs import Action, show_dialog, DialogType, get_dialogs_string, get_message
from .main_helper import get_base_model, get_iptv_url, on_popup_menu
from .uicommons import Gtk, Gdk, TEXT_DOMAIN, UI_RESOURCES_PATH, IPTV_ICON, Column, IS_GNOME_SESSION, KeyboardKey, \
get_yt_icon
from .uicommons import (Gtk, Gdk, TEXT_DOMAIN, UI_RESOURCES_PATH, IPTV_ICON, Column, IS_GNOME_SESSION, KeyboardKey,
get_yt_icon)
_DIGIT_ENTRY_NAME = "digit-entry"
_ENIGMA2_REFERENCE = "{}:0:{}:{:X}:{:X}:{:X}:{:X}:0:0:0"
@@ -38,12 +38,14 @@ def get_stream_type(box):
return StreamType.NONE_TS.value
elif active == 2:
return StreamType.NONE_REC_1.value
return StreamType.NONE_REC_2.value
elif active == 3:
return StreamType.NONE_REC_2.value
return StreamType.E_SERVICE_URI.value
class IptvDialog:
def __init__(self, transient, view, services, bouquet, profile=SettingsType.ENIGMA_2, action=Action.ADD):
def __init__(self, transient, view, services, bouquet, settings, action=Action.ADD):
handlers = {"on_response": self.on_response,
"on_entry_changed": self.on_entry_changed,
"on_url_changed": self.on_url_changed,
@@ -52,18 +54,20 @@ class IptvDialog:
"on_yt_quality_changed": self.on_yt_quality_changed,
"on_info_bar_close": self.on_info_bar_close}
self._action = action
self._s_type = settings.setting_type
self._settings = settings
self._bouquet = bouquet
self._services = services
self._yt_links = None
self._yt_dl = None
builder = Gtk.Builder()
builder.set_translation_domain(TEXT_DOMAIN)
builder.add_objects_from_string(get_dialogs_string(_UI_PATH).format(use_header=IS_GNOME_SESSION),
("iptv_dialog", "stream_type_liststore", "yt_quality_liststore"))
builder.connect_signals(handlers)
self._action = action
self._profile = profile
self._bouquet = bouquet
self._services = services
self._yt_links = None
self._dialog = builder.get_object("iptv_dialog")
self._dialog.set_transient_for(transient)
self._name_entry = builder.get_object("name_entry")
@@ -91,7 +95,7 @@ class IptvDialog:
for el in self._digit_elems:
el.get_style_context().add_provider_for_screen(Gdk.Screen.get_default(), self._style_provider,
Gtk.STYLE_PROVIDER_PRIORITY_USER)
if profile is SettingsType.NEUTRINO_MP:
if self._s_type is SettingsType.NEUTRINO_MP:
builder.get_object("iptv_dialog_ts_data_frame").set_visible(False)
builder.get_object("iptv_type_label").set_visible(False)
builder.get_object("reference_entry").set_visible(False)
@@ -104,8 +108,8 @@ class IptvDialog:
if self._action is Action.ADD:
self._save_button.set_visible(False)
self._add_button.set_visible(True)
if self._profile is SettingsType.ENIGMA_2:
self._update_reference_entry()
if self._s_type is SettingsType.ENIGMA_2:
self.update_reference_entry()
self._stream_type_combobox.set_active(1)
elif self._action is Action.EDIT:
self._current_srv = get_base_model(self._model)[self._paths][:]
@@ -129,13 +133,13 @@ class IptvDialog:
if show_dialog(DialogType.QUESTION, self._dialog) == Gtk.ResponseType.CANCEL:
return
self.save_enigma2_data() if self._profile is SettingsType.ENIGMA_2 else self.save_neutrino_data()
self.save_enigma2_data() if self._s_type is SettingsType.ENIGMA_2 else self.save_neutrino_data()
self._dialog.destroy()
def init_data(self, srv):
name, fav_id = srv[2], srv[7]
self._name_entry.set_text(name)
self.init_enigma2_data(fav_id) if self._profile is SettingsType.ENIGMA_2 else self.init_neutrino_data(fav_id)
self.init_enigma2_data(fav_id) if self._s_type is SettingsType.ENIGMA_2 else self.init_neutrino_data(fav_id)
def init_enigma2_data(self, fav_id):
data, sep, desc = fav_id.partition("#DESCRIPTION")
@@ -155,6 +159,8 @@ class IptvDialog:
self._stream_type_combobox.set_active(2)
elif stream_type is StreamType.NONE_REC_2:
self._stream_type_combobox.set_active(3)
elif stream_type is StreamType.E_SERVICE_URI:
self._stream_type_combobox.set_active(4)
except ValueError:
self.show_info_message("Unknown stream type {}".format(s_type), Gtk.MessageType.ERROR)
@@ -163,16 +169,17 @@ class IptvDialog:
self._tr_id_entry.set_text(str(int(data[4], 16)))
self._net_id_entry.set_text(str(int(data[5], 16)))
self._namespace_entry.set_text(str(int(data[6], 16)))
self._url_entry.set_text(urllib.request.unquote(data[10].strip()))
self._update_reference_entry()
self._url_entry.set_text(unquote(data[10].strip()))
self.update_reference_entry()
def init_neutrino_data(self, fav_id):
data = fav_id.split("::")
self._url_entry.set_text(data[0])
self._description_entry.set_text(data[1])
def _update_reference_entry(self):
if self._profile is SettingsType.ENIGMA_2:
def update_reference_entry(self):
if self._s_type is SettingsType.ENIGMA_2:
self.on_url_changed(self._url_entry)
self._reference_entry.set_text(_ENIGMA2_REFERENCE.format(self.get_type(),
self._srv_type_entry.get_text(),
int(self._sid_entry.get_text()),
@@ -188,12 +195,13 @@ class IptvDialog:
entry.set_name(_DIGIT_ENTRY_NAME)
else:
entry.set_name("GtkEntry")
self._update_reference_entry()
self.update_reference_entry()
def on_url_changed(self, entry):
url_str = entry.get_text()
url = urlparse(url_str)
entry.set_name("GtkEntry" if all([url.scheme, url.netloc, url.path]) else _DIGIT_ENTRY_NAME)
cond = all([url.scheme, url.netloc, url.path]) or self.get_type() == StreamType.E_SERVICE_URI.value
entry.set_name("GtkEntry" if cond else _DIGIT_ENTRY_NAME)
yt_id = YouTube.get_yt_id(url_str)
if yt_id:
@@ -211,10 +219,22 @@ class IptvDialog:
def set_yt_url(self, entry, video_id):
try:
links, title = YouTube.get_yt_link(video_id)
if self._settings.enable_yt_dl:
if not self._yt_dl:
def callback(message, error=True):
msg_type = Gtk.MessageType.ERROR if error else Gtk.MessageType.INFO
self.show_info_message(message, msg_type)
self._yt_dl = YouTubeDL.get_instance(self._settings, callback=callback)
yield True
links, title = self._yt_dl.get_yt_link(entry.get_text())
else:
links, title = YouTube.get_yt_link(video_id)
except urllib.error.URLError as e:
self.show_info_message(get_message("Getting link error:") + (str(e)), Gtk.MessageType.ERROR)
return
except YouTubeDL.YouTubeDLException as e:
self.show_info_message((str(e)), Gtk.MessageType.ERROR)
return
else:
if self._action is Action.ADD:
self._name_entry.set_text(title)
@@ -232,7 +252,9 @@ class IptvDialog:
yield True
def on_stream_type_changed(self, item):
self._update_reference_entry()
if self.get_type() == StreamType.E_SERVICE_URI.value:
self.show_info_message("DreamOS only!", Gtk.MessageType.WARNING)
self.update_reference_entry()
def on_yt_quality_changed(self, box):
model = box.get_model()
@@ -248,7 +270,7 @@ class IptvDialog:
int(self._tr_id_entry.get_text()),
int(self._net_id_entry.get_text()),
int(self._namespace_entry.get_text()),
urllib.request.quote(self._url_entry.get_text()),
quote(self._url_entry.get_text()),
name, name)
self.update_bouquet_data(name, fav_id)
@@ -291,7 +313,7 @@ class IptvDialog:
class SearchUnavailableDialog:
def __init__(self, transient, model, fav_bouquet, iptv_rows, profile):
def __init__(self, transient, model, fav_bouquet, iptv_rows, s_type):
handlers = {"on_response": self.on_response}
builder = Gtk.Builder()
@@ -305,7 +327,7 @@ class SearchUnavailableDialog:
self._counter_label = builder.get_object("streams_rows_counter_label")
self._level_bar = builder.get_object("unavailable_streams_level_bar")
self._bouquet = fav_bouquet
self._profile = profile
self._s_type = s_type
self._iptv_rows = iptv_rows
self._counter = -1
self._max_rows = len(self._iptv_rows)
@@ -333,7 +355,7 @@ class SearchUnavailableDialog:
if not self._download_task:
return
try:
req = Request(get_iptv_url(row, self._profile))
req = Request(get_iptv_url(row, self._s_type))
self.update_bar()
urlopen(req, timeout=2)
except HTTPError as e:
@@ -375,7 +397,7 @@ class SearchUnavailableDialog:
class IptvListConfigurationDialog:
def __init__(self, transient, services, iptv_rows, bouquet, fav_model, profile):
def __init__(self, transient, services, iptv_rows, bouquet, fav_model, s_type):
handlers = {"on_apply": self.on_apply,
"on_response": self.on_response,
"on_stream_type_default_togged": self.on_stream_type_default_togged,
@@ -389,18 +411,18 @@ class IptvListConfigurationDialog:
"on_entry_changed": self.on_entry_changed,
"on_info_bar_close": self.on_info_bar_close}
self._rows = iptv_rows
self._services = services
self._bouquet = bouquet
self._fav_model = fav_model
self._s_type = s_type
builder = Gtk.Builder()
builder.set_translation_domain(TEXT_DOMAIN)
builder.add_objects_from_string(get_dialogs_string(_UI_PATH).format(use_header=IS_GNOME_SESSION),
("iptv_list_configuration_dialog", "stream_type_liststore"))
builder.connect_signals(handlers)
self._rows = iptv_rows
self._services = services
self._bouquet = bouquet
self._fav_model = fav_model
self._profile = profile
self._dialog = builder.get_object("iptv_list_configuration_dialog")
self._dialog.set_transient_for(transient)
self._info_bar = builder.get_object("list_configuration_info_bar")
@@ -487,7 +509,7 @@ class IptvListConfigurationDialog:
show_dialog(DialogType.ERROR, self._dialog, "Error. Verify the data!")
return
if self._profile is SettingsType.ENIGMA_2:
if self._s_type is SettingsType.ENIGMA_2:
reset = self._reset_to_default_switch.get_active()
type_default = self._type_check_button.get_active()
tid_default = self._tid_check_button.get_active()
@@ -541,7 +563,7 @@ class IptvListConfigurationDialog:
class YtListImportDialog:
def __init__(self, transient, profile, appender):
def __init__(self, transient, settings, appender):
handlers = {"on_import": self.on_import,
"on_receive": self.on_receive,
"on_yt_url_entry_changed": self.on_url_entry_changed,
@@ -553,6 +575,14 @@ class YtListImportDialog:
"on_key_press": self.on_key_press,
"on_close": self.on_close}
self.appender = appender
self._s_type = settings.setting_type
self._download_task = False
self._yt_list_id = None
self._yt_list_title = None
self._settings = settings
self._yt_dl = None
builder = Gtk.Builder()
builder.set_translation_domain(TEXT_DOMAIN)
builder.add_objects_from_string(get_dialogs_string(_UI_PATH).format(use_header=IS_GNOME_SESSION),
@@ -584,12 +614,6 @@ class YtListImportDialog:
self._url_entry.get_style_context().add_provider_for_screen(Gdk.Screen.get_default(), self._style_provider,
Gtk.STYLE_PROVIDER_PRIORITY_USER)
self.appender = appender
self._profile = profile
self._download_task = False
self._yt_list_id = None
self._yt_list_title = None
def show(self):
self._dialog.show()
@@ -603,7 +627,14 @@ class YtListImportDialog:
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
done_links = {}
rows = list(filter(lambda r: r[2], self._model))
futures = {executor.submit(YouTube.get_yt_link, r[1]): r for r in rows}
if self._settings.enable_yt_dl:
if not self._yt_dl:
self._yt_dl = YouTubeDL.get_instance(self._settings)
futures = {executor.submit(self._yt_dl.get_yt_link,
YouTubeDL.VIDEO_LINK.format(r[1]),
True): r for r in rows}
else:
futures = {executor.submit(YouTube.get_yt_link, r[1]): r for r in rows}
size = len(futures)
counter = 0
@@ -615,6 +646,8 @@ class YtListImportDialog:
done_links[futures[future]] = future.result()
counter += 1
self.update_progress_bar(counter / size)
except YouTubeDL.YouTubeDLException as e:
self.show_info_message(str(e), Gtk.MessageType.ERROR)
except Exception as e:
self.show_info_message(str(e), Gtk.MessageType.ERROR)
else:
@@ -648,8 +681,8 @@ class YtListImportDialog:
self.update_active_elements(True)
def update_links(self, links):
for l in links:
yield self._model.append((l[0], l[1], True, None))
for link in links:
yield self._model.append((link[0], link[1], True, None))
size = len(self._model)
self._yt_count_label.set_text(str(size))
@@ -669,11 +702,11 @@ class YtListImportDialog:
act = self._quality_model.get_value(self._quality_box.get_active_iter(), 0)
for link in links:
lnk, title = link
lnk, title = link or (None, None)
if not lnk:
continue
ln = lnk.get(act) if act in lnk else lnk[sorted(lnk, key=lambda x: int(x.rstrip("p")), reverse=True)[0]]
fav_id = get_fav_id(ln, title, self._profile)
fav_id = get_fav_id(ln, title, self._s_type)
srv = Service(None, None, IPTV_ICON, title, *aggr[0:3], BqServiceType.IPTV.name, *aggr, None, fav_id, None)
srvs.append(srv)
self.appender(srvs)

View File

@@ -1739,7 +1739,7 @@ class Application(Gtk.Application):
self._fav_view,
self._services,
self._bouquets.get(self._bq_selected, None),
self._s_type,
self._settings,
Action.ADD).show()
if response != Gtk.ResponseType.CANCEL:
self.update_fav_num_column(self._fav_model)
@@ -1802,7 +1802,7 @@ class Application(Gtk.Application):
if not self._bq_selected:
return
YtListImportDialog(self._main_window, self._s_type, self.append_imported_services).show()
YtListImportDialog(self._main_window, self._settings, self.append_imported_services).show()
def on_import_m3u(self, action, value=None):
""" Imports iptv from m3u files. """
@@ -1910,6 +1910,8 @@ class Application(Gtk.Application):
url = get_iptv_url(row, self._s_type)
self.update_player_buttons()
if not url:
self.show_error_dialog("No reference is present!")
self.set_playback_elms_active()
return
self.play(url)
@@ -2460,7 +2462,7 @@ class Application(Gtk.Application):
self._fav_view,
self._services,
self._bouquets.get(self._bq_selected, None),
self._s_type,
self._settings,
Action.EDIT).show()
self.on_locate_in_services(view)