diff --git a/source/ChanSort.Api/Controller/SerializerBase.cs b/source/ChanSort.Api/Controller/SerializerBase.cs index 37046fc..02ba5ea 100644 --- a/source/ChanSort.Api/Controller/SerializerBase.cs +++ b/source/ChanSort.Api/Controller/SerializerBase.cs @@ -242,5 +242,17 @@ namespace ChanSort.Api return 0; } #endregion + + #region ParseDecimal() + protected decimal ParseDecimal(string input) + { + if (string.IsNullOrWhiteSpace(input)) + return 0; + if (decimal.TryParse(input, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out var value)) + return value; + return 0; + } + #endregion + } } diff --git a/source/ChanSort.Api/Utils/Tools.cs b/source/ChanSort.Api/Utils/Tools.cs index 3a7c6b4..216499c 100644 --- a/source/ChanSort.Api/Utils/Tools.cs +++ b/source/ChanSort.Api/Utils/Tools.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Drawing; using System.Linq; using System.Text; +using System.Xml; namespace ChanSort.Api { @@ -149,11 +150,15 @@ namespace ChanSort.Api /// /// This method tests whether the binary data can be interpreted as valid UTF-8. If not, it might be encoded with a locale specific encoding /// - public static bool IsUtf8(byte[] buffer) + public static bool IsUtf8(byte[] buffer, int start=0, int count=-1) { + if (count < 0) + count = buffer.Length - start; + int followBytes = 0; - foreach (byte b in buffer) + for (int i = start, e=Math.Min(start+count, buffer.Length); i 0) { if ((b & 0xC0) != 0x80) // follow-up bytes must be 10xx xxxx @@ -227,6 +232,12 @@ namespace ChanSort.Api } #endregion + #region TrimGarbage() + /// + /// Remove a \0 and everything following it from a string + /// + /// + /// public static string TrimGarbage(this string input) { if (input == null) return null; @@ -235,6 +246,21 @@ namespace ChanSort.Api return input.Substring(0, i); return input; } + #endregion + + #region XmlNode: GetElement(), GetElementString(), GetElementInt() + public static XmlElement GetElement(this XmlNode node, string localName) + { + if (node is not XmlElement element) + return null; + var children = element.GetElementsByTagName(localName, node.NamespaceURI); + return children.Count >= 1 ? children[0] as XmlElement : null; + } + + public static string GetElementString(this XmlNode node, string localName) => GetElement(node, localName)?.InnerText; + public static int GetElementInt(this XmlNode node, string localName, int defaultValue = 0) => int.TryParse(GetElementString(node, localName), out var value) ? value : defaultValue; + + #endregion } } diff --git a/source/ChanSort.Loader.MediaTek/ChanSort.Loader.MediaTek.csproj b/source/ChanSort.Loader.MediaTek/ChanSort.Loader.MediaTek.csproj new file mode 100644 index 0000000..2842cd2 --- /dev/null +++ b/source/ChanSort.Loader.MediaTek/ChanSort.Loader.MediaTek.csproj @@ -0,0 +1,28 @@ + + + + net48 + Library + false + + + ..\Debug\ + latest + + + latest + + + ..\Debug\ + MinimumRecommendedRules.ruleset + latest + + + ..\Release\ + MinimumRecommendedRules.ruleset + latest + + + + + diff --git a/source/ChanSort.Loader.MediaTek/Channel.cs b/source/ChanSort.Loader.MediaTek/Channel.cs new file mode 100644 index 0000000..9af742e --- /dev/null +++ b/source/ChanSort.Loader.MediaTek/Channel.cs @@ -0,0 +1,15 @@ +using System.Xml; +using ChanSort.Api; + +namespace ChanSort.Loader.MediaTek +{ + internal class Channel : ChannelInfo + { + public XmlElement Xml { get; } + + internal Channel(SignalSource source, int index, int oldProgNr, string name, XmlElement element) :base(source, index, oldProgNr, name) + { + this.Xml = element; + } + } +} diff --git a/source/ChanSort.Loader.MediaTek/MediatekPlugin.cs b/source/ChanSort.Loader.MediaTek/MediatekPlugin.cs new file mode 100644 index 0000000..10abb77 --- /dev/null +++ b/source/ChanSort.Loader.MediaTek/MediatekPlugin.cs @@ -0,0 +1,22 @@ +using System.IO; +using ChanSort.Api; + +namespace ChanSort.Loader.MediaTek; + +public class MediatekPlugin : ISerializerPlugin +{ + public string DllName { get; set; } + public string PluginName => "MediaTek (MtkChannelList.xml)"; + public string FileFilter => "Mtk*.xml"; + + public SerializerBase CreateSerializer(string inputFile) + { + var dir = Path.GetDirectoryName(inputFile); + + // if there is a chanLst.bin file, let the Philips module handle the channel list + //if (File.Exists(Path.Combine(dir, "chanLst.bin"))) + // return null; + + return new Serializer(inputFile); + } +} \ No newline at end of file diff --git a/source/ChanSort.Loader.MediaTek/Properties/AssemblyInfo.cs b/source/ChanSort.Loader.MediaTek/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..8053afb --- /dev/null +++ b/source/ChanSort.Loader.MediaTek/Properties/AssemblyInfo.cs @@ -0,0 +1,33 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("ChanSort.Loader.MediaTek")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("ChanSort.Loader.MediaTek")] +[assembly: AssemblyCopyright("Copyright © 2024")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("5fc54726-b7ec-4a81-919f-f924110c723e")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/source/ChanSort.Loader.MediaTek/Serializer.cs b/source/ChanSort.Loader.MediaTek/Serializer.cs new file mode 100644 index 0000000..1475e81 --- /dev/null +++ b/source/ChanSort.Loader.MediaTek/Serializer.cs @@ -0,0 +1,250 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Xml; +using System.Xml.Schema; +using ChanSort.Api; + +namespace ChanSort.Loader.MediaTek; + +public class Serializer : SerializerBase +{ + /* + * Some Android based TVs export (in addition to the brand specific channel list files) a file named MtkChannelList.xml + * Examples are Philips channel list formats 120 and 125 + * + * + * + * + * + * + * + * 1=TV, 2=Radio + * + * + * + * service://SERVICE_LIST_GENERAL_SATELLITE/[service_list_id]/[major_channel_number] + * + * SID + * TSID + * NID + * (DVB-S2: MHz) + * ONID + * + * + * + * + * 0=false + * + * + * (base64 encoded Java serialized binary) + * (base64 encoded Java serialized binary) + * (base64 encoded Java serialized binary, which contains proprietary MediaTek compressed/encrypted cl_Zip data) + */ + + private XmlDocument doc; + private byte[] content; + private string textContent; + private readonly StringBuilder fileInfo = new(); + + private readonly Dictionary listsById = new(); + + + #region ctor() + public Serializer(string inputFile) : base(inputFile) + { + this.Features.ChannelNameEdit = ChannelNameEditMode.All; + this.Features.DeleteMode = DeleteMode.NotSupported; + this.Features.FavoritesMode = FavoritesMode.None; + this.Features.CanSkipChannels = false; + this.Features.CanLockChannels = true; + this.Features.CanHideChannels = false; + this.Features.CanSaveAs = true; + } + + #endregion + + #region Load() + public override void Load() + { + bool fail = false; + try + { + this.doc = new XmlDocument(); + this.content = File.ReadAllBytes(this.FileName); + this.textContent = Encoding.UTF8.GetString(this.content); + + var settings = new XmlReaderSettings + { + CheckCharacters = false, + IgnoreProcessingInstructions = true, + ValidationFlags = XmlSchemaValidationFlags.None, + DtdProcessing = DtdProcessing.Ignore + }; + using var reader = XmlReader.Create(new StringReader(textContent), settings); + doc.Load(reader); + } + catch + { + fail = true; + } + + var root = doc.FirstChild; + if (root is XmlDeclaration) + root = root.NextSibling; + if (fail || root == null || root.LocalName != "service_list_transfer") + throw LoaderException.TryNext("\"" + this.FileName + "\" is not a supported MediaTek XML file"); + + foreach (XmlNode child in root.ChildNodes) + { + switch (child.LocalName) + { + case "service_list_infos": + ReadServiceListInfos(child); + break; + case "internal": + // child elements: summary, scan, service_database + break; + } + } + } + #endregion + + #region ReadServiceListInfos() + private void ReadServiceListInfos(XmlNode serviceListInfosNode) + { + foreach (var sli in serviceListInfosNode.ChildNodes) + { + if (sli is XmlElement serviceListInfo) + this.ReadServiceList(serviceListInfo); + } + + foreach (var list in this.DataRoot.ChannelLists) + { + list.VisibleColumnFieldNames = ChannelList.DefaultVisibleColumns.ToList(); + list.VisibleColumnFieldNames.Remove("PcrPid"); + list.VisibleColumnFieldNames.Remove("VideoPid"); + list.VisibleColumnFieldNames.Remove("AudioPid"); + list.VisibleColumnFieldNames.Remove("ShortName"); + } + } + #endregion + + #region ReadServiceList() + private void ReadServiceList(XmlElement node) + { + SignalSource ss = SignalSource.Tv | SignalSource.Radio | SignalSource.Data | SignalSource.Dvb; + var slt = node.GetAttribute("service_list_type"); + if (slt.Contains("SATELLITE")) + ss |= SignalSource.Sat; + else if (slt.Contains("CABLE")) + ss |= SignalSource.Cable; + else if (slt.Contains("TERR")) + ss |= SignalSource.Antenna; + + + // service_list_id example: SERVICE_LIST_GENERAL_SATELLITE/17 + var serviceListId = node.GetAttribute("service_list_id"); + + var list = new ChannelList(ss, serviceListId); + this.listsById[serviceListId] = list; + + int idx = 0; + foreach (var child in node.ChildNodes) + { + if (!(child is XmlElement si && si.LocalName == "service_info")) + continue; + + ReadChannel(si, ss, idx++, list); + } + + this.DataRoot.AddChannelList(list); + } + #endregion + + #region ReadChannel() + + private ChannelInfo ReadChannel(XmlElement si, SignalSource ss, int idx, ChannelList list) + { + // record_id example: service://SERVICE_LIST_GENERAL_SATELLITE/17/1 + var recIdUri = si.GetElementString("record_id") ?? ""; + var i = recIdUri.LastIndexOf('/'); + var recId = int.Parse("0" + recIdUri.Substring(i + 1)); + + var chan = new Channel(ss, recId, -1, "", si); + chan.RecordOrder = idx; + + chan.OldProgramNr = si.GetElementInt("major_channel_number"); + // user_edit_flag ("none" in all observed records) + chan.Name = si.GetElementString("service_name"); + chan.ServiceType = si.GetElementInt("sdt_service_type"); + // visible_service ("3" in all observed records) + chan.ServiceId = si.GetElementInt("service_id"); + chan.TransportStreamId = si.GetElementInt("transport_stream_id"); + chan.FreqInMhz = si.GetElementInt("frequency"); + chan.OriginalNetworkId = si.GetElementInt("original_network_id"); + chan.SymbolRate = si.GetElementInt("symbol_rate"); + // modulation (not used by ChanSort) + var pol = si.GetElementInt("polarization"); + chan.Polarity = pol == 1 ? 'H' : pol == 2 ? 'V' : '\0'; + chan.Lock = si.GetElementInt("lock") != 0; + chan.Encrypted = si.GetElementInt("scrambled") != 0; + chan.Satellite = si.GetElementString("satelliteName"); + + if ((ss & SignalSource.Antenna) != 0) + chan.ChannelOrTransponder = LookupData.Instance.GetDvbtTransponder(chan.FreqInMhz).ToString(); + else if ((ss & SignalSource.Cable) != 0) + chan.ChannelOrTransponder = LookupData.Instance.GetDvbcTransponder(chan.FreqInMhz).ToString(); + + var elements = si.GetElementsByTagName("major_channel_number", si.NamespaceURI); + list.ReadOnly |= elements.Count == 1 && elements[0].Attributes["editable", si.NamespaceURI].InnerText == "false"; + + list.AddChannel(chan); + + + return chan; + } + #endregion + + + + #region GetFileInformation() + + public override string GetFileInformation() + { + var txt = base.GetFileInformation(); + return txt + "\n\n" + this.fileInfo; + } + + #endregion + + + #region Save() + public override void Save() + { + foreach (var list in this.DataRoot.ChannelLists) + { + foreach (var chan in list.Channels) + { + if (chan is not Channel ch || ch.IsProxy) + continue; + + var si = ch.Xml; + si["major_channel_number"].InnerText = ch.NewProgramNr.ToString(); + si["service_name"].InnerText = ch.Name; + si["lock"].InnerText = ch.Lock ? "1" : "0"; + si["visible_service"].InnerText = ch.Hidden ? "1" : "3"; + } + } + + var filePath = this.SaveAsFileName ?? this.FileName; + var settings = new XmlWriterSettings(); + settings.Indent = true; + settings.Encoding = new UTF8Encoding(false); + using var w = XmlWriter.Create(filePath, settings); + this.doc.WriteTo(w); + this.FileName = filePath; + } + #endregion +} diff --git a/source/ChanSort.Loader.Philips/ChanSort.Loader.Philips.csproj b/source/ChanSort.Loader.Philips/ChanSort.Loader.Philips.csproj index 1f6c35c..547c0b7 100644 --- a/source/ChanSort.Loader.Philips/ChanSort.Loader.Philips.csproj +++ b/source/ChanSort.Loader.Philips/ChanSort.Loader.Philips.csproj @@ -30,6 +30,7 @@ + diff --git a/source/ChanSort.Loader.Philips/ChanSort.Loader.Philips.ini b/source/ChanSort.Loader.Philips/ChanSort.Loader.Philips.ini index 5b0f774..34d8c61 100644 --- a/source/ChanSort.Loader.Philips/ChanSort.Loader.Philips.ini +++ b/source/ChanSort.Loader.Philips/ChanSort.Loader.Philips.ini @@ -454,3 +454,17 @@ incrementFavListVersion=false allowDelete=false userHiddenDefaultValue=3 dvbXmlLowercaseHexDigits=true + +############################################################################ +# ChannelMap_125: same as 120 + +[Map125] +padChannelName=true +setFavoriteNumber=false +setReorderedFavNumber=false +userReorderChannel=0 +reorderRecordsByChannelNumber=true +incrementFavListVersion=false +allowDelete=false +userHiddenDefaultValue=3 +dvbXmlLowercaseHexDigits=true diff --git a/source/ChanSort.Loader.Philips/Channel.cs b/source/ChanSort.Loader.Philips/Channel.cs index 796129b..a660caf 100644 --- a/source/ChanSort.Loader.Philips/Channel.cs +++ b/source/ChanSort.Loader.Philips/Channel.cs @@ -42,5 +42,7 @@ namespace ChanSort.Loader.Philips public int FlashFileOffset; public int DbFileOffset; + + public ChannelInfo MtkChannel; } } diff --git a/source/ChanSort.Loader.Philips/PhilipsPlugin.cs b/source/ChanSort.Loader.Philips/PhilipsPlugin.cs index 9fd1240..832cd85 100644 --- a/source/ChanSort.Loader.Philips/PhilipsPlugin.cs +++ b/source/ChanSort.Loader.Philips/PhilipsPlugin.cs @@ -77,7 +77,9 @@ namespace ChanSort.Loader.Philips * * version 120.0 * same as 105 plus additional ChannelList\MtkChannelList.xml - * + * + * version 125.0 + * same as 120 * * Version 0.1 and 100-120 are XML based and loaded through the XmlSerializer. * Version 1.1 and 1.2 are loaded through the BinSerializer. @@ -137,7 +139,7 @@ namespace ChanSort.Loader.Philips } } - if (majorVersion == 0 || majorVersion >= 100 && majorVersion <= 120) + if (majorVersion == 0 || majorVersion >= 100 && majorVersion <= 125) return new XmlSerializer(inputFile); if (majorVersion == 1 || majorVersion == 2 || majorVersion == 30 || majorVersion == 45) // || majorVersion == 11 // format version 11 is similar to 1.x, but not (yet) supported return new BinarySerializer(inputFile); diff --git a/source/ChanSort.Loader.Philips/XmlSerializer.cs b/source/ChanSort.Loader.Philips/XmlSerializer.cs index 1633808..4d06518 100644 --- a/source/ChanSort.Loader.Philips/XmlSerializer.cs +++ b/source/ChanSort.Loader.Philips/XmlSerializer.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Data; using System.IO; using System.Linq; using System.Reflection; @@ -8,6 +7,7 @@ using System.Text; using System.Xml; using System.Xml.Schema; using ChanSort.Api; +using ChanSort.Loader.MediaTek; namespace ChanSort.Loader.Philips { @@ -56,7 +56,7 @@ namespace ChanSort.Loader.Philips Onka does not update chanLst.bin (which isn't required when only DVBS.xml is modified since that file has no checksum in chanLst.bin) Nevertheless a user reported that swapping DVB-S channels 1 and 2 with Onka on a TV that uses this xml-only format 110 worked for him. - There seem to be 3 different flavors or the "100" format: + There seem to be 3 different flavors of the "100" format: One has only .xml files in the channellib and s2channellib folders, does not indent lines in the .xml files, has a fixed number of bytes for channel and satellite names (padded with 0x00), has no "Scramble" attribute and values 1 and 0 for "Polarization". And a version that has dtv_cmdb_*.bin next to the .xml files, uses 4 spaces for indentation, only writes as many bytes for names as needed, has a "Scramble" attribute and uses values 1 and 2 for "Polarization". While the first seems to work fine when XML nodes are reordered by their new programNr, the latter seems to get confused when the .bin and .xml files have different data record orders. This is still under investigation. @@ -90,6 +90,11 @@ namespace ChanSort.Loader.Philips DVB-T and DVB-C share the same number range, so they are treated as a unified logical list + + Version 120 and 125 (MediaTek platform for Google TVs) contain an additional MtkChannelList.xml file, which is also used by other brands. + This file contains both a readable channel list as XML as well as a base64 encoded Java serialized Blob containing a cl_Zip compressed/encrypted binary channel list. + A separate Loader module is used for this file to keep data in-sync. + */ class XmlSerializer : SerializerBase { @@ -106,6 +111,7 @@ namespace ChanSort.Loader.Philips private readonly IniFile ini; private IniFile.Section iniMapSection; private string polarizationValueForHorizontal = "1"; + private MediaTek.Serializer mtkSerializer; #region ctor() public XmlSerializer(string inputFile) : base(inputFile) @@ -169,6 +175,7 @@ namespace ChanSort.Loader.Philips // ChannelMap_100/ChannelList/channelFile.bin // ChannelMap_105/ChannelList/Favorite.xml // ChannelMap_100/ChannelList/satInfo.bin + // ChannelMap_120/ChannelList/MtkChannelList.xml var dataFiles = new[] { @"channellib\DVBC.xml", @"channellib\DVBT.xml", @"s2channellib\DVBS.xml", @"s2channellib\DVBSall.xml", @"Favorite.xml" }; @@ -218,6 +225,8 @@ namespace ChanSort.Loader.Philips } if (this.fileDataList.Count == 0) throw LoaderException.TryNext("No XML files found in folder structure"); + + LoadAndValidateMtkChannelList(dir); } else { @@ -491,10 +500,10 @@ namespace ChanSort.Loader.Philips { chan.Format = 2; chan.RawSatellite = data.TryGet("SatelliteName"); - chan.Satellite = DecodeName(chan.RawSatellite); + chan.Satellite = DecodeName(chan.RawSatellite, NameType.Satellite); chan.OldProgramNr = ParseInt(data.TryGet("ChannelNumber")); chan.RawName = data.TryGet("ChannelName"); - chan.Name = DecodeName(chan.RawName); + chan.Name = DecodeName(chan.RawName, NameType.Channel); chan.Lock = data.TryGet("ChannelLock") == "1"; chan.Hidden = data.TryGet("UserHidden") == "1"; // can be "3" instead of "0", at least in format 120 var fav = ParseInt(data.TryGet("FavoriteNumber")); @@ -540,7 +549,7 @@ namespace ChanSort.Loader.Philips private void ReadFavList(XmlNode node) { int index = ParseInt(node.Attributes["Index"].InnerText); - string name = DecodeName(node.Attributes["Name"].InnerText); + string name = DecodeName(node.Attributes["Name"].InnerText, NameType.FavList); this.Features.FavoritesMode = FavoritesMode.MixedSource; this.Features.MaxFavoriteLists = Math.Max(this.Features.MaxFavoriteLists, index); @@ -561,48 +570,138 @@ namespace ChanSort.Loader.Philips } } - var startNrBias = (chanLstBin?.VersionMajor ?? 0) >= 120 ? 0 : +1; - + int favNr = 0; foreach (XmlNode child in node.ChildNodes) { if (child.LocalName == "FavoriteChannel") { var uniqueId = ParseLong(child["UniqueID"].InnerText); - var favNumber = ParseInt(child["FavNumber"].InnerText); + //var favNumber = ParseDecimal(child["FavNumber"].InnerText); // this is a Decimal like "1.5" when the previous "0" was moved behind "1", which makes the new list start at 1 instead of 2 var chan = this.favChannels.Channels.FirstOrDefault(ch => ch.RecordIndex == uniqueId && ch.GetOldPosition(index) <= 0); - chan?.SetOldPosition(index, favNumber + startNrBias); + chan?.SetOldPosition(index, ++favNr); } } } #endregion #region DecodeName() - private string DecodeName(string input) + private string DecodeName(string input, NameType nameType) { if (input == null || !input.StartsWith("0x")) // fallback for unknown input return input; - // according to https://github.com/PredatH0r/ChanSort/issues/347 Philips seems to not use UTF 16, but instead use locale dependent encoding and - // writing "0xAA 0x00" to the file for an 8 bit code point. At least for the favorite list captions. Congratulations, well done! + // The Philips encodes names is a complete mess. + // Each character is represented as two bytes, with the low byte first and the high second, but this isn't utf16. + // All observed files have the "high" byte always as 0x00 + // If looking only at the odd bytes, this can either be encoded in some random locale, a valid utf8 sequence or 1 byte characters mixed with big-endian utf16 double-bytes characters. + + // according to https://github.com/PredatH0r/ChanSort/issues/347 Philips seems use a locale dependent encoding for favorite list names, + // writing "0xAA 0x00" to the file for an 8 bit code point. Congratulations, well done! + + // In version 120/125 umlauts in channel names are encoded as 1 byte CP-1252/UTF16 code point + 0xFF as the second byte (i.e. for "Ä" it is 0xC4 0xFF instead of 0xC4 0x00) + // Also: 0x62 0x00 0x65 0x00 0x49 0x00 0x4e 0x00 0x20 0x00 0x01 0x00 0x30 0x00 0x5a - here 0x01 0x00 0x30 0x00 refers to U+0130 (the upper case I with dot), in "beIN İZ" + + // https://github.com/PredatH0r/ChanSort/issues/421: 0x38 0x00 0x20 0x00 0xD0 0x00 0xBA ... seems to contain cyrillic UTF-8 encoding in channel names instead of UTF-16 - // In version 120 umlauts in channel names are encoded as 1 byte CP-1252 (= low byte UTF16) code point + 0xFF as the second byte var hexParts = input.Split(' '); - var buffer = new MemoryStream(); + var utf16 = new MemoryStream(); + var utf8 = new MemoryStream(); bool highByte = false; + bool invalidUtf8 = false; + byte bigEndianUnicodeHighByte = 0; + int bigEndianUnicodeIndex = -1; foreach (var part in hexParts) { if (part == "") continue; var val = (byte)ParseInt(part); + invalidUtf8 |= highByte && val != 0; if (highByte && val == 0xff) // hack-around for version 120 val = 0; - buffer.WriteByte(val); + + if (bigEndianUnicodeIndex >= 0) // special handling when a character < 32 was detected, which means we have a messed up "HI 00 LO 00" encoding for an UTF16 character (where HI is < 32) + { + ++bigEndianUnicodeIndex; + if (bigEndianUnicodeIndex == 2) + { + utf16.WriteByte(val); + utf16.WriteByte(bigEndianUnicodeHighByte); + bigEndianUnicodeHighByte = 0; + } + else if (bigEndianUnicodeIndex == 3) + bigEndianUnicodeIndex = -1; + } + else + { + if (!highByte) + { + if (val < 32 && val != 0) // a char < 32 is likely the high byte of a "HI 00 LO 00" encoded UTF16 character + { + bigEndianUnicodeHighByte = val; + bigEndianUnicodeIndex = 0; + invalidUtf8 = true; + } + else if (!invalidUtf8) + utf8.WriteByte(val); + } + if (bigEndianUnicodeIndex < 0) + utf16.WriteByte(val); + } + + highByte = !highByte; } - return Encoding.Unicode.GetString(buffer.GetBuffer(), 0, (int) buffer.Length).TrimEnd('\x0'); + // in the FavList the name can be a random locale based on the country setting (other than CP-1252 or U-0000-00FF) + if (nameType == NameType.FavList) + return this.DefaultEncoding.GetString(utf8.GetBuffer(), 0, (int)utf8.Length).TrimGarbage(); + + // e.g. for cyrillic names, where only the low-byte is used for an utf8 encoding while the high-byte is always 0 + if (!invalidUtf8 && Tools.IsUtf8(utf8.GetBuffer(), 0, (int)utf8.Length)) + return Encoding.UTF8.GetString(utf8.GetBuffer(), 0, (int)utf8.Length).TrimGarbage(); + + return Encoding.Unicode.GetString(utf16.GetBuffer(), 0, (int) utf16.Length).TrimGarbage(); + } + #endregion + + #region LoadAndValidateMtkChannelList() + private void LoadAndValidateMtkChannelList(string dir) + { + var path = Path.Combine(dir, "MtkChannelList.xml"); + if (!File.Exists(path)) + return; + + this.mtkSerializer = new Serializer(path); + this.mtkSerializer.Load(); + foreach (var list1 in this.DataRoot.ChannelLists) + { + if (list1.Channels.Count == 0 || (list1.SignalSource & SignalSource.Analog) != 0) + continue; + + var list2 = mtkSerializer.DataRoot.GetChannelList(list1.SignalSource); + if (list2 == null) + throw LoaderException.Fail("MtkChannelList.xml doesn't contain a list for " + list1.SignalSource); + + if (list1.Channels.Count != list2.Channels.Count) + throw LoaderException.Fail("MtkChannelList.xml contains a different number of channels for " + list1.SignalSource); + + foreach (var ch1 in list1.Channels) + { + var others = list2.GetChannelByUid(ch1.Uid).Where(c => Math.Abs(ch1.FreqInMhz - c.FreqInMhz) < 2).ToList(); + if (others.Count == 0) + throw LoaderException.Fail("MtkChannelList.xml doesn't contain a matching channel for " + ch1); + if (others.Count > 1) + throw LoaderException.Fail("MtkChannelList.xml contains multiple matching channel for " + ch1); + + var ch2 = others[0]; + if (ch1.OldProgramNr != ch2.OldProgramNr) + throw LoaderException.Fail("MtkChannelList.xml contains a different channel number for " + ch1); + + ((Channel)ch1).MtkChannel = ch2; + } + } } #endregion @@ -613,6 +712,10 @@ namespace ChanSort.Loader.Philips if (this.chanLstBin != null) list.Add(this.FileName); list.AddRange(this.fileDataList.Select(f => f.path)); + + if (this.mtkSerializer != null) + list.AddRange(this.mtkSerializer.GetDataFilePaths()); + return list; } #endregion @@ -634,7 +737,7 @@ namespace ChanSort.Loader.Philips // It is unclear whether XML nodes must be sorted by the new program number or kept in the original order. This may be different for the various format versions. // Onka, which was made for the ChannelMap_100 flavor that doesn't export dtv_cmdb_2.bin files, reorders the XML nodes and users reported that it works. // The official Philips Editor 6.61.22 does not reorder the XML nodes and does not change dtv_cmdb_*.bin when editing a ChannelMap_100 folder. But it is unclear if this editor is designed to handle the cmdb flavor. - + // A user with a ChannelMap_100 export including a dtv_cmdb_2.bin reported, that the TV shows the reordered list in the menu, but tunes the channels based on the original numbers. // It's unclear if that happens because the XML was reordered and out-of-sync with the .bin, or if the TV always uses the .bin for tuning and XML edits are moot. // On top of that this TV messed up Umlauts during the import, despite ChanSort writing the exact same name data in hex-encoded UTF16. The result was as if the string was exported as UTF-8 bytes and then parsed with an 8-bit code page. @@ -649,6 +752,7 @@ namespace ChanSort.Loader.Philips } this.chanLstBin?.Save(this.FileName); + this.mtkSerializer?.Save(); } #endregion @@ -676,7 +780,11 @@ namespace ChanSort.Loader.Philips if (ch.Format == 1) this.UpdateRepairXml(ch); else if (ch.Format == 2) + { this.UpdateChannelMapXml(ch, padChannelNameBytes, setFavoriteNumber, userReorderChannel, uppercaseHexDigits); + if (ch.MtkChannel != null) + this.UpdateMtkChannel(ch); + } } } #endregion @@ -747,6 +855,18 @@ namespace ChanSort.Loader.Philips #endregion + #region UpdateMtkChannel() + private void UpdateMtkChannel(Channel channel) + { + var mtk = channel.MtkChannel; + //mtk.Name = channel.Name; + mtk.NewProgramNr = channel.NewProgramNr; + mtk.Lock = channel.Lock; + mtk.Skip = channel.Skip; + mtk.Hidden = channel.Hidden; + } + #endregion + #region UpdateFavList() private void UpdateFavList() { @@ -771,9 +891,9 @@ namespace ChanSort.Loader.Philips attr.InnerText = (version + 1).ToString(); } - var startNrBias = (chanLstBin?.VersionMajor ?? 0) >= 120 ? 0 : -1; - var uniqueIdFormat = (chanLstBin?.VersionMajor ?? 0) >= 120 ? "d10" : "d"; // v120 writes 10 digits as uniqueID, including leading zeros + var uniqueIdFormat = (chanLstBin?.VersionMajor ?? 0) == 120 ? "d10" : "d"; // v120 writes 10 digits as uniqueID, including leading zeros; v100,v110 and v125 don't + int favNr = 0; foreach (var ch in favChannels.Channels.OrderBy(ch => ch.GetPosition(index))) { var nr = ch.GetPosition(index); @@ -782,13 +902,13 @@ namespace ChanSort.Loader.Philips var uniqueIdNode = favFile.doc.CreateElement("UniqueID"); uniqueIdNode.InnerText = ch.RecordIndex.ToString(uniqueIdFormat); var favNrNode = favFile.doc.CreateElement("FavNumber"); - favNrNode.InnerText = (nr + startNrBias).ToString(); + favNrNode.InnerText = (++favNr).ToString(); var channelNode = favFile.doc.CreateElement("FavoriteChannel"); channelNode.AppendChild(uniqueIdNode); channelNode.AppendChild(favNrNode); - // Version 120 also stores a element in the XML - if ((chanLstBin?.VersionMajor ?? 0) >= 120) + // Version 120 also stores a element in the XML, but not 100, 110, 115, 125 + if ((chanLstBin?.VersionMajor ?? 0) == 120) { var chanTypeNode = favFile.doc.CreateElement("ChannelType"); string type = null; @@ -904,6 +1024,10 @@ namespace ChanSort.Loader.Philips } #endregion + #region enum NameType + private enum NameType { Channel, Satellite, FavList } + #endregion + #region class FileData private class FileData { diff --git a/source/ChanSort.Loader.Samsung/ChanSort.Loader.Samsung.ini b/source/ChanSort.Loader.Samsung/ChanSort.Loader.Samsung.ini index f9a9340..26e82fe 100644 --- a/source/ChanSort.Loader.Samsung/ChanSort.Loader.Samsung.ini +++ b/source/ChanSort.Loader.Samsung/ChanSort.Loader.Samsung.ini @@ -125,6 +125,44 @@ offIndex = 2 offLenName = 6 offName = 8 + +[DvbCT:112] + ; map-AirD, map-CableD for HG40NC460KFXZA hospitality model (USA, T-NT14LAKUCB) + offProgramNr = 0 + offVideoPid = 2 + offPcrPid = 4 + offServiceId = 6 + offDeleted = 8 + maskDeleted = 0x01 + offSignalSource = 10 + offQam = 12 + offSkip = 13 + maskSkip = 0x01 + offBandwidth = 14 + offServiceType = 15 + offCodec = 16 + offHRes = 20 + offVRes = 22 + offEncrypted = 24 + maskEncrypted = 0x01 + offHidden = 25 + offHiddenAlt = 26 + offSymbolRate = 28 + offLock = 31 + maskLock = 0x01 + offOriginalNetworkId = 32 + offNetworkId = 34 + offServiceProviderId = 40 + offChannelTransponder = 42 + offLogicalProgramNr = 44 + offTransportStreamId = 48 + offName = 64 + lenName = 100 + offShortName = 264 + lenShortName = 18 + offVideoFormat = 282 + offFavorites = 292,296,300,304,308 + offChecksum = 319 [DvbCT:248] ; map-AirD and map-CableD for B models diff --git a/source/ChanSort.sln b/source/ChanSort.sln index 9297de6..cccd911 100644 --- a/source/ChanSort.sln +++ b/source/ChanSort.sln @@ -160,6 +160,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ChanSort.Loader.TechniSat", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ChanSort.Loader.Amdb", "ChanSort.Loader.Amdb\ChanSort.Loader.Amdb.csproj", "{30E9D084-6F3C-41A9-9B46-846178C91BDB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ChanSort.Loader.MediaTek", "ChanSort.Loader.MediaTek\ChanSort.Loader.MediaTek.csproj", "{5FC54726-B7EC-4A81-919F-F924110C723E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution All_Debug|Any CPU = All_Debug|Any CPU @@ -1467,6 +1469,36 @@ Global {30E9D084-6F3C-41A9-9B46-846178C91BDB}.Release|Mixed Platforms.Build.0 = Release|Any CPU {30E9D084-6F3C-41A9-9B46-846178C91BDB}.Release|x86.ActiveCfg = Release|Any CPU {30E9D084-6F3C-41A9-9B46-846178C91BDB}.Release|x86.Build.0 = Release|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.All_Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.All_Debug|Any CPU.Build.0 = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.All_Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.All_Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.All_Debug|x86.ActiveCfg = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.All_Debug|x86.Build.0 = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.All_Release|Any CPU.ActiveCfg = Release|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.All_Release|Any CPU.Build.0 = Release|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.All_Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.All_Release|Mixed Platforms.Build.0 = Release|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.All_Release|x86.ActiveCfg = Release|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.All_Release|x86.Build.0 = Release|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.Debug|x86.ActiveCfg = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.Debug|x86.Build.0 = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.NoDevExpress_Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.NoDevExpress_Debug|Any CPU.Build.0 = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.NoDevExpress_Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.NoDevExpress_Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.NoDevExpress_Debug|x86.ActiveCfg = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.NoDevExpress_Debug|x86.Build.0 = Debug|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.Release|Any CPU.Build.0 = Release|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.Release|x86.ActiveCfg = Release|Any CPU + {5FC54726-B7EC-4A81-919F-F924110C723E}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/source/ChanSort/ChanSort.csproj b/source/ChanSort/ChanSort.csproj index 94a04f8..4cf1f3c 100644 --- a/source/ChanSort/ChanSort.csproj +++ b/source/ChanSort/ChanSort.csproj @@ -136,6 +136,7 @@ + diff --git a/source/Test.Api/Test.Api.csproj b/source/Test.Api/Test.Api.csproj index e2939ab..7ce2f41 100644 --- a/source/Test.Api/Test.Api.csproj +++ b/source/Test.Api/Test.Api.csproj @@ -17,8 +17,8 @@ - - + + \ No newline at end of file diff --git a/source/Test.Loader.CmdbBin/Test.Loader.CmdbBin.csproj b/source/Test.Loader.CmdbBin/Test.Loader.CmdbBin.csproj index cfb5d11..58264da 100644 --- a/source/Test.Loader.CmdbBin/Test.Loader.CmdbBin.csproj +++ b/source/Test.Loader.CmdbBin/Test.Loader.CmdbBin.csproj @@ -5,8 +5,8 @@ - - + + diff --git a/source/Test.Loader.Enigma2/Test.Loader.Enigma2.csproj b/source/Test.Loader.Enigma2/Test.Loader.Enigma2.csproj index 712bb71..5265e03 100644 --- a/source/Test.Loader.Enigma2/Test.Loader.Enigma2.csproj +++ b/source/Test.Loader.Enigma2/Test.Loader.Enigma2.csproj @@ -18,8 +18,8 @@ - - + + \ No newline at end of file diff --git a/source/Test.Loader.Grundig/Test.Loader.Grundig.csproj b/source/Test.Loader.Grundig/Test.Loader.Grundig.csproj index 9e4ff48..53dd00d 100644 --- a/source/Test.Loader.Grundig/Test.Loader.Grundig.csproj +++ b/source/Test.Loader.Grundig/Test.Loader.Grundig.csproj @@ -18,8 +18,8 @@ - - + + \ No newline at end of file diff --git a/source/Test.Loader.Hisense/Test.Loader.Hisense.csproj b/source/Test.Loader.Hisense/Test.Loader.Hisense.csproj index 7ee6abf..20516ad 100644 --- a/source/Test.Loader.Hisense/Test.Loader.Hisense.csproj +++ b/source/Test.Loader.Hisense/Test.Loader.Hisense.csproj @@ -13,8 +13,8 @@ - - + + diff --git a/source/Test.Loader.LG/Test.Loader.LG.csproj b/source/Test.Loader.LG/Test.Loader.LG.csproj index 0d93689..326441f 100644 --- a/source/Test.Loader.LG/Test.Loader.LG.csproj +++ b/source/Test.Loader.LG/Test.Loader.LG.csproj @@ -35,8 +35,8 @@ - - + + \ No newline at end of file diff --git a/source/Test.Loader.M3u/Test.Loader.M3u.csproj b/source/Test.Loader.M3u/Test.Loader.M3u.csproj index ea18df4..6169987 100644 --- a/source/Test.Loader.M3u/Test.Loader.M3u.csproj +++ b/source/Test.Loader.M3u/Test.Loader.M3u.csproj @@ -26,8 +26,8 @@ - - + + diff --git a/source/Test.Loader.Panasonic/Test.Loader.Panasonic.csproj b/source/Test.Loader.Panasonic/Test.Loader.Panasonic.csproj index f07ea84..5ae253e 100644 --- a/source/Test.Loader.Panasonic/Test.Loader.Panasonic.csproj +++ b/source/Test.Loader.Panasonic/Test.Loader.Panasonic.csproj @@ -13,8 +13,8 @@ - - + + diff --git a/source/Test.Loader.Philips/Test.Loader.Philips.csproj b/source/Test.Loader.Philips/Test.Loader.Philips.csproj index b29f2d3..329307a 100644 --- a/source/Test.Loader.Philips/Test.Loader.Philips.csproj +++ b/source/Test.Loader.Philips/Test.Loader.Philips.csproj @@ -205,8 +205,8 @@ - - + + \ No newline at end of file diff --git a/source/Test.Loader.Samsung/Test.Loader.Samsung.csproj b/source/Test.Loader.Samsung/Test.Loader.Samsung.csproj index a2de908..84b7f37 100644 --- a/source/Test.Loader.Samsung/Test.Loader.Samsung.csproj +++ b/source/Test.Loader.Samsung/Test.Loader.Samsung.csproj @@ -21,8 +21,8 @@ - - + + diff --git a/source/Test.Loader.SatcoDX/Test.Loader.SatcoDX.csproj b/source/Test.Loader.SatcoDX/Test.Loader.SatcoDX.csproj index f1b11a1..ae17a1b 100644 --- a/source/Test.Loader.SatcoDX/Test.Loader.SatcoDX.csproj +++ b/source/Test.Loader.SatcoDX/Test.Loader.SatcoDX.csproj @@ -21,8 +21,8 @@ - - + + \ No newline at end of file diff --git a/source/Test.Loader.Sharp/Test.Loader.Sharp.csproj b/source/Test.Loader.Sharp/Test.Loader.Sharp.csproj index ee48789..67b248e 100644 --- a/source/Test.Loader.Sharp/Test.Loader.Sharp.csproj +++ b/source/Test.Loader.Sharp/Test.Loader.Sharp.csproj @@ -18,7 +18,7 @@ - - + + \ No newline at end of file diff --git a/source/Test.Loader.Sony/Test.Loader.Sony.csproj b/source/Test.Loader.Sony/Test.Loader.Sony.csproj index a67f71a..a67d568 100644 --- a/source/Test.Loader.Sony/Test.Loader.Sony.csproj +++ b/source/Test.Loader.Sony/Test.Loader.Sony.csproj @@ -17,8 +17,8 @@ - - + + \ No newline at end of file diff --git a/source/Test.Loader.Toshiba/Test.Loader.Toshiba.csproj b/source/Test.Loader.Toshiba/Test.Loader.Toshiba.csproj index dee569a..3595c88 100644 --- a/source/Test.Loader.Toshiba/Test.Loader.Toshiba.csproj +++ b/source/Test.Loader.Toshiba/Test.Loader.Toshiba.csproj @@ -21,8 +21,8 @@ - - + + diff --git a/source/Test.Loader.VDR/Test.Loader.VDR.csproj b/source/Test.Loader.VDR/Test.Loader.VDR.csproj index f099332..6436c8c 100644 --- a/source/Test.Loader.VDR/Test.Loader.VDR.csproj +++ b/source/Test.Loader.VDR/Test.Loader.VDR.csproj @@ -22,8 +22,8 @@ - - + + \ No newline at end of file diff --git a/source/Test.Loader/Test.Loader.csproj b/source/Test.Loader/Test.Loader.csproj index 33d7cd5..6304f21 100644 --- a/source/Test.Loader/Test.Loader.csproj +++ b/source/Test.Loader/Test.Loader.csproj @@ -1,4 +1,5 @@  + net48 Library @@ -23,7 +24,7 @@ - - + + \ No newline at end of file diff --git a/source/changelog.md b/source/changelog.md index 1693f23..3fb7b04 100644 --- a/source/changelog.md +++ b/source/changelog.md @@ -1,6 +1,11 @@ ChanSort Change Log =================== +2024-09-31 +- experimental support for Philips channel list format 125 (with automatic sync to MtkChannelList.xml) +- experimental support for MtkChannelList.xml (which is part of several MediaTek based Google TVs, e.g. Philips formats 120 and 125) +- Philips formats 100-125: improved decoding of non-latin characters (turkish, cyrillic, ...) + 2024-08-18 - added support for dtv_cmdb_3.bin with file size 1323920 (e.g. Grundig 37 VLE 9270 SL) - fixed error in the reference list import dialog when using Italian language diff --git a/source/makeDistribZip.cmd b/source/makeDistribZip.cmd index 8b07992..70c1150 100644 --- a/source/makeDistribZip.cmd +++ b/source/makeDistribZip.cmd @@ -69,15 +69,14 @@ goto:eof set oldcd=%cd% cd %target% -call :signBatch ChanSort.exe ChanSort*.dll -if errorlevel 1 goto :error -set files= +set filesToSign=ChanSort.exe ChanSort*.dll for %%l in (%languages%) do ( - call :signBatch "%%l\ChanSort*.dll" - if errorlevel 1 goto :error + set filesToSign=!filesToSign! "%%l\ChanSort*.dll" ) +call :signBatch !filesToSign! cd %oldcd% goto:eof + :signBatch set todo= for %%f in (%*) do (