ChanSort/source/ChanSort.Loader.Panasonic/XmlSerializer.cs

using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Xml;
using System.Xml.Schema;
using ChanSort.Api;

namespace ChanSort.Loader.Panasonic
{
  /*

  MediaTek based Android TVs (e.g. Panasonic 2020 and later, Nokia, ...) use the same unstandardized compressed .bin binary format as many 
  Philips lists and some Sharp TVs.
  Additionally it exports a .xml file with very limited information that only includes channel numbers and channel names truncated at 8 bytes. 
  
  This truncation makes it impossible for a user to distinguish between channels that have longer names like "Sky Bundesliga ...", therefore
  this loader adds the "SvlId" as the ShortName. This SvlId is probably a "service list id" and refers to a a data record inside the .bin file.
  The truncation can also happen in the middle of a multi-type UTF-8 character sequence. Non-latin characters, including German umlauts or all
  cyrillic characters require 2 bytes/character, effectively reducing the channel name length to 4-8 characters.

  Another severe issue with these files is that XML special characters in channel names are not escaped properly. Some preprocessing is required
  in order to guess if a "&" is meant as an &amp; data value or an XML attribute. It's likely that < and > inside channel names have the same problem.

  When the TV has channels from various sources, it is not possible to determine to which internal source a channel belongs, making sorting of 
  sub-lists more or less impossible.

  */
  class XmlSerializer : SerializerBase
  {
    private readonly Dictionary<string, ChannelList> channelLists = new();
    public XmlDocument doc;
    public readonly IniFile ini;

    #region ctor()
    public XmlSerializer(string inputFile) : base(inputFile)
    {
      this.Features.ChannelNameEdit = ChannelNameEditMode.None;
      this.Features.CanSkipChannels = false;
      this.Features.CanLockChannels = false;
      this.Features.CanHideChannels = false;
      this.Features.DeleteMode = DeleteMode.NotSupported;
      this.Features.AllowGapsInFavNumbers = true;
      this.Features.CanEditFavListNames = false;

      string iniFile = Assembly.GetExecutingAssembly().Location.Replace(".dll", ".ini");
      this.ini = new IniFile(iniFile);
    }
    #endregion

    #region GetOrCreateList()
    private ChannelList GetOrCreateList(string name)
    {
      if (this.channelLists.TryGetValue(name, out var list))
        return list;
      
      list = new ChannelList(SignalSource.All, name);
      this.channelLists[name] = list;
      var cols = list.VisibleColumnFieldNames;
      cols.Clear();
      cols.Add("Position");
      cols.Add("OldPosition");
      cols.Add(nameof(ChannelInfo.Name));
      cols.Add(nameof(ChannelInfo.ShortName));
      this.DataRoot.AddChannelList(list);
      return list;
    }
    #endregion

    #region Load()
    public override void Load()
    {
      bool fail = false;
      try
      {
        doc = new XmlDocument();
        var content = File.ReadAllBytes(this.FileName);
        var textContent = Encoding.UTF8.GetString(content);
        textContent = FixUnescapedXmlChars(textContent);

        var settings = new XmlReaderSettings
        {
          CheckCharacters = false,
          IgnoreProcessingInstructions = true,
          ValidationFlags = XmlSchemaValidationFlags.None,
          DtdProcessing = DtdProcessing.Ignore
        };
        using var reader = XmlReader.Create(new StringReader(textContent), settings);
        doc.Load(reader);
      }
      catch
      {
        fail = true;
      }

      var root = doc.FirstChild;
      if (root is XmlDeclaration)
        root = root.NextSibling;
      if (fail || root == null || root.LocalName != "ChannelList" || !root.HasChildNodes || root.ChildNodes[0].LocalName != "ChannelInfo")
        throw LoaderException.TryNext("File is not a supported Panasonic XML file");

      string curListName = null;
      int index = 0;
      foreach (XmlNode child in root.ChildNodes)
      {
        switch (child.LocalName)
        {
          case "ChannelInfo":
            this.ReadChannel(child, index++, ref curListName);
            break;
        }
      }
    }
    #endregion

    #region FixUnescapedXmlChars()
    private string FixUnescapedXmlChars(string textContent)
    {
      var sb = new StringBuilder((int)(textContent.Length * 1.1));
      var inQuotes = false;
      foreach (var c in textContent)
      {
        if (c == '\"')
          inQuotes = !inQuotes;

        if (c == '&' && inQuotes)
          sb.Append("&amp;");
        else if (c == '<' && inQuotes)
          sb.Append("&lt;");
        else if (c == '>' && inQuotes)
          sb.Append("&gt;");
        else
          sb.Append(c);
      }

      return sb.ToString();
    }
    #endregion

    #region ReadChannel()
    private void ReadChannel(XmlNode node, int index, ref string curListName)
    {
      curListName ??= GetXmlValue(node, "ChannelType");

      var chan = new XmlChannel(index, node);

      // There is no clean indicator that distinguishes between Analog, DVB-C, DVB-T and DVB-S channel list. All nodes are in one flat list
      // The best guess is to start a new list when the number sequence resets
      var list = this.GetOrCreateList(curListName);
      if (list.Count > 0 && chan.OldProgramNr < list.Channels[list.Count - 1].OldProgramNr)
      {
        curListName = GetXmlValue(node, "ChannelType") + " " + (DataRoot.ChannelLists.Count() + 1);
        list = GetOrCreateList(curListName);
      }

      DataRoot.AddChannel(list, chan);
    }
    #endregion


    #region Save()

    public override void Save()
    {
      var sec = ini.GetSection("channel_list.xml");
      var reorder = sec?.GetBool("reorderRecordsByChannelNumber", true) ?? true;
      var setIsModified = sec?.GetBool("setIsModified", false) ?? false;

      foreach (var list in this.DataRoot.ChannelLists)
      {
        var seq = reorder ? list.Channels.OrderBy(c => c.NewProgramNr).ThenBy(c => c.RecordIndex).ToList() : list.Channels;
        XmlNode prevNode = null;
        foreach (var chan in seq)
        {
          if (chan is not XmlChannel ch)
            continue;
          SetXmlValue(ch.Node, "ChannelNumber", ch.NewProgramNr.ToString());
          
          if (setIsModified && ch.NewProgramNr != ch.OldProgramNr)
            SetXmlValue(ch.Node, "IsModified", "1");

          if (reorder)
          {
            var parent = ch.Node.ParentNode;
            parent.RemoveChild(ch.Node);
            parent.InsertAfter(ch.Node, prevNode);
            prevNode = ch.Node;
          }
        }
      }

      var xmlSettings = new XmlWriterSettings();
      xmlSettings.Encoding = new UTF8Encoding(false);
      xmlSettings.CheckCharacters = false;
      xmlSettings.Indent = true;
      xmlSettings.IndentChars = "";
      xmlSettings.NewLineHandling = NewLineHandling.None;
      xmlSettings.NewLineChars = "\n";
      xmlSettings.OmitXmlDeclaration = true;

      // write to a string so that we can patch the result to be binary identical to the original file (if there are no changes)
      using var stringWriter = new StringWriter();
      using var w = XmlWriter.Create(stringWriter, xmlSettings);
      doc.WriteTo(w);
      w.Flush();
      stringWriter.Write('\n'); // original file has a trailing \x0A
      var xml = stringWriter.ToString();
      xml = UnescapeXmlChars(xml); // create same broken XML as the original export with unescaped entities
      xml = xml.Replace(" />", "/>"); // original file has no space before the element end
      File.WriteAllText(this.FileName, xml, xmlSettings.Encoding);
    }

    #endregion

    #region UnescapeXmlChars()
    /// <summary>
    /// Generate the same broken XML with unescaped XML-entities as the original Panasonic XML export does (i.e. literal '&' character in channel names)
    /// </summary>
    private string UnescapeXmlChars(string xml)
    {
      bool inQuotes = false;
      bool inEntity = false;
      string entity = "";
      var sb = new StringBuilder(xml.Length);
      foreach (var c in xml)
      {
        if (inEntity)
        {
          if (c == ';')
          {
            switch (entity)
            {
              case "lt":
                sb.Append("<");
                break;
              case "gt":
                sb.Append(">");
                break;
              case "amp":
                sb.Append("&");
                break;
            }

            inEntity = false;
          }
          else
            entity += c;
          continue;
        }

        if (c == '"')
          inQuotes = !inQuotes;

        if (c == '&' && inQuotes)
        {
          inEntity = true;
          entity = "";
          continue;
        }

        sb.Append(c);
      }

      return sb.ToString();
    }
    #endregion

    #region GetXmlValue()
    static string GetXmlValue(XmlNode node, string field)
    {
      // old format stored all values as attributes of <ChannelInfo ...>
      if (node.Attributes != null && node.Attributes.Count > 0)
        return node.Attributes[field]?.InnerText;

      // new format with meaningful channel names stores all values as child elements
      foreach (XmlNode child in node.ChildNodes)
      {
        if (child is XmlElement elem && elem.LocalName == field)
          return elem.InnerText;
      }

      return "";
    }
    #endregion

    #region SetXmlValue()
    static void SetXmlValue(XmlNode node, string field, string value)
    {
      // old format stored all values as attributes of <ChannelInfo ...>
      if (node.Attributes != null && node.Attributes.Count > 0)
      {
        node.Attributes[field].InnerText = value;
        return;
      }

      // new format with meaningful channel names stores all values as child elements
      foreach (XmlNode child in node.ChildNodes)
      {
        if (child is XmlElement elem && elem.LocalName == field)
        {
          elem.InnerText = value;
          return;
        }
      }
    }
    #endregion

    #region class XmlChannel

    class XmlChannel : ChannelInfo
    {
      internal XmlNode Node;

      public XmlChannel(int index, XmlNode node) : base(0, index, 0, null)
      {
        this.Node = node;

        this.OldProgramNr = int.Parse(GetXmlValue(node, "ChannelNumber"));
        this.Name = GetXmlValue(node, "ChannelName");
        var svlId = GetXmlValue(node, "SvlId");
        if (svlId == "")
          svlId = GetXmlValue(node, "SvlRecId");
        this.ShortName = $"SvlId: {svlId}";
        if (int.TryParse(svlId, out var id))
          this.RecordOrder = id;
      }
    }
    #endregion
  }
}