- Philips: added support for CM\_*.xml variant that uses a <ECSM> root element around the <ChannelMap>

- Philips: ability to read/write broken CM\_*.xml files that contain channel names with an unescaped & character
- Philips: enabled write mode for Repair\\FLASH\_\*/\*.db file format (one variant was confirmed to work)
  Favorite lists for this format are disabled for now (TV ignored them).
- Panasonic: importing a modified svl.bin file caused the TV to use case-sensitive sorting when using the
  function to list the names sorted alphabetically. This is now fixed.

source/ChanSort.Loader.Panasonic/DbChannel.cs

@@ -133,9 +133,6 @@ namespace ChanSort.Loader.Panasonic
     {
       byte[] buffer = new byte[100];
       int len = (int)r.GetBytes(field["sname"], 0, buffer, 0, buffer.Length);
-      int end = Array.IndexOf<byte>(buffer, 0, 0, len);
-      if (end >= 0)
-        len = end;
       this.RawName = new byte[len];
       Array.Copy(buffer, 0, this.RawName, 0, len);
       this.ChangeEncoding(encoding);      
@@ -152,22 +149,25 @@ namespace ChanSort.Loader.Panasonic
       // it can be code page encoded without any clue to what the code page is
       // it can have DVB-control characters inside an UTF-8 stream
 
-      if (RawName.Length == 0)
+      int len = Array.IndexOf<byte>(this.RawName, 0, 0, this.RawName.Length);
+      if (len < 0)
+        len = this.RawName.Length;
+      if (len == 0)
         return;
 
-      int startOffset;
-      int bytesPerChar;
-      if (!GetRecommendedEncoding(ref encoding, out startOffset, out bytesPerChar)) 
+      if (!GetRecommendedEncoding(ref encoding, out var startOffset, out var bytesPerChar)) 
         return;
 
       // single byte code pages might have UTF-8 code mixed in, so we have to parse it manually
       StringBuilder sb = new StringBuilder();
       this.NonAscii = false;
       this.ValidUtf8 = true;
-      for (int i = startOffset; i < this.RawName.Length; i+=bytesPerChar)
+      for (int i = startOffset; i < len; i+=bytesPerChar)
       {
         byte c = this.RawName[i];
-        byte c2 = i + 1 < this.RawName.Length ? this.RawName[i + 1] : (byte)0;
+        byte c2 = i + 1 < len ? this.RawName[i + 1] : (byte)0;
+        byte c3 = i + 2 < len ? this.RawName[i + 2] : (byte)0;
+        byte c4 = i + 4 < len ? this.RawName[i + 3] : (byte)0;
         if (c >= 0x80)
           NonAscii = true;
 
@@ -178,10 +178,28 @@ namespace ChanSort.Loader.Panasonic
           ValidUtf8 = false;
           sb.Append((char) c);
         }
-        else if (bytesPerChar == 1 && c >= 0xC0 && c <= 0xDF && c2 >= 0x80 && c2 <= 0xBF) // 2 byte UTF-8
+        else if (bytesPerChar == 1)
         {
-          sb.Append((char)(((c & 0x1F) << 6) | (c2 & 0x3F)));
-          ++i;
+          if (c >= 0xC0 && c <= 0xDF && c2 >= 0x80 && c2 <= 0xBF) // 2 byte UTF-8
+          {
+            sb.Append((char)(((c & 0x1F) << 6) | (c2 & 0x3F)));
+            ++i;
+          }
+          else if (c >= 0xE0 && c <= 0xEF && (c2 & 0xC0) == 0x80 && (c3 & 0xC0) == 0x80) // 3 byte UTF-8
+          {
+            sb.Append((char)(((c & 0x0F) << 12) | ((c2 & 0x3F) << 6) | (c3 & 0x3F)));
+            i += 2;
+          }
+          else if (c >= 0xF0 && c <= 0xF7 && (c2 & 0xC0) == 0x80 && (c3 & 0xC0) == 0x80 && (c4 & 0xC0) == 0x80) // 4 byte UTF-8
+          {
+            sb.Append((char)(((c & 0x07) << 18) | ((c2 & 0x3F) << 12) | ((c3 & 0x3F) << 6) | (c4 & 0x3F)));
+            i += 3;
+          }
+          else
+          {
+            ValidUtf8 = false;
+            sb.Append(encoding.GetString(this.RawName, i, bytesPerChar));
+          }
         }
         else
         {
@@ -190,8 +208,7 @@ namespace ChanSort.Loader.Panasonic
         }
       }
 
-      string longName, shortName;
-      this.GetChannelNames(sb.ToString(), out longName, out shortName);
+      this.GetChannelNames(sb.ToString(), out var longName, out var shortName);
       this.Name = longName;
       this.ShortName = shortName;
     }

source/ChanSort.Loader.Panasonic/Serializer.cs

@@ -20,7 +20,7 @@ namespace ChanSort.Loader.Panasonic
 
     private string workFile;
     private CypherMode cypherMode;
-    private byte[] fileHeader = new byte[0];
+    private byte[] fileHeader = Array.Empty<byte>();
     private int dbSizeOffset;
     private bool littleEndianByteOrder;
     private string charEncoding;
@@ -329,11 +329,14 @@ order by s.ntype,major_channel
     #region WriteChannels()
     private void WriteChannels(SqliteCommand cmd, ChannelList channelList)
     {
+      if (channelList.Channels.Count == 0)
+        return;
+
       cmd.CommandText = "update SVL set major_channel=@progNr, sname=@sname, profile1index=@fav1, profile2index=@fav2, profile3index=@fav3, profile4index=@fav4, child_lock=@lock, skip=@skip where rowid=@rowid";
       cmd.Parameters.Clear();
       cmd.Parameters.Add("@rowid", SqliteType.Integer);
       cmd.Parameters.Add("@progNr", SqliteType.Integer);
-      cmd.Parameters.Add("@sname", SqliteType.Blob);
+      cmd.Parameters.Add("@sname", this.implicitUtf8 ? SqliteType.Text : SqliteType.Blob); // must use "TEXT" when possible to preserve collation / case-insensitive sorting for the TV
       cmd.Parameters.Add("@fav1", SqliteType.Integer);
       cmd.Parameters.Add("@fav2", SqliteType.Integer);
       cmd.Parameters.Add("@fav3", SqliteType.Integer);
@@ -351,7 +354,7 @@ order by s.ntype,major_channel
         channel.UpdateRawData(this.explicitUtf8, this.implicitUtf8);
         cmd.Parameters["@rowid"].Value = channel.RecordIndex;
         cmd.Parameters["@progNr"].Value = channel.NewProgramNr;
-        cmd.Parameters["@sname"].Value = channel.RawName;
+        cmd.Parameters["@sname"].Value = this.implicitUtf8 ? channel.Name : channel.RawName; // must use a string when possible to preserve collation / case-insensitive sorting for the TV
         for (int fav = 0; fav < 4; fav++)
           cmd.Parameters["@fav" + (fav + 1)].Value = Math.Max(0, channel.GetPosition(fav+1));
         cmd.Parameters["@lock"].Value = channel.Lock;

source/ChanSort.Loader.Philips/ChanSort.Loader.Philips.ini

@@ -19,7 +19,7 @@
 
 [flash_db]
 reorderRecordsByChannelNumber=true
-allowEdit=false
+allowEdit=true
 
 [mgr_chan_s_fta.db]
 lenHeader=64

source/ChanSort.Loader.Philips/DbSerializer.cs

@@ -50,8 +50,8 @@ namespace ChanSort.Loader.Philips
     #region ctor()
     public DbSerializer(string inputFile) : base(inputFile)
     {
-      this.Features.MaxFavoriteLists = 1;
-      this.Features.FavoritesMode = FavoritesMode.OrderedPerSource;
+      this.Features.MaxFavoriteLists = 0; //1;
+      this.Features.FavoritesMode = FavoritesMode.None; // FavoritesMode.OrderedPerSource; // doesn't work yet, must be hidden somewhere inside the FLASH files too
       this.Features.DeleteMode = DeleteMode.NotSupported;
       this.Features.CanHaveGaps = true; // the mgr_chan_s_pkg can have gaps
 

source/ChanSort.Loader.Philips/XmlSerializer.cs

@@ -20,12 +20,19 @@ namespace ChanSort.Loader.Philips
     The .BIN file itself is compressed with some unknown cl_Zip compression / archive format and can't be edited with ChanSort.
     Deleting a channel is not possible by modifiying the .xml file. Omitting a channel only results in duplicate numbers with the TV still showing the missing channels at their old numbers.
     The channel nodes in the .XML must be kept in the original order with "oldpresetnumber" keeping the original value and only "presetnumber" being updated.
+    There are (at least) two versions of this format. One starts with <ChannelMap> as the root node, one has a <ECSM> root node wrapping <ChannelMap> and other elements.
 
     <Channel>
     <Setup oldpresetnumber="1" presetnumber="1" name="Das Erste" ></Setup>
     <Broadcast medium="dvbc" frequency="410000" system="west" serviceID="1" ONID="41985" TSID="1101" modulation="256" symbolrate="6901000" bandwidth="Unknown"></Broadcast>
     </Channel>
   
+    <Channel>
+    <Setup presetnumber="1" name="Das Erste HD" ></Setup>
+    <Broadcast medium="dvbs" satellitename="Astra1F-1L 19.2E
+    @" frequency="11494" system="west" serviceID="2604" ONID="1" TSID="1019" modulation="8-VSB" symbolrate="22000"></Broadcast>
+    </Channel>
+
   
     Newer channel lists from Philips contain multiple XML files with a different internal structure, which also varies based on the version number in the ChannelMap_xxx folder name.
     The official Philips Channel Editor 6.61.22 supports the binary file format 1.1 and 1.2 as well as the XML file format "ChannelMap_100" (but not 45, 105 nor 110).
@@ -238,7 +245,8 @@ namespace ChanSort.Loader.Philips
         fileData.doc = new XmlDocument();
         fileData.content = File.ReadAllBytes(fileName);
         fileData.textContent = Encoding.UTF8.GetString(fileData.content);
-        fileData.newline = fileData.textContent.Contains("\r\n") ? "\r\n" : "\n";
+        var idx = fileData.textContent.IndexOf('\n');
+        fileData.newline = idx < 0 ? "" : idx > 0 && fileData.textContent[idx-1] == '\r' ? "\r\n" : "\n"; // there are Repair\*.xml files with <ECSM> root that use \n normally but contain a \n\r\n near the end
 
         // indentation can be 0, 2 or 4 spaces
         var idx1 = fileData.textContent.IndexOf("<Channel>");
@@ -255,7 +263,13 @@ namespace ChanSort.Loader.Philips
           ValidationFlags = XmlSchemaValidationFlags.None,
           DtdProcessing = DtdProcessing.Ignore
         };
-        using (var reader = XmlReader.Create(new StringReader(fileData.textContent), settings))
+
+        fileData.formatVersion = Path.GetFileName(fileName).ToLowerInvariant().StartsWith("cm_") ? FormatVersion.RepairXml : FormatVersion.ChannelMapXml; // first guess, will be set based on file content later
+
+        var xml = fileData.textContent;
+        if (fileData.formatVersion == FormatVersion.RepairXml)
+          xml = xml.Replace("&", "&amp;"); // Philips exports broken XML with unescaped & instead of &amp;
+        using (var reader = XmlReader.Create(new StringReader(xml), settings))
         {
           fileData.doc.Load(reader);
         }
@@ -268,6 +282,8 @@ namespace ChanSort.Loader.Philips
       var root = fileData.doc.FirstChild;
       if (root is XmlDeclaration)
         root = root.NextSibling;
+      if (root?.LocalName == "ECSM")
+        root = root.FirstChild;
       if (fail || root == null || (root.LocalName != "ChannelMap" && root.LocalName != "FavoriteListMAP"))
         throw new FileLoadException("\"" + fileName + "\" is not a supported Philips XML file");
 
@@ -313,7 +329,7 @@ namespace ChanSort.Loader.Philips
 
       if (setupNode.HasAttribute("name"))
       {
-        file.formatVersion = 1;
+        file.formatVersion = FormatVersion.RepairXml;
         this.iniMapSection = ini.GetSection("Repair_xml");
         this.Features.FavoritesMode = FavoritesMode.None;
         foreach (var list in this.DataRoot.ChannelLists)
@@ -321,13 +337,14 @@ namespace ChanSort.Loader.Philips
           list.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.Favorites));
           list.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.Lock));
           list.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.Hidden));
-          list.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.ServiceTypeName));
+          list.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.ServiceType));
+          list.VisibleColumnFieldNames.Add("-" + nameof(ChannelInfo.ServiceTypeName));
           list.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.Encrypted));
         }
       }
       else if (setupNode.HasAttribute("ChannelName"))
       {
-        file.formatVersion = 2;
+        file.formatVersion = FormatVersion.ChannelMapXml;
         this.Features.FavoritesMode = FavoritesMode.Flags;
         this.Features.MaxFavoriteLists = 1;
 
@@ -417,9 +434,9 @@ namespace ChanSort.Loader.Philips
       var chan = new Channel(curList.SignalSource & SignalSource.MaskAdInput, rowId, uniqueId, setupNode);
       chan.OldProgramNr = -1;
       chan.IsDeleted = false;
-      if (file.formatVersion == 1)
+      if (file.formatVersion == FormatVersion.RepairXml)
         this.ParseRepairXml(data, chan);
-      else if (file.formatVersion == 2)
+      else if (file.formatVersion == FormatVersion.ChannelMapXml)
         this.ParseChannelMapXml(data, chan);
 
       if ((chan.SignalSource & SignalSource.MaskAdInput) == SignalSource.DvbT)
@@ -447,7 +464,10 @@ namespace ChanSort.Loader.Philips
       chan.OriginalNetworkId = ParseInt(data.TryGet("ONID"));
       chan.TransportStreamId = ParseInt(data.TryGet("TSID"));
       chan.ServiceType = ParseInt(data.TryGet("serviceType"));
-      chan.SymbolRate = ParseInt(data.TryGet("symbolrate")) / 1000;
+      chan.SymbolRate = ParseInt(data.TryGet("symbolrate"));
+      if (chan.SymbolRate > 100000) // DVB-C/T specify it in Sym/s, DVB-S in kSym/sec
+        chan.SymbolRate /= 1000;
+      chan.Satellite = data.TryGet("satellitename")?.TrimEnd('@', '\n', '\r'); // the satellitename can have a "\n@" at the end
     }
     #endregion
 
@@ -583,7 +603,7 @@ namespace ChanSort.Loader.Philips
 
       foreach (var file in this.fileDataList)
       {
-        if (reorderNodes && (file.formatVersion == 1 || Path.GetFileName(file.path).ToLowerInvariant().StartsWith("dvb")))
+        if (reorderNodes && (file.formatVersion == FormatVersion.RepairXml || Path.GetFileName(file.path).ToLowerInvariant().StartsWith("dvb")))
           this.ReorderNodes(file);
         var satelliteListcopy = this.iniMapSection?.GetString("satelliteListcopy") ?? "";
         var nodeList = file.doc.GetElementsByTagName("SatelliteListcopy");
@@ -765,7 +785,7 @@ namespace ChanSort.Loader.Philips
     #region ReorderNodes()
     private void ReorderNodes(FileData file)
     {
-      var progNrAttrib = file.formatVersion == 1 ? "presetnumber" : "ChannelNumber";
+      var progNrAttrib = file.formatVersion == FormatVersion.RepairXml ? "presetnumber" : "ChannelNumber";
 
       var nodes = file.doc.DocumentElement.GetElementsByTagName("Channel");
       var list = new List<XmlElement>();
@@ -810,6 +830,8 @@ namespace ChanSort.Loader.Philips
       // append trailing newline, if the original file had one
       if (file.textContent.EndsWith(file.newline) && !xml.EndsWith(file.newline))
         xml += file.newline;
+      if (file.formatVersion == FormatVersion.RepairXml)
+        xml = xml.Replace("&amp;", "&"); // Philips uses broken XML with unescaped & instead of &amp;
 
       var enc = new UTF8Encoding(false, false);
       File.WriteAllText(file.path, xml, enc);
@@ -824,6 +846,14 @@ namespace ChanSort.Loader.Philips
     #endregion
 
 
+    #region enum FormatVersion
+
+    private enum FormatVersion
+    {
+      RepairXml = 1, ChannelMapXml = 2
+    }
+    #endregion
+
     #region class FileData
     private class FileData
     {
@@ -833,7 +863,7 @@ namespace ChanSort.Loader.Philips
       public string textContent;
       public string newline;
       public string indent;
-      public int formatVersion;
+      public FormatVersion formatVersion;
     }
     #endregion
   }

source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_476.h

@@ -216,4 +216,37 @@ public struct Philips_FLASH_DTVINFO_S_PKG
     //s_unknown unknownTable[numUnknown];
 	BYTE filler[0x10000-current_offset];
     s_channelBlock channelBlocks[*];
+};
+
+#pragma byte_order(BigEndian)		
+
+public struct Philips_FLASH_SDTSECTS_S_PKG
+{
+	BYTE u1[4];
+	BYTE allocationBitmap[64];
+	struct
+	{
+		var off0 = current_offset;
+		DWORD ffff;
+		BYTE u1[3];
+		WORD tsid;
+		BYTE u2[6];
+		struct Channel
+		{
+			WORD serviceId;
+			BYTE u1[6];
+			BYTE lenProviderName;
+			char provider[lenProviderName];
+			BYTE lenChannelName;
+			char channelName[lenChannelName];		
+		};		
+		Channel chan0;
+		BYTE u3[51];		
+		Channel chan1;
+		BYTE u4[60];
+		Channel chan2;
+		BYTE u5[18];
+		Channel chans[16];
+		BYTE filler[0x44C - current_offset];
+	} transponder[*];
 };

source/Test.Loader.Philips/Test.Loader.Philips.csproj

@@ -260,6 +260,7 @@
     <Content Include="TestFiles\ChannelMap_100\ChannelList\s2channellib\DVBSall.xml">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="TestFiles\Repair\CM_T923E_LA_CK.xml" />
   </ItemGroup>
   <ItemGroup>
     <None Include="app.config" />

source/changelog.md

@@ -1,6 +1,14 @@
 ChanSort Change Log 
 ===================
 
+2021-09-22
+- Philips: added support for CM\_*.xml variant that uses a <ECSM> root element around the <ChannelMap>
+- Philips: ability to read/write broken CM\_*.xml files that contain channel names with an unescaped & character
+- Philips: enabled write mode for Repair\\FLASH\_\*/\*.db file format (one variant was confirmed to work)
+  Favorite lists for this format are disabled for now (TV ignored them).
+- Panasonic: importing a modified svl.bin file caused the TV to use case-sensitive sorting when using the 
+  function to list the names sorted alphabetically. This is now fixed.
+   
 2021-09-19
 - Philips: added support for ChannelMap_115 format
 - Philips: ChannelMap formats 100-115 did not always fill "Source" and "Polarity" columns correctly