ChanSort/source/ChanSort.Loader.TCL/DtvDataSerializer.cs

//#define TestBuild

using System.Text;
using Microsoft.Data.Sqlite;
using ChanSort.Api;

namespace ChanSort.Loader.TCL
{
  /*
   * This class loads TCL / Thomson channel lists from a directory or a .tar file containing cloneCRC.bin, DtvData.db and satellite.db.
   *
   * None of the sample files contained more than a single input source (DVB-C/T/S), so for the time being this loader puts everything into a single list
   *
   * When a channel is added to favorites, it will: EditFlag |= 0x01, IsFavor=1, but will keep FavChannelNo=65535
   * When a channel is hidden through the TV's menu, it will result in: EditFlag |= 0x08, IsSkipped=1
   * When a channel is deleted in the menu: EditFlag |= 0x10, IsDelete=1, but it will keep its unique ProgNum
   * When a channel is moved in the menu: EditFlag |= 0x02, but no change to IsMove(=0)
   */
  class DtvDataSerializer : SerializerBase
  {
    private const int CrcMaxDataLength = 0x4B000;

    [Flags]
    enum EditFlags
    {
      Favorite = 0x01,
      CustomProgNum = 0x02,
      Skip = 0x08,
      Delete = 0x10,

      AllKnown = Favorite|CustomProgNum|Skip|Delete
    }

    private readonly ChannelList channels = new (SignalSource.All, "All");
    private string dbDir;
    private string dtvFile;
    private string satFile;
    private string crcFile;

    private readonly HashSet<string> tableNames = new();
    private readonly StringBuilder protocol = new();

    private GnuTar tar;

    #region ctor()
    public DtvDataSerializer(string inputFile) : base(inputFile)
    {
      this.Features.ChannelNameEdit = ChannelNameEditMode.All;
#if TestBuild
      this.Features.DeleteMode = DeleteMode.NotSupported;
#else
      this.Features.DeleteMode = DeleteMode.FlagWithoutPrNr;
#endif
      this.Features.CanSkipChannels = true;
      this.Features.CanLockChannels = true;
      this.Features.CanHideChannels = true;
      this.Features.FavoritesMode = FavoritesMode.Flags;
      this.Features.MaxFavoriteLists = 1;

      this.DataRoot.AddChannelList(this.channels);
      channels.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.AudioPid));
      channels.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.Provider));
      channels.VisibleColumnFieldNames.Add(nameof(ChannelInfo.ServiceType));
      channels.VisibleColumnFieldNames.Add(nameof(ChannelInfo.Source));
    }
    #endregion

    #region GetDataFilePaths()
    public override IEnumerable<string> GetDataFilePaths()
    {
      var list = new List<string>();
      list.Add(this.FileName);
      var backupFile = GetBackupFilePath();
      if (File.Exists(backupFile))
        list.Add(backupFile);
      return list;
    }

    private string GetBackupFilePath()
    {
      var dir = Path.GetDirectoryName(this.FileName) ?? ".";
      var name = Path.GetFileNameWithoutExtension(this.FileName);
      var ext = Path.GetExtension(this.FileName);
      var backupFile = Path.Combine(dir, name + "Backup" + ext);
      return backupFile;
    }

    #endregion

    #region Load()
    public override void Load()
    {
      PrepareWorkingDirectory();
      ValidateCrc();
      ReadSattelliteDb();
      ReadDtvDataDb();
    }
    #endregion
    
    #region PrepareWorkingDirectory()
    /// <summary>
    /// this.FileName might be
    /// - a .tar file containing database/cloneCRC.bin, database/userdata/DtvData.db, database/userdata/satellite.db
    /// - a .db file in a folder with DtvData.db and satellite.db and a cloneCRC.bin in either the same dir or the parent dir
    /// Other situations have already been handled in the <see cref="TclPlugin"/>
    /// </summary>
    private void PrepareWorkingDirectory()
    {
      var ext = Path.GetExtension(this.FileName).ToLowerInvariant();
      if (ext == ".tar")
      {
        UntarToTempDir();
        this.crcFile = Path.Combine(this.TempPath, "database", "cloneCRC.bin");
        this.dbDir = Path.Combine(this.TempPath, "database", "userdata");
      }
      else if (ext == ".db")
      {
        this.dbDir = Path.GetDirectoryName(this.FileName);
        this.crcFile = Path.Combine(this.dbDir, "cloneCRC.bin");
        if (!File.Exists(crcFile))
          this.crcFile = Path.Combine(Path.GetDirectoryName(this.dbDir), "cloneCRC.bin");
      }
      else
        throw LoaderException.TryNext("unrecognized TCL/Thomson directory structure");

      this.dtvFile = Path.Combine(dbDir, "DtvData.db");
      if (!File.Exists(dtvFile))
        throw LoaderException.TryNext("Missing DtvData.db file");

      this.satFile = Path.Combine(dbDir, "satellite.db");
      if (!File.Exists(satFile))
        satFile = null;

      if (!File.Exists(crcFile))
        crcFile = null;
    }
    #endregion

    #region UntarToTempDir()
    private void UntarToTempDir()
    {
      this.TempPath = Path.Combine(Path.GetTempPath(), "ChanSort_" + DateTime.Now.ToString("yyyyMMdd-HHmmss"));
      Directory.CreateDirectory(this.TempPath);

      this.tar = new GnuTar();
      tar.ExtractToDirectory(this.FileName, this.TempPath);
    }
    #endregion

    #region ValidateCrc()
    private void ValidateCrc()
    {
      if (!File.Exists(crcFile)) 
        return;

      var crcData = File.ReadAllBytes(crcFile);
      var crc = Crc16.CCITT;

      var data = File.ReadAllBytes(dtvFile);
      var actual = crc.Calc(data, 0, Math.Min(data.Length, CrcMaxDataLength));
      var expected = BitConverter.ToUInt16(crcData, 2);
      if (actual != expected)
      {
        var msg = $"Invalid CRC16-CCITT check sum for {dtvFile}. Expected {expected:X4} but calculated {actual:X4}";
        protocol.AppendLine(msg);
        throw LoaderException.Fail(msg);
      }

      if (satFile != null)
      {
        data = File.ReadAllBytes(satFile);
        actual = crc.Calc(data);
        expected = BitConverter.ToUInt16(crcData, 4);
        if (actual != expected)
        {
          var msg = $"Invalid CRC16-CCITT check sum for {satFile}. Expected {expected:X4} but calculated {actual:X4}";
          protocol.AppendLine(msg);
          throw LoaderException.Fail(msg);
        }
      }
    }

    #endregion

    #region ReadSattelliteDb()
    private void ReadSattelliteDb()
    {
      if (this.satFile == null)
        return;
      string satConnString = $"Data Source={satFile};Pooling=False";
      using var conn = new SqliteConnection(satConnString);
      conn.Open();
      using var cmd = conn.CreateCommand();

      cmd.CommandText = "SELECT name FROM sqlite_master WHERE type = 'table'";
      using (var r = cmd.ExecuteReader())
      {
        while (r.Read())
          this.tableNames.Add(r.GetString(0).ToLowerInvariant());
      }

      if (!this.tableNames.Contains("sateliteinfotbl") || !this.tableNames.Contains("transponderinfotbl"))
        throw LoaderException.TryNext("File doesn't contain the expected tables");

      this.ReadSatellites(cmd);
    }

    #endregion

    #region ReadDtvDataDb()
    private void ReadDtvDataDb()
    {
      string dtvConnString = $"Data Source={dtvFile};Pooling=False";
      using var conn = new SqliteConnection(dtvConnString);
      conn.Open();
      using var cmd = conn.CreateCommand();

      this.ReadTransponders(cmd);
      this.ReadChannels(cmd);
    }
    #endregion

    #region ReadSatellites()
    private void ReadSatellites(SqliteCommand cmd)
    {
      cmd.CommandText = "select SateliteID, SateliteName, Longitude from SateliteInfoTbl";
      using var r = cmd.ExecuteReader();
      while (r.Read())
      {
        Satellite sat = new Satellite(r.GetInt32(0));
        string eastWest = "E";
        int pos = r.IsDBNull(2) ? 0 : r.GetInt32(2);
        if (pos != 0)
        {
          if (pos < 0)
          {
            pos = -pos;
            eastWest = "W";
          }

          sat.OrbitalPosition = $"{pos / 100}.{pos % 100}{eastWest}";
        }

        sat.Name = r.GetString(1);
        this.DataRoot.AddSatellite(sat);
      }
    }
    #endregion

    #region ReadTransponders()
    private void ReadTransponders(SqliteCommand cmd)
    {
      //cmd.CommandText = "select TransponderId, SateliteId, Freq, Polarisation, SymbolRate from TransponderInfoTbl";
      cmd.CommandText = "select u16MuxTblID, SatTblID, Freq, null, SymbolRate, TransportStreamId, OriginalNetworkId from MuxInfoTbl";
      using var r = cmd.ExecuteReader();
      while (r.Read())
      {
        int id = r.GetInt32(0);
        int satId = r.IsDBNull(1) ? -1 : r.GetInt32(1);
        int freq = r.GetInt32(2);
        
        if (this.DataRoot.Transponder.TryGet(id) != null)
          continue;
        Transponder tp = new Transponder(id);
        tp.FrequencyInMhz = (decimal)freq / 1000;
        //tp.Polarity = r.GetInt32(3) == 0 ? 'H' : 'V';
        tp.Satellite = this.DataRoot.Satellites.TryGet(satId);
        tp.SymbolRate = r.GetInt32(4);
        tp.TransportStreamId = r.GetInt32(5);
        tp.OriginalNetworkId = r.GetInt32(6);
        this.DataRoot.AddTransponder(tp.Satellite, tp);
      }
    }
    #endregion

    #region ReadChannels()
    private void ReadChannels(SqliteCommand cmd)
    {
      cmd.CommandText = @"
select 
  p.u32Index, p.ProgNum, p.ServiceName, p.ShortServiceName, p.ServiceID, p.VideoType, p.PCRPID, p.VideoPID, p.unlockedFlag, p.LCN, p.LCNAssignmentType, p.EditFlag,
  m.OriginalNetworkId, m.TransportStreamId, m.Freq, m.SymbolRate,
  c.RouteName,
  a.RealServiceType, a.IsScramble, a.VisibleFlag, a.IsDelete, a.IsSkipped, a.IsLock, a.IsFavor, a.IsRename, a.IsMove, a.NumSelectFlag, a.FavChannelNo
from ProgramInfoTbl p 
left outer join AtrributeTbl a on a.u32index=p.u32index
left outer join MuxInfoTbl m on m.u16MuxTblID=p.u16MuxTblID
left outer join CurCIOPSerType c on c.u8DtvRoute=p.u8DtvRoute
";

      using var r = cmd.ExecuteReader();
      while (r.Read())
      {
        var handle = r.GetInt32(0);
        var oldProgNr = r.GetInt32(1);
        if (oldProgNr == 65535)
          oldProgNr = -1;

        var name = r.GetString(2)?.TrimEnd(' ', '\0');
        ChannelInfo channel = new ChannelInfo(0, handle, oldProgNr, name);
        channel.ShortName = r.GetString(3).TrimEnd(' ', '\0');
        channel.ServiceId = r.GetInt32(4);
        var vtype = r.GetInt32(5);
        channel.ServiceTypeName = vtype == 1 ? "SD-TV" : vtype == 4 ? "HD-TV" : vtype == 6 ? "UHD-TV" : null;
        channel.PcrPid = r.GetInt32(6);
        channel.VideoPid = r.GetInt32(7);
        channel.Hidden = r.GetBoolean(8);
        var edit = (EditFlags)r.GetInt32(11);
        channel.Favorites = (edit & EditFlags.Favorite) != 0 ? Favorites.A : 0;
        channel.Skip = (edit & EditFlags.Skip) != 0;
        channel.AddDebug($"LCN={r.GetValue(9)}, edit={(int)edit:x4}");

        // DVB
        var ixD = 12;
        var ixC = ixD + 4;
        var ixA = ixC + 1;
        if (!r.IsDBNull(ixD))
        {
          channel.OriginalNetworkId = r.GetInt32(ixD + 0);
          channel.TransportStreamId = r.GetInt32(ixD + 1);
          channel.FreqInMhz = (decimal) r.GetInt32(ixD + 2) / 1000;
          channel.SymbolRate = r.GetInt32(ixD + 3);
          if (channel.FreqInMhz > 10000)
            channel.FreqInMhz = (int) channel.FreqInMhz;
          channel.Source = r.GetString(ixC);
        }

        // AtrributeTbl (actual typo in the TV's table name!)
        if (!r.IsDBNull(ixA))
        {
          channel.ServiceType = r.GetInt32(ixA + 0);
          channel.ServiceTypeName = LookupData.Instance.GetServiceTypeDescription(channel.ServiceType);
          channel.SignalSource |= LookupData.Instance.IsRadioTvOrData(channel.ServiceType);
          channel.Encrypted = r.GetInt32(ixA + 1) != 0;
          channel.Hidden = !r.GetBoolean(ixA + 2);
          channel.IsDeleted |= r.GetBoolean(ixA + 3);
          channel.Skip |= r.GetBoolean(ixA + 4);
          channel.Lock = r.GetBoolean(ixA + 5);
          if (r.GetBoolean(ixA + 6))
            channel.Favorites |= Favorites.A;
          channel.IsNameModified = r.GetBoolean(ixA + 7);
          channel.AddDebug($", FavChannelNo={r.GetInt32(ixA + 10)}");
        }

        //if (!channel.IsDeleted)
          this.DataRoot.AddChannel(this.channels, channel);
      }
    }
    #endregion


    #region Save()
    public override void Save()
    {
      string channelConnString = $"Data Source={dtvFile};Pooling=False";
      using (var conn = new SqliteConnection(channelConnString))
      {
        conn.Open();
        using var trans = conn.BeginTransaction();
        using var cmd = conn.CreateCommand();
        using var cmd2 = conn.CreateCommand();

        this.WriteChannels(cmd, cmd2, this.channels);
        trans.Commit();

        cmd.Transaction = null;
      }

      UpdateCrc();

      if (Path.GetExtension(this.FileName).ToLowerInvariant() == ".tar")
        WriteToTar();
    }
    #endregion

    #region WriteChannels()
    private void WriteChannels(SqliteCommand cmd, SqliteCommand cmdAttrib, ChannelList channelList)
    {
      // what the TV shows as "hide" in the menu is actually "skip" in the database

      cmd.CommandText = "update PrograminfoTbl set ProgNum=@nr"
#if !TestBuild      
        + ", ServiceName=@name, EditFlag=(EditFlag & " + ~(EditFlags.AllKnown) + ") | @editflag" // unlockedFlag=@hide,
#endif
        + " where u32Index=@handle";
      cmd.Parameters.Add("@handle", SqliteType.Integer);
      cmd.Parameters.Add("@nr", SqliteType.Integer);
#if !TestBuild
      cmd.Parameters.Add("@name", SqliteType.Blob, 64);
      //cmd.Parameters.Add("@hide", SqliteType.Integer);
      cmd.Parameters.Add("@editflag", SqliteType.Integer);
#endif
      cmd.Prepare();

#if !TestBuild
      cmdAttrib.CommandText = @"update AtrributeTbl set VisibleFlag=@vis, IsDelete=@del, IsSkipped=@skip, IsLock=@lock, IsRename=@ren, IsFavor=@fav where u32Index=@handle;"; // IsMove=IsMove|@mov,
      cmdAttrib.Parameters.Add("@handle", SqliteType.Integer);
      cmdAttrib.Parameters.Add("@vis", SqliteType.Integer);
      cmdAttrib.Parameters.Add("@del", SqliteType.Integer);
      cmdAttrib.Parameters.Add("@mov", SqliteType.Integer);
      cmdAttrib.Parameters.Add("@skip", SqliteType.Integer);
      cmdAttrib.Parameters.Add("@lock", SqliteType.Integer);
      cmdAttrib.Parameters.Add("@ren", SqliteType.Integer);
      cmdAttrib.Parameters.Add("@fav", SqliteType.Integer);
      cmdAttrib.Prepare();
#endif

      foreach (ChannelInfo channel in channelList.Channels)
      {
        if (channel.IsProxy) // ignore reference list proxy channels
          continue;

        channel.UpdateRawData();
        cmd.Parameters["@handle"].Value = channel.RecordIndex;
        cmd.Parameters["@nr"].Value = channel.IsDeleted ? 65535 : channel.NewProgramNr;
#if !TestBuild
        var bytes = Encoding.UTF8.GetBytes(channel.Name);
        var blob = new byte[64];
        Tools.MemCopy(bytes, 0, blob, 0, 64);
        cmd.Parameters["@name"].Value = blob;
        //cmd.Parameters["@hide"].Value = channel.Hidden;
        EditFlags flags = 0;
        if (channel.Favorites != 0)
          flags |= EditFlags.Favorite;
        if (channel.Skip)
          flags |= EditFlags.Skip;
        if (channel.IsDeleted)
          flags |= EditFlags.Delete;
        else
          flags |= EditFlags.CustomProgNum;
        cmd.Parameters["@editflag"].Value = (int)flags;

        cmdAttrib.Parameters["@handle"].Value = channel.RecordIndex;
        cmdAttrib.Parameters["@vis"].Value = channel.Hidden ? 0 : 1;
        cmdAttrib.Parameters["@del"].Value = channel.IsDeleted ? 1 : 0;
        cmdAttrib.Parameters["@skip"].Value = channel.Skip ? 1 : 0;
        cmdAttrib.Parameters["@lock"].Value = channel.Lock ? 1 : 0;
        cmdAttrib.Parameters["@ren"].Value = channel.IsNameModified ? 1 : 0;
        //cmdAttrib.Parameters["@mov"].Value = 1;
        cmdAttrib.Parameters["@fav"].Value = channel.Favorites != 0 ? 1 : 0;
        cmdAttrib.ExecuteNonQuery();
#endif
        cmd.ExecuteNonQuery();
      }
    }
    #endregion

    #region UpdateCrc
    /// <summary>
    /// update CRC in cloneCRC.bin
    /// </summary>
    private void UpdateCrc()
    {
      if (this.crcFile == null)
        return;

      var dtvData = File.ReadAllBytes(dtvFile);
      var crc = Crc16.CCITT.Calc(dtvData, 0, Math.Min(dtvData.Length, CrcMaxDataLength));
      var crcData = File.ReadAllBytes(this.crcFile);
      crcData[2] = (byte)(crc & 0xFF);
      crcData[3] = (byte)(crc >> 8);
      File.WriteAllBytes(crcFile, crcData);
    }
    #endregion

    #region WriteToTar()
    private void WriteToTar()
    {
      // delete old .tar file and create a new one from temp dir
      File.Delete(this.FileName);
      this.tar.UpdateFromDirectory(this.FileName);
    }
    #endregion
  }
}