diff --git a/source/ChanSort.Loader.TCL/DtvDataSerializer.cs b/source/ChanSort.Loader.TCL/DtvDataSerializer.cs index 89c29b7..1082c8a 100644 --- a/source/ChanSort.Loader.TCL/DtvDataSerializer.cs +++ b/source/ChanSort.Loader.TCL/DtvDataSerializer.cs @@ -1,11 +1,14 @@ -#define Win10_TAR -#define TestBuild +//#define Win10_TAR +#define GNU_TAR +//#define TestBuild -using System.Diagnostics; using System.Text; using Microsoft.Data.Sqlite; using ChanSort.Api; -#if !Win10_TAR +#if Win10_TAR +using System.Diagnostics; +#elif GNU_TAR +#else using SharpCompress.Archives; using SharpCompress.Archives.Tar; using SharpCompress.Common; @@ -31,6 +34,10 @@ namespace ChanSort.Loader.TCL private readonly HashSet tableNames = new(); private readonly StringBuilder protocol = new(); +#if GNU_TAR + private GnuTar gnuTar; +#endif + #region ctor() public DtvDataSerializer(string inputFile) : base(inputFile) { @@ -140,6 +147,9 @@ namespace ChanSort.Loader.TCL psi.Arguments = $"xf \"{this.FileName}\""; var proc = Process.Start(psi); proc.WaitForExit(); +#elif GNU_TAR + this.gnuTar = new GnuTar(); + gnuTar.ExtractToDirectory(this.FileName, this.TempPath); #else using var tar = TarArchive.Open(this.FileName); var rdr = tar.ExtractAllEntries(); @@ -442,6 +452,8 @@ left outer join CurCIOPSerType c on c.u8DtvRoute=p.u8DtvRoute psi.Arguments = $"cf \"{this.FileName}\" *"; var proc = Process.Start(psi); proc.WaitForExit(); +#elif GNU_TAR + this.gnuTar.UpdateFromDirectory(this.FileName); #else using var tar = TarArchive.Create(); tar.AddAllFromDirectory(this.TempPath); diff --git a/source/ChanSort.Loader.TCL/GnuTar.cs b/source/ChanSort.Loader.TCL/GnuTar.cs new file mode 100644 index 0000000..33668ca --- /dev/null +++ b/source/ChanSort.Loader.TCL/GnuTar.cs @@ -0,0 +1,323 @@ +using System.Text; + +namespace ChanSort.Loader.TCL; + +/// +/// Minimal implementation to support the contents of TCL's .tar. +/// Reading supports all "ustar" formats including old-GNU and POSIX 1990 .tar flavors. +/// Saving uses the "old-GNU" format, that's used by TCL. +/// +/// Unlike all tools and libraries available under Windows, this implementation preserves unix metadata like: +/// file mode, user-id, group-id, user name, group name and device major/minor number +/// +/// Information about GNU tar can be found on https://www.gnu.org/software/tar/manual/html_node/Standard.html +/// +internal class GnuTar +{ + private static readonly DateTime Epoc = new (1970, 1, 1); + private static readonly byte[] Padding = new byte[512]; + + public Encoding Encoding { get; set; } = Encoding.UTF8; + public List Entries { get; } = new(); + + #region ExtractToDirectory() + public void ExtractToDirectory(string tarPath, string targetDir) + { + var data = File.ReadAllBytes(tarPath); + this.Read(data); + + foreach (var entry in this.Entries) + { + if (entry.TypeFlag == TarEntryTypes.Directory) + { + entry.Path = Path.Combine(targetDir, entry.Name.TrimEnd('/', '\\')); + Directory.CreateDirectory(entry.Path); + } + else if (entry.TypeFlag == TarEntryTypes.File || entry.TypeFlag == TarEntryTypes.File0) + { + var path = Path.Combine(targetDir, entry.Name); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + using (var outStream = new FileStream(path, FileMode.Create)) + outStream.Write(data, entry.Position, entry.Size); + entry.Path = path; + File.SetLastWriteTimeUtc(path, entry.LastModified); + } + else + throw new NotImplementedException($"unsupported tar entry type {(char)entry.TypeFlag} for {entry.Name}"); + } + } + #endregion + + #region Read() + public void Read(string path) + { + var stream = new FileStream(path, FileMode.Open); + Read(stream, false); + } + + public void Read(byte[] data) + { + using var stream = new MemoryStream(data); + Read(stream); + } + + public void Read(Stream stream, bool keepOpen = true) + { + try + { + Entries.Clear(); + var header = new byte[512]; + var memStream = stream as MemoryStream; + while (true) + { + var len = stream.Read(header, 0, 512); + if (len < 512) + break; + + if (header.All(b => b == 0)) // end block is all-zero + break; + + var position = stream.Position; + var entry = ReadEntryHeader(header); + entry.Data = memStream?.ToArray(); + entry.Position = (int)position; + Entries.Add(entry); + + var extra = entry.Size % 512; + if (extra != 0) + extra = 512 - extra; + stream.Seek(entry.Size + extra, SeekOrigin.Current); + } + } + finally + { + if (!keepOpen) + stream.Dispose(); + } + } + #endregion + + #region ReadEntryHeader() + private TarEntry ReadEntryHeader(byte[] header) + { + var e = new TarEntry(); + e.Name = ReadString(header, 0, 100, this.Encoding); + e.Mode = (ushort)ReadNumber(header, 100, 8); + e.UserId = ReadNumber(header, 108, 8); + e.GroupId = ReadNumber(header, 116, 8); + e.Size = ReadNumber(header, 124, 12); + var mtime = (uint)ReadNumber(header, 136, 12); + e.LastModified = Epoc.AddSeconds(mtime); + e.Checksum = (uint)ReadNumber(header, 148, 8); + e.TypeFlag = (TarEntryTypes)header[156]; + e.LinkName = ReadString(header, 157, 100, this.Encoding); + + e.Magic = ReadString(header, 257, 6); + e.Version = ReadString(header, 263, 2); + if (e.Magic != "ustar" && !(e.Magic == "ustar " && e.Version == " ")) + throw new InvalidOperationException("not a POSIX/GNU or old-GNU tar"); + e.Username = ReadString(header, 265, 32, this.Encoding); + e.Groupname = ReadString(header, 297, 32, this.Encoding); + e.DeviceMajor = ReadOptionalNumber(header, 329, 8); + e.DeviceMinor = ReadOptionalNumber(header, 337, 8); + e.Prefix = ReadString(header, 345, 155, this.Encoding); + + return e; + } + #endregion + + #region ReadString(), ReadNumber() + private string ReadString(byte[] data, int offset, int length, Encoding encoding = null) + { + encoding ??= Encoding.ASCII; + int idx = Array.IndexOf(data, (byte)0, offset); + if (idx == 0) + idx = data.Length; + if (idx > 0) + length = Math.Min(length, idx - offset); + return encoding.GetString(data, offset, length); + } + + private int ReadNumber(byte[] data, int offset, int length) + { + var val = ReadOptionalNumber(data, offset, length); + return val ?? 0; + } + + private int? ReadOptionalNumber(byte[] data, int offset, int length) + { + var nr = ReadString(data, offset, length).TrimEnd(' '); + return nr.Length == 0 ? null : Convert.ToInt32(nr, 8); + } + + #endregion + + + #region UpdateFromDirectory() + public void UpdateFromDirectory(string tarPath) + { + using var outStream = new FileStream(tarPath, FileMode.Create); + using var memStream = new MemoryStream(512); + foreach (var entry in this.Entries) + { + byte[] data = null; + if (entry.TypeFlag == TarEntryTypes.Directory) + { + if (!Directory.Exists(entry.Path)) + continue; + } + else + { + var info = new FileInfo(entry.Path); + if (!info.Exists) + continue; + + data = File.ReadAllBytes(entry.Path); + entry.Size = data.Length; + entry.LastModified = info.LastWriteTimeUtc; + } + + // prepare header in a memory stream and patch checksum into it + memStream.Seek(0, SeekOrigin.Begin); + WriteEntryHeader(entry, memStream); + entry.Checksum = CalcChecksum(memStream.GetBuffer()); + memStream.Seek(148, SeekOrigin.Begin); + WriteNumber(memStream, (ushort)entry.Checksum, 7); + outStream.Write(memStream.GetBuffer(), 0, 512); + + // write file data + if (data != null) + outStream.Write(data, 0, data.Length); + + // padding zeros to 512 + var padlen = entry.Size % 512; + if (padlen != 0) + outStream.Write(Padding, 0, 512 - padlen); + } + + // end-of-file marker: 2x 512 byte blocks with all zeros + outStream.Write(Padding, 0, Padding.Length); + outStream.Write(Padding, 0, Padding.Length); + } + #endregion + + #region CalcChecksum() + private uint CalcChecksum(byte[] data) + { + uint sum = 0; + int count = data.Length; + for (int i=0; count>0; i++, count--) + sum += data[i]; + return sum; + } + #endregion + + #region WriteEntryHeader() + private void WriteEntryHeader(TarEntry e, Stream strm) + { + WriteString(strm, e.Name, 100, this.Encoding); + WriteNumber(strm, e.Mode, 8); + WriteNumber(strm, e.UserId, 8); + WriteNumber(strm, e.GroupId, 8); + WriteNumber(strm, e.Size, 12); + WriteNumber(strm, (int)(e.LastModified - Epoc).TotalSeconds, 12); + strm.Write(" "u8.ToArray(), 0, 8); // placeholder for checksum + strm.WriteByte((byte)e.TypeFlag); + WriteString(strm, e.LinkName, 100, this.Encoding); + strm.Write(Encoding.ASCII.GetBytes(e.Magic), 0, 6); + WriteString(strm, e.Version, 2); + WriteString(strm, e.Username, 32); + WriteString(strm, e.Groupname, 32); + WriteNumber(strm, e.DeviceMajor, 8); + WriteNumber(strm, e.DeviceMinor, 8); + WriteString(strm, e.Prefix, 155, this.Encoding); + strm.Write(Padding, 0, 512 - 500); + } + #endregion + + #region WriteString() + private void WriteString(Stream strm, string str, int length, Encoding enc = null) + { + enc ??= Encoding.UTF8; + var bytes = enc.GetBytes(str); + if (bytes.Length >= length) + { + strm.Write(bytes, 0, length-1); + strm.WriteByte(0); + return; + } + + strm.Write(bytes, 0, bytes.Length); + strm.Write(Padding, 0, length - bytes.Length); + } + #endregion + + #region WriteNumber() + private void WriteNumber(Stream strm, int number, int length) + { + var str = Convert.ToString((uint)number, 8); + if (str.Length >= length) + throw new ArgumentException($"{number} is too long for {length} octal digits"); + for (int i=length - str.Length - 1; i>0; i--) + strm.WriteByte((byte)'0'); + var bytes = Encoding.ASCII.GetBytes(str); + strm.Write(bytes, 0, bytes.Length); + strm.WriteByte(0); + } + + private void WriteNumber(Stream strm, int? number, int length) + { + if (number.HasValue) + WriteNumber(strm, number.Value, length); + else + WriteString(strm, "", length); + } + #endregion +} + +#region enum TarEntryTypes +public enum TarEntryTypes : byte +{ + File0 = (byte)'\0', + File = (byte)'0', + Link = (byte)'1', + Sym = (byte)'2', + CharDevice = (byte)'3', + BlockDevice = (byte)'4', + Directory = (byte)'5', + FiFo = (byte)'6', + Cont = (byte)'7', + ExtendedFileHeader = (byte)'x', + GlobalHeader = (byte)'g' +} +#endregion + +#region class TarEntry +class TarEntry +{ + // original tar V7 + public string Name { get; set; } + public ushort Mode { get; set; } + public int UserId { get; set; } + public int GroupId { get; set; } + public int Size { get; set; } + public DateTime LastModified { get; set; } + public uint Checksum { get; set; } + public TarEntryTypes TypeFlag { get; set; } + public string LinkName { get; set; } + + // UStar (POSIX 1003.1-1990) / GNU / old-GNU + public string Magic { get; set; } + public string Version { get; set; } + public string Username { get; set; } + public string Groupname { get; set; } + public int? DeviceMajor { get; set; } + public int? DeviceMinor { get; set; } + public string Prefix { get; set; } + + // internal + public byte[] Data { get; set; } + public int Position { get; set; } + public string Path { get; set; } +} +#endregion \ No newline at end of file diff --git a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/LaSat_lst.h b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/LaSat_lst.h index eb916d0..c433bb9 100644 --- a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/LaSat_lst.h +++ b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/LaSat_lst.h @@ -1,4 +1,4 @@ -#include +#include "chansort.h" // CRCs are calculated MSB first (left-shift with initial mask 0x80000000), polynomial 0x04C11DB7, init-value 0xFFFFFFFF and exit-XOR 0x00000000 diff --git a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/chansort.h b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/chansort.h index 8e30a9c..3de7a07 100644 --- a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/chansort.h +++ b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/chansort.h @@ -1,8 +1,30 @@ typedef unsigned char byte; typedef unsigned short word; typedef unsigned int dword; +typedef unsigned __int64 qword; + +typedef unsigned char BYTE; +typedef unsigned short WORD; +typedef unsigned int DWORD; +typedef unsigned __int64 QWORD; + typedef big_endian unsigned short uc16be; +#ifndef stddef + +typedef char int8; +typedef short int16; +typedef long int32; +typedef __int64 int64; + +typedef unsigned int uint; +typedef unsigned char uint8; +typedef unsigned short uint16; +typedef unsigned long uint32; +typedef unsigned __int64 uint64; + +#endif + enum ServiceType : byte { SDTV = 1, @@ -11,4 +33,5 @@ enum ServiceType : byte SDTV_MPEG4 = 22, HDTV = 25, Option = 211 -}; \ No newline at end of file +}; + diff --git a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/cvt_database-dat.h b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/cvt_database-dat.h index 1942cd3..280c038 100644 --- a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/cvt_database-dat.h +++ b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/cvt_database-dat.h @@ -1,4 +1,4 @@ -#include +#include "chansort.h" struct StringChar { diff --git a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_472.h b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_472.h index 4f4c270..b0ee137 100644 --- a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_472.h +++ b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_472.h @@ -1,4 +1,4 @@ -#include +#include "chansort.h" #pragma byte_order(LittleEndian) diff --git a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_476.h b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_476.h index d84193d..c45d611 100644 --- a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_476.h +++ b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_476.h @@ -1,4 +1,4 @@ -#include +#include "chansort.h" #pragma byte_order(LittleEndian) diff --git a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_480.h b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_480.h index 7713cdf..da06dbc 100644 --- a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_480.h +++ b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_480.h @@ -1,4 +1,4 @@ -#include +#include "chansort.h" #pragma byte_order(LittleEndian) diff --git a/source/changelog.md b/source/changelog.md index 2c0cc78..29f013c 100644 --- a/source/changelog.md +++ b/source/changelog.md @@ -1,6 +1,10 @@ ChanSort Change Log =================== +2023-01-08 +- TCL/Thomson .tar: custom implementation for reading/writing .tar archives, preserving all + unix file metadata (based on "old-GNU" .tar flavor, like the files exported by the TV) + 2023-01-06_1450 - .HDB: added support for "Hide"-flag, added Skip/Lock/Fav for TechniSat DVB-C file format