path: root/LibGit/GitPack.cs

using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.IO.Compression;
using System.Runtime.CompilerServices;
using LibGit.Extensions;

namespace LibGit;

// https://shafiul.github.io//gitbook/7_the_packfile.html - easier to digest than the git documentation
public class GitPack
{
    private const bool _log = false;
    public string PackId { get; set; }
    public GitRepo Repo { get; set; }

    public int Version { get; set; }
    public int ObjectCount { get; set; }
    public GitPackIndex Index { get; set; }
    public List<GitPackObject> Objects { get; set; } = new();

    public GitPack Read(Stream packStream, Stream? idxStream = null, GitPackIndex? index = null)
    {
        if (_log)
        {
            Console.Write(" Header: ");
            packStream.Peek(04).ToArray()[0..].HexDump(4);
            Console.Write("Version: ");
            packStream.Peek(08).ToArray()[4..].HexDump(4);
            Console.Write(" ObjCnt: ");
            packStream.Peek(12).ToArray()[8..].HexDump(4);
        }

        if (!packStream.StartsWith("PACK"))
            throw new Exception("Invalid pack file header");

        packStream.Skip(4);

        Version = packStream.ReadInt32BE();
        ObjectCount = packStream.ReadInt32BE();
        if (_log)
        {
            Console.WriteLine($"Got git v{Version} pack with {ObjectCount} objects");
            Console.WriteLine("Reading index...");
        }

        Index = new GitPackIndex().Read(idxStream);
        if (_log) Console.WriteLine("Reading pack objects...");

        var ordered = Index.Entries.OrderBy(x => x.Offset).ToArray();
        // prevent spamming the console
        var sw = Stopwatch.StartNew();
        var tsw = Stopwatch.StartNew();
        for (int i = 0; i < ObjectCount; i++)
        {
            var obj = ordered[i];
            if (sw.ElapsedMilliseconds >= 50 || i == ObjectCount - 1)
            {
                Console.Write($"\r[{tsw.Elapsed}] Reading object {i + 1}/{ObjectCount} ({ordered[i].Sha}) @ {obj.Offset}");
                sw.Restart();
            }

            Objects.Add(new GitPackObject().ReadHeader(packStream, obj.Offset));
        }

        Console.WriteLine();

        foreach (var group in Objects.GroupBy(x => x.ObjType))
        {
            Console.WriteLine($"  - {group.Key}: {group.Count()} objects");
        }

        return this;
    }

    public GitPack(string packId, GitRepo repo)
    {
        PackId = packId;
        Repo = repo;
    }
}

public class GitPackIndex
{
    private const bool _log = false;
    public int Version { get; set; }
    public int[] fanOutTable = new int[256];
    public List<IndexEntry> Entries { get; set; } = [];
    public Sha1Value PackSHA { get; set; }
    public Sha1Value IndexSHA { get; set; }

    [InlineArray(20)]
    public struct Sha1Value
    {
        private byte _e0;

        public override string ToString()
        {
            ReadOnlySpan<byte> bytes = this;
            return Convert.ToHexStringLower(bytes);
        }
    }

    public struct IndexEntry
    {
        public Sha1Value Sha;
        public uint Crc32;
        public ulong Offset;
    }

    public GitPackIndex Read(Stream stream)
    {
        if (!stream.StartsWith([0xff, 0x74, 0x4f, 0x63]))
            throw new Exception("Invalid pack index file header or pack is v1");

        stream.Skip(4);
        Version = stream.ReadInt32BE();
        if (_log) Console.WriteLine($"Got git v{Version} pack index");

        fanOutTable = stream.MultiReadInt32BE(255);

        var size = stream.ReadInt32BE(); // aka "fanout[255]"
        if (_log) Console.WriteLine($"Index contains {size} objects");
        Entries = new List<IndexEntry>(new IndexEntry[size]);

        if (_log) Console.Write("Reading SHA1...");
        var sha1Values = stream.MultiReadInlineArray<Sha1Value>(size);

        if (_log) Console.Write(" CRC...");
        var crcValues = stream.MultiReadUInt32BE(size);

        if (_log) Console.Write(" OFS...");
        var raw32 = stream.MultiReadUInt32BE(size); // uint[]
        var raw64 = stream.MultiReadUInt64BE((int)((stream.Remaining() - 40) / sizeof(ulong)));

        if (_log) Console.WriteLine(" Hashes...");
        var h = new Sha1Value();
        stream.ReadBlock(20).CopyTo(h);
        PackSHA = h;
        if (_log) Console.WriteLine($"Pack SHA: {PackSHA}");

        h = new Sha1Value();
        stream.ReadBlock(20).CopyTo(h);
        IndexSHA = h;
        if (_log) Console.WriteLine($"Index SHA: {IndexSHA}");

        if (_log) Console.WriteLine("Constructing entries...");
        for (var i = 0; i < size; i++)
        {
            Entries[i] = new IndexEntry
            {
                Sha = sha1Values[i],
                Crc32 = crcValues[i],
                Offset = (raw32[i] & 0x80000000) == 0
                    ? raw32[i]
                    : raw64[raw32[i] & 0x7FFFFFFF]
            };
        }

        return this;
    }
}

public class GitPackObject
{
    private const bool _debug = false;

    public GitPackObject ReadHeader(Stream stream, ulong offset)
    {
        Offset = offset;
        stream.Seek((long)offset, SeekOrigin.Begin);
        if (_debug) Console.WriteLine($"Reading pack object at offset {offset}, stream position {stream.Position}");
        var headerPos = stream.Position;
        var data = stream.ReadByte();
        if (_debug) Console.WriteLine($"data: {data:X8} ({data}/{data:b8})");

        //format: 1 bit continue, 3 bits type, 4 bits size (A), continued by up to 3 more bytes of size (B, C and D), A is least significant
        ObjType = (GitObjectType)((data >> 4) & 0b0000_0111);
        var sizeBits = data & 0b0000_1111; // Lower 4 bits are the initial size
        var restOfSize = (data & 0b1000_0000) != 0 ? stream.ReadVLQ() : 0;
        UncompressedSize = (restOfSize << 4) | sizeBits;

        // handle delta objects
        if (ObjType == GitObjectType.RefDelta)
        {
            RefDeltaBaseObjectId = stream.ReadBlock(20);
            if (_debug) Console.WriteLine($"Ref delta base object id: {RefDeltaBaseObjectId.AsHexString()}");
        }
        else if (ObjType == GitObjectType.OffsDelta)
        {
            OffsDeltaBaseOffset = stream.ReadGitPackOffsetModifiedVLQ();
            if (_debug) Console.WriteLine($"Offset delta base offset: {OffsDeltaBaseOffset}");
        }

        // var dataPos = stream.Position;
        // if (_debug) Console.WriteLine($"pack objType: {ObjType} ({(int)ObjType}), uncompressed size: {UncompressedSize}, position: HDR={headerPos}, DATA={dataPos}");

        DataOffset = (ulong) stream.Position;

        return this;
    }

    public GitPackObject Read(Stream stream, ulong offset)
    {
        ReadHeader(stream, offset);

        try
        {
            using var zlibStream = new ZLibStream(stream, CompressionMode.Decompress, true);
            var decompressedData = new Span<byte>(new byte[UncompressedSize]);
            int totalRead = 0;
            while (totalRead < UncompressedSize)
            {
                int bytesRead = zlibStream.Read(decompressedData); //, totalRead, UncompressedSize - totalRead);
                if (bytesRead == 0)
                    throw new Exception("Unexpected end of zlib stream");
                totalRead += bytesRead;
            }
        }
        catch (Exception ex)
        {
            Console.WriteLine($"Error during zlib decompression: {ex.Message}");
            stream.Seek((long)Offset, SeekOrigin.Begin);
            stream.Peek(32).HexDump();
            throw;
        }

        // var endPos = stream.Position;
        // if (_debug) Console.WriteLine($"Decompressed data ({UncompressedSize} bytes/{endPos - dataPos} compressed, stream @ {endPos}):");
        // if (_debug) decompressedData.ToArray().HexDump(32);

        // Environment.Exit(1);
        return this;
    }

    public ulong Offset { get; set; }
    public ulong DataOffset { get; set; }

    public GitObjectType ObjType { get; set; }

    public int UncompressedSize { get; set; }

    public byte[]? RefDeltaBaseObjectId { get; set; }

    public int? OffsDeltaBaseOffset { get; set; }
}

public enum GitObjectType
{
    Commit = 1,
    Tree,
    Blob,
    Tag,
    Invalid, // Reserved for future expansion, see https://git-scm.com/docs/pack-format#_object_types
    OffsDelta,
    RefDelta
}