diff --git a/Rhea.Nar/NarConsts.cs b/Rhea.Nar/NarConsts.cs
new file mode 100644
index 0000000..f821449
--- /dev/null
+++ b/Rhea.Nar/NarConsts.cs
@@ -0,0 +1,44 @@
+namespace Rhea.Nar;
+
+/// <summary>Common strings/flags in NAR archives</summary>
+public static class NarConsts
+{
+ /// <summary>Nix archive header</summary>
+ public static readonly byte[] Header = NarWriter.GetLengthPrefixedPadded("nix-archive-1"u8);
+
+ /// <summary>Start of object marker</summary>
+ public static readonly byte[] LeftParen = NarWriter.GetLengthPrefixedPadded("("u8);
+
+ /// <summary>End of object marker</summary>
+ public static readonly byte[] RightParen = NarWriter.GetLengthPrefixedPadded(")"u8);
+
+ /// <summary>Indicates the node is a regular file</summary>
+ public static readonly byte[] Regular = NarWriter.GetLengthPrefixedPadded("regular"u8);
+
+ /// <summary>Indicates the node is a directory</summary>
+ public static readonly byte[] Directory = NarWriter.GetLengthPrefixedPadded("directory"u8);
+
+ /// <summary>Indicates the node is a symlink</summary>
+ public static readonly byte[] Symlink = NarWriter.GetLengthPrefixedPadded("symlink"u8);
+
+ /// <summary>Indicates a directory entry</summary>
+ public static readonly byte[] Entry = NarWriter.GetLengthPrefixedPadded("entry"u8);
+
+ /// <summary>Indicates a directory entry</summary>
+ public static readonly byte[] Node = NarWriter.GetLengthPrefixedPadded("node"u8);
+
+ /// <summary>Indicates the name of a directory entry</summary>
+ public static readonly byte[] Name = NarWriter.GetLengthPrefixedPadded("name"u8);
+
+ /// <summary>Indicates the target of a symlink</summary>
+ public static readonly byte[] Target = NarWriter.GetLengthPrefixedPadded("target"u8);
+
+ /// <summary>Indicates the type of an entry</summary>
+ public static readonly byte[] Type = NarWriter.GetLengthPrefixedPadded("type"u8);
+
+ /// <summary>Indicates that a regular file is executable</summary>
+ public static readonly byte[] Executable = NarWriter.GetLengthPrefixedPadded("executable"u8);
+
+ /// <summary>Indicates the start of regular file contents</summary>
+ public static readonly byte[] Contents = NarWriter.GetLengthPrefixedPadded("contents"u8);
+}
\ No newline at end of file
diff --git a/Rhea.Nar/NarWriter.cs b/Rhea.Nar/NarWriter.cs
index 424f52a..49b5eb6 100644
--- a/Rhea.Nar/NarWriter.cs
+++ b/Rhea.Nar/NarWriter.cs
@@ -1,31 +1,8 @@
-using System.IO.Enumeration;
-using System.Runtime.InteropServices;
using ArcaneLibs;
using ArcaneLibs.Extensions;
namespace Rhea.Nar;
-public ref struct NarConsts
-{
- public static readonly byte[] Header = NarWriter.GetLengthPrefixedPadded("nix-archive-1"u8);
-
- public static readonly byte[] LeftParen = NarWriter.GetLengthPrefixedPadded("("u8);
- public static readonly byte[] RightParen = NarWriter.GetLengthPrefixedPadded(")"u8);
-
- public static readonly byte[] Regular = NarWriter.GetLengthPrefixedPadded("regular"u8);
- public static readonly byte[] Directory = NarWriter.GetLengthPrefixedPadded("directory"u8);
- public static readonly byte[] Symlink = NarWriter.GetLengthPrefixedPadded("symlink"u8);
-
- public static readonly byte[] Entry = NarWriter.GetLengthPrefixedPadded("entry"u8);
- public static readonly byte[] Name = NarWriter.GetLengthPrefixedPadded("name"u8);
- public static readonly byte[] Target = NarWriter.GetLengthPrefixedPadded("target"u8);
- public static readonly byte[] Node = NarWriter.GetLengthPrefixedPadded("node"u8);
- public static readonly byte[] Type = NarWriter.GetLengthPrefixedPadded("type"u8);
- public static readonly byte[] Executable = NarWriter.GetLengthPrefixedPadded("executable"u8);
-
- public static readonly byte[] Contents = NarWriter.GetLengthPrefixedPadded("contents"u8);
-}
-
public class NarWriter
{
public Span<byte> GetHeader()
@@ -42,19 +19,19 @@ public class NarWriter
public IEnumerable<byte[]> GetNarStreamContentsInner(string path)
{
FileAttributes fattr = File.GetAttributes(path);
- if (fattr.HasFlag(FileAttributes.Directory))
- foreach (var chunk in GetNarDirectoryContents(new DirectoryInfo(path)))
- yield return chunk;
- else
+ var linkTarget = new FileInfo(path).LinkTarget;
+ if (string.IsNullOrWhiteSpace(linkTarget))
{
- var linkTarget = new FileInfo(path).LinkTarget;
- if (string.IsNullOrWhiteSpace(linkTarget))
- foreach (var chunk in GetNarFileContents(new FileInfo(path)))
+ if (fattr.HasFlag(FileAttributes.Directory))
+ foreach (var chunk in GetNarDirectoryContents(new DirectoryInfo(path)))
yield return chunk;
else
- foreach (var chunk in GetNarSymlinkContents(new FileInfo(path)))
+ foreach (var chunk in GetNarFileContents(new FileInfo(path)))
yield return chunk;
}
+ else
+ foreach (var chunk in GetNarSymlinkContents(new FileInfo(path)))
+ yield return chunk;
}
private IEnumerable<byte[]> GetNarDirectoryContents(DirectoryInfo de)
@@ -62,15 +39,19 @@ public class NarWriter
yield return NarConsts.LeftParen;
yield return NarConsts.Type;
yield return NarConsts.Directory;
-
- foreach (var fe in FileUtils.EnumerateDirectory(de.FullName).OrderBy(x => x.Name))
+
+ foreach (var fe in FileUtils.EnumerateDirectory(de.FullName)
+ // .OrderBy(x=>x is DirectoryInfo ? 0 : 1)
+ // .ThenBy(x => x.Name, StringComparer.InvariantCultureIgnoreCase)
+ .OrderBy(x => x.Name, StringComparer.Ordinal)
+ )
{
yield return NarConsts.Entry;
yield return NarConsts.LeftParen;
yield return NarConsts.Name;
yield return GetLengthPrefixedPadded(fe.Name.AsBytes().ToArray());
-
+
yield return NarConsts.Node;
foreach (var chunk in GetNarStreamContentsInner(fe.FullName))
yield return chunk;
@@ -83,14 +64,26 @@ public class NarWriter
private IEnumerable<byte[]> GetNarSymlinkContents(FileInfo fe)
{
- yield return NarConsts.LeftParen;
+ // yield return NarConsts.LeftParen;
+ //
+ // yield return NarConsts.Type;
+ // yield return NarConsts.Symlink;
+ // yield return NarConsts.Target;
+ // yield return GetLengthPrefixedPadded(fe.LinkTarget!.AsBytes().ToArray());
+ //
+ // yield return NarConsts.RightParen;
- yield return NarConsts.Type;
- yield return NarConsts.Symlink;
- yield return NarConsts.Target;
- yield return GetLengthPrefixedPadded(fe.LinkTarget!.AsBytes().ToArray());
+ yield return
+ [
+ ..NarConsts.LeftParen,
- yield return NarConsts.RightParen;
+ ..NarConsts.Type,
+ ..NarConsts.Symlink,
+ ..NarConsts.Target,
+ ..GetLengthPrefixedPadded(fe.LinkTarget!.AsBytes().ToArray()),
+
+ ..NarConsts.RightParen
+ ];
}
private IEnumerable<byte[]> GetNarFileContents(FileInfo fe)
@@ -104,7 +97,10 @@ public class NarWriter
|| fe.UnixFileMode.HasFlag(UnixFileMode.GroupExecute)
|| fe.UnixFileMode.HasFlag(UnixFileMode.OtherExecute)
)
+ {
yield return NarConsts.Executable;
+ yield return new byte[8];
+ }
var targetLength = Math.Ceiling(fe.Length / 8d) * 8UL;
yield return NarConsts.Contents;
@@ -135,11 +131,7 @@ public class NarWriter
{
byte[] unpadded = [..BitConverter.GetBytes((ulong)data.Length), .. data];
var targetLength = Math.Ceiling(unpadded.Length / 8d) * 8;
- // Console.Write($"{unpadded.Length:X2} - ");
- // unpadded.HexDump();
Array.Resize(ref unpadded, (int)targetLength);
- // Console.Write($"{unpadded.Length:X2} - ");
- // unpadded.HexDump();
return unpadded;
}
diff --git a/Rhea.Nar/Rhea.Nar.csproj b/Rhea.Nar/Rhea.Nar.csproj
index bcfc328..7057327 100644
--- a/Rhea.Nar/Rhea.Nar.csproj
+++ b/Rhea.Nar/Rhea.Nar.csproj
@@ -5,6 +5,8 @@
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+ <PublishAot>true</PublishAot>
+ <GenerateDocumentationFile>true</GenerateDocumentationFile>
</PropertyGroup>
<ItemGroup>
diff --git a/Rhea/Program.cs b/Rhea/Program.cs
index 5d3f242..a50de4d 100644
--- a/Rhea/Program.cs
+++ b/Rhea/Program.cs
@@ -1,10 +1,142 @@
using System.Diagnostics;
+using System.Reflection;
+using System.Text;
using ArcaneLibs;
using ArcaneLibs.Extensions;
-using ArcaneLibs.Extensions.Streams;
using Rhea;
using Rhea.Nar;
+
+// new NarWriter().GetHeader().ToArray().HexDump(64);
+
+foreach (var fieldInfo in typeof(NarConsts).GetFields())
+{
+ // Console.WriteLine(fieldInfo);
+ var rawBytes = (byte[])fieldInfo.GetValue(null)!;
+ var sizeBytes = rawBytes[..8];
+ var dataBytes = rawBytes[8..];
+ Console.WriteLine(
+ $$"""
+ public static readonly byte[] {{fieldInfo.Name}} = new byte[] {
+ {{string.Join(", ", sizeBytes.Select(x => x.ToString()))}}, // {{dataBytes.Trim((byte)0).ToArray().Length}} bytes
+ {{string.Join(", ", dataBytes.Select(x => x.ToString()))}}, // "{{Encoding.UTF8.GetString(dataBytes)}}"u8 + padding
+ {{string.Join(", ", rawBytes.Select(x => x.ToString()))}}
+ };
+ """
+ );
+}
+
+int maxVerifySize = 8_192_000;
+
+// return;
+// while (true)
+// foreach (var srcPath in (string[])[
+// "/etc/nix/nix.conf",
+// FileUtils.GetBinDir() + "/Rhea.runtimeconfig.json",
+// "/etc/nix",
+// ".",
+// "/nix/store/06d2xkrl3h1pbyg3m1whz1mvhafqgsqq-source",
+// "/nix/store/0l93s9x3z2s9p4r58jls0xiyk64hwkxz-source",
+// "/nix/store/2g4sa9xcir93nckzib9iq1ky6ly6azgw-7j1q60m17gf95k3s0gk2qvjqygnp1q36-source"
+// ])
+// foreach (var srcPath in (string[])["/nix/store/7j1q60m17gf95k3s0gk2qvjqygnp1q36-source"])
+Parallel.ForEach(FileUtils.EnumerateDirectory("/nix/store")
+ // .Where(x => x is not DirectoryInfo)
+ .Select(x => x.FullName),
+ new() { MaxDegreeOfParallelism = 12 },
+ srcPath =>
+ // foreach (var srcPath in FileUtils.EnumerateDirectory("/nix/store").Select(x=>x.FullName))
+ {
+ var sw = Stopwatch.StartNew();
+ TimeSpan oldTime, newTime;
+ var log = "";
+ log += ($"\n - {srcPath}:");
+ bool tooLarge = false;
+ // Console.WriteLine($"Original {srcPath}:");
+
+ var origBuf = new List<byte>();
+ var nixProcess = Process.Start(new ProcessStartInfo("nix", ["nar", "dump-path", srcPath])
+ {
+ RedirectStandardOutput = true
+ });
+ int b;
+ byte[] oldBuf = new byte[16384];
+ while ((b = nixProcess.StandardOutput.BaseStream.Read(oldBuf)) != 0)
+ {
+ if (!tooLarge)
+ {
+ if (origBuf.Count < maxVerifySize)
+ origBuf.AddRange(oldBuf[..b]);
+ else tooLarge = true;
+ }
+ // Console.WriteLine(b + " @ " + sw.Elapsed + " -> " + Util.BytesToString(origBuf.Count));
+ // Thread.Sleep(1);
+ }
+
+ // origBuf.ToArray().HexDump(64);
+ oldTime = sw.GetElapsedAndRestart();
+
+ // Console.WriteLine($"\nNew {srcPath}:");
+ List<byte> tbuf = [];
+ // List<byte> buf = [];
+ foreach (var chunk in new NarWriter().GetNarStreamContents(srcPath))
+ {
+ // Console.WriteLine($"ret Chunk({chunk.Length})");
+ if (!tooLarge)
+ {
+ if (tbuf.Count < maxVerifySize)
+ tbuf.AddRange(chunk);
+ else tooLarge = true;
+ }
+ // buf.AddRange(chunk);
+ // while (buf.Count > 64)
+ // {
+ // byte[] data = buf[..64].ToArray();
+ // buf = buf[64..];
+ // data.HexDump(64);
+ // }
+ }
+
+ // buf.ToArray().HexDump(64);
+ newTime = sw.GetElapsedAndRestart();
+
+ log += ($"\nOld time: {oldTime}, new time: {newTime}");
+ if (!tooLarge)
+ {
+ log += ("\nIs equal: " + origBuf.SequenceEqual(tbuf));
+ if (!origBuf.SequenceEqual(tbuf))
+ {
+ // File.WriteAllBytes("a", origBuf.ToArray().AsHexString().AsBytes().ToArray());
+ // File.WriteAllBytes("b", tbuf.ToArray().AsHexString().AsBytes().ToArray());
+ var origStdout = Console.Out;
+ using (var fs = File.OpenWrite("a"))
+ {
+ using var fsw = new StreamWriter(fs);
+ Console.SetOut(fsw);
+ origBuf.HexDump(width: 16, colorize: false);
+ Console.SetOut(origStdout);
+ }
+
+ using (var fs = File.OpenWrite("b"))
+ {
+ using var fsw = new StreamWriter(fs);
+ Console.SetOut(fsw);
+ tbuf.HexDump(width: 16, colorize: false);
+ Console.SetOut(origStdout);
+ }
+
+ Console.WriteLine(log);
+ Environment.Exit(0);
+ }
+ }
+ else log += "\nToo large to check consistency";
+
+ Console.WriteLine(log);
+ }
+);
+
+return;
+
var builder = WebApplication.CreateBuilder(args);
// Add services to the container.
@@ -27,41 +159,4 @@ app.UseAuthorization();
app.MapControllers();
-new NarWriter().GetHeader().ToArray().HexDump(64);
-
-
-foreach (var srcPath in (string[])["/etc/nix/nix.conf", FileUtils.GetBinDir() + "/Rhea.runtimeconfig.json", "/etc/nix"])
-{
- Console.WriteLine($"Original {srcPath}:");
-
- var pe = new ProcessStartInfo("nix", ["nar", "dump-path", srcPath])
- {
- RedirectStandardOutput = true
- };
- var origBuf = new List<byte>();
- var pr = Process.Start(pe);
- int b;
- while ((b = pr.StandardOutput.BaseStream.ReadByte()) != -1) origBuf.Add((byte)b);
- origBuf.ToArray().HexDump(64);
-
- Console.WriteLine($"\nNew {srcPath}:");
- List<byte> tbuf = [];
- List<byte> buf = [];
- foreach (var chunk in new NarWriter().GetNarStreamContents(srcPath))
- {
- tbuf.AddRange(chunk);
- buf.AddRange(chunk);
- while (buf.Count > 64)
- {
- byte[] data = buf[..64].ToArray();
- buf = buf[64..];
- data.HexDump(64);
- }
- }
-
- buf.ToArray().HexDump(64);
-
- Console.WriteLine("Is equal: " + origBuf.SequenceEqual(tbuf));
-}
-
// app.Run();
\ No newline at end of file
|
