Normalize entry paths like the engine, add editing, nested reading and Verify

README.md

@@ -43,6 +43,8 @@ if (file != null)
 
 Do note that files such as `pak01_001.vpk` are just data files, you have to open `pak01_dir.vpk`.
 
+`FindEntry` normalizes the path like added files are: `\` is a directory separator and ASCII letters are lowercased.
+
 To read from a stream instead of a path, set the file name first so that the chunk files can be located:
 
 ```csharp
@@ -113,6 +115,14 @@ var entry = package.FindEntry("path/to/file.txt");
 using var stream = package.GetMemoryMappedStreamIfPossible(entry);
 ```
 
+`OpenEntryStream` returns a seekable stream that reads an entry in place. A VPK stored in an entry of another VPK can be read from it without extracting it:
+
+```csharp
+using var inner = new Package();
+inner.SetFileName(entry.GetFullPath());
+inner.Read(package.OpenEntryStream(entry));
+```
+
 ## Optimize for many lookups
 
 By default, `FindEntry` performs a linear scan. If you need to look up many files, call `OptimizeEntriesForBinarySearch()` before `Read()` to sort entries and use binary search instead. You can also pass `StringComparison.OrdinalIgnoreCase` for case-insensitive lookups.
@@ -128,13 +138,18 @@ package.Read("pak01_dir.vpk");
 var file = package.FindEntry("path/to/file.txt");
 ```
 
+`Package.Open("pak01_dir.vpk")` does both in one call.
+
 ## Verification
 
 ```csharp
 using var package = new Package();
 package.Read("pak01_dir.vpk");
 
-// Verify MD5 hashes of the directory tree and whole file (throws on version 1 packages)
+// Verify the signature, all hashes, and the CRC32 checksums of files no hash covers in one call
+package.Verify();
+
+// Verify MD5 hashes of the directory tree and whole file (version 1 packages have none)
 package.VerifyHashes();
 
 // Verify MD5/Blake3 hashes of individual chunk files (pak01_000.vpk, pak01_001.vpk, ...)
@@ -167,7 +182,7 @@ All file data is stored in the single written file, which is limited to 2 GiB. T
 
 ### Modify an existing VPK
 
-A package that was read can have files added or removed and then be written to a new file. This only works for packages that are not split into chunk files. Writing back a package opened from a `_dir.vpk` is not supported.
+A package that was read can have files added, removed or renamed and then be written to a new file, or back over the file it was read from, including a `_dir.vpk` with chunk files.
 
 ```csharp
 using var package = new Package();
@@ -179,18 +194,20 @@ package.RemoveFile(package.FindEntry("old/file.txt")!);
 package.Write("pak01_modified.vpk");
 ```
 
+Chunk files are named after the directory file, so a package with chunk files can only be written to another path with `repack: true`, which `RequiresRepack(path)` tells. Repacking a package without chunk files into a `_dir.vpk` moves its data into chunk files.
+
 ### Split into chunk files
 
-Files added with `multiChunk` are written right away into numbered chunk files (`pak01_000.vpk`, `pak01_001.vpk`, ...) next to the directory file, so their data is not kept in memory. A new chunk file is started once the current one reaches `WriteChunkSize` (200 MiB by default). `Write` then only writes the directory file, which contains MD5 hashes of the chunk files that can be verified with `VerifyChunkHashes`.
+Files added with `multiChunk` are written right away into numbered chunk files (`pak01_000.vpk`, `pak01_001.vpk`, ...) next to the directory file, so their data is not kept in memory. A new chunk file is started once the current one is larger than `WriteChunkSize` (100 MiB by default). `Write` then only writes the directory file, which contains Blake3 hashes of the chunk files that can be verified with `VerifyChunkHashes`.
 
 ```csharp
 using var package = new Package();
 
 // Chunk files are created next to the directory file, so its path must be set first
 package.SetFileName("pak01_dir.vpk");
 
-// Optionally lower the maximum chunk file size, in bytes
-package.WriteChunkSize = 100 * 1024 * 1024;
+// Optionally change the chunk file size, in bytes
+package.WriteChunkSize = 50 * 1024 * 1024;
 
 // Data can be streamed from a file instead of being read into memory
 using (var stream = File.OpenRead("example.vmdl"))
@@ -205,4 +222,4 @@ package.AddFile("manifest.txt", File.ReadAllBytes("manifest.txt"));
 package.Write("pak01_dir.vpk");
 ```
 
-Removing a file that is already in a chunk file leaves its data in place. Chunk files stay on disk if the package is disposed without being written.
+Removing a file that is already in a chunk file leaves its data in place until the package is written with `repack: true`. Chunk files stay on disk if the package is disposed without being written.

ValvePak/ValvePak.Test/EntryPathTest.cs

@@ -0,0 +1,80 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace ValvePak.Test
+{
+	internal sealed class EntryPathTest
+	{
+		[Test]
+		[Arguments("A\\B//./c.TXT", "a/b/c.txt")]
+		[Arguments("a/x/../b.txt", "a/b.txt")]
+		[Arguments("/a/b.txt", "a/b.txt")]
+		[Arguments("README", "readme")]
+		[Arguments("", null)]
+		[Arguments("../b.txt", null)]
+		[Arguments("a/b/", null)]
+		[Arguments(".gitignore", null)]
+		[Arguments("x.. ", null)]
+		[Arguments("a/b\0c.txt", null)]
+		[Arguments("a/\ud800.txt", null)]
+		public async Task NormalizesEntryPaths(string path, string? expected)
+		{
+			using (Assert.Multiple())
+			{
+				await Assert.That(Package.TryParseEntryPath(path, out var normalizedPath)).IsEqualTo(expected != null);
+				await Assert.That(normalizedPath).IsEqualTo(expected);
+			}
+		}
+
+		[Test]
+		public async Task RejectsNamesThatAreTooLong()
+		{
+			using (Assert.Multiple())
+			{
+				await Assert.That(Package.TryParseEntryPath($"a.{new string('y', 129)}", out _)).IsFalse();
+				await Assert.That(Package.TryParseEntryPath($"{new string('é', 131)}.txt", out _)).IsFalse();
+			}
+		}
+
+		[Test]
+		[Arguments("Materials\\Models/", "materials/models")]
+		[Arguments(" ", "")]
+		public async Task NormalizesDirectoryPaths(string path, string expected)
+		{
+			await Assert.That(Package.NormalizeDirectoryPath(path)).IsEqualTo(expected);
+		}
+
+		[Test]
+		[Arguments(" ")]
+		[Arguments("")]
+		public async Task GetsDirectoryPathWithRootAsEmptyString(string directoryName)
+		{
+			var entry = new PackageEntry
+			{
+				DirectoryName = directoryName,
+				FileName = "file",
+				TypeName = "txt",
+			};
+
+			using (Assert.Multiple())
+			{
+				await Assert.That(entry.GetDirectoryPath()).IsEmpty();
+				await Assert.That(entry.GetFullPath()).IsEqualTo("file.txt");
+			}
+		}
+
+		[Test]
+		public async Task PathComparerIgnoresOnlyTheCaseOfAsciiLetters()
+		{
+			var folders = new Dictionary<string, int>(Package.PathComparer) { ["Models"] = 1 };
+
+			using (Assert.Multiple())
+			{
+				await Assert.That(Package.PathComparer.Equals("MODELS/A.txt", "models/a.TXT")).IsTrue();
+				await Assert.That(Package.PathComparer.Equals("Ä", "ä")).IsFalse();
+				await Assert.That(folders.GetAlternateLookup<ReadOnlySpan<char>>().ContainsKey("mODELS")).IsTrue();
+			}
+		}
+	}
+}

ValvePak/ValvePak.Test/MultiChunkWriteTest.cs

@@ -88,6 +88,22 @@ public async Task WriteMultiChunkPackage()
 			await Assert.That(dirEntry.ArchiveIndex).IsEqualTo((ushort)0x7FFF);
 		}
 
+		[Test]
+		public async Task WriteDeletesStaleChunkFilesOfNewPackage()
+		{
+			var dirPath = TempPath("stale_dir.vpk");
+			await File.WriteAllBytesAsync(TempPath("stale_001.vpk"), [1]);
+
+			using (var package = new Package())
+			{
+				package.SetFileName(dirPath);
+				package.AddFile("a.bin", CreateTestData(200, 1), multiChunk: true);
+				package.Write(dirPath);
+			}
+
+			await Assert.That(File.Exists(TempPath("stale_001.vpk"))).IsFalse();
+		}
+
 		[Test]
 		public async Task WritesChunkHashFractions()
 		{
@@ -609,8 +625,8 @@ public async Task EditsReadPackageWithChunkFiles()
 				await AssertEntryData(readBack, "added/embedded.bin", 300, 2);
 				await Assert.That(readBack.FindEntry("steammessages_base.proto")!.SmallData).Count().IsEqualTo(preload.Length);
 
-				// The read chunk file had no hashes, and data in the directory file is not hashed when there are chunk files
-				await Assert.That(readBack.AccessPackFileHashes.Select(static fraction => fraction.ArchiveIndex).ToArray()).IsEquivalentTo(new ushort[] { 2, 4 }, CollectionOrdering.Matching);
+				// Chunk files without hashes are hashed, and data in the directory file is not hashed when there are chunk files
+				await Assert.That(readBack.AccessPackFileHashes.Select(static fraction => fraction.ArchiveIndex).ToArray()).IsEquivalentTo(new ushort[] { 0, 1, 2, 3, 4 }, CollectionOrdering.Matching);
 			}
 
 			// Hashes that cover the whole chunk file are kept even when its data changed, and are computed again once it grew
@@ -847,6 +863,142 @@ public async Task RepackRefusesToReplaceChunkFilesOfTheReadFileFromAnotherPath()
 			await AssertPackageVerifies(package);
 		}
 
+		[Test]
+		public async Task RequiresRepackForPathsTheChunkFilesAreNotNamedAfter()
+		{
+			var dirPath = CopyFixtures("test_dir.vpk", "test_000.vpk");
+			var otherChunkNames = TempPath("copy_dir.vpk");
+
+			using var package = new Package();
+			package.Read(dirPath);
+
+			using (Assert.Multiple())
+			{
+				await Assert.That(package.RequiresRepack(dirPath)).IsFalse();
+				await Assert.That(package.RequiresRepack(TempPath("test.vpk"))).IsFalse();
+				await Assert.That(package.RequiresRepack(otherChunkNames)).IsTrue();
+				await Assert.That(() => package.Write(otherChunkNames)).ThrowsExactly<InvalidOperationException>();
+			}
+
+			package.Write(otherChunkNames, package.RequiresRepack(otherChunkNames));
+			await AssertPackageVerifies(package);
+		}
+
+		[Test]
+		public async Task RepackIntoDirVpkMovesDirectoryFileData()
+		{
+			var path = TempPath("single.vpk");
+			var dirPath = TempPath("single_dir.vpk");
+
+			using (var package = new Package())
+			{
+				package.AddFile("a.bin", CreateTestData(400, 1));
+				package.Write(path);
+			}
+
+			using (var package = new Package())
+			{
+				package.Read(path);
+				package.AddFile("b.bin", CreateTestData(300, 2));
+				package.Write(dirPath, repack: true);
+			}
+
+			using var readBack = new Package();
+			readBack.Read(dirPath);
+
+			await Assert.That(readBack.Entries!.Values.SelectMany(static entries => entries).All(static entry => entry.ArchiveIndex == 0)).IsTrue();
+			await AssertEntryData(readBack, "a.bin", 400, 1);
+			await AssertEntryData(readBack, "b.bin", 300, 2);
+			await AssertPackageVerifies(readBack);
+		}
+
+		[Test]
+		public async Task OpensNestedPackagesInPlace()
+		{
+			var leafData = CreateTestData(10000, 1);
+			var level2 = CreatePackageBytes(package => package.AddFile("data/leaf.bin", leafData));
+			var level1 = CreatePackageBytes(package =>
+			{
+				var entry = package.AddFile("maps/level2.vpk", level2[20..]);
+				entry.SmallData = level2[..20];
+				entry.CRC32 = Crc32.HashToUInt32(level2);
+			});
+
+			var path = TempPath("level0_dir.vpk");
+
+			using (var created = new Package())
+			{
+				created.SetFileName(path);
+				created.AddFile("maps/level1.vpk", level1, multiChunk: true);
+				created.Write(path);
+			}
+
+			using var level0Package = Package.Open(path);
+			using var level1Package = OpenNestedPackage(level0Package, level0Package.FindEntry("maps/level1.vpk")!);
+			using var level2Package = OpenNestedPackage(level1Package, level1Package.FindEntry("maps/level2.vpk")!);
+
+			var leafEntry = level2Package.FindEntry("data/leaf.bin")!;
+			level2Package.ReadEntry(leafEntry, out var leafRead);
+
+			using var mapped = level2Package.GetMemoryMappedStreamIfPossible(leafEntry);
+
+			using (Assert.Multiple())
+			{
+				await Assert.That(leafRead).IsEquivalentTo(leafData, CollectionOrdering.Matching);
+				await Assert.That(mapped is UnmanagedMemoryStream).IsTrue();
+			}
+
+			level1Package.Dispose();
+
+			await Assert.That(() => level2Package.ReadEntry(leafEntry, out _)).ThrowsExactly<ObjectDisposedException>();
+		}
+
+		[Test]
+		public async Task NestedPackagesDoNotUseChunkFiles()
+		{
+			using var innerDirectoryFile = new MemoryStream();
+
+			using (var inner = new Package())
+			{
+				inner.SetFileName(TempPath("inner_dir.vpk"));
+				inner.AddFile("chunked.bin", CreateTestData(100, 1), multiChunk: true);
+				inner.AddFile("embedded.bin", CreateTestData(100, 2));
+				inner.Write(innerDirectoryFile);
+			}
+
+			using var outer = new Package();
+			outer.AddFile("maps/inner_dir.vpk", innerDirectoryFile.ToArray());
+
+			using var nested = OpenNestedPackage(outer, outer.FindEntry("maps/inner_dir.vpk")!);
+
+			using (Assert.Multiple())
+			{
+				await Assert.That(() => nested.ReadEntry(nested.FindEntry("embedded.bin")!, out _)).ThrowsNothing();
+				await Assert.That(() => nested.ReadEntry(nested.FindEntry("chunked.bin")!, out _)).ThrowsExactly<NotSupportedException>();
+			}
+		}
+
+		private static Package OpenNestedPackage(Package outer, PackageEntry entry)
+		{
+			var package = new Package();
+			package.OptimizeEntriesForBinarySearch();
+			package.SetFileName(entry.GetFullPath());
+			package.Read(outer.OpenEntryStream(entry));
+
+			return package;
+		}
+
+		private static byte[] CreatePackageBytes(Action<Package> addFiles)
+		{
+			using var package = new Package();
+			addFiles(package);
+
+			using var output = new MemoryStream();
+			package.Write(output);
+
+			return output.ToArray();
+		}
+
 		private string TempPath(string fileName)
 		{
 			return Path.Combine(TempDirectory, fileName);