diff --git a/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs b/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs index 3745a4a..0786e9d 100644 --- a/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs +++ b/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs @@ -1,9 +1,12 @@ using System; +using System.Collections.Generic; using System.IO; -using System.Security.Cryptography; +using System.IO.Hashing; +using System.Linq; using System.Text; using System.Threading.Tasks; using TUnit.Assertions.Enums; +using TUnit.Core.Enums; namespace ValvePak.Test { @@ -34,13 +37,13 @@ public async Task WriteMultiChunkPackage() using (var package = new Package()) { package.SetFileName(dirPath); - package.WriteChunkSize = 1024; + package.WriteChunkSize = 800; for (var i = 0; i < 10; i++) { var entry = package.AddFile($"files/chunked_{i}.bin", CreateTestData(400, (byte)i), multiChunk: true); - // 400 byte files with a 1024 chunk size, so three files per chunk + // 400 byte files with an 800 byte chunk size: a chunk file at exactly 800 bytes is not full yet, so each one holds three files await Assert.That(entry.ArchiveIndex).IsEqualTo((ushort)(i / 3)); } @@ -112,8 +115,8 @@ public async Task WritesChunkHashFractions() await Assert.That(readBack.AccessPackFileHashes[0].ArchiveIndex).IsZero(); await Assert.That(readBack.AccessPackFileHashes[0].Offset).IsZero(); await Assert.That(readBack.AccessPackFileHashes[0].Length).IsEqualTo((uint)FractionSize); - await Assert.That(readBack.AccessPackFileHashes[0].HashType).IsEqualTo(EHashType.MD5); - await Assert.That(readBack.AccessPackFileHashes[0].Checksum).IsEquivalentTo(MD5.HashData(fileA.AsSpan(0, FractionSize)), CollectionOrdering.Matching); + await Assert.That(readBack.AccessPackFileHashes[0].HashType).IsEqualTo(EHashType.Blake3); + await Assert.That(readBack.AccessPackFileHashes[0].Checksum).IsEquivalentTo(HashBlake3(fileA.AsSpan(0, FractionSize)), CollectionOrdering.Matching); await Assert.That(readBack.AccessPackFileHashes[1].ArchiveIndex).IsZero(); await Assert.That(readBack.AccessPackFileHashes[1].Offset).IsEqualTo((uint)FractionSize); @@ -161,11 +164,11 @@ public async Task HashesFractionSpanningMultipleFiles() { await Assert.That(readBack.AccessPackFileHashes[0].Offset).IsZero(); await Assert.That(readBack.AccessPackFileHashes[0].Length).IsEqualTo((uint)FractionSize); - await Assert.That(readBack.AccessPackFileHashes[0].Checksum).IsEquivalentTo(MD5.HashData(fraction0), CollectionOrdering.Matching); + await Assert.That(readBack.AccessPackFileHashes[0].Checksum).IsEquivalentTo(HashBlake3(fraction0), CollectionOrdering.Matching); await Assert.That(readBack.AccessPackFileHashes[1].Offset).IsEqualTo((uint)FractionSize); await Assert.That(readBack.AccessPackFileHashes[1].Length).IsEqualTo((uint)(fileA.Length + fileB.Length - FractionSize)); - await Assert.That(readBack.AccessPackFileHashes[1].Checksum).IsEquivalentTo(MD5.HashData(fileB.AsSpan(FractionSize - fileA.Length)), CollectionOrdering.Matching); + await Assert.That(readBack.AccessPackFileHashes[1].Checksum).IsEquivalentTo(HashBlake3(fileB.AsSpan(FractionSize - fileA.Length)), CollectionOrdering.Matching); } await AssertPackageVerifies(readBack); @@ -274,17 +277,17 @@ public async Task AddFileThrowsWhenExceedingChunkLimit() package.SetFileName(TempPath("limit_dir.vpk")); package.WriteChunkSize = 1; - // With a chunk size of 1 every file starts a new chunk + // With a chunk size of 1 every file of 2 bytes starts a new chunk PackageEntry? entry = null; for (var i = 0; i < 0x7FFF; i++) { - entry = package.AddFile($"{i}.bin", [1], multiChunk: true); + entry = package.AddFile($"{i}.bin", [1, 2], multiChunk: true); } await Assert.That(entry!.ArchiveIndex).IsEqualTo((ushort)0x7FFE); - await Assert.That(() => package.AddFile("one too many.bin", [1], multiChunk: true)).ThrowsExactly() + await Assert.That(() => package.AddFile("one too many.bin", [1, 2], multiChunk: true)).ThrowsExactly() .WithMessageContaining("maximum amount of chunk files", StringComparison.Ordinal); } @@ -368,6 +371,9 @@ public async Task WritesChunkDataWhenFilesAreAdded() { var dirPath = TempPath("streamed_dir.vpk"); + // A package that was not read replaces existing chunk files + await File.WriteAllBytesAsync(TempPath("streamed_000.vpk"), new byte[4096]); + using var package = new Package(); package.SetFileName(dirPath); package.WriteChunkSize = 1024; @@ -492,16 +498,333 @@ public async Task WritesAgainAfterAddingMoreFiles() public async Task WriteChunkSizeValidation() { using var package = new Package(); - await Assert.That(package.WriteChunkSize).IsEqualTo(200 * 1024 * 1024); + await Assert.That(package.WriteChunkSize).IsEqualTo(100 * 1024 * 1024); await Assert.That(() => package.WriteChunkSize = 0).ThrowsExactly(); await Assert.That(() => package.WriteChunkSize = -1).ThrowsExactly(); } + [Test] + public async Task WriteRefusesFilesItReadsFromAndMissingChunkFiles() + { + var dirPath = CopyFixtures("test_dir.vpk", "test_000.vpk"); + var originalDir = await File.ReadAllBytesAsync(dirPath); + + using (var package = new Package()) + { + package.Read(dirPath); + + foreach (var path in new[] { dirPath, TempPath("test_000.vpk") }) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite); + await Assert.That(() => package.Write(stream)).ThrowsExactly(); + } + } + + using var memoryPackage = new Package(); + var input = ReadFromMemory(memoryPackage, dirPath); + await Assert.That(() => memoryPackage.Write(input)).ThrowsExactly(); + + File.Delete(TempPath("test_000.vpk")); + + await Assert.That(() => memoryPackage.Write(dirPath)).ThrowsExactly(); + await Assert.That(await File.ReadAllBytesAsync(dirPath)).IsEquivalentTo(originalDir, CollectionOrdering.Matching); + } + + [Test] + public async Task EditsReadPackageWithChunkFiles() + { + var dirPath = CopyFixtures("test_dir.vpk", "test_000.vpk"); + var originalChunk = await File.ReadAllBytesAsync(TempPath("test_000.vpk")); + var stale = CreateTestData(100, 9); + var preload = CreateTestData(10, 7); + + // Chunk files that the directory file does not refer to are skipped + await File.WriteAllBytesAsync(TempPath("test_001.vpk"), stale); + await File.WriteAllBytesAsync(TempPath("test_003.vpk"), stale); + + using (var package = new Package()) + { + ReadFromMemory(package, dirPath); + package.WriteChunkSize = 400; + package.RemoveFile(package.FindEntry("steammessages_base.proto")!); + + var replaced = package.AddFile("steammessages_base.proto", CreateTestData(500, 3), multiChunk: true); + replaced.SmallData = preload; + replaced.CRC32 = Crc32.HashToUInt32([.. preload, .. CreateTestData(500, 3)]); + var second = package.AddFile("added/second.bin", CreateTestData(100, 4), multiChunk: true); + package.AddFile("added/embedded.bin", CreateTestData(300, 2)); + + using (Assert.Multiple()) + { + await Assert.That(replaced.ArchiveIndex).IsEqualTo((ushort)2); + await Assert.That(second.ArchiveIndex).IsEqualTo((ushort)4); + } + + package.Write(dirPath); + } + + // Data of removed files stays in the chunk file it is in + using (Assert.Multiple()) + { + await Assert.That(await File.ReadAllBytesAsync(TempPath("test_000.vpk"))).IsEquivalentTo(originalChunk, CollectionOrdering.Matching); + await Assert.That(Directory.GetFiles(TempDirectory)).Count().IsEqualTo(6); + } + + using (var readBack = new Package()) + { + readBack.Read(dirPath); + + await AssertPackageVerifies(readBack); + await AssertEntryData(readBack, "added/second.bin", 100, 4); + await AssertEntryData(readBack, "added/embedded.bin", 300, 2); + await Assert.That(readBack.FindEntry("steammessages_base.proto")!.SmallData).Count().IsEqualTo(preload.Length); + + // The read chunk file had no hashes, and data in the directory file is not hashed when there are chunk files + await Assert.That(readBack.AccessPackFileHashes.Select(static fraction => fraction.ArchiveIndex).ToArray()).IsEquivalentTo(new ushort[] { 2, 4 }, CollectionOrdering.Matching); + } + + // Hashes that cover the whole chunk file are kept even when its data changed, and are computed again once it grew + await File.WriteAllBytesAsync(TempPath("test_004.vpk"), stale); + await Assert.That(() => WriteFromMemoryAndVerifyChunkHashes(dirPath)).ThrowsExactly(); + + await File.AppendAllTextAsync(TempPath("test_004.vpk"), "grown"); + await Assert.That(() => WriteFromMemoryAndVerifyChunkHashes(dirPath)).ThrowsNothing(); + } + + [Test] + public async Task WritesOverTheFileItWasReadFrom() + { + var path = CopyFixtures("preload.vpk"); + + using (var package = new Package()) + { + package.Read(path); + package.AddFile("added.bin", CreateTestData(300, 2)); + package.AddFile("chunked.bin", CreateTestData(500, 3), multiChunk: true); + package.Write(path); + + // The package reads from the written file, where the data of every entry has moved + await AssertPackageVerifies(package); + await Assert.That(package.FindEntry("lorem.txt")!.SmallData).Count().IsEqualTo(56); + + package.Write(path); + } + + await Assert.That(Directory.GetFiles(TempDirectory)).Count().IsEqualTo(2); + + using var readBack = new Package(); + readBack.Read(path); + + await AssertPackageVerifies(readBack); + await AssertEntryData(readBack, "added.bin", 300, 2); + await AssertEntryData(readBack, "chunked.bin", 500, 3); + await Assert.That(readBack.FindEntry("lorem.txt")!.SmallData).Count().IsEqualTo(56); + } + + [Test] + [RunOn(OS.Windows)] + public async Task WritesOtherFilesInPlaceAndKeepsTheReadFileWhenItCanNotBeReplaced() + { + var path = CopyFixtures("preload.vpk"); + var original = await File.ReadAllBytesAsync(path); + var otherPath = TempPath("other.vpk"); + await File.WriteAllBytesAsync(otherPath, new byte[4096]); + File.SetAttributes(otherPath, FileAttributes.Hidden); + + using var package = new Package(); + package.Read(path); + package.AddFile("added.bin", CreateTestData(300, 2)); + + File.SetAttributes(path, FileAttributes.ReadOnly); + await Assert.That(() => package.Write(path)).ThrowsExactly(); + File.SetAttributes(path, FileAttributes.Normal); + + package.Write(otherPath); + + using var readBack = new Package(); + readBack.Read(otherPath); + + using (Assert.Multiple()) + { + await Assert.That(await File.ReadAllBytesAsync(path)).IsEquivalentTo(original, CollectionOrdering.Matching); + await Assert.That(Directory.GetFiles(TempDirectory)).Count().IsEqualTo(2); + await Assert.That(File.GetAttributes(otherPath).HasFlag(FileAttributes.Hidden)).IsTrue(); + await Assert.That(() => readBack.VerifyFileChecksums()).ThrowsNothing(); + } + } + + [Test] + public async Task RepackDropsDataOfRemovedFiles() + { + var dirPath = TempPath("repack_dir.vpk"); + + using (var package = new Package()) + { + package.SetFileName(dirPath); + package.WriteChunkSize = 1000; + + for (var i = 0; i < 9; i++) + { + package.AddFile($"file_{i}.bin", CreateTestData(400, (byte)i), multiChunk: true); + } + + package.AddFile("in_dir.bin", CreateTestData(300, 9)); + package.Write(dirPath); + + foreach (var i in new[] { 0, 1, 3, 4, 5, 7, 8 }) + { + package.RemoveFile(package.FindEntry($"file_{i}.bin")!); + } + + package.AddFile("last.bin", CreateTestData(100, 9), multiChunk: true); + package.Write(dirPath, repack: true); + + var next = package.AddFile("next.bin", CreateTestData(400, 8), multiChunk: true); + await Assert.That(next.ArchiveIndex).IsEqualTo((ushort)1); + + package.Write(dirPath); + } + + await Assert.That(Directory.GetFiles(TempDirectory)).Count().IsEqualTo(3); + File.SetAttributes(TempPath("repack_000.vpk"), FileAttributes.ReadOnly); + File.SetAttributes(TempPath("repack_001.vpk"), FileAttributes.ReadOnly); + + using (var package = new Package()) + { + package.Read(dirPath); + package.RemoveFile(package.FindEntry("file_2.bin")!); + package.Write(dirPath, repack: true); + + // The package now reads from the repacked files + await AssertPackageVerifies(package); + await AssertEntryData(package, "in_dir.bin", 300, 9); + } + + await Assert.That(Directory.GetFiles(TempDirectory)).Count().IsEqualTo(2); + + using var readBack = new Package(); + readBack.Read(dirPath); + + // Data is written in directory tree order + await Assert.That((await AssertEntryData(readBack, "file_6.bin", 400, 6)).Offset).IsZero(); + await Assert.That((await AssertEntryData(readBack, "next.bin", 400, 8)).Offset).IsEqualTo(500u); + } + + [Test] + public async Task RepackWritesToNewBaseName() + { + var dirPath = CopyFixtures("test_dir.vpk", "test_000.vpk"); + var originalFiles = Directory.GetFiles(TempDirectory).ToDictionary(static path => path, File.ReadAllBytes); + var newPath = TempPath("copy_dir.vpk"); + await File.WriteAllTextAsync(TempPath("copy_001.vpk"), "stale"); + await File.WriteAllTextAsync(TempPath("copy_003.vpk"), "stale"); + + using (var package = new Package()) + { + package.Read(dirPath); + package.Write(newPath, repack: true); + + await Assert.That(package.FileName).IsEqualTo(TempPath("copy")); + await Assert.That(() => package.Write(dirPath)).ThrowsExactly(); + await AssertPackageVerifies(package); + } + + // A package without chunk files leaves the chunk files of another package with the same name alone + using (var package = new Package()) + { + package.AddFile("a.bin", CreateTestData(10, 1)); + package.Write(TempPath("test.vpk"), repack: true); + } + + await AssertFilesUnchanged(originalFiles); + + // Deleting stops at the first chunk file that does not exist + await Assert.That(File.Exists(TempPath("copy_001.vpk"))).IsFalse(); + await Assert.That(File.Exists(TempPath("copy_003.vpk"))).IsTrue(); + } + + [Test] + [RunOn(OS.Windows)] + public async Task RepackKeepsAllFilesWhenOneCanNotBeReplaced() + { + var dirPath = CopyFixtures("test_dir.vpk", "test_000.vpk"); + var originalFiles = Directory.GetFiles(TempDirectory).ToDictionary(static path => path, File.ReadAllBytes); + + using var package = new Package(); + package.Read(dirPath); + package.RemoveFile(package.FindEntry("kitten.jpg")!); + + // The chunk file is replaced before the directory file, which can not be + File.SetAttributes(dirPath, FileAttributes.ReadOnly); + await Assert.That(() => package.Write(dirPath, repack: true)).ThrowsExactly(); + File.SetAttributes(dirPath, FileAttributes.Normal); + + await Assert.That(Directory.GetFiles(TempDirectory)).Count().IsEqualTo(originalFiles.Count); + + await AssertFilesUnchanged(originalFiles); + + // The package still reads the original files, and can be repacked once they can be replaced + await AssertPackageVerifies(package); + package.Write(dirPath, repack: true); + await AssertPackageVerifies(package); + } + private string TempPath(string fileName) { return Path.Combine(TempDirectory, fileName); } + private string CopyFixtures(params string[] fileNames) + { + foreach (var fileName in fileNames) + { + File.Copy(Path.Combine(AppContext.BaseDirectory, "Files", fileName), TempPath(fileName)); + } + + return TempPath(fileNames[0]); + } + + private static async Task AssertFilesUnchanged(Dictionary originalFiles) + { + foreach (var (path, data) in originalFiles) + { + await Assert.That(await File.ReadAllBytesAsync(path)).IsEquivalentTo(data, CollectionOrdering.Matching); + } + } + + private static MemoryStream ReadFromMemory(Package package, string dirPath) + { + var input = new MemoryStream(File.ReadAllBytes(dirPath)); + package.SetFileName(dirPath); + package.Read(input); + + return input; + } + + private static void WriteFromMemoryAndVerifyChunkHashes(string dirPath) + { + using (var package = new Package()) + { + ReadFromMemory(package, dirPath); + package.Write(dirPath); + } + + using var readBack = new Package(); + readBack.Read(dirPath); + readBack.VerifyChunkHashes(); + } + + private static byte[] HashBlake3(ReadOnlySpan data) + { + using var hasher = Blake3.Hasher.New(); + hasher.Update(data); + + var hash = new byte[16]; + hasher.Finalize(hash); + + return hash; + } + private static async Task AssertPackageVerifies(Package package) { using (Assert.Multiple()) diff --git a/ValvePak/ValvePak.Test/PackageTest.cs b/ValvePak/ValvePak.Test/PackageTest.cs index 4fd7063..7c8b7d7 100644 --- a/ValvePak/ValvePak.Test/PackageTest.cs +++ b/ValvePak/ValvePak.Test/PackageTest.cs @@ -524,6 +524,44 @@ public async Task InvalidWholeFileChecksumThrows() await Assert.That(() => package.VerifyHashes()).ThrowsExactly(); } + [Test] + [Arguments(0, "File tree")] + [Arguments(16, "Archive MD5 entries")] + public async Task SectionChecksumMismatchIsReportedWhenWholeFileChecksumMatches(int checksumOffset, string sectionName) + { + var data = CreateSmallPackageBytes(); + var otherMD5SectionOffset = GetOtherMD5SectionOffset(data); + + data.AsSpan(otherMD5SectionOffset + checksumOffset, 16).Fill(0xAB); + MD5.HashData(data.AsSpan(0, otherMD5SectionOffset + 32)).CopyTo(data, otherMD5SectionOffset + 32); + + using var package = new Package(); + package.SetFileName("a.vpk"); + package.Read(new MemoryStream(data)); + + var progress = new SynchronousProgress(); + + await Assert.That(() => package.VerifyHashes(progress)).ThrowsNothing(); + await Assert.That(progress.Reports).Count().IsEqualTo(1); + await Assert.That(progress.Reports[0]).StartsWith($"{sectionName} checksum mismatch (", StringComparison.Ordinal); + } + + [Test] + public async Task SectionChecksumMismatchThrowsWhenWholeFileChecksumMismatches() + { + var data = CreateSmallPackageBytes(); + var otherMD5SectionOffset = GetOtherMD5SectionOffset(data); + + data.AsSpan(otherMD5SectionOffset, 16).Fill(0xAB); + + using var package = new Package(); + package.SetFileName("a.vpk"); + package.Read(new MemoryStream(data)); + + await Assert.That(() => package.VerifyHashes()).ThrowsExactly() + .WithMessageContaining("File tree checksum mismatch", StringComparison.Ordinal); + } + [Test] public async Task InvalidSignatureFails() { @@ -629,7 +667,7 @@ public async Task ThrowsWhenReadTwice() } [Test] - public async Task VerifyHashesThrowsOnVersion1() + public async Task VerifyHashesSkipsVersion1() { using var package = new Package(); package.SetFileName("a.vpk"); @@ -638,8 +676,10 @@ public async Task VerifyHashesThrowsOnVersion1() using var ms = new MemoryStream([0x34, 0x12, 0xAA, 0x55, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00]); package.Read(ms); - await Assert.That(() => package.VerifyHashes()).ThrowsExactly() - .WithMessage("Only version 2 is supported.", StringComparison.Ordinal); + var progress = new SynchronousProgress(); + + await Assert.That(() => package.VerifyHashes(progress)).ThrowsNothing(); + await Assert.That(progress.Reports).IsEquivalentTo(["Version 1 packages have no checksums to verify."]); } [Test] @@ -870,8 +910,11 @@ public async Task ThrowsOnDirectoryTreeEndingInTheMiddleOfString(string tree) } private const int TreeSizeOffset = 8; + private const int FileDataSectionSizeOffset = 12; private const int ArchiveMD5SectionSizeOffset = 16; private const int SignatureSectionSizeOffset = 24; + private const int HeaderSize = 28; + private const int UnsignedSignatureSectionSize = 20; private const int EmbeddedPrefixLength = 1000; private const int EmbeddedSuffixLength = 50; @@ -886,16 +929,26 @@ private static byte[] CreateSmallPackageBytes() return output.ToArray(); } + private static int GetOtherMD5SectionOffset(byte[] package) + { + var span = package.AsSpan(); + var treeSize = BinaryPrimitives.ReadInt32LittleEndian(span[TreeSizeOffset..]); + var fileDataSectionSize = BinaryPrimitives.ReadInt32LittleEndian(span[FileDataSectionSizeOffset..]); + var archiveMD5SectionSize = BinaryPrimitives.ReadInt32LittleEndian(span[ArchiveMD5SectionSizeOffset..]); + + return HeaderSize + treeSize + fileDataSectionSize + archiveMD5SectionSize; + } + /// /// Creates a package whose signature section holds the given little-endian integers. /// private static byte[] CreateSignedPackageBytes(params int[] signatureSection) { - var package = CreateSmallPackageBytes(); + var package = CreateSmallPackageBytes().AsSpan(..^UnsignedSignatureSectionSize); var signatureSize = signatureSection.Length * sizeof(int); var result = new byte[package.Length + signatureSize]; - package.CopyTo(result, 0); + package.CopyTo(result); BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(SignatureSectionSizeOffset), (uint)signatureSize); for (var i = 0; i < signatureSection.Length; i++) diff --git a/ValvePak/ValvePak.Test/WriteTest.cs b/ValvePak/ValvePak.Test/WriteTest.cs index cd9078c..f4d54d5 100644 --- a/ValvePak/ValvePak.Test/WriteTest.cs +++ b/ValvePak/ValvePak.Test/WriteTest.cs @@ -1,6 +1,8 @@ using System; +using System.Buffers.Binary; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using System.Threading.Tasks; using TUnit.Assertions.Enums; @@ -9,6 +11,8 @@ namespace ValvePak.Test { internal sealed class WriteTest { + private const int FractionSize = 1024 * 1024; + [Test] public async Task CreateNewPackage() { @@ -35,9 +39,7 @@ public async Task CreateNewPackage() output.Position = 0; // Verify - using var packageWritten = new Package(); - packageWritten.SetFileName("test.vpk"); - packageWritten.Read(output); + using var packageWritten = ReadBack(output); packageWritten.VerifyHashes(); var newEntry = packageWritten.FindEntry(newName); @@ -76,9 +78,7 @@ public async Task WriteManyFiles() output.Position = 0; // Verify - using var packageWritten = new Package(); - packageWritten.SetFileName("test.vpk"); - packageWritten.Read(output); + using var packageWritten = ReadBack(output); packageWritten.VerifyHashes(); await Assert.That(packageWritten.Entries!["txt"]).Count().IsEqualTo(1000); @@ -118,15 +118,7 @@ await Assert.That(package.RemoveFile(new PackageEntry public async Task SetsSpaces() { using var package = new Package(); - var file = package.AddFile("", []); - using (Assert.Multiple()) - { - await Assert.That(file.TypeName).IsEqualTo(" "); - await Assert.That(file.DirectoryName).IsEqualTo(" "); - await Assert.That(file.FileName).IsEqualTo(""); - await Assert.That(package.Entries!.ContainsKey(" ")).IsTrue(); - await Assert.That(package.Entries[" "][0]).IsEqualTo(file); - } + await Assert.That(() => package.AddFile("", [])).ThrowsExactly(); var file2 = package.AddFile("hello", []); using (Assert.Multiple()) @@ -134,6 +126,8 @@ public async Task SetsSpaces() await Assert.That(file2.TypeName).IsEqualTo(" "); await Assert.That(file2.DirectoryName).IsEqualTo(" "); await Assert.That(file2.FileName).IsEqualTo("hello"); + await Assert.That(package.Entries!.ContainsKey(" ")).IsTrue(); + await Assert.That(package.Entries[" "][0]).IsEqualTo(file2); } var file3 = package.AddFile("hello.txt", []); @@ -167,16 +161,198 @@ public async Task NormalizesSlashes() } [Test] - public async Task WriteThrowsWhenIsDirVPK() + [Arguments(StringComparison.Ordinal)] + [Arguments(StringComparison.OrdinalIgnoreCase)] + public async Task NormalizesPathsTheSameWhenAddingAndFinding(StringComparison comparison) { - var path = Path.Combine(AppContext.BaseDirectory, "Files", "test_dir.vpk"); + using var package = new Package(); + package.OptimizeEntriesForBinarySearch(comparison); + var entry = package.AddFile("\\Materials//./Models/./Crate.VMAT_C/", Encoding.UTF8.GetBytes("crate")); + var noExtension = package.AddFile("./readme.", []); + var trailingDots = package.AddFile("notes..", [1]); + var trailingDotAfterExtension = package.AddFile("archive.tar.", [2]); + + using (Assert.Multiple()) + { + await Assert.That(entry.DirectoryName).IsEqualTo("materials/models"); + await Assert.That(entry.FileName).IsEqualTo("crate"); + await Assert.That(entry.TypeName).IsEqualTo("vmat_c"); + await Assert.That(noExtension.DirectoryName).IsEqualTo(" "); + await Assert.That(noExtension.TypeName).IsEqualTo(" "); + await Assert.That(package.FindEntry("materials/models/crate.vmat_c")).IsEqualTo(entry); + await Assert.That(package.FindEntry("/materials\\models//crate.vmat_c")).IsEqualTo(entry); + await Assert.That(package.FindEntry("./materials/./models/crate.vmat_c/".AsSpan())).IsEqualTo(entry); + await Assert.That(package.FindEntry("readme")).IsEqualTo(noExtension); + await Assert.That(package.FindEntry("/readme.")).IsEqualTo(noExtension); + await Assert.That(trailingDots.FileName).IsEqualTo("notes"); + await Assert.That(trailingDots.TypeName).IsEqualTo(" "); + await Assert.That(trailingDotAfterExtension.FileName).IsEqualTo("archive"); + await Assert.That(trailingDotAfterExtension.TypeName).IsEqualTo("tar"); + + foreach (var added in new[] { entry, noExtension, trailingDots, trailingDotAfterExtension }) + { + await Assert.That(package.FindEntry(added.GetFullPath())).IsEqualTo(added); + } + } + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task FindsReadEntriesWhoseNamesAreNotNormalized(bool optimize) + { + (string Directory, string File, string Type)[] names = + [ + ("a//b", "file", "txt"), + ("a/./b", "file", "txt"), + ("./rel", "file", "txt"), + ("/leading", "file", "txt"), + ("trailing/", "file", "txt"), + ("back\\slash", "file", "txt"), + (" ", "has.dot", " "), + ("dir", "noext.", " "), + (" ", "archive", "tar.gz"), + ("dir", "archive.tar", "gz"), + ]; + + using var output = new MemoryStream(); + + using (var package = new Package()) + { + for (var i = 0; i < names.Length; i++) + { + var entry = package.AddFile($"file{i}.txt", [(byte)i]); + (entry.DirectoryName, entry.FileName, entry.TypeName) = names[i]; + } + + package.Write(output); + } + + output.Position = 0; + + using var readBack = new Package(); + + if (optimize) + { + readBack.OptimizeEntriesForBinarySearch(StringComparison.OrdinalIgnoreCase); + } + + readBack.SetFileName("test.vpk"); + readBack.Read(output); + + var entries = readBack.Entries!.Values.SelectMany(static entries => entries).ToList(); + await Assert.That(entries).Count().IsEqualTo(names.Length); + + foreach (var entry in entries) + { + await Assert.That(readBack.FindEntry(entry.GetFullPath())).IsSameReferenceAs(entry); + } + + using (Assert.Multiple()) + { + await Assert.That(readBack.FindEntry("dir/noext")).IsNull(); + await Assert.That(readBack.FindEntry("has")).IsNull(); + } + } + + [Test] + public async Task WritesDirectoryTreeSortedLikeValve() + { using var package = new Package(); - package.Read(path); + + string[] paths = + [ + "bc/b.txt", + "zeta/a.txt", + "b_c/a.txt", + "a/a.txt", + "root.txt", + "a/b.bin", + "a/a_b.txt", + "a/ab.txt", + "a/a.x_y", + ]; + + foreach (var path in paths) + { + package.AddFile(path, Encoding.UTF8.GetBytes(path)); + } + + // Names from read packages can have upper case letters, which sort as lower case + package.FindEntry("zeta/a.txt")!.DirectoryName = "Zeta"; + + using var output = new MemoryStream(); + package.Write(output); + output.Position = 0; + + using var readBack = ReadBack(output); + + string[] expected = + [ + "a/b.bin", + "root.txt", + "a/a.txt", + "a/a_b.txt", + "a/ab.txt", + "b_c/a.txt", + "bc/b.txt", + "Zeta/a.txt", + "a/a.x_y", + ]; + + await Assert.That(readBack.Entries!.Values.SelectMany(static entries => entries).Select(static entry => entry.GetFullPath()).ToArray()).IsEquivalentTo(expected, CollectionOrdering.Matching); + await Assert.That(() => readBack.VerifyHashes()).ThrowsNothing(); + } + + [Test] + [Arguments(2 * FractionSize, 2)] + [Arguments((2 * FractionSize) + 1, 3)] + [Arguments(100, 1)] + [Arguments(0, 0)] + public async Task WritesUnsignedSignatureSectionAndHashesEmbeddedData(int dataLength, int fractionCount) + { + var data = new byte[dataLength]; + + for (var i = 0; i < data.Length; i++) + { + data[i] = (byte)(i * 37); + } using var output = new MemoryStream(); - await Assert.That(() => package.Write(output)).ThrowsExactly() - .WithMessage("This package was opened from a _dir.vpk, writing back is currently unsupported.", StringComparison.Ordinal); + + using (var package = new Package()) + { + package.AddFile("data.bin", data); + package.AddFile("empty.bin", []); + package.Write(output); + } + + var bytes = output.ToArray(); + output.Position = 0; + + using var readBack = ReadBack(output); + + var signatureSection = bytes[(int)(readBack.HeaderSize + readBack.TreeSize + readBack.FileDataSectionSize + readBack.ArchiveMD5SectionSize + readBack.OtherMD5SectionSize)..]; + var emptyEntry = readBack.FindEntry("empty.bin"); + await Assert.That(emptyEntry).IsNotNull(); + + using (Assert.Multiple()) + { + await Assert.That(emptyEntry.ArchiveIndex).IsEqualTo((ushort)0x7FFF); + await Assert.That(emptyEntry.Offset).IsEqualTo(uint.MaxValue); + await Assert.That(emptyEntry.Length).IsZero(); + await Assert.That(readBack.ArchiveMD5SectionSize).IsEqualTo((uint)(fractionCount * ChunkHashFraction.SectionEntrySize)); + await Assert.That(signatureSection.Length).IsEqualTo(20); + await Assert.That(BinaryPrimitives.ReadInt32LittleEndian(signatureSection)).IsEqualTo(Package.MAGIC); + await Assert.That(BinaryPrimitives.ReadInt32LittleEndian(signatureSection.AsSpan(4))).IsEqualTo((int)ESignatureType.OnlyFileChecksum); + await Assert.That(signatureSection.AsSpan(8).ContainsAnyExcept((byte)0)).IsFalse(); + await Assert.That(readBack.IsSigned).IsFalse(); + await Assert.That(readBack.AccessPackFileHashes.TrueForAll(static fraction => fraction.ArchiveIndex == 0x7FFF && fraction.HashType == EHashType.Blake3 && fraction.Length > 0)).IsTrue(); + await Assert.That(() => readBack.VerifyHashes()).ThrowsNothing(); + await Assert.That(() => readBack.VerifyChunkHashes()).ThrowsNothing(); + await Assert.That(() => readBack.VerifyFileChecksums()).ThrowsNothing(); + } } [Test] @@ -194,9 +370,7 @@ public async Task WritesToNonSeekableStream(uint version) } using var output = new MemoryStream(nonSeekable.ToArray()); - using var readBack = new Package(); - readBack.SetFileName("test.vpk"); - readBack.Read(output); + using var readBack = ReadBack(output); if (version >= 2) { @@ -259,9 +433,7 @@ public async Task WriteAndVerifyRoundTrip() output.Position = 0; - using var readBack = new Package(); - readBack.SetFileName("test.vpk"); - readBack.Read(output); + using var readBack = ReadBack(output); using (Assert.Multiple()) { @@ -292,9 +464,7 @@ public async Task WriteVersion1Package() output.Position = 0; - using var readBack = new Package(); - readBack.SetFileName("test.vpk"); - readBack.Read(output); + using var readBack = ReadBack(output); using (Assert.Multiple()) { @@ -327,18 +497,14 @@ public async Task WritePreservesVersionOfReadPackage() output.Position = 0; - using var readBack = new Package(); - readBack.SetFileName("test.vpk"); - readBack.Read(output); + using var readBack = ReadBack(output); using var output2 = new MemoryStream(); readBack.Write(output2); output2.Position = 0; - using var readBack2 = new Package(); - readBack2.SetFileName("test.vpk"); - readBack2.Read(output2); + using var readBack2 = ReadBack(output2); await Assert.That(readBack2.Version).IsEqualTo(1u); @@ -428,9 +594,7 @@ public async Task WriteIntoLongerStream(int position, uint version) output.Position = position; - using var readBack = new Package(); - readBack.SetFileName("test.vpk"); - readBack.Read(output); + using var readBack = ReadBack(output); await Assert.That(readBack.Version).IsEqualTo(version); @@ -517,6 +681,35 @@ public async Task AddedFilesCanBeFoundAfterOptimizingForBinarySearch(StringCompa await Assert.That(package.FindEntry("addons/chess/missing.vdf")).IsNull(); } + [Test] + public async Task RenameFile() + { + using var package = new Package(); + package.OptimizeEntriesForBinarySearch(StringComparison.OrdinalIgnoreCase); + var entry = package.AddFile("kitten.jpg", Encoding.UTF8.GetBytes("meow")); + package.AddFile("other.txt", []); + + package.RenameFile(entry, "/Path\\To/Cat.PNG"); + + using (Assert.Multiple()) + { + await Assert.That(entry.GetFullPath()).IsEqualTo("path/to/cat.png"); + await Assert.That(package.FindEntry("kitten.jpg")).IsNull(); + await Assert.That(package.FindEntry("path/to/cat.png")).IsEqualTo(entry); + await Assert.That(package.Entries!.ContainsKey("jpg")).IsFalse(); + await Assert.That(() => package.RenameFile(entry, "other.txt")).ThrowsExactly(); + } + } + + private static Package ReadBack(Stream stream) + { + var package = new Package(); + package.SetFileName("test.vpk"); + package.Read(stream); + + return package; + } + /// /// Write only stream that can not seek, collecting the written bytes. /// diff --git a/ValvePak/ValvePak/Archive.cs b/ValvePak/ValvePak/Archive.cs index 861447a..2500c92 100644 --- a/ValvePak/ValvePak/Archive.cs +++ b/ValvePak/ValvePak/Archive.cs @@ -25,6 +25,11 @@ protected Archive(long origin, long dataOffset) /// protected long Origin { get; } + /// + /// Gets the size in bytes of the archive. + /// + public abstract long Length { get; } + /// /// Reads data at the given offset until the output is full or the end of the archive is reached. /// This method is safe to call from multiple threads. diff --git a/ValvePak/ValvePak/FileArchive.cs b/ValvePak/ValvePak/FileArchive.cs index 5a7668f..9196f4c 100644 --- a/ValvePak/ValvePak/FileArchive.cs +++ b/ValvePak/ValvePak/FileArchive.cs @@ -25,6 +25,8 @@ public FileArchive(SafeFileHandle handle, bool ownsHandle, long origin, long dat OwnsHandle = ownsHandle; } + public override long Length => RandomAccess.GetLength(Handle) - Origin; + public override int Read(long offset, Span output) { var totalRead = 0; diff --git a/ValvePak/ValvePak/Package.Read.cs b/ValvePak/ValvePak/Package.Read.cs index d77c38c..c304f44 100644 --- a/ValvePak/ValvePak/Package.Read.cs +++ b/ValvePak/ValvePak/Package.Read.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.IO; using System.IO.Hashing; +using System.Linq; using System.Runtime.CompilerServices; using System.Text; @@ -23,6 +24,8 @@ public partial class Package : IDisposable public void Read(string filename) { ObjectDisposedException.ThrowIf(IsDisposed, this); + ThrowIfAlreadyRead(); + ThrowIfFilesAdded(); SetFileName(filename); @@ -44,6 +47,7 @@ public void Read(Stream input) ArgumentNullException.ThrowIfNull(input); ObjectDisposedException.ThrowIf(IsDisposed, this); ThrowIfAlreadyRead(); + ThrowIfFilesAdded(); if (FileName == null) { @@ -51,7 +55,51 @@ public void Read(Stream input) } StreamOffset = input.Position; + + ReadHeader(input); + ReadEntries(); + ReadSections(); + + IsReadPackage = true; + HighestChunkIndex = GetChunkIndicesWithData() + .Concat(AccessPackFileHashes.Select(static fraction => fraction.ArchiveIndex)) + .Where(static archiveIndex => archiveIndex < DirectoryArchiveIndex) + .Max(static archiveIndex => (int?)archiveIndex) ?? -1; + + if (HighestChunkIndex >= 0) + { + ChunkBaseName = Path.GetFullPath(FileName); + } + } + + /// + /// Opens the directory file at the given path instead of the one read before, keeping the current . + /// + private void ReopenDirectoryFile(string path) + { + CloseDirectoryFile(); + ReadFilePath = path; + + var input = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + + try + { + StreamOffset = 0; + ReadHeader(input); + input.Position = HeaderSize + TreeSize; + ReadSections(); + } + catch + { + CloseDirectoryFile(); + throw; + } + } + + private void ReadHeader(Stream input) + { Reader = new BinaryReader(input); + ReadFilePath = (input as FileStream)?.Name; if (Reader.ReadUInt32() != MAGIC) { @@ -63,7 +111,10 @@ public void Read(Stream input) if (version == 1) { - // Nothing else + FileDataSectionSize = 0; + ArchiveMD5SectionSize = 0; + OtherMD5SectionSize = 0; + SignatureSectionSize = 0; } else if (version == 2) { @@ -90,24 +141,24 @@ public void Read(Stream input) // Taking the handle flushes the stream's buffer, so it is done here before any other thread can read from the stream if (input is FileStream fileStream) { - Archives[0x7FFF] = new FileArchive(fileStream.SafeFileHandle, ownsHandle: false, StreamOffset, embeddedDataOffset); + Archives[DirectoryArchiveIndex] = new FileArchive(fileStream.SafeFileHandle, ownsHandle: false, StreamOffset, embeddedDataOffset); } else { - Archives[0x7FFF] = new StreamArchive(input, StreamOffset, embeddedDataOffset); + Archives[DirectoryArchiveIndex] = new StreamArchive(input, StreamOffset, embeddedDataOffset); } + } - ReadEntries(); + private void ReadSections() + { + Debug.Assert(Reader != null); - if (Version == 2) - { - // Skip over file data, if any - input.Position += FileDataSectionSize; + // Skip over file data, if any + Reader.BaseStream.Position += FileDataSectionSize; - ReadArchiveMD5Section(); - ReadOtherMD5Section(); - ReadSignatureSection(); - } + ReadArchiveMD5Section(); + ReadOtherMD5Section(); + ReadSignatureSection(); } /// @@ -121,6 +172,8 @@ public void Read(Stream input) public void ReadEntry(PackageEntry entry, out byte[] output, bool validateCrc = true) { ArgumentNullException.ThrowIfNull(entry); + ObjectDisposedException.ThrowIf(IsDisposed, this); + ThrowIfEntryPastEndOfArchive(entry); output = new byte[entry.TotalLength]; @@ -141,7 +194,7 @@ public void ReadEntry(PackageEntry entry, byte[] output, bool validateCrc = true ArgumentNullException.ThrowIfNull(output); ObjectDisposedException.ThrowIf(IsDisposed, this); - var totalLength = (int)entry.TotalLength; + var totalLength = (long)entry.Length + entry.SmallData.Length; if (output.Length < totalLength) { @@ -153,7 +206,11 @@ public void ReadEntry(PackageEntry entry, byte[] output, bool validateCrc = true entry.SmallData.CopyTo(output, 0); } - if (entry.Length > 0) + if (entry.PendingData != null) + { + entry.PendingData.CopyTo(output, entry.SmallData.Length); + } + else if (entry.Length > 0) { var length = (int)entry.Length; var bytesRead = ReadArchiveData(entry.ArchiveIndex, entry.Offset, output.AsSpan(entry.SmallData.Length, length)); @@ -169,7 +226,7 @@ public void ReadEntry(PackageEntry entry, byte[] output, bool validateCrc = true return; } - var actualChecksum = Crc32.HashToUInt32(output.AsSpan(0, totalLength)); + var actualChecksum = Crc32.HashToUInt32(output.AsSpan(0, (int)totalLength)); if (entry.CRC32 != actualChecksum) { @@ -177,6 +234,22 @@ public void ReadEntry(PackageEntry entry, byte[] output, bool validateCrc = true } } + private void ThrowIfEntryPastEndOfArchive(PackageEntry entry) + { + if (entry.Length == 0 || entry.PendingData != null) + { + return; + } + + var archive = GetArchive(entry.ArchiveIndex); + var available = archive.Length - archive.DataOffset - entry.Offset; + + if (entry.Length > available) + { + throw new EndOfStreamException($"Archive {entry.ArchiveIndex} ends {Math.Max(available, 0)} bytes after the offset of '{entry.GetFullPath()}', which is {entry.Length} bytes long."); + } + } + /// /// Reads exactly bytes of the directory tree and parses the entries from them. /// @@ -288,13 +361,22 @@ private Dictionary> ParseEntries(ReadOnlySpan t } } + // Types that differ only in case share a list when the dictionary is case insensitive + if (typeEntries.TryGetValue(typeName, out var existingEntries)) + { + existingEntries.AddRange(entries); + entries = existingEntries; + } + else + { + typeEntries.Add(typeName, entries); + } + if (Comparer != null) { // Sorting at the end is faster than doing BinarySearch+Insert entries.Sort(Comparer); } - - typeEntries.Add(typeName, entries); } return typeEntries; @@ -317,6 +399,13 @@ private void ReadArchiveMD5Section() throw new InvalidDataException($"Archive MD5 section size ({ArchiveMD5SectionSize}) is not a multiple of {ChunkHashFraction.SectionEntrySize}."); } + var input = Reader.BaseStream; + + if (ArchiveMD5SectionSize > input.Length - input.Position) + { + throw new InvalidDataException($"Archive MD5 section size ({ArchiveMD5SectionSize}) is larger than the rest of the file ({input.Length - input.Position})."); + } + var entries = (int)(ArchiveMD5SectionSize / ChunkHashFraction.SectionEntrySize); AccessPackFileHashes = new List(entries); @@ -334,7 +423,7 @@ private void ReadArchiveMD5Section() if (hashFraction.ArchiveIndex == 0 && hashFraction.HashType == (EHashType)0x8000) { - hashFraction.ArchiveIndex = 0x7FFF; + hashFraction.ArchiveIndex = DirectoryArchiveIndex; hashFraction.HashType = EHashType.MD5; } @@ -344,17 +433,22 @@ private void ReadArchiveMD5Section() private void ReadOtherMD5Section() { - if (OtherMD5SectionSize != 48) + Debug.Assert(Reader != null); + + if (OtherMD5SectionSize < 48) { + TreeChecksum = null; + ArchiveMD5EntriesChecksum = null; + WholeFileChecksum = null; + Reader.BaseStream.Position += OtherMD5SectionSize; return; } - Debug.Assert(Reader != null); - TreeChecksum = Reader.ReadBytes(16); ArchiveMD5EntriesChecksum = Reader.ReadBytes(16); FileSizeBeforeWholeFileHash = (uint)(Reader.BaseStream.Position - StreamOffset); WholeFileChecksum = Reader.ReadBytes(16); + Reader.BaseStream.Position += OtherMD5SectionSize - 48; } private void ReadSignatureSection() @@ -362,6 +456,9 @@ private void ReadSignatureSection() Debug.Assert(Reader != null); FileSizeBeforeSignature = (uint)(Reader.BaseStream.Position - StreamOffset); + SignatureType = ESignatureType.Unknown; + PublicKey = null; + Signature = null; if (SignatureSectionSize == 0) { @@ -413,7 +510,7 @@ private byte[] ReadSignatureBytes(int size, string name) /// /// Returns the archive for the given index, opening its chunk file on first use. - /// The embedded archive (0x7FFF) is added by . + /// The embedded archive () is added when the directory file is read. /// private Archive GetArchive(ushort archiveIndex) { @@ -431,12 +528,12 @@ private Archive GetArchive(ushort archiveIndex) return archive; } - if (archiveIndex == 0x7FFF) + if (archiveIndex == DirectoryArchiveIndex) { - throw new InvalidOperationException("The package has not been read."); + ThrowIfDirectoryFileClosed(); } - var fileName = GetArchiveIndexFullFilePath(FileName, archiveIndex); + var fileName = GetArchiveIndexFullFilePath(GetChunkBaseName(), archiveIndex); var handle = File.OpenHandle(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); archive = new FileArchive(handle, ownsHandle: true, origin: 0, dataOffset: 0); @@ -458,6 +555,34 @@ private int ReadArchiveData(ushort archiveIndex, long offset, Span output) return archive.Read(archive.DataOffset + offset, output); } + private void ReadArchiveDataExactly(ushort archiveIndex, long offset, Span output) + { + var bytesRead = ReadArchiveData(archiveIndex, offset, output); + + if (bytesRead != output.Length) + { + throw new EndOfStreamException($"Reached the end of archive {archiveIndex} after reading {bytesRead} of {output.Length} bytes at offset {offset}."); + } + } + + private void CloseArchive(ushort archiveIndex) + { + lock (ArchivesLock) + { + if (Archives.TryRemove(archiveIndex, out var archive)) + { + archive.Dispose(); + } + } + } + + private void CloseDirectoryFile() + { + CloseArchive(DirectoryArchiveIndex); + Reader?.Dispose(); + Reader = null; + } + /// /// Returns a stream that reads the entry directly from a memory mapping of its archive when possible, /// otherwise reads entry into a byte array and returns . @@ -467,9 +592,9 @@ private int ReadArchiveData(ushort archiveIndex, long offset, Span output) /// When trying to map a file in a non-split packages (), /// this will only memory map if the package was read by providing or using . /// - /// Each archive is mapped once and stays mapped until the package is disposed. + /// Each archive is mapped once. /// Returned streams can only be read while the package is open, they throw - /// once the package is disposed. + /// once the package is disposed, once replaces the file they read from, or once it repacks, which closes all files the package read from. /// This method is safe to call from multiple threads. /// /// Package entry. @@ -480,7 +605,7 @@ public Stream GetMemoryMappedStreamIfPossible(PackageEntry entry) ArgumentNullException.ThrowIfNull(entry); ObjectDisposedException.ThrowIf(IsDisposed, this); - if (entry.Length > 4096 && entry.SmallData.Length == 0) + if (entry.Length > 4096 && entry.SmallData.Length == 0 && entry.PendingData == null) { var archive = GetArchive(entry.ArchiveIndex); var stream = archive.OpenMappedStream(archive.DataOffset + entry.Offset, entry.Length); diff --git a/ValvePak/ValvePak/Package.Save.cs b/ValvePak/ValvePak/Package.Save.cs index 1f535bb..6af2734 100644 --- a/ValvePak/ValvePak/Package.Save.cs +++ b/ValvePak/ValvePak/Package.Save.cs @@ -1,9 +1,12 @@ using System; using System.Buffers; +using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.IO.Hashing; +using System.Linq; using System.Security.Cryptography; using System.Text; @@ -12,9 +15,10 @@ namespace ValvePak public partial class Package { /// - /// Gets or sets the maximum size in bytes of chunk files created when adding files with multi chunk enabled, - /// see . A new chunk file is started once the current one reaches this size. - /// Defaults to 200 MiB. + /// Gets or sets the size in bytes after which files added with multi chunk enabled (see ) + /// or repacked (see ) go into a new chunk file. + /// A new chunk file is started once the current one is larger than this size, so a chunk file can end past it by up to the size of its last file. + /// Defaults to 100 MiB. /// public int WriteChunkSize { @@ -25,12 +29,14 @@ public int WriteChunkSize field = value; } - } = 200 * 1024 * 1024; + } = 100 * 1024 * 1024; /// - /// Size in bytes of the buffer that file data is copied through when it is added to a chunk file. + /// Size in bytes of the buffer that file data is copied through. /// - private const int AddFileBufferSize = 1024 * 1024; + private const int CopyBufferSize = 1024 * 1024; + + private const int UnsignedSignatureSectionSize = 20; /// /// The chunk file that files added with multi chunk enabled are currently written to. @@ -38,9 +44,18 @@ public int WriteChunkSize private ChunkFileWriter? CurrentChunkFile; /// - /// Hashes of the chunk files before . + /// Hashes of chunk files that this package wrote or hashed again, which take precedence over . + /// + private readonly Dictionary> UpdatedChunkHashFractions = []; + + private string? ChunkBaseName; + + /// + /// Highest index of the chunk files that the package refers to or created, or -1 if there are none. /// - private readonly List FinishedChunkHashFractions = []; + private int HighestChunkIndex = -1; + + private bool IsReadPackage; /// /// Remove file from current package. @@ -50,6 +65,7 @@ public int WriteChunkSize public bool RemoveFile(PackageEntry entry) { ArgumentNullException.ThrowIfNull(entry); + ObjectDisposedException.ThrowIf(IsDisposed, this); if (Entries == null) { @@ -71,56 +87,120 @@ public bool RemoveFile(PackageEntry entry) return removed; } + /// + /// Moves an entry in current package to a new path. Only the directory tree changes, the file data is not touched. + /// The path is normalized and lowercased like Valve's packer does, see . + /// + /// The package entry to rename. + /// Full new file path for this entry. + /// The entry is not in this package, another entry already has the new path ignoring case, or the new path has an empty file name or contains a null character. + public void RenameFile(PackageEntry entry, string newFilePath) + { + ArgumentNullException.ThrowIfNull(entry); + ArgumentNullException.ThrowIfNull(newFilePath); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var (directoryName, fileName, typeName) = ParseEntryPath(newFilePath, nameof(newFilePath)); + var existingEntry = FindEntryIgnoringCase(directoryName, fileName, typeName); + + if (existingEntry != null && existingEntry != entry) + { + throw new ArgumentException($"An entry already exists at '{existingEntry.GetFullPath()}'.", nameof(newFilePath)); + } + + if (!RemoveFile(entry)) + { + throw new ArgumentException("The entry is not in this package.", nameof(entry)); + } + + (entry.DirectoryName, entry.FileName, entry.TypeName) = (directoryName, fileName, typeName); + + InsertEntry(entry); + } + + private PackageEntry? FindEntryIgnoringCase(string directoryName, string fileName, string typeName) + { + foreach (var (entryTypeName, typeEntries) in Entries ?? []) + { + if (!entryTypeName.Equals(typeName, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var match = typeEntries.Find(entry => entry.FileName.Equals(fileName, StringComparison.OrdinalIgnoreCase) + && entry.DirectoryName.Equals(directoryName, StringComparison.OrdinalIgnoreCase)); + + if (match != null) + { + return match; + } + } + + return null; + } + /// /// Add file to current package. Be careful to not add duplicate entries, because this does not check for duplicates. + /// The path is normalized and lowercased like Valve's packer does, see . /// /// Full file path for this entry. /// File data for this entry. /// If true, the data is written right away into a numbered chunk file (such as "example_001.vpk") instead of being kept in memory for the directory file. /// must be called first with the path of the directory file, the chunk files are created next to it. - /// A new chunk file is started whenever the current one reaches . + /// New chunk files are numbered after the highest chunk file of the package, and a read package also skips numbers that already have a file. + /// A new chunk file is started whenever the current one is larger than . /// Chunk files stay on disk even if the directory file is never written. /// The added entry. + /// The path has an empty file name, or contains a null character. public PackageEntry AddFile(string filePath, byte[] fileData, bool multiChunk = false) { ArgumentNullException.ThrowIfNull(filePath); ArgumentNullException.ThrowIfNull(fileData); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var entry = CreateEntry(filePath, nameof(filePath)); // Files with no data are only ever written into the directory file if (!multiChunk || fileData.Length == 0) { - return AddFileToDirectoryFile(filePath, fileData); + return AddFileToDirectoryFile(entry, fileData); } var chunkFile = GetChunkFileForNextFile(); var offset = chunkFile.Length; chunkFile.Write(fileData); - return AddChunkFileEntry(filePath, chunkFile, offset, Crc32.HashToUInt32(fileData)); + return AddChunkFileEntry(entry, chunkFile, offset, Crc32.HashToUInt32(fileData)); } /// /// Add file to current package, reading its data from the current position of the stream to its end. /// Be careful to not add duplicate entries, because this does not check for duplicates. + /// The path is normalized and lowercased like Valve's packer does, see . /// /// Full file path for this entry. /// Stream to read the file data from. It is not disposed. /// If true, the data is copied right away into a numbered chunk file (such as "example_001.vpk") instead of being kept in memory for the directory file. /// must be called first with the path of the directory file, the chunk files are created next to it. - /// A new chunk file is started whenever the current one reaches . + /// New chunk files are numbered after the highest chunk file of the package, and a read package also skips numbers that already have a file. + /// A new chunk file is started whenever the current one is larger than . /// Chunk files stay on disk even if the directory file is never written. /// The added entry. + /// The path has an empty file name, or contains a null character. public PackageEntry AddFile(string filePath, Stream fileData, bool multiChunk = false) { ArgumentNullException.ThrowIfNull(filePath); ArgumentNullException.ThrowIfNull(fileData); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var entry = CreateEntry(filePath, nameof(filePath)); if (!multiChunk) { - return AddFileToDirectoryFile(filePath, ReadToEnd(fileData)); + return AddFileToDirectoryFile(entry, ReadToEnd(fileData)); } - var buffer = ArrayPool.Shared.Rent(AddFileBufferSize); + var buffer = ArrayPool.Shared.Rent(CopyBufferSize); try { @@ -129,7 +209,7 @@ public PackageEntry AddFile(string filePath, Stream fileData, bool multiChunk = if (bytesRead == 0) { // Files with no data are only ever written into the directory file - return AddFileToDirectoryFile(filePath, []); + return AddFileToDirectoryFile(entry, []); } var chunkFile = GetChunkFileForNextFile(); @@ -146,7 +226,7 @@ public PackageEntry AddFile(string filePath, Stream fileData, bool multiChunk = } while (bytesRead > 0); - return AddChunkFileEntry(filePath, chunkFile, offset, crc.GetCurrentHashAsUInt32()); + return AddChunkFileEntry(entry, chunkFile, offset, crc.GetCurrentHashAsUInt32()); } finally { @@ -176,12 +256,12 @@ private static byte[] ReadToEnd(Stream stream) /// /// Adds an entry whose data is kept in memory until it is written into the directory file. /// - private PackageEntry AddFileToDirectoryFile(string filePath, byte[] fileData) + private PackageEntry AddFileToDirectoryFile(PackageEntry entry, byte[] fileData) { - var entry = CreateEntry(filePath); - entry.SmallData = fileData; + entry.PendingData = fileData; + entry.Length = (uint)fileData.Length; entry.CRC32 = Crc32.HashToUInt32(fileData); - entry.ArchiveIndex = 0x7FFF; + entry.ArchiveIndex = DirectoryArchiveIndex; InsertEntry(entry); @@ -191,12 +271,11 @@ private PackageEntry AddFileToDirectoryFile(string filePath, byte[] fileData) /// /// Adds an entry for the data that was written to the chunk file since it was at the given offset. /// - private PackageEntry AddChunkFileEntry(string filePath, ChunkFileWriter chunkFile, uint offset, uint crc) + private PackageEntry AddChunkFileEntry(PackageEntry entry, ChunkFileWriter chunkFile, uint offset, uint crc) { // Hand the data to the OS so the entry can be read back before the package is written chunkFile.Flush(); - var entry = CreateEntry(filePath); entry.CRC32 = crc; entry.ArchiveIndex = chunkFile.ArchiveIndex; entry.Offset = offset; @@ -212,79 +291,125 @@ private PackageEntry AddChunkFileEntry(string filePath, ChunkFileWriter chunkFil /// private ChunkFileWriter GetChunkFileForNextFile() { - if (FileName == null) + if (HasRoomForNextFile(CurrentChunkFile)) { - throw new InvalidOperationException("Call SetFileName() with the path of the directory file before adding files with multi chunk enabled, chunk files are created next to it."); + return CurrentChunkFile; } - var archiveIndex = 0; + var baseName = GetChunkBaseName(); + var archiveIndex = HighestChunkIndex + 1; - if (CurrentChunkFile != null) + while (IsReadPackage && archiveIndex < DirectoryArchiveIndex && File.Exists(GetArchiveIndexFullFilePath(baseName, (ushort)archiveIndex))) { - if (CurrentChunkFile.Length < (uint)WriteChunkSize) - { - return CurrentChunkFile; - } - - archiveIndex = CurrentChunkFile.ArchiveIndex + 1; - - if (archiveIndex >= 0x7FFF) - { - throw new InvalidOperationException("Reached the maximum amount of chunk files (32767)."); - } + archiveIndex++; } - var chunkFile = new ChunkFileWriter((ushort)archiveIndex, GetArchiveIndexFullFilePath(FileName, (ushort)archiveIndex)); + ThrowIfChunkLimitReached(archiveIndex); - if (CurrentChunkFile != null) - { - FinishedChunkHashFractions.AddRange(CurrentChunkFile.GetHashFractions()); - CurrentChunkFile.Dispose(); - } + var chunkFile = new ChunkFileWriter((ushort)archiveIndex, GetArchiveIndexFullFilePath(baseName, (ushort)archiveIndex), IsReadPackage ? FileMode.CreateNew : FileMode.Create); + FinishCurrentChunkFile(); CurrentChunkFile = chunkFile; + HighestChunkIndex = archiveIndex; + ChunkBaseName = baseName; return chunkFile; } /// - /// Creates an entry with the directory, file and type names split from the given path. + /// Returns whether the chunk file is open and not larger than , so that the next file still goes into it. /// - private static PackageEntry CreateEntry(string filePath) + private bool HasRoomForNextFile([NotNullWhen(true)] ChunkFileWriter? chunkFile) { - filePath = filePath.Replace(WindowsDirectorySeparator, DirectorySeparatorChar); - - var lastSeparator = filePath.LastIndexOf(DirectorySeparatorChar); - var directory = lastSeparator > -1 ? filePath[..lastSeparator] : string.Empty; - var fileName = filePath[(lastSeparator + 1)..]; - - var dot = fileName.LastIndexOf('.'); - string extension; + return chunkFile != null && chunkFile.Length <= (uint)WriteChunkSize; + } - if (dot > -1) + private static void ThrowIfChunkLimitReached(int archiveIndex) + { + if (archiveIndex >= DirectoryArchiveIndex) { - extension = fileName[(dot + 1)..]; - fileName = fileName[..dot]; + throw new InvalidOperationException("Reached the maximum amount of chunk files (32767)."); } - else + } + + private void FinishCurrentChunkFile() + { + if (CurrentChunkFile == null) { - // Valve uses a space for missing extensions - extension = Space; + return; } - if (directory.Length == 0) + UpdatedChunkHashFractions[CurrentChunkFile.ArchiveIndex] = CurrentChunkFile.GetHashFractions(); + CurrentChunkFile.Dispose(); + CurrentChunkFile = null; + } + + private string GetChunkBaseName() + { + if (FileName == null) { - directory = Space; + throw new InvalidOperationException("Call SetFileName() with the path of the directory file before adding files with multi chunk enabled, chunk files are created next to it."); } + return ChunkBaseName ?? Path.GetFullPath(FileName); + } + + /// + /// Creates an entry with the directory, file and type names split from the normalized path. + /// + private static PackageEntry CreateEntry(string filePath, string paramName) + { + var (directoryName, fileName, typeName) = ParseEntryPath(filePath, paramName); + return new PackageEntry { + DirectoryName = directoryName, FileName = fileName, - DirectoryName = directory, - TypeName = extension, + TypeName = typeName, }; } + /// + /// Splits the given path into lowercased directory, file and type names. + /// + private static (string DirectoryName, string FileName, string TypeName) ParseEntryPath(string filePath, string paramName) + { + if (filePath.Contains('\0', StringComparison.Ordinal)) + { + throw new ArgumentException($"'{filePath}' contains a null character.", paramName); + } + + SplitPath(NormalizePath(filePath, out _), out var directory, out var fileName, out var extension); + + if (fileName.IsEmpty) + { + throw new ArgumentException($"'{filePath}' has an empty file name.", paramName); + } + + return (ToLowerAscii(directory), ToLowerAscii(fileName), ToLowerAscii(extension)); + } + + private static string ToLowerAscii(ReadOnlySpan value) + { + if (!value.ContainsAnyInRange('A', 'Z')) + { + return value.ToString(); + } + + return string.Create(value.Length, value, static (span, source) => + { + for (var i = 0; i < source.Length; i++) + { + span[i] = ToLowerAscii(source[i]); + } + }); + } + + private static char ToLowerAscii(char c) + { + return c is >= 'A' and <= 'Z' ? (char)(c | 0x20) : c; + } + /// /// Adds the entry to , at its sorted position if the package is optimized for binary search. /// @@ -309,45 +434,375 @@ private void InsertEntry(PackageEntry entry) } } + private IEnumerable GetAllEntries() + { + return Entries?.Values.SelectMany(static entries => entries) ?? []; + } + + /// + /// Returns the archive indices of the entries with data in chunk files. + /// + private IEnumerable GetChunkIndicesWithData() + { + return GetAllEntries().Where(IsInChunkFile).Select(static entry => entry.ArchiveIndex); + } + + private static bool IsInChunkFile(PackageEntry entry) + { + return entry.ArchiveIndex != DirectoryArchiveIndex && entry.Length > 0; + } + + private static bool IsInDirectoryFile(PackageEntry entry) + { + return entry.ArchiveIndex == DirectoryArchiveIndex && entry.Length > 0; + } + /// /// Opens and writes the given filename. /// - /// If any files were added with multi chunk enabled (see ), - /// their data is already in the chunk files, and this only writes the directory file. - /// It must be written to the path given to , which should end with "_dir.vpk". + /// If the package has chunk files, from being read or from files added with multi chunk enabled (see ), + /// this only writes the directory file, which refers to the data where it is in the chunk files. + /// Unless is enabled, it must be written to the path the chunk files are named after, which should end with "_dir.vpk". + /// Hashes of chunk files are kept if they cover the whole file, computed again otherwise, and left out for chunk files that had none. + /// + /// Writing over the file the package was read from, or repacking, writes temporary files that replace the existing files once all were written, + /// and the package reads from the written files afterwards. This must not run concurrently with reads from this package, + /// and streams from can no longer be read once they read from a replaced file, or from any file after repacking. + /// Any other file is written in place. /// /// The VPK version that is written is controlled by . /// /// The file to open and write. - public void Write(string filename) + /// If true, the data in chunk files is copied into new chunk files named after in directory tree order, + /// which drops the data of removed and replaced files. Either all existing files are replaced or none are. + /// If new chunk files were written or the package had chunk files with that name, existing chunk files numbered after the new ones are then deleted where possible + /// until a number has no file, and in the latter case also up to the highest number the package had. + public void Write(string filename, bool repack = false) { ArgumentNullException.ThrowIfNull(filename); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var fullPath = Path.GetFullPath(filename); + var replacesReadFile = ReadFilePath != null && PathsEqual(fullPath, ReadFilePath); + + if (replacesReadFile && StreamOffset != 0) + { + throw new InvalidOperationException("This package was read from inside a larger file, which can not be written over."); + } + + var baseName = StripDirVpkSuffixes(fullPath, out var isDirVpk); + var writesOverChunkFiles = HighestChunkIndex >= 0 && PathsEqual(baseName, GetChunkBaseName()); + + // Readers find chunk files by appending "_###.vpk" to the directory file name without the "_dir.vpk" suffix + if (!repack && HighestChunkIndex >= 0 && !writesOverChunkFiles) + { + throw new InvalidOperationException($"This package has chunk files named after '{GetChunkBaseName()}_dir.vpk', the directory file must be written to that path so that they can be found, or be written with repack enabled."); + } + + var entries = GetEntriesInTreeOrder(); + + if (!replacesReadFile && !repack) + { + var chunkHashFractions = GetChunkHashFractions(); + + using var output = new FileStream(fullPath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None); + WriteDirectoryFile(output, entries, chunkHashFractions, chunkLocations: null); + output.SetLength(output.Position); + + return; + } + + var token = Path.GetRandomFileName(); + var chunkReplacements = new List(); + var directoryReplacement = new FileReplacement(GetTemporaryPath(fullPath, token), fullPath); - if (CurrentChunkFile != null) + try { - Debug.Assert(FileName != null); + Dictionary? chunkLocations = null; + List? chunkHashFractions; + + if (repack) + { + (chunkLocations, chunkHashFractions) = WriteRepackedChunkFiles(entries, baseName, token, chunkReplacements); + } + else + { + chunkHashFractions = GetChunkHashFractions(); + } + + using (var stream = new FileStream(directoryReplacement.TemporaryPath, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + WriteDirectoryFile(stream, entries, chunkHashFractions, chunkLocations); + } + + var readsWrittenFile = replacesReadFile || (repack && Reader != null); + + if (repack) + { + FinishCurrentChunkFile(); + + foreach (var archiveIndex in Archives.Keys.Where(static archiveIndex => archiveIndex != DirectoryArchiveIndex)) + { + CloseArchive(archiveIndex); + } + } + + var backupPaths = ReplaceFilesOrReopen(chunkReplacements, directoryReplacement, token, replacesReadFile); - // Readers find chunk files by appending "_###.vpk" to the directory file name without the "_dir.vpk" suffix - var expectedBaseName = Path.GetFullPath(FileName); - var writtenBaseName = Path.GetFullPath(StripDirVpkSuffixes(filename, out _)); + if (chunkLocations != null) + { + var staleHighestIndex = writesOverChunkFiles ? HighestChunkIndex : -1; + + ApplyRepackedLayout(chunkLocations, chunkHashFractions, baseName, isDirVpk, chunkReplacements.Count); + backupPaths.ForEach(TryDeleteFile); - if (writtenBaseName != expectedBaseName) + if (chunkReplacements.Count > 0 || writesOverChunkFiles) + { + DeleteStaleChunkFiles(baseName, chunkReplacements.Count, staleHighestIndex); + } + } + + if (readsWrittenFile) + { + ReopenWrittenDirectoryFile(fullPath, entries); + } + } + finally + { + foreach (var (temporaryPath, _) in chunkReplacements) { - throw new InvalidOperationException($"Chunk files were created for '{FileName}_dir.vpk', the directory file must be written to that path so that they can be found."); + TryDeleteFile(temporaryPath); } + + TryDeleteFile(directoryReplacement.TemporaryPath); + } + } + + private static string GetTemporaryPath(string path, string token) + { + return $"{path}.{token}.tmp"; + } + + /// + /// Points the copied entries at the repacked chunk files and makes the package refer to the chunk files named after . + /// + private void ApplyRepackedLayout(Dictionary chunkLocations, List? chunkHashFractions, string baseName, bool isDirVpk, int chunkFileCount) + { + foreach (var (entry, (archiveIndex, offset)) in chunkLocations) + { + entry.ArchiveIndex = archiveIndex; + entry.Offset = offset; + } + + FileName = baseName; + ChunkBaseName = chunkFileCount > 0 ? baseName : null; + IsDirVPK = isDirVpk; + HighestChunkIndex = chunkFileCount - 1; + UpdatedChunkHashFractions.Clear(); + + foreach (var fractions in (chunkHashFractions ?? []).GroupBy(static fraction => fraction.ArchiveIndex)) + { + UpdatedChunkHashFractions[fractions.Key] = [.. fractions]; } + } + + /// + /// Deletes chunk files from on until a number has no file, and up to regardless. + /// + private static void DeleteStaleChunkFiles(string baseName, int firstIndex, int staleHighestIndex) + { + for (var i = firstIndex; i < DirectoryArchiveIndex; i++) + { + var path = GetArchiveIndexFullFilePath(baseName, (ushort)i); - using var fs = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None); + if (i > staleHighestIndex && !File.Exists(path)) + { + break; + } - Write(fs); + TryDeleteFile(path); + } } /// - /// Writes to the given . + /// Replaces the files, closing the directory file first when it is the one the package reads from, and opening it again if replacing fails. + /// + /// The paths that the replaced chunk files were moved aside to. + private List ReplaceFilesOrReopen(List chunkReplacements, FileReplacement directoryReplacement, string token, bool replacesReadFile) + { + if (!replacesReadFile) + { + return ReplaceFiles(chunkReplacements, directoryReplacement, token); + } + + CloseDirectoryFile(); + + try + { + return ReplaceFiles(chunkReplacements, directoryReplacement, token); + } + catch (Exception exception) + { + try + { + ReopenDirectoryFile(directoryReplacement.Path); + } + catch (Exception reopenException) + { + throw new AggregateException(exception, reopenException); + } + + throw; + } + } + + /// + /// Copies the data of all entries in chunk files into new chunk files under temporary names, adding each of them to . + /// + /// Where the data of each copied entry was written to, and the hashes of the new chunk files or null if there are none. + private (Dictionary Locations, List? HashFractions) WriteRepackedChunkFiles( + List entries, string baseName, string token, List replacements) + { + var locations = new Dictionary(); + var hashFractions = new List(); + ChunkFileWriter? chunkFile = null; + + try + { + foreach (var entry in entries) + { + if (!IsInChunkFile(entry)) + { + continue; + } + + if (!HasRoomForNextFile(chunkFile)) + { + var archiveIndex = chunkFile == null ? 0 : chunkFile.ArchiveIndex + 1; + ThrowIfChunkLimitReached(archiveIndex); + + var path = GetArchiveIndexFullFilePath(baseName, (ushort)archiveIndex); + var temporaryPath = GetTemporaryPath(path, token); + var nextChunkFile = new ChunkFileWriter((ushort)archiveIndex, temporaryPath); + replacements.Add(new FileReplacement(temporaryPath, path)); + + if (chunkFile != null) + { + hashFractions.AddRange(chunkFile.GetHashFractions()); + chunkFile.Dispose(); + } + + chunkFile = nextChunkFile; + } + + locations[entry] = new ChunkLocation(chunkFile.ArchiveIndex, chunkFile.Length); + ReadArchiveDataInBlocks(entry.ArchiveIndex, entry.Offset, entry.Length, chunkFile.Write); + } + + if (chunkFile == null) + { + return (locations, null); + } + + hashFractions.AddRange(chunkFile.GetHashFractions()); + + return (locations, hashFractions); + } + finally + { + chunkFile?.Dispose(); + } + } + + /// + /// Moves the temporary files over their targets, the directory file last. + /// Existing chunk files are moved aside first and are moved back if any move fails, so that either all files are replaced or none are. + /// + /// The paths that the replaced chunk files were moved aside to. + private static List ReplaceFiles(List chunkReplacements, FileReplacement directoryReplacement, string token) + { + var backups = new Dictionary(); + var replaced = new List(); + + try + { + foreach (var (_, path) in chunkReplacements) + { + if (File.Exists(path)) + { + var backupPath = $"{path}.{token}.bak"; + File.Move(path, backupPath); + backups.Add(path, backupPath); + } + } + + foreach (var (temporaryPath, path) in chunkReplacements.Append(directoryReplacement)) + { + File.Move(temporaryPath, path, overwrite: true); + replaced.Add(path); + } + } + catch (Exception exception) + { + var failures = new List { exception }; + + void Undo(Action action) + { + try + { + action(); + } + catch (Exception undoException) when (undoException is IOException or UnauthorizedAccessException) + { + failures.Add(undoException); + } + } + + foreach (var path in replaced.Where(path => !backups.ContainsKey(path))) + { + Undo(() => File.Delete(path)); + } + + foreach (var (path, backupPath) in backups) + { + Undo(() => File.Move(backupPath, path, overwrite: true)); + } + + if (failures.Count > 1) + { + throw new AggregateException(failures); + } + + throw; + } + + return [.. backups.Values]; + } + + private static void TryDeleteFile(string path) + { + try + { + var file = new FileInfo(path); + + if (file.Exists) + { + file.IsReadOnly = false; + file.Delete(); + } + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) + { + } + } + + /// + /// Writes to the given , which must not be the stream or a file this package reads from. /// - /// If any files were added with multi chunk enabled (see ), - /// their data is already in the chunk files, and this only writes the directory file. - /// Readers find the chunk files next to the directory file by name, so the stream should be the file at the path given to . + /// If the package has chunk files, from being read or from files added with multi chunk enabled (see ), + /// this only writes the directory file. Readers find the chunk files next to the directory file by name, + /// so the written data should end up at the path they are named after, which is the path the package was read from, given to , + /// or last repacked to with . /// /// The VPK version that is written is controlled by . /// @@ -356,35 +811,114 @@ public void Write(string filename) /// The input to write to. public void Write(Stream stream) { - if (Reader != null && IsDirVPK) + ArgumentNullException.ThrowIfNull(stream); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + if (ReferenceEquals(stream, Reader?.BaseStream) || (stream is FileStream fileStream && IsReadFile(fileStream.Name))) { - throw new InvalidOperationException("This package was opened from a _dir.vpk, writing back is currently unsupported."); + throw new InvalidOperationException("This package reads from the given stream or file, use Write(string) to write over the directory file it was read from."); } - ArgumentNullException.ThrowIfNull(stream); + WriteDirectoryFile(stream, GetEntriesInTreeOrder(), GetChunkHashFractions(), chunkLocations: null); + } - var tree = new Dictionary>>(); + /// + /// Points the entries with data in the directory file at where their data was written, and reads from the written directory file from then on. + /// + private void ReopenWrittenDirectoryFile(string path, List entries) + { + var offset = 0u; - // Group the entries by type and directory - foreach (var typeEntries in Entries ?? []) + foreach (var entry in entries) + { + if (IsInDirectoryFile(entry)) + { + entry.Offset = offset; + entry.PendingData = null; + offset += entry.Length; + } + } + + try + { + ReopenDirectoryFile(path); + } + catch (Exception exception) { - var typeTree = new Dictionary>(); - tree[typeEntries.Key] = typeTree; + throw new IOException($"'{path}' was written, but could not be opened again to read from it.", exception); + } + } + + /// + /// Returns all entries sorted the way Valve's packer writes the directory tree: by type, then directory, then file name, + /// and checks that the data of entries in the directory file can be read. + /// + private List GetEntriesInTreeOrder() + { + var entries = GetAllEntries().ToList(); - foreach (var entry in typeEntries.Value) + foreach (var entry in entries) + { + if (entry.FileName.Length == 0 || entry.TypeName.Length == 0) { - var directoryName = entry.DirectoryName.Length == 0 ? Space : entry.DirectoryName; + throw new InvalidOperationException($"Entry '{entry.GetFullPath()}' has an empty file name or extension, which the directory tree can not store."); + } + } - if (!typeTree.TryGetValue(directoryName, out var directoryEntries)) - { - directoryEntries = []; - typeTree[directoryName] = directoryEntries; - } + if (entries.Exists(static entry => IsInDirectoryFile(entry) && entry.PendingData == null)) + { + ThrowIfDirectoryFileClosed(); + } + + entries.Sort(CompareTreeOrder); - directoryEntries.Add(entry); + return entries; + } + + private static int CompareTreeOrder(PackageEntry x, PackageEntry y) + { + var comparison = CompareNames(x.TypeName, y.TypeName); + + if (comparison == 0) + { + comparison = CompareNames(GetTreeDirectoryName(x), GetTreeDirectoryName(y)); + } + + return comparison != 0 ? comparison : CompareNames(x.FileName, y.FileName); + } + + /// + /// Compares names with ASCII letters lowercased, and names that are equal that way ordinally. + /// + private static int CompareNames(string x, string y) + { + var length = Math.Min(x.Length, y.Length); + + for (var i = 0; i < length; i++) + { + var comparison = ToLowerAscii(x[i]).CompareTo(ToLowerAscii(y[i])); + + if (comparison != 0) + { + return comparison; } } + var lengthComparison = x.Length.CompareTo(y.Length); + + return lengthComparison != 0 ? lengthComparison : string.CompareOrdinal(x, y); + } + + private static string GetTreeDirectoryName(PackageEntry entry) + { + return entry.DirectoryName.Length == 0 ? Space : entry.DirectoryName; + } + + /// + /// Writes the directory file, with entries in chunk files where they are unless they are in . + /// + private void WriteDirectoryFile(Stream stream, List entries, List? chunkHashFractions, Dictionary? chunkLocations) + { // The file tree is built in memory first, so that the header can be written with its size and the tree can be hashed using var treeStream = new MemoryStream(); using var writer = new BinaryWriter(treeStream); @@ -395,43 +929,59 @@ public void Write(Stream stream) const byte NullByte = 0; - foreach (var typeEntries in tree) + void WriteNullTerminated(string value) { - writer.Write(Encoding.UTF8.GetBytes(typeEntries.Key)); + writer.Write(Encoding.UTF8.GetBytes(value)); writer.Write(NullByte); + } - foreach (var directoryEntries in typeEntries.Value) + foreach (var typeEntries in entries.GroupBy(static entry => entry.TypeName)) + { + WriteNullTerminated(typeEntries.Key); + + foreach (var directoryEntries in typeEntries.GroupBy(GetTreeDirectoryName)) { - writer.Write(Encoding.UTF8.GetBytes(directoryEntries.Key)); - writer.Write(NullByte); + WriteNullTerminated(directoryEntries.Key); - foreach (var entry in directoryEntries.Value) + foreach (var entry in directoryEntries) { - var fileLength = entry.TotalLength; - - // Data in chunk files was written when the file was added + var archiveIndex = entry.ArchiveIndex; var fileOffset = entry.Offset; - if (entry.ArchiveIndex == 0x7FFF) + if (entry.Length == 0) { - if ((ulong)embeddedDataOffset + fileLength > int.MaxValue) + archiveIndex = DirectoryArchiveIndex; + fileOffset = uint.MaxValue; + } + else if (chunkLocations != null && chunkLocations.TryGetValue(entry, out var location)) + { + (archiveIndex, fileOffset) = location; + } + else if (archiveIndex == DirectoryArchiveIndex) + { + if ((ulong)embeddedDataOffset + entry.Length > int.MaxValue) { throw new InvalidOperationException("Package contents exceed 2GiB, add files with multi chunk enabled to split them into chunk files."); } fileOffset = embeddedDataOffset; - embeddedDataOffset += fileLength; + embeddedDataOffset += entry.Length; embeddedEntries.Add(entry); } - writer.Write(Encoding.UTF8.GetBytes(entry.FileName)); - writer.Write(NullByte); + if (entry.SmallData.Length > ushort.MaxValue) + { + throw new InvalidOperationException($"Entry '{entry.GetFullPath()}' has {entry.SmallData.Length} preload bytes, the directory tree can store at most {ushort.MaxValue}."); + } + + WriteNullTerminated(entry.FileName); writer.Write(entry.CRC32); - writer.Write((short)0); // SmallData, we will put it into data instead - writer.Write(entry.ArchiveIndex); + writer.Write((ushort)entry.SmallData.Length); + writer.Write(archiveIndex); writer.Write(fileOffset); - writer.Write(fileLength); + writer.Write(entry.Length); writer.Write(ushort.MaxValue); // terminator, 0xFFFF + writer.Write(entry.SmallData); } writer.Write(NullByte); @@ -444,33 +994,23 @@ public void Write(Stream stream) var treeData = treeStream.GetBuffer().AsSpan(0, (int)treeStream.Length); - // Archive MD5 section, contains hashes of chunk files - var chunkHashFractions = new List(FinishedChunkHashFractions); + var fractionCount = chunkHashFractions?.Count ?? (int)((embeddedDataOffset + (ChunkHasher.FileFractionSize - 1L)) / ChunkHasher.FileFractionSize); - if (CurrentChunkFile != null) - { - chunkHashFractions.AddRange(CurrentChunkFile.GetHashFractions()); - } - - var archiveMd5SectionBytes = SerializeChunkHashFractions(chunkHashFractions); - - using var headerStream = new MemoryStream(); - using var headerWriter = new BinaryWriter(headerStream); - headerWriter.Write(MAGIC); - headerWriter.Write(Version); - headerWriter.Write(treeData.Length); // TreeSize + Span header = stackalloc byte[Version >= 2 ? 28 : 12]; + BinaryPrimitives.WriteInt32LittleEndian(header, MAGIC); + BinaryPrimitives.WriteUInt32LittleEndian(header[4..], Version); + BinaryPrimitives.WriteInt32LittleEndian(header[8..], treeData.Length); // TreeSize if (Version >= 2) { - headerWriter.Write(embeddedDataOffset); // FileDataSectionSize - headerWriter.Write(archiveMd5SectionBytes.Length); // ArchiveMD5SectionSize - headerWriter.Write(48); // OtherMD5SectionSize - headerWriter.Write(0); // SignatureSectionSize + BinaryPrimitives.WriteUInt32LittleEndian(header[12..], embeddedDataOffset); // FileDataSectionSize + BinaryPrimitives.WriteInt32LittleEndian(header[16..], fractionCount * ChunkHashFraction.SectionEntrySize); // ArchiveMD5SectionSize + BinaryPrimitives.WriteInt32LittleEndian(header[20..], 48); // OtherMD5SectionSize + BinaryPrimitives.WriteInt32LittleEndian(header[24..], UnsignedSignatureSectionSize); // SignatureSectionSize } - var header = headerStream.GetBuffer().AsSpan(0, (int)headerStream.Length); - using var fullFileMD5 = IncrementalHash.CreateHash(HashAlgorithmName.MD5); + using var embeddedDataHashes = new ChunkHasher(DirectoryArchiveIndex); void WriteAndHash(ReadOnlySpan data) { @@ -483,7 +1023,14 @@ void WriteAndHash(ReadOnlySpan data) foreach (var entry in embeddedEntries) { - WriteAndHash(GetEntryData(entry)); + var data = GetEmbeddedData(entry); + WriteAndHash(data); + + // Data in the directory file is only hashed in packages without chunk files + if (chunkHashFractions == null) + { + embeddedDataHashes.Append(data); + } } if (Version < 2) @@ -492,26 +1039,158 @@ void WriteAndHash(ReadOnlySpan data) return; } + var archiveMd5SectionBytes = SerializeChunkHashFractions(chunkHashFractions ?? embeddedDataHashes.GetHashFractions(includeEmptyLast: false)); + Debug.Assert(archiveMd5SectionBytes.Length == fractionCount * ChunkHashFraction.SectionEntrySize); + WriteAndHash(archiveMd5SectionBytes); WriteAndHash(MD5.HashData(treeData)); WriteAndHash(MD5.HashData(archiveMd5SectionBytes)); stream.Write(fullFileMD5.GetHashAndReset()); + + Span signatureSection = stackalloc byte[UnsignedSignatureSectionSize]; + BinaryPrimitives.WriteInt32LittleEndian(signatureSection, MAGIC); + BinaryPrimitives.WriteInt32LittleEndian(signatureSection[4..], (int)ESignatureType.OnlyFileChecksum); + stream.Write(signatureSection); + } + + private bool IsReadFile(string path) + { + if (ReadFilePath != null && PathsEqual(path, ReadFilePath)) + { + return true; + } + + if (HighestChunkIndex < 0) + { + return false; + } + + var baseName = GetChunkBaseName(); + + return Enumerable.Range(0, HighestChunkIndex + 1).Any(archiveIndex => PathsEqual(path, GetArchiveIndexFullFilePath(baseName, (ushort)archiveIndex))); + } + + private static bool PathsEqual(string path1, string path2) + { + return string.Equals(path1, path2, OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal); } /// - /// Returns the data for an entry that is being written out. + /// Returns the archive MD5 section entries for the chunk files, or null if the package has none. /// - private byte[] GetEntryData(PackageEntry entry) + private List? GetChunkHashFractions() { - if (entry.Length == 0) + if (HighestChunkIndex < 0) + { + return null; + } + + var baseName = GetChunkBaseName(); + var readFractions = AccessPackFileHashes + .Where(static fraction => fraction.ArchiveIndex != DirectoryArchiveIndex) + .GroupBy(static fraction => fraction.ArchiveIndex) + .ToDictionary(static fractions => fractions.Key, static fractions => fractions.OrderBy(static fraction => fraction.Offset).ToList()); + var usedArchiveIndices = GetChunkIndicesWithData().ToHashSet(); + var hashFractions = new List(); + + for (var i = 0; i <= HighestChunkIndex; i++) { - // Data added by AddFile lives entirely in SmallData, avoid copying it - return entry.SmallData; + var archiveIndex = (ushort)i; + + if (CurrentChunkFile?.ArchiveIndex == archiveIndex) + { + hashFractions.AddRange(CurrentChunkFile.GetHashFractions()); + continue; + } + + var file = new FileInfo(GetArchiveIndexFullFilePath(baseName, archiveIndex)); + + if (!file.Exists) + { + if (usedArchiveIndices.Contains(archiveIndex)) + { + throw new FileNotFoundException($"Chunk file {archiveIndex} does not exist, but entries are in it.", file.FullName); + } + + continue; + } + + if (!UpdatedChunkHashFractions.TryGetValue(archiveIndex, out var fractions) && !readFractions.TryGetValue(archiveIndex, out fractions)) + { + continue; + } + + if (!CoversFile(fractions, file.Length)) + { + if (file.Length > uint.MaxValue) + { + throw new InvalidOperationException($"Chunk file {archiveIndex} is {file.Length} bytes, its hashes can only be computed for chunk files smaller than 4 GiB."); + } + + using var hashes = new ChunkHasher(archiveIndex); + ReadArchiveDataInBlocks(archiveIndex, 0, file.Length, hashes.Append); + fractions = hashes.GetHashFractions(); + UpdatedChunkHashFractions[archiveIndex] = fractions; + } + + hashFractions.AddRange(fractions); } - ReadEntry(entry, out var fileData, validateCrc: false); - return fileData; + return hashFractions; + } + + private static bool CoversFile(List fractions, long fileLength) + { + var offset = 0L; + + foreach (var fraction in fractions) + { + if (fraction.Offset != offset) + { + return false; + } + + offset += fraction.Length; + } + + return offset == fileLength; + } + + private void ReadArchiveDataInBlocks(ushort archiveIndex, long offset, long length, Action> action) + { + var buffer = ArrayPool.Shared.Rent(CopyBufferSize); + + try + { + for (var end = offset + length; offset < end;) + { + var data = buffer.AsSpan(0, (int)Math.Min(end - offset, CopyBufferSize)); + ReadArchiveDataExactly(archiveIndex, offset, data); + action(data); + offset += data.Length; + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + /// + /// Returns the data of an entry that is written into the directory file, without its preload bytes. + /// + private byte[] GetEmbeddedData(PackageEntry entry) + { + if (entry.PendingData != null) + { + return entry.PendingData; + } + + var data = new byte[entry.Length]; + ReadArchiveDataExactly(entry.ArchiveIndex, entry.Offset, data); + + return data; } private static byte[] SerializeChunkHashFractions(List fractions) @@ -533,41 +1212,92 @@ private static byte[] SerializeChunkHashFractions(List fracti return bytes; } + private readonly record struct ChunkLocation(ushort ArchiveIndex, uint Offset); + + private readonly record struct FileReplacement(string TemporaryPath, string Path); + /// - /// Writes data for a single chunk file, hashing it in 1 MiB fractions for the archive MD5 section as it is written. + /// Writes data for a single chunk file, hashing it for the archive MD5 section as it is written. /// private sealed class ChunkFileWriter : IDisposable { - private const int FileFractionSize = 0x00100000; // 1 MiB - private readonly FileStream stream; - private readonly IncrementalHash md5 = IncrementalHash.CreateHash(HashAlgorithmName.MD5); - private readonly List fractions = []; - private uint fractionOffset; - private uint bytesInFraction; + private readonly ChunkHasher hasher; /// /// Gets the index of this chunk file. /// - public ushort ArchiveIndex { get; } + public ushort ArchiveIndex => hasher.ArchiveIndex; /// /// Gets the size in bytes of the data written so far. /// - public uint Length => fractionOffset + bytesInFraction; + public uint Length => hasher.Length; /// - /// Creates the chunk file, replacing an existing file. + /// Creates the chunk file. /// /// Index of this chunk file. /// Path of the chunk file. - public ChunkFileWriter(ushort archiveIndex, string filePath) + /// Either to replace an existing file, or to fail if one exists. + public ChunkFileWriter(ushort archiveIndex, string filePath, FileMode mode = FileMode.Create) { - ArchiveIndex = archiveIndex; - stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read, bufferSize: 65536); + stream = new FileStream(filePath, mode, FileAccess.Write, FileShare.Read, bufferSize: 65536); + hasher = new ChunkHasher(archiveIndex); } public void Write(ReadOnlySpan data) + { + hasher.Append(data); + stream.Write(data); + } + + public void Flush() + { + stream.Flush(); + } + + /// + public List GetHashFractions() + { + return hasher.GetHashFractions(); + } + + public void Dispose() + { + stream.Dispose(); + hasher.Dispose(); + } + } + + /// + /// Hashes the data of a single archive in 1 MiB fractions for the archive MD5 section. + /// + private sealed class ChunkHasher : IDisposable + { + public const int FileFractionSize = 0x00100000; // 1 MiB + + private Blake3.Hasher hasher = Blake3.Hasher.New(); + private readonly List fractions = []; + private uint fractionOffset; + private uint bytesInFraction; + + /// + /// Gets the index of the archive the hashed data is in. + /// + public ushort ArchiveIndex { get; } + + /// + /// Gets the size in bytes of the data hashed so far. + /// + public uint Length => fractionOffset + bytesInFraction; + + public ChunkHasher(ushort archiveIndex) + { + ArchiveIndex = archiveIndex; + } + + public void Append(ReadOnlySpan data) { // Offsets and lengths in the directory tree are 32-bit if ((long)Length + data.Length > uint.MaxValue) @@ -575,47 +1305,42 @@ public void Write(ReadOnlySpan data) throw new InvalidOperationException($"Chunk file {ArchiveIndex} would exceed 4 GiB."); } - stream.Write(data); - while (!data.IsEmpty) { var length = Math.Min(FileFractionSize - (int)bytesInFraction, data.Length); - md5.AppendData(data[..length]); + hasher.Update(data[..length]); bytesInFraction += (uint)length; data = data[length..]; if (bytesInFraction == FileFractionSize) { - fractions.Add(CreateFraction(md5.GetHashAndReset())); + fractions.Add(CreateFraction()); + hasher.Reset(); fractionOffset += bytesInFraction; bytesInFraction = 0; } } } - public void Flush() - { - stream.Flush(); - } - /// /// Returns the hashes of the data written so far. The last fraction covers the data after the last full fraction, /// and is zero sized when the chunk file size is an exact multiple of the fraction size. /// - public List GetHashFractions() + /// If false, a zero sized last fraction is left out. + public List GetHashFractions(bool includeEmptyLast = true) { - var allFractions = new List(fractions); - allFractions.Add(CreateFraction(md5.GetCurrentHash())); - - return allFractions; + return bytesInFraction > 0 || includeEmptyLast ? [.. fractions, CreateFraction()] : [.. fractions]; } - private ChunkHashFraction CreateFraction(byte[] checksum) + private ChunkHashFraction CreateFraction() { + var checksum = new byte[16]; + hasher.Finalize(checksum); + return new ChunkHashFraction { ArchiveIndex = ArchiveIndex, - HashType = EHashType.MD5, + HashType = EHashType.Blake3, Offset = fractionOffset, Length = bytesInFraction, Checksum = checksum, @@ -624,8 +1349,7 @@ private ChunkHashFraction CreateFraction(byte[] checksum) public void Dispose() { - stream.Dispose(); - md5.Dispose(); + hasher.Dispose(); } } } diff --git a/ValvePak/ValvePak/Package.Verify.cs b/ValvePak/ValvePak/Package.Verify.cs index 66ba961..54179fc 100644 --- a/ValvePak/ValvePak/Package.Verify.cs +++ b/ValvePak/ValvePak/Package.Verify.cs @@ -34,20 +34,24 @@ public partial class Package : IDisposable /// /// Verify MD5 hashes provided in the VPK. /// - /// The package has not been read. - /// The package has no checksums, or a checksum does not match. - public void VerifyHashes() + /// + /// Version 1 packages have no checksums, so nothing is verified for them. + /// The whole file checksum covers the file tree and archive MD5 section checksums, so when it matches, + /// a mismatching file tree or archive MD5 section checksum is reported to instead of thrown. + /// + /// If provided, will report a string for each checksum mismatch that is not thrown, or that a version 1 package has no checksums. + /// The package has not been read, or its directory file could not be opened again after writing. + /// The package has no checksums, or the whole file checksum does not match. + public void VerifyHashes(IProgress? progressReporter = null) { ObjectDisposedException.ThrowIf(IsDisposed, this); - if (Reader == null) - { - throw new InvalidOperationException("The package has not been read."); - } + ThrowIfDirectoryFileClosed(); - if (Version != 2) + if (Version == 1) { - throw new InvalidDataException("Only version 2 is supported."); + progressReporter?.Report("Version 1 packages have no checksums to verify."); + return; } if (TreeChecksum == null || ArchiveMD5EntriesChecksum == null || WholeFileChecksum == null) @@ -55,26 +59,35 @@ public void VerifyHashes() throw new InvalidDataException("The package has no checksum section."); } - var hash = HashPackageRange(HeaderSize, TreeSize, HashAlgorithmName.MD5); + var wholeFileHash = HashPackageRange(0, FileSizeBeforeWholeFileHash, HashAlgorithmName.MD5); + var wholeFileHashMatches = wholeFileHash.SequenceEqual(WholeFileChecksum); + + VerifySectionHash("File tree", HeaderSize, TreeSize, TreeChecksum, wholeFileHashMatches, progressReporter); + VerifySectionHash("Archive MD5 entries", FileSizeBeforeArchiveMD5Entries, ArchiveMD5SectionSize, ArchiveMD5EntriesChecksum, wholeFileHashMatches, progressReporter); - if (!hash.SequenceEqual(TreeChecksum)) + if (!wholeFileHashMatches) { - throw new InvalidDataException($"File tree checksum mismatch ({BitConverter.ToString(hash)} != expected {BitConverter.ToString(TreeChecksum)})"); + throw new InvalidDataException($"Package checksum mismatch ({BitConverter.ToString(wholeFileHash)} != expected {BitConverter.ToString(WholeFileChecksum)})"); } + } - hash = HashPackageRange(FileSizeBeforeArchiveMD5Entries, ArchiveMD5SectionSize, HashAlgorithmName.MD5); + private void VerifySectionHash(string sectionName, long offset, long length, byte[] expected, bool wholeFileHashMatches, IProgress? progressReporter) + { + var hash = HashPackageRange(offset, length, HashAlgorithmName.MD5); - if (!hash.SequenceEqual(ArchiveMD5EntriesChecksum)) + if (hash.SequenceEqual(expected)) { - throw new InvalidDataException($"Archive MD5 entries checksum mismatch ({BitConverter.ToString(hash)} != expected {BitConverter.ToString(ArchiveMD5EntriesChecksum)})"); + return; } - hash = HashPackageRange(0, FileSizeBeforeWholeFileHash, HashAlgorithmName.MD5); + var message = $"{sectionName} checksum mismatch ({BitConverter.ToString(hash)} != expected {BitConverter.ToString(expected)})"; - if (!hash.SequenceEqual(WholeFileChecksum)) + if (!wholeFileHashMatches) { - throw new InvalidDataException($"Package checksum mismatch ({BitConverter.ToString(hash)} != expected {BitConverter.ToString(WholeFileChecksum)})"); + throw new InvalidDataException(message); } + + progressReporter?.Report($"{message}, ignored because the package checksum matches."); } /// @@ -83,7 +96,7 @@ public void VerifyHashes() /// private byte[] HashPackageRange(long offset, long length, HashAlgorithmName hashAlgorithm) { - var archive = GetArchive(0x7FFF); + var archive = GetArchive(DirectoryArchiveIndex); var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, ChunkHashBufferSize)); try @@ -190,6 +203,7 @@ public void VerifyChunkHashes(IProgress? progressReporter = null) /// Verify CRC32 checksums of all files in the package. /// /// If provided, will report a string with the current verification progress. + /// An archive ends before all bytes of an entry could be read. public void VerifyFileChecksums(IProgress? progressReporter = null) { ObjectDisposedException.ThrowIf(IsDisposed, this); @@ -210,6 +224,8 @@ public void VerifyFileChecksums(IProgress? progressReporter = null) { progressReporter?.Report($"Verifying CRC32 checksum for '{entry.GetFullPath()}' in archive {entry.ArchiveIndex}."); + ThrowIfEntryPastEndOfArchive(entry); + if (buffer.Length < entry.TotalLength) { buffer = new byte[entry.TotalLength]; @@ -224,7 +240,7 @@ public void VerifyFileChecksums(IProgress? progressReporter = null) /// /// Verifies the RSA signature. /// - /// True if signature is valid, false otherwise. + /// True if the signature is valid or the package has no signature (see ), false otherwise. public bool IsSignatureValid() { if (PublicKey == null || Signature == null) diff --git a/ValvePak/ValvePak/Package.cs b/ValvePak/ValvePak/Package.cs index 51abece..20ece28 100644 --- a/ValvePak/ValvePak/Package.cs +++ b/ValvePak/ValvePak/Package.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.Text; using System.Threading; namespace ValvePak @@ -20,6 +21,11 @@ public partial class Package : IDisposable internal const string Dot = "."; internal const string DirectorySeparator = "/"; + /// + /// Archive index of entries whose data is in the directory file instead of a chunk file. + /// + internal const ushort DirectoryArchiveIndex = 0x7FFF; + /// /// Always '/' as per Valve's vpk implementation. /// @@ -33,6 +39,8 @@ public partial class Package : IDisposable /// private long StreamOffset; + private string? ReadFilePath; + private readonly ConcurrentDictionary Archives = new(); private readonly Lock ArchivesLock = new(); @@ -120,6 +128,11 @@ public uint Version /// public byte[]? Signature { get; private set; } + /// + /// Gets whether the package has both a public key and a signature. + /// + public bool IsSigned => PublicKey != null && Signature != null; + /// /// Gets the signature type. /// @@ -179,12 +192,32 @@ protected virtual void Dispose(bool disposing) private void ThrowIfAlreadyRead() { - if (Reader != null) + if (Reader != null || IsReadPackage) { throw new InvalidOperationException("This package has already been read, create a new instance to read another package."); } } + private void ThrowIfFilesAdded() + { + if (Reader == null && Entries != null) + { + throw new InvalidOperationException("Files were added to this package, create a new instance to read a package."); + } + } + + private void ThrowIfDirectoryFileClosed() + { + if (Reader != null) + { + return; + } + + throw new InvalidOperationException(ReadFilePath == null + ? "The package has not been read." + : $"The directory file '{ReadFilePath}' could not be opened again after writing, so data in it can not be read."); + } + private Dictionary> CreateEntriesDictionary() { var stringComparer = Comparer == null ? null : StringComparer.FromComparison(Comparer.Comparison); @@ -202,7 +235,7 @@ public void SetFileName(string fileName) ArgumentNullException.ThrowIfNull(fileName); ThrowIfAlreadyRead(); - if (CurrentChunkFile != null) + if (HighestChunkIndex >= 0) { throw new InvalidOperationException("The file name can not be changed after files were added to chunk files."); } @@ -239,16 +272,13 @@ private static string StripDirVpkSuffixes(string fileName, out bool isDirVpk) /// /// Full path to the file to find. /// - /// Normalizes the directory path separator from (\) to (/) in . + /// The path is normalized the same way as paths of added files, see . /// public PackageEntry? FindEntry(string filePath) { ArgumentNullException.ThrowIfNull(filePath); - // Normalize path separators when reading the file list - var filePathSpan = filePath.Replace(WindowsDirectorySeparator, DirectorySeparatorChar).AsSpan(); - - return FindEntry(filePathSpan); + return FindEntry(filePath.AsSpan()); } /// @@ -259,40 +289,55 @@ private static string StripDirVpkSuffixes(string fileName, out bool isDirVpk) /// /// Full path to the file to find. /// - /// Unlike the version, this one does not normalize the directory path separator. + /// The path is normalized the same way as paths of added files: (\) becomes (/), + /// empty and "." segments and trailing dots are removed, and a missing directory or extension is a space. + /// If that finds no entry, the path is also looked up as it is, split at its last and at each dot of the file name, + /// which finds entries of read packages whose names normalizing changes. /// public PackageEntry? FindEntry(ReadOnlySpan filePath) { - var lastSeparator = filePath.LastIndexOf(DirectorySeparatorChar); - var directory = lastSeparator > -1 ? filePath[..lastSeparator] : string.Empty; - var fileName = filePath[(lastSeparator + 1)..]; + SplitPath(NormalizePath(filePath, out var changed), out var directory, out var fileName, out var extension); - var dot = fileName.LastIndexOf('.'); - string extension; + return FindEntry(directory, fileName, extension) ?? FindEntryByStoredName(filePath, changed ? -1 : fileName.Length); + } - if (dot > -1) - { - extension = fileName[(dot + 1)..].ToString(); - fileName = fileName[..dot]; - } - else - { - // Valve uses a space for missing extensions - extension = Space; - } + /// + /// Finds the entry whose is the given path, trying each split of the file name into a file and type name. + /// + /// The path as it was given. + /// Position in the file name that was already looked up, where the length of the file name means without a type, or -1. + private PackageEntry? FindEntryByStoredName(ReadOnlySpan path, int skipSplit) + { + var lastSeparator = path.LastIndexOf(DirectorySeparatorChar); + var directory = lastSeparator > -1 ? path[..lastSeparator] : Space; + var name = path[(lastSeparator + 1)..]; + PackageEntry? entry = null; - if (Entries == null || !Entries.TryGetValue(extension, out var entriesForExtension)) + for (var split = 1; entry == null && split <= name.Length; split++) { - return default; + if (split == skipSplit) + { + continue; + } + + if (split == name.Length) + { + entry = FindEntry(directory, name, Space); + } + else if (name[split] == '.' && split < name.Length - 1) + { + entry = FindEntry(directory, name[..split], name[(split + 1)..]); + } } - // Remove the trailing and leading slash - directory = directory.Trim(DirectorySeparatorChar); + return entry; + } - // If the directory is empty after trimming, set it to a space to match Valve's behaviour - if (directory.Length == 0) + private PackageEntry? FindEntry(ReadOnlySpan directory, ReadOnlySpan fileName, ReadOnlySpan extension) + { + if (Entries == null || !Entries.GetAlternateLookup>().TryGetValue(extension, out var entriesForExtension)) { - directory = Space; + return default; } int hi = entriesForExtension.Count - 1; @@ -359,6 +404,89 @@ private static string StripDirVpkSuffixes(string fileName, out bool isDirVpk) return default; } + /// + /// Replaces with , removes empty and "." segments, and removes trailing dots. + /// + private static ReadOnlySpan NormalizePath(ReadOnlySpan path, out bool changed) + { + changed = !IsNormalizedPath(path); + + if (!changed) + { + return path; + } + + var builder = new StringBuilder(path.Length); + + foreach (var range in path.SplitAny(DirectorySeparatorChar, WindowsDirectorySeparator)) + { + var segment = path[range]; + + if (IsSkippedSegment(segment)) + { + continue; + } + + if (builder.Length > 0) + { + builder.Append(DirectorySeparatorChar); + } + + builder.Append(segment); + } + + var length = builder.Length; + + while (length > 0 && builder[length - 1] == '.') + { + length--; + } + + return builder.ToString(0, length); + } + + private static bool IsNormalizedPath(ReadOnlySpan path) + { + if (path.Contains(WindowsDirectorySeparator)) + { + return false; + } + + foreach (var range in path.Split(DirectorySeparatorChar)) + { + if (IsSkippedSegment(path[range])) + { + return false; + } + } + + return !path.EndsWith('.'); + } + + private static bool IsSkippedSegment(ReadOnlySpan segment) + { + return segment.IsEmpty || segment is Dot; + } + + /// + /// Splits a normalized path into the directory, file and type names of an entry, using a space for a missing directory or extension. + /// + private static void SplitPath(ReadOnlySpan path, out ReadOnlySpan directory, out ReadOnlySpan fileName, out ReadOnlySpan extension) + { + var lastSeparator = path.LastIndexOf(DirectorySeparatorChar); + directory = lastSeparator > -1 ? path[..lastSeparator] : Space; + fileName = path[(lastSeparator + 1)..]; + + var dot = fileName.LastIndexOf('.'); + extension = Space; + + if (dot > -1) + { + extension = fileName[(dot + 1)..]; + fileName = fileName[..dot]; + } + } + /// /// This sorts so that it can be searched through using binary search. /// Use if you want to search case insensitively. diff --git a/ValvePak/ValvePak/PackageEntry.cs b/ValvePak/ValvePak/PackageEntry.cs index 8cd0cc0..31d04ad 100644 --- a/ValvePak/ValvePak/PackageEntry.cs +++ b/ValvePak/ValvePak/PackageEntry.cs @@ -22,7 +22,7 @@ public class PackageEntry /// /// Gets or sets the file extension. - /// If the file has no extension, this is an empty string. + /// If the file has no extension, this is a single space. /// public required string TypeName { get; set; } @@ -69,6 +69,11 @@ public uint TotalLength /// public byte[] SmallData { get; set; } = []; + /// + /// Gets or sets the data of an added file that is written into the directory file and is not stored in an archive yet. + /// + internal byte[]? PendingData { get; set; } + /// /// Returns the file name and extension. /// diff --git a/ValvePak/ValvePak/StreamArchive.cs b/ValvePak/ValvePak/StreamArchive.cs index 95b7740..58b6f1e 100644 --- a/ValvePak/ValvePak/StreamArchive.cs +++ b/ValvePak/ValvePak/StreamArchive.cs @@ -21,6 +21,17 @@ public StreamArchive(Stream stream, long origin, long dataOffset) Stream = stream; } + public override long Length + { + get + { + lock (StreamLock) + { + return Stream.Length - Origin; + } + } + } + public override int Read(long offset, Span output) { lock (StreamLock)