diff --git a/README.md b/README.md index 99c80fc..6d92dcd 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ if (file != null) Do note that files such as `pak01_001.vpk` are just data files, you have to open `pak01_dir.vpk`. +`FindEntry` normalizes the path like added files are: `\` is a directory separator and ASCII letters are lowercased. + To read from a stream instead of a path, set the file name first so that the chunk files can be located: ```csharp @@ -113,6 +115,14 @@ var entry = package.FindEntry("path/to/file.txt"); using var stream = package.GetMemoryMappedStreamIfPossible(entry); ``` +`OpenEntryStream` returns a seekable stream that reads an entry in place. A VPK stored in an entry of another VPK can be read from it without extracting it: + +```csharp +using var inner = new Package(); +inner.SetFileName(entry.GetFullPath()); +inner.Read(package.OpenEntryStream(entry)); +``` + ## Optimize for many lookups By default, `FindEntry` performs a linear scan. If you need to look up many files, call `OptimizeEntriesForBinarySearch()` before `Read()` to sort entries and use binary search instead. You can also pass `StringComparison.OrdinalIgnoreCase` for case-insensitive lookups. @@ -128,13 +138,18 @@ package.Read("pak01_dir.vpk"); var file = package.FindEntry("path/to/file.txt"); ``` +`Package.Open("pak01_dir.vpk")` does both in one call. + ## Verification ```csharp using var package = new Package(); package.Read("pak01_dir.vpk"); -// Verify MD5 hashes of the directory tree and whole file (throws on version 1 packages) +// Verify the signature, all hashes, and the CRC32 checksums of files no hash covers in one call +package.Verify(); + +// Verify MD5 hashes of the directory tree and whole file (version 1 packages have none) package.VerifyHashes(); // Verify MD5/Blake3 hashes of individual chunk files (pak01_000.vpk, pak01_001.vpk, ...) @@ -167,7 +182,7 @@ All file data is stored in the single written file, which is limited to 2 GiB. T ### Modify an existing VPK -A package that was read can have files added or removed and then be written to a new file. This only works for packages that are not split into chunk files. Writing back a package opened from a `_dir.vpk` is not supported. +A package that was read can have files added, removed or renamed and then be written to a new file, or back over the file it was read from, including a `_dir.vpk` with chunk files. ```csharp using var package = new Package(); @@ -179,9 +194,11 @@ package.RemoveFile(package.FindEntry("old/file.txt")!); package.Write("pak01_modified.vpk"); ``` +Chunk files are named after the directory file, so a package with chunk files can only be written to another path with `repack: true`, which `RequiresRepack(path)` tells. Repacking a package without chunk files into a `_dir.vpk` moves its data into chunk files. + ### Split into chunk files -Files added with `multiChunk` are written right away into numbered chunk files (`pak01_000.vpk`, `pak01_001.vpk`, ...) next to the directory file, so their data is not kept in memory. A new chunk file is started once the current one reaches `WriteChunkSize` (200 MiB by default). `Write` then only writes the directory file, which contains MD5 hashes of the chunk files that can be verified with `VerifyChunkHashes`. +Files added with `multiChunk` are written right away into numbered chunk files (`pak01_000.vpk`, `pak01_001.vpk`, ...) next to the directory file, so their data is not kept in memory. A new chunk file is started once the current one is larger than `WriteChunkSize` (100 MiB by default). `Write` then only writes the directory file, which contains Blake3 hashes of the chunk files that can be verified with `VerifyChunkHashes`. ```csharp using var package = new Package(); @@ -189,8 +206,8 @@ using var package = new Package(); // Chunk files are created next to the directory file, so its path must be set first package.SetFileName("pak01_dir.vpk"); -// Optionally lower the maximum chunk file size, in bytes -package.WriteChunkSize = 100 * 1024 * 1024; +// Optionally change the chunk file size, in bytes +package.WriteChunkSize = 50 * 1024 * 1024; // Data can be streamed from a file instead of being read into memory using (var stream = File.OpenRead("example.vmdl")) @@ -205,4 +222,4 @@ package.AddFile("manifest.txt", File.ReadAllBytes("manifest.txt")); package.Write("pak01_dir.vpk"); ``` -Removing a file that is already in a chunk file leaves its data in place. Chunk files stay on disk if the package is disposed without being written. +Removing a file that is already in a chunk file leaves its data in place until the package is written with `repack: true`. Chunk files stay on disk if the package is disposed without being written. diff --git a/ValvePak/ValvePak.Test/EntryPathTest.cs b/ValvePak/ValvePak.Test/EntryPathTest.cs new file mode 100644 index 0000000..3f3042b --- /dev/null +++ b/ValvePak/ValvePak.Test/EntryPathTest.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace ValvePak.Test +{ + internal sealed class EntryPathTest + { + [Test] + [Arguments("A\\B//./c.TXT", "a/b/c.txt")] + [Arguments("a/x/../b.txt", "a/b.txt")] + [Arguments("/a/b.txt", "a/b.txt")] + [Arguments("README", "readme")] + [Arguments("", null)] + [Arguments("../b.txt", null)] + [Arguments("a/b/", null)] + [Arguments(".gitignore", null)] + [Arguments("x.. ", null)] + [Arguments("a/b\0c.txt", null)] + [Arguments("a/\ud800.txt", null)] + public async Task NormalizesEntryPaths(string path, string? expected) + { + using (Assert.Multiple()) + { + await Assert.That(Package.TryParseEntryPath(path, out var normalizedPath)).IsEqualTo(expected != null); + await Assert.That(normalizedPath).IsEqualTo(expected); + } + } + + [Test] + public async Task RejectsNamesThatAreTooLong() + { + using (Assert.Multiple()) + { + await Assert.That(Package.TryParseEntryPath($"a.{new string('y', 129)}", out _)).IsFalse(); + await Assert.That(Package.TryParseEntryPath($"{new string('é', 131)}.txt", out _)).IsFalse(); + } + } + + [Test] + [Arguments("Materials\\Models/", "materials/models")] + [Arguments(" ", "")] + public async Task NormalizesDirectoryPaths(string path, string expected) + { + await Assert.That(Package.NormalizeDirectoryPath(path)).IsEqualTo(expected); + } + + [Test] + [Arguments(" ")] + [Arguments("")] + public async Task GetsDirectoryPathWithRootAsEmptyString(string directoryName) + { + var entry = new PackageEntry + { + DirectoryName = directoryName, + FileName = "file", + TypeName = "txt", + }; + + using (Assert.Multiple()) + { + await Assert.That(entry.GetDirectoryPath()).IsEmpty(); + await Assert.That(entry.GetFullPath()).IsEqualTo("file.txt"); + } + } + + [Test] + public async Task PathComparerIgnoresOnlyTheCaseOfAsciiLetters() + { + var folders = new Dictionary(Package.PathComparer) { ["Models"] = 1 }; + + using (Assert.Multiple()) + { + await Assert.That(Package.PathComparer.Equals("MODELS/A.txt", "models/a.TXT")).IsTrue(); + await Assert.That(Package.PathComparer.Equals("Ä", "ä")).IsFalse(); + await Assert.That(folders.GetAlternateLookup>().ContainsKey("mODELS")).IsTrue(); + } + } + } +} diff --git a/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs b/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs index 8b1babc..3282991 100644 --- a/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs +++ b/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs @@ -88,6 +88,22 @@ public async Task WriteMultiChunkPackage() await Assert.That(dirEntry.ArchiveIndex).IsEqualTo((ushort)0x7FFF); } + [Test] + public async Task WriteDeletesStaleChunkFilesOfNewPackage() + { + var dirPath = TempPath("stale_dir.vpk"); + await File.WriteAllBytesAsync(TempPath("stale_001.vpk"), [1]); + + using (var package = new Package()) + { + package.SetFileName(dirPath); + package.AddFile("a.bin", CreateTestData(200, 1), multiChunk: true); + package.Write(dirPath); + } + + await Assert.That(File.Exists(TempPath("stale_001.vpk"))).IsFalse(); + } + [Test] public async Task WritesChunkHashFractions() { @@ -609,8 +625,8 @@ public async Task EditsReadPackageWithChunkFiles() await AssertEntryData(readBack, "added/embedded.bin", 300, 2); await Assert.That(readBack.FindEntry("steammessages_base.proto")!.SmallData).Count().IsEqualTo(preload.Length); - // The read chunk file had no hashes, and data in the directory file is not hashed when there are chunk files - await Assert.That(readBack.AccessPackFileHashes.Select(static fraction => fraction.ArchiveIndex).ToArray()).IsEquivalentTo(new ushort[] { 2, 4 }, CollectionOrdering.Matching); + // Chunk files without hashes are hashed, and data in the directory file is not hashed when there are chunk files + await Assert.That(readBack.AccessPackFileHashes.Select(static fraction => fraction.ArchiveIndex).ToArray()).IsEquivalentTo(new ushort[] { 0, 1, 2, 3, 4 }, CollectionOrdering.Matching); } // Hashes that cover the whole chunk file are kept even when its data changed, and are computed again once it grew @@ -847,6 +863,142 @@ public async Task RepackRefusesToReplaceChunkFilesOfTheReadFileFromAnotherPath() await AssertPackageVerifies(package); } + [Test] + public async Task RequiresRepackForPathsTheChunkFilesAreNotNamedAfter() + { + var dirPath = CopyFixtures("test_dir.vpk", "test_000.vpk"); + var otherChunkNames = TempPath("copy_dir.vpk"); + + using var package = new Package(); + package.Read(dirPath); + + using (Assert.Multiple()) + { + await Assert.That(package.RequiresRepack(dirPath)).IsFalse(); + await Assert.That(package.RequiresRepack(TempPath("test.vpk"))).IsFalse(); + await Assert.That(package.RequiresRepack(otherChunkNames)).IsTrue(); + await Assert.That(() => package.Write(otherChunkNames)).ThrowsExactly(); + } + + package.Write(otherChunkNames, package.RequiresRepack(otherChunkNames)); + await AssertPackageVerifies(package); + } + + [Test] + public async Task RepackIntoDirVpkMovesDirectoryFileData() + { + var path = TempPath("single.vpk"); + var dirPath = TempPath("single_dir.vpk"); + + using (var package = new Package()) + { + package.AddFile("a.bin", CreateTestData(400, 1)); + package.Write(path); + } + + using (var package = new Package()) + { + package.Read(path); + package.AddFile("b.bin", CreateTestData(300, 2)); + package.Write(dirPath, repack: true); + } + + using var readBack = new Package(); + readBack.Read(dirPath); + + await Assert.That(readBack.Entries!.Values.SelectMany(static entries => entries).All(static entry => entry.ArchiveIndex == 0)).IsTrue(); + await AssertEntryData(readBack, "a.bin", 400, 1); + await AssertEntryData(readBack, "b.bin", 300, 2); + await AssertPackageVerifies(readBack); + } + + [Test] + public async Task OpensNestedPackagesInPlace() + { + var leafData = CreateTestData(10000, 1); + var level2 = CreatePackageBytes(package => package.AddFile("data/leaf.bin", leafData)); + var level1 = CreatePackageBytes(package => + { + var entry = package.AddFile("maps/level2.vpk", level2[20..]); + entry.SmallData = level2[..20]; + entry.CRC32 = Crc32.HashToUInt32(level2); + }); + + var path = TempPath("level0_dir.vpk"); + + using (var created = new Package()) + { + created.SetFileName(path); + created.AddFile("maps/level1.vpk", level1, multiChunk: true); + created.Write(path); + } + + using var level0Package = Package.Open(path); + using var level1Package = OpenNestedPackage(level0Package, level0Package.FindEntry("maps/level1.vpk")!); + using var level2Package = OpenNestedPackage(level1Package, level1Package.FindEntry("maps/level2.vpk")!); + + var leafEntry = level2Package.FindEntry("data/leaf.bin")!; + level2Package.ReadEntry(leafEntry, out var leafRead); + + using var mapped = level2Package.GetMemoryMappedStreamIfPossible(leafEntry); + + using (Assert.Multiple()) + { + await Assert.That(leafRead).IsEquivalentTo(leafData, CollectionOrdering.Matching); + await Assert.That(mapped is UnmanagedMemoryStream).IsTrue(); + } + + level1Package.Dispose(); + + await Assert.That(() => level2Package.ReadEntry(leafEntry, out _)).ThrowsExactly(); + } + + [Test] + public async Task NestedPackagesDoNotUseChunkFiles() + { + using var innerDirectoryFile = new MemoryStream(); + + using (var inner = new Package()) + { + inner.SetFileName(TempPath("inner_dir.vpk")); + inner.AddFile("chunked.bin", CreateTestData(100, 1), multiChunk: true); + inner.AddFile("embedded.bin", CreateTestData(100, 2)); + inner.Write(innerDirectoryFile); + } + + using var outer = new Package(); + outer.AddFile("maps/inner_dir.vpk", innerDirectoryFile.ToArray()); + + using var nested = OpenNestedPackage(outer, outer.FindEntry("maps/inner_dir.vpk")!); + + using (Assert.Multiple()) + { + await Assert.That(() => nested.ReadEntry(nested.FindEntry("embedded.bin")!, out _)).ThrowsNothing(); + await Assert.That(() => nested.ReadEntry(nested.FindEntry("chunked.bin")!, out _)).ThrowsExactly(); + } + } + + private static Package OpenNestedPackage(Package outer, PackageEntry entry) + { + var package = new Package(); + package.OptimizeEntriesForBinarySearch(); + package.SetFileName(entry.GetFullPath()); + package.Read(outer.OpenEntryStream(entry)); + + return package; + } + + private static byte[] CreatePackageBytes(Action addFiles) + { + using var package = new Package(); + addFiles(package); + + using var output = new MemoryStream(); + package.Write(output); + + return output.ToArray(); + } + private string TempPath(string fileName) { return Path.Combine(TempDirectory, fileName); diff --git a/ValvePak/ValvePak.Test/PackageTest.cs b/ValvePak/ValvePak.Test/PackageTest.cs index 7c8b7d7..ce99b9a 100644 --- a/ValvePak/ValvePak.Test/PackageTest.cs +++ b/ValvePak/ValvePak.Test/PackageTest.cs @@ -2,6 +2,7 @@ using System.Buffers.Binary; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; @@ -213,14 +214,22 @@ public async Task FindEntrySpacesAndExtensionless() await Assert.That(package.FindEntry("folder with space\\space_extension. txt")?.CRC32).IsEqualTo(0x09321FC0u); await Assert.That(package.FindEntry("folder with space/file name with space.txt")?.CRC32).IsEqualTo(0x76D91432u); await Assert.That(package.FindEntry("uppercasefolder/bad_file_forfun.txt")?.CRC32).IsEqualTo(0x15C1490Fu); - await Assert.That(package.FindEntry("UpperCaseFolder/UpperCaseFile.txt")?.CRC32).IsEqualTo(0x32CFF012u); - await Assert.That(package.FindEntry("UpperCaseFolder/bad_file_forfun.txt")).IsNull(); + await Assert.That(package.FindEntry("UpperCaseFolder/bad_file_forfun.txt")?.CRC32).IsEqualTo(0x15C1490Fu); + await Assert.That(package.FindEntry("uppercasefolder/bad_file_forfun.TXT")?.CRC32).IsEqualTo(0x15C1490Fu); + await Assert.That(package.FindEntry("UpperCaseFolder/UpperCaseFile.txt")).IsNull(); await Assert.That(package.FindEntry("uppercasefolder/UpperCaseFile.txt")).IsNull(); - await Assert.That(package.FindEntry("uppercasefolder/bad_file_forfun.TXT")).IsNull(); await Assert.That(package.FindEntry("uppercasefolder/bad_file_forfun.txt2")).IsNull(); } } + /// + /// Returns the entry of broken_dir.vpk whose stored names have upper case letters, which does not find. + /// + private static PackageEntry? GetUpperCaseEntry(Package package) + { + return package.Entries!["txt"].Find(static entry => entry.FileName == "UpperCaseFile"); + } + [Test] public async Task ThrowsOnInvalidCRC32() { @@ -229,7 +238,7 @@ public async Task ThrowsOnInvalidCRC32() using var package = new Package(); package.Read(path); - var file = package.FindEntry("UpperCaseFolder/UpperCaseFile.txt"); + var file = GetUpperCaseEntry(package); await Assert.That(file).IsNotNull(); await Assert.That(file.CRC32).IsEqualTo(0x32CFF012u); @@ -264,7 +273,7 @@ public async Task TestGetFullPath() await Assert.That(package.FindEntry("folder with space/test")?.GetFullPath()).IsEqualTo("folder with space/test"); await Assert.That(package.FindEntry("folder with space\\space_extension. txt")?.GetFullPath()).IsEqualTo("folder with space/space_extension. txt"); await Assert.That(package.FindEntry("uppercasefolder/bad_file_forfun.txt")?.GetFullPath()).IsEqualTo("uppercasefolder/bad_file_forfun.txt"); - await Assert.That(package.FindEntry("UpperCaseFolder/UpperCaseFile.txt")?.GetFullPath()).IsEqualTo("UpperCaseFolder/UpperCaseFile.txt"); + await Assert.That(GetUpperCaseEntry(package)?.GetFullPath()).IsEqualTo("UpperCaseFolder/UpperCaseFile.txt"); await Assert.That(package.FindEntry("test")?.GetFileName()).IsEqualTo("test"); await Assert.That(package.FindEntry("folder with space/test")?.GetFileName()).IsEqualTo("test"); @@ -287,7 +296,7 @@ public async Task TestPackageEntryToString() await Assert.That(package.FindEntry("folder with space/test")?.ToString()).IsEqualTo("folder with space/test crc=0xbf108706 metadatasz=0 fnumber=0 ofs=0x52 sz=41"); await Assert.That(package.FindEntry("folder with space\\space_extension. txt")?.ToString()).IsEqualTo("folder with space/space_extension. txt crc=0x9321fc0 metadatasz=0 fnumber=0 ofs=0x7b sz=30"); await Assert.That(package.FindEntry("uppercasefolder/bad_file_forfun.txt")?.ToString()).IsEqualTo("uppercasefolder/bad_file_forfun.txt crc=0x15c1490f metadatasz=0 fnumber=0 ofs=0xa2 sz=2"); - await Assert.That(package.FindEntry("UpperCaseFolder/UpperCaseFile.txt")?.ToString()).IsEqualTo("UpperCaseFolder/UpperCaseFile.txt crc=0x32cff012 metadatasz=0 fnumber=0 ofs=0x27 sz=43"); + await Assert.That(GetUpperCaseEntry(package)?.ToString()).IsEqualTo("UpperCaseFolder/UpperCaseFile.txt crc=0x32cff012 metadatasz=0 fnumber=0 ofs=0x27 sz=43"); } } @@ -442,7 +451,7 @@ public async Task TestFileChecksums() package.Read(path); await Assert.That(() => package.VerifyFileChecksums()).ThrowsNothing(); - var file = package.FindEntry("UpperCaseFolder/UpperCaseFile.txt"); + var file = GetUpperCaseEntry(package); await Assert.That(file).IsNotNull(); await Assert.That(file.CRC32).IsEqualTo(0x32CFF012u); @@ -525,11 +534,12 @@ public async Task InvalidWholeFileChecksumThrows() } [Test] - [Arguments(0, "File tree")] - [Arguments(16, "Archive MD5 entries")] - public async Task SectionChecksumMismatchIsReportedWhenWholeFileChecksumMatches(int checksumOffset, string sectionName) + [Arguments(0, "File tree", false)] + [Arguments(16, "Archive MD5 entries", false)] + [Arguments(0, "File tree", true)] + public async Task SectionChecksumMismatchIsReportedOnlyWithoutSignatureHeader(int checksumOffset, string sectionName, bool hasSignatureHeader) { - var data = CreateSmallPackageBytes(); + var data = hasSignatureHeader ? CreateSmallPackageBytes() : CreateSmallPackageBytesWithoutSignatureSection(); var otherMD5SectionOffset = GetOtherMD5SectionOffset(data); data.AsSpan(otherMD5SectionOffset + checksumOffset, 16).Fill(0xAB); @@ -541,6 +551,13 @@ public async Task SectionChecksumMismatchIsReportedWhenWholeFileChecksumMatches( var progress = new SynchronousProgress(); + if (hasSignatureHeader) + { + await Assert.That(() => package.VerifyHashes(progress)).ThrowsExactly() + .WithMessageContaining($"{sectionName} checksum mismatch (", StringComparison.Ordinal); + return; + } + await Assert.That(() => package.VerifyHashes(progress)).ThrowsNothing(); await Assert.That(progress.Reports).Count().IsEqualTo(1); await Assert.That(progress.Reports[0]).StartsWith($"{sectionName} checksum mismatch (", StringComparison.Ordinal); @@ -629,6 +646,92 @@ public async Task TestVerifyChunkHashesBlake3() await Assert.That(() => package.VerifyChunkHashes(null)).ThrowsNothing(); } + [Test] + [Arguments("cs2_new_signature_actually_signed.vpk", "Verified the package signature.", 0)] + [Arguments("test_single.vpk", "The package is not signed.", 0)] + [Arguments("test_dir.vpk", "The package is not signed.", 3)] + [Arguments("broken_dir.vpk", "The package is not signed.", 6)] + public async Task VerifyReportsStepsAndChecksCrcOfUnhashedEntries(string fileName, string signatureReport, int crcReports) + { + var path = Path.Combine(AppContext.BaseDirectory, "Files", fileName); + + using var package = new Package(); + package.Read(path); + + var reports = new List<(int Step, int StepCount, string Message)>(); + package.Verify(new SynchronousProgress<(int Step, int StepCount, string Message)>(reports.Add)); + + using (Assert.Multiple()) + { + await Assert.That(reports.Select(static report => report.Step)).IsEquivalentTo(Enumerable.Range(1, reports.Count), CollectionOrdering.Matching); + await Assert.That(reports.All(report => report.StepCount == reports.Count)).IsTrue(); + await Assert.That(reports[0].Message).IsEqualTo(signatureReport); + await Assert.That(reports.Count(static report => report.Message.StartsWith("Verifying CRC32", StringComparison.Ordinal))).IsEqualTo(crcReports); + await Assert.That(reports[^1].Message).IsEqualTo("Verified the package."); + } + } + + [Test] + public async Task VerifyThrowsOnCrcMismatchOfUnhashedEntry() + { + using var package = new Package(); + package.Read(Path.Combine(AppContext.BaseDirectory, "Files", "test_dir.vpk")); + package.FindEntry("kitten.jpg")!.CRC32 = 0xDEADBEEF; + + await Assert.That(() => package.Verify()).ThrowsExactly(); + } + + [Test] + public async Task BuildsAndParsesChunkFilePaths() + { + using var package = new Package(); + package.Read(Path.Combine(AppContext.BaseDirectory, "Files", "test_dir.vpk")); + + using (Assert.Multiple()) + { + await Assert.That(package.GetChunkFilePath(1)).IsEqualTo(Path.Combine(AppContext.BaseDirectory, "Files", "test_001.vpk")); + await Assert.That(Package.TryParseChunkFilePath("a_b_042.VPK", out var baseName, out var archiveIndex)).IsTrue(); + await Assert.That(baseName).IsEqualTo("a_b"); + await Assert.That(archiveIndex).IsEqualTo((ushort)42); + await Assert.That(Package.TryParseChunkFilePath("a_dir.vpk", out _, out _)).IsFalse(); + } + } + + [Test] + public async Task HasChunkFilesWithoutDirSuffix() + { + using var package = new Package(); + package.Read(Path.Combine(AppContext.BaseDirectory, "Files", "test_without_suffix.vpk")); + + using (Assert.Multiple()) + { + await Assert.That(package.HasChunkFiles).IsTrue(); + await Assert.That(package.IsDirVPK).IsFalse(); + } + } + + [Test] + [Arguments("test_dir.vpk", (ushort)0)] + [Arguments("test_single.vpk", Package.DirectoryArchiveIndex)] + public async Task OpensArchiveStreamsAtEntryOffsets(string fileName, ushort archiveIndex) + { + using var package = new Package(); + package.Read(Path.Combine(AppContext.BaseDirectory, "Files", fileName)); + + using var stream = package.OpenArchiveStream(archiveIndex); + + foreach (var entry in package.Entries!.Values.SelectMany(static entries => entries).Where(entry => entry.ArchiveIndex == archiveIndex)) + { + package.ReadEntry(entry, out var expected); + var data = new byte[entry.Length]; + + stream.Position = entry.Offset; + stream.ReadExactly(data); + + await Assert.That(data).IsEquivalentTo(expected[entry.SmallData.Length..], CollectionOrdering.Matching); + } + } + [Test] public async Task SetFileNameStripsVpkExtensionAndDirSuffix() { @@ -709,6 +812,11 @@ private sealed class SynchronousProgress : IProgress public void Report(string value) => Reports.Add(value); } + private sealed class SynchronousProgress(Action report) : IProgress + { + public void Report(T value) => report(value); + } + [Test] public async Task DisposeCanBeCalledMultipleTimes() { @@ -929,6 +1037,14 @@ private static byte[] CreateSmallPackageBytes() return output.ToArray(); } + private static byte[] CreateSmallPackageBytesWithoutSignatureSection() + { + var package = CreateSmallPackageBytes()[..^UnsignedSignatureSectionSize]; + BinaryPrimitives.WriteUInt32LittleEndian(package.AsSpan(SignatureSectionSizeOffset), 0); + + return package; + } + private static int GetOtherMD5SectionOffset(byte[] package) { var span = package.AsSpan(); diff --git a/ValvePak/ValvePak.Test/WriteTest.cs b/ValvePak/ValvePak.Test/WriteTest.cs index c3a7d33..25fe62d 100644 --- a/ValvePak/ValvePak.Test/WriteTest.cs +++ b/ValvePak/ValvePak.Test/WriteTest.cs @@ -168,51 +168,36 @@ public async Task NormalizesPathsTheSameWhenAddingAndFinding(StringComparison co using var package = new Package(); package.OptimizeEntriesForBinarySearch(comparison); - var entry = package.AddFile("\\Materials//./Models/./Crate.VMAT_C/", Encoding.UTF8.GetBytes("crate")); - var noExtension = package.AddFile("./readme.", []); - var trailingDots = package.AddFile("notes..", [1]); - var trailingDotAfterExtension = package.AddFile("archive.tar.", [2]); + var entry = package.AddFile("\\Materials//./Models/../Models/Crate.VMAT_C", Encoding.UTF8.GetBytes("crate")); + var noExtension = package.AddFile("./Docs/ReadMe", []); using (Assert.Multiple()) { await Assert.That(entry.DirectoryName).IsEqualTo("materials/models"); await Assert.That(entry.FileName).IsEqualTo("crate"); await Assert.That(entry.TypeName).IsEqualTo("vmat_c"); - await Assert.That(noExtension.DirectoryName).IsEqualTo(" "); await Assert.That(noExtension.TypeName).IsEqualTo(" "); - await Assert.That(package.FindEntry("materials/models/crate.vmat_c")).IsEqualTo(entry); - await Assert.That(package.FindEntry("/materials\\models//crate.vmat_c")).IsEqualTo(entry); - await Assert.That(package.FindEntry("./materials/./models/crate.vmat_c/".AsSpan())).IsEqualTo(entry); - await Assert.That(package.FindEntry("readme")).IsEqualTo(noExtension); - await Assert.That(package.FindEntry("/readme.")).IsEqualTo(noExtension); - await Assert.That(trailingDots.FileName).IsEqualTo("notes"); - await Assert.That(trailingDots.TypeName).IsEqualTo(" "); - await Assert.That(trailingDotAfterExtension.FileName).IsEqualTo("archive"); - await Assert.That(trailingDotAfterExtension.TypeName).IsEqualTo("tar"); - - foreach (var added in new[] { entry, noExtension, trailingDots, trailingDotAfterExtension }) - { - await Assert.That(package.FindEntry(added.GetFullPath())).IsEqualTo(added); - } + await Assert.That(package.FindEntry("/MATERIALS\\models//crate.vmat_c")).IsEqualTo(entry); + await Assert.That(package.FindEntry("./materials/./textures/../models/crate.vmat_c".AsSpan())).IsEqualTo(entry); + await Assert.That(package.FindEntry("DOCS/README")).IsEqualTo(noExtension); + await Assert.That(package.FindEntry("materials/models/crate.vmat_c/")).IsNull(); + await Assert.That(package.FindEntry("../materials/models/crate.vmat_c")).IsNull(); } } [Test] [Arguments(false)] [Arguments(true)] - public async Task FindsReadEntriesWhoseNamesAreNotNormalized(bool optimize) + public async Task DoesNotFindReadEntriesWhoseNamesAreNotNormalized(bool optimize) { (string Directory, string File, string Type)[] names = [ ("a//b", "file", "txt"), - ("a/./b", "file", "txt"), - ("./rel", "file", "txt"), - ("/leading", "file", "txt"), - ("trailing/", "file", "txt"), + ("a/../b", "file", "txt"), ("back\\slash", "file", "txt"), (" ", "has.dot", " "), - ("dir", "noext.", " "), (" ", "archive", "tar.gz"), + ("Upper", "File", "TXT"), ("dir", "archive.tar", "gz"), ]; @@ -241,18 +226,22 @@ public async Task FindsReadEntriesWhoseNamesAreNotNormalized(bool optimize) readBack.SetFileName("test.vpk"); readBack.Read(output); - var entries = readBack.Entries!.Values.SelectMany(static entries => entries).ToList(); + var entries = readBack.Entries!.Values.SelectMany(static entries => entries).ToDictionary(static entry => entry.GetFullPath()); await Assert.That(entries).Count().IsEqualTo(names.Length); - foreach (var entry in entries) - { - await Assert.That(readBack.FindEntry(entry.GetFullPath())).IsSameReferenceAs(entry); - } - using (Assert.Multiple()) { - await Assert.That(readBack.FindEntry("dir/noext")).IsNull(); - await Assert.That(readBack.FindEntry("has")).IsNull(); + foreach (var (path, entry) in entries) + { + var expected = path switch + { + "dir/archive.tar.gz" => entry, + "Upper/File.TXT" when optimize => entry, + _ => null, + }; + + await Assert.That(readBack.FindEntry(path)).IsSameReferenceAs(expected); + } } } @@ -340,7 +329,7 @@ public async Task WritesUnsignedSignatureSectionAndHashesEmbeddedData(int dataLe using (Assert.Multiple()) { await Assert.That(emptyEntry.ArchiveIndex).IsEqualTo((ushort)0x7FFF); - await Assert.That(emptyEntry.Offset).IsEqualTo(uint.MaxValue); + await Assert.That(emptyEntry.Offset).IsEqualTo((uint)dataLength); await Assert.That(emptyEntry.Length).IsZero(); await Assert.That(readBack.ArchiveMD5SectionSize).IsEqualTo((uint)(fractionCount * ChunkHashFraction.SectionEntrySize)); await Assert.That(signatureSection.Length).IsEqualTo(20); @@ -667,17 +656,7 @@ public async Task AddedFilesCanBeFoundAfterOptimizingForBinarySearch(StringCompa await Assert.That(package.FindEntry(entry.GetFullPath())).IsEqualTo(entry); } - var upperCaseEntry = package.FindEntry("ADDONS/CHESS/ZZZ.VDF"); - - if (comparison == StringComparison.OrdinalIgnoreCase) - { - await Assert.That(upperCaseEntry).IsEqualTo(addedEntries[0]); - } - else - { - await Assert.That(upperCaseEntry).IsNull(); - } - + await Assert.That(package.FindEntry("ADDONS/CHESS/ZZZ.VDF")).IsEqualTo(addedEntries[0]); await Assert.That(package.FindEntry("addons/chess/missing.vdf")).IsNull(); } @@ -703,6 +682,24 @@ public async Task RenameFile() } } + [Test] + public async Task RenamesAndRemovesDirectories() + { + using var package = new Package(); + var file = package.AddFile("materials/a.txt", [1]); + var nested = package.AddFile("materials/sub/b.txt", [2]); + var sibling = package.AddFile("materialsx/c.txt", [3]); + + using (Assert.Multiple()) + { + await Assert.That(package.RenameDirectory("materials/", "Renamed")).IsEquivalentTo([file, nested], CollectionOrdering.Any); + await Assert.That(nested.GetFullPath()).IsEqualTo("renamed/sub/b.txt"); + await Assert.That(sibling.GetFullPath()).IsEqualTo("materialsx/c.txt"); + await Assert.That(package.RemoveDirectory("renamed")).IsEquivalentTo([file, nested], CollectionOrdering.Any); + await Assert.That(package.Entries!["txt"]).IsEquivalentTo([sibling], CollectionOrdering.Matching); + } + } + private static Package ReadBack(Stream stream) { var package = new Package(); diff --git a/ValvePak/ValvePak/Archive.cs b/ValvePak/ValvePak/Archive.cs index 861447a..8136bf3 100644 --- a/ValvePak/ValvePak/Archive.cs +++ b/ValvePak/ValvePak/Archive.cs @@ -25,6 +25,11 @@ protected Archive(long origin, long dataOffset) /// protected long Origin { get; } + /// + /// Gets the current length of the archive from its start. + /// + public abstract long Length { get; } + /// /// Reads data at the given offset until the output is full or the end of the archive is reached. /// This method is safe to call from multiple threads. diff --git a/ValvePak/ValvePak/FileArchive.cs b/ValvePak/ValvePak/FileArchive.cs index 5a7668f..9196f4c 100644 --- a/ValvePak/ValvePak/FileArchive.cs +++ b/ValvePak/ValvePak/FileArchive.cs @@ -25,6 +25,8 @@ public FileArchive(SafeFileHandle handle, bool ownsHandle, long origin, long dat OwnsHandle = ownsHandle; } + public override long Length => RandomAccess.GetLength(Handle) - Origin; + public override int Read(long offset, Span output) { var totalRead = 0; diff --git a/ValvePak/ValvePak/Package.Read.cs b/ValvePak/ValvePak/Package.Read.cs index c20d6b1..052a1ec 100644 --- a/ValvePak/ValvePak/Package.Read.cs +++ b/ValvePak/ValvePak/Package.Read.cs @@ -3,6 +3,8 @@ using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; using System.IO; using System.IO.Hashing; using System.Linq; @@ -66,12 +68,64 @@ public void Read(Stream input) .Where(static archiveIndex => archiveIndex < DirectoryArchiveIndex) .Max(static archiveIndex => (int?)archiveIndex) ?? -1; - if (HighestChunkIndex >= 0) + if (HighestChunkIndex >= 0 && !IsReadFromAnotherPackage) { ChunkBaseName = Path.GetFullPath(FileName); } } + /// + /// Opens and reads the package at the given path, with its entries sorted for binary search, see . + /// + /// Path of the directory file. + /// Comparison that compares names with. + /// The read package, which holds the file open until it is disposed. + public static Package Open(string path, StringComparison comparison = StringComparison.Ordinal) + { + ArgumentNullException.ThrowIfNull(path); + + var package = new Package(); + + try + { + package.OptimizeEntriesForBinarySearch(comparison); + package.Read(path); + } + catch + { + package.Dispose(); + throw; + } + + return package; + } + + /// + /// Returns a read-only seekable stream over the data of the entry including its preload bytes, which reads in place from the archive the entry is in + /// unless its data is held in memory. Passing it directly to reads a package stored in the entry in place, + /// and that package throws when reading its entries in chunk files. + /// Like streams from , it can only be read while the package is open and has not replaced the file it reads from. + /// + /// Package entry. + /// Stream over the entry contents. + /// The chunk file of the entry does not exist. + public Stream OpenEntryStream(PackageEntry entry) + { + ArgumentNullException.ThrowIfNull(entry); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + if (entry.PendingData != null || entry.Length == 0) + { + ReadEntry(entry, out var data, validateCrc: false); + + return new PackageDataStream(data); + } + + var archive = GetArchive(entry.ArchiveIndex); + + return new PackageDataStream(archive, archive.DataOffset + entry.Offset, entry.Length, entry.SmallData); + } + /// /// Opens the directory file at the given path instead of the one read before, keeping the current . /// @@ -79,6 +133,7 @@ private void ReopenDirectoryFile(string path) { CloseDirectoryFile(); ReadFilePath = path; + IsReadFromAnotherPackage = false; var input = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); @@ -100,6 +155,7 @@ private void ReadHeader(Stream input) { Reader = new BinaryReader(input); ReadFilePath = (input as FileStream)?.Name; + IsReadFromAnotherPackage = input is PackageDataStream; if (Reader.ReadUInt32() != MAGIC) { @@ -139,14 +195,12 @@ private void ReadHeader(Stream input) var embeddedDataOffset = (long)HeaderSize + TreeSize; // Taking the handle flushes the stream's buffer, so it is done here before any other thread can read from the stream - if (input is FileStream fileStream) + Archives[DirectoryArchiveIndex] = input switch { - Archives[DirectoryArchiveIndex] = new FileArchive(fileStream.SafeFileHandle, ownsHandle: false, StreamOffset, embeddedDataOffset); - } - else - { - Archives[DirectoryArchiveIndex] = new StreamArchive(input, StreamOffset, embeddedDataOffset); - } + FileStream fileStream => new FileArchive(fileStream.SafeFileHandle, ownsHandle: false, StreamOffset, embeddedDataOffset), + PackageDataStream dataStream => new PackageDataStreamArchive(dataStream, StreamOffset, embeddedDataOffset), + _ => new StreamArchive(input, StreamOffset, embeddedDataOffset), + }; } private void ReadSections() @@ -573,7 +627,8 @@ private void CloseDirectoryFile() /// Files smaller or equal to 4096 bytes will be read into memory. /// /// When trying to map a file in a non-split packages (), - /// this will only memory map if the package was read by providing or using . + /// this will only memory map if the package was read by providing or using , + /// or was read from of such a package. /// /// Each archive is mapped once. /// Returned streams can only be read while the package is open, they throw @@ -604,6 +659,90 @@ public Stream GetMemoryMappedStreamIfPossible(PackageEntry entry) return new MemoryStream(output); } + /// + /// Returns a read-only stream over a whole chunk file, or for over the file data in the directory file, at positions. + /// Like streams from , it can only be read while the package is open and has not replaced the file it reads from. + /// + /// Index of the chunk file, or . + /// The chunk file does not exist. + public Stream OpenArchiveStream(ushort archiveIndex) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan(archiveIndex, DirectoryArchiveIndex); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var archive = GetArchive(archiveIndex); + var length = Math.Max(0, archive.Length - archive.DataOffset); + + if (archiveIndex == DirectoryArchiveIndex && Version != 1) + { + length = Math.Min(length, FileDataSectionSize); + } + + return archive.OpenMappedStream(archive.DataOffset, length) ?? new PackageDataStream(archive, archive.DataOffset, length); + } + + /// + /// Returns the full path of a chunk file of this package, which does not have to exist, such as "pak01_003.vpk" for "pak01_dir.vpk". + /// + /// Index of the chunk file, up to . + /// The package has no file name. + public string GetChunkFilePath(ushort archiveIndex) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan(archiveIndex, MaxChunkIndex); + + return GetArchiveIndexFullFilePath(GetChunkBaseName(), archiveIndex); + } + + /// + /// Parses a path in the form that returns: a base name, an underscore, + /// three digits or four to five digits without a leading zero, and ".vpk". + /// + /// The path to parse. + /// The path without the underscore, the index and ".vpk". + /// The index of the chunk file. + /// True if the path is in the form of a chunk file path. + public static bool TryParseChunkFilePath(string path, [NotNullWhen(true)] out string? baseName, out ushort archiveIndex) + { + ArgumentNullException.ThrowIfNull(path); + + const string Extension = ".vpk"; + + baseName = null; + archiveIndex = 0; + + if (!path.EndsWith(Extension, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var name = path.AsSpan(0, path.Length - Extension.Length); + var separator = name.LastIndexOf('_'); + + if (separator < 1) + { + return false; + } + + var digits = name[(separator + 1)..]; + + if (digits.Length is < 3 or > 5 || digits.ContainsAnyExceptInRange('0', '9') || (digits.Length > 3 && digits[0] == '0')) + { + return false; + } + + var index = int.Parse(digits, NumberStyles.None, CultureInfo.InvariantCulture); + + if (index > MaxChunkIndex) + { + return false; + } + + baseName = path[..separator]; + archiveIndex = (ushort)index; + + return true; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string GetArchiveIndexFullFilePath(string? baseFileName, ushort archiveIndex) { diff --git a/ValvePak/ValvePak/Package.Save.cs b/ValvePak/ValvePak/Package.Save.cs index cb8add9..6e420a0 100644 --- a/ValvePak/ValvePak/Package.Save.cs +++ b/ValvePak/ValvePak/Package.Save.cs @@ -90,11 +90,11 @@ public bool RemoveFile(PackageEntry entry) /// /// Moves an entry in current package to a new path. Only the directory tree changes, the file data is not touched. /// Be careful to not move it onto the path of another entry, because this does not check for duplicates. - /// The path is normalized and lowercased like Valve's packer does, see . + /// The path is normalized like does. /// /// The package entry to rename. /// Full new file path for this entry. - /// The entry is not in this package, or the new path has an empty file name or contains a null character. + /// The entry is not in this package, or the new path is rejected, see . public void RenameFile(PackageEntry entry, string newFilePath) { ArgumentNullException.ThrowIfNull(entry); @@ -113,9 +113,104 @@ public void RenameFile(PackageEntry entry, string newFilePath) InsertEntry(entry); } + /// + /// Removes every entry in a directory and its subdirectories from current package. + /// + /// The directory as entries store it in , with or without a trailing '/'. + /// The removed entries. + /// is the root. + public IReadOnlyList RemoveDirectory(string directory) + { + ArgumentNullException.ThrowIfNull(directory); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var removed = GetEntriesInDirectory(TrimStoredDirectory(directory, nameof(directory))); + + RemoveEntries(removed); + + return removed; + } + + /// + /// Moves every entry in a directory and its subdirectories into another directory. Only the directory tree changes, the file data is not touched. + /// Be careful to not move entries onto the paths of other entries, because this does not check for duplicates. + /// + /// The directory as entries store it, see . + /// The directory to move the entries into, normalized like does. + /// The moved entries. + /// or is the root. + public IReadOnlyList RenameDirectory(string directory, string newDirectory) + { + ArgumentNullException.ThrowIfNull(directory); + ArgumentNullException.ThrowIfNull(newDirectory); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var storedDirectory = TrimStoredDirectory(directory, nameof(directory)); + var targetDirectory = NormalizeDirectoryPath(newDirectory); + + if (targetDirectory.Length == 0) + { + throw new ArgumentException($"'{newDirectory}' is the root.", nameof(newDirectory)); + } + + var moved = GetEntriesInDirectory(storedDirectory); + + RemoveEntries(moved); + + foreach (var entry in moved) + { + entry.DirectoryName = string.Concat(targetDirectory, entry.DirectoryName.AsSpan(storedDirectory.Length)); + + InsertEntry(entry); + } + + return moved; + } + + private static string TrimStoredDirectory(string directory, string paramName) + { + if (directory.EndsWith(DirectorySeparatorChar)) + { + directory = directory[..^1]; + } + + if (directory is "" or Space) + { + throw new ArgumentException("The root directory can not be renamed or removed.", paramName); + } + + return directory; + } + + /// + /// Returns the entries whose stored directory is the given directory or one of its subdirectories, with ASCII letters case insensitive. + /// + private List GetEntriesInDirectory(string directory) + { + return [.. GetAllEntries().Where(entry => entry.DirectoryName.Length >= directory.Length + && (entry.DirectoryName.Length == directory.Length || entry.DirectoryName[directory.Length] == DirectorySeparatorChar) + && AsciiComparer.Equals(entry.DirectoryName.AsSpan(0, directory.Length), directory))]; + } + + private void RemoveEntries(List entries) + { + var removed = entries.ToHashSet(); + + foreach (var typeName in removed.Select(static entry => entry.TypeName).Distinct()) + { + var typeEntries = Entries![typeName]; + typeEntries.RemoveAll(removed.Contains); + + if (typeEntries.Count == 0) + { + Entries.Remove(typeName); + } + } + } + /// /// Add file to current package. Be careful to not add duplicate entries, because this does not check for duplicates. - /// The path is normalized and lowercased like Valve's packer does, see . + /// The path is normalized like does. /// /// Full file path for this entry. /// File data for this entry. @@ -125,7 +220,7 @@ public void RenameFile(PackageEntry entry, string newFilePath) /// A new chunk file is started whenever the current one is larger than . /// Chunk files stay on disk even if the directory file is never written. /// The added entry. - /// The path has an empty file name, or contains a null character. + /// The path is rejected, see . public PackageEntry AddFile(string filePath, byte[] fileData, bool multiChunk = false) { ArgumentNullException.ThrowIfNull(filePath); @@ -150,7 +245,7 @@ public PackageEntry AddFile(string filePath, byte[] fileData, bool multiChunk = /// /// Add file to current package, reading its data from the current position of the stream to its end. /// Be careful to not add duplicate entries, because this does not check for duplicates. - /// The path is normalized and lowercased like Valve's packer does, see . + /// The path is normalized like does. /// /// Full file path for this entry. /// Stream to read the file data from. It is not disposed. @@ -160,7 +255,7 @@ public PackageEntry AddFile(string filePath, byte[] fileData, bool multiChunk = /// A new chunk file is started whenever the current one is larger than . /// Chunk files stay on disk even if the directory file is never written. /// The added entry. - /// The path has an empty file name, or contains a null character. + /// The path is rejected, see . public PackageEntry AddFile(string filePath, Stream fileData, bool multiChunk = false) { ArgumentNullException.ThrowIfNull(filePath); @@ -176,9 +271,9 @@ public PackageEntry AddFile(string filePath, Stream fileData, bool multiChunk = var knownLength = fileData.CanSeek ? fileData.Length - fileData.Position : 0; - if (knownLength > uint.MaxValue) + if (knownLength > MaxChunkFileSize) { - throw new ArgumentException($"The stream has {knownLength} bytes left, a chunk file can hold at most {uint.MaxValue}.", nameof(fileData)); + throw new ArgumentException($"The stream has {knownLength} bytes left, a chunk file can hold at most {MaxChunkFileSize}.", nameof(fileData)); } var buffer = ArrayPool.Shared.Rent(CopyBufferSize); @@ -306,12 +401,12 @@ private ChunkFileWriter GetChunkFileForNextFile(long length) /// Size in bytes of the next file, or 0 if it is not known. private bool HasRoomForNextFile([NotNullWhen(true)] ChunkFileWriter? chunkFile, long length) { - return chunkFile != null && chunkFile.Length <= (uint)WriteChunkSize && chunkFile.Length + length <= uint.MaxValue; + return chunkFile != null && chunkFile.Length <= (uint)WriteChunkSize && chunkFile.Length + length <= MaxChunkFileSize; } private static void ThrowIfChunkLimitReached(int archiveIndex) { - if (archiveIndex >= DirectoryArchiveIndex) + if (archiveIndex > MaxChunkIndex) { throw new InvalidOperationException("Reached the maximum amount of chunk files (32767)."); } @@ -331,6 +426,11 @@ private void FinishCurrentChunkFile() private string GetChunkBaseName() { + if (IsReadFromAnotherPackage) + { + throw new NotSupportedException("Chunk files of a package read from an entry of another package are not supported."); + } + if (FileName == null) { throw new InvalidOperationException("Call SetFileName() with the path of the directory file before adding files with multi chunk enabled, chunk files are created next to it."); @@ -355,39 +455,18 @@ private static PackageEntry CreateEntry(string filePath, string paramName) } /// - /// Splits the given path into lowercased directory, file and type names. + /// Splits a file path into the directory, file and type names that stores it with, normalized like does. /// private static (string DirectoryName, string FileName, string TypeName) ParseEntryPath(string filePath, string paramName) { - if (filePath.Contains('\0', StringComparison.Ordinal)) - { - throw new ArgumentException($"'{filePath}' contains a null character.", paramName); - } - - SplitPath(NormalizePath(filePath, out _), out var directory, out var fileName, out var extension); - - if (fileName.IsEmpty) - { - throw new ArgumentException($"'{filePath}' has an empty file name.", paramName); - } - - return (ToLowerAscii(directory), ToLowerAscii(fileName), ToLowerAscii(extension)); - } + Span buffer = filePath.Length <= MaxStackPathLength ? stackalloc char[MaxStackPathLength] : new char[filePath.Length]; - private static string ToLowerAscii(ReadOnlySpan value) - { - if (!value.ContainsAnyInRange('A', 'Z')) + if (GetEntryPathError(filePath, buffer, out var directory, out var fileName, out var typeName) is { } error) { - return value.ToString(); + throw new ArgumentException($"'{filePath}' {error}.", paramName); } - return string.Create(value.Length, value, static (span, source) => - { - for (var i = 0; i < source.Length; i++) - { - span[i] = ToLowerAscii(source[i]); - } - }); + return (directory.ToString(), fileName.ToString(), typeName.ToString()); } private static char ToLowerAscii(char c) @@ -448,7 +527,8 @@ private static bool IsInDirectoryFile(PackageEntry entry) /// If the package has chunk files, from being read or from files added with multi chunk enabled (see ), /// this only writes the directory file, which refers to the data where it is in the chunk files. /// Unless is enabled, it must be written to the path the chunk files are named after, which should end with "_dir.vpk". - /// Hashes of chunk files are kept if they cover the whole file, computed again otherwise, and left out for chunk files that had none. + /// A package that was not read also deletes existing chunk files numbered after the ones it wrote, until a number has no file. + /// Hashes of chunk files are kept if they cover the whole file, and computed otherwise. /// /// Writing over the file the package was read from, or repacking, writes temporary files that replace the existing files once all were written, /// and the package reads from the written files afterwards. This must not run concurrently with reads from this package, @@ -459,10 +539,14 @@ private static bool IsInDirectoryFile(PackageEntry entry) /// /// The file to open and write. /// If true, the data in chunk files is copied into new chunk files named after in directory tree order, + /// or for a package without chunk files written to a "_dir.vpk", the data in the directory file, /// which drops the data of removed and replaced files. Either all existing files are replaced or none are. /// If the package has chunk files with that name and was read from a file, it must be written to that file. /// If new chunk files were written or the package had chunk files with that name, existing chunk files numbered after the new ones are then deleted where possible /// until a number has no file, and in the latter case also up to the highest number the package had. + /// is false and is true, + /// is true and the package has chunk files named after but was read from another file, + /// or is the file the package was read from inside a larger file. public void Write(string filename, bool repack = false) { ArgumentNullException.ThrowIfNull(filename); @@ -477,7 +561,7 @@ public void Write(string filename, bool repack = false) } var baseName = StripDirVpkSuffixes(fullPath, out var isDirVpk); - var writesOverChunkFiles = HighestChunkIndex >= 0 && PathsEqual(baseName, GetChunkBaseName()); + var writesOverChunkFiles = WritesOverChunkFiles(baseName); // Readers find chunk files by appending "_###.vpk" to the directory file name without the "_dir.vpk" suffix if (!repack && HighestChunkIndex >= 0 && !writesOverChunkFiles) @@ -512,6 +596,11 @@ public void Write(string filename, bool repack = false) output.SetLength(output.Position); + if (!IsReadPackage && writesOverChunkFiles) + { + DeleteStaleChunkFiles(baseName, HighestChunkIndex + 1, staleHighestIndex: -1); + } + return; } @@ -526,7 +615,7 @@ public void Write(string filename, bool repack = false) if (repack) { - (chunkLocations, chunkHashFractions) = WriteRepackedChunkFiles(entries, baseName, token, chunkReplacements); + (chunkLocations, chunkHashFractions) = WriteRepackedChunkFiles(entries, baseName, isDirVpk && HighestChunkIndex < 0, token, chunkReplacements); } else { @@ -581,6 +670,26 @@ public void Write(string filename, bool repack = false) } } + /// + /// Returns whether has to repack to write the directory file to the given path, because the package has chunk files named after another path. + /// + /// The file the directory file is going to be written to. + public bool RequiresRepack(string filename) + { + ArgumentNullException.ThrowIfNull(filename); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + return HighestChunkIndex >= 0 && !WritesOverChunkFiles(StripDirVpkSuffixes(Path.GetFullPath(filename), out _)); + } + + /// + /// Returns whether the package has chunk files and they are named after the given base name. + /// + private bool WritesOverChunkFiles(string baseName) + { + return HighestChunkIndex >= 0 && PathsEqual(baseName, GetChunkBaseName()); + } + private static string GetTemporaryPath(string path, string token) { return $"{path}.{token}.tmp"; @@ -595,6 +704,7 @@ private void ApplyRepackedLayout(Dictionary chunkLo { entry.ArchiveIndex = archiveIndex; entry.Offset = offset; + entry.PendingData = null; } FileName = baseName; @@ -660,11 +770,11 @@ private List ReplaceFilesOrReopen(List chunkReplacement } /// - /// Copies the data of all entries in chunk files into new chunk files under temporary names, adding each of them to . + /// Copies the data of all entries in chunk files, and optionally in the directory file, into new chunk files under temporary names, adding each of them to . /// /// Where the data of each copied entry was written to, and the hashes of the new chunk files or null if there are none. private (Dictionary Locations, List? HashFractions) WriteRepackedChunkFiles( - List entries, string baseName, string token, List replacements) + List entries, string baseName, bool includesDirectoryFileData, string token, List replacements) { var locations = new Dictionary(); var hashFractions = new List(); @@ -674,7 +784,7 @@ private List ReplaceFilesOrReopen(List chunkReplacement { foreach (var entry in entries) { - if (!IsInChunkFile(entry)) + if (!IsInChunkFile(entry) && !(includesDirectoryFileData && IsInDirectoryFile(entry))) { continue; } @@ -699,7 +809,15 @@ private List ReplaceFilesOrReopen(List chunkReplacement } locations[entry] = new ChunkLocation(chunkFile.ArchiveIndex, chunkFile.Length); - ReadArchiveDataInBlocks(entry.ArchiveIndex, entry.Offset, entry.Length, chunkFile.Write); + + if (entry.PendingData != null) + { + chunkFile.Write(entry.PendingData); + } + else + { + ReadArchiveDataInBlocks(entry.ArchiveIndex, entry.Offset, entry.Length, chunkFile.Write); + } } if (chunkFile == null) @@ -895,21 +1013,9 @@ private static int CompareTreeOrder(PackageEntry x, PackageEntry y) /// private static int CompareNames(string x, string y) { - var length = Math.Min(x.Length, y.Length); - - for (var i = 0; i < length; i++) - { - var comparison = ToLowerAscii(x[i]).CompareTo(ToLowerAscii(y[i])); + var comparison = AsciiComparer.Compare(x, y); - if (comparison != 0) - { - return comparison; - } - } - - var lengthComparison = x.Length.CompareTo(y.Length); - - return lengthComparison != 0 ? lengthComparison : string.CompareOrdinal(x, y); + return comparison != 0 ? comparison : string.CompareOrdinal(x, y); } private static string GetTreeDirectoryName(PackageEntry entry) @@ -954,7 +1060,7 @@ void WriteNullTerminated(string value) if (entry.Length == 0) { archiveIndex = DirectoryArchiveIndex; - fileOffset = uint.MaxValue; + fileOffset = embeddedDataOffset; } else if (chunkLocations != null && chunkLocations.TryGetValue(entry, out var location)) { @@ -962,7 +1068,7 @@ void WriteNullTerminated(string value) } else if (archiveIndex == DirectoryArchiveIndex) { - if ((ulong)embeddedDataOffset + entry.Length > int.MaxValue) + if ((ulong)embeddedDataOffset + entry.Length > MaxDirectoryFileDataSize) { throw new InvalidOperationException("Package contents exceed 2GiB, add files with multi chunk enabled to split them into chunk files."); } @@ -972,9 +1078,9 @@ void WriteNullTerminated(string value) embeddedEntries.Add(entry); } - if (entry.SmallData.Length > ushort.MaxValue) + if (entry.SmallData.Length > MaxPreloadSize) { - throw new InvalidOperationException($"Entry '{entry.GetFullPath()}' has {entry.SmallData.Length} preload bytes, the directory tree can store at most {ushort.MaxValue}."); + throw new InvalidOperationException($"Entry '{entry.GetFullPath()}' has {entry.SmallData.Length} preload bytes, the directory tree can store at most {MaxPreloadSize}."); } WriteNullTerminated(entry.FileName); @@ -1121,12 +1227,12 @@ private static bool PathsEqual(string path1, string path2) if (!UpdatedChunkHashFractions.TryGetValue(archiveIndex, out var fractions) && !readFractions.TryGetValue(archiveIndex, out fractions)) { - continue; + fractions = []; } if (!CoversFile(fractions, file.Length)) { - if (file.Length > uint.MaxValue) + if (file.Length > MaxChunkFileSize) { throw new InvalidOperationException($"Chunk file {archiveIndex} is {file.Length} bytes, its hashes can only be computed for chunk files smaller than 4 GiB."); } @@ -1303,7 +1409,7 @@ public ChunkHasher(ushort archiveIndex) public void Append(ReadOnlySpan data) { // Offsets and lengths in the directory tree are 32-bit - if ((long)Length + data.Length > uint.MaxValue) + if ((long)Length + data.Length > MaxChunkFileSize) { throw new InvalidOperationException($"Chunk file {ArchiveIndex} would exceed 4 GiB."); } diff --git a/ValvePak/ValvePak/Package.Verify.cs b/ValvePak/ValvePak/Package.Verify.cs index 1745009..aacc60b 100644 --- a/ValvePak/ValvePak/Package.Verify.cs +++ b/ValvePak/ValvePak/Package.Verify.cs @@ -1,8 +1,11 @@ using System; using System.Buffers; +using System.Collections.Generic; using System.IO; +using System.IO.Hashing; using System.Linq; using System.Security.Cryptography; +using System.Threading; namespace ValvePak { @@ -31,18 +34,151 @@ public partial class Package : IDisposable /// private uint FileSizeBeforeSignature; + /// + /// Verifies the signature, the checksums, the chunk hashes, and the CRC32 checksum of each entry whose data no checksum or chunk hash covers. + /// + /// If provided, will report each step with its number starting at 1, the number of steps, and a message. + /// A token to cancel the verification. + /// The package has not been read. + /// The signature is not valid, or a checksum, chunk hash or CRC32 checksum does not match. + public void Verify(IProgress<(int Step, int StepCount, string Message)>? progressReporter = null, CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(IsDisposed, this); + + ThrowIfDirectoryFileClosed(); + + var entriesWithUnhashedData = GetEntriesWithUnhashedData(); + var stepProgress = progressReporter == null + ? null + : new StepProgress(progressReporter, 3 + AccessPackFileHashes.Count + entriesWithUnhashedData.Count); + + if (IsSigned && !IsSignatureValid(cancellationToken)) + { + throw new InvalidDataException("The package signature is not valid."); + } + + stepProgress?.Report(IsSigned ? "Verified the package signature." : "The package is not signed."); + + var checksumMessages = VerifyChecksums(cancellationToken); + + stepProgress?.Report(checksumMessages.Count > 0 ? string.Join(' ', checksumMessages) : "Verified the package checksums."); + + VerifyChunkHashFractions(stepProgress, cancellationToken); + VerifyEntryChecksums(entriesWithUnhashedData, stepProgress, cancellationToken); + + stepProgress?.Report("Verified the package."); + } + + /// + /// Numbers the messages of before passing them on. + /// + private sealed class StepProgress(IProgress<(int Step, int StepCount, string Message)> progressReporter, int stepCount) : IProgress + { + private int Step; + + public void Report(string value) + { + progressReporter.Report((++Step, stepCount, value)); + } + } + + /// + /// Returns the entries, sorted by archive and offset, whose data no checksum or chunk hash covers. + /// + private List GetEntriesWithUnhashedData() + { + var ranges = AccessPackFileHashes.Select(static fraction => (fraction.ArchiveIndex, Start: (long)fraction.Offset, End: (long)fraction.Offset + fraction.Length)); + + if (Version != 1) + { + ranges = ranges.Append((DirectoryArchiveIndex, 0, FileDataSectionSize)); + } + + var hashedRanges = ranges + .GroupBy(static range => range.ArchiveIndex) + .ToDictionary(static group => group.Key, static group => MergeRanges(group.Select(static range => (range.Start, range.End)))); + + return [.. GetAllEntries() + .Where(entry => !IsDataHashed(entry, hashedRanges)) + .OrderBy(static entry => entry.ArchiveIndex) + .ThenBy(static entry => entry.Offset)]; + } + + private bool IsDataHashed(PackageEntry entry, Dictionary> hashedRanges) + { + if (Version == 1 || entry.PendingData != null) + { + return false; + } + + if (entry.Length == 0) + { + return true; + } + + if (!hashedRanges.TryGetValue(entry.ArchiveIndex, out var ranges)) + { + return false; + } + + var end = (long)entry.Offset + entry.Length; + + foreach (var (start, rangeEnd) in ranges) + { + if (start <= entry.Offset && end <= rangeEnd) + { + return true; + } + } + + return false; + } + + /// + /// Sorts the ranges and joins those that overlap or touch. + /// + private static List<(long Start, long End)> MergeRanges(IEnumerable<(long Start, long End)> ranges) + { + var merged = new List<(long Start, long End)>(); + + foreach (var range in ranges.OrderBy(static range => range.Start)) + { + if (merged.Count > 0 && range.Start <= merged[^1].End) + { + merged[^1] = (merged[^1].Start, Math.Max(merged[^1].End, range.End)); + } + else + { + merged.Add(range); + } + } + + return merged; + } + /// /// Verify MD5 hashes provided in the VPK. /// /// /// Version 1 packages have no checksums, so nothing is verified for them. - /// The whole file checksum covers the file tree and archive MD5 section checksums, so when it matches, + /// The whole file checksum covers the file tree and archive MD5 section checksums, so when it matches in a package without a signature header, /// a mismatching file tree or archive MD5 section checksum is reported to instead of thrown. /// /// If provided, will report a string for each checksum mismatch that is not thrown, or that a version 1 package has no checksums. /// The package has not been read, or its directory file could not be opened again after writing. - /// The package has no checksums, or the whole file checksum does not match. + /// The package has no checksums, or a checksum does not match. public void VerifyHashes(IProgress? progressReporter = null) + { + foreach (var message in VerifyChecksums(CancellationToken.None)) + { + progressReporter?.Report(message); + } + } + + /// + /// Verifies the checksums like does, returning the messages it reports. + /// + private List VerifyChecksums(CancellationToken cancellationToken) { ObjectDisposedException.ThrowIf(IsDisposed, this); @@ -50,8 +186,7 @@ public void VerifyHashes(IProgress? progressReporter = null) if (Version == 1) { - progressReporter?.Report("Version 1 packages have no checksums to verify."); - return; + return ["Version 1 packages have no checksums to verify."]; } if (TreeChecksum == null || ArchiveMD5EntriesChecksum == null || WholeFileChecksum == null) @@ -59,21 +194,25 @@ public void VerifyHashes(IProgress? progressReporter = null) throw new InvalidDataException("The package has no checksum section."); } - var wholeFileHash = HashPackageRange(0, FileSizeBeforeWholeFileHash, HashAlgorithmName.MD5); + var wholeFileHash = HashPackageRange(0, FileSizeBeforeWholeFileHash, HashAlgorithmName.MD5, cancellationToken); var wholeFileHashMatches = wholeFileHash.SequenceEqual(WholeFileChecksum); + var toleratesSectionMismatch = wholeFileHashMatches && SignatureType != ESignatureType.OnlyFileChecksum; + var messages = new List(); - VerifySectionHash("File tree", HeaderSize, TreeSize, TreeChecksum, wholeFileHashMatches, progressReporter); - VerifySectionHash("Archive MD5 entries", FileSizeBeforeArchiveMD5Entries, ArchiveMD5SectionSize, ArchiveMD5EntriesChecksum, wholeFileHashMatches, progressReporter); + VerifySectionHash("File tree", HeaderSize, TreeSize, TreeChecksum, toleratesSectionMismatch, messages, cancellationToken); + VerifySectionHash("Archive MD5 entries", FileSizeBeforeArchiveMD5Entries, ArchiveMD5SectionSize, ArchiveMD5EntriesChecksum, toleratesSectionMismatch, messages, cancellationToken); if (!wholeFileHashMatches) { throw new InvalidDataException($"Package checksum mismatch ({BitConverter.ToString(wholeFileHash)} != expected {BitConverter.ToString(WholeFileChecksum)})"); } + + return messages; } - private void VerifySectionHash(string sectionName, long offset, long length, byte[] expected, bool wholeFileHashMatches, IProgress? progressReporter) + private void VerifySectionHash(string sectionName, long offset, long length, byte[] expected, bool toleratesMismatch, List messages, CancellationToken cancellationToken) { - var hash = HashPackageRange(offset, length, HashAlgorithmName.MD5); + var hash = HashPackageRange(offset, length, HashAlgorithmName.MD5, cancellationToken); if (hash.SequenceEqual(expected)) { @@ -82,19 +221,19 @@ private void VerifySectionHash(string sectionName, long offset, long length, byt var message = $"{sectionName} checksum mismatch ({BitConverter.ToString(hash)} != expected {BitConverter.ToString(expected)})"; - if (!wholeFileHashMatches) + if (!toleratesMismatch) { throw new InvalidDataException(message); } - progressReporter?.Report($"{message}, ignored because the package checksum matches."); + messages.Add($"{message}, ignored because the package checksum matches."); } /// /// Computes the hash of a range of the package, at an offset from the start of the package. /// The hash covers fewer bytes if the package ends before the range does. /// - private byte[] HashPackageRange(long offset, long length, HashAlgorithmName hashAlgorithm) + private byte[] HashPackageRange(long offset, long length, HashAlgorithmName hashAlgorithm, CancellationToken cancellationToken) { var archive = GetArchive(DirectoryArchiveIndex); var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, ChunkHashBufferSize)); @@ -105,6 +244,8 @@ private byte[] HashPackageRange(long offset, long length, HashAlgorithmName hash while (length > 0) { + cancellationToken.ThrowIfCancellationRequested(); + var count = (int)Math.Min(length, buffer.Length); var bytesRead = archive.Read(offset, buffer.AsSpan(0, count)); @@ -134,6 +275,16 @@ public void VerifyChunkHashes(IProgress? progressReporter = null) { ObjectDisposedException.ThrowIf(IsDisposed, this); + VerifyChunkHashFractions(progressReporter, CancellationToken.None); + + progressReporter?.Report("Successfully verified archive hashes."); + } + + /// + /// Verifies every entry of , reporting a string before each. + /// + private void VerifyChunkHashFractions(IProgress? progressReporter, CancellationToken cancellationToken) + { // When created by Valve, entries are sorted, and are 1MB chunks var allEntries = AccessPackFileHashes .OrderBy(file => file.ArchiveIndex) @@ -147,6 +298,8 @@ public void VerifyChunkHashes(IProgress? progressReporter = null) foreach (var entry in allEntries) { + cancellationToken.ThrowIfCancellationRequested(); + var hashTypeName = entry.HashType switch { EHashType.Blake3 => "Blake3", @@ -195,8 +348,6 @@ public void VerifyChunkHashes(IProgress? progressReporter = null) throw new InvalidDataException($"Package checksum mismatch ({hashTypeName}) in archive {entry.ArchiveIndex} at {entry.Offset} ({Convert.ToHexString(hash)} != expected {Convert.ToHexString(entry.Checksum)})"); } } - - progressReporter?.Report("Successfully verified archive hashes."); } /// @@ -213,26 +364,48 @@ public void VerifyFileChecksums(IProgress? progressReporter = null) return; } - var allEntries = Entries - .SelectMany(file => file.Value) + var allEntries = GetAllEntries() .OrderBy(file => file.ArchiveIndex) .ThenBy(file => file.Offset); - var buffer = Array.Empty(); + VerifyEntryChecksums(allEntries, progressReporter, CancellationToken.None); - foreach (var entry in allEntries) + progressReporter?.Report("Successfully verified file CRC32 checksums."); + } + + /// + /// Verifies the CRC32 checksum of each of the entries, reporting a string before each. + /// + private void VerifyEntryChecksums(IEnumerable entries, IProgress? progressReporter, CancellationToken cancellationToken) + { + var crc = new Crc32(); + Action> appendToCrc = crc.Append; + + foreach (var entry in entries) { + cancellationToken.ThrowIfCancellationRequested(); + progressReporter?.Report($"Verifying CRC32 checksum for '{entry.GetFullPath()}' in archive {entry.ArchiveIndex}."); - if (buffer.Length < entry.TotalLength) + crc.Append(entry.SmallData); + + if (entry.PendingData != null) { - buffer = new byte[entry.TotalLength]; + crc.Append(entry.PendingData); + } + else + { + ReadArchiveDataInBlocks(entry.ArchiveIndex, entry.Offset, entry.Length, appendToCrc); } - ReadEntry(entry, buffer, validateCrc: true); - } + var actualChecksum = crc.GetCurrentHashAsUInt32(); + crc.Reset(); - progressReporter?.Report("Successfully verified file CRC32 checksums."); + if (entry.CRC32 != actualChecksum) + { + throw new InvalidDataException($"CRC32 mismatch for '{entry.GetFullPath()}' (expected {entry.CRC32:X2}, got {actualChecksum:X2})."); + } + } } /// @@ -240,6 +413,11 @@ public void VerifyFileChecksums(IProgress? progressReporter = null) /// /// True if the signature is valid or the package has no signature (see ), false otherwise. public bool IsSignatureValid() + { + return IsSignatureValid(CancellationToken.None); + } + + private bool IsSignatureValid(CancellationToken cancellationToken) { if (PublicKey == null || Signature == null) { @@ -261,7 +439,7 @@ public bool IsSignatureValid() return false; } - var hash = HashPackageRange(0, FileSizeBeforeWholeFileHash, HashAlgorithmName.MD5); + var hash = HashPackageRange(0, FileSizeBeforeWholeFileHash, HashAlgorithmName.MD5, cancellationToken); if (!hash.SequenceEqual(WholeFileChecksum)) { @@ -277,7 +455,7 @@ public bool IsSignatureValid() return false; } - var signedHash = HashPackageRange(0, FileSizeBeforeSignature, HashAlgorithmName.SHA256); + var signedHash = HashPackageRange(0, FileSizeBeforeSignature, HashAlgorithmName.SHA256, cancellationToken); return rsa.VerifyHash(signedHash, Signature, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); } diff --git a/ValvePak/ValvePak/Package.cs b/ValvePak/ValvePak/Package.cs index 16cae96..7a3914d 100644 --- a/ValvePak/ValvePak/Package.cs +++ b/ValvePak/ValvePak/Package.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Text; using System.Threading; @@ -19,12 +20,45 @@ public partial class Package : IDisposable internal const string Space = " "; internal const string Dot = "."; + internal const string DotDot = ".."; internal const string DirectorySeparator = "/"; /// /// Archive index of entries whose data is in the directory file instead of a chunk file. /// - internal const ushort DirectoryArchiveIndex = 0x7FFF; + public const ushort DirectoryArchiveIndex = 0x7FFF; + + /// + /// Highest archive index of a chunk file. + /// + public const ushort MaxChunkIndex = 0x7FFE; + + /// + /// Maximum size in bytes of the file data in a directory file. + /// + public const int MaxDirectoryFileDataSize = int.MaxValue; + + /// + /// Maximum size in bytes of a chunk file. + /// + public const uint MaxChunkFileSize = uint.MaxValue; + + /// + /// Maximum number of preload bytes () of an entry. + /// + public const ushort MaxPreloadSize = ushort.MaxValue; + + /// + /// Maximum length in UTF-8 bytes of a . + /// + public const int MaxTypeNameLength = 128; + + /// + /// Maximum length in UTF-8 bytes of a . + /// + public const int MaxFileNameLength = 260; + + private const int MaxStackPathLength = 256; /// /// Always '/' as per Valve's vpk implementation. @@ -40,6 +74,7 @@ public partial class Package : IDisposable private long StreamOffset; private string? ReadFilePath; + private bool IsReadFromAnotherPackage; private readonly ConcurrentDictionary Archives = new(); private readonly Lock ArchivesLock = new(); @@ -49,11 +84,22 @@ public partial class Package : IDisposable /// public string? FileName { get; private set; } + /// + /// Gets the full path of the directory file that this package reads from, or null if it does not read from the start of a file. + /// + public string? FilePath => StreamOffset == 0 ? ReadFilePath : null; + /// /// Gets whether this package had "_dir" in the name, indicating it has multiple chunk files. /// public bool IsDirVPK { get; private set; } + /// + /// Gets whether entries or chunk hashes of this package refer to chunk files, or files were added to chunk files. + /// Unlike , this does not depend on the file name. + /// + public bool HasChunkFiles => HighestChunkIndex >= 0; + /// /// Gets or sets the VPK version. This version is also used when writing the package. /// @@ -265,15 +311,13 @@ private static string StripDirVpkSuffixes(string fileName, out bool isDirVpk) } /// - /// Searches for a given file entry in the file list. + /// Searches for the entry at the given path, which is normalized like does. /// - /// If was called on this package, this method will use . + /// If was called on this package, this method uses a binary search. /// Optimized packages also support case insensitive search by using a different . /// /// Full path to the file to find. - /// - /// The path is normalized the same way as paths of added files, see . - /// + /// The entry, or null if there is none or the path is rejected. public PackageEntry? FindEntry(string filePath) { ArgumentNullException.ThrowIfNull(filePath); @@ -281,58 +325,17 @@ private static string StripDirVpkSuffixes(string fileName, out bool isDirVpk) return FindEntry(filePath.AsSpan()); } - /// - /// Searches for a given file entry in the file list. - /// - /// If was called on this package, this method will use . - /// Optimized packages also support case insensitive search by using a different . - /// - /// Full path to the file to find. - /// - /// The path is normalized the same way as paths of added files: (\) becomes (/), - /// empty and "." segments and trailing dots are removed, and a missing directory or extension is a space. - /// If that finds no entry, the path is also looked up as it is, split at its last and at each dot of the file name, - /// which finds entries of read packages whose names normalizing changes. - /// The path is not lowercased like added paths are, so the search is case sensitive unless - /// was called with a case insensitive . - /// + /// public PackageEntry? FindEntry(ReadOnlySpan filePath) { - SplitPath(NormalizePath(filePath, out var changed), out var directory, out var fileName, out var extension); + Span buffer = filePath.Length <= MaxStackPathLength || IsNormalizedPath(filePath) ? stackalloc char[MaxStackPathLength] : new char[filePath.Length]; - return FindEntry(directory, fileName, extension) ?? FindEntryByStoredName(filePath, changed ? -1 : fileName.Length); - } - - /// - /// Finds the entry whose is the given path, trying each split of the file name into a file and type name. - /// - /// The path as it was given. - /// Position in the file name that was already looked up, where the length of the file name means without a type, or -1. - private PackageEntry? FindEntryByStoredName(ReadOnlySpan path, int skipSplit) - { - var lastSeparator = path.LastIndexOf(DirectorySeparatorChar); - var directory = lastSeparator > -1 ? path[..lastSeparator] : Space; - var name = path[(lastSeparator + 1)..]; - PackageEntry? entry = null; - - for (var split = 1; entry == null && split <= name.Length; split++) + if (SplitEntryPath(filePath, buffer, out var directory, out var fileName, out var typeName) != null) { - if (split == skipSplit) - { - continue; - } - - if (split == name.Length) - { - entry = FindEntry(directory, name, Space); - } - else if (name[split] == '.' && split < name.Length - 1) - { - entry = FindEntry(directory, name[..split], name[(split + 1)..]); - } + return null; } - return entry; + return FindEntry(directory, fileName, typeName); } private PackageEntry? FindEntry(ReadOnlySpan directory, ReadOnlySpan fileName, ReadOnlySpan extension) @@ -407,95 +410,229 @@ private static string StripDirVpkSuffixes(string fileName, out bool isDirVpk) } /// - /// Replaces with , removes empty and "." segments, and removes trailing dots. + /// Normalizes a file path to the path that stores it at: '\' is a directory separator, + /// empty and "." segments are removed, ".." removes the segment before it, ASCII letters are lowercased, and the type name is split from the file name at its last dot. /// - private static ReadOnlySpan NormalizePath(ReadOnlySpan path, out bool changed) + /// Full file path. + /// The normalized path, or null if the path is rejected. + /// True if a file can be stored at the path, false if it does not end with a file name, has a ".." above the root, + /// a null character or an unpaired surrogate, or has a name longer than or . + public static bool TryParseEntryPath(string filePath, [NotNullWhen(true)] out string? normalizedPath) { - changed = !IsNormalizedPath(path); + ArgumentNullException.ThrowIfNull(filePath); + + Span buffer = filePath.Length <= MaxStackPathLength ? stackalloc char[MaxStackPathLength] : new char[filePath.Length]; - if (!changed) + if (GetEntryPathError(filePath, buffer, out var directory, out var fileName, out var typeName) != null) { - return path; + normalizedPath = null; + return false; } - var builder = new StringBuilder(path.Length); + normalizedPath = JoinEntryPath(directory, fileName, typeName); + return true; + } - foreach (var range in path.SplitAny(DirectorySeparatorChar, WindowsDirectorySeparator)) + /// + /// Normalizes a directory path to the of the files in it, like normalizes directories. + /// + /// Directory path, which may end with a directory separator. + /// The normalized path, or an empty string for the root directory or if the path has a ".." above the root. + public static string NormalizeDirectoryPath(string directoryPath) + { + ArgumentNullException.ThrowIfNull(directoryPath); + + var filePath = string.Concat(directoryPath, "/x"); + Span buffer = filePath.Length <= MaxStackPathLength ? stackalloc char[MaxStackPathLength] : new char[filePath.Length]; + + if (SplitEntryPath(filePath, buffer, out var directory, out _, out _) != null || directory is Space) { - var segment = path[range]; + return string.Empty; + } + + return directory.ToString(); + } + + /// + /// Splits a file path like does and checks the lengths of the names. + /// + /// Why the path can not be stored, or null. + private static string? GetEntryPathError(ReadOnlySpan path, Span buffer, out ReadOnlySpan directory, out ReadOnlySpan fileName, out ReadOnlySpan typeName) + { + var error = SplitEntryPath(path, buffer, out directory, out fileName, out typeName); + + if (error != null) + { + return error; + } + + if (Encoding.UTF8.GetByteCount(typeName) > MaxTypeNameLength) + { + return "has a type name longer than 128 bytes in UTF-8"; + } + + if (Encoding.UTF8.GetByteCount(fileName) > MaxFileNameLength) + { + return "has a file name longer than 260 bytes in UTF-8"; + } + + return null; + } + + /// + /// Normalizes a file path like does and splits it into the directory, file and type names it is stored as. + /// A path that normalizing changes is written into the buffer, which must be at least as long as the path. + /// + /// Why no file can be at the path, or null. + private static string? SplitEntryPath(ReadOnlySpan path, Span buffer, out ReadOnlySpan directory, out ReadOnlySpan fileName, out ReadOnlySpan typeName) + { + directory = fileName = typeName = default; + + if (path.IsEmpty || IsDirectorySeparator(path[^1]) || path[(path.LastIndexOfAny(DirectorySeparatorChar, WindowsDirectorySeparator) + 1)..] is Dot or DotDot) + { + return "does not end with a file name"; + } + + if (HasInvalidCharacter(path)) + { + return "has a null character or an unpaired surrogate"; + } + + var normalizedPath = path; + + if (!IsNormalizedPath(path)) + { + var length = ResolveSegments(path, buffer); - if (IsSkippedSegment(segment)) + if (length < 0) { - continue; + return "has a \"..\" segment above the root"; } - if (builder.Length > 0) + normalizedPath = buffer[..length]; + } + + var lastSeparator = normalizedPath.LastIndexOf(DirectorySeparatorChar); + var file = normalizedPath[(lastSeparator + 1)..]; + var dot = file.LastIndexOf('.'); + + directory = lastSeparator > -1 ? normalizedPath[..lastSeparator] : Space; + fileName = dot > -1 ? file[..dot] : file; + typeName = dot > -1 ? file[(dot + 1)..] : Space; + + if (dot == 0 || dot == file.Length - 1 || (typeName is Space && fileName.EndsWith('.'))) + { + directory = fileName = typeName = default; + return "has a file name that is empty or ends with a dot"; + } + + return null; + } + + /// + /// Returns whether the path has a null character, which ends a name in the directory tree, or a surrogate that is not part of a pair, which UTF-8 can not encode. + /// + private static bool HasInvalidCharacter(ReadOnlySpan path) + { + if (path.Contains('\0')) + { + return true; + } + + int i; + + while ((i = path.IndexOfAnyInRange('\uD800', '\uDFFF')) >= 0) + { + if (!char.IsHighSurrogate(path[i]) || i + 1 >= path.Length || !char.IsLowSurrogate(path[i + 1])) { - builder.Append(DirectorySeparatorChar); + return true; } - builder.Append(segment); + path = path[(i + 2)..]; } - var length = builder.Length; + return false; + } - while (length > 0 && builder[length - 1] == '.') + /// + /// Joins the segments of the path with '/' into the buffer and lowercases ASCII letters, leaving out empty and "." segments and removing the segment before each "..". + /// + /// The length of the resolved path, or -1 if a ".." has no segment before it. + private static int ResolveSegments(ReadOnlySpan path, Span buffer) + { + var length = 0; + + foreach (var range in path.SplitAny(DirectorySeparatorChar, WindowsDirectorySeparator)) { - length--; + var segment = path[range]; + + if (segment.IsEmpty || segment is Dot) + { + continue; + } + + if (segment is DotDot) + { + if (length == 0) + { + return -1; + } + + length = Math.Max(buffer[..length].LastIndexOf(DirectorySeparatorChar), 0); + continue; + } + + if (length > 0) + { + buffer[length++] = DirectorySeparatorChar; + } + + foreach (var c in segment) + { + buffer[length++] = ToLowerAscii(c); + } } - return builder.ToString(0, length); + return length; } + /// + /// Returns whether normalizing leaves the path as it is, because it has no '\', no upper case ASCII letters and no empty, "." or ".." segments. + /// private static bool IsNormalizedPath(ReadOnlySpan path) { - if (path.Contains(WindowsDirectorySeparator)) + if (path.Contains(WindowsDirectorySeparator) || path.ContainsAnyInRange('A', 'Z')) { return false; } foreach (var range in path.Split(DirectorySeparatorChar)) { - if (IsSkippedSegment(path[range])) + if (path[range] is "" or Dot or DotDot) { return false; } } - return !path.EndsWith('.'); + return true; } - private static bool IsSkippedSegment(ReadOnlySpan segment) + private static string JoinEntryPath(ReadOnlySpan directory, ReadOnlySpan fileName, ReadOnlySpan typeName) { - return segment.IsEmpty || segment is Dot; + var file = typeName is Space ? fileName.ToString() : string.Concat(fileName, Dot, typeName); + + return directory is Space ? file : string.Concat(directory, DirectorySeparator, file); } - /// - /// Splits a normalized path into the directory, file and type names of an entry, using a space for a missing directory or extension. - /// - private static void SplitPath(ReadOnlySpan path, out ReadOnlySpan directory, out ReadOnlySpan fileName, out ReadOnlySpan extension) + private static bool IsDirectorySeparator(char c) { - var lastSeparator = path.LastIndexOf(DirectorySeparatorChar); - directory = lastSeparator > -1 ? path[..lastSeparator] : Space; - fileName = path[(lastSeparator + 1)..]; - - var dot = fileName.LastIndexOf('.'); - extension = Space; - - if (dot > -1) - { - extension = fileName[(dot + 1)..]; - fileName = fileName[..dot]; - } + return c is DirectorySeparatorChar or WindowsDirectorySeparator; } /// /// This sorts so that it can be searched through using binary search. /// Use if you want to search case insensitively. /// - /// - /// This is experimental and may be removed in a future release. - /// /// Comparison method to use. public void OptimizeEntriesForBinarySearch(StringComparison comparison = StringComparison.Ordinal) { @@ -506,5 +643,62 @@ public void OptimizeEntriesForBinarySearch(StringComparison comparison = StringC Comparer = new CaseInsensitivePackageEntryComparer(comparison); } + + private static readonly AsciiCaseInsensitiveComparer AsciiComparer = new(); + + /// + /// Gets the comparer that compares paths and names ordinally with only ASCII letters case insensitive, which also looks up keys in dictionaries. + /// + public static StringComparer PathComparer => AsciiComparer; + + private sealed class AsciiCaseInsensitiveComparer : StringComparer, IAlternateEqualityComparer, string> + { + public override int Compare(string? x, string? y) + { + return x == null || y == null ? string.CompareOrdinal(x, y) : CompareAscii(x, y); + } + + public override bool Equals(string? x, string? y) + { + return x == null || y == null ? x == y : Equals(x.AsSpan(), y); + } + + public bool Equals(ReadOnlySpan alternate, string other) + { + return alternate.Length == other.Length && CompareAscii(alternate, other) == 0; + } + + public override int GetHashCode(string obj) + { + return GetHashCode(obj.AsSpan()); + } + + public int GetHashCode(ReadOnlySpan alternate) + { + return string.GetHashCode(alternate, StringComparison.OrdinalIgnoreCase); + } + + public string Create(ReadOnlySpan alternate) + { + return alternate.ToString(); + } + + private static int CompareAscii(ReadOnlySpan x, ReadOnlySpan y) + { + var length = Math.Min(x.Length, y.Length); + + for (var i = 0; i < length; i++) + { + var comparison = ToLowerAscii(x[i]).CompareTo(ToLowerAscii(y[i])); + + if (comparison != 0) + { + return comparison; + } + } + + return x.Length.CompareTo(y.Length); + } + } } } diff --git a/ValvePak/ValvePak/PackageDataStream.cs b/ValvePak/ValvePak/PackageDataStream.cs new file mode 100644 index 0000000..72e07e4 --- /dev/null +++ b/ValvePak/ValvePak/PackageDataStream.cs @@ -0,0 +1,170 @@ +using System; +using System.IO; + +namespace ValvePak +{ + /// + /// Read-only seekable stream over data stored in a package: bytes held in memory such as the preload bytes of an entry, + /// followed by a range of an archive, which it reads from without owning it. Either part can be empty. + /// + internal sealed class PackageDataStream : Stream + { +#pragma warning disable CA2213 // Disposable fields should be disposed + private readonly Archive? Archive; +#pragma warning restore CA2213 + private readonly long Start; + private readonly long RangeLength; + private readonly byte[] Prefix; + private long CurrentPosition; + private bool IsDisposed; + + /// + /// Creates a stream over the given range of the archive, which starts with if given. + /// + public PackageDataStream(Archive archive, long start, long length, byte[]? prefix = null) + { + Archive = archive; + Start = start; + RangeLength = length; + Prefix = prefix ?? []; + } + + /// + /// Creates a stream over bytes held in memory. + /// + public PackageDataStream(byte[] data) + { + Prefix = data; + } + + public override bool CanRead => !IsDisposed; + + public override bool CanSeek => !IsDisposed; + + public override bool CanWrite => false; + + public override long Length => Prefix.Length + RangeLength; + + public override long Position + { + get => CurrentPosition; + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + CurrentPosition = value; + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + ValidateBufferArguments(buffer, offset, count); + + return Read(buffer.AsSpan(offset, count)); + } + + public override int Read(Span buffer) + { + var bytesRead = ReadAt(CurrentPosition, buffer); + CurrentPosition += bytesRead; + + return bytesRead; + } + + /// + /// Reads at the given position of the stream without moving its position. + /// This method is safe to call from multiple threads. + /// + /// The number of bytes read, which is less than requested only at the end of the stream or of the archive. + public int ReadAt(long position, Span buffer) + { + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var prefixRead = 0; + + if (position < Prefix.Length) + { + prefixRead = Math.Min(Prefix.Length - (int)position, buffer.Length); + Prefix.AsSpan((int)position, prefixRead).CopyTo(buffer); + position += prefixRead; + buffer = buffer[prefixRead..]; + } + + var available = Length - position; + + if (available <= 0 || buffer.IsEmpty || Archive == null) + { + return prefixRead; + } + + if (buffer.Length > available) + { + buffer = buffer[..(int)available]; + } + + return prefixRead + Archive.Read(Start + position - Prefix.Length, buffer); + } + + /// + /// Returns a stream that reads the given range of this stream directly from a memory mapping of the archive. + /// + /// The stream, or null if the archive can not be mapped or the range is not entirely in the archive range. + public Stream? OpenMappedStream(long position, long length) + { + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var offset = position - Prefix.Length; + + if (Archive == null || offset < 0 || length > RangeLength - offset) + { + return null; + } + + return Archive.OpenMappedStream(Start + offset, length); + } + + public override long Seek(long offset, SeekOrigin origin) + { + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => CurrentPosition + offset, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentOutOfRangeException(nameof(origin)), + }; + + if (position < 0) + { + throw new IOException("An attempt was made to move the position before the beginning of the stream."); + } + + CurrentPosition = position; + + return position; + } + + public override void Flush() + { + } + + public override void SetLength(long value) + { + throw new NotSupportedException("The stream is read-only."); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException("The stream is read-only."); + } + + protected override void Dispose(bool disposing) + { + IsDisposed = true; + + base.Dispose(disposing); + } + } +} diff --git a/ValvePak/ValvePak/PackageDataStreamArchive.cs b/ValvePak/ValvePak/PackageDataStreamArchive.cs new file mode 100644 index 0000000..58764e9 --- /dev/null +++ b/ValvePak/ValvePak/PackageDataStreamArchive.cs @@ -0,0 +1,41 @@ +using System; +using System.IO; + +namespace ValvePak +{ + /// + /// Archive data embedded in a package that was read from a , such as a package inside another package. + /// Reads go to the archive of the stream without moving the stream position, so they are not serialized. + /// + internal sealed class PackageDataStreamArchive : Archive + { +#pragma warning disable CA2213 // Disposable fields should be disposed + private readonly PackageDataStream DataStream; +#pragma warning restore CA2213 + + public PackageDataStreamArchive(PackageDataStream stream, long origin, long dataOffset) + : base(origin, dataOffset) + { + DataStream = stream; + } + + public override long Length => DataStream.Length - Origin; + + public override int Read(long offset, Span output) + { + return DataStream.ReadAt(Origin + offset, output); + } + + public override Stream? OpenMappedStream(long offset, long length) + { + return DataStream.OpenMappedStream(Origin + offset, length); + } + + /// + /// Does nothing, the stream is owned by the package. + /// + public override void Dispose() + { + } + } +} diff --git a/ValvePak/ValvePak/PackageEntry.cs b/ValvePak/ValvePak/PackageEntry.cs index 31d04ad..fbe2f4e 100644 --- a/ValvePak/ValvePak/PackageEntry.cs +++ b/ValvePak/ValvePak/PackageEntry.cs @@ -90,18 +90,30 @@ public string GetFileName() return string.Concat(fileName, Package.Dot, TypeName); } + /// + /// Returns , or an empty string for the root directory, which is stored as a single space or an empty string. + /// +#pragma warning disable CA1024 // Use properties where appropriate + public string GetDirectoryPath() +#pragma warning restore CA1024 + { + return DirectoryName is Package.Space or "" ? string.Empty : DirectoryName; + } + /// /// Returns the absolute path of the file in the package. /// /// Absolute path. public string GetFullPath() { - if (DirectoryName == Package.Space) + var directory = GetDirectoryPath(); + + if (directory.Length == 0) { return GetFileName(); } - return string.Concat(DirectoryName, Package.DirectorySeparator, GetFileName()); + return string.Concat(directory, Package.DirectorySeparator, GetFileName()); } /// diff --git a/ValvePak/ValvePak/StreamArchive.cs b/ValvePak/ValvePak/StreamArchive.cs index 95b7740..58b6f1e 100644 --- a/ValvePak/ValvePak/StreamArchive.cs +++ b/ValvePak/ValvePak/StreamArchive.cs @@ -21,6 +21,17 @@ public StreamArchive(Stream stream, long origin, long dataOffset) Stream = stream; } + public override long Length + { + get + { + lock (StreamLock) + { + return Stream.Length - Origin; + } + } + } + public override int Read(long offset, Span output) { lock (StreamLock)