From 950e39fbf8628fbd044013be8fadc5ff28ae9703 Mon Sep 17 00:00:00 2001 From: w1tcherrr <70418164+w1tcherrr@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:14:05 +0200 Subject: [PATCH 1/7] Add NormalizeEntryPath, NormalizeDirectoryPath and PathComparer, find entries by their normalized names and refuse names that can not be stored --- ValvePak/ValvePak.Test/EntryPathTest.cs | 73 ++++++ ValvePak/ValvePak.Test/PackageTest.cs | 22 +- ValvePak/ValvePak.Test/WriteTest.cs | 67 ++---- ValvePak/ValvePak/Package.Save.cs | 63 ++--- ValvePak/ValvePak/Package.cs | 302 ++++++++++++++++-------- ValvePak/ValvePak/PackageEntry.cs | 16 +- 6 files changed, 353 insertions(+), 190 deletions(-) create mode 100644 ValvePak/ValvePak.Test/EntryPathTest.cs diff --git a/ValvePak/ValvePak.Test/EntryPathTest.cs b/ValvePak/ValvePak.Test/EntryPathTest.cs new file mode 100644 index 0000000..964743a --- /dev/null +++ b/ValvePak/ValvePak.Test/EntryPathTest.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace ValvePak.Test +{ + internal sealed class EntryPathTest + { + [Test] + [Arguments("A\\B//./c.TXT", "a/b/c.txt")] + [Arguments("a/x/../b.txt", "a/b.txt")] + [Arguments("/a/b.txt", "a/b.txt")] + [Arguments("README", "readme")] + [Arguments("", "")] + [Arguments("../b.txt", "")] + [Arguments("a/b/", "")] + [Arguments(".gitignore", "")] + public async Task NormalizesEntryPaths(string path, string expected) + { + await Assert.That(Package.NormalizeEntryPath(path)).IsEqualTo(expected); + } + + [Test] + public async Task RejectsNamesThatAreTooLong() + { + using (Assert.Multiple()) + { + await Assert.That(Package.NormalizeEntryPath($"a.{new string('y', 129)}")).IsEmpty(); + await Assert.That(Package.NormalizeEntryPath($"{new string('é', 131)}.txt")).IsEmpty(); + } + } + + [Test] + [Arguments("Materials\\Models/", "materials/models")] + [Arguments(" ", "")] + public async Task NormalizesDirectoryPaths(string path, string expected) + { + await Assert.That(Package.NormalizeDirectoryPath(path)).IsEqualTo(expected); + } + + [Test] + [Arguments(" ")] + [Arguments("")] + public async Task GetsDirectoryPathWithRootAsEmptyString(string directoryName) + { + var entry = new PackageEntry + { + DirectoryName = directoryName, + FileName = "file", + TypeName = "txt", + }; + + using (Assert.Multiple()) + { + await Assert.That(entry.GetDirectoryPath()).IsEmpty(); + await Assert.That(entry.GetFullPath()).IsEqualTo("file.txt"); + } + } + + [Test] + public async Task PathComparerIgnoresOnlyTheCaseOfAsciiLetters() + { + var folders = new Dictionary(Package.PathComparer) { ["Models"] = 1 }; + + using (Assert.Multiple()) + { + await Assert.That(Package.PathComparer.Equals("MODELS/A.txt", "models/a.TXT")).IsTrue(); + await Assert.That(Package.PathComparer.Equals("Ä", "ä")).IsFalse(); + await Assert.That(folders.GetAlternateLookup>().ContainsKey("mODELS")).IsTrue(); + } + } + } +} diff --git a/ValvePak/ValvePak.Test/PackageTest.cs b/ValvePak/ValvePak.Test/PackageTest.cs index 7c8b7d7..fd6f4a8 100644 --- a/ValvePak/ValvePak.Test/PackageTest.cs +++ b/ValvePak/ValvePak.Test/PackageTest.cs @@ -213,14 +213,22 @@ public async Task FindEntrySpacesAndExtensionless() await Assert.That(package.FindEntry("folder with space\\space_extension. txt")?.CRC32).IsEqualTo(0x09321FC0u); await Assert.That(package.FindEntry("folder with space/file name with space.txt")?.CRC32).IsEqualTo(0x76D91432u); await Assert.That(package.FindEntry("uppercasefolder/bad_file_forfun.txt")?.CRC32).IsEqualTo(0x15C1490Fu); - await Assert.That(package.FindEntry("UpperCaseFolder/UpperCaseFile.txt")?.CRC32).IsEqualTo(0x32CFF012u); - await Assert.That(package.FindEntry("UpperCaseFolder/bad_file_forfun.txt")).IsNull(); + await Assert.That(package.FindEntry("UpperCaseFolder/bad_file_forfun.txt")?.CRC32).IsEqualTo(0x15C1490Fu); + await Assert.That(package.FindEntry("uppercasefolder/bad_file_forfun.TXT")?.CRC32).IsEqualTo(0x15C1490Fu); + await Assert.That(package.FindEntry("UpperCaseFolder/UpperCaseFile.txt")).IsNull(); await Assert.That(package.FindEntry("uppercasefolder/UpperCaseFile.txt")).IsNull(); - await Assert.That(package.FindEntry("uppercasefolder/bad_file_forfun.TXT")).IsNull(); await Assert.That(package.FindEntry("uppercasefolder/bad_file_forfun.txt2")).IsNull(); } } + /// + /// Returns the entry of broken_dir.vpk whose stored names have upper case letters, which does not find. + /// + private static PackageEntry? GetUpperCaseEntry(Package package) + { + return package.Entries!["txt"].Find(static entry => entry.FileName == "UpperCaseFile"); + } + [Test] public async Task ThrowsOnInvalidCRC32() { @@ -229,7 +237,7 @@ public async Task ThrowsOnInvalidCRC32() using var package = new Package(); package.Read(path); - var file = package.FindEntry("UpperCaseFolder/UpperCaseFile.txt"); + var file = GetUpperCaseEntry(package); await Assert.That(file).IsNotNull(); await Assert.That(file.CRC32).IsEqualTo(0x32CFF012u); @@ -264,7 +272,7 @@ public async Task TestGetFullPath() await Assert.That(package.FindEntry("folder with space/test")?.GetFullPath()).IsEqualTo("folder with space/test"); await Assert.That(package.FindEntry("folder with space\\space_extension. txt")?.GetFullPath()).IsEqualTo("folder with space/space_extension. txt"); await Assert.That(package.FindEntry("uppercasefolder/bad_file_forfun.txt")?.GetFullPath()).IsEqualTo("uppercasefolder/bad_file_forfun.txt"); - await Assert.That(package.FindEntry("UpperCaseFolder/UpperCaseFile.txt")?.GetFullPath()).IsEqualTo("UpperCaseFolder/UpperCaseFile.txt"); + await Assert.That(GetUpperCaseEntry(package)?.GetFullPath()).IsEqualTo("UpperCaseFolder/UpperCaseFile.txt"); await Assert.That(package.FindEntry("test")?.GetFileName()).IsEqualTo("test"); await Assert.That(package.FindEntry("folder with space/test")?.GetFileName()).IsEqualTo("test"); @@ -287,7 +295,7 @@ public async Task TestPackageEntryToString() await Assert.That(package.FindEntry("folder with space/test")?.ToString()).IsEqualTo("folder with space/test crc=0xbf108706 metadatasz=0 fnumber=0 ofs=0x52 sz=41"); await Assert.That(package.FindEntry("folder with space\\space_extension. txt")?.ToString()).IsEqualTo("folder with space/space_extension. txt crc=0x9321fc0 metadatasz=0 fnumber=0 ofs=0x7b sz=30"); await Assert.That(package.FindEntry("uppercasefolder/bad_file_forfun.txt")?.ToString()).IsEqualTo("uppercasefolder/bad_file_forfun.txt crc=0x15c1490f metadatasz=0 fnumber=0 ofs=0xa2 sz=2"); - await Assert.That(package.FindEntry("UpperCaseFolder/UpperCaseFile.txt")?.ToString()).IsEqualTo("UpperCaseFolder/UpperCaseFile.txt crc=0x32cff012 metadatasz=0 fnumber=0 ofs=0x27 sz=43"); + await Assert.That(GetUpperCaseEntry(package)?.ToString()).IsEqualTo("UpperCaseFolder/UpperCaseFile.txt crc=0x32cff012 metadatasz=0 fnumber=0 ofs=0x27 sz=43"); } } @@ -442,7 +450,7 @@ public async Task TestFileChecksums() package.Read(path); await Assert.That(() => package.VerifyFileChecksums()).ThrowsNothing(); - var file = package.FindEntry("UpperCaseFolder/UpperCaseFile.txt"); + var file = GetUpperCaseEntry(package); await Assert.That(file).IsNotNull(); await Assert.That(file.CRC32).IsEqualTo(0x32CFF012u); diff --git a/ValvePak/ValvePak.Test/WriteTest.cs b/ValvePak/ValvePak.Test/WriteTest.cs index c3a7d33..934f96d 100644 --- a/ValvePak/ValvePak.Test/WriteTest.cs +++ b/ValvePak/ValvePak.Test/WriteTest.cs @@ -168,51 +168,36 @@ public async Task NormalizesPathsTheSameWhenAddingAndFinding(StringComparison co using var package = new Package(); package.OptimizeEntriesForBinarySearch(comparison); - var entry = package.AddFile("\\Materials//./Models/./Crate.VMAT_C/", Encoding.UTF8.GetBytes("crate")); - var noExtension = package.AddFile("./readme.", []); - var trailingDots = package.AddFile("notes..", [1]); - var trailingDotAfterExtension = package.AddFile("archive.tar.", [2]); + var entry = package.AddFile("\\Materials//./Models/../Models/Crate.VMAT_C", Encoding.UTF8.GetBytes("crate")); + var noExtension = package.AddFile("./Docs/ReadMe", []); using (Assert.Multiple()) { await Assert.That(entry.DirectoryName).IsEqualTo("materials/models"); await Assert.That(entry.FileName).IsEqualTo("crate"); await Assert.That(entry.TypeName).IsEqualTo("vmat_c"); - await Assert.That(noExtension.DirectoryName).IsEqualTo(" "); await Assert.That(noExtension.TypeName).IsEqualTo(" "); - await Assert.That(package.FindEntry("materials/models/crate.vmat_c")).IsEqualTo(entry); - await Assert.That(package.FindEntry("/materials\\models//crate.vmat_c")).IsEqualTo(entry); - await Assert.That(package.FindEntry("./materials/./models/crate.vmat_c/".AsSpan())).IsEqualTo(entry); - await Assert.That(package.FindEntry("readme")).IsEqualTo(noExtension); - await Assert.That(package.FindEntry("/readme.")).IsEqualTo(noExtension); - await Assert.That(trailingDots.FileName).IsEqualTo("notes"); - await Assert.That(trailingDots.TypeName).IsEqualTo(" "); - await Assert.That(trailingDotAfterExtension.FileName).IsEqualTo("archive"); - await Assert.That(trailingDotAfterExtension.TypeName).IsEqualTo("tar"); - - foreach (var added in new[] { entry, noExtension, trailingDots, trailingDotAfterExtension }) - { - await Assert.That(package.FindEntry(added.GetFullPath())).IsEqualTo(added); - } + await Assert.That(package.FindEntry("/MATERIALS\\models//crate.vmat_c")).IsEqualTo(entry); + await Assert.That(package.FindEntry("./materials/./textures/../models/crate.vmat_c".AsSpan())).IsEqualTo(entry); + await Assert.That(package.FindEntry("DOCS/README")).IsEqualTo(noExtension); + await Assert.That(package.FindEntry("materials/models/crate.vmat_c/")).IsNull(); + await Assert.That(package.FindEntry("../materials/models/crate.vmat_c")).IsNull(); } } [Test] [Arguments(false)] [Arguments(true)] - public async Task FindsReadEntriesWhoseNamesAreNotNormalized(bool optimize) + public async Task DoesNotFindReadEntriesWhoseNamesAreNotNormalized(bool optimize) { (string Directory, string File, string Type)[] names = [ ("a//b", "file", "txt"), - ("a/./b", "file", "txt"), - ("./rel", "file", "txt"), - ("/leading", "file", "txt"), - ("trailing/", "file", "txt"), + ("a/../b", "file", "txt"), ("back\\slash", "file", "txt"), (" ", "has.dot", " "), - ("dir", "noext.", " "), (" ", "archive", "tar.gz"), + ("Upper", "File", "TXT"), ("dir", "archive.tar", "gz"), ]; @@ -241,18 +226,22 @@ public async Task FindsReadEntriesWhoseNamesAreNotNormalized(bool optimize) readBack.SetFileName("test.vpk"); readBack.Read(output); - var entries = readBack.Entries!.Values.SelectMany(static entries => entries).ToList(); + var entries = readBack.Entries!.Values.SelectMany(static entries => entries).ToDictionary(static entry => entry.GetFullPath()); await Assert.That(entries).Count().IsEqualTo(names.Length); - foreach (var entry in entries) - { - await Assert.That(readBack.FindEntry(entry.GetFullPath())).IsSameReferenceAs(entry); - } - using (Assert.Multiple()) { - await Assert.That(readBack.FindEntry("dir/noext")).IsNull(); - await Assert.That(readBack.FindEntry("has")).IsNull(); + foreach (var (path, entry) in entries) + { + var expected = path switch + { + "dir/archive.tar.gz" => entry, + "Upper/File.TXT" when optimize => entry, + _ => null, + }; + + await Assert.That(readBack.FindEntry(path)).IsSameReferenceAs(expected); + } } } @@ -667,17 +656,7 @@ public async Task AddedFilesCanBeFoundAfterOptimizingForBinarySearch(StringCompa await Assert.That(package.FindEntry(entry.GetFullPath())).IsEqualTo(entry); } - var upperCaseEntry = package.FindEntry("ADDONS/CHESS/ZZZ.VDF"); - - if (comparison == StringComparison.OrdinalIgnoreCase) - { - await Assert.That(upperCaseEntry).IsEqualTo(addedEntries[0]); - } - else - { - await Assert.That(upperCaseEntry).IsNull(); - } - + await Assert.That(package.FindEntry("ADDONS/CHESS/ZZZ.VDF")).IsEqualTo(addedEntries[0]); await Assert.That(package.FindEntry("addons/chess/missing.vdf")).IsNull(); } diff --git a/ValvePak/ValvePak/Package.Save.cs b/ValvePak/ValvePak/Package.Save.cs index cb8add9..5924e99 100644 --- a/ValvePak/ValvePak/Package.Save.cs +++ b/ValvePak/ValvePak/Package.Save.cs @@ -90,11 +90,11 @@ public bool RemoveFile(PackageEntry entry) /// /// Moves an entry in current package to a new path. Only the directory tree changes, the file data is not touched. /// Be careful to not move it onto the path of another entry, because this does not check for duplicates. - /// The path is normalized and lowercased like Valve's packer does, see . + /// The path is normalized like does. /// /// The package entry to rename. /// Full new file path for this entry. - /// The entry is not in this package, or the new path has an empty file name or contains a null character. + /// The entry is not in this package, or the new path is rejected, see . public void RenameFile(PackageEntry entry, string newFilePath) { ArgumentNullException.ThrowIfNull(entry); @@ -115,7 +115,7 @@ public void RenameFile(PackageEntry entry, string newFilePath) /// /// Add file to current package. Be careful to not add duplicate entries, because this does not check for duplicates. - /// The path is normalized and lowercased like Valve's packer does, see . + /// The path is normalized like does. /// /// Full file path for this entry. /// File data for this entry. @@ -125,7 +125,7 @@ public void RenameFile(PackageEntry entry, string newFilePath) /// A new chunk file is started whenever the current one is larger than . /// Chunk files stay on disk even if the directory file is never written. /// The added entry. - /// The path has an empty file name, or contains a null character. + /// The path is rejected, see . public PackageEntry AddFile(string filePath, byte[] fileData, bool multiChunk = false) { ArgumentNullException.ThrowIfNull(filePath); @@ -150,7 +150,7 @@ public PackageEntry AddFile(string filePath, byte[] fileData, bool multiChunk = /// /// Add file to current package, reading its data from the current position of the stream to its end. /// Be careful to not add duplicate entries, because this does not check for duplicates. - /// The path is normalized and lowercased like Valve's packer does, see . + /// The path is normalized like does. /// /// Full file path for this entry. /// Stream to read the file data from. It is not disposed. @@ -160,7 +160,7 @@ public PackageEntry AddFile(string filePath, byte[] fileData, bool multiChunk = /// A new chunk file is started whenever the current one is larger than . /// Chunk files stay on disk even if the directory file is never written. /// The added entry. - /// The path has an empty file name, or contains a null character. + /// The path is rejected, see . public PackageEntry AddFile(string filePath, Stream fileData, bool multiChunk = false) { ArgumentNullException.ThrowIfNull(filePath); @@ -355,39 +355,26 @@ private static PackageEntry CreateEntry(string filePath, string paramName) } /// - /// Splits the given path into lowercased directory, file and type names. + /// Splits a file path into the directory, file and type names that stores it with, normalized like does. /// - private static (string DirectoryName, string FileName, string TypeName) ParseEntryPath(string filePath, string paramName) + /// Full file path. + /// The names, with a space for a missing directory or type name. + /// The path is rejected. + public static (string DirectoryName, string FileName, string TypeName) ParseEntryPath(string filePath) { - if (filePath.Contains('\0', StringComparison.Ordinal)) - { - throw new ArgumentException($"'{filePath}' contains a null character.", paramName); - } - - SplitPath(NormalizePath(filePath, out _), out var directory, out var fileName, out var extension); - - if (fileName.IsEmpty) - { - throw new ArgumentException($"'{filePath}' has an empty file name.", paramName); - } + ArgumentNullException.ThrowIfNull(filePath); - return (ToLowerAscii(directory), ToLowerAscii(fileName), ToLowerAscii(extension)); + return ParseEntryPath(filePath, nameof(filePath)); } - private static string ToLowerAscii(ReadOnlySpan value) + private static (string DirectoryName, string FileName, string TypeName) ParseEntryPath(string filePath, string paramName) { - if (!value.ContainsAnyInRange('A', 'Z')) + if (GetEntryPathError(filePath, out var directory, out var fileName, out var typeName) is { } error) { - return value.ToString(); + throw new ArgumentException($"'{filePath}' {error}.", paramName); } - return string.Create(value.Length, value, static (span, source) => - { - for (var i = 0; i < source.Length; i++) - { - span[i] = ToLowerAscii(source[i]); - } - }); + return (directory.ToString(), fileName.ToString(), typeName.ToString()); } private static char ToLowerAscii(char c) @@ -895,21 +882,9 @@ private static int CompareTreeOrder(PackageEntry x, PackageEntry y) /// private static int CompareNames(string x, string y) { - var length = Math.Min(x.Length, y.Length); - - for (var i = 0; i < length; i++) - { - var comparison = ToLowerAscii(x[i]).CompareTo(ToLowerAscii(y[i])); - - if (comparison != 0) - { - return comparison; - } - } - - var lengthComparison = x.Length.CompareTo(y.Length); + var comparison = AsciiComparer.Compare(x, y); - return lengthComparison != 0 ? lengthComparison : string.CompareOrdinal(x, y); + return comparison != 0 ? comparison : string.CompareOrdinal(x, y); } private static string GetTreeDirectoryName(PackageEntry entry) diff --git a/ValvePak/ValvePak/Package.cs b/ValvePak/ValvePak/Package.cs index 16cae96..dc0c5e2 100644 --- a/ValvePak/ValvePak/Package.cs +++ b/ValvePak/ValvePak/Package.cs @@ -19,6 +19,7 @@ public partial class Package : IDisposable internal const string Space = " "; internal const string Dot = "."; + internal const string DotDot = ".."; internal const string DirectorySeparator = "/"; /// @@ -26,6 +27,16 @@ public partial class Package : IDisposable /// internal const ushort DirectoryArchiveIndex = 0x7FFF; + /// + /// Maximum length in UTF-8 bytes of a . + /// + public const int MaxTypeNameLength = 128; + + /// + /// Maximum length in UTF-8 bytes of a . + /// + public const int MaxFileNameLength = 260; + /// /// Always '/' as per Valve's vpk implementation. /// @@ -265,15 +276,13 @@ private static string StripDirVpkSuffixes(string fileName, out bool isDirVpk) } /// - /// Searches for a given file entry in the file list. + /// Searches for the entry at the given path, which is normalized like does. /// - /// If was called on this package, this method will use . + /// If was called on this package, this method uses a binary search. /// Optimized packages also support case insensitive search by using a different . /// /// Full path to the file to find. - /// - /// The path is normalized the same way as paths of added files, see . - /// + /// The entry, or null if there is none or the path is rejected. public PackageEntry? FindEntry(string filePath) { ArgumentNullException.ThrowIfNull(filePath); @@ -281,58 +290,15 @@ private static string StripDirVpkSuffixes(string fileName, out bool isDirVpk) return FindEntry(filePath.AsSpan()); } - /// - /// Searches for a given file entry in the file list. - /// - /// If was called on this package, this method will use . - /// Optimized packages also support case insensitive search by using a different . - /// - /// Full path to the file to find. - /// - /// The path is normalized the same way as paths of added files: (\) becomes (/), - /// empty and "." segments and trailing dots are removed, and a missing directory or extension is a space. - /// If that finds no entry, the path is also looked up as it is, split at its last and at each dot of the file name, - /// which finds entries of read packages whose names normalizing changes. - /// The path is not lowercased like added paths are, so the search is case sensitive unless - /// was called with a case insensitive . - /// + /// public PackageEntry? FindEntry(ReadOnlySpan filePath) { - SplitPath(NormalizePath(filePath, out var changed), out var directory, out var fileName, out var extension); - - return FindEntry(directory, fileName, extension) ?? FindEntryByStoredName(filePath, changed ? -1 : fileName.Length); - } - - /// - /// Finds the entry whose is the given path, trying each split of the file name into a file and type name. - /// - /// The path as it was given. - /// Position in the file name that was already looked up, where the length of the file name means without a type, or -1. - private PackageEntry? FindEntryByStoredName(ReadOnlySpan path, int skipSplit) - { - var lastSeparator = path.LastIndexOf(DirectorySeparatorChar); - var directory = lastSeparator > -1 ? path[..lastSeparator] : Space; - var name = path[(lastSeparator + 1)..]; - PackageEntry? entry = null; - - for (var split = 1; entry == null && split <= name.Length; split++) + if (SplitEntryPath(filePath, out var directory, out var fileName, out var typeName) != null) { - if (split == skipSplit) - { - continue; - } - - if (split == name.Length) - { - entry = FindEntry(directory, name, Space); - } - else if (name[split] == '.' && split < name.Length - 1) - { - entry = FindEntry(directory, name[..split], name[(split + 1)..]); - } + return null; } - return entry; + return FindEntry(directory, fileName, typeName); } private PackageEntry? FindEntry(ReadOnlySpan directory, ReadOnlySpan fileName, ReadOnlySpan extension) @@ -407,95 +373,188 @@ private static string StripDirVpkSuffixes(string fileName, out bool isDirVpk) } /// - /// Replaces with , removes empty and "." segments, and removes trailing dots. + /// Normalizes a file path to the path that stores it at: '\' is a directory separator, + /// empty and "." segments are removed, ".." removes the segment before it, ASCII letters are lowercased, and the type name is split from the file name at its last dot. + /// + /// Full file path. + /// The normalized path, or an empty string if the path does not end with a file name, has a ".." above the root, + /// or has a name longer than or . + public static string NormalizeEntryPath(string filePath) + { + ArgumentNullException.ThrowIfNull(filePath); + + if (GetEntryPathError(filePath, out var directory, out var fileName, out var typeName) != null) + { + return string.Empty; + } + + return JoinEntryPath(directory, fileName, typeName); + } + + /// + /// Normalizes a directory path to the of the files in it, like normalizes directories. /// - private static ReadOnlySpan NormalizePath(ReadOnlySpan path, out bool changed) + /// Directory path, which may end with a directory separator. + /// The normalized path, or an empty string for the root directory or if the path has a ".." above the root. + public static string NormalizeDirectoryPath(string directoryPath) { - changed = !IsNormalizedPath(path); + ArgumentNullException.ThrowIfNull(directoryPath); - if (!changed) + if (SplitEntryPath(string.Concat(directoryPath, "/x"), out var directory, out _, out _) != null || directory is Space) { - return path; + return string.Empty; } - var builder = new StringBuilder(path.Length); + return directory.ToString(); + } + + /// + /// Splits a file path like does and checks the lengths of the names. + /// + /// Why the path can not be stored, or null. + private static string? GetEntryPathError(ReadOnlySpan path, out ReadOnlySpan directory, out ReadOnlySpan fileName, out ReadOnlySpan typeName) + { + var error = SplitEntryPath(path, out directory, out fileName, out typeName); + + if (error != null) + { + return error; + } + + if (Encoding.UTF8.GetByteCount(typeName) > MaxTypeNameLength) + { + return "has a type name longer than 128 bytes in UTF-8"; + } + + if (Encoding.UTF8.GetByteCount(fileName) > MaxFileNameLength) + { + return "has a file name longer than 260 bytes in UTF-8"; + } + + return null; + } + + /// + /// Normalizes a file path like does and splits it into the directory, file and type names it is stored as. + /// + /// Why no file can be at the path, or null. + private static string? SplitEntryPath(ReadOnlySpan path, out ReadOnlySpan directory, out ReadOnlySpan fileName, out ReadOnlySpan typeName) + { + directory = fileName = typeName = default; + + if (path.IsEmpty || IsDirectorySeparator(path[^1]) || path[(path.LastIndexOfAny(DirectorySeparatorChar, WindowsDirectorySeparator) + 1)..] is Dot or DotDot) + { + return "does not end with a file name"; + } + + var normalizedPath = path; + + if (!IsNormalizedPath(path)) + { + if (ResolveSegments(path) is not { } resolvedPath) + { + return "has a \"..\" segment above the root"; + } + + normalizedPath = resolvedPath; + } + + var lastSeparator = normalizedPath.LastIndexOf(DirectorySeparatorChar); + var file = normalizedPath[(lastSeparator + 1)..]; + var dot = file.LastIndexOf('.'); + + if (dot == 0 || dot == file.Length - 1) + { + return "has a file name that is empty or ends with a dot"; + } + + directory = lastSeparator > -1 ? normalizedPath[..lastSeparator] : Space; + fileName = dot > -1 ? file[..dot] : file; + typeName = dot > -1 ? file[(dot + 1)..] : Space; + + return null; + } + + /// + /// Joins the segments of the path with '/' and lowercases ASCII letters, leaving out empty and "." segments and removing the segment before each "..". + /// + /// The resolved path, or null if a ".." has no segment before it. + private static string? ResolveSegments(ReadOnlySpan path) + { + var buffer = new char[path.Length]; + var length = 0; foreach (var range in path.SplitAny(DirectorySeparatorChar, WindowsDirectorySeparator)) { var segment = path[range]; - if (IsSkippedSegment(segment)) + if (segment.IsEmpty || segment is Dot) { continue; } - if (builder.Length > 0) + if (segment is DotDot) { - builder.Append(DirectorySeparatorChar); - } + if (length == 0) + { + return null; + } - builder.Append(segment); - } + length = Math.Max(buffer.AsSpan(0, length).LastIndexOf(DirectorySeparatorChar), 0); + continue; + } - var length = builder.Length; + if (length > 0) + { + buffer[length++] = DirectorySeparatorChar; + } - while (length > 0 && builder[length - 1] == '.') - { - length--; + foreach (var c in segment) + { + buffer[length++] = ToLowerAscii(c); + } } - return builder.ToString(0, length); + return new string(buffer, 0, length); } + /// + /// Returns whether normalizing leaves the path as it is, because it has no '\', no upper case ASCII letters and no empty, "." or ".." segments. + /// private static bool IsNormalizedPath(ReadOnlySpan path) { - if (path.Contains(WindowsDirectorySeparator)) + if (path.Contains(WindowsDirectorySeparator) || path.ContainsAnyInRange('A', 'Z')) { return false; } foreach (var range in path.Split(DirectorySeparatorChar)) { - if (IsSkippedSegment(path[range])) + if (path[range] is "" or Dot or DotDot) { return false; } } - return !path.EndsWith('.'); + return true; } - private static bool IsSkippedSegment(ReadOnlySpan segment) + private static string JoinEntryPath(ReadOnlySpan directory, ReadOnlySpan fileName, ReadOnlySpan typeName) { - return segment.IsEmpty || segment is Dot; + var file = typeName is Space ? fileName.ToString() : string.Concat(fileName, Dot, typeName); + + return directory is Space ? file : string.Concat(directory, DirectorySeparator, file); } - /// - /// Splits a normalized path into the directory, file and type names of an entry, using a space for a missing directory or extension. - /// - private static void SplitPath(ReadOnlySpan path, out ReadOnlySpan directory, out ReadOnlySpan fileName, out ReadOnlySpan extension) + private static bool IsDirectorySeparator(char c) { - var lastSeparator = path.LastIndexOf(DirectorySeparatorChar); - directory = lastSeparator > -1 ? path[..lastSeparator] : Space; - fileName = path[(lastSeparator + 1)..]; - - var dot = fileName.LastIndexOf('.'); - extension = Space; - - if (dot > -1) - { - extension = fileName[(dot + 1)..]; - fileName = fileName[..dot]; - } + return c is DirectorySeparatorChar or WindowsDirectorySeparator; } /// /// This sorts so that it can be searched through using binary search. /// Use if you want to search case insensitively. /// - /// - /// This is experimental and may be removed in a future release. - /// /// Comparison method to use. public void OptimizeEntriesForBinarySearch(StringComparison comparison = StringComparison.Ordinal) { @@ -506,5 +565,62 @@ public void OptimizeEntriesForBinarySearch(StringComparison comparison = StringC Comparer = new CaseInsensitivePackageEntryComparer(comparison); } + + private static readonly AsciiCaseInsensitiveComparer AsciiComparer = new(); + + /// + /// Gets the comparer that compares paths and names ordinally with only ASCII letters case insensitive, which also looks up keys in dictionaries. + /// + public static StringComparer PathComparer => AsciiComparer; + + private sealed class AsciiCaseInsensitiveComparer : StringComparer, IAlternateEqualityComparer, string> + { + public override int Compare(string? x, string? y) + { + return x == null || y == null ? string.CompareOrdinal(x, y) : CompareAscii(x, y); + } + + public override bool Equals(string? x, string? y) + { + return x == null || y == null ? x == y : Equals(x.AsSpan(), y); + } + + public bool Equals(ReadOnlySpan alternate, string other) + { + return alternate.Length == other.Length && CompareAscii(alternate, other) == 0; + } + + public override int GetHashCode(string obj) + { + return GetHashCode(obj.AsSpan()); + } + + public int GetHashCode(ReadOnlySpan alternate) + { + return string.GetHashCode(alternate, StringComparison.OrdinalIgnoreCase); + } + + public string Create(ReadOnlySpan alternate) + { + return alternate.ToString(); + } + + private static int CompareAscii(ReadOnlySpan x, ReadOnlySpan y) + { + var length = Math.Min(x.Length, y.Length); + + for (var i = 0; i < length; i++) + { + var comparison = ToLowerAscii(x[i]).CompareTo(ToLowerAscii(y[i])); + + if (comparison != 0) + { + return comparison; + } + } + + return x.Length.CompareTo(y.Length); + } + } } } diff --git a/ValvePak/ValvePak/PackageEntry.cs b/ValvePak/ValvePak/PackageEntry.cs index 31d04ad..fbe2f4e 100644 --- a/ValvePak/ValvePak/PackageEntry.cs +++ b/ValvePak/ValvePak/PackageEntry.cs @@ -90,18 +90,30 @@ public string GetFileName() return string.Concat(fileName, Package.Dot, TypeName); } + /// + /// Returns , or an empty string for the root directory, which is stored as a single space or an empty string. + /// +#pragma warning disable CA1024 // Use properties where appropriate + public string GetDirectoryPath() +#pragma warning restore CA1024 + { + return DirectoryName is Package.Space or "" ? string.Empty : DirectoryName; + } + /// /// Returns the absolute path of the file in the package. /// /// Absolute path. public string GetFullPath() { - if (DirectoryName == Package.Space) + var directory = GetDirectoryPath(); + + if (directory.Length == 0) { return GetFileName(); } - return string.Concat(DirectoryName, Package.DirectorySeparator, GetFileName()); + return string.Concat(directory, Package.DirectorySeparator, GetFileName()); } /// From 5ef2c4e6b8d7d47ce8e3b34837ef4e8b92a87d53 Mon Sep 17 00:00:00 2001 From: w1tcherrr <70418164+w1tcherrr@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:14:06 +0200 Subject: [PATCH 2/7] Add RenameDirectory, RemoveDirectory and RequiresRepack, move the data of a package without chunk files into chunk files when it is repacked into a _dir.vpk, and write zero-length entries at the data offset --- ValvePak/ValvePak.Test/MultiChunkWriteTest.cs | 65 ++++++++ ValvePak/ValvePak.Test/WriteTest.cs | 20 ++- ValvePak/ValvePak/Package.Save.cs | 148 +++++++++++++++++- 3 files changed, 225 insertions(+), 8 deletions(-) diff --git a/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs b/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs index 8b1babc..c169f17 100644 --- a/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs +++ b/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs @@ -88,6 +88,22 @@ public async Task WriteMultiChunkPackage() await Assert.That(dirEntry.ArchiveIndex).IsEqualTo((ushort)0x7FFF); } + [Test] + public async Task WriteDeletesStaleChunkFilesOfNewPackage() + { + var dirPath = TempPath("stale_dir.vpk"); + await File.WriteAllBytesAsync(TempPath("stale_001.vpk"), [1]); + + using (var package = new Package()) + { + package.SetFileName(dirPath); + package.AddFile("a.bin", CreateTestData(200, 1), multiChunk: true); + package.Write(dirPath); + } + + await Assert.That(File.Exists(TempPath("stale_001.vpk"))).IsFalse(); + } + [Test] public async Task WritesChunkHashFractions() { @@ -847,6 +863,55 @@ public async Task RepackRefusesToReplaceChunkFilesOfTheReadFileFromAnotherPath() await AssertPackageVerifies(package); } + [Test] + public async Task RequiresRepackForPathsTheChunkFilesAreNotNamedAfter() + { + var dirPath = CopyFixtures("test_dir.vpk", "test_000.vpk"); + var otherChunkNames = TempPath("copy_dir.vpk"); + + using var package = new Package(); + package.Read(dirPath); + + using (Assert.Multiple()) + { + await Assert.That(package.RequiresRepack(dirPath)).IsFalse(); + await Assert.That(package.RequiresRepack(TempPath("test.vpk"))).IsFalse(); + await Assert.That(package.RequiresRepack(otherChunkNames)).IsTrue(); + await Assert.That(() => package.Write(otherChunkNames)).ThrowsExactly(); + } + + package.Write(otherChunkNames, package.RequiresRepack(otherChunkNames)); + await AssertPackageVerifies(package); + } + + [Test] + public async Task RepackIntoDirVpkMovesDirectoryFileData() + { + var path = TempPath("single.vpk"); + var dirPath = TempPath("single_dir.vpk"); + + using (var package = new Package()) + { + package.AddFile("a.bin", CreateTestData(400, 1)); + package.Write(path); + } + + using (var package = new Package()) + { + package.Read(path); + package.AddFile("b.bin", CreateTestData(300, 2)); + package.Write(dirPath, repack: true); + } + + using var readBack = new Package(); + readBack.Read(dirPath); + + await Assert.That(readBack.EnumerateEntries().All(static entry => entry.ArchiveIndex == 0)).IsTrue(); + await AssertEntryData(readBack, "a.bin", 400, 1); + await AssertEntryData(readBack, "b.bin", 300, 2); + await AssertPackageVerifies(readBack); + } + private string TempPath(string fileName) { return Path.Combine(TempDirectory, fileName); diff --git a/ValvePak/ValvePak.Test/WriteTest.cs b/ValvePak/ValvePak.Test/WriteTest.cs index 934f96d..25fe62d 100644 --- a/ValvePak/ValvePak.Test/WriteTest.cs +++ b/ValvePak/ValvePak.Test/WriteTest.cs @@ -329,7 +329,7 @@ public async Task WritesUnsignedSignatureSectionAndHashesEmbeddedData(int dataLe using (Assert.Multiple()) { await Assert.That(emptyEntry.ArchiveIndex).IsEqualTo((ushort)0x7FFF); - await Assert.That(emptyEntry.Offset).IsEqualTo(uint.MaxValue); + await Assert.That(emptyEntry.Offset).IsEqualTo((uint)dataLength); await Assert.That(emptyEntry.Length).IsZero(); await Assert.That(readBack.ArchiveMD5SectionSize).IsEqualTo((uint)(fractionCount * ChunkHashFraction.SectionEntrySize)); await Assert.That(signatureSection.Length).IsEqualTo(20); @@ -682,6 +682,24 @@ public async Task RenameFile() } } + [Test] + public async Task RenamesAndRemovesDirectories() + { + using var package = new Package(); + var file = package.AddFile("materials/a.txt", [1]); + var nested = package.AddFile("materials/sub/b.txt", [2]); + var sibling = package.AddFile("materialsx/c.txt", [3]); + + using (Assert.Multiple()) + { + await Assert.That(package.RenameDirectory("materials/", "Renamed")).IsEquivalentTo([file, nested], CollectionOrdering.Any); + await Assert.That(nested.GetFullPath()).IsEqualTo("renamed/sub/b.txt"); + await Assert.That(sibling.GetFullPath()).IsEqualTo("materialsx/c.txt"); + await Assert.That(package.RemoveDirectory("renamed")).IsEquivalentTo([file, nested], CollectionOrdering.Any); + await Assert.That(package.Entries!["txt"]).IsEquivalentTo([sibling], CollectionOrdering.Matching); + } + } + private static Package ReadBack(Stream stream) { var package = new Package(); diff --git a/ValvePak/ValvePak/Package.Save.cs b/ValvePak/ValvePak/Package.Save.cs index 5924e99..6ae6b8c 100644 --- a/ValvePak/ValvePak/Package.Save.cs +++ b/ValvePak/ValvePak/Package.Save.cs @@ -113,6 +113,101 @@ public void RenameFile(PackageEntry entry, string newFilePath) InsertEntry(entry); } + /// + /// Removes every entry in a directory and its subdirectories from current package. + /// + /// The directory as entries store it in , with or without a trailing '/'. + /// The removed entries. + /// is the root. + public IReadOnlyList RemoveDirectory(string directory) + { + ArgumentNullException.ThrowIfNull(directory); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var removed = GetEntriesInDirectory(TrimStoredDirectory(directory, nameof(directory))); + + RemoveEntries(removed); + + return removed; + } + + /// + /// Moves every entry in a directory and its subdirectories into another directory. Only the directory tree changes, the file data is not touched. + /// Be careful to not move entries onto the paths of other entries, because this does not check for duplicates. + /// + /// The directory as entries store it, see . + /// The directory to move the entries into, normalized like does. + /// The moved entries. + /// or is the root. + public IReadOnlyList RenameDirectory(string directory, string newDirectory) + { + ArgumentNullException.ThrowIfNull(directory); + ArgumentNullException.ThrowIfNull(newDirectory); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var storedDirectory = TrimStoredDirectory(directory, nameof(directory)); + var targetDirectory = NormalizeDirectoryPath(newDirectory); + + if (targetDirectory.Length == 0) + { + throw new ArgumentException($"'{newDirectory}' is the root.", nameof(newDirectory)); + } + + var moved = GetEntriesInDirectory(storedDirectory); + + RemoveEntries(moved); + + foreach (var entry in moved) + { + entry.DirectoryName = string.Concat(targetDirectory, entry.DirectoryName.AsSpan(storedDirectory.Length)); + + InsertEntry(entry); + } + + return moved; + } + + private static string TrimStoredDirectory(string directory, string paramName) + { + if (directory.EndsWith(DirectorySeparatorChar)) + { + directory = directory[..^1]; + } + + if (directory is "" or Space) + { + throw new ArgumentException("The root directory can not be renamed or removed.", paramName); + } + + return directory; + } + + /// + /// Returns the entries whose stored directory is the given directory or one of its subdirectories, with ASCII letters case insensitive. + /// + private List GetEntriesInDirectory(string directory) + { + return [.. GetAllEntries().Where(entry => entry.DirectoryName.Length >= directory.Length + && (entry.DirectoryName.Length == directory.Length || entry.DirectoryName[directory.Length] == DirectorySeparatorChar) + && AsciiComparer.Equals(entry.DirectoryName.AsSpan(0, directory.Length), directory))]; + } + + private void RemoveEntries(List entries) + { + var removed = entries.ToHashSet(); + + foreach (var typeName in removed.Select(static entry => entry.TypeName).Distinct()) + { + var typeEntries = Entries![typeName]; + typeEntries.RemoveAll(removed.Contains); + + if (typeEntries.Count == 0) + { + Entries.Remove(typeName); + } + } + } + /// /// Add file to current package. Be careful to not add duplicate entries, because this does not check for duplicates. /// The path is normalized like does. @@ -435,6 +530,7 @@ private static bool IsInDirectoryFile(PackageEntry entry) /// If the package has chunk files, from being read or from files added with multi chunk enabled (see ), /// this only writes the directory file, which refers to the data where it is in the chunk files. /// Unless is enabled, it must be written to the path the chunk files are named after, which should end with "_dir.vpk". + /// A package that was not read also deletes existing chunk files numbered after the ones it wrote, until a number has no file. /// Hashes of chunk files are kept if they cover the whole file, computed again otherwise, and left out for chunk files that had none. /// /// Writing over the file the package was read from, or repacking, writes temporary files that replace the existing files once all were written, @@ -446,10 +542,14 @@ private static bool IsInDirectoryFile(PackageEntry entry) /// /// The file to open and write. /// If true, the data in chunk files is copied into new chunk files named after in directory tree order, + /// or for a package without chunk files written to a "_dir.vpk", the data in the directory file, /// which drops the data of removed and replaced files. Either all existing files are replaced or none are. /// If the package has chunk files with that name and was read from a file, it must be written to that file. /// If new chunk files were written or the package had chunk files with that name, existing chunk files numbered after the new ones are then deleted where possible /// until a number has no file, and in the latter case also up to the highest number the package had. + /// is false and is true, + /// is true and the package has chunk files named after but was read from another file, + /// or is the file the package was read from inside a larger file. public void Write(string filename, bool repack = false) { ArgumentNullException.ThrowIfNull(filename); @@ -464,7 +564,7 @@ public void Write(string filename, bool repack = false) } var baseName = StripDirVpkSuffixes(fullPath, out var isDirVpk); - var writesOverChunkFiles = HighestChunkIndex >= 0 && PathsEqual(baseName, GetChunkBaseName()); + var writesOverChunkFiles = WritesOverChunkFiles(baseName); // Readers find chunk files by appending "_###.vpk" to the directory file name without the "_dir.vpk" suffix if (!repack && HighestChunkIndex >= 0 && !writesOverChunkFiles) @@ -499,6 +599,11 @@ public void Write(string filename, bool repack = false) output.SetLength(output.Position); + if (!IsReadPackage && writesOverChunkFiles) + { + DeleteStaleChunkFiles(baseName, HighestChunkIndex + 1, staleHighestIndex: -1); + } + return; } @@ -513,7 +618,7 @@ public void Write(string filename, bool repack = false) if (repack) { - (chunkLocations, chunkHashFractions) = WriteRepackedChunkFiles(entries, baseName, token, chunkReplacements); + (chunkLocations, chunkHashFractions) = WriteRepackedChunkFiles(entries, baseName, isDirVpk && HighestChunkIndex < 0, token, chunkReplacements); } else { @@ -568,6 +673,26 @@ public void Write(string filename, bool repack = false) } } + /// + /// Returns whether has to repack to write the directory file to the given path, because the package has chunk files named after another path. + /// + /// The file the directory file is going to be written to. + public bool RequiresRepack(string filename) + { + ArgumentNullException.ThrowIfNull(filename); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + return HighestChunkIndex >= 0 && !WritesOverChunkFiles(StripDirVpkSuffixes(Path.GetFullPath(filename), out _)); + } + + /// + /// Returns whether the package has chunk files and they are named after the given base name. + /// + private bool WritesOverChunkFiles(string baseName) + { + return HighestChunkIndex >= 0 && PathsEqual(baseName, GetChunkBaseName()); + } + private static string GetTemporaryPath(string path, string token) { return $"{path}.{token}.tmp"; @@ -582,6 +707,7 @@ private void ApplyRepackedLayout(Dictionary chunkLo { entry.ArchiveIndex = archiveIndex; entry.Offset = offset; + entry.PendingData = null; } FileName = baseName; @@ -647,11 +773,11 @@ private List ReplaceFilesOrReopen(List chunkReplacement } /// - /// Copies the data of all entries in chunk files into new chunk files under temporary names, adding each of them to . + /// Copies the data of all entries in chunk files, and optionally in the directory file, into new chunk files under temporary names, adding each of them to . /// /// Where the data of each copied entry was written to, and the hashes of the new chunk files or null if there are none. private (Dictionary Locations, List? HashFractions) WriteRepackedChunkFiles( - List entries, string baseName, string token, List replacements) + List entries, string baseName, bool includesDirectoryFileData, string token, List replacements) { var locations = new Dictionary(); var hashFractions = new List(); @@ -661,7 +787,7 @@ private List ReplaceFilesOrReopen(List chunkReplacement { foreach (var entry in entries) { - if (!IsInChunkFile(entry)) + if (!IsInChunkFile(entry) && !(includesDirectoryFileData && IsInDirectoryFile(entry))) { continue; } @@ -686,7 +812,15 @@ private List ReplaceFilesOrReopen(List chunkReplacement } locations[entry] = new ChunkLocation(chunkFile.ArchiveIndex, chunkFile.Length); - ReadArchiveDataInBlocks(entry.ArchiveIndex, entry.Offset, entry.Length, chunkFile.Write); + + if (entry.PendingData != null) + { + chunkFile.Write(entry.PendingData); + } + else + { + ReadArchiveDataInBlocks(entry.ArchiveIndex, entry.Offset, entry.Length, chunkFile.Write); + } } if (chunkFile == null) @@ -929,7 +1063,7 @@ void WriteNullTerminated(string value) if (entry.Length == 0) { archiveIndex = DirectoryArchiveIndex; - fileOffset = uint.MaxValue; + fileOffset = embeddedDataOffset; } else if (chunkLocations != null && chunkLocations.TryGetValue(entry, out var location)) { From 946c0aebbee2f9034a8d4fd9f6a1bd5c3f4ea455 Mon Sep 17 00:00:00 2001 From: w1tcherrr <70418164+w1tcherrr@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:14:06 +0200 Subject: [PATCH 3/7] Add DirectoryArchiveIndex, HasChunkFiles, chunk file paths, FilePath, EnumerateEntries, EntryCount, size limit constants and OpenArchiveStream --- ValvePak/ValvePak.Test/PackageTest.cs | 52 ++++++++ ValvePak/ValvePak/Archive.cs | 5 + ValvePak/ValvePak/ArchiveRangeStream.cs | 162 ++++++++++++++++++++++++ ValvePak/ValvePak/FileArchive.cs | 2 + ValvePak/ValvePak/Package.Read.cs | 86 +++++++++++++ ValvePak/ValvePak/Package.Save.cs | 18 +-- ValvePak/ValvePak/Package.cs | 47 ++++++- ValvePak/ValvePak/StreamArchive.cs | 11 ++ 8 files changed, 373 insertions(+), 10 deletions(-) create mode 100644 ValvePak/ValvePak/ArchiveRangeStream.cs diff --git a/ValvePak/ValvePak.Test/PackageTest.cs b/ValvePak/ValvePak.Test/PackageTest.cs index fd6f4a8..4b804e2 100644 --- a/ValvePak/ValvePak.Test/PackageTest.cs +++ b/ValvePak/ValvePak.Test/PackageTest.cs @@ -2,6 +2,7 @@ using System.Buffers.Binary; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; @@ -637,6 +638,57 @@ public async Task TestVerifyChunkHashesBlake3() await Assert.That(() => package.VerifyChunkHashes(null)).ThrowsNothing(); } + [Test] + public async Task BuildsAndParsesChunkFilePaths() + { + using var package = new Package(); + package.Read(Path.Combine(AppContext.BaseDirectory, "Files", "test_dir.vpk")); + + using (Assert.Multiple()) + { + await Assert.That(package.GetChunkFilePath(1)).IsEqualTo(Path.Combine(AppContext.BaseDirectory, "Files", "test_001.vpk")); + await Assert.That(Package.TryParseChunkFilePath("a_b_042.VPK", out var baseName, out var archiveIndex)).IsTrue(); + await Assert.That(baseName).IsEqualTo("a_b"); + await Assert.That(archiveIndex).IsEqualTo((ushort)42); + await Assert.That(Package.TryParseChunkFilePath("a_dir.vpk", out _, out _)).IsFalse(); + } + } + + [Test] + public async Task HasChunkFilesWithoutDirSuffix() + { + using var package = new Package(); + package.Read(Path.Combine(AppContext.BaseDirectory, "Files", "test_without_suffix.vpk")); + + using (Assert.Multiple()) + { + await Assert.That(package.HasChunkFiles).IsTrue(); + await Assert.That(package.IsDirVPK).IsFalse(); + } + } + + [Test] + [Arguments("test_dir.vpk", (ushort)0)] + [Arguments("test_single.vpk", Package.DirectoryArchiveIndex)] + public async Task OpensArchiveStreamsAtEntryOffsets(string fileName, ushort archiveIndex) + { + using var package = new Package(); + package.Read(Path.Combine(AppContext.BaseDirectory, "Files", fileName)); + + using var stream = package.OpenArchiveStream(archiveIndex); + + foreach (var entry in package.EnumerateEntries().Where(entry => entry.ArchiveIndex == archiveIndex)) + { + package.ReadEntry(entry, out var expected); + var data = new byte[entry.Length]; + + stream.Position = entry.Offset; + stream.ReadExactly(data); + + await Assert.That(data).IsEquivalentTo(expected[entry.SmallData.Length..], CollectionOrdering.Matching); + } + } + [Test] public async Task SetFileNameStripsVpkExtensionAndDirSuffix() { diff --git a/ValvePak/ValvePak/Archive.cs b/ValvePak/ValvePak/Archive.cs index 861447a..8136bf3 100644 --- a/ValvePak/ValvePak/Archive.cs +++ b/ValvePak/ValvePak/Archive.cs @@ -25,6 +25,11 @@ protected Archive(long origin, long dataOffset) /// protected long Origin { get; } + /// + /// Gets the current length of the archive from its start. + /// + public abstract long Length { get; } + /// /// Reads data at the given offset until the output is full or the end of the archive is reached. /// This method is safe to call from multiple threads. diff --git a/ValvePak/ValvePak/ArchiveRangeStream.cs b/ValvePak/ValvePak/ArchiveRangeStream.cs new file mode 100644 index 0000000..1d5e7a8 --- /dev/null +++ b/ValvePak/ValvePak/ArchiveRangeStream.cs @@ -0,0 +1,162 @@ +using System; +using System.IO; + +namespace ValvePak +{ + /// + /// Read-only seekable stream over a range of an archive, which it reads from without owning it, + /// optionally preceded by bytes held in memory such as the preload bytes of an entry. + /// + internal sealed class ArchiveRangeStream : Stream + { +#pragma warning disable CA2213 // Disposable fields should be disposed + private readonly Archive Archive; +#pragma warning restore CA2213 + private readonly long Start; + private readonly long RangeLength; + private readonly byte[] Prefix; + private long CurrentPosition; + private bool IsDisposed; + + /// + /// Creates a stream over the given range of the archive, which starts with if given. + /// + public ArchiveRangeStream(Archive archive, long start, long length, byte[]? prefix = null) + { + Archive = archive; + Start = start; + RangeLength = length; + Prefix = prefix ?? []; + } + + public override bool CanRead => !IsDisposed; + + public override bool CanSeek => !IsDisposed; + + public override bool CanWrite => false; + + public override long Length => Prefix.Length + RangeLength; + + public override long Position + { + get => CurrentPosition; + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + CurrentPosition = value; + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + ValidateBufferArguments(buffer, offset, count); + + return Read(buffer.AsSpan(offset, count)); + } + + public override int Read(Span buffer) + { + var bytesRead = ReadAt(CurrentPosition, buffer); + CurrentPosition += bytesRead; + + return bytesRead; + } + + /// + /// Reads at the given position of the stream without moving its position. + /// This method is safe to call from multiple threads. + /// + /// The number of bytes read, which is less than requested only at the end of the stream or of the archive. + public int ReadAt(long position, Span buffer) + { + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var prefixRead = 0; + + if (position < Prefix.Length) + { + prefixRead = Math.Min(Prefix.Length - (int)position, buffer.Length); + Prefix.AsSpan((int)position, prefixRead).CopyTo(buffer); + position += prefixRead; + buffer = buffer[prefixRead..]; + } + + var available = Length - position; + + if (available <= 0 || buffer.IsEmpty) + { + return prefixRead; + } + + if (buffer.Length > available) + { + buffer = buffer[..(int)available]; + } + + return prefixRead + Archive.Read(Start + position - Prefix.Length, buffer); + } + + /// + /// Returns a stream that reads the given range of this stream directly from a memory mapping of the archive. + /// + /// The stream, or null if the archive can not be mapped or the range is not entirely in the archive range. + public Stream? OpenMappedStream(long position, long length) + { + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var offset = position - Prefix.Length; + + if (offset < 0 || length > RangeLength - offset) + { + return null; + } + + return Archive.OpenMappedStream(Start + offset, length); + } + + public override long Seek(long offset, SeekOrigin origin) + { + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => CurrentPosition + offset, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentOutOfRangeException(nameof(origin)), + }; + + if (position < 0) + { + throw new IOException("An attempt was made to move the position before the beginning of the stream."); + } + + CurrentPosition = position; + + return position; + } + + public override void Flush() + { + } + + public override void SetLength(long value) + { + throw new NotSupportedException("The stream is read-only."); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException("The stream is read-only."); + } + + protected override void Dispose(bool disposing) + { + IsDisposed = true; + + base.Dispose(disposing); + } + } +} diff --git a/ValvePak/ValvePak/FileArchive.cs b/ValvePak/ValvePak/FileArchive.cs index 5a7668f..9196f4c 100644 --- a/ValvePak/ValvePak/FileArchive.cs +++ b/ValvePak/ValvePak/FileArchive.cs @@ -25,6 +25,8 @@ public FileArchive(SafeFileHandle handle, bool ownsHandle, long origin, long dat OwnsHandle = ownsHandle; } + public override long Length => RandomAccess.GetLength(Handle) - Origin; + public override int Read(long offset, Span output) { var totalRead = 0; diff --git a/ValvePak/ValvePak/Package.Read.cs b/ValvePak/ValvePak/Package.Read.cs index c20d6b1..975a6d8 100644 --- a/ValvePak/ValvePak/Package.Read.cs +++ b/ValvePak/ValvePak/Package.Read.cs @@ -3,6 +3,8 @@ using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; using System.IO; using System.IO.Hashing; using System.Linq; @@ -604,6 +606,90 @@ public Stream GetMemoryMappedStreamIfPossible(PackageEntry entry) return new MemoryStream(output); } + /// + /// Returns a read-only stream over a whole chunk file, or for over the file data in the directory file, at positions. + /// Like streams from , it can only be read while the package is open and has not replaced the file it reads from. + /// + /// Index of the chunk file, or . + /// The chunk file does not exist. + public Stream OpenArchiveStream(ushort archiveIndex) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan(archiveIndex, DirectoryArchiveIndex); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var archive = GetArchive(archiveIndex); + var length = Math.Max(0, archive.Length - archive.DataOffset); + + if (archiveIndex == DirectoryArchiveIndex && Version != 1) + { + length = Math.Min(length, FileDataSectionSize); + } + + return archive.OpenMappedStream(archive.DataOffset, length) ?? new ArchiveRangeStream(archive, archive.DataOffset, length); + } + + /// + /// Returns the full path of a chunk file of this package, which does not have to exist, such as "pak01_003.vpk" for "pak01_dir.vpk". + /// + /// Index of the chunk file, up to . + /// The package has no file name. + public string GetChunkFilePath(ushort archiveIndex) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan(archiveIndex, MaxChunkIndex); + + return GetArchiveIndexFullFilePath(GetChunkBaseName(), archiveIndex); + } + + /// + /// Parses a path in the form that returns: a base name, an underscore, + /// three digits or four to five digits without a leading zero, and ".vpk". + /// + /// The path to parse. + /// The path without the underscore, the index and ".vpk". + /// The index of the chunk file. + /// True if the path is in the form of a chunk file path. + public static bool TryParseChunkFilePath(string path, [NotNullWhen(true)] out string? baseName, out ushort archiveIndex) + { + ArgumentNullException.ThrowIfNull(path); + + const string Extension = ".vpk"; + + baseName = null; + archiveIndex = 0; + + if (!path.EndsWith(Extension, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var name = path.AsSpan(0, path.Length - Extension.Length); + var separator = name.LastIndexOf('_'); + + if (separator < 1) + { + return false; + } + + var digits = name[(separator + 1)..]; + + if (digits.Length is < 3 or > 5 || digits.ContainsAnyExceptInRange('0', '9') || (digits.Length > 3 && digits[0] == '0')) + { + return false; + } + + var index = int.Parse(digits, NumberStyles.None, CultureInfo.InvariantCulture); + + if (index > MaxChunkIndex) + { + return false; + } + + baseName = path[..separator]; + archiveIndex = (ushort)index; + + return true; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string GetArchiveIndexFullFilePath(string? baseFileName, ushort archiveIndex) { diff --git a/ValvePak/ValvePak/Package.Save.cs b/ValvePak/ValvePak/Package.Save.cs index 6ae6b8c..a643bc6 100644 --- a/ValvePak/ValvePak/Package.Save.cs +++ b/ValvePak/ValvePak/Package.Save.cs @@ -271,9 +271,9 @@ public PackageEntry AddFile(string filePath, Stream fileData, bool multiChunk = var knownLength = fileData.CanSeek ? fileData.Length - fileData.Position : 0; - if (knownLength > uint.MaxValue) + if (knownLength > MaxChunkFileSize) { - throw new ArgumentException($"The stream has {knownLength} bytes left, a chunk file can hold at most {uint.MaxValue}.", nameof(fileData)); + throw new ArgumentException($"The stream has {knownLength} bytes left, a chunk file can hold at most {MaxChunkFileSize}.", nameof(fileData)); } var buffer = ArrayPool.Shared.Rent(CopyBufferSize); @@ -401,12 +401,12 @@ private ChunkFileWriter GetChunkFileForNextFile(long length) /// Size in bytes of the next file, or 0 if it is not known. private bool HasRoomForNextFile([NotNullWhen(true)] ChunkFileWriter? chunkFile, long length) { - return chunkFile != null && chunkFile.Length <= (uint)WriteChunkSize && chunkFile.Length + length <= uint.MaxValue; + return chunkFile != null && chunkFile.Length <= (uint)WriteChunkSize && chunkFile.Length + length <= MaxChunkFileSize; } private static void ThrowIfChunkLimitReached(int archiveIndex) { - if (archiveIndex >= DirectoryArchiveIndex) + if (archiveIndex > MaxChunkIndex) { throw new InvalidOperationException("Reached the maximum amount of chunk files (32767)."); } @@ -1071,7 +1071,7 @@ void WriteNullTerminated(string value) } else if (archiveIndex == DirectoryArchiveIndex) { - if ((ulong)embeddedDataOffset + entry.Length > int.MaxValue) + if ((ulong)embeddedDataOffset + entry.Length > MaxDirectoryFileDataSize) { throw new InvalidOperationException("Package contents exceed 2GiB, add files with multi chunk enabled to split them into chunk files."); } @@ -1081,9 +1081,9 @@ void WriteNullTerminated(string value) embeddedEntries.Add(entry); } - if (entry.SmallData.Length > ushort.MaxValue) + if (entry.SmallData.Length > MaxPreloadSize) { - throw new InvalidOperationException($"Entry '{entry.GetFullPath()}' has {entry.SmallData.Length} preload bytes, the directory tree can store at most {ushort.MaxValue}."); + throw new InvalidOperationException($"Entry '{entry.GetFullPath()}' has {entry.SmallData.Length} preload bytes, the directory tree can store at most {MaxPreloadSize}."); } WriteNullTerminated(entry.FileName); @@ -1235,7 +1235,7 @@ private static bool PathsEqual(string path1, string path2) if (!CoversFile(fractions, file.Length)) { - if (file.Length > uint.MaxValue) + if (file.Length > MaxChunkFileSize) { throw new InvalidOperationException($"Chunk file {archiveIndex} is {file.Length} bytes, its hashes can only be computed for chunk files smaller than 4 GiB."); } @@ -1412,7 +1412,7 @@ public ChunkHasher(ushort archiveIndex) public void Append(ReadOnlySpan data) { // Offsets and lengths in the directory tree are 32-bit - if ((long)Length + data.Length > uint.MaxValue) + if ((long)Length + data.Length > MaxChunkFileSize) { throw new InvalidOperationException($"Chunk file {ArchiveIndex} would exceed 4 GiB."); } diff --git a/ValvePak/ValvePak/Package.cs b/ValvePak/ValvePak/Package.cs index dc0c5e2..b40fc3b 100644 --- a/ValvePak/ValvePak/Package.cs +++ b/ValvePak/ValvePak/Package.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using System.Threading; @@ -25,7 +26,27 @@ public partial class Package : IDisposable /// /// Archive index of entries whose data is in the directory file instead of a chunk file. /// - internal const ushort DirectoryArchiveIndex = 0x7FFF; + public const ushort DirectoryArchiveIndex = 0x7FFF; + + /// + /// Highest archive index of a chunk file. + /// + public const ushort MaxChunkIndex = 0x7FFE; + + /// + /// Maximum size in bytes of the file data in a directory file. + /// + public const int MaxDirectoryFileDataSize = int.MaxValue; + + /// + /// Maximum size in bytes of a chunk file. + /// + public const uint MaxChunkFileSize = uint.MaxValue; + + /// + /// Maximum number of preload bytes () of an entry. + /// + public const ushort MaxPreloadSize = ushort.MaxValue; /// /// Maximum length in UTF-8 bytes of a . @@ -60,11 +81,22 @@ public partial class Package : IDisposable /// public string? FileName { get; private set; } + /// + /// Gets the full path of the directory file that this package reads from, or null if it does not read from the start of a file. + /// + public string? FilePath => StreamOffset == 0 ? ReadFilePath : null; + /// /// Gets whether this package had "_dir" in the name, indicating it has multiple chunk files. /// public bool IsDirVPK { get; private set; } + /// + /// Gets whether entries or chunk hashes of this package refer to chunk files, or files were added to chunk files. + /// Unlike , this does not depend on the file name. + /// + public bool HasChunkFiles => HighestChunkIndex >= 0; + /// /// Gets or sets the VPK version. This version is also used when writing the package. /// @@ -154,6 +186,11 @@ public uint Version /// public Dictionary>? Entries { get; private set; } + /// + /// Gets the number of entries in the package. + /// + public int EntryCount => Entries?.Values.Sum(static entries => entries.Count) ?? 0; + /// /// Gets the archive MD5 checksum section entries. Also known as cache line hashes. /// @@ -201,6 +238,14 @@ protected virtual void Dispose(bool disposing) } } + /// + /// Returns all entries of the package in no particular order, which must not be changed while they are enumerated. + /// + public IEnumerable EnumerateEntries() + { + return GetAllEntries(); + } + private void ThrowIfAlreadyRead() { if (Reader != null || IsReadPackage) diff --git a/ValvePak/ValvePak/StreamArchive.cs b/ValvePak/ValvePak/StreamArchive.cs index 95b7740..58b6f1e 100644 --- a/ValvePak/ValvePak/StreamArchive.cs +++ b/ValvePak/ValvePak/StreamArchive.cs @@ -21,6 +21,17 @@ public StreamArchive(Stream stream, long origin, long dataOffset) Stream = stream; } + public override long Length + { + get + { + lock (StreamLock) + { + return Stream.Length - Origin; + } + } + } + public override int Read(long offset, Span output) { lock (StreamLock) From 6cc62132bfc986a375a0bf7e9b9cc30d8dc64ec6 Mon Sep 17 00:00:00 2001 From: w1tcherrr <70418164+w1tcherrr@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:14:07 +0200 Subject: [PATCH 4/7] Add Open and OpenPackage, which reads a package inside another package in place at any depth --- ValvePak/ValvePak.Test/MultiChunkWriteTest.cs | 80 +++++++++++++++ ValvePak/ValvePak/Package.Read.cs | 98 +++++++++++++++++-- ValvePak/ValvePak/Package.Save.cs | 5 + ValvePak/ValvePak/Package.cs | 10 ++ ValvePak/ValvePak/RangeStreamArchive.cs | 41 ++++++++ 5 files changed, 225 insertions(+), 9 deletions(-) create mode 100644 ValvePak/ValvePak/RangeStreamArchive.cs diff --git a/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs b/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs index c169f17..e62c450 100644 --- a/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs +++ b/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs @@ -912,6 +912,86 @@ public async Task RepackIntoDirVpkMovesDirectoryFileData() await AssertPackageVerifies(readBack); } + [Test] + public async Task OpensNestedPackagesInPlace() + { + var leafData = CreateTestData(10000, 1); + var level2 = CreatePackageBytes(package => package.AddFile("data/leaf.bin", leafData)); + var level1 = CreatePackageBytes(package => + { + var entry = package.AddFile("maps/level2.vpk", level2[20..]); + entry.SmallData = level2[..20]; + entry.CRC32 = Crc32.HashToUInt32(level2); + }); + + var path = TempPath("level0_dir.vpk"); + + using (var created = new Package()) + { + created.SetFileName(path); + created.AddFile("maps/level1.vpk", level1, multiChunk: true); + created.Write(path); + } + + using var level0Package = Package.Open(path); + using var level1Package = level0Package.OpenPackage(level0Package.FindEntry("maps/level1.vpk")!); + var level2Entry = level1Package.FindEntry("maps/level2.vpk")!; + using var level2Package = level1Package.OpenPackage(level2Entry); + + var leafEntry = level2Package.FindEntry("data/leaf.bin")!; + level2Package.ReadEntry(leafEntry, out var leafRead); + + using var mapped = level2Package.GetMemoryMappedStreamIfPossible(leafEntry); + + using (Assert.Multiple()) + { + await Assert.That(leafRead).IsEquivalentTo(leafData, CollectionOrdering.Matching); + await Assert.That(mapped is UnmanagedMemoryStream).IsTrue(); + await Assert.That(level2Package.Parent).IsSameReferenceAs(level1Package); + await Assert.That(level2Package.ParentEntry).IsSameReferenceAs(level2Entry); + } + + level1Package.Dispose(); + + await Assert.That(() => level2Package.ReadEntry(leafEntry, out _)).ThrowsExactly(); + } + + [Test] + public async Task NestedPackagesDoNotUseChunkFiles() + { + using var innerDirectoryFile = new MemoryStream(); + + using (var inner = new Package()) + { + inner.SetFileName(TempPath("inner_dir.vpk")); + inner.AddFile("chunked.bin", CreateTestData(100, 1), multiChunk: true); + inner.AddFile("embedded.bin", CreateTestData(100, 2)); + inner.Write(innerDirectoryFile); + } + + using var outer = new Package(); + outer.AddFile("maps/inner_dir.vpk", innerDirectoryFile.ToArray()); + + using var nested = outer.OpenPackage(outer.FindEntry("maps/inner_dir.vpk")!); + + using (Assert.Multiple()) + { + await Assert.That(() => nested.ReadEntry(nested.FindEntry("embedded.bin")!, out _)).ThrowsNothing(); + await Assert.That(() => nested.ReadEntry(nested.FindEntry("chunked.bin")!, out _)).ThrowsExactly(); + } + } + + private static byte[] CreatePackageBytes(Action addFiles) + { + using var package = new Package(); + addFiles(package); + + using var output = new MemoryStream(); + package.Write(output); + + return output.ToArray(); + } + private string TempPath(string fileName) { return Path.Combine(TempDirectory, fileName); diff --git a/ValvePak/ValvePak/Package.Read.cs b/ValvePak/ValvePak/Package.Read.cs index 975a6d8..d9ad190 100644 --- a/ValvePak/ValvePak/Package.Read.cs +++ b/ValvePak/ValvePak/Package.Read.cs @@ -68,12 +68,91 @@ public void Read(Stream input) .Where(static archiveIndex => archiveIndex < DirectoryArchiveIndex) .Max(static archiveIndex => (int?)archiveIndex) ?? -1; - if (HighestChunkIndex >= 0) + if (HighestChunkIndex >= 0 && Parent == null) { ChunkBaseName = Path.GetFullPath(FileName); } } + /// + /// Opens and reads the package at the given path, with its entries sorted for binary search, see . + /// + /// Path of the directory file. + /// Comparison that compares names with. + /// The read package, which holds the file open until it is disposed. + public static Package Open(string path, StringComparison comparison = StringComparison.Ordinal) + { + ArgumentNullException.ThrowIfNull(path); + + var package = new Package(); + + try + { + package.OptimizeEntriesForBinarySearch(comparison); + package.Read(path); + } + catch + { + package.Dispose(); + throw; + } + + return package; + } + + /// + /// Opens the package stored in an entry of this package like does, reading it in place from the archive the entry is in. + /// This package must stay open and must not replace the file the entry is in while the inner package is used, + /// and reading entries of the inner package that are in chunk files throws . + /// + /// The entry of this package that holds the inner package. + /// Comparison that of the inner package compares names with. + /// The inner package, which is disposed separately. + public Package OpenPackage(PackageEntry entry, StringComparison comparison = StringComparison.Ordinal) + { + ArgumentNullException.ThrowIfNull(entry); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var stream = OpenEntryStream(entry); + var package = new Package + { + Parent = this, + ParentEntry = entry, + }; + + try + { + package.OptimizeEntriesForBinarySearch(comparison); + package.SetFileName(entry.GetFullPath()); + package.Read(stream); + } + catch + { + package.Dispose(); + stream.Dispose(); + throw; + } + + return package; + } + + /// + /// Returns a stream over the data of the entry including its preload bytes, reading from the archive the entry is in unless its data is held in memory. + /// + private Stream OpenEntryStream(PackageEntry entry) + { + if (entry.PendingData != null || entry.Length == 0) + { + ReadEntry(entry, out var data, validateCrc: false); + + return new MemoryStream(data, writable: false); + } + + var archive = GetArchive(entry.ArchiveIndex); + + return new ArchiveRangeStream(archive, archive.DataOffset + entry.Offset, entry.Length, entry.SmallData); + } + /// /// Opens the directory file at the given path instead of the one read before, keeping the current . /// @@ -81,6 +160,8 @@ private void ReopenDirectoryFile(string path) { CloseDirectoryFile(); ReadFilePath = path; + Parent = null; + ParentEntry = null; var input = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); @@ -141,14 +222,12 @@ private void ReadHeader(Stream input) var embeddedDataOffset = (long)HeaderSize + TreeSize; // Taking the handle flushes the stream's buffer, so it is done here before any other thread can read from the stream - if (input is FileStream fileStream) - { - Archives[DirectoryArchiveIndex] = new FileArchive(fileStream.SafeFileHandle, ownsHandle: false, StreamOffset, embeddedDataOffset); - } - else + Archives[DirectoryArchiveIndex] = input switch { - Archives[DirectoryArchiveIndex] = new StreamArchive(input, StreamOffset, embeddedDataOffset); - } + FileStream fileStream => new FileArchive(fileStream.SafeFileHandle, ownsHandle: false, StreamOffset, embeddedDataOffset), + ArchiveRangeStream rangeStream => new RangeStreamArchive(rangeStream, StreamOffset, embeddedDataOffset), + _ => new StreamArchive(input, StreamOffset, embeddedDataOffset), + }; } private void ReadSections() @@ -575,7 +654,8 @@ private void CloseDirectoryFile() /// Files smaller or equal to 4096 bytes will be read into memory. /// /// When trying to map a file in a non-split packages (), - /// this will only memory map if the package was read by providing or using . + /// this will only memory map if the package was read by providing or using , + /// or was opened with from such a package. /// /// Each archive is mapped once. /// Returned streams can only be read while the package is open, they throw diff --git a/ValvePak/ValvePak/Package.Save.cs b/ValvePak/ValvePak/Package.Save.cs index a643bc6..4b41394 100644 --- a/ValvePak/ValvePak/Package.Save.cs +++ b/ValvePak/ValvePak/Package.Save.cs @@ -426,6 +426,11 @@ private void FinishCurrentChunkFile() private string GetChunkBaseName() { + if (Parent != null) + { + throw new NotSupportedException("Chunk files of a package opened from an entry of another package are not supported."); + } + if (FileName == null) { throw new InvalidOperationException("Call SetFileName() with the path of the directory file before adding files with multi chunk enabled, chunk files are created next to it."); diff --git a/ValvePak/ValvePak/Package.cs b/ValvePak/ValvePak/Package.cs index b40fc3b..0171d98 100644 --- a/ValvePak/ValvePak/Package.cs +++ b/ValvePak/ValvePak/Package.cs @@ -86,6 +86,16 @@ public partial class Package : IDisposable /// public string? FilePath => StreamOffset == 0 ? ReadFilePath : null; + /// + /// Gets the package that this package was opened from with , which must stay open while this package is used. + /// + public Package? Parent { get; private set; } + + /// + /// Gets the entry of that this package was opened from. + /// + public PackageEntry? ParentEntry { get; private set; } + /// /// Gets whether this package had "_dir" in the name, indicating it has multiple chunk files. /// diff --git a/ValvePak/ValvePak/RangeStreamArchive.cs b/ValvePak/ValvePak/RangeStreamArchive.cs new file mode 100644 index 0000000..c74ec46 --- /dev/null +++ b/ValvePak/ValvePak/RangeStreamArchive.cs @@ -0,0 +1,41 @@ +using System; +using System.IO; + +namespace ValvePak +{ + /// + /// Archive data embedded in a package that was read from an , such as a package inside another package. + /// Reads go to the archive of the stream without moving the stream position, so they are not serialized. + /// + internal sealed class RangeStreamArchive : Archive + { +#pragma warning disable CA2213 // Disposable fields should be disposed + private readonly ArchiveRangeStream RangeStream; +#pragma warning restore CA2213 + + public RangeStreamArchive(ArchiveRangeStream stream, long origin, long dataOffset) + : base(origin, dataOffset) + { + RangeStream = stream; + } + + public override long Length => RangeStream.Length - Origin; + + public override int Read(long offset, Span output) + { + return RangeStream.ReadAt(Origin + offset, output); + } + + public override Stream? OpenMappedStream(long offset, long length) + { + return RangeStream.OpenMappedStream(Origin + offset, length); + } + + /// + /// Does nothing, the stream is owned by the package. + /// + public override void Dispose() + { + } + } +} From 41389b44fb1b066975c09b0c28465a43e089662d Mon Sep 17 00:00:00 2001 From: w1tcherrr <70418164+w1tcherrr@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:14:08 +0200 Subject: [PATCH 5/7] Add Package.Verify, which can be cancelled, and throw on section checksum mismatches when the package has a signature header --- ValvePak/ValvePak.Test/PackageTest.cs | 59 +++++++- ValvePak/ValvePak/Package.Verify.cs | 202 ++++++++++++++++++++++---- 2 files changed, 232 insertions(+), 29 deletions(-) diff --git a/ValvePak/ValvePak.Test/PackageTest.cs b/ValvePak/ValvePak.Test/PackageTest.cs index 4b804e2..ed21d4f 100644 --- a/ValvePak/ValvePak.Test/PackageTest.cs +++ b/ValvePak/ValvePak.Test/PackageTest.cs @@ -1,6 +1,7 @@ using System; using System.Buffers.Binary; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Security.Cryptography; @@ -534,11 +535,12 @@ public async Task InvalidWholeFileChecksumThrows() } [Test] - [Arguments(0, "File tree")] - [Arguments(16, "Archive MD5 entries")] - public async Task SectionChecksumMismatchIsReportedWhenWholeFileChecksumMatches(int checksumOffset, string sectionName) + [Arguments(0, "File tree", false)] + [Arguments(16, "Archive MD5 entries", false)] + [Arguments(0, "File tree", true)] + public async Task SectionChecksumMismatchIsReportedOnlyWithoutSignatureHeader(int checksumOffset, string sectionName, bool hasSignatureHeader) { - var data = CreateSmallPackageBytes(); + var data = hasSignatureHeader ? CreateSmallPackageBytes() : CreateSmallPackageBytesWithoutSignatureSection(); var otherMD5SectionOffset = GetOtherMD5SectionOffset(data); data.AsSpan(otherMD5SectionOffset + checksumOffset, 16).Fill(0xAB); @@ -550,6 +552,13 @@ public async Task SectionChecksumMismatchIsReportedWhenWholeFileChecksumMatches( var progress = new SynchronousProgress(); + if (hasSignatureHeader) + { + await Assert.That(() => package.VerifyHashes(progress)).ThrowsExactly() + .WithMessageContaining($"{sectionName} checksum mismatch (", StringComparison.Ordinal); + return; + } + await Assert.That(() => package.VerifyHashes(progress)).ThrowsNothing(); await Assert.That(progress.Reports).Count().IsEqualTo(1); await Assert.That(progress.Reports[0]).StartsWith($"{sectionName} checksum mismatch (", StringComparison.Ordinal); @@ -638,6 +647,40 @@ public async Task TestVerifyChunkHashesBlake3() await Assert.That(() => package.VerifyChunkHashes(null)).ThrowsNothing(); } + [Test] + [Arguments("cs2_new_signature_actually_signed.vpk", "Verified the package signature.", 0)] + [Arguments("test_single.vpk", "The package is not signed.", 0)] + [Arguments("test_dir.vpk", "The package is not signed.", 3)] + [Arguments("broken_dir.vpk", "The package is not signed.", 6)] + public async Task VerifyReportsCountFirstAndChecksCrcOfUnhashedEntries(string fileName, string signatureReport, int crcReports) + { + var path = Path.Combine(AppContext.BaseDirectory, "Files", fileName); + + using var package = new Package(); + package.Read(path); + + var progress = new SynchronousProgress(); + package.Verify(progress); + + using (Assert.Multiple()) + { + await Assert.That(int.Parse(progress.Reports[0], CultureInfo.InvariantCulture)).IsEqualTo(progress.Reports.Count - 1); + await Assert.That(progress.Reports[1]).IsEqualTo(signatureReport); + await Assert.That(progress.Reports.Count(report => report.StartsWith("Verifying CRC32", StringComparison.Ordinal))).IsEqualTo(crcReports); + await Assert.That(progress.Reports[^1]).IsEqualTo("Verified the package."); + } + } + + [Test] + public async Task VerifyThrowsOnCrcMismatchOfUnhashedEntry() + { + using var package = new Package(); + package.Read(Path.Combine(AppContext.BaseDirectory, "Files", "test_dir.vpk")); + package.FindEntry("kitten.jpg")!.CRC32 = 0xDEADBEEF; + + await Assert.That(() => package.Verify()).ThrowsExactly(); + } + [Test] public async Task BuildsAndParsesChunkFilePaths() { @@ -989,6 +1032,14 @@ private static byte[] CreateSmallPackageBytes() return output.ToArray(); } + private static byte[] CreateSmallPackageBytesWithoutSignatureSection() + { + var package = CreateSmallPackageBytes()[..^UnsignedSignatureSectionSize]; + BinaryPrimitives.WriteUInt32LittleEndian(package.AsSpan(SignatureSectionSizeOffset), 0); + + return package; + } + private static int GetOtherMD5SectionOffset(byte[] package) { var span = package.AsSpan(); diff --git a/ValvePak/ValvePak/Package.Verify.cs b/ValvePak/ValvePak/Package.Verify.cs index 1745009..43607c4 100644 --- a/ValvePak/ValvePak/Package.Verify.cs +++ b/ValvePak/ValvePak/Package.Verify.cs @@ -1,8 +1,12 @@ using System; using System.Buffers; +using System.Collections.Generic; +using System.Globalization; using System.IO; +using System.IO.Hashing; using System.Linq; using System.Security.Cryptography; +using System.Threading; namespace ValvePak { @@ -31,18 +35,125 @@ public partial class Package : IDisposable /// private uint FileSizeBeforeSignature; + /// + /// Verifies the signature, the checksums, the chunk hashes, and the CRC32 checksum of each entry whose data no checksum or chunk hash covers. + /// + /// If provided, will report the number of reports that follow first, then a string for each step. + /// A token to cancel the verification. + /// The package has not been read. + /// The signature is not valid, or a checksum, chunk hash or CRC32 checksum does not match. + public void Verify(IProgress? progressReporter = null, CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(IsDisposed, this); + + ThrowIfDirectoryFileClosed(); + + var entriesWithUnhashedData = GetEntriesWithUnhashedData(); + var reportCount = 3 + AccessPackFileHashes.Count + entriesWithUnhashedData.Count; + + progressReporter?.Report(reportCount.ToString(CultureInfo.InvariantCulture)); + + if (IsSigned && !IsSignatureValid(cancellationToken)) + { + throw new InvalidDataException("The package signature is not valid."); + } + + progressReporter?.Report(IsSigned ? "Verified the package signature." : "The package is not signed."); + + var checksumMessages = VerifyChecksums(cancellationToken); + + progressReporter?.Report(checksumMessages.Count > 0 ? string.Join(' ', checksumMessages) : "Verified the package checksums."); + + VerifyChunkHashFractions(progressReporter, cancellationToken); + VerifyEntryChecksums(entriesWithUnhashedData, progressReporter, cancellationToken); + + progressReporter?.Report("Verified the package."); + } + + /// + /// Returns the entries, sorted by archive and offset, whose data no checksum or chunk hash covers. + /// + private List GetEntriesWithUnhashedData() + { + var ranges = AccessPackFileHashes.Select(static fraction => (fraction.ArchiveIndex, Start: (long)fraction.Offset, End: (long)fraction.Offset + fraction.Length)); + + if (Version != 1) + { + ranges = ranges.Append((DirectoryArchiveIndex, 0, FileDataSectionSize)); + } + + var hashedRanges = ranges + .GroupBy(static range => range.ArchiveIndex) + .ToDictionary(static group => group.Key, static group => MergeRanges(group.Select(static range => (range.Start, range.End)))); + + return [.. GetAllEntries() + .Where(entry => !IsDataHashed(entry, hashedRanges)) + .OrderBy(static entry => entry.ArchiveIndex) + .ThenBy(static entry => entry.Offset)]; + } + + private bool IsDataHashed(PackageEntry entry, Dictionary> hashedRanges) + { + if (Version == 1 || entry.PendingData != null) + { + return false; + } + + if (entry.Length == 0) + { + return true; + } + + var end = (long)entry.Offset + entry.Length; + + return hashedRanges.TryGetValue(entry.ArchiveIndex, out var ranges) && ranges.Exists(range => range.Start <= entry.Offset && end <= range.End); + } + + /// + /// Sorts the ranges and joins those that overlap or touch. + /// + private static List<(long Start, long End)> MergeRanges(IEnumerable<(long Start, long End)> ranges) + { + var merged = new List<(long Start, long End)>(); + + foreach (var range in ranges.OrderBy(static range => range.Start)) + { + if (merged.Count > 0 && range.Start <= merged[^1].End) + { + merged[^1] = (merged[^1].Start, Math.Max(merged[^1].End, range.End)); + } + else + { + merged.Add(range); + } + } + + return merged; + } + /// /// Verify MD5 hashes provided in the VPK. /// /// /// Version 1 packages have no checksums, so nothing is verified for them. - /// The whole file checksum covers the file tree and archive MD5 section checksums, so when it matches, + /// The whole file checksum covers the file tree and archive MD5 section checksums, so when it matches in a package without a signature header, /// a mismatching file tree or archive MD5 section checksum is reported to instead of thrown. /// /// If provided, will report a string for each checksum mismatch that is not thrown, or that a version 1 package has no checksums. /// The package has not been read, or its directory file could not be opened again after writing. - /// The package has no checksums, or the whole file checksum does not match. + /// The package has no checksums, or a checksum does not match. public void VerifyHashes(IProgress? progressReporter = null) + { + foreach (var message in VerifyChecksums(CancellationToken.None)) + { + progressReporter?.Report(message); + } + } + + /// + /// Verifies the checksums like does, returning the messages it reports. + /// + private List VerifyChecksums(CancellationToken cancellationToken) { ObjectDisposedException.ThrowIf(IsDisposed, this); @@ -50,8 +161,7 @@ public void VerifyHashes(IProgress? progressReporter = null) if (Version == 1) { - progressReporter?.Report("Version 1 packages have no checksums to verify."); - return; + return ["Version 1 packages have no checksums to verify."]; } if (TreeChecksum == null || ArchiveMD5EntriesChecksum == null || WholeFileChecksum == null) @@ -59,21 +169,25 @@ public void VerifyHashes(IProgress? progressReporter = null) throw new InvalidDataException("The package has no checksum section."); } - var wholeFileHash = HashPackageRange(0, FileSizeBeforeWholeFileHash, HashAlgorithmName.MD5); + var wholeFileHash = HashPackageRange(0, FileSizeBeforeWholeFileHash, HashAlgorithmName.MD5, cancellationToken); var wholeFileHashMatches = wholeFileHash.SequenceEqual(WholeFileChecksum); + var toleratesSectionMismatch = wholeFileHashMatches && SignatureType != ESignatureType.OnlyFileChecksum; + var messages = new List(); - VerifySectionHash("File tree", HeaderSize, TreeSize, TreeChecksum, wholeFileHashMatches, progressReporter); - VerifySectionHash("Archive MD5 entries", FileSizeBeforeArchiveMD5Entries, ArchiveMD5SectionSize, ArchiveMD5EntriesChecksum, wholeFileHashMatches, progressReporter); + VerifySectionHash("File tree", HeaderSize, TreeSize, TreeChecksum, toleratesSectionMismatch, messages, cancellationToken); + VerifySectionHash("Archive MD5 entries", FileSizeBeforeArchiveMD5Entries, ArchiveMD5SectionSize, ArchiveMD5EntriesChecksum, toleratesSectionMismatch, messages, cancellationToken); if (!wholeFileHashMatches) { throw new InvalidDataException($"Package checksum mismatch ({BitConverter.ToString(wholeFileHash)} != expected {BitConverter.ToString(WholeFileChecksum)})"); } + + return messages; } - private void VerifySectionHash(string sectionName, long offset, long length, byte[] expected, bool wholeFileHashMatches, IProgress? progressReporter) + private void VerifySectionHash(string sectionName, long offset, long length, byte[] expected, bool toleratesMismatch, List messages, CancellationToken cancellationToken) { - var hash = HashPackageRange(offset, length, HashAlgorithmName.MD5); + var hash = HashPackageRange(offset, length, HashAlgorithmName.MD5, cancellationToken); if (hash.SequenceEqual(expected)) { @@ -82,19 +196,19 @@ private void VerifySectionHash(string sectionName, long offset, long length, byt var message = $"{sectionName} checksum mismatch ({BitConverter.ToString(hash)} != expected {BitConverter.ToString(expected)})"; - if (!wholeFileHashMatches) + if (!toleratesMismatch) { throw new InvalidDataException(message); } - progressReporter?.Report($"{message}, ignored because the package checksum matches."); + messages.Add($"{message}, ignored because the package checksum matches."); } /// /// Computes the hash of a range of the package, at an offset from the start of the package. /// The hash covers fewer bytes if the package ends before the range does. /// - private byte[] HashPackageRange(long offset, long length, HashAlgorithmName hashAlgorithm) + private byte[] HashPackageRange(long offset, long length, HashAlgorithmName hashAlgorithm, CancellationToken cancellationToken) { var archive = GetArchive(DirectoryArchiveIndex); var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, ChunkHashBufferSize)); @@ -105,6 +219,8 @@ private byte[] HashPackageRange(long offset, long length, HashAlgorithmName hash while (length > 0) { + cancellationToken.ThrowIfCancellationRequested(); + var count = (int)Math.Min(length, buffer.Length); var bytesRead = archive.Read(offset, buffer.AsSpan(0, count)); @@ -134,6 +250,16 @@ public void VerifyChunkHashes(IProgress? progressReporter = null) { ObjectDisposedException.ThrowIf(IsDisposed, this); + VerifyChunkHashFractions(progressReporter, CancellationToken.None); + + progressReporter?.Report("Successfully verified archive hashes."); + } + + /// + /// Verifies every entry of , reporting a string before each. + /// + private void VerifyChunkHashFractions(IProgress? progressReporter, CancellationToken cancellationToken) + { // When created by Valve, entries are sorted, and are 1MB chunks var allEntries = AccessPackFileHashes .OrderBy(file => file.ArchiveIndex) @@ -147,6 +273,8 @@ public void VerifyChunkHashes(IProgress? progressReporter = null) foreach (var entry in allEntries) { + cancellationToken.ThrowIfCancellationRequested(); + var hashTypeName = entry.HashType switch { EHashType.Blake3 => "Blake3", @@ -195,8 +323,6 @@ public void VerifyChunkHashes(IProgress? progressReporter = null) throw new InvalidDataException($"Package checksum mismatch ({hashTypeName}) in archive {entry.ArchiveIndex} at {entry.Offset} ({Convert.ToHexString(hash)} != expected {Convert.ToHexString(entry.Checksum)})"); } } - - progressReporter?.Report("Successfully verified archive hashes."); } /// @@ -213,26 +339,47 @@ public void VerifyFileChecksums(IProgress? progressReporter = null) return; } - var allEntries = Entries - .SelectMany(file => file.Value) + var allEntries = GetAllEntries() .OrderBy(file => file.ArchiveIndex) .ThenBy(file => file.Offset); - var buffer = Array.Empty(); + VerifyEntryChecksums(allEntries, progressReporter, CancellationToken.None); - foreach (var entry in allEntries) + progressReporter?.Report("Successfully verified file CRC32 checksums."); + } + + /// + /// Verifies the CRC32 checksum of each of the entries, reporting a string before each. + /// + private void VerifyEntryChecksums(IEnumerable entries, IProgress? progressReporter, CancellationToken cancellationToken) + { + var crc = new Crc32(); + + foreach (var entry in entries) { + cancellationToken.ThrowIfCancellationRequested(); + progressReporter?.Report($"Verifying CRC32 checksum for '{entry.GetFullPath()}' in archive {entry.ArchiveIndex}."); - if (buffer.Length < entry.TotalLength) + crc.Append(entry.SmallData); + + if (entry.PendingData != null) + { + crc.Append(entry.PendingData); + } + else { - buffer = new byte[entry.TotalLength]; + ReadArchiveDataInBlocks(entry.ArchiveIndex, entry.Offset, entry.Length, crc.Append); } - ReadEntry(entry, buffer, validateCrc: true); - } + var actualChecksum = crc.GetCurrentHashAsUInt32(); + crc.Reset(); - progressReporter?.Report("Successfully verified file CRC32 checksums."); + if (entry.CRC32 != actualChecksum) + { + throw new InvalidDataException($"CRC32 mismatch for '{entry.GetFullPath()}' (expected {entry.CRC32:X2}, got {actualChecksum:X2})."); + } + } } /// @@ -240,6 +387,11 @@ public void VerifyFileChecksums(IProgress? progressReporter = null) /// /// True if the signature is valid or the package has no signature (see ), false otherwise. public bool IsSignatureValid() + { + return IsSignatureValid(CancellationToken.None); + } + + private bool IsSignatureValid(CancellationToken cancellationToken) { if (PublicKey == null || Signature == null) { @@ -261,7 +413,7 @@ public bool IsSignatureValid() return false; } - var hash = HashPackageRange(0, FileSizeBeforeWholeFileHash, HashAlgorithmName.MD5); + var hash = HashPackageRange(0, FileSizeBeforeWholeFileHash, HashAlgorithmName.MD5, cancellationToken); if (!hash.SequenceEqual(WholeFileChecksum)) { @@ -277,7 +429,7 @@ public bool IsSignatureValid() return false; } - var signedHash = HashPackageRange(0, FileSizeBeforeSignature, HashAlgorithmName.SHA256); + var signedHash = HashPackageRange(0, FileSizeBeforeSignature, HashAlgorithmName.SHA256, cancellationToken); return rsa.VerifyHash(signedHash, Signature, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); } From 88e8f4fce02c467bacd3625fa9c9aa4ff6db5cb3 Mon Sep 17 00:00:00 2001 From: w1tcherrr <70418164+w1tcherrr@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:14:09 +0200 Subject: [PATCH 6/7] Update the README for the editing, reading and verification APIs --- README.md | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 99c80fc..5ec8df1 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ if (file != null) Do note that files such as `pak01_001.vpk` are just data files, you have to open `pak01_dir.vpk`. +`FindEntry` normalizes the path like added files are: `\` is a directory separator and ASCII letters are lowercased. + To read from a stream instead of a path, set the file name first so that the chunk files can be located: ```csharp @@ -113,6 +115,8 @@ var entry = package.FindEntry("path/to/file.txt"); using var stream = package.GetMemoryMappedStreamIfPossible(entry); ``` +`OpenPackage` opens a VPK stored in an entry of another VPK without extracting it. + ## Optimize for many lookups By default, `FindEntry` performs a linear scan. If you need to look up many files, call `OptimizeEntriesForBinarySearch()` before `Read()` to sort entries and use binary search instead. You can also pass `StringComparison.OrdinalIgnoreCase` for case-insensitive lookups. @@ -128,13 +132,18 @@ package.Read("pak01_dir.vpk"); var file = package.FindEntry("path/to/file.txt"); ``` +`Package.Open("pak01_dir.vpk")` does both in one call. + ## Verification ```csharp using var package = new Package(); package.Read("pak01_dir.vpk"); -// Verify MD5 hashes of the directory tree and whole file (throws on version 1 packages) +// Verify the signature, all hashes, and the CRC32 checksums of files no hash covers in one call +package.Verify(); + +// Verify MD5 hashes of the directory tree and whole file (version 1 packages have none) package.VerifyHashes(); // Verify MD5/Blake3 hashes of individual chunk files (pak01_000.vpk, pak01_001.vpk, ...) @@ -167,7 +176,7 @@ All file data is stored in the single written file, which is limited to 2 GiB. T ### Modify an existing VPK -A package that was read can have files added or removed and then be written to a new file. This only works for packages that are not split into chunk files. Writing back a package opened from a `_dir.vpk` is not supported. +A package that was read can have files added, removed or renamed and then be written to a new file, or back over the file it was read from, including a `_dir.vpk` with chunk files. ```csharp using var package = new Package(); @@ -179,9 +188,11 @@ package.RemoveFile(package.FindEntry("old/file.txt")!); package.Write("pak01_modified.vpk"); ``` +Chunk files are named after the directory file, so a package with chunk files can only be written to another path with `repack: true`, which `RequiresRepack(path)` tells. Repacking a package without chunk files into a `_dir.vpk` moves its data into chunk files. + ### Split into chunk files -Files added with `multiChunk` are written right away into numbered chunk files (`pak01_000.vpk`, `pak01_001.vpk`, ...) next to the directory file, so their data is not kept in memory. A new chunk file is started once the current one reaches `WriteChunkSize` (200 MiB by default). `Write` then only writes the directory file, which contains MD5 hashes of the chunk files that can be verified with `VerifyChunkHashes`. +Files added with `multiChunk` are written right away into numbered chunk files (`pak01_000.vpk`, `pak01_001.vpk`, ...) next to the directory file, so their data is not kept in memory. A new chunk file is started once the current one is larger than `WriteChunkSize` (100 MiB by default). `Write` then only writes the directory file, which contains Blake3 hashes of the chunk files that can be verified with `VerifyChunkHashes`. ```csharp using var package = new Package(); @@ -189,8 +200,8 @@ using var package = new Package(); // Chunk files are created next to the directory file, so its path must be set first package.SetFileName("pak01_dir.vpk"); -// Optionally lower the maximum chunk file size, in bytes -package.WriteChunkSize = 100 * 1024 * 1024; +// Optionally change the chunk file size, in bytes +package.WriteChunkSize = 50 * 1024 * 1024; // Data can be streamed from a file instead of being read into memory using (var stream = File.OpenRead("example.vmdl")) @@ -205,4 +216,4 @@ package.AddFile("manifest.txt", File.ReadAllBytes("manifest.txt")); package.Write("pak01_dir.vpk"); ``` -Removing a file that is already in a chunk file leaves its data in place. Chunk files stay on disk if the package is disposed without being written. +Removing a file that is already in a chunk file leaves its data in place until the package is written with `repack: true`. Chunk files stay on disk if the package is disposed without being written. From 5ab74aa4b70bba6913a8699904dce9b447255e1f Mon Sep 17 00:00:00 2001 From: w1tcherrr <70418164+w1tcherrr@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:00:11 +0200 Subject: [PATCH 7/7] Replace OpenPackage with OpenEntryStream, merge the path normalizers into TryParseEntryPath, remove EntryCount and EnumerateEntries, report Verify steps with their count, find entries without allocating, reject null characters, unpaired surrogates and names ending in a dot without an extension, and hash chunk files that have no hashes when writing --- README.md | 8 +- ValvePak/ValvePak.Test/EntryPathTest.cs | 23 ++-- ValvePak/ValvePak.Test/MultiChunkWriteTest.cs | 25 ++-- ValvePak/ValvePak.Test/PackageTest.cs | 23 ++-- ValvePak/ValvePak/Package.Read.cs | 59 +++----- ValvePak/ValvePak/Package.Save.cs | 36 ++--- ValvePak/ValvePak/Package.Verify.cs | 52 +++++-- ValvePak/ValvePak/Package.cs | 127 +++++++++++------- ...iveRangeStream.cs => PackageDataStream.cs} | 22 ++- ...Archive.cs => PackageDataStreamArchive.cs} | 16 +-- 10 files changed, 219 insertions(+), 172 deletions(-) rename ValvePak/ValvePak/{ArchiveRangeStream.cs => PackageDataStream.cs} (84%) rename ValvePak/ValvePak/{RangeStreamArchive.cs => PackageDataStreamArchive.cs} (53%) diff --git a/README.md b/README.md index 5ec8df1..6d92dcd 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,13 @@ var entry = package.FindEntry("path/to/file.txt"); using var stream = package.GetMemoryMappedStreamIfPossible(entry); ``` -`OpenPackage` opens a VPK stored in an entry of another VPK without extracting it. +`OpenEntryStream` returns a seekable stream that reads an entry in place. A VPK stored in an entry of another VPK can be read from it without extracting it: + +```csharp +using var inner = new Package(); +inner.SetFileName(entry.GetFullPath()); +inner.Read(package.OpenEntryStream(entry)); +``` ## Optimize for many lookups diff --git a/ValvePak/ValvePak.Test/EntryPathTest.cs b/ValvePak/ValvePak.Test/EntryPathTest.cs index 964743a..3f3042b 100644 --- a/ValvePak/ValvePak.Test/EntryPathTest.cs +++ b/ValvePak/ValvePak.Test/EntryPathTest.cs @@ -11,13 +11,20 @@ internal sealed class EntryPathTest [Arguments("a/x/../b.txt", "a/b.txt")] [Arguments("/a/b.txt", "a/b.txt")] [Arguments("README", "readme")] - [Arguments("", "")] - [Arguments("../b.txt", "")] - [Arguments("a/b/", "")] - [Arguments(".gitignore", "")] - public async Task NormalizesEntryPaths(string path, string expected) + [Arguments("", null)] + [Arguments("../b.txt", null)] + [Arguments("a/b/", null)] + [Arguments(".gitignore", null)] + [Arguments("x.. ", null)] + [Arguments("a/b\0c.txt", null)] + [Arguments("a/\ud800.txt", null)] + public async Task NormalizesEntryPaths(string path, string? expected) { - await Assert.That(Package.NormalizeEntryPath(path)).IsEqualTo(expected); + using (Assert.Multiple()) + { + await Assert.That(Package.TryParseEntryPath(path, out var normalizedPath)).IsEqualTo(expected != null); + await Assert.That(normalizedPath).IsEqualTo(expected); + } } [Test] @@ -25,8 +32,8 @@ public async Task RejectsNamesThatAreTooLong() { using (Assert.Multiple()) { - await Assert.That(Package.NormalizeEntryPath($"a.{new string('y', 129)}")).IsEmpty(); - await Assert.That(Package.NormalizeEntryPath($"{new string('é', 131)}.txt")).IsEmpty(); + await Assert.That(Package.TryParseEntryPath($"a.{new string('y', 129)}", out _)).IsFalse(); + await Assert.That(Package.TryParseEntryPath($"{new string('é', 131)}.txt", out _)).IsFalse(); } } diff --git a/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs b/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs index e62c450..3282991 100644 --- a/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs +++ b/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs @@ -625,8 +625,8 @@ public async Task EditsReadPackageWithChunkFiles() await AssertEntryData(readBack, "added/embedded.bin", 300, 2); await Assert.That(readBack.FindEntry("steammessages_base.proto")!.SmallData).Count().IsEqualTo(preload.Length); - // The read chunk file had no hashes, and data in the directory file is not hashed when there are chunk files - await Assert.That(readBack.AccessPackFileHashes.Select(static fraction => fraction.ArchiveIndex).ToArray()).IsEquivalentTo(new ushort[] { 2, 4 }, CollectionOrdering.Matching); + // Chunk files without hashes are hashed, and data in the directory file is not hashed when there are chunk files + await Assert.That(readBack.AccessPackFileHashes.Select(static fraction => fraction.ArchiveIndex).ToArray()).IsEquivalentTo(new ushort[] { 0, 1, 2, 3, 4 }, CollectionOrdering.Matching); } // Hashes that cover the whole chunk file are kept even when its data changed, and are computed again once it grew @@ -906,7 +906,7 @@ public async Task RepackIntoDirVpkMovesDirectoryFileData() using var readBack = new Package(); readBack.Read(dirPath); - await Assert.That(readBack.EnumerateEntries().All(static entry => entry.ArchiveIndex == 0)).IsTrue(); + await Assert.That(readBack.Entries!.Values.SelectMany(static entries => entries).All(static entry => entry.ArchiveIndex == 0)).IsTrue(); await AssertEntryData(readBack, "a.bin", 400, 1); await AssertEntryData(readBack, "b.bin", 300, 2); await AssertPackageVerifies(readBack); @@ -934,9 +934,8 @@ public async Task OpensNestedPackagesInPlace() } using var level0Package = Package.Open(path); - using var level1Package = level0Package.OpenPackage(level0Package.FindEntry("maps/level1.vpk")!); - var level2Entry = level1Package.FindEntry("maps/level2.vpk")!; - using var level2Package = level1Package.OpenPackage(level2Entry); + using var level1Package = OpenNestedPackage(level0Package, level0Package.FindEntry("maps/level1.vpk")!); + using var level2Package = OpenNestedPackage(level1Package, level1Package.FindEntry("maps/level2.vpk")!); var leafEntry = level2Package.FindEntry("data/leaf.bin")!; level2Package.ReadEntry(leafEntry, out var leafRead); @@ -947,8 +946,6 @@ public async Task OpensNestedPackagesInPlace() { await Assert.That(leafRead).IsEquivalentTo(leafData, CollectionOrdering.Matching); await Assert.That(mapped is UnmanagedMemoryStream).IsTrue(); - await Assert.That(level2Package.Parent).IsSameReferenceAs(level1Package); - await Assert.That(level2Package.ParentEntry).IsSameReferenceAs(level2Entry); } level1Package.Dispose(); @@ -972,7 +969,7 @@ public async Task NestedPackagesDoNotUseChunkFiles() using var outer = new Package(); outer.AddFile("maps/inner_dir.vpk", innerDirectoryFile.ToArray()); - using var nested = outer.OpenPackage(outer.FindEntry("maps/inner_dir.vpk")!); + using var nested = OpenNestedPackage(outer, outer.FindEntry("maps/inner_dir.vpk")!); using (Assert.Multiple()) { @@ -981,6 +978,16 @@ public async Task NestedPackagesDoNotUseChunkFiles() } } + private static Package OpenNestedPackage(Package outer, PackageEntry entry) + { + var package = new Package(); + package.OptimizeEntriesForBinarySearch(); + package.SetFileName(entry.GetFullPath()); + package.Read(outer.OpenEntryStream(entry)); + + return package; + } + private static byte[] CreatePackageBytes(Action addFiles) { using var package = new Package(); diff --git a/ValvePak/ValvePak.Test/PackageTest.cs b/ValvePak/ValvePak.Test/PackageTest.cs index ed21d4f..ce99b9a 100644 --- a/ValvePak/ValvePak.Test/PackageTest.cs +++ b/ValvePak/ValvePak.Test/PackageTest.cs @@ -1,7 +1,6 @@ using System; using System.Buffers.Binary; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Linq; using System.Security.Cryptography; @@ -652,22 +651,23 @@ public async Task TestVerifyChunkHashesBlake3() [Arguments("test_single.vpk", "The package is not signed.", 0)] [Arguments("test_dir.vpk", "The package is not signed.", 3)] [Arguments("broken_dir.vpk", "The package is not signed.", 6)] - public async Task VerifyReportsCountFirstAndChecksCrcOfUnhashedEntries(string fileName, string signatureReport, int crcReports) + public async Task VerifyReportsStepsAndChecksCrcOfUnhashedEntries(string fileName, string signatureReport, int crcReports) { var path = Path.Combine(AppContext.BaseDirectory, "Files", fileName); using var package = new Package(); package.Read(path); - var progress = new SynchronousProgress(); - package.Verify(progress); + var reports = new List<(int Step, int StepCount, string Message)>(); + package.Verify(new SynchronousProgress<(int Step, int StepCount, string Message)>(reports.Add)); using (Assert.Multiple()) { - await Assert.That(int.Parse(progress.Reports[0], CultureInfo.InvariantCulture)).IsEqualTo(progress.Reports.Count - 1); - await Assert.That(progress.Reports[1]).IsEqualTo(signatureReport); - await Assert.That(progress.Reports.Count(report => report.StartsWith("Verifying CRC32", StringComparison.Ordinal))).IsEqualTo(crcReports); - await Assert.That(progress.Reports[^1]).IsEqualTo("Verified the package."); + await Assert.That(reports.Select(static report => report.Step)).IsEquivalentTo(Enumerable.Range(1, reports.Count), CollectionOrdering.Matching); + await Assert.That(reports.All(report => report.StepCount == reports.Count)).IsTrue(); + await Assert.That(reports[0].Message).IsEqualTo(signatureReport); + await Assert.That(reports.Count(static report => report.Message.StartsWith("Verifying CRC32", StringComparison.Ordinal))).IsEqualTo(crcReports); + await Assert.That(reports[^1].Message).IsEqualTo("Verified the package."); } } @@ -720,7 +720,7 @@ public async Task OpensArchiveStreamsAtEntryOffsets(string fileName, ushort arch using var stream = package.OpenArchiveStream(archiveIndex); - foreach (var entry in package.EnumerateEntries().Where(entry => entry.ArchiveIndex == archiveIndex)) + foreach (var entry in package.Entries!.Values.SelectMany(static entries => entries).Where(entry => entry.ArchiveIndex == archiveIndex)) { package.ReadEntry(entry, out var expected); var data = new byte[entry.Length]; @@ -812,6 +812,11 @@ private sealed class SynchronousProgress : IProgress public void Report(string value) => Reports.Add(value); } + private sealed class SynchronousProgress(Action report) : IProgress + { + public void Report(T value) => report(value); + } + [Test] public async Task DisposeCanBeCalledMultipleTimes() { diff --git a/ValvePak/ValvePak/Package.Read.cs b/ValvePak/ValvePak/Package.Read.cs index d9ad190..052a1ec 100644 --- a/ValvePak/ValvePak/Package.Read.cs +++ b/ValvePak/ValvePak/Package.Read.cs @@ -68,7 +68,7 @@ public void Read(Stream input) .Where(static archiveIndex => archiveIndex < DirectoryArchiveIndex) .Max(static archiveIndex => (int?)archiveIndex) ?? -1; - if (HighestChunkIndex >= 0 && Parent == null) + if (HighestChunkIndex >= 0 && !IsReadFromAnotherPackage) { ChunkBaseName = Path.GetFullPath(FileName); } @@ -101,56 +101,29 @@ public static Package Open(string path, StringComparison comparison = StringComp } /// - /// Opens the package stored in an entry of this package like does, reading it in place from the archive the entry is in. - /// This package must stay open and must not replace the file the entry is in while the inner package is used, - /// and reading entries of the inner package that are in chunk files throws . + /// Returns a read-only seekable stream over the data of the entry including its preload bytes, which reads in place from the archive the entry is in + /// unless its data is held in memory. Passing it directly to reads a package stored in the entry in place, + /// and that package throws when reading its entries in chunk files. + /// Like streams from , it can only be read while the package is open and has not replaced the file it reads from. /// - /// The entry of this package that holds the inner package. - /// Comparison that of the inner package compares names with. - /// The inner package, which is disposed separately. - public Package OpenPackage(PackageEntry entry, StringComparison comparison = StringComparison.Ordinal) + /// Package entry. + /// Stream over the entry contents. + /// The chunk file of the entry does not exist. + public Stream OpenEntryStream(PackageEntry entry) { ArgumentNullException.ThrowIfNull(entry); ObjectDisposedException.ThrowIf(IsDisposed, this); - var stream = OpenEntryStream(entry); - var package = new Package - { - Parent = this, - ParentEntry = entry, - }; - - try - { - package.OptimizeEntriesForBinarySearch(comparison); - package.SetFileName(entry.GetFullPath()); - package.Read(stream); - } - catch - { - package.Dispose(); - stream.Dispose(); - throw; - } - - return package; - } - - /// - /// Returns a stream over the data of the entry including its preload bytes, reading from the archive the entry is in unless its data is held in memory. - /// - private Stream OpenEntryStream(PackageEntry entry) - { if (entry.PendingData != null || entry.Length == 0) { ReadEntry(entry, out var data, validateCrc: false); - return new MemoryStream(data, writable: false); + return new PackageDataStream(data); } var archive = GetArchive(entry.ArchiveIndex); - return new ArchiveRangeStream(archive, archive.DataOffset + entry.Offset, entry.Length, entry.SmallData); + return new PackageDataStream(archive, archive.DataOffset + entry.Offset, entry.Length, entry.SmallData); } /// @@ -160,8 +133,7 @@ private void ReopenDirectoryFile(string path) { CloseDirectoryFile(); ReadFilePath = path; - Parent = null; - ParentEntry = null; + IsReadFromAnotherPackage = false; var input = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); @@ -183,6 +155,7 @@ private void ReadHeader(Stream input) { Reader = new BinaryReader(input); ReadFilePath = (input as FileStream)?.Name; + IsReadFromAnotherPackage = input is PackageDataStream; if (Reader.ReadUInt32() != MAGIC) { @@ -225,7 +198,7 @@ private void ReadHeader(Stream input) Archives[DirectoryArchiveIndex] = input switch { FileStream fileStream => new FileArchive(fileStream.SafeFileHandle, ownsHandle: false, StreamOffset, embeddedDataOffset), - ArchiveRangeStream rangeStream => new RangeStreamArchive(rangeStream, StreamOffset, embeddedDataOffset), + PackageDataStream dataStream => new PackageDataStreamArchive(dataStream, StreamOffset, embeddedDataOffset), _ => new StreamArchive(input, StreamOffset, embeddedDataOffset), }; } @@ -655,7 +628,7 @@ private void CloseDirectoryFile() /// /// When trying to map a file in a non-split packages (), /// this will only memory map if the package was read by providing or using , - /// or was opened with from such a package. + /// or was read from of such a package. /// /// Each archive is mapped once. /// Returned streams can only be read while the package is open, they throw @@ -705,7 +678,7 @@ public Stream OpenArchiveStream(ushort archiveIndex) length = Math.Min(length, FileDataSectionSize); } - return archive.OpenMappedStream(archive.DataOffset, length) ?? new ArchiveRangeStream(archive, archive.DataOffset, length); + return archive.OpenMappedStream(archive.DataOffset, length) ?? new PackageDataStream(archive, archive.DataOffset, length); } /// diff --git a/ValvePak/ValvePak/Package.Save.cs b/ValvePak/ValvePak/Package.Save.cs index 4b41394..6e420a0 100644 --- a/ValvePak/ValvePak/Package.Save.cs +++ b/ValvePak/ValvePak/Package.Save.cs @@ -90,11 +90,11 @@ public bool RemoveFile(PackageEntry entry) /// /// Moves an entry in current package to a new path. Only the directory tree changes, the file data is not touched. /// Be careful to not move it onto the path of another entry, because this does not check for duplicates. - /// The path is normalized like does. + /// The path is normalized like does. /// /// The package entry to rename. /// Full new file path for this entry. - /// The entry is not in this package, or the new path is rejected, see . + /// The entry is not in this package, or the new path is rejected, see . public void RenameFile(PackageEntry entry, string newFilePath) { ArgumentNullException.ThrowIfNull(entry); @@ -210,7 +210,7 @@ private void RemoveEntries(List entries) /// /// Add file to current package. Be careful to not add duplicate entries, because this does not check for duplicates. - /// The path is normalized like does. + /// The path is normalized like does. /// /// Full file path for this entry. /// File data for this entry. @@ -220,7 +220,7 @@ private void RemoveEntries(List entries) /// A new chunk file is started whenever the current one is larger than . /// Chunk files stay on disk even if the directory file is never written. /// The added entry. - /// The path is rejected, see . + /// The path is rejected, see . public PackageEntry AddFile(string filePath, byte[] fileData, bool multiChunk = false) { ArgumentNullException.ThrowIfNull(filePath); @@ -245,7 +245,7 @@ public PackageEntry AddFile(string filePath, byte[] fileData, bool multiChunk = /// /// Add file to current package, reading its data from the current position of the stream to its end. /// Be careful to not add duplicate entries, because this does not check for duplicates. - /// The path is normalized like does. + /// The path is normalized like does. /// /// Full file path for this entry. /// Stream to read the file data from. It is not disposed. @@ -255,7 +255,7 @@ public PackageEntry AddFile(string filePath, byte[] fileData, bool multiChunk = /// A new chunk file is started whenever the current one is larger than . /// Chunk files stay on disk even if the directory file is never written. /// The added entry. - /// The path is rejected, see . + /// The path is rejected, see . public PackageEntry AddFile(string filePath, Stream fileData, bool multiChunk = false) { ArgumentNullException.ThrowIfNull(filePath); @@ -426,9 +426,9 @@ private void FinishCurrentChunkFile() private string GetChunkBaseName() { - if (Parent != null) + if (IsReadFromAnotherPackage) { - throw new NotSupportedException("Chunk files of a package opened from an entry of another package are not supported."); + throw new NotSupportedException("Chunk files of a package read from an entry of another package are not supported."); } if (FileName == null) @@ -455,21 +455,13 @@ private static PackageEntry CreateEntry(string filePath, string paramName) } /// - /// Splits a file path into the directory, file and type names that stores it with, normalized like does. + /// Splits a file path into the directory, file and type names that stores it with, normalized like does. /// - /// Full file path. - /// The names, with a space for a missing directory or type name. - /// The path is rejected. - public static (string DirectoryName, string FileName, string TypeName) ParseEntryPath(string filePath) - { - ArgumentNullException.ThrowIfNull(filePath); - - return ParseEntryPath(filePath, nameof(filePath)); - } - private static (string DirectoryName, string FileName, string TypeName) ParseEntryPath(string filePath, string paramName) { - if (GetEntryPathError(filePath, out var directory, out var fileName, out var typeName) is { } error) + Span buffer = filePath.Length <= MaxStackPathLength ? stackalloc char[MaxStackPathLength] : new char[filePath.Length]; + + if (GetEntryPathError(filePath, buffer, out var directory, out var fileName, out var typeName) is { } error) { throw new ArgumentException($"'{filePath}' {error}.", paramName); } @@ -536,7 +528,7 @@ private static bool IsInDirectoryFile(PackageEntry entry) /// this only writes the directory file, which refers to the data where it is in the chunk files. /// Unless is enabled, it must be written to the path the chunk files are named after, which should end with "_dir.vpk". /// A package that was not read also deletes existing chunk files numbered after the ones it wrote, until a number has no file. - /// Hashes of chunk files are kept if they cover the whole file, computed again otherwise, and left out for chunk files that had none. + /// Hashes of chunk files are kept if they cover the whole file, and computed otherwise. /// /// Writing over the file the package was read from, or repacking, writes temporary files that replace the existing files once all were written, /// and the package reads from the written files afterwards. This must not run concurrently with reads from this package, @@ -1235,7 +1227,7 @@ private static bool PathsEqual(string path1, string path2) if (!UpdatedChunkHashFractions.TryGetValue(archiveIndex, out var fractions) && !readFractions.TryGetValue(archiveIndex, out fractions)) { - continue; + fractions = []; } if (!CoversFile(fractions, file.Length)) diff --git a/ValvePak/ValvePak/Package.Verify.cs b/ValvePak/ValvePak/Package.Verify.cs index 43607c4..aacc60b 100644 --- a/ValvePak/ValvePak/Package.Verify.cs +++ b/ValvePak/ValvePak/Package.Verify.cs @@ -1,7 +1,6 @@ using System; using System.Buffers; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.IO.Hashing; using System.Linq; @@ -38,36 +37,49 @@ public partial class Package : IDisposable /// /// Verifies the signature, the checksums, the chunk hashes, and the CRC32 checksum of each entry whose data no checksum or chunk hash covers. /// - /// If provided, will report the number of reports that follow first, then a string for each step. + /// If provided, will report each step with its number starting at 1, the number of steps, and a message. /// A token to cancel the verification. /// The package has not been read. /// The signature is not valid, or a checksum, chunk hash or CRC32 checksum does not match. - public void Verify(IProgress? progressReporter = null, CancellationToken cancellationToken = default) + public void Verify(IProgress<(int Step, int StepCount, string Message)>? progressReporter = null, CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(IsDisposed, this); ThrowIfDirectoryFileClosed(); var entriesWithUnhashedData = GetEntriesWithUnhashedData(); - var reportCount = 3 + AccessPackFileHashes.Count + entriesWithUnhashedData.Count; - - progressReporter?.Report(reportCount.ToString(CultureInfo.InvariantCulture)); + var stepProgress = progressReporter == null + ? null + : new StepProgress(progressReporter, 3 + AccessPackFileHashes.Count + entriesWithUnhashedData.Count); if (IsSigned && !IsSignatureValid(cancellationToken)) { throw new InvalidDataException("The package signature is not valid."); } - progressReporter?.Report(IsSigned ? "Verified the package signature." : "The package is not signed."); + stepProgress?.Report(IsSigned ? "Verified the package signature." : "The package is not signed."); var checksumMessages = VerifyChecksums(cancellationToken); - progressReporter?.Report(checksumMessages.Count > 0 ? string.Join(' ', checksumMessages) : "Verified the package checksums."); + stepProgress?.Report(checksumMessages.Count > 0 ? string.Join(' ', checksumMessages) : "Verified the package checksums."); + + VerifyChunkHashFractions(stepProgress, cancellationToken); + VerifyEntryChecksums(entriesWithUnhashedData, stepProgress, cancellationToken); - VerifyChunkHashFractions(progressReporter, cancellationToken); - VerifyEntryChecksums(entriesWithUnhashedData, progressReporter, cancellationToken); + stepProgress?.Report("Verified the package."); + } - progressReporter?.Report("Verified the package."); + /// + /// Numbers the messages of before passing them on. + /// + private sealed class StepProgress(IProgress<(int Step, int StepCount, string Message)> progressReporter, int stepCount) : IProgress + { + private int Step; + + public void Report(string value) + { + progressReporter.Report((++Step, stepCount, value)); + } } /// @@ -104,9 +116,22 @@ private bool IsDataHashed(PackageEntry entry, Dictionary range.Start <= entry.Offset && end <= range.End); + foreach (var (start, rangeEnd) in ranges) + { + if (start <= entry.Offset && end <= rangeEnd) + { + return true; + } + } + + return false; } /// @@ -354,6 +379,7 @@ public void VerifyFileChecksums(IProgress? progressReporter = null) private void VerifyEntryChecksums(IEnumerable entries, IProgress? progressReporter, CancellationToken cancellationToken) { var crc = new Crc32(); + Action> appendToCrc = crc.Append; foreach (var entry in entries) { @@ -369,7 +395,7 @@ private void VerifyEntryChecksums(IEnumerable entries, IProgress public const int MaxFileNameLength = 260; + private const int MaxStackPathLength = 256; + /// /// Always '/' as per Valve's vpk implementation. /// @@ -72,6 +74,7 @@ public partial class Package : IDisposable private long StreamOffset; private string? ReadFilePath; + private bool IsReadFromAnotherPackage; private readonly ConcurrentDictionary Archives = new(); private readonly Lock ArchivesLock = new(); @@ -86,16 +89,6 @@ public partial class Package : IDisposable /// public string? FilePath => StreamOffset == 0 ? ReadFilePath : null; - /// - /// Gets the package that this package was opened from with , which must stay open while this package is used. - /// - public Package? Parent { get; private set; } - - /// - /// Gets the entry of that this package was opened from. - /// - public PackageEntry? ParentEntry { get; private set; } - /// /// Gets whether this package had "_dir" in the name, indicating it has multiple chunk files. /// @@ -196,11 +189,6 @@ public uint Version /// public Dictionary>? Entries { get; private set; } - /// - /// Gets the number of entries in the package. - /// - public int EntryCount => Entries?.Values.Sum(static entries => entries.Count) ?? 0; - /// /// Gets the archive MD5 checksum section entries. Also known as cache line hashes. /// @@ -248,14 +236,6 @@ protected virtual void Dispose(bool disposing) } } - /// - /// Returns all entries of the package in no particular order, which must not be changed while they are enumerated. - /// - public IEnumerable EnumerateEntries() - { - return GetAllEntries(); - } - private void ThrowIfAlreadyRead() { if (Reader != null || IsReadPackage) @@ -331,7 +311,7 @@ private static string StripDirVpkSuffixes(string fileName, out bool isDirVpk) } /// - /// Searches for the entry at the given path, which is normalized like does. + /// Searches for the entry at the given path, which is normalized like does. /// /// If was called on this package, this method uses a binary search. /// Optimized packages also support case insensitive search by using a different . @@ -348,7 +328,9 @@ private static string StripDirVpkSuffixes(string fileName, out bool isDirVpk) /// public PackageEntry? FindEntry(ReadOnlySpan filePath) { - if (SplitEntryPath(filePath, out var directory, out var fileName, out var typeName) != null) + Span buffer = filePath.Length <= MaxStackPathLength || IsNormalizedPath(filePath) ? stackalloc char[MaxStackPathLength] : new char[filePath.Length]; + + if (SplitEntryPath(filePath, buffer, out var directory, out var fileName, out var typeName) != null) { return null; } @@ -432,22 +414,27 @@ private static string StripDirVpkSuffixes(string fileName, out bool isDirVpk) /// empty and "." segments are removed, ".." removes the segment before it, ASCII letters are lowercased, and the type name is split from the file name at its last dot. /// /// Full file path. - /// The normalized path, or an empty string if the path does not end with a file name, has a ".." above the root, - /// or has a name longer than or . - public static string NormalizeEntryPath(string filePath) + /// The normalized path, or null if the path is rejected. + /// True if a file can be stored at the path, false if it does not end with a file name, has a ".." above the root, + /// a null character or an unpaired surrogate, or has a name longer than or . + public static bool TryParseEntryPath(string filePath, [NotNullWhen(true)] out string? normalizedPath) { ArgumentNullException.ThrowIfNull(filePath); - if (GetEntryPathError(filePath, out var directory, out var fileName, out var typeName) != null) + Span buffer = filePath.Length <= MaxStackPathLength ? stackalloc char[MaxStackPathLength] : new char[filePath.Length]; + + if (GetEntryPathError(filePath, buffer, out var directory, out var fileName, out var typeName) != null) { - return string.Empty; + normalizedPath = null; + return false; } - return JoinEntryPath(directory, fileName, typeName); + normalizedPath = JoinEntryPath(directory, fileName, typeName); + return true; } /// - /// Normalizes a directory path to the of the files in it, like normalizes directories. + /// Normalizes a directory path to the of the files in it, like normalizes directories. /// /// Directory path, which may end with a directory separator. /// The normalized path, or an empty string for the root directory or if the path has a ".." above the root. @@ -455,7 +442,10 @@ public static string NormalizeDirectoryPath(string directoryPath) { ArgumentNullException.ThrowIfNull(directoryPath); - if (SplitEntryPath(string.Concat(directoryPath, "/x"), out var directory, out _, out _) != null || directory is Space) + var filePath = string.Concat(directoryPath, "/x"); + Span buffer = filePath.Length <= MaxStackPathLength ? stackalloc char[MaxStackPathLength] : new char[filePath.Length]; + + if (SplitEntryPath(filePath, buffer, out var directory, out _, out _) != null || directory is Space) { return string.Empty; } @@ -467,9 +457,9 @@ public static string NormalizeDirectoryPath(string directoryPath) /// Splits a file path like does and checks the lengths of the names. /// /// Why the path can not be stored, or null. - private static string? GetEntryPathError(ReadOnlySpan path, out ReadOnlySpan directory, out ReadOnlySpan fileName, out ReadOnlySpan typeName) + private static string? GetEntryPathError(ReadOnlySpan path, Span buffer, out ReadOnlySpan directory, out ReadOnlySpan fileName, out ReadOnlySpan typeName) { - var error = SplitEntryPath(path, out directory, out fileName, out typeName); + var error = SplitEntryPath(path, buffer, out directory, out fileName, out typeName); if (error != null) { @@ -490,10 +480,11 @@ public static string NormalizeDirectoryPath(string directoryPath) } /// - /// Normalizes a file path like does and splits it into the directory, file and type names it is stored as. + /// Normalizes a file path like does and splits it into the directory, file and type names it is stored as. + /// A path that normalizing changes is written into the buffer, which must be at least as long as the path. /// /// Why no file can be at the path, or null. - private static string? SplitEntryPath(ReadOnlySpan path, out ReadOnlySpan directory, out ReadOnlySpan fileName, out ReadOnlySpan typeName) + private static string? SplitEntryPath(ReadOnlySpan path, Span buffer, out ReadOnlySpan directory, out ReadOnlySpan fileName, out ReadOnlySpan typeName) { directory = fileName = typeName = default; @@ -502,41 +493,73 @@ public static string NormalizeDirectoryPath(string directoryPath) return "does not end with a file name"; } + if (HasInvalidCharacter(path)) + { + return "has a null character or an unpaired surrogate"; + } + var normalizedPath = path; if (!IsNormalizedPath(path)) { - if (ResolveSegments(path) is not { } resolvedPath) + var length = ResolveSegments(path, buffer); + + if (length < 0) { return "has a \"..\" segment above the root"; } - normalizedPath = resolvedPath; + normalizedPath = buffer[..length]; } var lastSeparator = normalizedPath.LastIndexOf(DirectorySeparatorChar); var file = normalizedPath[(lastSeparator + 1)..]; var dot = file.LastIndexOf('.'); - if (dot == 0 || dot == file.Length - 1) - { - return "has a file name that is empty or ends with a dot"; - } - directory = lastSeparator > -1 ? normalizedPath[..lastSeparator] : Space; fileName = dot > -1 ? file[..dot] : file; typeName = dot > -1 ? file[(dot + 1)..] : Space; + if (dot == 0 || dot == file.Length - 1 || (typeName is Space && fileName.EndsWith('.'))) + { + directory = fileName = typeName = default; + return "has a file name that is empty or ends with a dot"; + } + return null; } /// - /// Joins the segments of the path with '/' and lowercases ASCII letters, leaving out empty and "." segments and removing the segment before each "..". + /// Returns whether the path has a null character, which ends a name in the directory tree, or a surrogate that is not part of a pair, which UTF-8 can not encode. + /// + private static bool HasInvalidCharacter(ReadOnlySpan path) + { + if (path.Contains('\0')) + { + return true; + } + + int i; + + while ((i = path.IndexOfAnyInRange('\uD800', '\uDFFF')) >= 0) + { + if (!char.IsHighSurrogate(path[i]) || i + 1 >= path.Length || !char.IsLowSurrogate(path[i + 1])) + { + return true; + } + + path = path[(i + 2)..]; + } + + return false; + } + + /// + /// Joins the segments of the path with '/' into the buffer and lowercases ASCII letters, leaving out empty and "." segments and removing the segment before each "..". /// - /// The resolved path, or null if a ".." has no segment before it. - private static string? ResolveSegments(ReadOnlySpan path) + /// The length of the resolved path, or -1 if a ".." has no segment before it. + private static int ResolveSegments(ReadOnlySpan path, Span buffer) { - var buffer = new char[path.Length]; var length = 0; foreach (var range in path.SplitAny(DirectorySeparatorChar, WindowsDirectorySeparator)) @@ -552,10 +575,10 @@ public static string NormalizeDirectoryPath(string directoryPath) { if (length == 0) { - return null; + return -1; } - length = Math.Max(buffer.AsSpan(0, length).LastIndexOf(DirectorySeparatorChar), 0); + length = Math.Max(buffer[..length].LastIndexOf(DirectorySeparatorChar), 0); continue; } @@ -570,7 +593,7 @@ public static string NormalizeDirectoryPath(string directoryPath) } } - return new string(buffer, 0, length); + return length; } /// diff --git a/ValvePak/ValvePak/ArchiveRangeStream.cs b/ValvePak/ValvePak/PackageDataStream.cs similarity index 84% rename from ValvePak/ValvePak/ArchiveRangeStream.cs rename to ValvePak/ValvePak/PackageDataStream.cs index 1d5e7a8..72e07e4 100644 --- a/ValvePak/ValvePak/ArchiveRangeStream.cs +++ b/ValvePak/ValvePak/PackageDataStream.cs @@ -4,13 +4,13 @@ namespace ValvePak { /// - /// Read-only seekable stream over a range of an archive, which it reads from without owning it, - /// optionally preceded by bytes held in memory such as the preload bytes of an entry. + /// Read-only seekable stream over data stored in a package: bytes held in memory such as the preload bytes of an entry, + /// followed by a range of an archive, which it reads from without owning it. Either part can be empty. /// - internal sealed class ArchiveRangeStream : Stream + internal sealed class PackageDataStream : Stream { #pragma warning disable CA2213 // Disposable fields should be disposed - private readonly Archive Archive; + private readonly Archive? Archive; #pragma warning restore CA2213 private readonly long Start; private readonly long RangeLength; @@ -21,7 +21,7 @@ internal sealed class ArchiveRangeStream : Stream /// /// Creates a stream over the given range of the archive, which starts with if given. /// - public ArchiveRangeStream(Archive archive, long start, long length, byte[]? prefix = null) + public PackageDataStream(Archive archive, long start, long length, byte[]? prefix = null) { Archive = archive; Start = start; @@ -29,6 +29,14 @@ public ArchiveRangeStream(Archive archive, long start, long length, byte[]? pref Prefix = prefix ?? []; } + /// + /// Creates a stream over bytes held in memory. + /// + public PackageDataStream(byte[] data) + { + Prefix = data; + } + public override bool CanRead => !IsDisposed; public override bool CanSeek => !IsDisposed; @@ -85,7 +93,7 @@ public int ReadAt(long position, Span buffer) var available = Length - position; - if (available <= 0 || buffer.IsEmpty) + if (available <= 0 || buffer.IsEmpty || Archive == null) { return prefixRead; } @@ -108,7 +116,7 @@ public int ReadAt(long position, Span buffer) var offset = position - Prefix.Length; - if (offset < 0 || length > RangeLength - offset) + if (Archive == null || offset < 0 || length > RangeLength - offset) { return null; } diff --git a/ValvePak/ValvePak/RangeStreamArchive.cs b/ValvePak/ValvePak/PackageDataStreamArchive.cs similarity index 53% rename from ValvePak/ValvePak/RangeStreamArchive.cs rename to ValvePak/ValvePak/PackageDataStreamArchive.cs index c74ec46..58764e9 100644 --- a/ValvePak/ValvePak/RangeStreamArchive.cs +++ b/ValvePak/ValvePak/PackageDataStreamArchive.cs @@ -4,31 +4,31 @@ namespace ValvePak { /// - /// Archive data embedded in a package that was read from an , such as a package inside another package. + /// Archive data embedded in a package that was read from a , such as a package inside another package. /// Reads go to the archive of the stream without moving the stream position, so they are not serialized. /// - internal sealed class RangeStreamArchive : Archive + internal sealed class PackageDataStreamArchive : Archive { #pragma warning disable CA2213 // Disposable fields should be disposed - private readonly ArchiveRangeStream RangeStream; + private readonly PackageDataStream DataStream; #pragma warning restore CA2213 - public RangeStreamArchive(ArchiveRangeStream stream, long origin, long dataOffset) + public PackageDataStreamArchive(PackageDataStream stream, long origin, long dataOffset) : base(origin, dataOffset) { - RangeStream = stream; + DataStream = stream; } - public override long Length => RangeStream.Length - Origin; + public override long Length => DataStream.Length - Origin; public override int Read(long offset, Span output) { - return RangeStream.ReadAt(Origin + offset, output); + return DataStream.ReadAt(Origin + offset, output); } public override Stream? OpenMappedStream(long offset, long length) { - return RangeStream.OpenMappedStream(Origin + offset, length); + return DataStream.OpenMappedStream(Origin + offset, length); } ///