Skip to content

Edit and repack packages with chunk files #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
w1tcherrr wants to merge 7 commits into from
Closed

Edit and repack packages with chunk files #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
665759e
Add Package.RenameFile, lowercase and check added paths, and truncate…
w1tcherrr Oct 18, 2026
56f1e6d
Throw on use after Dispose and on malformed section sizes, add IsSign…
w1tcherrr Oct 18, 2026
7c78eb0
Default chunk size to 100 MiB and roll over only past it
w1tcherrr Oct 18, 2026
2f088b1
Normalize paths in one place
w1tcherrr Oct 18, 2026
34c42c1
Edit read packages with chunk files and write over the read file
w1tcherrr Oct 18, 2026
293a93e
Add repack to Write
w1tcherrr Oct 18, 2026
8140e31
Write directory files like Valve's packer with a sorted tree, BLAKE3 …
w1tcherrr Oct 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
345 changes: 334 additions & 11 deletions ValvePak/ValvePak.Test/MultiChunkWriteTest.cs
View file
Open in desktop

Large diffs are not rendered by default.

63 changes: 58 additions & 5 deletions ValvePak/ValvePak.Test/PackageTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,44 @@ public async Task InvalidWholeFileChecksumThrows()
await Assert.That(() => package.VerifyHashes()).ThrowsExactly<InvalidDataException>();
}

[Test]
[Arguments(0, "File tree")]
[Arguments(16, "Archive MD5 entries")]
public async Task SectionChecksumMismatchIsReportedWhenWholeFileChecksumMatches(int checksumOffset, string sectionName)
{
var data = CreateSmallPackageBytes();
var otherMD5SectionOffset = GetOtherMD5SectionOffset(data);

data.AsSpan(otherMD5SectionOffset + checksumOffset, 16).Fill(0xAB);
MD5.HashData(data.AsSpan(0, otherMD5SectionOffset + 32)).CopyTo(data, otherMD5SectionOffset + 32);

using var package = new Package();
package.SetFileName("a.vpk");
package.Read(new MemoryStream(data));

var progress = new SynchronousProgress();

await Assert.That(() => package.VerifyHashes(progress)).ThrowsNothing();
await Assert.That(progress.Reports).Count().IsEqualTo(1);
await Assert.That(progress.Reports[0]).StartsWith($"{sectionName} checksum mismatch (", StringComparison.Ordinal);
}

[Test]
public async Task SectionChecksumMismatchThrowsWhenWholeFileChecksumMismatches()
{
var data = CreateSmallPackageBytes();
var otherMD5SectionOffset = GetOtherMD5SectionOffset(data);

data.AsSpan(otherMD5SectionOffset, 16).Fill(0xAB);

using var package = new Package();
package.SetFileName("a.vpk");
package.Read(new MemoryStream(data));

await Assert.That(() => package.VerifyHashes()).ThrowsExactly<InvalidDataException>()
.WithMessageContaining("File tree checksum mismatch", StringComparison.Ordinal);
}

[Test]
public async Task InvalidSignatureFails()
{
Expand Down Expand Up @@ -629,7 +667,7 @@ public async Task ThrowsWhenReadTwice()
}

[Test]
public async Task VerifyHashesThrowsOnVersion1()
public async Task VerifyHashesSkipsVersion1()
{
using var package = new Package();
package.SetFileName("a.vpk");
Expand All @@ -638,8 +676,10 @@ public async Task VerifyHashesThrowsOnVersion1()
using var ms = new MemoryStream([0x34, 0x12, 0xAA, 0x55, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00]);
package.Read(ms);

await Assert.That(() => package.VerifyHashes()).ThrowsExactly<InvalidDataException>()
.WithMessage("Only version 2 is supported.", StringComparison.Ordinal);
var progress = new SynchronousProgress();

await Assert.That(() => package.VerifyHashes(progress)).ThrowsNothing();
await Assert.That(progress.Reports).IsEquivalentTo(["Version 1 packages have no checksums to verify."]);
}

[Test]
Expand Down Expand Up @@ -870,8 +910,11 @@ public async Task ThrowsOnDirectoryTreeEndingInTheMiddleOfString(string tree)
}

private const int TreeSizeOffset = 8;
private const int FileDataSectionSizeOffset = 12;
private const int ArchiveMD5SectionSizeOffset = 16;
private const int SignatureSectionSizeOffset = 24;
private const int HeaderSize = 28;
private const int UnsignedSignatureSectionSize = 20;
private const int EmbeddedPrefixLength = 1000;
private const int EmbeddedSuffixLength = 50;

Expand All @@ -886,16 +929,26 @@ private static byte[] CreateSmallPackageBytes()
return output.ToArray();
}

private static int GetOtherMD5SectionOffset(byte[] package)
{
var span = package.AsSpan();
var treeSize = BinaryPrimitives.ReadInt32LittleEndian(span[TreeSizeOffset..]);
var fileDataSectionSize = BinaryPrimitives.ReadInt32LittleEndian(span[FileDataSectionSizeOffset..]);
var archiveMD5SectionSize = BinaryPrimitives.ReadInt32LittleEndian(span[ArchiveMD5SectionSizeOffset..]);

return HeaderSize + treeSize + fileDataSectionSize + archiveMD5SectionSize;
}

/// <summary>
/// Creates a package whose signature section holds the given little-endian integers.
/// </summary>
private static byte[] CreateSignedPackageBytes(params int[] signatureSection)
{
var package = CreateSmallPackageBytes();
var package = CreateSmallPackageBytes().AsSpan(..^UnsignedSignatureSectionSize);
var signatureSize = signatureSection.Length * sizeof(int);
var result = new byte[package.Length + signatureSize];

package.CopyTo(result, 0);
package.CopyTo(result);
BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(SignatureSectionSizeOffset), (uint)signatureSize);

for (var i = 0; i < signatureSection.Length; i++)
Expand Down
Loading
Loading