From 77ac4b381f7250eb79f4dfb7b4b55bd637693d02 Mon Sep 17 00:00:00 2001 From: somaz Date: Wed, 11 Nov 2026 15:40:03 +0900 Subject: [PATCH 1/2] mirror: stop re-downloading packages indexed by SHA512 only --- AUTHORS | 1 + deb/package_files.go | 4 +++ deb/remote.go | 61 ++++++++++++++++++++++++++++++++++++++++++++ deb/remote_test.go | 55 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+) diff --git a/AUTHORS b/AUTHORS index 8785e7f69..308dbea31 100644 --- a/AUTHORS +++ b/AUTHORS @@ -90,3 +90,4 @@ List of contributors, in chronological order: * Luan Dang (https://github.com/luanmdang) * Martin Simon (https://github.com/barnumbirr) * Nathan Gray (https://github.com/NathanGray-ChurchillNavigation) +* somaz (https://github.com/somaz94) diff --git a/deb/package_files.go b/deb/package_files.go index 2a805d83a..337ef7b9e 100644 --- a/deb/package_files.go +++ b/deb/package_files.go @@ -48,6 +48,10 @@ func (f *PackageFile) GetPoolPath(packagePool aptly.PackagePool) (string, error) return f.PoolPath, err } +func (f *PackageFile) isSHA512Only() bool { + return f.Checksums.MD5 == "" && f.Checksums.SHA256 == "" && f.Checksums.SHA512 != "" +} + // DownloadURL return relative URL to package download location func (f *PackageFile) DownloadURL() string { return filepath.Join(f.downloadPath, f.Filename) diff --git a/deb/remote.go b/deb/remote.go index 92274afa8..1654096f8 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -718,6 +718,63 @@ func (repo *RemoteRepo) FilterLatest(count int) (oldLen, newLen int, err error) return } +// restoreSHA512OnlyFiles reuses the previous update's pool path and checksums +// for files indexed by SHA512 alone, whose pool path cannot be derived. +func (repo *RemoteRepo) restoreSHA512OnlyFiles(p *Package, packageCollection *PackageCollection) error { + if repo.packageRefs == nil { + return nil + } + + files := p.Files() + sha512Only := false + for _, f := range files { + if f.isSHA512Only() { + sha512Only = true + break + } + } + if !sha512Only { + return nil + } + + // packageRefs is built from a list unique by ShortKey, so at most one ref matches. + prefix := append(p.ShortKey(""), ' ') + refs := repo.packageRefs.Refs + i := sort.Search(len(refs), func(j int) bool { return bytes.Compare(refs[j], prefix) >= 0 }) + if i == len(refs) || !bytes.HasPrefix(refs[i], prefix) { + return nil + } + + prevP, err := packageCollection.ByKey(refs[i]) + if errors.Is(err, database.ErrNotFound) { + return nil + } + if err != nil { + return err + } + + restored := false + for idx := range files { + f := &files[idx] + if !f.isSHA512Only() { + continue + } + for _, prevF := range prevP.Files() { + if prevF.Filename == f.Filename && prevF.Checksums.Size == f.Checksums.Size && prevF.Checksums.SHA512 == f.Checksums.SHA512 { + f.Checksums = prevF.Checksums + f.PoolPath = prevF.PoolPath + restored = true + break + } + } + } + + if restored { + p.UpdateFiles(files) + } + return nil +} + // BuildDownloadQueue builds queue, discards current PackageList func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) { if repo.packageList == nil { @@ -729,6 +786,10 @@ func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packag seen := make(map[string]int, repo.packageList.Len()) err = repo.packageList.ForEach(func(p *Package) error { + if err2 := repo.restoreSHA512OnlyFiles(p, packageCollection); err2 != nil { + return err2 + } + if repo.packageRefs != nil && skipExistingPackages { if repo.packageRefs.Has(p) { // skip this package, but load checksums/files from package in DB diff --git a/deb/remote_test.go b/deb/remote_test.go index f154d1d0c..cace40141 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -7,6 +7,7 @@ import ( "io" "os" "sort" + "strings" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/console" @@ -731,6 +732,50 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(s.flat.packageRefs, NotNil) } +func (s *RemoteRepoSuite) TestDownloadSHA512OnlyFlat(c *C) { + downloader := http.NewFakeDownloader() + fetchIndexes := func(packages string) { + downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile) + downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404}) + downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404}) + downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404}) + downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", packages) + + c.Assert(s.flat.Fetch(downloader, nil, true), IsNil) + c.Assert(s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, true), IsNil) + c.Assert(downloader.Empty(), Equals, true) + } + + fetchIndexes(exampleSHA512OnlyPackagesFile) + queue, _, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + c.Assert(err, IsNil) + c.Assert(queue, HasLen, 1) + + tmpPath := c.MkDir() + "/amanda-client.deb" + c.Assert(os.WriteFile(tmpPath, []byte("abc"), 0644), IsNil) + queue[0].File.PoolPath, err = s.packagePool.Import(tmpPath, queue[0].File.Filename, &queue[0].File.Checksums, true, s.cs) + c.Assert(err, IsNil) + c.Assert(s.flat.FinalizeDownload(s.collectionFactory, nil), IsNil) + + for _, skipExisting := range []bool{true, false} { + fetchIndexes(exampleSHA512OnlyPackagesFile) + queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, skipExisting) + c.Assert(err, IsNil) + c.Check(size, Equals, int64(0)) + c.Check(queue, HasLen, 0) + c.Check(s.flat.packageList.ForEach(func(p *Package) error { + c.Check(s.flat.packageRefs.Has(p), Equals, true) + return nil + }), IsNil) + c.Assert(s.flat.FinalizeDownload(s.collectionFactory, nil), IsNil) + } + + fetchIndexes(strings.Replace(exampleSHA512OnlyPackagesFile, "SHA512: ddaf35", "SHA512: 00af35", 1)) + queue, _, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + c.Assert(err, IsNil) + c.Check(queue, HasLen, 1) +} + func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { s.flat.DownloadSources = true @@ -1174,6 +1219,16 @@ SHA1: 66b27417d37e024c46526c2f6d358a754fc552f3 SHA256: 3608bca1e44ea6c4d268eb6db02260269892c0b42b86bbf1e77a6fa16c3c9282 ` +const exampleSHA512OnlyPackagesFile = `Package: amanda-client +Source: amanda +Version: 1:3.3.1-3~bpo60+1 +Architecture: i386 +Description: Advanced Maryland Automatic Network Disk Archiver (Client) +Filename: pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb +Size: 3 +SHA512: ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f +` + const exampleInstallerHashSumFile = `82f69d557f0004d2923fb03e4fb47d18187e37768dbfd0c99756f8a6c68a6d3a ./MANIFEST ` From 9b5ac2081c8b73474d465ea1931e98f67b7c59f8 Mon Sep 17 00:00:00 2001 From: somaz Date: Wed, 18 Nov 2026 10:26:47 +0900 Subject: [PATCH 2/2] Add system test for mirrors indexed by SHA512 only --- system/t04_mirror/UpdateMirror33Test_gold | 7 +++++++ .../test_release_sha512/dists/hardy/Release | 12 ++++++++++++ .../dists/hardy/main/binary-amd64/Packages | 12 ++++++++++++ .../amanda-client_3.3.1-3~bpo60+1_amd64.deb | 1 + system/t04_mirror/update.py | 18 ++++++++++++++++++ 5 files changed, 50 insertions(+) create mode 100644 system/t04_mirror/UpdateMirror33Test_gold create mode 100644 system/t04_mirror/test_release_sha512/dists/hardy/Release create mode 100644 system/t04_mirror/test_release_sha512/dists/hardy/main/binary-amd64/Packages create mode 100644 system/t04_mirror/test_release_sha512/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb diff --git a/system/t04_mirror/UpdateMirror33Test_gold b/system/t04_mirror/UpdateMirror33Test_gold new file mode 100644 index 000000000..99bb22ea5 --- /dev/null +++ b/system/t04_mirror/UpdateMirror33Test_gold @@ -0,0 +1,7 @@ +Downloading: ${url}dists/hardy/Release +Downloading & parsing package files... +Downloading: ${url}dists/hardy/main/binary-amd64/Packages +Building download queue... +Download queue: 0 items (0 B) + +Mirror `sha512-only` has been updated successfully. diff --git a/system/t04_mirror/test_release_sha512/dists/hardy/Release b/system/t04_mirror/test_release_sha512/dists/hardy/Release new file mode 100644 index 000000000..e658501c8 --- /dev/null +++ b/system/t04_mirror/test_release_sha512/dists/hardy/Release @@ -0,0 +1,12 @@ +Origin: test +Label: sha512-only +Suite: test +Codename: hardy +Date: Sat, 19 Oct 2013 13:54:21 UTC +Architectures: amd64 +Components: main +Description: Packages indexed by SHA512 only +SHA256: + ad011bf6c8594241540f8786cf2f91b3854dd3ad39581b96ee8ccaf685675c24 467 main/binary-amd64/Packages +SHA512: + bad5e2fc368c8f1db04bdb9c79ff5c590bf0e4c31e5afb0b55f9235812aeb304e3fe6ec8ba486783582aa327d240c030162086c0b67af7bc7a29b1115da3adc5 467 main/binary-amd64/Packages diff --git a/system/t04_mirror/test_release_sha512/dists/hardy/main/binary-amd64/Packages b/system/t04_mirror/test_release_sha512/dists/hardy/main/binary-amd64/Packages new file mode 100644 index 000000000..ef95c8604 --- /dev/null +++ b/system/t04_mirror/test_release_sha512/dists/hardy/main/binary-amd64/Packages @@ -0,0 +1,12 @@ +Package: amanda-client +Source: amanda +Version: 1:3.3.1-3~bpo60+1 +Installed-Size: 880 +Maintainer: Bdale Garbee +Architecture: amd64 +Description: Advanced Maryland Automatic Network Disk Archiver (Client) +Section: utils +Priority: optional +Filename: pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb +Size: 30 +SHA512: 4f1f50282331161be460e31c042e3ed1c36d003b7daf08fe9e31596ca8e3508c9d15a6bc416602684de32392a44c9d604a22a42a6611c198c223bad9c36730de diff --git a/system/t04_mirror/test_release_sha512/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb b/system/t04_mirror/test_release_sha512/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb new file mode 100644 index 000000000..6e6e144ae --- /dev/null +++ b/system/t04_mirror/test_release_sha512/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb @@ -0,0 +1 @@ +012345678901234567890123456789 \ No newline at end of file diff --git a/system/t04_mirror/update.py b/system/t04_mirror/update.py index 39315649f..a59b9db68 100644 --- a/system/t04_mirror/update.py +++ b/system/t04_mirror/update.py @@ -566,3 +566,21 @@ def check(self): "aptly mirror search -format='{{.ShortKey}}' arch-variant 'Name (3cpio)'", "packages", ) + + +class UpdateMirror33Test(BaseTest): + """ + update mirrors: packages indexed by SHA512 only are not downloaded again + """ + fixtureCmds = [ + "aptly mirror create --ignore-signatures sha512-only ${url} hardy main", + "aptly mirror update --ignore-signatures sha512-only", + ] + fixtureWebServer = "test_release_sha512" + configOverride = { + "downloadRetries": 0, + } + runCmd = "aptly mirror update --ignore-signatures sha512-only" + + def gold_processor(self, gold): + return string.Template(gold).substitute({'url': self.webServerUrl})