Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AUTHORS
Original file line number Diff line number Diff line change
Expand Up @@ -90,3 +90,4 @@ List of contributors, in chronological order:
* Luan Dang (https://github.com/luanmdang)
* Martin Simon (https://github.com/barnumbirr)
* Nathan Gray (https://github.com/NathanGray-ChurchillNavigation)
* somaz (https://github.com/somaz94)
4 changes: 4 additions & 0 deletions deb/package_files.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ func (f *PackageFile) GetPoolPath(packagePool aptly.PackagePool) (string, error)
return f.PoolPath, err
}

func (f *PackageFile) isSHA512Only() bool {
return f.Checksums.MD5 == "" && f.Checksums.SHA256 == "" && f.Checksums.SHA512 != ""
}

// DownloadURL return relative URL to package download location
func (f *PackageFile) DownloadURL() string {
return filepath.Join(f.downloadPath, f.Filename)
Expand Down
61 changes: 61 additions & 0 deletions deb/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -718,6 +718,63 @@ func (repo *RemoteRepo) FilterLatest(count int) (oldLen, newLen int, err error)
return
}

// restoreSHA512OnlyFiles reuses the previous update's pool path and checksums
// for files indexed by SHA512 alone, whose pool path cannot be derived.
func (repo *RemoteRepo) restoreSHA512OnlyFiles(p *Package, packageCollection *PackageCollection) error {
if repo.packageRefs == nil {
return nil
}

files := p.Files()
sha512Only := false
for _, f := range files {
if f.isSHA512Only() {
sha512Only = true
break
}
}
if !sha512Only {
return nil
}

// packageRefs is built from a list unique by ShortKey, so at most one ref matches.
prefix := append(p.ShortKey(""), ' ')
refs := repo.packageRefs.Refs
i := sort.Search(len(refs), func(j int) bool { return bytes.Compare(refs[j], prefix) >= 0 })
if i == len(refs) || !bytes.HasPrefix(refs[i], prefix) {
return nil
}

prevP, err := packageCollection.ByKey(refs[i])
if errors.Is(err, database.ErrNotFound) {
return nil
}
if err != nil {
return err
}

restored := false
for idx := range files {
f := &files[idx]
if !f.isSHA512Only() {
continue
}
for _, prevF := range prevP.Files() {
if prevF.Filename == f.Filename && prevF.Checksums.Size == f.Checksums.Size && prevF.Checksums.SHA512 == f.Checksums.SHA512 {
f.Checksums = prevF.Checksums
f.PoolPath = prevF.PoolPath
restored = true
break
}
}
}

if restored {
p.UpdateFiles(files)
}
return nil
}

// BuildDownloadQueue builds queue, discards current PackageList
func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
if repo.packageList == nil {
Expand All @@ -729,6 +786,10 @@ func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packag
seen := make(map[string]int, repo.packageList.Len())

err = repo.packageList.ForEach(func(p *Package) error {
if err2 := repo.restoreSHA512OnlyFiles(p, packageCollection); err2 != nil {
return err2
}

if repo.packageRefs != nil && skipExistingPackages {
if repo.packageRefs.Has(p) {
// skip this package, but load checksums/files from package in DB
Expand Down
55 changes: 55 additions & 0 deletions deb/remote_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io"
"os"
"sort"
"strings"

"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/console"
Expand Down Expand Up @@ -731,6 +732,50 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
c.Assert(s.flat.packageRefs, NotNil)
}

func (s *RemoteRepoSuite) TestDownloadSHA512OnlyFlat(c *C) {
downloader := http.NewFakeDownloader()
fetchIndexes := func(packages string) {
downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", packages)

c.Assert(s.flat.Fetch(downloader, nil, true), IsNil)
c.Assert(s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, true), IsNil)
c.Assert(downloader.Empty(), Equals, true)
}

fetchIndexes(exampleSHA512OnlyPackagesFile)
queue, _, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
c.Assert(err, IsNil)
c.Assert(queue, HasLen, 1)

tmpPath := c.MkDir() + "/amanda-client.deb"
c.Assert(os.WriteFile(tmpPath, []byte("abc"), 0644), IsNil)
queue[0].File.PoolPath, err = s.packagePool.Import(tmpPath, queue[0].File.Filename, &queue[0].File.Checksums, true, s.cs)
c.Assert(err, IsNil)
c.Assert(s.flat.FinalizeDownload(s.collectionFactory, nil), IsNil)

for _, skipExisting := range []bool{true, false} {
fetchIndexes(exampleSHA512OnlyPackagesFile)
queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, skipExisting)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(0))
c.Check(queue, HasLen, 0)
c.Check(s.flat.packageList.ForEach(func(p *Package) error {
c.Check(s.flat.packageRefs.Has(p), Equals, true)
return nil
}), IsNil)
c.Assert(s.flat.FinalizeDownload(s.collectionFactory, nil), IsNil)
}

fetchIndexes(strings.Replace(exampleSHA512OnlyPackagesFile, "SHA512: ddaf35", "SHA512: 00af35", 1))
queue, _, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
c.Assert(err, IsNil)
c.Check(queue, HasLen, 1)
}

func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
s.flat.DownloadSources = true

Expand Down Expand Up @@ -1174,6 +1219,16 @@ SHA1: 66b27417d37e024c46526c2f6d358a754fc552f3
SHA256: 3608bca1e44ea6c4d268eb6db02260269892c0b42b86bbf1e77a6fa16c3c9282
`

const exampleSHA512OnlyPackagesFile = `Package: amanda-client
Source: amanda
Version: 1:3.3.1-3~bpo60+1
Architecture: i386
Description: Advanced Maryland Automatic Network Disk Archiver (Client)
Filename: pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb
Size: 3
SHA512: ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f
`

const exampleInstallerHashSumFile = `82f69d557f0004d2923fb03e4fb47d18187e37768dbfd0c99756f8a6c68a6d3a ./MANIFEST
`

Expand Down
7 changes: 7 additions & 0 deletions system/t04_mirror/UpdateMirror33Test_gold
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Downloading: ${url}dists/hardy/Release
Downloading & parsing package files...
Downloading: ${url}dists/hardy/main/binary-amd64/Packages
Building download queue...
Download queue: 0 items (0 B)

Mirror `sha512-only` has been updated successfully.
12 changes: 12 additions & 0 deletions system/t04_mirror/test_release_sha512/dists/hardy/Release
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
Origin: test
Label: sha512-only
Suite: test
Codename: hardy
Date: Sat, 19 Oct 2013 13:54:21 UTC
Architectures: amd64
Components: main
Description: Packages indexed by SHA512 only
SHA256:
ad011bf6c8594241540f8786cf2f91b3854dd3ad39581b96ee8ccaf685675c24 467 main/binary-amd64/Packages
SHA512:
bad5e2fc368c8f1db04bdb9c79ff5c590bf0e4c31e5afb0b55f9235812aeb304e3fe6ec8ba486783582aa327d240c030162086c0b67af7bc7a29b1115da3adc5 467 main/binary-amd64/Packages
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
Package: amanda-client
Source: amanda
Version: 1:3.3.1-3~bpo60+1
Installed-Size: 880
Maintainer: Bdale Garbee <[email protected]>
Architecture: amd64
Description: Advanced Maryland Automatic Network Disk Archiver (Client)
Section: utils
Priority: optional
Filename: pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb
Size: 30
SHA512: 4f1f50282331161be460e31c042e3ed1c36d003b7daf08fe9e31596ca8e3508c9d15a6bc416602684de32392a44c9d604a22a42a6611c198c223bad9c36730de
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
012345678901234567890123456789
18 changes: 18 additions & 0 deletions system/t04_mirror/update.py
Original file line number Diff line number Diff line change
Expand Up @@ -566,3 +566,21 @@ def check(self):
"aptly mirror search -format='{{.ShortKey}}' arch-variant 'Name (3cpio)'",
"packages",
)


class UpdateMirror33Test(BaseTest):
"""
update mirrors: packages indexed by SHA512 only are not downloaded again
"""
fixtureCmds = [
"aptly mirror create --ignore-signatures sha512-only ${url} hardy main",
"aptly mirror update --ignore-signatures sha512-only",
]
fixtureWebServer = "test_release_sha512"
configOverride = {
"downloadRetries": 0,
}
runCmd = "aptly mirror update --ignore-signatures sha512-only"

def gold_processor(self, gold):
return string.Template(gold).substitute({'url': self.webServerUrl})
Loading