From 213fea63a05dc24bf37ea13fe40a2336e069ba93 Mon Sep 17 00:00:00 2001 From: Walid Moghrabi Date: Thu, 19 Nov 2026 14:49:50 +0000 Subject: [PATCH 1/2] Optimized publish and drop operations for local filesystem storage --- database/goleveldb/database.go | 5 + deb/index_files.go | 110 ++++++++++- deb/publish.go | 349 ++++++++++++++++++++++++++++----- docs/Publish.md | 30 +++ files/public.go | 177 ++++++++++++++++- utils/compress.go | 66 +++++-- 6 files changed, 669 insertions(+), 68 deletions(-) diff --git a/database/goleveldb/database.go b/database/goleveldb/database.go index 98960a2cd..1afe08e84 100644 --- a/database/goleveldb/database.go +++ b/database/goleveldb/database.go @@ -19,6 +19,11 @@ func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) { o.CompactionL0Trigger = 32 o.WriteL0PauseTrigger = 96 o.WriteL0SlowdownTrigger = 64 + o.BlockCacheCapacity = 128 * opt.MiB + o.CompactionTableSize = 64 * opt.MiB + o.CompactionTableSizeMultiplier = 2 + o.NoSync = true + o.WriteBuffer = 256 * opt.MiB } return leveldb.OpenFile(path, o) diff --git a/deb/index_files.go b/deb/index_files.go index 0b9cedcd9..bea6db670 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -6,6 +6,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strings" "sync" @@ -25,6 +26,7 @@ type indexFiles struct { acquireByHash bool byHashAlgorithms []string skipBz2 bool + mu sync.Mutex } type indexFile struct { @@ -102,7 +104,7 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { if err != nil { return fmt.Errorf("unable to collect checksums: %s", err) } - file.parent.generatedFiles[file.relativePath+ext] = checksumInfo + file.parent.recordGeneratedFile(file.relativePath+ext, checksumInfo) } filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)) @@ -133,12 +135,14 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { } if file.parent.suffix != "" { - file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] = - filepath.Join(file.parent.basePath, file.relativePath+ext) + file.parent.recordRename( + filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext), + filepath.Join(file.parent.basePath, file.relativePath+ext), + ) } if file.acquireByHash { - sums := file.parent.generatedFiles[file.relativePath+ext] + sums := file.parent.generatedFile(file.relativePath + ext) all := map[string]string{"SHA512": sums.SHA512, "SHA256": sums.SHA256, "SHA1": sums.SHA1, "MD5Sum": sums.MD5} algorithms, algErr := file.parent.hashAlgorithms() if algErr != nil { @@ -162,8 +166,10 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { } if file.parent.suffix != "" { - file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+gpgExt)] = - filepath.Join(file.parent.basePath, file.relativePath+gpgExt) + file.parent.recordRename( + filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+gpgExt), + filepath.Join(file.parent.basePath, file.relativePath+gpgExt), + ) } err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+gpgExt), @@ -181,8 +187,10 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { } if file.parent.suffix != "" { - file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] = - filepath.Join(file.parent.basePath, "In"+file.relativePath) + file.parent.recordRename( + filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix), + filepath.Join(file.parent.basePath, "In"+file.relativePath), + ) } err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix), @@ -306,7 +314,31 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s } } +func (files *indexFiles) recordGeneratedFile(relativePath string, checksumInfo utils.ChecksumInfo) { + files.mu.Lock() + defer files.mu.Unlock() + + files.generatedFiles[relativePath] = checksumInfo +} + +func (files *indexFiles) generatedFile(relativePath string) utils.ChecksumInfo { + files.mu.Lock() + defer files.mu.Unlock() + + return files.generatedFiles[relativePath] +} + +func (files *indexFiles) recordRename(oldName, newName string) { + files.mu.Lock() + defer files.mu.Unlock() + + files.renameMap[oldName] = newName +} + func (files *indexFiles) PackageIndex(component, arch string, udeb bool, installer bool, distribution string) *indexFile { + files.mu.Lock() + defer files.mu.Unlock() + if arch == ArchitectureSource { udeb = false } @@ -348,6 +380,9 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool, install } func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile { + files.mu.Lock() + defer files.mu.Unlock() + if arch == ArchitectureSource { udeb = false } @@ -383,6 +418,9 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF } func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *indexFile { + files.mu.Lock() + defer files.mu.Unlock() + if arch == ArchitectureSource { udeb = false } @@ -415,6 +453,9 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index } func (files *indexFiles) LegacyContentsIndex(arch string, udeb bool) *indexFile { + files.mu.Lock() + defer files.mu.Unlock() + if arch == ArchitectureSource { udeb = false } @@ -447,6 +488,9 @@ func (files *indexFiles) LegacyContentsIndex(arch string, udeb bool) *indexFile } func (files *indexFiles) SkelIndex(component, path string) *indexFile { + files.mu.Lock() + defer files.mu.Unlock() + key := fmt.Sprintf("si-%s-%s", component, path) file, ok := files.indexes[key] @@ -485,10 +529,58 @@ func (files *indexFiles) FinalizeAll(progress aptly.Progress, signer pgp.Signer) defer progress.ShutdownBar() } + parallelFiles := make([]*indexFile, 0, len(files.indexes)) + serialFiles := make([]*indexFile, 0) + for _, file := range files.indexes { + if signer != nil && (file.detachedSign || file.clearSign) { + serialFiles = append(serialFiles, file) + continue + } + parallelFiles = append(parallelFiles, file) + } + + if len(parallelFiles) > 0 { + workerCount := min(max(runtime.NumCPU(), 1), len(parallelFiles)) + jobs := make(chan *indexFile) + errs := make(chan error, len(parallelFiles)) + + var wg sync.WaitGroup + wg.Add(workerCount) + for i := 0; i < workerCount; i++ { + go func() { + defer wg.Done() + for file := range jobs { + errs <- file.Finalize(nil) + } + }() + } + + for _, file := range parallelFiles { + jobs <- file + } + close(jobs) + + wg.Wait() + close(errs) + + for finalizeErr := range errs { + if finalizeErr != nil && err == nil { + err = finalizeErr + } + if progress != nil { + progress.AddBar(1) + } + } + if err != nil { + return err + } + } + + for _, file := range serialFiles { err = file.Finalize(signer) if err != nil { - return + return err } if progress != nil { progress.AddBar(1) diff --git a/deb/publish.go b/deb/publish.go index 087267950..34053c86a 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -8,9 +8,11 @@ import ( "log" "os" "path/filepath" + "runtime" "sort" "strconv" "strings" + "sync" "time" "github.com/google/uuid" @@ -41,6 +43,15 @@ type repoSourceItem struct { packageRefs *PackageRefList } +type asyncPublishedStorageRemover interface { + CommitAsyncRemoves() error + RollbackAsyncRemoves() error +} + +type asyncPublishedStorageRemoveBatcher interface { + BeginAsyncRemoveBatch() aptly.PublishedStorage +} + // PublishedRepo is a published for http/ftp representation of snapshot as Debian repository type PublishedRepo struct { // Internal unique ID @@ -923,7 +934,73 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP progress.InitBar(count, false, aptly.BarPublishGeneratePackageFiles) } - for component, list := range lists { + var legacyContentIndexesMu sync.Mutex + var progressMu sync.Mutex + var linkLocksMu sync.Mutex + linkLocks := map[string]*sync.Mutex{} + + addProgress := func(n int) { + if progress == nil { + return + } + progressMu.Lock() + defer progressMu.Unlock() + progress.AddBar(n) + } + + legacyContentIndex := func(key string) *ContentsIndex { + legacyContentIndexesMu.Lock() + defer legacyContentIndexesMu.Unlock() + + contentIndex := legacyContentIndexes[key] + if contentIndex == nil { + contentIndex = NewContentsIndex(tempDB) + legacyContentIndexes[key] = contentIndex + } + return contentIndex + } + + linkLock := func(key string) *sync.Mutex { + linkLocksMu.Lock() + defer linkLocksMu.Unlock() + + mu := linkLocks[key] + if mu == nil { + mu = &sync.Mutex{} + linkLocks[key] = mu + } + return mu + } + + parallelPackages := os.Getenv("APTLY_PARALLEL_PACKAGES") != "0" + packageWorkerCount := runtime.NumCPU() + if workerCountEnv := os.Getenv("APTLY_PARALLEL_PACKAGE_WORKERS"); workerCountEnv != "" { + if workerCount, parseErr := strconv.Atoi(workerCountEnv); parseErr == nil && workerCount > 0 { + packageWorkerCount = workerCount + } + } + if packageWorkerCount < 1 { + packageWorkerCount = 1 + } + + var packageWorkerTokens chan struct{} + if parallelPackages { + packageWorkerTokens = make(chan struct{}, packageWorkerCount) + } + + type packageIndexStanza struct { + arch string + udeb bool + installer bool + data []byte + } + + type packageWorkResult struct { + hadUdeb bool + stanzas []packageIndexStanza + } + + processComponent := func(component string, list *PackageList) error { hadUdebs := false // For all architectures, pregenerate packages/sources files @@ -934,21 +1011,31 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP list.PrepareIndex() contentIndexes := map[string]*ContentsIndex{} + var contentIndexesMu sync.Mutex - err = list.ForEachIndexed(func(pkg *Package) error { - if progress != nil { - progress.AddBar(1) + componentContentIndex := func(key string) *ContentsIndex { + contentIndexesMu.Lock() + defer contentIndexesMu.Unlock() + + contentIndex := contentIndexes[key] + if contentIndex == nil { + contentIndex = NewContentsIndex(tempDB) + contentIndexes[key] = contentIndex } + return contentIndex + } + + processPackage := func(pkg *Package) (packageWorkResult, error) { + result := packageWorkResult{hadUdeb: pkg.IsUdeb} + addProgress(1) for _, arch := range p.Architectures { if pkg.MatchesIndexArchitecture(arch) { - hadUdebs = hadUdebs || pkg.IsUdeb - var relPath string if !pkg.IsInstaller { poolDir, err2 := pkg.PoolDirectory() if err2 != nil { - return err2 + return result, err2 } if p.MultiDist { relPath = filepath.Join("pool", p.Distribution, component, poolDir) @@ -964,9 +1051,20 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } } - err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, relPath, forceOverwrite) + var linkKey strings.Builder + linkKey.WriteString(p.Prefix) + linkKey.WriteByte(0) + linkKey.WriteString(relPath) + for _, file := range pkg.Files() { + linkKey.WriteByte(0) + linkKey.WriteString(file.Filename) + } + linkMu := linkLock(linkKey.String()) + linkMu.Lock() + err := pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, relPath, forceOverwrite) + linkMu.Unlock() if err != nil { - return err + return result, err } break } @@ -980,38 +1078,42 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP for _, arch := range p.Architectures { if pkg.MatchesIndexArchitecture(arch) { - var bufWriter *bufio.Writer - if !p.SkipContents && !pkg.IsInstaller { key := fmt.Sprintf("%s-%v", arch, pkg.IsUdeb) qualifiedName := []byte(pkg.QualifiedName()) - contents := pkg.Contents(packagePool, progress) - - for _, contentIndexesMap := range []map[string]*ContentsIndex{contentIndexes, legacyContentIndexes} { - contentIndex := contentIndexesMap[key] - - if contentIndex == nil { - contentIndex = NewContentsIndex(tempDB) - contentIndexesMap[key] = contentIndex - } + contentsProgress := progress + if parallelPackages { + contentsProgress = nil + } + contents := pkg.Contents(packagePool, contentsProgress) - _ = contentIndex.Push(qualifiedName, contents, batch) + if err := componentContentIndex(key).Push(qualifiedName, contents, batch); err != nil { + return result, err + } + if err := legacyContentIndex(key).Push(qualifiedName, contents, batch); err != nil { + return result, err } } - bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb, pkg.IsInstaller, p.Distribution).BufWriter() + var stanzaBuffer bytes.Buffer + stanzaWriter := bufio.NewWriter(&stanzaBuffer) + err := pkg.Stanza().WriteTo(stanzaWriter, pkg.IsSource, false, pkg.IsInstaller) if err != nil { - return err + return result, err } - - err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false, pkg.IsInstaller) - if err != nil { - return err + if err := stanzaWriter.WriteByte('\n'); err != nil { + return result, err } - err = bufWriter.WriteByte('\n') - if err != nil { - return err + if err := stanzaWriter.Flush(); err != nil { + return result, err } + + result.stanzas = append(result.stanzas, packageIndexStanza{ + arch: arch, + udeb: pkg.IsUdeb, + installer: pkg.IsInstaller, + data: stanzaBuffer.Bytes(), + }) } } @@ -1020,13 +1122,91 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP pkg.extra = nil pkg.contents = nil - return batch.Write() - }) + err := batch.Write() + return result, err + } - if err != nil { - return fmt.Errorf("unable to process packages: %s", err) + writePackageStanzas := func(result packageWorkResult) error { + for _, stanza := range result.stanzas { + bufWriter, err := indexes.PackageIndex(component, stanza.arch, stanza.udeb, stanza.installer, p.Distribution).BufWriter() + if err != nil { + return err + } + + if _, err := bufWriter.Write(stanza.data); err != nil { + return err + } + } + return nil } + if parallelPackages && list.Len() > 1 { + results := make([]packageWorkResult, len(list.packagesIndex)) + jobs := make(chan int) + + workerCount := min(packageWorkerCount, len(list.packagesIndex)) + var wg sync.WaitGroup + var errMu sync.Mutex + var firstErr error + + setErr := func(err error) { + if err == nil { + return + } + errMu.Lock() + defer errMu.Unlock() + if firstErr == nil { + firstErr = err + } + } + + wg.Add(workerCount) + for i := 0; i < workerCount; i++ { + go func() { + defer wg.Done() + for index := range jobs { + packageWorkerTokens <- struct{}{} + result, err := processPackage(list.packagesIndex[index]) + <-packageWorkerTokens + if err != nil { + setErr(err) + continue + } + results[index] = result + } + }() + } + + for index := range list.packagesIndex { + jobs <- index + } + close(jobs) + wg.Wait() + + if firstErr != nil { + return fmt.Errorf("unable to process packages: %s", firstErr) + } + + for _, result := range results { + hadUdebs = hadUdebs || result.hadUdeb + if err := writePackageStanzas(result); err != nil { + return fmt.Errorf("unable to process packages: %s", err) + } + } + } else { + err = list.ForEachIndexed(func(pkg *Package) error { + result, err := processPackage(pkg) + if err != nil { + return err + } + hadUdebs = hadUdebs || result.hadUdeb + return writePackageStanzas(result) + }) + + if err != nil { + return fmt.Errorf("unable to process packages: %s", err) + } + } for _, arch := range p.Architectures { for _, udeb := range []bool{true, false} { index := contentIndexes[fmt.Sprintf("%s-%v", arch, udeb)] @@ -1034,8 +1214,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP continue } - var bufWriter *bufio.Writer - bufWriter, err = indexes.ContentsIndex(component, arch, udeb).BufWriter() + bufWriter, err := indexes.ContentsIndex(component, arch, udeb).BufWriter() if err != nil { return fmt.Errorf("unable to generate contents index: %v", err) } @@ -1078,8 +1257,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP release["Version"] = p.Version } - var bufWriter *bufio.Writer - bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter() + bufWriter, err := indexes.ReleaseIndex(component, arch, udeb).BufWriter() if err != nil { return fmt.Errorf("unable to get ReleaseIndex writer: %s", err) } @@ -1090,8 +1268,45 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } } } + return nil } + componentNames := make([]string, 0, len(lists)) + for component := range lists { + componentNames = append(componentNames, component) + } + sort.Strings(componentNames) + + if os.Getenv("APTLY_PARALLEL_COMPONENTS") != "0" { + var wg sync.WaitGroup + var errMu sync.Mutex + var firstErr error + for _, component := range componentNames { + component := component + list := lists[component] + wg.Add(1) + go func() { + defer wg.Done() + if err := processComponent(component, list); err != nil { + errMu.Lock() + if firstErr == nil { + firstErr = err + } + errMu.Unlock() + } + }() + } + wg.Wait() + if firstErr != nil { + return firstErr + } + } else { + for _, component := range componentNames { + if err := processComponent(component, lists[component]); err != nil { + return err + } + } + } // Skeleton and AppStream (DEP-11) files are keyed by component and already // iterate over every component, so they must be emitted once for the whole // publish -- not once per component, which would append each payload again. @@ -1261,7 +1476,8 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP return err } - return indexes.RenameFiles() + err = indexes.RenameFiles() + return err } // RemoveFiles removes files that were created by Publish @@ -1273,7 +1489,11 @@ func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStor if err != nil { return err } + return p.removeFiles(publishedStorage, removePrefix, removePoolComponents, progress) +} +func (p *PublishedRepo) removeFiles(publishedStorage aptly.PublishedStorage, removePrefix bool, + removePoolComponents []string, progress aptly.Progress) error { // I. Easy: remove whole prefix (meta+packages) if removePrefix { err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists"), progress) @@ -1285,7 +1505,7 @@ func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStor } // II. Medium: remove metadata, it can't be shared as prefix/distribution as unique - err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress) + err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress) if err != nil { return err } @@ -1665,6 +1885,16 @@ func (collection *PublishedRepoCollection) CleanupAfterMultiDistToggle(published func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(publishedStorageProvider aptly.PublishedStorageProvider, published *PublishedRepo, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error { + publishedStorage, err := publishedStorageProvider.GetPublishedStorage(published.Storage) + if err != nil { + return err + } + return collection.cleanupPrefixComponentFiles(publishedStorage, published, cleanComponents, collectionFactory, progress) +} + +func (collection *PublishedRepoCollection) cleanupPrefixComponentFiles(publishedStorage aptly.PublishedStorage, + published *PublishedRepo, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error { + var err error collection.loadList() @@ -1674,10 +1904,6 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(published distribution := published.Distribution rootPath := filepath.Join(prefix, "dists", distribution) - publishedStorage, err := publishedStorageProvider.GetPublishedStorage(published.Storage) - if err != nil { - return err - } sort.Strings(cleanComponents) publishedComponents := published.Components() @@ -1807,6 +2033,22 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly return err } + publishedStorage, err := publishedStorageProvider.GetPublishedStorage(repo.Storage) + if err != nil { + return err + } + removeStorage := publishedStorage + if batcher, ok := publishedStorage.(asyncPublishedStorageRemoveBatcher); ok { + removeStorage = batcher.BeginAsyncRemoveBatch() + } + asyncRemover, _ := removeStorage.(asyncPublishedStorageRemover) + asyncRemovesCommitted := false + defer func() { + if asyncRemover != nil && !asyncRemovesCommitted { + _ = asyncRemover.RollbackAsyncRemoves() + } + }() + removePrefix := true removePoolComponents := repo.Components() cleanComponents := []string{} @@ -1830,7 +2072,7 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly } } - err = repo.RemoveFiles(publishedStorageProvider, removePrefix, removePoolComponents, progress) + err = repo.removeFiles(removeStorage, removePrefix, removePoolComponents, progress) if err != nil { if !force { return fmt.Errorf("published files removal failed, use -force-drop to override: %s", err) @@ -1842,7 +2084,7 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1] if !skipCleanup && len(cleanComponents) > 0 { - err = collection.CleanupPrefixComponentFiles(publishedStorageProvider, repo, cleanComponents, collectionFactory, progress) + err = collection.cleanupPrefixComponentFiles(removeStorage, repo, cleanComponents, collectionFactory, progress) if err != nil { if !force { return fmt.Errorf("cleanup failed, use -force-drop to override: %s", err) @@ -1857,5 +2099,18 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly _ = batch.Delete(repo.RefKey(component)) } - return batch.Write() + err = batch.Write() + if err != nil { + return err + } + + asyncRemovesCommitted = true + if asyncRemover != nil { + err = asyncRemover.CommitAsyncRemoves() + if err != nil { + return err + } + } + + return nil } diff --git a/docs/Publish.md b/docs/Publish.md index 36b0edc46..435390056 100644 --- a/docs/Publish.md +++ b/docs/Publish.md @@ -37,4 +37,34 @@ aptly publish repo my-repo --gpg-key=KEY_ID_a --gpg-key=KEY_ID_b Publish APIs use following convention to identify published repositories: `/api/publish/:prefix/:distribution`. `:distribution` is distribution name, while `:prefix` is `[:]` (storage is optional, it defaults to empty string), if publishing prefix contains slashes `/`, they should be replaced with underscores (`_`) and underscores should be replaced with double underscore (`__`). To specify root `:prefix`, use `:.`, as `.` is ambiguous in URLs. +#### Performance-related environment variables + +The following environment variables tune publish and drop performance. They are intended for installations where aptly publishes to a local filesystem and can aggressively use local CPU, memory, and filesystem throughput. + +| Variable | Default | Description | +| --- | --- | --- | +| `APTLY_PARALLEL_COMPONENTS` | enabled | Controls concurrent processing of publish components, for example `main`, `restricted`, `universe`, and `multiverse`. Set to `0` to disable component-level parallelism. | +| `APTLY_PARALLEL_PACKAGES` | enabled | Controls concurrent package processing inside each component. Set to `0` to disable package-level parallelism. Package index output order is preserved. | +| `APTLY_PARALLEL_PACKAGE_WORKERS` | number of CPU cores | Overrides the number of package workers used when `APTLY_PARALLEL_PACKAGES` is enabled. For example, set `APTLY_PARALLEL_PACKAGE_WORKERS=4` to force four package workers. | +| `APTLY_FAST_REMOVE` | enabled | Uses external `rm -rf -- ` instead of Go `os.RemoveAll` for local filesystem cleanup. Set to `0` to restore Go's built-in removal path. | +| `APTLY_ASYNC_REMOVE` | enabled | Makes local filesystem removals asynchronous. Removed paths are first renamed into `.aptly-trash//`; the aptly DB is updated; physical deletion starts only after the DB update succeeds. Set to `0` to make cleanup synchronous. | + +`APTLY_ASYNC_REMOVE` changes the operational semantics of `publish drop`: when the command returns, the published repository has already been removed from aptly metadata and moved out of the served path, but disk space may still be reclaimed in the background by `rm -rf`. If the DB update fails, staged removals are rolled back to their original paths. + +Examples: + +``` +# Disable asynchronous removal and wait for physical cleanup before returning. +APTLY_ASYNC_REMOVE=0 aptly publish drop noble filesystem:public:test-ubuntu + +# Disable publish parallelism for troubleshooting. +APTLY_PARALLEL_COMPONENTS=0 APTLY_PARALLEL_PACKAGES=0 aptly publish snapshot ... + +# Override package worker count. +APTLY_PARALLEL_PACKAGE_WORKERS=4 aptly publish snapshot ... + +# Use Go os.RemoveAll instead of external rm -rf. +APTLY_ASYNC_REMOVE=0 APTLY_FAST_REMOVE=0 aptly publish drop noble filesystem:public:test-ubuntu +``` + diff --git a/files/public.go b/files/public.go index dea35ea81..9b9c30251 100644 --- a/files/public.go +++ b/files/public.go @@ -4,11 +4,14 @@ import ( "fmt" "io" "os" + "os/exec" "path/filepath" "sort" "strings" "sync" + "sync/atomic" "syscall" + "time" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" @@ -19,6 +22,20 @@ import ( // In production it calls (*os.File).Sync(). var syncFile = func(f *os.File) error { return f.Sync() } +type asyncRemoveEntry struct { + original string + target string +} + +var asyncRemoveSequence uint64 + +type asyncRemoveBatchPublishedStorage struct { + *PublishedStorage + batchID string + mu sync.Mutex + entries []asyncRemoveEntry +} + // PublishedStorage abstract file system with public dirs (published repos) type PublishedStorage struct { rootPath string @@ -131,10 +148,168 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress panic("trying to remove the root directory") } filepath := filepath.Join(storage.rootPath, path) + if os.Getenv("APTLY_ASYNC_REMOVE") != "0" { + batch := storage.newAsyncRemoveBatch() + if err := batch.RemoveDirs(path, progress); err != nil { + _ = batch.RollbackAsyncRemoves() + return err + } + return batch.CommitAsyncRemoves() + } if progress != nil { progress.Printf("Removing %s...\n", filepath) } - return os.RemoveAll(filepath) + if os.Getenv("APTLY_FAST_REMOVE") == "0" { + return os.RemoveAll(filepath) + } + return removeAllFast(filepath) +} + +// BeginAsyncRemoveBatch creates an isolated removal batch. Directories removed +// through the returned storage are renamed into a batch-specific trash +// directory, then physically deleted only when CommitAsyncRemoves is called. +func (storage *PublishedStorage) BeginAsyncRemoveBatch() aptly.PublishedStorage { + return storage.newAsyncRemoveBatch() +} + +func (storage *PublishedStorage) newAsyncRemoveBatch() *asyncRemoveBatchPublishedStorage { + sequence := atomic.AddUint64(&asyncRemoveSequence, 1) + batchID := fmt.Sprintf("%d-%d-%d", time.Now().UnixNano(), os.Getpid(), sequence) + return &asyncRemoveBatchPublishedStorage{ + PublishedStorage: storage, + batchID: batchID, + } +} + +func (storage *asyncRemoveBatchPublishedStorage) RemoveDirs(path string, progress aptly.Progress) error { + if len(path) <= 0 { + panic("trying to remove the root directory") + } + fullPath := filepath.Join(storage.rootPath, path) + if progress != nil { + progress.Printf("Removing %s...\n", fullPath) + } + if os.Getenv("APTLY_ASYNC_REMOVE") == "0" { + return storage.PublishedStorage.RemoveDirs(path, nil) + } + return storage.stageAsyncRemove(path, fullPath) +} + +func (storage *asyncRemoveBatchPublishedStorage) Remove(path string) error { + if len(path) <= 0 { + panic("trying to remove empty path") + } + fullPath := filepath.Join(storage.rootPath, path) + if os.Getenv("APTLY_ASYNC_REMOVE") == "0" { + return storage.PublishedStorage.Remove(path) + } + return storage.stageAsyncRemove(path, fullPath) +} + +func (storage *asyncRemoveBatchPublishedStorage) stageAsyncRemove(path, fullPath string) error { + trashDir := filepath.Join(storage.rootPath, ".aptly-trash", storage.batchID) + if err := os.MkdirAll(trashDir, 0777); err != nil { + return err + } + + sequence := atomic.AddUint64(&asyncRemoveSequence, 1) + target := filepath.Join(trashDir, fmt.Sprintf("%d-%s", sequence, trashName(path))) + if err := os.Rename(fullPath, target); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + storage.mu.Lock() + storage.entries = append(storage.entries, asyncRemoveEntry{ + original: fullPath, + target: target, + }) + storage.mu.Unlock() + + return nil +} + +// CommitAsyncRemoves starts physical deletion of directories previously staged +// by RemoveDirs in async mode. It is intentionally separate from RemoveDirs so +// callers can update their metadata first, then make the staged deletion final. +func (storage *asyncRemoveBatchPublishedStorage) CommitAsyncRemoves() error { + entries := storage.popAsyncRemoves() + if len(entries) == 0 { + return nil + } + + return removeAllFastAsync(filepath.Join(storage.rootPath, ".aptly-trash", storage.batchID)) +} + +// RollbackAsyncRemoves moves directories staged by RemoveDirs in async mode back +// to their original location. It is used when metadata update fails. +func (storage *asyncRemoveBatchPublishedStorage) RollbackAsyncRemoves() error { + entries := storage.popAsyncRemoves() + + for i := len(entries) - 1; i >= 0; i-- { + entry := entries[i] + if err := os.MkdirAll(filepath.Dir(entry.original), 0777); err != nil { + return err + } + if err := os.Rename(entry.target, entry.original); err != nil && !os.IsNotExist(err) { + return err + } + } + + if len(entries) > 0 { + _ = os.Remove(filepath.Join(storage.rootPath, ".aptly-trash", storage.batchID)) + } + + return nil +} + +func (storage *asyncRemoveBatchPublishedStorage) popAsyncRemoves() []asyncRemoveEntry { + storage.mu.Lock() + defer storage.mu.Unlock() + + entries := storage.entries + storage.entries = nil + + return entries +} + +func trashName(path string) string { + name := filepath.Clean(path) + name = strings.TrimPrefix(name, string(filepath.Separator)) + name = strings.Trim(name, ".") + name = strings.ReplaceAll(name, string(filepath.Separator), "_") + if name == "" { + return "removed" + } + return name +} + +func removeAllFast(path string) error { + cmd := exec.Command("rm", "-rf", "--", path) + output, err := cmd.CombinedOutput() + if err == nil { + return nil + } + if _, lookupErr := exec.LookPath("rm"); lookupErr != nil { + return os.RemoveAll(path) + } + return fmt.Errorf("rm -rf failed: %w: %s", err, strings.TrimSpace(string(output))) +} + +func removeAllFastAsync(path string) error { + cmd := exec.Command("rm", "-rf", "--", path) + if err := cmd.Start(); err != nil { + if _, lookupErr := exec.LookPath("rm"); lookupErr != nil { + return os.RemoveAll(path) + } + return fmt.Errorf("unable to start async rm -rf: %w", err) + } + if err := cmd.Process.Release(); err != nil { + return err + } + return nil } // LinkFromPool links package file from pool to dist's pool location diff --git a/utils/compress.go b/utils/compress.go index e194c3e19..a09910485 100644 --- a/utils/compress.go +++ b/utils/compress.go @@ -1,9 +1,11 @@ package utils import ( + "compress/gzip" "io" "os" "os/exec" + "sync" "github.com/klauspost/pgzip" ) @@ -13,26 +15,68 @@ import ( // It uses internal gzip and external bzip2, see: // https://code.google.com/p/go/issues/detail?id=4828 func CompressFile(source *os.File, onlyGzip bool) error { - gzPath := source.Name() + ".gz" - gzFile, err := os.Create(gzPath) + if onlyGzip { + return compressGzip(source.Name()) + } + + errs := make(chan error, 2) + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + errs <- compressGzip(source.Name()) + }() + + go func() { + defer wg.Done() + cmd := exec.Command("bzip2", "-1", "-k", "-f", source.Name()) + errs <- cmd.Run() + }() + + wg.Wait() + close(errs) + + var firstErr error + for err := range errs { + if firstErr == nil && err != nil { + firstErr = err + } + } + return firstErr +} + +func compressGzip(sourceFilename string) error { + source, err := os.Open(sourceFilename) if err != nil { return err } defer func() { - _ = gzFile.Close() + _ = source.Close() }() - gzWriter := pgzip.NewWriter(gzFile) - defer func() { - _ = gzWriter.Close() - }() + gzPath := sourceFilename + ".gz" + gzFile, err := os.Create(gzPath) + if err != nil { + return err + } + + gzWriter, err := pgzip.NewWriterLevel(gzFile, gzip.BestSpeed) + if err != nil { + _ = gzFile.Close() + return err + } - _, _ = source.Seek(0, 0) _, err = io.Copy(gzWriter, source) - if err != nil || onlyGzip { + if closeErr := gzWriter.Close(); err == nil { + err = closeErr + } + if closeErr := gzFile.Close(); err == nil { + err = closeErr + } + if err != nil { return err } - cmd := exec.Command("bzip2", "-k", "-f", source.Name()) - return cmd.Run() + return nil } From cc7ac3eacc357cc1312fe6f155489fa0bd672fb4 Mon Sep 17 00:00:00 2001 From: Walid Moghrabi Date: Thu, 19 Nov 2026 14:58:37 +0000 Subject: [PATCH 2/2] Optimized publish and drop operations for local filesystem storage --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 267d9bc57..37b5979f4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -89,3 +89,4 @@ List of contributors, in chronological order: * James Munson (https://github.com/jmunson) * Luan Dang (https://github.com/luanmdang) * Martin Simon (https://github.com/barnumbirr) +* Walid Moghrabi (https://github.com/Hybrid512) \ No newline at end of file