mirror of https://github.com/aptly-dev/aptly
436 lines
12 KiB
Go
436 lines
12 KiB
Go
package deb
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/aptly-dev/aptly/aptly"
|
|
"github.com/aptly-dev/aptly/pgp"
|
|
"github.com/aptly-dev/aptly/utils"
|
|
)
|
|
|
|
type indexFiles struct {
|
|
publishedStorage aptly.PublishedStorage
|
|
basePath string
|
|
renameMap map[string]string
|
|
generatedFiles map[string]utils.ChecksumInfo
|
|
tempDir string
|
|
suffix string
|
|
indexes map[string]*indexFile
|
|
acquireByHash bool
|
|
skipBz2 bool
|
|
}
|
|
|
|
type indexFile struct {
|
|
parent *indexFiles
|
|
discardable bool
|
|
compressable bool
|
|
onlyGzip bool
|
|
clearSign bool
|
|
detachedSign bool
|
|
acquireByHash bool
|
|
relativePath string
|
|
tempFilename string
|
|
tempFile *os.File
|
|
w *bufio.Writer
|
|
}
|
|
|
|
func (file *indexFile) BufWriter() (*bufio.Writer, error) {
|
|
if file.w == nil {
|
|
var err error
|
|
file.tempFilename = filepath.Join(file.parent.tempDir, strings.Replace(file.relativePath, "/", "_", -1))
|
|
file.tempFile, err = os.Create(file.tempFilename)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to create temporary index file: %s", err)
|
|
}
|
|
|
|
file.w = bufio.NewWriter(file.tempFile)
|
|
}
|
|
|
|
return file.w, nil
|
|
}
|
|
|
|
func (file *indexFile) Finalize(signer pgp.Signer) error {
|
|
if file.w == nil {
|
|
if file.discardable {
|
|
return nil
|
|
}
|
|
file.BufWriter()
|
|
}
|
|
|
|
err := file.w.Flush()
|
|
if err != nil {
|
|
file.tempFile.Close()
|
|
return fmt.Errorf("unable to write to index file: %s", err)
|
|
}
|
|
|
|
if file.compressable {
|
|
err = utils.CompressFile(file.tempFile, file.onlyGzip || file.parent.skipBz2)
|
|
if err != nil {
|
|
file.tempFile.Close()
|
|
return fmt.Errorf("unable to compress index file: %s", err)
|
|
}
|
|
}
|
|
|
|
file.tempFile.Close()
|
|
|
|
exts := []string{""}
|
|
cksumExts := exts
|
|
if file.compressable {
|
|
if file.onlyGzip {
|
|
exts = []string{".gz"}
|
|
cksumExts = []string{"", ".gz"}
|
|
} else {
|
|
exts = append(exts, ".gz")
|
|
if !file.parent.skipBz2 {
|
|
exts = append(exts, ".bz2")
|
|
}
|
|
cksumExts = exts
|
|
}
|
|
}
|
|
|
|
for _, ext := range cksumExts {
|
|
var checksumInfo utils.ChecksumInfo
|
|
|
|
checksumInfo, err = utils.ChecksumsForFile(file.tempFilename + ext)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to collect checksums: %s", err)
|
|
}
|
|
file.parent.generatedFiles[file.relativePath+ext] = checksumInfo
|
|
}
|
|
|
|
filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
|
|
|
|
err = file.parent.publishedStorage.MkDir(filedir)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to create dir: %s", err)
|
|
}
|
|
|
|
if file.acquireByHash {
|
|
for _, hash := range []string{"MD5Sum", "SHA1", "SHA256", "SHA512"} {
|
|
err = file.parent.publishedStorage.MkDir(filepath.Join(filedir, "by-hash", hash))
|
|
if err != nil {
|
|
return fmt.Errorf("unable to create dir: %s", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, ext := range exts {
|
|
err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
|
|
file.tempFilename+ext)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to publish file: %s", err)
|
|
}
|
|
|
|
if file.parent.suffix != "" {
|
|
file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] =
|
|
filepath.Join(file.parent.basePath, file.relativePath+ext)
|
|
}
|
|
|
|
if file.acquireByHash {
|
|
sums := file.parent.generatedFiles[file.relativePath+ext]
|
|
for hash, sum := range map[string]string{"SHA512": sums.SHA512, "SHA256": sums.SHA256, "SHA1": sums.SHA1, "MD5Sum": sums.MD5} {
|
|
err = packageIndexByHash(file, ext, hash, sum)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to build hash file: %s", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if signer != nil {
|
|
gpgExt := ".gpg"
|
|
if file.detachedSign {
|
|
err = signer.DetachedSign(file.tempFilename, file.tempFilename+gpgExt)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to detached sign file: %s", err)
|
|
}
|
|
|
|
if file.parent.suffix != "" {
|
|
file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+gpgExt)] =
|
|
filepath.Join(file.parent.basePath, file.relativePath+gpgExt)
|
|
}
|
|
|
|
err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+gpgExt),
|
|
file.tempFilename+gpgExt)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to publish file: %s", err)
|
|
}
|
|
|
|
}
|
|
|
|
if file.clearSign {
|
|
err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
|
|
if err != nil {
|
|
return fmt.Errorf("unable to clearsign file: %s", err)
|
|
}
|
|
|
|
if file.parent.suffix != "" {
|
|
file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
|
|
filepath.Join(file.parent.basePath, "In"+file.relativePath)
|
|
}
|
|
|
|
err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
|
|
filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
|
|
if err != nil {
|
|
return fmt.Errorf("unable to publish file: %s", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func packageIndexByHash(file *indexFile, ext string, hash string, sum string) error {
|
|
src := filepath.Join(file.parent.basePath, file.relativePath)
|
|
indexfile := path.Base(src + ext)
|
|
src = src + file.parent.suffix + ext
|
|
filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
|
|
dst := filepath.Join(filedir, "by-hash", hash)
|
|
sumfilePath := filepath.Join(dst, sum)
|
|
|
|
// link already exists? do nothing
|
|
exists, err := file.parent.publishedStorage.FileExists(sumfilePath)
|
|
if err != nil {
|
|
return fmt.Errorf("Acquire-By-Hash: error checking exists of file %s: %s", sumfilePath, err)
|
|
}
|
|
if exists {
|
|
return nil
|
|
}
|
|
|
|
// create the link
|
|
err = file.parent.publishedStorage.HardLink(src, sumfilePath)
|
|
if err != nil {
|
|
return fmt.Errorf("Acquire-By-Hash: error creating hardlink %s: %s", sumfilePath, err)
|
|
}
|
|
|
|
// if a previous index file already exists exists, backup symlink
|
|
indexPath := filepath.Join(dst, indexfile)
|
|
oldIndexPath := filepath.Join(dst, indexfile+".old")
|
|
if exists, _ = file.parent.publishedStorage.FileExists(indexPath); exists {
|
|
// if exists, remove old symlink
|
|
if exists, _ = file.parent.publishedStorage.FileExists(oldIndexPath); exists {
|
|
var linkTarget string
|
|
linkTarget, err = file.parent.publishedStorage.ReadLink(oldIndexPath)
|
|
if err == nil {
|
|
// If we managed to resolve the link target: delete it. This is the
|
|
// oldest physical index file we no longer need. Once we drop our
|
|
// old symlink we'll essentially forget about it existing at all.
|
|
file.parent.publishedStorage.Remove(linkTarget)
|
|
}
|
|
file.parent.publishedStorage.Remove(oldIndexPath)
|
|
}
|
|
file.parent.publishedStorage.RenameFile(indexPath, oldIndexPath)
|
|
}
|
|
|
|
// create symlink
|
|
err = file.parent.publishedStorage.SymLink(filepath.Join(dst, sum), filepath.Join(dst, indexfile))
|
|
if err != nil {
|
|
return fmt.Errorf("Acquire-By-Hash: error creating symlink %s: %s", filepath.Join(dst, indexfile), err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool, skipBz2 bool) *indexFiles {
|
|
return &indexFiles{
|
|
publishedStorage: publishedStorage,
|
|
basePath: basePath,
|
|
renameMap: make(map[string]string),
|
|
generatedFiles: make(map[string]utils.ChecksumInfo),
|
|
tempDir: tempDir,
|
|
suffix: suffix,
|
|
indexes: make(map[string]*indexFile),
|
|
acquireByHash: acquireByHash,
|
|
skipBz2: skipBz2,
|
|
}
|
|
}
|
|
|
|
func (files *indexFiles) PackageIndex(component, arch string, udeb bool, installer bool, distribution string) *indexFile {
|
|
if arch == ArchitectureSource {
|
|
udeb = false
|
|
}
|
|
key := fmt.Sprintf("pi-%s-%s-%v-%v", component, arch, udeb, installer)
|
|
file, ok := files.indexes[key]
|
|
if !ok {
|
|
var relativePath string
|
|
|
|
if arch == ArchitectureSource {
|
|
relativePath = filepath.Join(component, "source", "Sources")
|
|
} else {
|
|
if udeb {
|
|
relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
|
|
} else if installer {
|
|
if distribution == aptly.DistributionFocal {
|
|
relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "legacy-images", "SHA256SUMS")
|
|
} else {
|
|
relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "images", "SHA256SUMS")
|
|
}
|
|
} else {
|
|
relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
|
|
}
|
|
}
|
|
|
|
file = &indexFile{
|
|
parent: files,
|
|
discardable: false,
|
|
compressable: !installer,
|
|
detachedSign: installer,
|
|
clearSign: false,
|
|
acquireByHash: files.acquireByHash,
|
|
relativePath: relativePath,
|
|
}
|
|
|
|
files.indexes[key] = file
|
|
}
|
|
|
|
return file
|
|
}
|
|
|
|
func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile {
|
|
if arch == ArchitectureSource {
|
|
udeb = false
|
|
}
|
|
key := fmt.Sprintf("ri-%s-%s-%v", component, arch, udeb)
|
|
file, ok := files.indexes[key]
|
|
if !ok {
|
|
var relativePath string
|
|
|
|
if arch == ArchitectureSource {
|
|
relativePath = filepath.Join(component, "source", "Release")
|
|
} else {
|
|
if udeb {
|
|
relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Release")
|
|
} else {
|
|
relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Release")
|
|
}
|
|
}
|
|
|
|
file = &indexFile{
|
|
parent: files,
|
|
discardable: udeb,
|
|
compressable: false,
|
|
detachedSign: false,
|
|
clearSign: false,
|
|
acquireByHash: files.acquireByHash,
|
|
relativePath: relativePath,
|
|
}
|
|
|
|
files.indexes[key] = file
|
|
}
|
|
|
|
return file
|
|
}
|
|
|
|
func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *indexFile {
|
|
if arch == ArchitectureSource {
|
|
udeb = false
|
|
}
|
|
key := fmt.Sprintf("ci-%s-%s-%v", component, arch, udeb)
|
|
file, ok := files.indexes[key]
|
|
if !ok {
|
|
var relativePath string
|
|
|
|
if udeb {
|
|
relativePath = filepath.Join(component, fmt.Sprintf("Contents-udeb-%s", arch))
|
|
} else {
|
|
relativePath = filepath.Join(component, fmt.Sprintf("Contents-%s", arch))
|
|
}
|
|
|
|
file = &indexFile{
|
|
parent: files,
|
|
discardable: true,
|
|
compressable: true,
|
|
onlyGzip: true,
|
|
detachedSign: false,
|
|
clearSign: false,
|
|
acquireByHash: files.acquireByHash,
|
|
relativePath: relativePath,
|
|
}
|
|
|
|
files.indexes[key] = file
|
|
}
|
|
|
|
return file
|
|
}
|
|
|
|
func (files *indexFiles) LegacyContentsIndex(arch string, udeb bool) *indexFile {
|
|
if arch == ArchitectureSource {
|
|
udeb = false
|
|
}
|
|
key := fmt.Sprintf("lci-%s-%v", arch, udeb)
|
|
file, ok := files.indexes[key]
|
|
if !ok {
|
|
var relativePath string
|
|
|
|
if udeb {
|
|
relativePath = fmt.Sprintf("Contents-udeb-%s", arch)
|
|
} else {
|
|
relativePath = fmt.Sprintf("Contents-%s", arch)
|
|
}
|
|
|
|
file = &indexFile{
|
|
parent: files,
|
|
discardable: true,
|
|
compressable: true,
|
|
onlyGzip: true,
|
|
detachedSign: false,
|
|
clearSign: false,
|
|
acquireByHash: files.acquireByHash,
|
|
relativePath: relativePath,
|
|
}
|
|
|
|
files.indexes[key] = file
|
|
}
|
|
|
|
return file
|
|
}
|
|
|
|
func (files *indexFiles) ReleaseFile() *indexFile {
|
|
return &indexFile{
|
|
parent: files,
|
|
discardable: false,
|
|
compressable: false,
|
|
detachedSign: true,
|
|
clearSign: true,
|
|
relativePath: "Release",
|
|
}
|
|
}
|
|
|
|
func (files *indexFiles) FinalizeAll(progress aptly.Progress, signer pgp.Signer) (err error) {
|
|
if progress != nil {
|
|
progress.InitBar(int64(len(files.indexes)), false, aptly.BarPublishFinalizeIndexes)
|
|
defer progress.ShutdownBar()
|
|
}
|
|
|
|
for _, file := range files.indexes {
|
|
err = file.Finalize(signer)
|
|
if err != nil {
|
|
return
|
|
}
|
|
if progress != nil {
|
|
progress.AddBar(1)
|
|
}
|
|
}
|
|
|
|
files.indexes = make(map[string]*indexFile)
|
|
|
|
return
|
|
}
|
|
|
|
func (files *indexFiles) RenameFiles() error {
|
|
var err error
|
|
|
|
for oldName, newName := range files.renameMap {
|
|
err = files.publishedStorage.RenameFile(oldName, newName)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to rename: %s", err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|