aptly/swift/public.go

package swift

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/aptly-dev/aptly/aptly"
	"github.com/aptly-dev/aptly/utils"
	"github.com/ncw/swift"
	"github.com/pkg/errors"
)

// PublishedStorage abstract file system with published files (actually hosted on Swift)
type PublishedStorage struct {
	conn              *swift.Connection
	container         string
	prefix            string
	supportBulkDelete bool
}

type swiftInfo map[string]interface{}

// Check interface
var (
	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
)

// NewPublishedStorage creates new instance of PublishedStorage with specified Swift access
// keys, tenant and tenantId
func NewPublishedStorage(username string, password string, authURL string, tenant string, tenantID string, domain string, domainID string, tenantDomain string, tenantDomainID string, container string, prefix string) (*PublishedStorage, error) {
	if username == "" {
		if username = os.Getenv("OS_USERNAME"); username == "" {
			username = os.Getenv("ST_USER")
		}
	}
	if password == "" {
		if password = os.Getenv("OS_PASSWORD"); password == "" {
			password = os.Getenv("ST_KEY")
		}
	}
	if authURL == "" {
		if authURL = os.Getenv("OS_AUTH_URL"); authURL == "" {
			authURL = os.Getenv("ST_AUTH")
		}
	}
	if tenant == "" {
		tenant = os.Getenv("OS_TENANT_NAME")
	}
	if tenantID == "" {
		tenantID = os.Getenv("OS_TENANT_ID")
	}
	if domain == "" {
		domain = os.Getenv("OS_USER_DOMAIN_NAME")
	}
	if domainID == "" {
		domainID = os.Getenv("OS_USER_DOMAIN_ID")
	}
	if tenantDomain == "" {
		tenantDomain = os.Getenv("OS_PROJECT_DOMAIN")
	}
	if tenantDomainID == "" {
		tenantDomainID = os.Getenv("OS_PROJECT_DOMAIN_ID")
	}

	ct := &swift.Connection{
		UserName:       username,
		ApiKey:         password,
		AuthUrl:        authURL,
		UserAgent:      "aptly/" + aptly.Version,
		Tenant:         tenant,
		TenantId:       tenantID,
		Domain:         domain,
		DomainId:       domainID,
		TenantDomain:   tenantDomain,
		TenantDomainId: tenantDomainID,
		ConnectTimeout: 60 * time.Second,
		Timeout:        60 * time.Second,
	}
	err := ct.Authenticate()
	if err != nil {
		return nil, fmt.Errorf("swift authentication failed: %s", err)
	}

	var bulkDelete bool
	resp, err := http.Get(filepath.Join(ct.StorageUrl, "..", "..") + "/info")
	if err == nil {
		defer resp.Body.Close()
		decoder := json.NewDecoder(resp.Body)
		var infos swiftInfo
		if decoder.Decode(&infos) == nil {
			_, bulkDelete = infos["bulk_delete"]
		}
	}

	result := &PublishedStorage{
		conn:              ct,
		container:         container,
		prefix:            prefix,
		supportBulkDelete: bulkDelete,
	}

	return result, nil
}

// String
func (storage *PublishedStorage) String() string {
	return fmt.Sprintf("Swift: %s/%s/%s", storage.conn.StorageUrl, storage.container, storage.prefix)
}

// MkDir creates directory recursively under public path
func (storage *PublishedStorage) MkDir(_ string) error {
	// no op for Swift
	return nil
}

// PutFile puts file into published storage at specified path
func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
	var (
		source *os.File
		err    error
	)
	source, err = os.Open(sourceFilename)
	if err != nil {
		return err
	}
	defer source.Close()

	err = storage.putFile(path, source)
	if err != nil {
		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
	}

	return err
}

func (storage *PublishedStorage) putFile(path string, source io.Reader) error {
	_, err := storage.conn.ObjectPut(storage.container, filepath.Join(storage.prefix, path), source, false, "", "", nil)

	return err
}

// Remove removes single file under public path
func (storage *PublishedStorage) Remove(path string) error {
	err := storage.conn.ObjectDelete(storage.container, filepath.Join(storage.prefix, path))

	if err != nil {
		return fmt.Errorf("error deleting %s from %s: %s", path, storage, err)
	}
	return nil
}

// RemoveDirs removes directory structure under public path
func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error {
	path = filepath.Join(storage.prefix, path)
	opts := swift.ObjectsOpts{
		Prefix: path,
	}

	objects, err := storage.conn.ObjectNamesAll(storage.container, &opts)
	if err != nil {
		return fmt.Errorf("error removing dir %s from %s: %s", path, storage, err)
	}

	for index, name := range objects {
		objects[index] = name[len(storage.prefix):]
	}

	multiDelete := true
	if storage.supportBulkDelete {
		_, err := storage.conn.BulkDelete(storage.container, objects)
		multiDelete = err != nil
	}
	if multiDelete {
		for _, name := range objects {
			if err := storage.conn.ObjectDelete(storage.container, name); err != nil {
				return err
			}
		}
	}

	return nil
}

// LinkFromPool links package file from pool to dist's pool location
//
// publishedPrefix is desired prefix for the location in the pool.
// publishedRelPath is desired location in pool (like pool/component/liba/libav/)
// sourcePool is instance of aptly.PackagePool
// sourcePath is filepath to package file in package pool
//
// LinkFromPool returns relative path for the published file to be included in package index
func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool,
	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {

	relPath := filepath.Join(publishedPrefix, publishedRelPath, fileName)
	poolPath := filepath.Join(storage.prefix, relPath)

	var (
		info swift.Object
		err  error
	)

	info, _, err = storage.conn.Object(storage.container, poolPath)
	if err != nil {
		if err != swift.ObjectNotFound {
			return fmt.Errorf("error getting information about %s from %s: %s", poolPath, storage, err)
		}
	} else {
		if sourceChecksums.MD5 == "" {
			return fmt.Errorf("unable to compare object, MD5 checksum missing")
		}

		if !force && info.Hash != sourceChecksums.MD5 {
			return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage)

		}
	}

	source, err := sourcePool.Open(sourcePath)
	if err != nil {
		return err
	}
	defer source.Close()

	err = storage.putFile(relPath, source)
	if err != nil {
		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
	}

	return err
}

// Filelist returns list of files under prefix
func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
	prefix = filepath.Join(storage.prefix, prefix)
	if prefix != "" {
		prefix += "/"
	}
	opts := swift.ObjectsOpts{
		Prefix: prefix,
	}
	contents, err := storage.conn.ObjectNamesAll(storage.container, &opts)
	if err != nil {
		return nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
	}

	for index, name := range contents {
		contents[index] = name[len(prefix):]
	}

	return contents, nil
}

// RenameFile renames (moves) file
func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
	err := storage.conn.ObjectMove(storage.container, filepath.Join(storage.prefix, oldName), storage.container, filepath.Join(storage.prefix, newName))
	if err != nil {
		return fmt.Errorf("error copying %s -> %s in %s: %s", oldName, newName, storage, err)
	}

	return nil
}

// SymLink creates a copy of src file and adds link information as meta data
func (storage *PublishedStorage) SymLink(src string, dst string) error {
	srcObjectName := filepath.Join(storage.prefix, src)
	dstObjectName := filepath.Join(storage.prefix, dst)

	headers := map[string]string{
		"SymLink": srcObjectName,
	}
	_, err := storage.conn.ObjectCopy(storage.container, srcObjectName, storage.container, dstObjectName, headers)
	if err != nil {
		return fmt.Errorf("error symlinking %s -> %s in %s: %s", srcObjectName, dstObjectName, storage, err)
	}
	return err
}

// HardLink using symlink functionality as hard links do not exist
func (storage *PublishedStorage) HardLink(src string, dst string) error {
	return storage.SymLink(src, dst)
}

// FileExists returns true if path exists
func (storage *PublishedStorage) FileExists(path string) (bool, error) {
	_, _, err := storage.conn.Object(storage.container, filepath.Join(storage.prefix, path))

	if err != nil {
		if err == swift.ObjectNotFound {
			return false, nil
		}

		return false, err
	}

	return true, nil
}

// ReadLink returns the symbolic link pointed to by path
func (storage *PublishedStorage) ReadLink(path string) (string, error) {
	srcObjectName := filepath.Join(storage.prefix, path)
	_, headers, err := storage.conn.Object(storage.container, srcObjectName)
	if err != nil {
		return "", fmt.Errorf("error reading symlink %s in %s: %s", srcObjectName, storage, err)
	}

	return headers["SymLink"], nil
}