container: add support for uploading to registries

Add a new generic container registry client via a new `container`
package. Use this to create a command line utility as well as a
new upload target for container registries.

The code uses the github.com/containers/* project and packages to
interact with container registires that is also used by skopeo,
podman et al. One if the dependencies is `proglottis/gpgme` that
is using cgo to bind libgpgme, so we have to add the corresponding
devel package to the BuildRequires as well as installing it on CI.

Checks will follow later via an integration test.

vendor/github.com/containers/image/v5/copy/digesting_reader.go

@@ -0,0 +1,62 @@
+package copy
+
+import (
+	"hash"
+	"io"
+
+	digest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+)
+
+type digestingReader struct {
+	source              io.Reader
+	digester            digest.Digester
+	hash                hash.Hash
+	expectedDigest      digest.Digest
+	validationFailed    bool
+	validationSucceeded bool
+}
+
+// newDigestingReader returns an io.Reader implementation with contents of source, which will eventually return a non-EOF error
+// or set validationSucceeded/validationFailed to true if the source stream does/does not match expectedDigest.
+// (neither is set if EOF is never reached).
+func newDigestingReader(source io.Reader, expectedDigest digest.Digest) (*digestingReader, error) {
+	var digester digest.Digester
+	if err := expectedDigest.Validate(); err != nil {
+		return nil, errors.Errorf("Invalid digest specification %s", expectedDigest)
+	}
+	digestAlgorithm := expectedDigest.Algorithm()
+	if !digestAlgorithm.Available() {
+		return nil, errors.Errorf("Invalid digest specification %s: unsupported digest algorithm %s", expectedDigest, digestAlgorithm)
+	}
+	digester = digestAlgorithm.Digester()
+
+	return &digestingReader{
+		source:           source,
+		digester:         digester,
+		hash:             digester.Hash(),
+		expectedDigest:   expectedDigest,
+		validationFailed: false,
+	}, nil
+}
+
+func (d *digestingReader) Read(p []byte) (int, error) {
+	n, err := d.source.Read(p)
+	if n > 0 {
+		if n2, err := d.hash.Write(p[:n]); n2 != n || err != nil {
+			// Coverage: This should not happen, the hash.Hash interface requires
+			// d.digest.Write to never return an error, and the io.Writer interface
+			// requires n2 == len(input) if no error is returned.
+			return 0, errors.Wrapf(err, "updating digest during verification: %d vs. %d", n2, n)
+		}
+	}
+	if err == io.EOF {
+		actualDigest := d.digester.Digest()
+		if actualDigest != d.expectedDigest {
+			d.validationFailed = true
+			return 0, errors.Errorf("Digest did not match, expected %s, got %s", d.expectedDigest, actualDigest)
+		}
+		d.validationSucceeded = true
+	}
+	return n, err
+}

vendor/github.com/containers/image/v5/copy/encrypt.go

@@ -0,0 +1,24 @@
+package copy
+
+import (
+	"strings"
+
+	"github.com/containers/image/v5/types"
+)
+
+// isOciEncrypted returns a bool indicating if a mediatype is encrypted
+// This function will be moved to be part of OCI spec when adopted.
+func isOciEncrypted(mediatype string) bool {
+	return strings.HasSuffix(mediatype, "+encrypted")
+}
+
+// isEncrypted checks if an image is encrypted
+func isEncrypted(i types.Image) bool {
+	layers := i.LayerInfos()
+	for _, l := range layers {
+		if isOciEncrypted(l.MediaType) {
+			return true
+		}
+	}
+	return false
+}

vendor/github.com/containers/image/v5/copy/manifest.go

@@ -0,0 +1,170 @@
+package copy
+
+import (
+	"context"
+	"strings"
+
+	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/types"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// preferredManifestMIMETypes lists manifest MIME types in order of our preference, if we can't use the original manifest and need to convert.
+// Prefer v2s2 to v2s1 because v2s2 does not need to be changed when uploading to a different location.
+// Include v2s1 signed but not v2s1 unsigned, because docker/distribution requires a signature even if the unsigned MIME type is used.
+var preferredManifestMIMETypes = []string{manifest.DockerV2Schema2MediaType, manifest.DockerV2Schema1SignedMediaType}
+
+// orderedSet is a list of strings (MIME types or platform descriptors in our case), with each string appearing at most once.
+type orderedSet struct {
+	list     []string
+	included map[string]struct{}
+}
+
+// newOrderedSet creates a correctly initialized orderedSet.
+// [Sometimes it would be really nice if Golang had constructors…]
+func newOrderedSet() *orderedSet {
+	return &orderedSet{
+		list:     []string{},
+		included: map[string]struct{}{},
+	}
+}
+
+// append adds s to the end of os, only if it is not included already.
+func (os *orderedSet) append(s string) {
+	if _, ok := os.included[s]; !ok {
+		os.list = append(os.list, s)
+		os.included[s] = struct{}{}
+	}
+}
+
+// determineManifestConversion updates ic.manifestUpdates to convert manifest to a supported MIME type, if necessary and ic.canModifyManifest.
+// Note that the conversion will only happen later, through ic.src.UpdatedImage
+// Returns the preferred manifest MIME type (whether we are converting to it or using it unmodified),
+// and a list of other possible alternatives, in order.
+func (ic *imageCopier) determineManifestConversion(ctx context.Context, destSupportedManifestMIMETypes []string, forceManifestMIMEType string, requiresOciEncryption bool) (string, []string, error) {
+	_, srcType, err := ic.src.Manifest(ctx)
+	if err != nil { // This should have been cached?!
+		return "", nil, errors.Wrap(err, "reading manifest")
+	}
+	normalizedSrcType := manifest.NormalizedMIMEType(srcType)
+	if srcType != normalizedSrcType {
+		logrus.Debugf("Source manifest MIME type %s, treating it as %s", srcType, normalizedSrcType)
+		srcType = normalizedSrcType
+	}
+
+	if forceManifestMIMEType != "" {
+		destSupportedManifestMIMETypes = []string{forceManifestMIMEType}
+	}
+
+	if len(destSupportedManifestMIMETypes) == 0 && (!requiresOciEncryption || manifest.MIMETypeSupportsEncryption(srcType)) {
+		return srcType, []string{}, nil // Anything goes; just use the original as is, do not try any conversions.
+	}
+	supportedByDest := map[string]struct{}{}
+	for _, t := range destSupportedManifestMIMETypes {
+		if !requiresOciEncryption || manifest.MIMETypeSupportsEncryption(t) {
+			supportedByDest[t] = struct{}{}
+		}
+	}
+
+	// destSupportedManifestMIMETypes is a static guess; a particular registry may still only support a subset of the types.
+	// So, build a list of types to try in order of decreasing preference.
+	// FIXME? This treats manifest.DockerV2Schema1SignedMediaType and manifest.DockerV2Schema1MediaType as distinct,
+	// although we are not really making any conversion, and it is very unlikely that a destination would support one but not the other.
+	// In practice, schema1 is probably the lowest common denominator, so we would expect to try the first one of the MIME types
+	// and never attempt the other one.
+	prioritizedTypes := newOrderedSet()
+
+	// First of all, prefer to keep the original manifest unmodified.
+	if _, ok := supportedByDest[srcType]; ok {
+		prioritizedTypes.append(srcType)
+	}
+	if ic.cannotModifyManifestReason != "" {
+		// We could also drop this check and have the caller
+		// make the choice; it is already doing that to an extent, to improve error
+		// messages.  But it is nice to hide the “if we can't modify, do no conversion”
+		// special case in here; the caller can then worry (or not) only about a good UI.
+		logrus.Debugf("We can't modify the manifest, hoping for the best...")
+		return srcType, []string{}, nil // Take our chances - FIXME? Or should we fail without trying?
+	}
+
+	// Then use our list of preferred types.
+	for _, t := range preferredManifestMIMETypes {
+		if _, ok := supportedByDest[t]; ok {
+			prioritizedTypes.append(t)
+		}
+	}
+
+	// Finally, try anything else the destination supports.
+	for _, t := range destSupportedManifestMIMETypes {
+		prioritizedTypes.append(t)
+	}
+
+	logrus.Debugf("Manifest has MIME type %s, ordered candidate list [%s]", srcType, strings.Join(prioritizedTypes.list, ", "))
+	if len(prioritizedTypes.list) == 0 { // Coverage: destSupportedManifestMIMETypes is not empty (or we would have exited in the “Anything goes” case above), so this should never happen.
+		return "", nil, errors.New("Internal error: no candidate MIME types")
+	}
+	preferredType := prioritizedTypes.list[0]
+	if preferredType != srcType {
+		ic.manifestUpdates.ManifestMIMEType = preferredType
+	} else {
+		logrus.Debugf("... will first try using the original manifest unmodified")
+	}
+	return preferredType, prioritizedTypes.list[1:], nil
+}
+
+// isMultiImage returns true if img is a list of images
+func isMultiImage(ctx context.Context, img types.UnparsedImage) (bool, error) {
+	_, mt, err := img.Manifest(ctx)
+	if err != nil {
+		return false, err
+	}
+	return manifest.MIMETypeIsMultiImage(mt), nil
+}
+
+// determineListConversion takes the current MIME type of a list of manifests,
+// the list of MIME types supported for a given destination, and a possible
+// forced value, and returns the MIME type to which we should convert the list
+// of manifests (regardless of whether we are converting to it or using it
+// unmodified) and a slice of other list types which might be supported by the
+// destination.
+func (c *copier) determineListConversion(currentListMIMEType string, destSupportedMIMETypes []string, forcedListMIMEType string) (string, []string, error) {
+	// If there's no list of supported types, then anything we support is expected to be supported.
+	if len(destSupportedMIMETypes) == 0 {
+		destSupportedMIMETypes = manifest.SupportedListMIMETypes
+	}
+	// If we're forcing it, replace the list of supported types with the forced value.
+	if forcedListMIMEType != "" {
+		destSupportedMIMETypes = []string{forcedListMIMEType}
+	}
+
+	prioritizedTypes := newOrderedSet()
+	// The first priority is the current type, if it's in the list, since that lets us avoid a
+	// conversion that isn't strictly necessary.
+	for _, t := range destSupportedMIMETypes {
+		if t == currentListMIMEType {
+			prioritizedTypes.append(currentListMIMEType)
+			break
+		}
+	}
+	// Pick out the other list types that we support.
+	for _, t := range destSupportedMIMETypes {
+		if manifest.MIMETypeIsMultiImage(t) {
+			prioritizedTypes.append(t)
+		}
+	}
+
+	logrus.Debugf("Manifest list has MIME type %s, ordered candidate list [%s]", currentListMIMEType, strings.Join(destSupportedMIMETypes, ", "))
+	if len(prioritizedTypes.list) == 0 {
+		return "", nil, errors.Errorf("destination does not support any supported manifest list types (%v)", manifest.SupportedListMIMETypes)
+	}
+	selectedType := prioritizedTypes.list[0]
+	otherSupportedTypes := prioritizedTypes.list[1:]
+	if selectedType != currentListMIMEType {
+		logrus.Debugf("... will convert to %s first, and then try %v", selectedType, otherSupportedTypes)
+	} else {
+		logrus.Debugf("... will use the original manifest list type, and then try %v", otherSupportedTypes)
+	}
+	// Done.
+	return selectedType, otherSupportedTypes, nil
+}

vendor/github.com/containers/image/v5/copy/progress_bars.go

@@ -0,0 +1,148 @@
+package copy
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/containers/image/v5/internal/private"
+	"github.com/containers/image/v5/types"
+	"github.com/vbauerster/mpb/v7"
+	"github.com/vbauerster/mpb/v7/decor"
+)
+
+// newProgressPool creates a *mpb.Progress.
+// The caller must eventually call pool.Wait() after the pool will no longer be updated.
+// NOTE: Every progress bar created within the progress pool must either successfully
+// complete or be aborted, or pool.Wait() will hang. That is typically done
+// using "defer bar.Abort(false)", which must be called BEFORE pool.Wait() is called.
+func (c *copier) newProgressPool() *mpb.Progress {
+	return mpb.New(mpb.WithWidth(40), mpb.WithOutput(c.progressOutput))
+}
+
+// customPartialBlobDecorFunc implements mpb.DecorFunc for the partial blobs retrieval progress bar
+func customPartialBlobDecorFunc(s decor.Statistics) string {
+	if s.Total == 0 {
+		pairFmt := "%.1f / %.1f (skipped: %.1f)"
+		return fmt.Sprintf(pairFmt, decor.SizeB1024(s.Current), decor.SizeB1024(s.Total), decor.SizeB1024(s.Refill))
+	}
+	pairFmt := "%.1f / %.1f (skipped: %.1f = %.2f%%)"
+	percentage := 100.0 * float64(s.Refill) / float64(s.Total)
+	return fmt.Sprintf(pairFmt, decor.SizeB1024(s.Current), decor.SizeB1024(s.Total), decor.SizeB1024(s.Refill), percentage)
+}
+
+// progressBar wraps a *mpb.Bar, allowing us to add extra state and methods.
+type progressBar struct {
+	*mpb.Bar
+	originalSize int64 // or -1 if unknown
+}
+
+// createProgressBar creates a progressBar in pool.  Note that if the copier's reportWriter
+// is io.Discard, the progress bar's output will be discarded
+//
+// NOTE: Every progress bar created within a progress pool must either successfully
+// complete or be aborted, or pool.Wait() will hang. That is typically done
+// using "defer bar.Abort(false)", which must happen BEFORE pool.Wait() is called.
+//
+// As a convention, most users of progress bars should call mark100PercentComplete on full success;
+// by convention, we don't leave progress bars in partial state when fully done
+// (even if we copied much less data than anticipated).
+func (c *copier) createProgressBar(pool *mpb.Progress, partial bool, info types.BlobInfo, kind string, onComplete string) *progressBar {
+	// shortDigestLen is the length of the digest used for blobs.
+	const shortDigestLen = 12
+
+	prefix := fmt.Sprintf("Copying %s %s", kind, info.Digest.Encoded())
+	// Truncate the prefix (chopping of some part of the digest) to make all progress bars aligned in a column.
+	maxPrefixLen := len("Copying blob ") + shortDigestLen
+	if len(prefix) > maxPrefixLen {
+		prefix = prefix[:maxPrefixLen]
+	}
+
+	// onComplete will replace prefix once the bar/spinner has completed
+	onComplete = prefix + " " + onComplete
+
+	// Use a normal progress bar when we know the size (i.e., size > 0).
+	// Otherwise, use a spinner to indicate that something's happening.
+	var bar *mpb.Bar
+	if info.Size > 0 {
+		if partial {
+			bar = pool.AddBar(info.Size,
+				mpb.BarFillerClearOnComplete(),
+				mpb.PrependDecorators(
+					decor.OnComplete(decor.Name(prefix), onComplete),
+				),
+				mpb.AppendDecorators(
+					decor.Any(customPartialBlobDecorFunc),
+				),
+			)
+		} else {
+			bar = pool.AddBar(info.Size,
+				mpb.BarFillerClearOnComplete(),
+				mpb.PrependDecorators(
+					decor.OnComplete(decor.Name(prefix), onComplete),
+				),
+				mpb.AppendDecorators(
+					decor.OnComplete(decor.CountersKibiByte("%.1f / %.1f"), ""),
+				),
+			)
+		}
+	} else {
+		bar = pool.New(0,
+			mpb.SpinnerStyle(".", "..", "...", "....", "").PositionLeft(),
+			mpb.BarFillerClearOnComplete(),
+			mpb.PrependDecorators(
+				decor.OnComplete(decor.Name(prefix), onComplete),
+			),
+		)
+	}
+	if c.progressOutput == io.Discard {
+		c.Printf("Copying %s %s\n", kind, info.Digest)
+	}
+	return &progressBar{
+		Bar:          bar,
+		originalSize: info.Size,
+	}
+}
+
+// mark100PercentComplete marks the progres bars as 100% complete;
+// it may do so by possibly advancing the current state if it is below the known total.
+func (bar *progressBar) mark100PercentComplete() {
+	if bar.originalSize > 0 {
+		// We can't call bar.SetTotal even if we wanted to; the total can not be changed
+		// after a progress bar is created with a definite total.
+		bar.SetCurrent(bar.originalSize) // This triggers the completion condition.
+	} else {
+		// -1 = unknown size
+		// 0 is somewhat of a a special case: Unlike c/image, where 0 is a definite known
+		// size (possible at least in theory), in mpb, zero-sized progress bars are treated
+		// as unknown size, in particular they are not configured to be marked as
+		// complete on bar.Current() reaching bar.total (because that would happen already
+		// when creating the progress bar).
+		// That means that we are both _allowed_ to call SetTotal, and we _have to_.
+		bar.SetTotal(-1, true) // total < 0 = set it to bar.Current(), report it; and mark the bar as complete.
+	}
+}
+
+// blobChunkAccessorProxy wraps a BlobChunkAccessor and updates a *progressBar
+// with the number of received bytes.
+type blobChunkAccessorProxy struct {
+	wrapped private.BlobChunkAccessor // The underlying BlobChunkAccessor
+	bar     *progressBar              // A progress bar updated with the number of bytes read so far
+}
+
+// GetBlobAt returns a sequential channel of readers that contain data for the requested
+// blob chunks, and a channel that might get a single error value.
+// The specified chunks must be not overlapping and sorted by their offset.
+// The readers must be fully consumed, in the order they are returned, before blocking
+// to read the next chunk.
+func (s *blobChunkAccessorProxy) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
+	rc, errs, err := s.wrapped.GetBlobAt(ctx, info, chunks)
+	if err == nil {
+		total := int64(0)
+		for _, c := range chunks {
+			total += int64(c.Length)
+		}
+		s.bar.IncrInt64(total)
+	}
+	return rc, errs, err
+}

vendor/github.com/containers/image/v5/copy/progress_channel.go

@@ -0,0 +1,79 @@
+package copy
+
+import (
+	"io"
+	"time"
+
+	"github.com/containers/image/v5/types"
+)
+
+// progressReader is a reader that reports its progress to a types.ProgressProperties channel on an interval.
+type progressReader struct {
+	source       io.Reader
+	channel      chan<- types.ProgressProperties
+	interval     time.Duration
+	artifact     types.BlobInfo
+	lastUpdate   time.Time
+	offset       uint64
+	offsetUpdate uint64
+}
+
+// newProgressReader creates a new progress reader for:
+// `source`:   The source when internally reading bytes
+// `channel`:  The reporter channel to which the progress will be sent
+// `interval`: The update interval to indicate how often the progress should update
+// `artifact`: The blob metadata which is currently being progressed
+func newProgressReader(
+	source io.Reader,
+	channel chan<- types.ProgressProperties,
+	interval time.Duration,
+	artifact types.BlobInfo,
+) *progressReader {
+	// The progress reader constructor informs the progress channel
+	// that a new artifact will be read
+	channel <- types.ProgressProperties{
+		Event:    types.ProgressEventNewArtifact,
+		Artifact: artifact,
+	}
+	return &progressReader{
+		source:       source,
+		channel:      channel,
+		interval:     interval,
+		artifact:     artifact,
+		lastUpdate:   time.Now(),
+		offset:       0,
+		offsetUpdate: 0,
+	}
+}
+
+// reportDone indicates to the internal channel that the progress has been
+// finished
+func (r *progressReader) reportDone() {
+	r.channel <- types.ProgressProperties{
+		Event:        types.ProgressEventDone,
+		Artifact:     r.artifact,
+		Offset:       r.offset,
+		OffsetUpdate: r.offsetUpdate,
+	}
+}
+
+// Read continuously reads bytes into the progress reader and reports the
+// status via the internal channel
+func (r *progressReader) Read(p []byte) (int, error) {
+	n, err := r.source.Read(p)
+	r.offset += uint64(n)
+	r.offsetUpdate += uint64(n)
+
+	// Fire the progress reader in the provided interval
+	if time.Since(r.lastUpdate) > r.interval {
+		r.channel <- types.ProgressProperties{
+			Event:        types.ProgressEventRead,
+			Artifact:     r.artifact,
+			Offset:       r.offset,
+			OffsetUpdate: r.offsetUpdate,
+		}
+		r.lastUpdate = time.Now()
+		r.offsetUpdate = 0
+	}
+	return n, err
+}

vendor/github.com/containers/image/v5/copy/sign.go

@@ -0,0 +1,38 @@
+package copy
+
+import (
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/signature"
+	"github.com/containers/image/v5/transports"
+	"github.com/pkg/errors"
+)
+
+// createSignature creates a new signature of manifest using keyIdentity.
+func (c *copier) createSignature(manifest []byte, keyIdentity string, passphrase string, identity reference.Named) ([]byte, error) {
+	mech, err := signature.NewGPGSigningMechanism()
+	if err != nil {
+		return nil, errors.Wrap(err, "initializing GPG")
+	}
+	defer mech.Close()
+	if err := mech.SupportsSigning(); err != nil {
+		return nil, errors.Wrap(err, "Signing not supported")
+	}
+
+	if identity != nil {
+		if reference.IsNameOnly(identity) {
+			return nil, errors.Errorf("Sign identity must be a fully specified reference %s", identity)
+		}
+	} else {
+		identity = c.dest.Reference().DockerReference()
+		if identity == nil {
+			return nil, errors.Errorf("Cannot determine canonical Docker reference for destination %s", transports.ImageName(c.dest.Reference()))
+		}
+	}
+
+	c.Printf("Signing manifest\n")
+	newSig, err := signature.SignDockerManifestWithOptions(manifest, identity.String(), mech, keyIdentity, &signature.SignOptions{Passphrase: passphrase})
+	if err != nil {
+		return nil, errors.Wrap(err, "creating signature")
+	}
+	return newSig, nil
+}