container: add support for uploading to registries

Add a new generic container registry client via a new `container`
package. Use this to create a command line utility as well as a
new upload target for container registries.

The code uses the github.com/containers/* project and packages to
interact with container registires that is also used by skopeo,
podman et al. One if the dependencies is `proglottis/gpgme` that
is using cgo to bind libgpgme, so we have to add the corresponding
devel package to the BuildRequires as well as installing it on CI.

Checks will follow later via an integration test.

vendor/github.com/containers/image/v5/oci/archive/oci_dest.go

@@ -0,0 +1,168 @@
+package archive
+
+import (
+	"context"
+	"io"
+	"os"
+
+	"github.com/containers/image/v5/types"
+	"github.com/containers/storage/pkg/archive"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+type ociArchiveImageDestination struct {
+	ref          ociArchiveReference
+	unpackedDest types.ImageDestination
+	tempDirRef   tempDirOCIRef
+}
+
+// newImageDestination returns an ImageDestination for writing to an existing directory.
+func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageDestination, error) {
+	tempDirRef, err := createOCIRef(sys, ref.image)
+	if err != nil {
+		return nil, errors.Wrapf(err, "creating oci reference")
+	}
+	unpackedDest, err := tempDirRef.ociRefExtracted.NewImageDestination(ctx, sys)
+	if err != nil {
+		if err := tempDirRef.deleteTempDir(); err != nil {
+			return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
+		}
+		return nil, err
+	}
+	return &ociArchiveImageDestination{ref: ref,
+		unpackedDest: unpackedDest,
+		tempDirRef:   tempDirRef}, nil
+}
+
+// Reference returns the reference used to set up this destination.
+func (d *ociArchiveImageDestination) Reference() types.ImageReference {
+	return d.ref
+}
+
+// Close removes resources associated with an initialized ImageDestination, if any
+// Close deletes the temp directory of the oci-archive image
+func (d *ociArchiveImageDestination) Close() error {
+	defer func() {
+		err := d.tempDirRef.deleteTempDir()
+		logrus.Debugf("Error deleting temporary directory: %v", err)
+	}()
+	return d.unpackedDest.Close()
+}
+
+func (d *ociArchiveImageDestination) SupportedManifestMIMETypes() []string {
+	return d.unpackedDest.SupportedManifestMIMETypes()
+}
+
+// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures
+func (d *ociArchiveImageDestination) SupportsSignatures(ctx context.Context) error {
+	return d.unpackedDest.SupportsSignatures(ctx)
+}
+
+func (d *ociArchiveImageDestination) DesiredLayerCompression() types.LayerCompression {
+	return d.unpackedDest.DesiredLayerCompression()
+}
+
+// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
+// uploaded to the image destination, true otherwise.
+func (d *ociArchiveImageDestination) AcceptsForeignLayerURLs() bool {
+	return d.unpackedDest.AcceptsForeignLayerURLs()
+}
+
+// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise
+func (d *ociArchiveImageDestination) MustMatchRuntimeOS() bool {
+	return d.unpackedDest.MustMatchRuntimeOS()
+}
+
+// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(),
+// and would prefer to receive an unmodified manifest instead of one modified for the destination.
+// Does not make a difference if Reference().DockerReference() is nil.
+func (d *ociArchiveImageDestination) IgnoresEmbeddedDockerReference() bool {
+	return d.unpackedDest.IgnoresEmbeddedDockerReference()
+}
+
+// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently.
+func (d *ociArchiveImageDestination) HasThreadSafePutBlob() bool {
+	return false
+}
+
+// PutBlob writes contents of stream and returns data representing the result.
+// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents.
+// inputInfo.Size is the expected length of stream, if known.
+// inputInfo.MediaType describes the blob format, if known.
+// May update cache.
+// WARNING: The contents of stream are being verified on the fly.  Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
+// to any other readers for download using the supplied digest.
+// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
+func (d *ociArchiveImageDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) {
+	return d.unpackedDest.PutBlob(ctx, stream, inputInfo, cache, isConfig)
+}
+
+// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
+// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree).
+// info.Digest must not be empty.
+// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input.
+// If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may
+// include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be
+// reflected in the manifest that will be written.
+// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
+// May use and/or update cache.
+func (d *ociArchiveImageDestination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) {
+	return d.unpackedDest.TryReusingBlob(ctx, info, cache, canSubstitute)
+}
+
+// PutManifest writes the manifest to the destination.
+// If instanceDigest is not nil, it contains a digest of the specific manifest instance to overwrite the manifest for (when
+// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list.
+// It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated
+// by `manifest.Digest()`.
+func (d *ociArchiveImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error {
+	return d.unpackedDest.PutManifest(ctx, m, instanceDigest)
+}
+
+// PutSignatures writes a set of signatures to the destination.
+// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for
+// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list.
+func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error {
+	return d.unpackedDest.PutSignatures(ctx, signatures, instanceDigest)
+}
+
+// Commit marks the process of storing the image as successful and asks for the image to be persisted
+// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
+// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
+// original manifest list digest, if desired.
+// after the directory is made, it is tarred up into a file and the directory is deleted
+func (d *ociArchiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
+	if err := d.unpackedDest.Commit(ctx, unparsedToplevel); err != nil {
+		return errors.Wrapf(err, "storing image %q", d.ref.image)
+	}
+
+	// path of directory to tar up
+	src := d.tempDirRef.tempDirectory
+	// path to save tarred up file
+	dst := d.ref.resolvedFile
+	return tarDirectory(src, dst)
+}
+
+// tar converts the directory at src and saves it to dst
+func tarDirectory(src, dst string) error {
+	// input is a stream of bytes from the archive of the directory at path
+	input, err := archive.Tar(src, archive.Uncompressed)
+	if err != nil {
+		return errors.Wrapf(err, "retrieving stream of bytes from %q", src)
+	}
+
+	// creates the tar file
+	outFile, err := os.Create(dst)
+	if err != nil {
+		return errors.Wrapf(err, "creating tar file %q", dst)
+	}
+	defer outFile.Close()
+
+	// copies the contents of the directory to the tar file
+	// TODO: This can take quite some time, and should ideally be cancellable using a context.Context.
+	_, err = io.Copy(outFile, input)
+
+	return err
+}

vendor/github.com/containers/image/v5/oci/archive/oci_src.go

@@ -0,0 +1,122 @@
+package archive
+
+import (
+	"context"
+	"io"
+
+	ocilayout "github.com/containers/image/v5/oci/layout"
+	"github.com/containers/image/v5/types"
+	digest "github.com/opencontainers/go-digest"
+	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+type ociArchiveImageSource struct {
+	ref         ociArchiveReference
+	unpackedSrc types.ImageSource
+	tempDirRef  tempDirOCIRef
+}
+
+// newImageSource returns an ImageSource for reading from an existing directory.
+// newImageSource untars the file and saves it in a temp directory
+func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageSource, error) {
+	tempDirRef, err := createUntarTempDir(sys, ref)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating temp directory")
+	}
+
+	unpackedSrc, err := tempDirRef.ociRefExtracted.NewImageSource(ctx, sys)
+	if err != nil {
+		if err := tempDirRef.deleteTempDir(); err != nil {
+			return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
+		}
+		return nil, err
+	}
+	return &ociArchiveImageSource{ref: ref,
+		unpackedSrc: unpackedSrc,
+		tempDirRef:  tempDirRef}, nil
+}
+
+// LoadManifestDescriptor loads the manifest
+// Deprecated: use LoadManifestDescriptorWithContext instead
+func LoadManifestDescriptor(imgRef types.ImageReference) (imgspecv1.Descriptor, error) {
+	return LoadManifestDescriptorWithContext(nil, imgRef)
+}
+
+// LoadManifestDescriptorWithContext loads the manifest
+func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.ImageReference) (imgspecv1.Descriptor, error) {
+	ociArchRef, ok := imgRef.(ociArchiveReference)
+	if !ok {
+		return imgspecv1.Descriptor{}, errors.Errorf("error typecasting, need type ociArchiveReference")
+	}
+	tempDirRef, err := createUntarTempDir(sys, ociArchRef)
+	if err != nil {
+		return imgspecv1.Descriptor{}, errors.Wrap(err, "creating temp directory")
+	}
+	defer func() {
+		err := tempDirRef.deleteTempDir()
+		logrus.Debugf("Error deleting temporary directory: %v", err)
+	}()
+
+	descriptor, err := ocilayout.LoadManifestDescriptor(tempDirRef.ociRefExtracted)
+	if err != nil {
+		return imgspecv1.Descriptor{}, errors.Wrap(err, "loading index")
+	}
+	return descriptor, nil
+}
+
+// Reference returns the reference used to set up this source.
+func (s *ociArchiveImageSource) Reference() types.ImageReference {
+	return s.ref
+}
+
+// Close removes resources associated with an initialized ImageSource, if any.
+// Close deletes the temporary directory at dst
+func (s *ociArchiveImageSource) Close() error {
+	defer func() {
+		err := s.tempDirRef.deleteTempDir()
+		logrus.Debugf("error deleting tmp dir: %v", err)
+	}()
+	return s.unpackedSrc.Close()
+}
+
+// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
+// It may use a remote (= slow) service.
+// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list);
+// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists).
+func (s *ociArchiveImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
+	return s.unpackedSrc.GetManifest(ctx, instanceDigest)
+}
+
+// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently.
+func (s *ociArchiveImageSource) HasThreadSafeGetBlob() bool {
+	return false
+}
+
+// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown).
+// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
+// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
+func (s *ociArchiveImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
+	return s.unpackedSrc.GetBlob(ctx, info, cache)
+}
+
+// GetSignatures returns the image's signatures.  It may use a remote (= slow) service.
+// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for
+// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list
+// (e.g. if the source never returns manifest lists).
+func (s *ociArchiveImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) {
+	return s.unpackedSrc.GetSignatures(ctx, instanceDigest)
+}
+
+// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer
+// blobsums that are listed in the image's manifest.  If values are returned, they should be used when using GetBlob()
+// to read the image's layers.
+// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for
+// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list
+// (e.g. if the source never returns manifest lists).
+// The Digest field is guaranteed to be provided; Size may be -1.
+// WARNING: The list may contain duplicates, and they are semantically relevant.
+func (s *ociArchiveImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) {
+	return s.unpackedSrc.LayerInfosForCopy(ctx, instanceDigest)
+}

vendor/github.com/containers/image/v5/oci/archive/oci_transport.go

@@ -0,0 +1,197 @@
+package archive
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/containers/image/v5/directory/explicitfilepath"
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/image"
+	"github.com/containers/image/v5/internal/tmpdir"
+	"github.com/containers/image/v5/oci/internal"
+	ocilayout "github.com/containers/image/v5/oci/layout"
+	"github.com/containers/image/v5/transports"
+	"github.com/containers/image/v5/types"
+	"github.com/containers/storage/pkg/archive"
+	"github.com/pkg/errors"
+)
+
+func init() {
+	transports.Register(Transport)
+}
+
+// Transport is an ImageTransport for OCI archive
+// it creates an oci-archive tar file by calling into the OCI transport
+// tarring the directory created by oci and deleting the directory
+var Transport = ociArchiveTransport{}
+
+type ociArchiveTransport struct{}
+
+// ociArchiveReference is an ImageReference for OCI Archive paths
+type ociArchiveReference struct {
+	file         string
+	resolvedFile string
+	image        string
+}
+
+func (t ociArchiveTransport) Name() string {
+	return "oci-archive"
+}
+
+// ParseReference converts a string, which should not start with the ImageTransport.Name prefix
+// into an ImageReference.
+func (t ociArchiveTransport) ParseReference(reference string) (types.ImageReference, error) {
+	return ParseReference(reference)
+}
+
+// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys
+func (t ociArchiveTransport) ValidatePolicyConfigurationScope(scope string) error {
+	return internal.ValidateScope(scope)
+}
+
+// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference.
+func ParseReference(reference string) (types.ImageReference, error) {
+	file, image := internal.SplitPathAndImage(reference)
+	return NewReference(file, image)
+}
+
+// NewReference returns an OCI reference for a file and a image.
+func NewReference(file, image string) (types.ImageReference, error) {
+	resolved, err := explicitfilepath.ResolvePathToFullyExplicit(file)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := internal.ValidateOCIPath(file); err != nil {
+		return nil, err
+	}
+
+	if err := internal.ValidateImageName(image); err != nil {
+		return nil, err
+	}
+
+	return ociArchiveReference{file: file, resolvedFile: resolved, image: image}, nil
+}
+
+func (ref ociArchiveReference) Transport() types.ImageTransport {
+	return Transport
+}
+
+// StringWithinTransport returns a string representation of the reference, which MUST be such that
+// reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference.
+func (ref ociArchiveReference) StringWithinTransport() string {
+	return fmt.Sprintf("%s:%s", ref.file, ref.image)
+}
+
+// DockerReference returns a Docker reference associated with this reference
+func (ref ociArchiveReference) DockerReference() reference.Named {
+	return nil
+}
+
+// PolicyConfigurationIdentity returns a string representation of the reference, suitable for policy lookup.
+func (ref ociArchiveReference) PolicyConfigurationIdentity() string {
+	// NOTE: ref.image is not a part of the image identity, because "$dir:$someimage" and "$dir:" may mean the
+	// same image and the two can’t be statically disambiguated.  Using at least the repository directory is
+	// less granular but hopefully still useful.
+	return ref.resolvedFile
+}
+
+// PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search
+// for if explicit configuration for PolicyConfigurationIdentity() is not set
+func (ref ociArchiveReference) PolicyConfigurationNamespaces() []string {
+	res := []string{}
+	path := ref.resolvedFile
+	for {
+		lastSlash := strings.LastIndex(path, "/")
+		// Note that we do not include "/"; it is redundant with the default "" global default,
+		// and rejected by ociTransport.ValidatePolicyConfigurationScope above.
+		if lastSlash == -1 || path == "/" {
+			break
+		}
+		res = append(res, path)
+		path = path[:lastSlash]
+	}
+	return res
+}
+
+// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport.
+// The caller must call .Close() on the returned ImageCloser.
+// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource,
+// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
+// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details.
+func (ref ociArchiveReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) {
+	src, err := newImageSource(ctx, sys, ref)
+	if err != nil {
+		return nil, err
+	}
+	return image.FromSource(ctx, sys, src)
+}
+
+// NewImageSource returns a types.ImageSource for this reference.
+// The caller must call .Close() on the returned ImageSource.
+func (ref ociArchiveReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) {
+	return newImageSource(ctx, sys, ref)
+}
+
+// NewImageDestination returns a types.ImageDestination for this reference.
+// The caller must call .Close() on the returned ImageDestination.
+func (ref ociArchiveReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) {
+	return newImageDestination(ctx, sys, ref)
+}
+
+// DeleteImage deletes the named image from the registry, if supported.
+func (ref ociArchiveReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error {
+	return errors.Errorf("Deleting images not implemented for oci: images")
+}
+
+// struct to store the ociReference and temporary directory returned by createOCIRef
+type tempDirOCIRef struct {
+	tempDirectory   string
+	ociRefExtracted types.ImageReference
+}
+
+// deletes the temporary directory created
+func (t *tempDirOCIRef) deleteTempDir() error {
+	return os.RemoveAll(t.tempDirectory)
+}
+
+// createOCIRef creates the oci reference of the image
+// If SystemContext.BigFilesTemporaryDir not "", overrides the temporary directory to use for storing big files
+func createOCIRef(sys *types.SystemContext, image string) (tempDirOCIRef, error) {
+	dir, err := os.MkdirTemp(tmpdir.TemporaryDirectoryForBigFiles(sys), "oci")
+	if err != nil {
+		return tempDirOCIRef{}, errors.Wrapf(err, "creating temp directory")
+	}
+	ociRef, err := ocilayout.NewReference(dir, image)
+	if err != nil {
+		return tempDirOCIRef{}, err
+	}
+
+	tempDirRef := tempDirOCIRef{tempDirectory: dir, ociRefExtracted: ociRef}
+	return tempDirRef, nil
+}
+
+// creates the temporary directory and copies the tarred content to it
+func createUntarTempDir(sys *types.SystemContext, ref ociArchiveReference) (tempDirOCIRef, error) {
+	tempDirRef, err := createOCIRef(sys, ref.image)
+	if err != nil {
+		return tempDirOCIRef{}, errors.Wrap(err, "creating oci reference")
+	}
+	src := ref.resolvedFile
+	dst := tempDirRef.tempDirectory
+	// TODO: This can take quite some time, and should ideally be cancellable using a context.Context.
+	arch, err := os.Open(src)
+	if err != nil {
+		return tempDirOCIRef{}, err
+	}
+	defer arch.Close()
+	if err := archive.NewDefaultArchiver().Untar(arch, dst, &archive.TarOptions{NoLchown: true}); err != nil {
+		if err := tempDirRef.deleteTempDir(); err != nil {
+			return tempDirOCIRef{}, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
+		}
+		return tempDirOCIRef{}, errors.Wrapf(err, "untarring file %q", tempDirRef.tempDirectory)
+	}
+	return tempDirRef, nil
+}