container: add support for uploading to registries

Add a new generic container registry client via a new `container`
package. Use this to create a command line utility as well as a
new upload target for container registries.

The code uses the github.com/containers/* project and packages to
interact with container registires that is also used by skopeo,
podman et al. One if the dependencies is `proglottis/gpgme` that
is using cgo to bind libgpgme, so we have to add the corresponding
devel package to the BuildRequires as well as installing it on CI.

Checks will follow later via an integration test.

vendor/github.com/containers/image/v5/pkg/compression/compression.go

@@ -0,0 +1,166 @@
+package compression
+
+import (
+	"bytes"
+	"compress/bzip2"
+	"fmt"
+	"io"
+
+	"github.com/containers/image/v5/pkg/compression/internal"
+	"github.com/containers/image/v5/pkg/compression/types"
+	"github.com/containers/storage/pkg/chunked/compressor"
+	"github.com/klauspost/pgzip"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/ulikunitz/xz"
+)
+
+// Algorithm is a compression algorithm that can be used for CompressStream.
+type Algorithm = types.Algorithm
+
+var (
+	// Gzip compression.
+	Gzip = internal.NewAlgorithm(types.GzipAlgorithmName, types.GzipAlgorithmName,
+		[]byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor)
+	// Bzip2 compression.
+	Bzip2 = internal.NewAlgorithm(types.Bzip2AlgorithmName, types.Bzip2AlgorithmName,
+		[]byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor)
+	// Xz compression.
+	Xz = internal.NewAlgorithm(types.XzAlgorithmName, types.XzAlgorithmName,
+		[]byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor)
+	// Zstd compression.
+	Zstd = internal.NewAlgorithm(types.ZstdAlgorithmName, types.ZstdAlgorithmName,
+		[]byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor)
+	// Zstd:chunked compression.
+	ZstdChunked = internal.NewAlgorithm(types.ZstdChunkedAlgorithmName, types.ZstdAlgorithmName, /* Note: InternalUnstableUndocumentedMIMEQuestionMark is not ZstdChunkedAlgorithmName */
+		nil, ZstdDecompressor, compressor.ZstdCompressor)
+
+	compressionAlgorithms = map[string]Algorithm{
+		Gzip.Name():        Gzip,
+		Bzip2.Name():       Bzip2,
+		Xz.Name():          Xz,
+		Zstd.Name():        Zstd,
+		ZstdChunked.Name(): ZstdChunked,
+	}
+)
+
+// AlgorithmByName returns the compressor by its name
+func AlgorithmByName(name string) (Algorithm, error) {
+	algorithm, ok := compressionAlgorithms[name]
+	if ok {
+		return algorithm, nil
+	}
+	return Algorithm{}, fmt.Errorf("cannot find compressor for %q", name)
+}
+
+// DecompressorFunc returns the decompressed stream, given a compressed stream.
+// The caller must call Close() on the decompressed stream (even if the compressed input stream does not need closing!).
+type DecompressorFunc = internal.DecompressorFunc
+
+// GzipDecompressor is a DecompressorFunc for the gzip compression algorithm.
+func GzipDecompressor(r io.Reader) (io.ReadCloser, error) {
+	return pgzip.NewReader(r)
+}
+
+// Bzip2Decompressor is a DecompressorFunc for the bzip2 compression algorithm.
+func Bzip2Decompressor(r io.Reader) (io.ReadCloser, error) {
+	return io.NopCloser(bzip2.NewReader(r)), nil
+}
+
+// XzDecompressor is a DecompressorFunc for the xz compression algorithm.
+func XzDecompressor(r io.Reader) (io.ReadCloser, error) {
+	r, err := xz.NewReader(r)
+	if err != nil {
+		return nil, err
+	}
+	return io.NopCloser(r), nil
+}
+
+// gzipCompressor is a CompressorFunc for the gzip compression algorithm.
+func gzipCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {
+	if level != nil {
+		return pgzip.NewWriterLevel(r, *level)
+	}
+	return pgzip.NewWriter(r), nil
+}
+
+// bzip2Compressor is a CompressorFunc for the bzip2 compression algorithm.
+func bzip2Compressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {
+	return nil, fmt.Errorf("bzip2 compression not supported")
+}
+
+// xzCompressor is a CompressorFunc for the xz compression algorithm.
+func xzCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {
+	return xz.NewWriter(r)
+}
+
+// CompressStream returns the compressor by its name
+func CompressStream(dest io.Writer, algo Algorithm, level *int) (io.WriteCloser, error) {
+	m := map[string]string{}
+	return internal.AlgorithmCompressor(algo)(dest, m, level)
+}
+
+// CompressStreamWithMetadata returns the compressor by its name.  If the compression
+// generates any metadata, it is written to the provided metadata map.
+func CompressStreamWithMetadata(dest io.Writer, metadata map[string]string, algo Algorithm, level *int) (io.WriteCloser, error) {
+	return internal.AlgorithmCompressor(algo)(dest, metadata, level)
+}
+
+// DetectCompressionFormat returns an Algorithm and DecompressorFunc if the input is recognized as a compressed format, an invalid
+// value and nil otherwise.
+// Because it consumes the start of input, other consumers must use the returned io.Reader instead to also read from the beginning.
+func DetectCompressionFormat(input io.Reader) (Algorithm, DecompressorFunc, io.Reader, error) {
+	buffer := [8]byte{}
+
+	n, err := io.ReadAtLeast(input, buffer[:], len(buffer))
+	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+		// This is a “real” error. We could just ignore it this time, process the data we have, and hope that the source will report the same error again.
+		// Instead, fail immediately with the original error cause instead of a possibly secondary/misleading error returned later.
+		return Algorithm{}, nil, nil, err
+	}
+
+	var retAlgo Algorithm
+	var decompressor DecompressorFunc
+	for _, algo := range compressionAlgorithms {
+		prefix := internal.AlgorithmPrefix(algo)
+		if len(prefix) > 0 && bytes.HasPrefix(buffer[:n], prefix) {
+			logrus.Debugf("Detected compression format %s", algo.Name())
+			retAlgo = algo
+			decompressor = internal.AlgorithmDecompressor(algo)
+			break
+		}
+	}
+	if decompressor == nil {
+		logrus.Debugf("No compression detected")
+	}
+
+	return retAlgo, decompressor, io.MultiReader(bytes.NewReader(buffer[:n]), input), nil
+}
+
+// DetectCompression returns a DecompressorFunc if the input is recognized as a compressed format, nil otherwise.
+// Because it consumes the start of input, other consumers must use the returned io.Reader instead to also read from the beginning.
+func DetectCompression(input io.Reader) (DecompressorFunc, io.Reader, error) {
+	_, d, r, e := DetectCompressionFormat(input)
+	return d, r, e
+}
+
+// AutoDecompress takes a stream and returns an uncompressed version of the
+// same stream.
+// The caller must call Close() on the returned stream (even if the input does not need,
+// or does not even support, closing!).
+func AutoDecompress(stream io.Reader) (io.ReadCloser, bool, error) {
+	decompressor, stream, err := DetectCompression(stream)
+	if err != nil {
+		return nil, false, errors.Wrapf(err, "detecting compression")
+	}
+	var res io.ReadCloser
+	if decompressor != nil {
+		res, err = decompressor(stream)
+		if err != nil {
+			return nil, false, errors.Wrapf(err, "initializing decompression")
+		}
+	} else {
+		res = io.NopCloser(stream)
+	}
+	return res, decompressor != nil, nil
+}

vendor/github.com/containers/image/v5/pkg/compression/internal/types.go

@@ -0,0 +1,65 @@
+package internal
+
+import "io"
+
+// CompressorFunc writes the compressed stream to the given writer using the specified compression level.
+// The caller must call Close() on the stream (even if the input stream does not need closing!).
+type CompressorFunc func(io.Writer, map[string]string, *int) (io.WriteCloser, error)
+
+// DecompressorFunc returns the decompressed stream, given a compressed stream.
+// The caller must call Close() on the decompressed stream (even if the compressed input stream does not need closing!).
+type DecompressorFunc func(io.Reader) (io.ReadCloser, error)
+
+// Algorithm is a compression algorithm that can be used for CompressStream.
+type Algorithm struct {
+	name         string
+	mime         string
+	prefix       []byte // Initial bytes of a stream compressed using this algorithm, or empty to disable detection.
+	decompressor DecompressorFunc
+	compressor   CompressorFunc
+}
+
+// NewAlgorithm creates an Algorithm instance.
+// This function exists so that Algorithm instances can only be created by code that
+// is allowed to import this internal subpackage.
+func NewAlgorithm(name, mime string, prefix []byte, decompressor DecompressorFunc, compressor CompressorFunc) Algorithm {
+	return Algorithm{
+		name:         name,
+		mime:         mime,
+		prefix:       prefix,
+		decompressor: decompressor,
+		compressor:   compressor,
+	}
+}
+
+// Name returns the name for the compression algorithm.
+func (c Algorithm) Name() string {
+	return c.name
+}
+
+// InternalUnstableUndocumentedMIMEQuestionMark ???
+// DO NOT USE THIS anywhere outside of c/image until it is properly documented.
+func (c Algorithm) InternalUnstableUndocumentedMIMEQuestionMark() string {
+	return c.mime
+}
+
+// AlgorithmCompressor returns the compressor field of algo.
+// This is a function instead of a public method so that it is only callable from by code
+// that is allowed to import this internal subpackage.
+func AlgorithmCompressor(algo Algorithm) CompressorFunc {
+	return algo.compressor
+}
+
+// AlgorithmDecompressor returns the decompressor field of algo.
+// This is a function instead of a public method so that it is only callable from by code
+// that is allowed to import this internal subpackage.
+func AlgorithmDecompressor(algo Algorithm) DecompressorFunc {
+	return algo.decompressor
+}
+
+// AlgorithmPrefix returns the prefix field of algo.
+// This is a function instead of a public method so that it is only callable from by code
+// that is allowed to import this internal subpackage.
+func AlgorithmPrefix(algo Algorithm) []byte {
+	return algo.prefix
+}

vendor/github.com/containers/image/v5/pkg/compression/types/types.go

@@ -0,0 +1,41 @@
+package types
+
+import (
+	"github.com/containers/image/v5/pkg/compression/internal"
+)
+
+// DecompressorFunc returns the decompressed stream, given a compressed stream.
+// The caller must call Close() on the decompressed stream (even if the compressed input stream does not need closing!).
+type DecompressorFunc = internal.DecompressorFunc
+
+// Algorithm is a compression algorithm provided and supported by pkg/compression.
+// It can’t be supplied from the outside.
+type Algorithm = internal.Algorithm
+
+const (
+	// GzipAlgorithmName is the name used by pkg/compression.Gzip.
+	// NOTE: Importing only this /types package does not inherently guarantee a Gzip algorithm
+	// will actually be available. (In fact it is intended for this types package not to depend
+	// on any of the implementations.)
+	GzipAlgorithmName = "gzip"
+	// Bzip2AlgorithmName is the name used by pkg/compression.Bzip2.
+	// NOTE: Importing only this /types package does not inherently guarantee a Bzip2 algorithm
+	// will actually be available. (In fact it is intended for this types package not to depend
+	// on any of the implementations.)
+	Bzip2AlgorithmName = "bzip2"
+	// XzAlgorithmName is the name used by pkg/compression.Xz.
+	// NOTE: Importing only this /types package does not inherently guarantee a Xz algorithm
+	// will actually be available. (In fact it is intended for this types package not to depend
+	// on any of the implementations.)
+	XzAlgorithmName = "Xz"
+	// ZstdAlgorithmName is the name used by pkg/compression.Zstd.
+	// NOTE: Importing only this /types package does not inherently guarantee a Zstd algorithm
+	// will actually be available. (In fact it is intended for this types package not to depend
+	// on any of the implementations.)
+	ZstdAlgorithmName = "zstd"
+	// ZstdChunkedAlgorithmName is the name used by pkg/compression.ZstdChunked.
+	// NOTE: Importing only this /types package does not inherently guarantee a ZstdChunked algorithm
+	// will actually be available. (In fact it is intended for this types package not to depend
+	// on any of the implementations.)
+	ZstdChunkedAlgorithmName = "zstd:chunked"
+)

vendor/github.com/containers/image/v5/pkg/compression/zstd.go

@@ -0,0 +1,59 @@
+package compression
+
+import (
+	"io"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+type wrapperZstdDecoder struct {
+	decoder *zstd.Decoder
+}
+
+func (w *wrapperZstdDecoder) Close() error {
+	w.decoder.Close()
+	return nil
+}
+
+func (w *wrapperZstdDecoder) DecodeAll(input, dst []byte) ([]byte, error) {
+	return w.decoder.DecodeAll(input, dst)
+}
+
+func (w *wrapperZstdDecoder) Read(p []byte) (int, error) {
+	return w.decoder.Read(p)
+}
+
+func (w *wrapperZstdDecoder) Reset(r io.Reader) error {
+	return w.decoder.Reset(r)
+}
+
+func (w *wrapperZstdDecoder) WriteTo(wr io.Writer) (int64, error) {
+	return w.decoder.WriteTo(wr)
+}
+
+func zstdReader(buf io.Reader) (io.ReadCloser, error) {
+	decoder, err := zstd.NewReader(buf)
+	return &wrapperZstdDecoder{decoder: decoder}, err
+}
+
+func zstdWriter(dest io.Writer) (io.WriteCloser, error) {
+	return zstd.NewWriter(dest)
+}
+
+func zstdWriterWithLevel(dest io.Writer, level int) (*zstd.Encoder, error) {
+	el := zstd.EncoderLevelFromZstd(level)
+	return zstd.NewWriter(dest, zstd.WithEncoderLevel(el))
+}
+
+// zstdCompressor is a CompressorFunc for the zstd compression algorithm.
+func zstdCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {
+	if level == nil {
+		return zstdWriter(r)
+	}
+	return zstdWriterWithLevel(r, *level)
+}
+
+// ZstdDecompressor is a DecompressorFunc for the zstd compression algorithm.
+func ZstdDecompressor(r io.Reader) (io.ReadCloser, error) {
+	return zstdReader(r)
+}