container: add support for uploading to registries

Add a new generic container registry client via a new `container`
package. Use this to create a command line utility as well as a
new upload target for container registries.

The code uses the github.com/containers/* project and packages to
interact with container registires that is also used by skopeo,
podman et al. One if the dependencies is `proglottis/gpgme` that
is using cgo to bind libgpgme, so we have to add the corresponding
devel package to the BuildRequires as well as installing it on CI.

Checks will follow later via an integration test.

vendor/github.com/docker/distribution/registry/api/v2/doc.go

@@ -0,0 +1,9 @@
+// Package v2 describes routes, urls and the error codes used in the Docker
+// Registry JSON HTTP API V2. In addition to declarations, descriptors are
+// provided for routes and error codes that can be used for implementation and
+// automatically generating documentation.
+//
+// Definitions here are considered to be locked down for the V2 registry api.
+// Any changes must be considered carefully and should not proceed without a
+// change proposal in docker core.
+package v2

vendor/github.com/docker/distribution/registry/api/v2/errors.go

@@ -0,0 +1,136 @@
+package v2
+
+import (
+	"net/http"
+
+	"github.com/docker/distribution/registry/api/errcode"
+)
+
+const errGroup = "registry.api.v2"
+
+var (
+	// ErrorCodeDigestInvalid is returned when uploading a blob if the
+	// provided digest does not match the blob contents.
+	ErrorCodeDigestInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:   "DIGEST_INVALID",
+		Message: "provided digest did not match uploaded content",
+		Description: `When a blob is uploaded, the registry will check that
+		the content matches the digest provided by the client. The error may
+		include a detail structure with the key "digest", including the
+		invalid digest string. This error may also be returned when a manifest
+		includes an invalid layer digest.`,
+		HTTPStatusCode: http.StatusBadRequest,
+	})
+
+	// ErrorCodeSizeInvalid is returned when uploading a blob if the provided
+	ErrorCodeSizeInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:   "SIZE_INVALID",
+		Message: "provided length did not match content length",
+		Description: `When a layer is uploaded, the provided size will be
+		checked against the uploaded content. If they do not match, this error
+		will be returned.`,
+		HTTPStatusCode: http.StatusBadRequest,
+	})
+
+	// ErrorCodeNameInvalid is returned when the name in the manifest does not
+	// match the provided name.
+	ErrorCodeNameInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:   "NAME_INVALID",
+		Message: "invalid repository name",
+		Description: `Invalid repository name encountered either during
+		manifest validation or any API operation.`,
+		HTTPStatusCode: http.StatusBadRequest,
+	})
+
+	// ErrorCodeTagInvalid is returned when the tag in the manifest does not
+	// match the provided tag.
+	ErrorCodeTagInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:   "TAG_INVALID",
+		Message: "manifest tag did not match URI",
+		Description: `During a manifest upload, if the tag in the manifest
+		does not match the uri tag, this error will be returned.`,
+		HTTPStatusCode: http.StatusBadRequest,
+	})
+
+	// ErrorCodeNameUnknown when the repository name is not known.
+	ErrorCodeNameUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:   "NAME_UNKNOWN",
+		Message: "repository name not known to registry",
+		Description: `This is returned if the name used during an operation is
+		unknown to the registry.`,
+		HTTPStatusCode: http.StatusNotFound,
+	})
+
+	// ErrorCodeManifestUnknown returned when image manifest is unknown.
+	ErrorCodeManifestUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:   "MANIFEST_UNKNOWN",
+		Message: "manifest unknown",
+		Description: `This error is returned when the manifest, identified by
+		name and tag is unknown to the repository.`,
+		HTTPStatusCode: http.StatusNotFound,
+	})
+
+	// ErrorCodeManifestInvalid returned when an image manifest is invalid,
+	// typically during a PUT operation. This error encompasses all errors
+	// encountered during manifest validation that aren't signature errors.
+	ErrorCodeManifestInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:   "MANIFEST_INVALID",
+		Message: "manifest invalid",
+		Description: `During upload, manifests undergo several checks ensuring
+		validity. If those checks fail, this error may be returned, unless a
+		more specific error is included. The detail will contain information
+		the failed validation.`,
+		HTTPStatusCode: http.StatusBadRequest,
+	})
+
+	// ErrorCodeManifestUnverified is returned when the manifest fails
+	// signature verification.
+	ErrorCodeManifestUnverified = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:   "MANIFEST_UNVERIFIED",
+		Message: "manifest failed signature verification",
+		Description: `During manifest upload, if the manifest fails signature
+		verification, this error will be returned.`,
+		HTTPStatusCode: http.StatusBadRequest,
+	})
+
+	// ErrorCodeManifestBlobUnknown is returned when a manifest blob is
+	// unknown to the registry.
+	ErrorCodeManifestBlobUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:   "MANIFEST_BLOB_UNKNOWN",
+		Message: "blob unknown to registry",
+		Description: `This error may be returned when a manifest blob is 
+		unknown to the registry.`,
+		HTTPStatusCode: http.StatusBadRequest,
+	})
+
+	// ErrorCodeBlobUnknown is returned when a blob is unknown to the
+	// registry. This can happen when the manifest references a nonexistent
+	// layer or the result is not found by a blob fetch.
+	ErrorCodeBlobUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:   "BLOB_UNKNOWN",
+		Message: "blob unknown to registry",
+		Description: `This error may be returned when a blob is unknown to the
+		registry in a specified repository. This can be returned with a
+		standard get or if a manifest references an unknown layer during
+		upload.`,
+		HTTPStatusCode: http.StatusNotFound,
+	})
+
+	// ErrorCodeBlobUploadUnknown is returned when an upload is unknown.
+	ErrorCodeBlobUploadUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:   "BLOB_UPLOAD_UNKNOWN",
+		Message: "blob upload unknown to registry",
+		Description: `If a blob upload has been cancelled or was never
+		started, this error code may be returned.`,
+		HTTPStatusCode: http.StatusNotFound,
+	})
+
+	// ErrorCodeBlobUploadInvalid is returned when an upload is invalid.
+	ErrorCodeBlobUploadInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:   "BLOB_UPLOAD_INVALID",
+		Message: "blob upload invalid",
+		Description: `The blob upload encountered an error and can no
+		longer proceed.`,
+		HTTPStatusCode: http.StatusNotFound,
+	})
+)

vendor/github.com/docker/distribution/registry/api/v2/headerparser.go

@@ -0,0 +1,161 @@
+package v2
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+var (
+	// according to rfc7230
+	reToken            = regexp.MustCompile(`^[^"(),/:;<=>?@[\]{}[:space:][:cntrl:]]+`)
+	reQuotedValue      = regexp.MustCompile(`^[^\\"]+`)
+	reEscapedCharacter = regexp.MustCompile(`^[[:blank:][:graph:]]`)
+)
+
+// parseForwardedHeader is a benevolent parser of Forwarded header defined in rfc7239. The header contains
+// a comma-separated list of forwarding key-value pairs. Each list element is set by single proxy. The
+// function parses only the first element of the list, which is set by the very first proxy. It returns a map
+// of corresponding key-value pairs and an unparsed slice of the input string.
+//
+// Examples of Forwarded header values:
+//
+//  1. Forwarded: For=192.0.2.43; Proto=https,For="[2001:db8:cafe::17]",For=unknown
+//  2. Forwarded: for="192.0.2.43:443"; host="registry.example.org", for="10.10.05.40:80"
+//
+// The first will be parsed into {"for": "192.0.2.43", "proto": "https"} while the second into
+// {"for": "192.0.2.43:443", "host": "registry.example.org"}.
+func parseForwardedHeader(forwarded string) (map[string]string, string, error) {
+	// Following are states of forwarded header parser. Any state could transition to a failure.
+	const (
+		// terminating state; can transition to Parameter
+		stateElement = iota
+		// terminating state; can transition to KeyValueDelimiter
+		stateParameter
+		// can transition to Value
+		stateKeyValueDelimiter
+		// can transition to one of { QuotedValue, PairEnd }
+		stateValue
+		// can transition to one of { EscapedCharacter, PairEnd }
+		stateQuotedValue
+		// can transition to one of { QuotedValue }
+		stateEscapedCharacter
+		// terminating state; can transition to one of { Parameter, Element }
+		statePairEnd
+	)
+
+	var (
+		parameter string
+		value     string
+		parse     = forwarded[:]
+		res       = map[string]string{}
+		state     = stateElement
+	)
+
+Loop:
+	for {
+		// skip spaces unless in quoted value
+		if state != stateQuotedValue && state != stateEscapedCharacter {
+			parse = strings.TrimLeftFunc(parse, unicode.IsSpace)
+		}
+
+		if len(parse) == 0 {
+			if state != stateElement && state != statePairEnd && state != stateParameter {
+				return nil, parse, fmt.Errorf("unexpected end of input")
+			}
+			// terminating
+			break
+		}
+
+		switch state {
+		// terminate at list element delimiter
+		case stateElement:
+			if parse[0] == ',' {
+				parse = parse[1:]
+				break Loop
+			}
+			state = stateParameter
+
+		// parse parameter (the key of key-value pair)
+		case stateParameter:
+			match := reToken.FindString(parse)
+			if len(match) == 0 {
+				return nil, parse, fmt.Errorf("failed to parse token at position %d", len(forwarded)-len(parse))
+			}
+			parameter = strings.ToLower(match)
+			parse = parse[len(match):]
+			state = stateKeyValueDelimiter
+
+		// parse '='
+		case stateKeyValueDelimiter:
+			if parse[0] != '=' {
+				return nil, parse, fmt.Errorf("expected '=', not '%c' at position %d", parse[0], len(forwarded)-len(parse))
+			}
+			parse = parse[1:]
+			state = stateValue
+
+		// parse value or quoted value
+		case stateValue:
+			if parse[0] == '"' {
+				parse = parse[1:]
+				state = stateQuotedValue
+			} else {
+				value = reToken.FindString(parse)
+				if len(value) == 0 {
+					return nil, parse, fmt.Errorf("failed to parse value at position %d", len(forwarded)-len(parse))
+				}
+				if _, exists := res[parameter]; exists {
+					return nil, parse, fmt.Errorf("duplicate parameter %q at position %d", parameter, len(forwarded)-len(parse))
+				}
+				res[parameter] = value
+				parse = parse[len(value):]
+				value = ""
+				state = statePairEnd
+			}
+
+		// parse a part of quoted value until the first backslash
+		case stateQuotedValue:
+			match := reQuotedValue.FindString(parse)
+			value += match
+			parse = parse[len(match):]
+			switch {
+			case len(parse) == 0:
+				return nil, parse, fmt.Errorf("unterminated quoted string")
+			case parse[0] == '"':
+				res[parameter] = value
+				value = ""
+				parse = parse[1:]
+				state = statePairEnd
+			case parse[0] == '\\':
+				parse = parse[1:]
+				state = stateEscapedCharacter
+			}
+
+		// parse escaped character in a quoted string, ignore the backslash
+		// transition back to QuotedValue state
+		case stateEscapedCharacter:
+			c := reEscapedCharacter.FindString(parse)
+			if len(c) == 0 {
+				return nil, parse, fmt.Errorf("invalid escape sequence at position %d", len(forwarded)-len(parse)-1)
+			}
+			value += c
+			parse = parse[1:]
+			state = stateQuotedValue
+
+		// expect either a new key-value pair, new list or end of input
+		case statePairEnd:
+			switch parse[0] {
+			case ';':
+				parse = parse[1:]
+				state = stateParameter
+			case ',':
+				state = stateElement
+			default:
+				return nil, parse, fmt.Errorf("expected ',' or ';', not %c at position %d", parse[0], len(forwarded)-len(parse))
+			}
+		}
+	}
+
+	return res, parse, nil
+}

vendor/github.com/docker/distribution/registry/api/v2/routes.go

@@ -0,0 +1,40 @@
+package v2
+
+import "github.com/gorilla/mux"
+
+// The following are definitions of the name under which all V2 routes are
+// registered. These symbols can be used to look up a route based on the name.
+const (
+	RouteNameBase            = "base"
+	RouteNameManifest        = "manifest"
+	RouteNameTags            = "tags"
+	RouteNameBlob            = "blob"
+	RouteNameBlobUpload      = "blob-upload"
+	RouteNameBlobUploadChunk = "blob-upload-chunk"
+	RouteNameCatalog         = "catalog"
+)
+
+// Router builds a gorilla router with named routes for the various API
+// methods. This can be used directly by both server implementations and
+// clients.
+func Router() *mux.Router {
+	return RouterWithPrefix("")
+}
+
+// RouterWithPrefix builds a gorilla router with a configured prefix
+// on all routes.
+func RouterWithPrefix(prefix string) *mux.Router {
+	rootRouter := mux.NewRouter()
+	router := rootRouter
+	if prefix != "" {
+		router = router.PathPrefix(prefix).Subrouter()
+	}
+
+	router.StrictSlash(true)
+
+	for _, descriptor := range routeDescriptors {
+		router.Path(descriptor.Path).Name(descriptor.Name)
+	}
+
+	return rootRouter
+}

vendor/github.com/docker/distribution/registry/api/v2/urls.go

@@ -0,0 +1,254 @@
+package v2
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/docker/distribution/reference"
+	"github.com/gorilla/mux"
+)
+
+// URLBuilder creates registry API urls from a single base endpoint. It can be
+// used to create urls for use in a registry client or server.
+//
+// All urls will be created from the given base, including the api version.
+// For example, if a root of "/foo/" is provided, urls generated will be fall
+// under "/foo/v2/...". Most application will only provide a schema, host and
+// port, such as "https://localhost:5000/".
+type URLBuilder struct {
+	root     *url.URL // url root (ie http://localhost/)
+	router   *mux.Router
+	relative bool
+}
+
+// NewURLBuilder creates a URLBuilder with provided root url object.
+func NewURLBuilder(root *url.URL, relative bool) *URLBuilder {
+	return &URLBuilder{
+		root:     root,
+		router:   Router(),
+		relative: relative,
+	}
+}
+
+// NewURLBuilderFromString workes identically to NewURLBuilder except it takes
+// a string argument for the root, returning an error if it is not a valid
+// url.
+func NewURLBuilderFromString(root string, relative bool) (*URLBuilder, error) {
+	u, err := url.Parse(root)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewURLBuilder(u, relative), nil
+}
+
+// NewURLBuilderFromRequest uses information from an *http.Request to
+// construct the root url.
+func NewURLBuilderFromRequest(r *http.Request, relative bool) *URLBuilder {
+	var (
+		scheme = "http"
+		host   = r.Host
+	)
+
+	if r.TLS != nil {
+		scheme = "https"
+	} else if len(r.URL.Scheme) > 0 {
+		scheme = r.URL.Scheme
+	}
+
+	// Handle fowarded headers
+	// Prefer "Forwarded" header as defined by rfc7239 if given
+	// see https://tools.ietf.org/html/rfc7239
+	if forwarded := r.Header.Get("Forwarded"); len(forwarded) > 0 {
+		forwardedHeader, _, err := parseForwardedHeader(forwarded)
+		if err == nil {
+			if fproto := forwardedHeader["proto"]; len(fproto) > 0 {
+				scheme = fproto
+			}
+			if fhost := forwardedHeader["host"]; len(fhost) > 0 {
+				host = fhost
+			}
+		}
+	} else {
+		if forwardedProto := r.Header.Get("X-Forwarded-Proto"); len(forwardedProto) > 0 {
+			scheme = forwardedProto
+		}
+		if forwardedHost := r.Header.Get("X-Forwarded-Host"); len(forwardedHost) > 0 {
+			// According to the Apache mod_proxy docs, X-Forwarded-Host can be a
+			// comma-separated list of hosts, to which each proxy appends the
+			// requested host. We want to grab the first from this comma-separated
+			// list.
+			hosts := strings.SplitN(forwardedHost, ",", 2)
+			host = strings.TrimSpace(hosts[0])
+		}
+	}
+
+	basePath := routeDescriptorsMap[RouteNameBase].Path
+
+	requestPath := r.URL.Path
+	index := strings.Index(requestPath, basePath)
+
+	u := &url.URL{
+		Scheme: scheme,
+		Host:   host,
+	}
+
+	if index > 0 {
+		// N.B. index+1 is important because we want to include the trailing /
+		u.Path = requestPath[0 : index+1]
+	}
+
+	return NewURLBuilder(u, relative)
+}
+
+// BuildBaseURL constructs a base url for the API, typically just "/v2/".
+func (ub *URLBuilder) BuildBaseURL() (string, error) {
+	route := ub.cloneRoute(RouteNameBase)
+
+	baseURL, err := route.URL()
+	if err != nil {
+		return "", err
+	}
+
+	return baseURL.String(), nil
+}
+
+// BuildCatalogURL constructs a url get a catalog of repositories
+func (ub *URLBuilder) BuildCatalogURL(values ...url.Values) (string, error) {
+	route := ub.cloneRoute(RouteNameCatalog)
+
+	catalogURL, err := route.URL()
+	if err != nil {
+		return "", err
+	}
+
+	return appendValuesURL(catalogURL, values...).String(), nil
+}
+
+// BuildTagsURL constructs a url to list the tags in the named repository.
+func (ub *URLBuilder) BuildTagsURL(name reference.Named) (string, error) {
+	route := ub.cloneRoute(RouteNameTags)
+
+	tagsURL, err := route.URL("name", name.Name())
+	if err != nil {
+		return "", err
+	}
+
+	return tagsURL.String(), nil
+}
+
+// BuildManifestURL constructs a url for the manifest identified by name and
+// reference. The argument reference may be either a tag or digest.
+func (ub *URLBuilder) BuildManifestURL(ref reference.Named) (string, error) {
+	route := ub.cloneRoute(RouteNameManifest)
+
+	tagOrDigest := ""
+	switch v := ref.(type) {
+	case reference.Tagged:
+		tagOrDigest = v.Tag()
+	case reference.Digested:
+		tagOrDigest = v.Digest().String()
+	default:
+		return "", fmt.Errorf("reference must have a tag or digest")
+	}
+
+	manifestURL, err := route.URL("name", ref.Name(), "reference", tagOrDigest)
+	if err != nil {
+		return "", err
+	}
+
+	return manifestURL.String(), nil
+}
+
+// BuildBlobURL constructs the url for the blob identified by name and dgst.
+func (ub *URLBuilder) BuildBlobURL(ref reference.Canonical) (string, error) {
+	route := ub.cloneRoute(RouteNameBlob)
+
+	layerURL, err := route.URL("name", ref.Name(), "digest", ref.Digest().String())
+	if err != nil {
+		return "", err
+	}
+
+	return layerURL.String(), nil
+}
+
+// BuildBlobUploadURL constructs a url to begin a blob upload in the
+// repository identified by name.
+func (ub *URLBuilder) BuildBlobUploadURL(name reference.Named, values ...url.Values) (string, error) {
+	route := ub.cloneRoute(RouteNameBlobUpload)
+
+	uploadURL, err := route.URL("name", name.Name())
+	if err != nil {
+		return "", err
+	}
+
+	return appendValuesURL(uploadURL, values...).String(), nil
+}
+
+// BuildBlobUploadChunkURL constructs a url for the upload identified by uuid,
+// including any url values. This should generally not be used by clients, as
+// this url is provided by server implementations during the blob upload
+// process.
+func (ub *URLBuilder) BuildBlobUploadChunkURL(name reference.Named, uuid string, values ...url.Values) (string, error) {
+	route := ub.cloneRoute(RouteNameBlobUploadChunk)
+
+	uploadURL, err := route.URL("name", name.Name(), "uuid", uuid)
+	if err != nil {
+		return "", err
+	}
+
+	return appendValuesURL(uploadURL, values...).String(), nil
+}
+
+// clondedRoute returns a clone of the named route from the router. Routes
+// must be cloned to avoid modifying them during url generation.
+func (ub *URLBuilder) cloneRoute(name string) clonedRoute {
+	route := new(mux.Route)
+	root := new(url.URL)
+
+	*route = *ub.router.GetRoute(name) // clone the route
+	*root = *ub.root
+
+	return clonedRoute{Route: route, root: root, relative: ub.relative}
+}
+
+type clonedRoute struct {
+	*mux.Route
+	root     *url.URL
+	relative bool
+}
+
+func (cr clonedRoute) URL(pairs ...string) (*url.URL, error) {
+	routeURL, err := cr.Route.URL(pairs...)
+	if err != nil {
+		return nil, err
+	}
+
+	if cr.relative {
+		return routeURL, nil
+	}
+
+	if routeURL.Scheme == "" && routeURL.User == nil && routeURL.Host == "" {
+		routeURL.Path = routeURL.Path[1:]
+	}
+
+	url := cr.root.ResolveReference(routeURL)
+	url.Scheme = cr.root.Scheme
+	return url, nil
+}
+
+// appendValuesURL appends the parameters to the url.
+func appendValuesURL(u *url.URL, values ...url.Values) *url.URL {
+	merged := u.Query()
+
+	for _, v := range values {
+		for k, vv := range v {
+			merged[k] = append(merged[k], vv...)
+		}
+	}
+
+	u.RawQuery = merged.Encode()
+	return u
+}