Port osbuild/images v0.33.0 with dot-notation to composer

Update the osbuild/images to the version which introduces "dot notation"
for distro release versions.

 - Replace all uses of distroregistry by distrofactory.
 - Delete local version of reporegistry and use the one from the
   osbuild/images.
 - Weldr: unify `createWeldrAPI()` and `createWeldrAPI2()` into a single
   `createTestWeldrAPI()` function`.
 - store/fixture: rework fixtures to allow overriding the host distro
   name and host architecture name. A cleanup function to restore the
   host distro and arch names is always part of the fixture struct.
 - Delete `distro_mock` package, since it is no longer used.
 - Bump the required version of osbuild to 98, because the OSCAP
   customization is using the 'compress_results' stage option, which is
   not available in older versions of osbuild.

Signed-off-by: Tomáš Hozza <thozza@redhat.com>
This commit is contained in:
Tomáš Hozza 2024-01-08 17:58:49 +01:00 committed by Achilleas Koutsou
parent f6ff8c40dd
commit 625b1578fa
1166 changed files with 154457 additions and 5508 deletions

View file

@ -0,0 +1,284 @@
package directory
import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"github.com/containers/image/v5/internal/imagedestination/impl"
"github.com/containers/image/v5/internal/imagedestination/stubs"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/internal/putblobdigest"
"github.com/containers/image/v5/internal/signature"
"github.com/containers/image/v5/types"
"github.com/opencontainers/go-digest"
"github.com/sirupsen/logrus"
)
const version = "Directory Transport Version: 1.1\n"
// ErrNotContainerImageDir indicates that the directory doesn't match the expected contents of a directory created
// using the 'dir' transport
var ErrNotContainerImageDir = errors.New("not a containers image directory, don't want to overwrite important data")
type dirImageDestination struct {
impl.Compat
impl.PropertyMethodsInitialize
stubs.NoPutBlobPartialInitialize
stubs.AlwaysSupportsSignatures
ref dirReference
}
// newImageDestination returns an ImageDestination for writing to a directory.
func newImageDestination(sys *types.SystemContext, ref dirReference) (private.ImageDestination, error) {
desiredLayerCompression := types.PreserveOriginal
if sys != nil {
if sys.DirForceCompress {
desiredLayerCompression = types.Compress
if sys.DirForceDecompress {
return nil, fmt.Errorf("Cannot compress and decompress at the same time")
}
}
if sys.DirForceDecompress {
desiredLayerCompression = types.Decompress
}
}
// If directory exists check if it is empty
// if not empty, check whether the contents match that of a container image directory and overwrite the contents
// if the contents don't match throw an error
dirExists, err := pathExists(ref.resolvedPath)
if err != nil {
return nil, fmt.Errorf("checking for path %q: %w", ref.resolvedPath, err)
}
if dirExists {
isEmpty, err := isDirEmpty(ref.resolvedPath)
if err != nil {
return nil, err
}
if !isEmpty {
versionExists, err := pathExists(ref.versionPath())
if err != nil {
return nil, fmt.Errorf("checking if path exists %q: %w", ref.versionPath(), err)
}
if versionExists {
contents, err := os.ReadFile(ref.versionPath())
if err != nil {
return nil, err
}
// check if contents of version file is what we expect it to be
if string(contents) != version {
return nil, ErrNotContainerImageDir
}
} else {
return nil, ErrNotContainerImageDir
}
// delete directory contents so that only one image is in the directory at a time
if err = removeDirContents(ref.resolvedPath); err != nil {
return nil, fmt.Errorf("erasing contents in %q: %w", ref.resolvedPath, err)
}
logrus.Debugf("overwriting existing container image directory %q", ref.resolvedPath)
}
} else {
// create directory if it doesn't exist
if err := os.MkdirAll(ref.resolvedPath, 0755); err != nil {
return nil, fmt.Errorf("unable to create directory %q: %w", ref.resolvedPath, err)
}
}
// create version file
err = os.WriteFile(ref.versionPath(), []byte(version), 0644)
if err != nil {
return nil, fmt.Errorf("creating version file %q: %w", ref.versionPath(), err)
}
d := &dirImageDestination{
PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{
SupportedManifestMIMETypes: nil,
DesiredLayerCompression: desiredLayerCompression,
AcceptsForeignLayerURLs: false,
MustMatchRuntimeOS: false,
IgnoresEmbeddedDockerReference: false, // N/A, DockerReference() returns nil.
HasThreadSafePutBlob: true,
}),
NoPutBlobPartialInitialize: stubs.NoPutBlobPartial(ref),
ref: ref,
}
d.Compat = impl.AddCompat(d)
return d, nil
}
// Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent,
// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects.
func (d *dirImageDestination) Reference() types.ImageReference {
return d.ref
}
// Close removes resources associated with an initialized ImageDestination, if any.
func (d *dirImageDestination) Close() error {
return nil
}
// PutBlobWithOptions writes contents of stream and returns data representing the result.
// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents.
// inputInfo.Size is the expected length of stream, if known.
// inputInfo.MediaType describes the blob format, if known.
// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
// to any other readers for download using the supplied digest.
// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlobWithOptions MUST 1) fail, and 2) delete any data stored so far.
func (d *dirImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options private.PutBlobOptions) (private.UploadedBlob, error) {
blobFile, err := os.CreateTemp(d.ref.path, "dir-put-blob")
if err != nil {
return private.UploadedBlob{}, err
}
succeeded := false
explicitClosed := false
defer func() {
if !explicitClosed {
blobFile.Close()
}
if !succeeded {
os.Remove(blobFile.Name())
}
}()
digester, stream := putblobdigest.DigestIfCanonicalUnknown(stream, inputInfo)
// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
size, err := io.Copy(blobFile, stream)
if err != nil {
return private.UploadedBlob{}, err
}
blobDigest := digester.Digest()
if inputInfo.Size != -1 && size != inputInfo.Size {
return private.UploadedBlob{}, fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", blobDigest, inputInfo.Size, size)
}
if err := blobFile.Sync(); err != nil {
return private.UploadedBlob{}, err
}
// On POSIX systems, blobFile was created with mode 0600, so we need to make it readable.
// On Windows, the “permissions of newly created files” argument to syscall.Open is
// ignored and the file is already readable; besides, blobFile.Chmod, i.e. syscall.Fchmod,
// always fails on Windows.
if runtime.GOOS != "windows" {
if err := blobFile.Chmod(0644); err != nil {
return private.UploadedBlob{}, err
}
}
blobPath := d.ref.layerPath(blobDigest)
// need to explicitly close the file, since a rename won't otherwise not work on Windows
blobFile.Close()
explicitClosed = true
if err := os.Rename(blobFile.Name(), blobPath); err != nil {
return private.UploadedBlob{}, err
}
succeeded = true
return private.UploadedBlob{Digest: blobDigest, Size: size}, nil
}
// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree).
// info.Digest must not be empty.
// If the blob has been successfully reused, returns (true, info, nil).
// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
func (d *dirImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) {
if !impl.OriginalBlobMatchesRequiredCompression(options) {
return false, private.ReusedBlob{}, nil
}
if info.Digest == "" {
return false, private.ReusedBlob{}, fmt.Errorf("Can not check for a blob with unknown digest")
}
blobPath := d.ref.layerPath(info.Digest)
finfo, err := os.Stat(blobPath)
if err != nil && os.IsNotExist(err) {
return false, private.ReusedBlob{}, nil
}
if err != nil {
return false, private.ReusedBlob{}, err
}
return true, private.ReusedBlob{Digest: info.Digest, Size: finfo.Size()}, nil
}
// PutManifest writes manifest to the destination.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write the manifest for (when
// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list.
// It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated
// by `manifest.Digest()`.
// FIXME? This should also receive a MIME type if known, to differentiate between schema versions.
// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema),
// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
func (d *dirImageDestination) PutManifest(ctx context.Context, manifest []byte, instanceDigest *digest.Digest) error {
return os.WriteFile(d.ref.manifestPath(instanceDigest), manifest, 0644)
}
// PutSignaturesWithFormat writes a set of signatures to the destination.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for
// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list.
// MUST be called after PutManifest (signatures may reference manifest contents).
func (d *dirImageDestination) PutSignaturesWithFormat(ctx context.Context, signatures []signature.Signature, instanceDigest *digest.Digest) error {
for i, sig := range signatures {
blob, err := signature.Blob(sig)
if err != nil {
return err
}
if err := os.WriteFile(d.ref.signaturePath(i, instanceDigest), blob, 0644); err != nil {
return err
}
}
return nil
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
func (d *dirImageDestination) Commit(context.Context, types.UnparsedImage) error {
return nil
}
// returns true if path exists
func pathExists(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
// returns true if directory is empty
func isDirEmpty(path string) (bool, error) {
files, err := os.ReadDir(path)
if err != nil {
return false, err
}
return len(files) == 0, nil
}
// deletes the contents of a directory
func removeDirContents(path string) error {
files, err := os.ReadDir(path)
if err != nil {
return err
}
for _, file := range files {
if err := os.RemoveAll(filepath.Join(path, file.Name())); err != nil {
return err
}
}
return nil
}

View file

@ -0,0 +1,102 @@
package directory
import (
"context"
"fmt"
"io"
"os"
"github.com/containers/image/v5/internal/imagesource/impl"
"github.com/containers/image/v5/internal/imagesource/stubs"
"github.com/containers/image/v5/internal/manifest"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/internal/signature"
"github.com/containers/image/v5/types"
"github.com/opencontainers/go-digest"
)
type dirImageSource struct {
impl.Compat
impl.PropertyMethodsInitialize
impl.DoesNotAffectLayerInfosForCopy
stubs.NoGetBlobAtInitialize
ref dirReference
}
// newImageSource returns an ImageSource reading from an existing directory.
// The caller must call .Close() on the returned ImageSource.
func newImageSource(ref dirReference) private.ImageSource {
s := &dirImageSource{
PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{
HasThreadSafeGetBlob: false,
}),
NoGetBlobAtInitialize: stubs.NoGetBlobAt(ref),
ref: ref,
}
s.Compat = impl.AddCompat(s)
return s
}
// Reference returns the reference used to set up this source, _as specified by the user_
// (not as the image itself, or its underlying storage, claims). This can be used e.g. to determine which public keys are trusted for this image.
func (s *dirImageSource) Reference() types.ImageReference {
return s.ref
}
// Close removes resources associated with an initialized ImageSource, if any.
func (s *dirImageSource) Close() error {
return nil
}
// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
// It may use a remote (= slow) service.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list);
// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists).
func (s *dirImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
m, err := os.ReadFile(s.ref.manifestPath(instanceDigest))
if err != nil {
return nil, "", err
}
return m, manifest.GuessMIMEType(m), err
}
// GetBlob returns a stream for the specified blob, and the blobs size (or -1 if unknown).
// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
func (s *dirImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
r, err := os.Open(s.ref.layerPath(info.Digest))
if err != nil {
return nil, -1, err
}
fi, err := r.Stat()
if err != nil {
return nil, -1, err
}
return r, fi.Size(), nil
}
// GetSignaturesWithFormat returns the image's signatures. It may use a remote (= slow) service.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for
// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list
// (e.g. if the source never returns manifest lists).
func (s *dirImageSource) GetSignaturesWithFormat(ctx context.Context, instanceDigest *digest.Digest) ([]signature.Signature, error) {
signatures := []signature.Signature{}
for i := 0; ; i++ {
path := s.ref.signaturePath(i, instanceDigest)
sigBlob, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
break
}
return nil, err
}
signature, err := signature.FromBlob(sigBlob)
if err != nil {
return nil, fmt.Errorf("parsing signature %q: %w", path, err)
}
signatures = append(signatures, signature)
}
return signatures, nil
}

View file

@ -0,0 +1,188 @@
package directory
import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"github.com/containers/image/v5/directory/explicitfilepath"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/image"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
"github.com/opencontainers/go-digest"
)
func init() {
transports.Register(Transport)
}
// Transport is an ImageTransport for directory paths.
var Transport = dirTransport{}
type dirTransport struct{}
func (t dirTransport) Name() string {
return "dir"
}
// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an ImageReference.
func (t dirTransport) ParseReference(reference string) (types.ImageReference, error) {
return NewReference(reference)
}
// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys
// (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value).
// It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion.
// scope passed to this function will not be "", that value is always allowed.
func (t dirTransport) ValidatePolicyConfigurationScope(scope string) error {
if !strings.HasPrefix(scope, "/") {
return fmt.Errorf("Invalid scope %s: Must be an absolute path", scope)
}
// Refuse also "/", otherwise "/" and "" would have the same semantics,
// and "" could be unexpectedly shadowed by the "/" entry.
if scope == "/" {
return errors.New(`Invalid scope "/": Use the generic default scope ""`)
}
cleaned := filepath.Clean(scope)
if cleaned != scope {
return fmt.Errorf(`Invalid scope %s: Uses non-canonical format, perhaps try %s`, scope, cleaned)
}
return nil
}
// dirReference is an ImageReference for directory paths.
type dirReference struct {
// Note that the interpretation of paths below depends on the underlying filesystem state, which may change under us at any time!
// Either of the paths may point to a different, or no, inode over time. resolvedPath may contain symbolic links, and so on.
// Generally we follow the intent of the user, and use the "path" member for filesystem operations (e.g. the user can use a relative path to avoid
// being exposed to symlinks and renames in the parent directories to the working directory).
// (But in general, we make no attempt to be completely safe against concurrent hostile filesystem modifications.)
path string // As specified by the user. May be relative, contain symlinks, etc.
resolvedPath string // Absolute path with no symlinks, at least at the time of its creation. Primarily used for policy namespaces.
}
// There is no directory.ParseReference because it is rather pointless.
// Callers who need a transport-independent interface will go through
// dirTransport.ParseReference; callers who intentionally deal with directories
// can use directory.NewReference.
// NewReference returns a directory reference for a specified path.
//
// We do not expose an API supplying the resolvedPath; we could, but recomputing it
// is generally cheap enough that we prefer being confident about the properties of resolvedPath.
func NewReference(path string) (types.ImageReference, error) {
resolved, err := explicitfilepath.ResolvePathToFullyExplicit(path)
if err != nil {
return nil, err
}
return dirReference{path: path, resolvedPath: resolved}, nil
}
func (ref dirReference) Transport() types.ImageTransport {
return Transport
}
// StringWithinTransport returns a string representation of the reference, which MUST be such that
// reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference.
// NOTE: The returned string is not promised to be equal to the original input to ParseReference;
// e.g. default attribute values omitted by the user may be filled in the return value, or vice versa.
// WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix.
func (ref dirReference) StringWithinTransport() string {
return ref.path
}
// DockerReference returns a Docker reference associated with this reference
// (fully explicit, i.e. !reference.IsNameOnly, but reflecting user intent,
// not e.g. after redirect or alias processing), or nil if unknown/not applicable.
func (ref dirReference) DockerReference() reference.Named {
return nil
}
// PolicyConfigurationIdentity returns a string representation of the reference, suitable for policy lookup.
// This MUST reflect user intent, not e.g. after processing of third-party redirects or aliases;
// The value SHOULD be fully explicit about its semantics, with no hidden defaults, AND canonical
// (i.e. various references with exactly the same semantics should return the same configuration identity)
// It is fine for the return value to be equal to StringWithinTransport(), and it is desirable but
// not required/guaranteed that it will be a valid input to Transport().ParseReference().
// Returns "" if configuration identities for these references are not supported.
func (ref dirReference) PolicyConfigurationIdentity() string {
return ref.resolvedPath
}
// PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search
// for if explicit configuration for PolicyConfigurationIdentity() is not set. The list will be processed
// in order, terminating on first match, and an implicit "" is always checked at the end.
// It is STRONGLY recommended for the first element, if any, to be a prefix of PolicyConfigurationIdentity(),
// and each following element to be a prefix of the element preceding it.
func (ref dirReference) PolicyConfigurationNamespaces() []string {
res := []string{}
path := ref.resolvedPath
for {
lastSlash := strings.LastIndex(path, "/")
if lastSlash == -1 || lastSlash == 0 {
break
}
path = path[:lastSlash]
res = append(res, path)
}
// Note that we do not include "/"; it is redundant with the default "" global default,
// and rejected by dirTransport.ValidatePolicyConfigurationScope above.
return res
}
// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport.
// The caller must call .Close() on the returned ImageCloser.
// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource,
// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details.
func (ref dirReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) {
return image.FromReference(ctx, sys, ref)
}
// NewImageSource returns a types.ImageSource for this reference.
// The caller must call .Close() on the returned ImageSource.
func (ref dirReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) {
return newImageSource(ref), nil
}
// NewImageDestination returns a types.ImageDestination for this reference.
// The caller must call .Close() on the returned ImageDestination.
func (ref dirReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) {
return newImageDestination(sys, ref)
}
// DeleteImage deletes the named image from the registry, if supported.
func (ref dirReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error {
return errors.New("Deleting images not implemented for dir: images")
}
// manifestPath returns a path for the manifest within a directory using our conventions.
func (ref dirReference) manifestPath(instanceDigest *digest.Digest) string {
if instanceDigest != nil {
return filepath.Join(ref.path, instanceDigest.Encoded()+".manifest.json")
}
return filepath.Join(ref.path, "manifest.json")
}
// layerPath returns a path for a layer tarball within a directory using our conventions.
func (ref dirReference) layerPath(digest digest.Digest) string {
// FIXME: Should we keep the digest identification?
return filepath.Join(ref.path, digest.Encoded())
}
// signaturePath returns a path for a signature within a directory using our conventions.
func (ref dirReference) signaturePath(index int, instanceDigest *digest.Digest) string {
if instanceDigest != nil {
return filepath.Join(ref.path, fmt.Sprintf(instanceDigest.Encoded()+".signature-%d", index+1))
}
return filepath.Join(ref.path, fmt.Sprintf("signature-%d", index+1))
}
// versionPath returns a path for the version file within a directory using our conventions.
func (ref dirReference) versionPath() string {
return filepath.Join(ref.path, "version")
}

View file

@ -0,0 +1,101 @@
package daemon
import (
"net/http"
"path/filepath"
"github.com/containers/image/v5/types"
dockerclient "github.com/docker/docker/client"
"github.com/docker/go-connections/tlsconfig"
)
const (
// The default API version to be used in case none is explicitly specified
defaultAPIVersion = "1.22"
)
// NewDockerClient initializes a new API client based on the passed SystemContext.
func newDockerClient(sys *types.SystemContext) (*dockerclient.Client, error) {
host := dockerclient.DefaultDockerHost
if sys != nil && sys.DockerDaemonHost != "" {
host = sys.DockerDaemonHost
}
opts := []dockerclient.Opt{
dockerclient.WithHost(host),
dockerclient.WithVersion(defaultAPIVersion),
}
// We conditionalize building the TLS configuration only to TLS sockets:
//
// The dockerclient.Client implementation differentiates between
// - Client.proto, which is ~how the connection is establishe (IP / AF_UNIX/Windows)
// - Client.scheme, which is what is sent over the connection (HTTP with/without TLS).
//
// Only Client.proto is set from the URL in dockerclient.WithHost(),
// Client.scheme is detected based on a http.Client.TLSClientConfig presence;
// dockerclient.WithHTTPClient with a client that has TLSClientConfig set
// will, by default, trigger an attempt to use TLS.
//
// So, dont use WithHTTPClient for unix:// sockets at all.
//
// Similarly, if we want to communicate over plain HTTP on a TCP socket (http://),
// we also should not set TLSClientConfig. We continue to use WithHTTPClient
// with our slightly non-default settings to avoid a behavior change on updates of c/image.
//
// Alternatively we could use dockerclient.WithScheme to drive the TLS/non-TLS logic
// explicitly, but we would still want to set WithHTTPClient (differently) for https:// and http:// ;
// so that would not be any simpler.
serverURL, err := dockerclient.ParseHostURL(host)
if err != nil {
return nil, err
}
switch serverURL.Scheme {
case "unix": // Nothing
case "http":
hc := httpConfig()
opts = append(opts, dockerclient.WithHTTPClient(hc))
default:
hc, err := tlsConfig(sys)
if err != nil {
return nil, err
}
opts = append(opts, dockerclient.WithHTTPClient(hc))
}
return dockerclient.NewClientWithOpts(opts...)
}
func tlsConfig(sys *types.SystemContext) (*http.Client, error) {
options := tlsconfig.Options{}
if sys != nil && sys.DockerDaemonInsecureSkipTLSVerify {
options.InsecureSkipVerify = true
}
if sys != nil && sys.DockerDaemonCertPath != "" {
options.CAFile = filepath.Join(sys.DockerDaemonCertPath, "ca.pem")
options.CertFile = filepath.Join(sys.DockerDaemonCertPath, "cert.pem")
options.KeyFile = filepath.Join(sys.DockerDaemonCertPath, "key.pem")
}
tlsc, err := tlsconfig.Client(options)
if err != nil {
return nil, err
}
return &http.Client{
Transport: &http.Transport{
TLSClientConfig: tlsc,
},
CheckRedirect: dockerclient.CheckRedirect,
}, nil
}
func httpConfig() *http.Client {
return &http.Client{
Transport: &http.Transport{
TLSClientConfig: nil,
},
CheckRedirect: dockerclient.CheckRedirect,
}
}

View file

@ -0,0 +1,186 @@
package daemon
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"github.com/containers/image/v5/docker/internal/tarfile"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/types"
"github.com/docker/docker/client"
"github.com/sirupsen/logrus"
)
type daemonImageDestination struct {
ref daemonReference
mustMatchRuntimeOS bool
*tarfile.Destination // Implements most of types.ImageDestination
archive *tarfile.Writer
// For talking to imageLoadGoroutine
goroutineCancel context.CancelFunc
statusChannel <-chan error
writer *io.PipeWriter
// Other state
committed bool // writer has been closed
}
// newImageDestination returns a types.ImageDestination for the specified image reference.
func newImageDestination(ctx context.Context, sys *types.SystemContext, ref daemonReference) (private.ImageDestination, error) {
if ref.ref == nil {
return nil, fmt.Errorf("Invalid destination docker-daemon:%s: a destination must be a name:tag", ref.StringWithinTransport())
}
namedTaggedRef, ok := ref.ref.(reference.NamedTagged)
if !ok {
return nil, fmt.Errorf("Invalid destination docker-daemon:%s: a destination must be a name:tag", ref.StringWithinTransport())
}
var mustMatchRuntimeOS = true
if sys != nil && sys.DockerDaemonHost != client.DefaultDockerHost {
mustMatchRuntimeOS = false
}
c, err := newDockerClient(sys)
if err != nil {
return nil, fmt.Errorf("initializing docker engine client: %w", err)
}
reader, writer := io.Pipe()
archive := tarfile.NewWriter(writer)
// Commit() may never be called, so we may never read from this channel; so, make this buffered to allow imageLoadGoroutine to write status and terminate even if we never read it.
statusChannel := make(chan error, 1)
goroutineContext, goroutineCancel := context.WithCancel(ctx)
go imageLoadGoroutine(goroutineContext, c, reader, statusChannel)
return &daemonImageDestination{
ref: ref,
mustMatchRuntimeOS: mustMatchRuntimeOS,
Destination: tarfile.NewDestination(sys, archive, ref.Transport().Name(), namedTaggedRef),
archive: archive,
goroutineCancel: goroutineCancel,
statusChannel: statusChannel,
writer: writer,
committed: false,
}, nil
}
// imageLoadGoroutine accepts tar stream on reader, sends it to c, and reports error or success by writing to statusChannel
func imageLoadGoroutine(ctx context.Context, c *client.Client, reader *io.PipeReader, statusChannel chan<- error) {
defer c.Close()
err := errors.New("Internal error: unexpected panic in imageLoadGoroutine")
defer func() {
logrus.Debugf("docker-daemon: sending done, status %v", err)
statusChannel <- err
}()
defer func() {
if err == nil {
reader.Close()
} else {
if err := reader.CloseWithError(err); err != nil {
logrus.Debugf("imageLoadGoroutine: Error during reader.CloseWithError: %v", err)
}
}
}()
err = imageLoad(ctx, c, reader)
}
// imageLoad accepts tar stream on reader and sends it to c
func imageLoad(ctx context.Context, c *client.Client, reader *io.PipeReader) error {
resp, err := c.ImageLoad(ctx, reader, true)
if err != nil {
return fmt.Errorf("starting a load operation in docker engine: %w", err)
}
defer resp.Body.Close()
// jsonError and jsonMessage are small subsets of docker/docker/pkg/jsonmessage.JSONError and JSONMessage,
// copied here to minimize dependencies.
type jsonError struct {
Message string `json:"message,omitempty"`
}
type jsonMessage struct {
Error *jsonError `json:"errorDetail,omitempty"`
}
dec := json.NewDecoder(resp.Body)
for {
var msg jsonMessage
if err := dec.Decode(&msg); err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("parsing docker load progress: %w", err)
}
if msg.Error != nil {
return fmt.Errorf("docker engine reported: %s", msg.Error.Message)
}
}
return nil // No error reported = success
}
// DesiredLayerCompression indicates if layers must be compressed, decompressed or preserved
func (d *daemonImageDestination) DesiredLayerCompression() types.LayerCompression {
return types.PreserveOriginal
}
// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise.
func (d *daemonImageDestination) MustMatchRuntimeOS() bool {
return d.mustMatchRuntimeOS
}
// Close removes resources associated with an initialized ImageDestination, if any.
func (d *daemonImageDestination) Close() error {
if !d.committed {
logrus.Debugf("docker-daemon: Closing tar stream to abort loading")
// In principle, goroutineCancel() should abort the HTTP request and stop the process from continuing.
// In practice, though, various HTTP implementations used by client.Client.ImageLoad() (including
// https://github.com/golang/net/blob/master/context/ctxhttp/ctxhttp_pre17.go and the
// net/http version with native Context support in Go 1.7) do not always actually immediately cancel
// the operation: they may process the HTTP request, or a part of it, to completion in a goroutine, and
// return early if the context is canceled without terminating the goroutine at all.
// So we need this CloseWithError to terminate sending the HTTP request Body
// immediately, and hopefully, through terminating the sending which uses "Transfer-Encoding: chunked"" without sending
// the terminating zero-length chunk, prevent the docker daemon from processing the tar stream at all.
// Whether that works or not, closing the PipeWriter seems desirable in any case.
if err := d.writer.CloseWithError(errors.New("Aborting upload, daemonImageDestination closed without a previous .Commit()")); err != nil {
return err
}
}
d.goroutineCancel()
return nil
}
func (d *daemonImageDestination) Reference() types.ImageReference {
return d.ref
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
func (d *daemonImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
logrus.Debugf("docker-daemon: Closing tar stream")
if err := d.archive.Close(); err != nil {
return err
}
if err := d.writer.Close(); err != nil {
return err
}
d.committed = true // We may still fail, but we are done sending to imageLoadGoroutine.
logrus.Debugf("docker-daemon: Waiting for status")
select {
case <-ctx.Done():
return ctx.Err()
case err := <-d.statusChannel:
return err
}
}

View file

@ -0,0 +1,56 @@
package daemon
import (
"context"
"fmt"
"github.com/containers/image/v5/docker/internal/tarfile"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/types"
)
type daemonImageSource struct {
ref daemonReference
*tarfile.Source // Implements most of types.ImageSource
}
// newImageSource returns a types.ImageSource for the specified image reference.
// The caller must call .Close() on the returned ImageSource.
//
// It would be great if we were able to stream the input tar as it is being
// sent; but Docker sends the top-level manifest, which determines which paths
// to look for, at the end, so in we will need to seek back and re-read, several times.
// (We could, perhaps, expect an exact sequence, assume that the first plaintext file
// is the config, and that the following len(RootFS) files are the layers, but that feels
// way too brittle.)
func newImageSource(ctx context.Context, sys *types.SystemContext, ref daemonReference) (private.ImageSource, error) {
c, err := newDockerClient(sys)
if err != nil {
return nil, fmt.Errorf("initializing docker engine client: %w", err)
}
defer c.Close()
// Per NewReference(), ref.StringWithinTransport() is either an image ID (config digest), or a !reference.NameOnly() reference.
// Either way ImageSave should create a tarball with exactly one image.
inputStream, err := c.ImageSave(ctx, []string{ref.StringWithinTransport()})
if err != nil {
return nil, fmt.Errorf("loading image from docker engine: %w", err)
}
defer inputStream.Close()
archive, err := tarfile.NewReaderFromStream(sys, inputStream)
if err != nil {
return nil, err
}
src := tarfile.NewSource(archive, true, ref.Transport().Name(), nil, -1)
return &daemonImageSource{
ref: ref,
Source: src,
}, nil
}
// Reference returns the reference used to set up this source, _as specified by the user_
// (not as the image itself, or its underlying storage, claims). This can be used e.g. to determine which public keys are trusted for this image.
func (s *daemonImageSource) Reference() types.ImageReference {
return s.ref
}

View file

@ -0,0 +1,219 @@
package daemon
import (
"context"
"errors"
"fmt"
"github.com/containers/image/v5/docker/policyconfiguration"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/image"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
"github.com/opencontainers/go-digest"
)
func init() {
transports.Register(Transport)
}
// Transport is an ImageTransport for images managed by a local Docker daemon.
var Transport = daemonTransport{}
type daemonTransport struct{}
// Name returns the name of the transport, which must be unique among other transports.
func (t daemonTransport) Name() string {
return "docker-daemon"
}
// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an ImageReference.
func (t daemonTransport) ParseReference(reference string) (types.ImageReference, error) {
return ParseReference(reference)
}
// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys
// (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value).
// It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion.
// scope passed to this function will not be "", that value is always allowed.
func (t daemonTransport) ValidatePolicyConfigurationScope(scope string) error {
// ID values cannot be effectively namespaced, and are clearly invalid host:port values.
if _, err := digest.Parse(scope); err == nil {
return fmt.Errorf(`docker-daemon: can not use algo:digest value %s as a namespace`, scope)
}
// FIXME? We could be verifying the various character set and length restrictions
// from docker/distribution/reference.regexp.go, but other than that there
// are few semantically invalid strings.
return nil
}
// daemonReference is an ImageReference for images managed by a local Docker daemon
// Exactly one of id and ref can be set.
// For daemonImageSource, both id and ref are acceptable, ref must not be a NameOnly (interpreted as all tags in that repository by the daemon)
// For daemonImageDestination, it must be a ref, which is NamedTagged.
// (We could, in principle, also allow storing images without tagging them, and the user would have to refer to them using the docker image ID = config digest.
// Using the config digest requires the caller to parse the manifest themselves, which is very cumbersome; so, for now, we dont bother.)
type daemonReference struct {
id digest.Digest
ref reference.Named // !reference.IsNameOnly
}
// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an ImageReference.
func ParseReference(refString string) (types.ImageReference, error) {
// This is intended to be compatible with reference.ParseAnyReference, but more strict about refusing some of the ambiguous cases.
// In particular, this rejects unprefixed digest values (64 hex chars), and sha256 digest prefixes (sha256:fewer-than-64-hex-chars).
// digest:hexstring is structurally the same as a reponame:tag (meaning docker.io/library/reponame:tag).
// reference.ParseAnyReference interprets such strings as digests.
if dgst, err := digest.Parse(refString); err == nil {
// The daemon explicitly refuses to tag images with a reponame equal to digest.Canonical - but _only_ this digest name.
// Other digest references are ambiguous, so refuse them.
if dgst.Algorithm() != digest.Canonical {
return nil, fmt.Errorf("Invalid docker-daemon: reference %s: only digest algorithm %s accepted", refString, digest.Canonical)
}
return NewReference(dgst, nil)
}
ref, err := reference.ParseNormalizedNamed(refString) // This also rejects unprefixed digest values
if err != nil {
return nil, err
}
if reference.FamiliarName(ref) == digest.Canonical.String() {
return nil, fmt.Errorf("Invalid docker-daemon: reference %s: The %s repository name is reserved for (non-shortened) digest references", refString, digest.Canonical)
}
return NewReference("", ref)
}
// NewReference returns a docker-daemon reference for either the supplied image ID (config digest) or the supplied reference (which must satisfy !reference.IsNameOnly)
func NewReference(id digest.Digest, ref reference.Named) (types.ImageReference, error) {
if id != "" && ref != nil {
return nil, errors.New("docker-daemon: reference must not have an image ID and a reference string specified at the same time")
}
if ref != nil {
if reference.IsNameOnly(ref) {
return nil, fmt.Errorf("docker-daemon: reference %s has neither a tag nor a digest", reference.FamiliarString(ref))
}
// A github.com/distribution/reference value can have a tag and a digest at the same time!
// Most versions of docker/reference do not handle that (ignoring the tag), so reject such input.
// This MAY be accepted in the future.
// (Even if it were supported, the semantics of policy namespaces are unclear - should we drop
// the tag or the digest first?)
_, isTagged := ref.(reference.NamedTagged)
_, isDigested := ref.(reference.Canonical)
if isTagged && isDigested {
return nil, fmt.Errorf("docker-daemon: references with both a tag and digest are currently not supported")
}
}
return daemonReference{
id: id,
ref: ref,
}, nil
}
func (ref daemonReference) Transport() types.ImageTransport {
return Transport
}
// StringWithinTransport returns a string representation of the reference, which MUST be such that
// reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference.
// NOTE: The returned string is not promised to be equal to the original input to ParseReference;
// e.g. default attribute values omitted by the user may be filled in the return value, or vice versa.
// WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix;
// instead, see transports.ImageName().
func (ref daemonReference) StringWithinTransport() string {
switch {
case ref.id != "":
return ref.id.String()
case ref.ref != nil:
return reference.FamiliarString(ref.ref)
default: // Coverage: Should never happen, NewReference above should refuse such values.
panic("Internal inconsistency: daemonReference has empty id and nil ref")
}
}
// DockerReference returns a Docker reference associated with this reference
// (fully explicit, i.e. !reference.IsNameOnly, but reflecting user intent,
// not e.g. after redirect or alias processing), or nil if unknown/not applicable.
func (ref daemonReference) DockerReference() reference.Named {
return ref.ref // May be nil
}
// PolicyConfigurationIdentity returns a string representation of the reference, suitable for policy lookup.
// This MUST reflect user intent, not e.g. after processing of third-party redirects or aliases;
// The value SHOULD be fully explicit about its semantics, with no hidden defaults, AND canonical
// (i.e. various references with exactly the same semantics should return the same configuration identity)
// It is fine for the return value to be equal to StringWithinTransport(), and it is desirable but
// not required/guaranteed that it will be a valid input to Transport().ParseReference().
// Returns "" if configuration identities for these references are not supported.
func (ref daemonReference) PolicyConfigurationIdentity() string {
// We must allow referring to images in the daemon by image ID, otherwise untagged images would not be accessible.
// But the existence of image IDs means that we cant truly well namespace the input:
// a single image can be namespaced either using the name or the ID depending on how it is named.
//
// Thats fairly unexpected, but we have to cope somehow.
//
// So, use the ordinary docker/policyconfiguration namespacing for named images.
// image IDs all fall into the root namespace.
// Users can set up the root namespace to be either untrusted or rejected,
// and to set up specific trust for named namespaces. This allows verifying image
// identity when a name is known, and unnamed images would be untrusted or rejected.
switch {
case ref.id != "":
return "" // This still allows using the default "" scope to define a global policy for ID-identified images.
case ref.ref != nil:
res, err := policyconfiguration.DockerReferenceIdentity(ref.ref)
if res == "" || err != nil { // Coverage: Should never happen, NewReference above should refuse values which could cause a failure.
panic(fmt.Sprintf("Internal inconsistency: policyconfiguration.DockerReferenceIdentity returned %#v, %v", res, err))
}
return res
default: // Coverage: Should never happen, NewReference above should refuse such values.
panic("Internal inconsistency: daemonReference has empty id and nil ref")
}
}
// PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search
// for if explicit configuration for PolicyConfigurationIdentity() is not set. The list will be processed
// in order, terminating on first match, and an implicit "" is always checked at the end.
// It is STRONGLY recommended for the first element, if any, to be a prefix of PolicyConfigurationIdentity(),
// and each following element to be a prefix of the element preceding it.
func (ref daemonReference) PolicyConfigurationNamespaces() []string {
// See the explanation in daemonReference.PolicyConfigurationIdentity.
switch {
case ref.id != "":
return []string{}
case ref.ref != nil:
return policyconfiguration.DockerReferenceNamespaces(ref.ref)
default: // Coverage: Should never happen, NewReference above should refuse such values.
panic("Internal inconsistency: daemonReference has empty id and nil ref")
}
}
// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport.
// The caller must call .Close() on the returned ImageCloser.
// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource,
// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details.
func (ref daemonReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) {
return image.FromReference(ctx, sys, ref)
}
// NewImageSource returns a types.ImageSource for this reference.
// The caller must call .Close() on the returned ImageSource.
func (ref daemonReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) {
return newImageSource(ctx, sys, ref)
}
// NewImageDestination returns a types.ImageDestination for this reference.
// The caller must call .Close() on the returned ImageDestination.
func (ref daemonReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) {
return newImageDestination(ctx, sys, ref)
}
// DeleteImage deletes the named image from the registry, if supported.
func (ref daemonReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error {
// Should this just untag the image? Should this stop running containers?
// The semantics is not quite as clear as for remote repositories.
// The user can run (docker rmi) directly anyway, so, for now(?), punt instead of trying to guess what the user meant.
return errors.New("Deleting images not implemented for docker-daemon: images")
}

View file

@ -123,6 +123,9 @@ func GetDigest(ctx context.Context, sys *types.SystemContext, ref types.ImageRef
if !ok {
return "", errors.New("ref must be a dockerReference")
}
if dr.isUnknownDigest {
return "", fmt.Errorf("docker: reference %q is for unknown digest case; cannot get digest", dr.StringWithinTransport())
}
tagOrDigest, err := dr.tagOrDigest()
if err != nil {

View file

@ -452,7 +452,15 @@ func (d *dockerImageDestination) TryReusingBlobWithOptions(ctx context.Context,
// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
func (d *dockerImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error {
var refTail string
if instanceDigest != nil {
// If d.ref.isUnknownDigest=true, then we push without a tag, so get the
// digest that will be used
if d.ref.isUnknownDigest {
digest, err := manifest.Digest(m)
if err != nil {
return err
}
refTail = digest.String()
} else if instanceDigest != nil {
// If the instanceDigest is provided, then use it as the refTail, because the reference,
// whether it includes a tag or a digest, refers to the list as a whole, and not this
// particular instance.

View file

@ -38,8 +38,8 @@ type dockerImageSource struct {
impl.DoesNotAffectLayerInfosForCopy
stubs.ImplementsGetBlobAt
logicalRef dockerReference // The reference the user requested.
physicalRef dockerReference // The actual reference we are accessing (possibly a mirror)
logicalRef dockerReference // The reference the user requested. This must satisfy !isUnknownDigest
physicalRef dockerReference // The actual reference we are accessing (possibly a mirror). This must satisfy !isUnknownDigest
c *dockerClient
// State
cachedManifest []byte // nil if not loaded yet
@ -48,7 +48,12 @@ type dockerImageSource struct {
// newImageSource creates a new ImageSource for the specified image reference.
// The caller must call .Close() on the returned ImageSource.
// The caller must ensure !ref.isUnknownDigest.
func newImageSource(ctx context.Context, sys *types.SystemContext, ref dockerReference) (*dockerImageSource, error) {
if ref.isUnknownDigest {
return nil, fmt.Errorf("reading images from docker: reference %q without a tag or digest is not supported", ref.StringWithinTransport())
}
registryConfig, err := loadRegistryConfiguration(sys)
if err != nil {
return nil, err
@ -121,7 +126,7 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref dockerRef
// The caller must call .Close() on the returned ImageSource.
func newImageSourceAttempt(ctx context.Context, sys *types.SystemContext, logicalRef dockerReference, pullSource sysregistriesv2.PullSource,
registryConfig *registryConfiguration) (*dockerImageSource, error) {
physicalRef, err := newReference(pullSource.Reference)
physicalRef, err := newReference(pullSource.Reference, false)
if err != nil {
return nil, err
}
@ -591,6 +596,10 @@ func (s *dockerImageSource) getSignaturesFromSigstoreAttachments(ctx context.Con
// deleteImage deletes the named image from the registry, if supported.
func deleteImage(ctx context.Context, sys *types.SystemContext, ref dockerReference) error {
if ref.isUnknownDigest {
return fmt.Errorf("Docker reference without a tag or digest cannot be deleted")
}
registryConfig, err := loadRegistryConfiguration(sys)
if err != nil {
return err

View file

@ -12,6 +12,11 @@ import (
"github.com/containers/image/v5/types"
)
// UnknownDigestSuffix can be appended to a reference when the caller
// wants to push an image without a tag or digest.
// NewReferenceUnknownDigest() is called when this const is detected.
const UnknownDigestSuffix = "@@unknown-digest@@"
func init() {
transports.Register(Transport)
}
@ -43,7 +48,8 @@ func (t dockerTransport) ValidatePolicyConfigurationScope(scope string) error {
// dockerReference is an ImageReference for Docker images.
type dockerReference struct {
ref reference.Named // By construction we know that !reference.IsNameOnly(ref)
ref reference.Named // By construction we know that !reference.IsNameOnly(ref) unless isUnknownDigest=true
isUnknownDigest bool
}
// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an Docker ImageReference.
@ -51,23 +57,46 @@ func ParseReference(refString string) (types.ImageReference, error) {
if !strings.HasPrefix(refString, "//") {
return nil, fmt.Errorf("docker: image reference %s does not start with //", refString)
}
// Check if ref has UnknownDigestSuffix suffixed to it
unknownDigest := false
if strings.HasSuffix(refString, UnknownDigestSuffix) {
unknownDigest = true
refString = strings.TrimSuffix(refString, UnknownDigestSuffix)
}
ref, err := reference.ParseNormalizedNamed(strings.TrimPrefix(refString, "//"))
if err != nil {
return nil, err
}
if unknownDigest {
if !reference.IsNameOnly(ref) {
return nil, fmt.Errorf("docker: image reference %q has unknown digest set but it contains either a tag or digest", ref.String()+UnknownDigestSuffix)
}
return NewReferenceUnknownDigest(ref)
}
ref = reference.TagNameOnly(ref)
return NewReference(ref)
}
// NewReference returns a Docker reference for a named reference. The reference must satisfy !reference.IsNameOnly().
func NewReference(ref reference.Named) (types.ImageReference, error) {
return newReference(ref)
return newReference(ref, false)
}
// NewReferenceUnknownDigest returns a Docker reference for a named reference, which can be used to write images without setting
// a tag on the registry. The reference must satisfy reference.IsNameOnly()
func NewReferenceUnknownDigest(ref reference.Named) (types.ImageReference, error) {
return newReference(ref, true)
}
// newReference returns a dockerReference for a named reference.
func newReference(ref reference.Named) (dockerReference, error) {
if reference.IsNameOnly(ref) {
return dockerReference{}, fmt.Errorf("Docker reference %s has neither a tag nor a digest", reference.FamiliarString(ref))
func newReference(ref reference.Named, unknownDigest bool) (dockerReference, error) {
if reference.IsNameOnly(ref) && !unknownDigest {
return dockerReference{}, fmt.Errorf("Docker reference %s is not for an unknown digest case; tag or digest is needed", reference.FamiliarString(ref))
}
if !reference.IsNameOnly(ref) && unknownDigest {
return dockerReference{}, fmt.Errorf("Docker reference %s is for an unknown digest case but reference has a tag or digest", reference.FamiliarString(ref))
}
// A github.com/distribution/reference value can have a tag and a digest at the same time!
// The docker/distribution API does not really support that (we cant ask for an image with a specific
@ -81,7 +110,8 @@ func newReference(ref reference.Named) (dockerReference, error) {
}
return dockerReference{
ref: ref,
ref: ref,
isUnknownDigest: unknownDigest,
}, nil
}
@ -95,7 +125,11 @@ func (ref dockerReference) Transport() types.ImageTransport {
// e.g. default attribute values omitted by the user may be filled in the return value, or vice versa.
// WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix.
func (ref dockerReference) StringWithinTransport() string {
return "//" + reference.FamiliarString(ref.ref)
famString := "//" + reference.FamiliarString(ref.ref)
if ref.isUnknownDigest {
return famString + UnknownDigestSuffix
}
return famString
}
// DockerReference returns a Docker reference associated with this reference
@ -113,6 +147,9 @@ func (ref dockerReference) DockerReference() reference.Named {
// not required/guaranteed that it will be a valid input to Transport().ParseReference().
// Returns "" if configuration identities for these references are not supported.
func (ref dockerReference) PolicyConfigurationIdentity() string {
if ref.isUnknownDigest {
return ref.ref.Name()
}
res, err := policyconfiguration.DockerReferenceIdentity(ref.ref)
if res == "" || err != nil { // Coverage: Should never happen, NewReference above should refuse values which could cause a failure.
panic(fmt.Sprintf("Internal inconsistency: policyconfiguration.DockerReferenceIdentity returned %#v, %v", res, err))
@ -126,7 +163,13 @@ func (ref dockerReference) PolicyConfigurationIdentity() string {
// It is STRONGLY recommended for the first element, if any, to be a prefix of PolicyConfigurationIdentity(),
// and each following element to be a prefix of the element preceding it.
func (ref dockerReference) PolicyConfigurationNamespaces() []string {
return policyconfiguration.DockerReferenceNamespaces(ref.ref)
namespaces := policyconfiguration.DockerReferenceNamespaces(ref.ref)
if ref.isUnknownDigest {
if len(namespaces) != 0 && namespaces[0] == ref.ref.Name() {
namespaces = namespaces[1:]
}
}
return namespaces
}
// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport.
@ -163,6 +206,10 @@ func (ref dockerReference) tagOrDigest() (string, error) {
if ref, ok := ref.ref.(reference.NamedTagged); ok {
return ref.Tag(), nil
}
if ref.isUnknownDigest {
return "", fmt.Errorf("Docker reference %q is for an unknown digest case, has neither a digest nor a tag", reference.FamiliarString(ref.ref))
}
// This should not happen, NewReference above refuses reference.IsNameOnly values.
return "", fmt.Errorf("Internal inconsistency: Reference %s unexpectedly has neither a digest nor a tag", reference.FamiliarString(ref.ref))
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,226 @@
package openshift
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/iolimits"
"github.com/containers/image/v5/version"
"github.com/sirupsen/logrus"
)
// openshiftClient is configuration for dealing with a single image stream, for reading or writing.
type openshiftClient struct {
ref openshiftReference
baseURL *url.URL
// Values from Kubernetes configuration
httpClient *http.Client
bearerToken string // "" if not used
username string // "" if not used
password string // if username != ""
}
// newOpenshiftClient creates a new openshiftClient for the specified reference.
func newOpenshiftClient(ref openshiftReference) (*openshiftClient, error) {
// We have already done this parsing in ParseReference, but thrown away
// httpClient. So, parse again.
// (We could also rework/split restClientFor to "get base URL" to be done
// in ParseReference, and "get httpClient" to be done here. But until/unless
// we support non-default clusters, this is good enough.)
// Overall, this is modelled on openshift/origin/pkg/cmd/util/clientcmd.New().ClientConfig() and openshift/origin/pkg/client.
cmdConfig := defaultClientConfig()
logrus.Debugf("cmdConfig: %#v", cmdConfig)
restConfig, err := cmdConfig.ClientConfig()
if err != nil {
return nil, err
}
// REMOVED: SetOpenShiftDefaults (values are not overridable in config files, so hard-coded these defaults.)
logrus.Debugf("restConfig: %#v", restConfig)
baseURL, httpClient, err := restClientFor(restConfig)
if err != nil {
return nil, err
}
logrus.Debugf("URL: %#v", *baseURL)
if httpClient == nil {
httpClient = http.DefaultClient
}
return &openshiftClient{
ref: ref,
baseURL: baseURL,
httpClient: httpClient,
bearerToken: restConfig.BearerToken,
username: restConfig.Username,
password: restConfig.Password,
}, nil
}
func (c *openshiftClient) close() {
c.httpClient.CloseIdleConnections()
}
// doRequest performs a correctly authenticated request to a specified path, and returns response body or an error object.
func (c *openshiftClient) doRequest(ctx context.Context, method, path string, requestBody []byte) ([]byte, error) {
requestURL := *c.baseURL
requestURL.Path = path
var requestBodyReader io.Reader
if requestBody != nil {
logrus.Debugf("Will send body: %s", requestBody)
requestBodyReader = bytes.NewReader(requestBody)
}
req, err := http.NewRequestWithContext(ctx, method, requestURL.String(), requestBodyReader)
if err != nil {
return nil, err
}
if len(c.bearerToken) != 0 {
req.Header.Set("Authorization", "Bearer "+c.bearerToken)
} else if len(c.username) != 0 {
req.SetBasicAuth(c.username, c.password)
}
req.Header.Set("Accept", "application/json, */*")
req.Header.Set("User-Agent", fmt.Sprintf("skopeo/%s", version.Version))
if requestBody != nil {
req.Header.Set("Content-Type", "application/json")
}
logrus.Debugf("%s %s", method, requestURL.Redacted())
res, err := c.httpClient.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
body, err := iolimits.ReadAtMost(res.Body, iolimits.MaxOpenShiftStatusBody)
if err != nil {
return nil, err
}
logrus.Debugf("Got body: %s", body)
// FIXME: Just throwing this useful information away only to try to guess later...
logrus.Debugf("Got content-type: %s", res.Header.Get("Content-Type"))
var status status
statusValid := false
if err := json.Unmarshal(body, &status); err == nil && len(status.Status) > 0 {
statusValid = true
}
switch {
case res.StatusCode == http.StatusSwitchingProtocols: // FIXME?! No idea why this weird case exists in k8s.io/kubernetes/pkg/client/restclient.
if statusValid && status.Status != "Success" {
return nil, errors.New(status.Message)
}
case res.StatusCode >= http.StatusOK && res.StatusCode <= http.StatusPartialContent:
// OK.
default:
if statusValid {
return nil, errors.New(status.Message)
}
return nil, fmt.Errorf("HTTP error: status code: %d (%s), body: %s", res.StatusCode, http.StatusText(res.StatusCode), string(body))
}
return body, nil
}
// getImage loads the specified image object.
func (c *openshiftClient) getImage(ctx context.Context, imageStreamImageName string) (*image, error) {
// FIXME: validate components per validation.IsValidPathSegmentName?
path := fmt.Sprintf("/oapi/v1/namespaces/%s/imagestreamimages/%s@%s", c.ref.namespace, c.ref.stream, imageStreamImageName)
body, err := c.doRequest(ctx, http.MethodGet, path, nil)
if err != nil {
return nil, err
}
// Note: This does absolutely no kind/version checking or conversions.
var isi imageStreamImage
if err := json.Unmarshal(body, &isi); err != nil {
return nil, err
}
return &isi.Image, nil
}
// convertDockerImageReference takes an image API DockerImageReference value and returns a reference we can actually use;
// currently OpenShift stores the cluster-internal service IPs here, which are unusable from the outside.
func (c *openshiftClient) convertDockerImageReference(ref string) (string, error) {
_, repo, gotRepo := strings.Cut(ref, "/")
if !gotRepo {
return "", fmt.Errorf("Invalid format of docker reference %s: missing '/'", ref)
}
return reference.Domain(c.ref.dockerReference) + "/" + repo, nil
}
// These structs are subsets of github.com/openshift/origin/pkg/image/api/v1 and its dependencies.
type imageStream struct {
Status imageStreamStatus `json:"status,omitempty"`
}
type imageStreamStatus struct {
DockerImageRepository string `json:"dockerImageRepository"`
Tags []namedTagEventList `json:"tags,omitempty"`
}
type namedTagEventList struct {
Tag string `json:"tag"`
Items []tagEvent `json:"items"`
}
type tagEvent struct {
DockerImageReference string `json:"dockerImageReference"`
Image string `json:"image"`
}
type imageStreamImage struct {
Image image `json:"image"`
}
type image struct {
objectMeta `json:"metadata,omitempty"`
DockerImageReference string `json:"dockerImageReference,omitempty"`
// DockerImageMetadata runtime.RawExtension `json:"dockerImageMetadata,omitempty"`
DockerImageMetadataVersion string `json:"dockerImageMetadataVersion,omitempty"`
DockerImageManifest string `json:"dockerImageManifest,omitempty"`
// DockerImageLayers []ImageLayer `json:"dockerImageLayers"`
Signatures []imageSignature `json:"signatures,omitempty"`
}
const imageSignatureTypeAtomic string = "atomic"
type imageSignature struct {
typeMeta `json:",inline"`
objectMeta `json:"metadata,omitempty"`
Type string `json:"type"`
Content []byte `json:"content"`
// Conditions []SignatureCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
// ImageIdentity string `json:"imageIdentity,omitempty"`
// SignedClaims map[string]string `json:"signedClaims,omitempty"`
// Created *unversioned.Time `json:"created,omitempty"`
// IssuedBy SignatureIssuer `json:"issuedBy,omitempty"`
// IssuedTo SignatureSubject `json:"issuedTo,omitempty"`
}
type typeMeta struct {
Kind string `json:"kind,omitempty"`
APIVersion string `json:"apiVersion,omitempty"`
}
type objectMeta struct {
Name string `json:"name,omitempty"`
GenerateName string `json:"generateName,omitempty"`
Namespace string `json:"namespace,omitempty"`
SelfLink string `json:"selfLink,omitempty"`
ResourceVersion string `json:"resourceVersion,omitempty"`
Generation int64 `json:"generation,omitempty"`
DeletionGracePeriodSeconds *int64 `json:"deletionGracePeriodSeconds,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
}
// A subset of k8s.io/kubernetes/pkg/api/unversioned/Status
type status struct {
Status string `json:"status,omitempty"`
Message string `json:"message,omitempty"`
// Reason StatusReason `json:"reason,omitempty"`
// Details *StatusDetails `json:"details,omitempty"`
Code int32 `json:"code,omitempty"`
}

View file

@ -0,0 +1,248 @@
package openshift
import (
"bytes"
"context"
"crypto/rand"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"github.com/containers/image/v5/docker"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/blobinfocache"
"github.com/containers/image/v5/internal/imagedestination"
"github.com/containers/image/v5/internal/imagedestination/impl"
"github.com/containers/image/v5/internal/imagedestination/stubs"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/internal/set"
"github.com/containers/image/v5/internal/signature"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/types"
"github.com/opencontainers/go-digest"
"golang.org/x/exp/slices"
)
type openshiftImageDestination struct {
impl.Compat
stubs.AlwaysSupportsSignatures
client *openshiftClient
docker private.ImageDestination // The docker/distribution API endpoint
// State
imageStreamImageName string // "" if not yet known
}
// newImageDestination creates a new ImageDestination for the specified reference.
func newImageDestination(ctx context.Context, sys *types.SystemContext, ref openshiftReference) (private.ImageDestination, error) {
client, err := newOpenshiftClient(ref)
if err != nil {
return nil, err
}
// FIXME: Should this always use a digest, not a tag? Uploading to Docker by tag requires the tag _inside_ the manifest to match,
// i.e. a single signed image cannot be available under multiple tags. But with types.ImageDestination, we don't know
// the manifest digest at this point.
dockerRefString := fmt.Sprintf("//%s/%s/%s:%s", reference.Domain(client.ref.dockerReference), client.ref.namespace, client.ref.stream, client.ref.dockerReference.Tag())
dockerRef, err := docker.ParseReference(dockerRefString)
if err != nil {
return nil, err
}
docker, err := dockerRef.NewImageDestination(ctx, sys)
if err != nil {
return nil, err
}
d := &openshiftImageDestination{
client: client,
docker: imagedestination.FromPublic(docker),
}
d.Compat = impl.AddCompat(d)
return d, nil
}
// Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent,
// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects.
func (d *openshiftImageDestination) Reference() types.ImageReference {
return d.client.ref
}
// Close removes resources associated with an initialized ImageDestination, if any.
func (d *openshiftImageDestination) Close() error {
err := d.docker.Close()
d.client.close()
return err
}
func (d *openshiftImageDestination) SupportedManifestMIMETypes() []string {
return d.docker.SupportedManifestMIMETypes()
}
func (d *openshiftImageDestination) DesiredLayerCompression() types.LayerCompression {
return types.Compress
}
// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
// uploaded to the image destination, true otherwise.
func (d *openshiftImageDestination) AcceptsForeignLayerURLs() bool {
return true
}
// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise.
func (d *openshiftImageDestination) MustMatchRuntimeOS() bool {
return false
}
// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(),
// and would prefer to receive an unmodified manifest instead of one modified for the destination.
// Does not make a difference if Reference().DockerReference() is nil.
func (d *openshiftImageDestination) IgnoresEmbeddedDockerReference() bool {
return d.docker.IgnoresEmbeddedDockerReference()
}
// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently.
func (d *openshiftImageDestination) HasThreadSafePutBlob() bool {
return false
}
// SupportsPutBlobPartial returns true if PutBlobPartial is supported.
func (d *openshiftImageDestination) SupportsPutBlobPartial() bool {
return d.docker.SupportsPutBlobPartial()
}
// PutBlobWithOptions writes contents of stream and returns data representing the result.
// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents.
// inputInfo.Size is the expected length of stream, if known.
// inputInfo.MediaType describes the blob format, if known.
// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
// to any other readers for download using the supplied digest.
// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlobWithOptions MUST 1) fail, and 2) delete any data stored so far.
func (d *openshiftImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options private.PutBlobOptions) (private.UploadedBlob, error) {
return d.docker.PutBlobWithOptions(ctx, stream, inputInfo, options)
}
// PutBlobPartial attempts to create a blob using the data that is already present
// at the destination. chunkAccessor is accessed in a non-sequential way to retrieve the missing chunks.
// It is available only if SupportsPutBlobPartial().
// Even if SupportsPutBlobPartial() returns true, the call can fail, in which case the caller
// should fall back to PutBlobWithOptions.
func (d *openshiftImageDestination) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, cache blobinfocache.BlobInfoCache2) (private.UploadedBlob, error) {
return d.docker.PutBlobPartial(ctx, chunkAccessor, srcInfo, cache)
}
// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree).
// info.Digest must not be empty.
// If the blob has been successfully reused, returns (true, info, nil).
// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
func (d *openshiftImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) {
return d.docker.TryReusingBlobWithOptions(ctx, info, options)
}
// PutManifest writes manifest to the destination.
// FIXME? This should also receive a MIME type if known, to differentiate between schema versions.
// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema),
// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
func (d *openshiftImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error {
if instanceDigest == nil {
manifestDigest, err := manifest.Digest(m)
if err != nil {
return err
}
d.imageStreamImageName = manifestDigest.String()
}
return d.docker.PutManifest(ctx, m, instanceDigest)
}
// PutSignaturesWithFormat writes a set of signatures to the destination.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for
// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list.
// MUST be called after PutManifest (signatures may reference manifest contents).
func (d *openshiftImageDestination) PutSignaturesWithFormat(ctx context.Context, signatures []signature.Signature, instanceDigest *digest.Digest) error {
var imageStreamImageName string
if instanceDigest == nil {
if d.imageStreamImageName == "" {
return errors.New("Internal error: Unknown manifest digest, can't add signatures")
}
imageStreamImageName = d.imageStreamImageName
} else {
imageStreamImageName = instanceDigest.String()
}
// Because image signatures are a shared resource in Atomic Registry, the default upload
// always adds signatures. Eventually we should also allow removing signatures.
if len(signatures) == 0 {
return nil // No need to even read the old state.
}
image, err := d.client.getImage(ctx, imageStreamImageName)
if err != nil {
return err
}
existingSigNames := set.New[string]()
for _, sig := range image.Signatures {
existingSigNames.Add(sig.objectMeta.Name)
}
for _, newSigWithFormat := range signatures {
newSigSimple, ok := newSigWithFormat.(signature.SimpleSigning)
if !ok {
return signature.UnsupportedFormatError(newSigWithFormat)
}
newSig := newSigSimple.UntrustedSignature()
if slices.ContainsFunc(image.Signatures, func(existingSig imageSignature) bool {
return existingSig.Type == imageSignatureTypeAtomic && bytes.Equal(existingSig.Content, newSig)
}) {
continue
}
// The API expect us to invent a new unique name. This is racy, but hopefully good enough.
var signatureName string
for {
randBytes := make([]byte, 16)
n, err := rand.Read(randBytes)
if err != nil || n != 16 {
return fmt.Errorf("generating random signature len %d: %w", n, err)
}
signatureName = fmt.Sprintf("%s@%032x", imageStreamImageName, randBytes)
if !existingSigNames.Contains(signatureName) {
break
}
}
// Note: This does absolutely no kind/version checking or conversions.
sig := imageSignature{
typeMeta: typeMeta{
Kind: "ImageSignature",
APIVersion: "v1",
},
objectMeta: objectMeta{Name: signatureName},
Type: imageSignatureTypeAtomic,
Content: newSig,
}
body, err := json.Marshal(sig)
if err != nil {
return err
}
_, err = d.client.doRequest(ctx, http.MethodPost, "/oapi/v1/imagesignatures", body)
if err != nil {
return err
}
}
return nil
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
func (d *openshiftImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
return d.docker.Commit(ctx, unparsedToplevel)
}

View file

@ -0,0 +1,174 @@
package openshift
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"github.com/containers/image/v5/docker"
"github.com/containers/image/v5/internal/imagesource/impl"
"github.com/containers/image/v5/internal/imagesource/stubs"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/internal/signature"
"github.com/containers/image/v5/types"
"github.com/opencontainers/go-digest"
"github.com/sirupsen/logrus"
)
type openshiftImageSource struct {
impl.Compat
impl.DoesNotAffectLayerInfosForCopy
// This is slightly suboptimal. We could forward GetBlobAt(), but we need to call ensureImageIsResolved in SupportsGetBlobAt(),
// and that method doesnt provide a context for timing out. That could actually be fixed (SupportsGetBlobAt is private and we
// can change it), but this is a deprecated transport anyway, so for now we just punt.
stubs.NoGetBlobAtInitialize
client *openshiftClient
// Values specific to this image
sys *types.SystemContext
// State
docker types.ImageSource // The docker/distribution API endpoint, or nil if not resolved yet
imageStreamImageName string // Resolved image identifier, or "" if not known yet
}
// newImageSource creates a new ImageSource for the specified reference.
// The caller must call .Close() on the returned ImageSource.
func newImageSource(sys *types.SystemContext, ref openshiftReference) (private.ImageSource, error) {
client, err := newOpenshiftClient(ref)
if err != nil {
return nil, err
}
s := &openshiftImageSource{
NoGetBlobAtInitialize: stubs.NoGetBlobAt(ref),
client: client,
sys: sys,
}
s.Compat = impl.AddCompat(s)
return s, nil
}
// Reference returns the reference used to set up this source, _as specified by the user_
// (not as the image itself, or its underlying storage, claims). This can be used e.g. to determine which public keys are trusted for this image.
func (s *openshiftImageSource) Reference() types.ImageReference {
return s.client.ref
}
// Close removes resources associated with an initialized ImageSource, if any.
func (s *openshiftImageSource) Close() error {
var err error
if s.docker != nil {
err = s.docker.Close()
s.docker = nil
}
s.client.close()
return err
}
// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
// It may use a remote (= slow) service.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list);
// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists).
func (s *openshiftImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
if err := s.ensureImageIsResolved(ctx); err != nil {
return nil, "", err
}
return s.docker.GetManifest(ctx, instanceDigest)
}
// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently.
func (s *openshiftImageSource) HasThreadSafeGetBlob() bool {
return false
}
// GetBlob returns a stream for the specified blob, and the blobs size (or -1 if unknown).
// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
func (s *openshiftImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
if err := s.ensureImageIsResolved(ctx); err != nil {
return nil, 0, err
}
return s.docker.GetBlob(ctx, info, cache)
}
// GetSignaturesWithFormat returns the image's signatures. It may use a remote (= slow) service.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for
// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list
// (e.g. if the source never returns manifest lists).
func (s *openshiftImageSource) GetSignaturesWithFormat(ctx context.Context, instanceDigest *digest.Digest) ([]signature.Signature, error) {
var imageStreamImageName string
if instanceDigest == nil {
if err := s.ensureImageIsResolved(ctx); err != nil {
return nil, err
}
imageStreamImageName = s.imageStreamImageName
} else {
imageStreamImageName = instanceDigest.String()
}
image, err := s.client.getImage(ctx, imageStreamImageName)
if err != nil {
return nil, err
}
var sigs []signature.Signature
for _, sig := range image.Signatures {
if sig.Type == imageSignatureTypeAtomic {
sigs = append(sigs, signature.SimpleSigningFromBlob(sig.Content))
}
}
return sigs, nil
}
// ensureImageIsResolved sets up s.docker and s.imageStreamImageName
func (s *openshiftImageSource) ensureImageIsResolved(ctx context.Context) error {
if s.docker != nil {
return nil
}
// FIXME: validate components per validation.IsValidPathSegmentName?
path := fmt.Sprintf("/oapi/v1/namespaces/%s/imagestreams/%s", s.client.ref.namespace, s.client.ref.stream)
body, err := s.client.doRequest(ctx, http.MethodGet, path, nil)
if err != nil {
return err
}
// Note: This does absolutely no kind/version checking or conversions.
var is imageStream
if err := json.Unmarshal(body, &is); err != nil {
return err
}
var te *tagEvent
for _, tag := range is.Status.Tags {
if tag.Tag != s.client.ref.dockerReference.Tag() {
continue
}
if len(tag.Items) > 0 {
te = &tag.Items[0]
break
}
}
if te == nil {
return errors.New("No matching tag found")
}
logrus.Debugf("tag event %#v", te)
dockerRefString, err := s.client.convertDockerImageReference(te.DockerImageReference)
if err != nil {
return err
}
logrus.Debugf("Resolved reference %#v", dockerRefString)
dockerRef, err := docker.ParseReference("//" + dockerRefString)
if err != nil {
return err
}
d, err := dockerRef.NewImageSource(ctx, s.sys)
if err != nil {
return err
}
s.docker = d
s.imageStreamImageName = te.Image
return nil
}

View file

@ -0,0 +1,153 @@
package openshift
import (
"context"
"errors"
"fmt"
"strings"
"github.com/containers/image/v5/docker/policyconfiguration"
"github.com/containers/image/v5/docker/reference"
genericImage "github.com/containers/image/v5/internal/image"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
"github.com/containers/storage/pkg/regexp"
)
func init() {
transports.Register(Transport)
}
// Transport is an ImageTransport for OpenShift registry-hosted images.
var Transport = openshiftTransport{}
type openshiftTransport struct{}
func (t openshiftTransport) Name() string {
return "atomic"
}
// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an ImageReference.
func (t openshiftTransport) ParseReference(reference string) (types.ImageReference, error) {
return ParseReference(reference)
}
// Note that imageNameRegexp is namespace/stream:tag, this
// is HOSTNAME/namespace/stream:tag or parent prefixes.
// Keep this in sync with imageNameRegexp!
var scopeRegexp = regexp.Delayed("^[^/]*(/[^:/]*(/[^:/]*(:[^:/]*)?)?)?$")
// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys
// (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value).
// It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion.
// scope passed to this function will not be "", that value is always allowed.
func (t openshiftTransport) ValidatePolicyConfigurationScope(scope string) error {
if scopeRegexp.FindStringIndex(scope) == nil {
return fmt.Errorf("Invalid scope name %s", scope)
}
return nil
}
// openshiftReference is an ImageReference for OpenShift images.
type openshiftReference struct {
dockerReference reference.NamedTagged
namespace string // Computed from dockerReference in advance.
stream string // Computed from dockerReference in advance.
}
// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OpenShift ImageReference.
func ParseReference(ref string) (types.ImageReference, error) {
r, err := reference.ParseNormalizedNamed(ref)
if err != nil {
return nil, fmt.Errorf("failed to parse image reference %q: %w", ref, err)
}
tagged, ok := r.(reference.NamedTagged)
if !ok {
return nil, fmt.Errorf("invalid image reference %s, expected format: 'hostname/namespace/stream:tag'", ref)
}
return NewReference(tagged)
}
// NewReference returns an OpenShift reference for a reference.NamedTagged
func NewReference(dockerRef reference.NamedTagged) (types.ImageReference, error) {
r := strings.SplitN(reference.Path(dockerRef), "/", 3)
if len(r) != 2 {
return nil, fmt.Errorf("invalid image reference: %s, expected format: 'hostname/namespace/stream:tag'",
reference.FamiliarString(dockerRef))
}
return openshiftReference{
namespace: r[0],
stream: r[1],
dockerReference: dockerRef,
}, nil
}
func (ref openshiftReference) Transport() types.ImageTransport {
return Transport
}
// StringWithinTransport returns a string representation of the reference, which MUST be such that
// reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference.
// NOTE: The returned string is not promised to be equal to the original input to ParseReference;
// e.g. default attribute values omitted by the user may be filled in the return value, or vice versa.
// WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix.
func (ref openshiftReference) StringWithinTransport() string {
return reference.FamiliarString(ref.dockerReference)
}
// DockerReference returns a Docker reference associated with this reference
// (fully explicit, i.e. !reference.IsNameOnly, but reflecting user intent,
// not e.g. after redirect or alias processing), or nil if unknown/not applicable.
func (ref openshiftReference) DockerReference() reference.Named {
return ref.dockerReference
}
// PolicyConfigurationIdentity returns a string representation of the reference, suitable for policy lookup.
// This MUST reflect user intent, not e.g. after processing of third-party redirects or aliases;
// The value SHOULD be fully explicit about its semantics, with no hidden defaults, AND canonical
// (i.e. various references with exactly the same semantics should return the same configuration identity)
// It is fine for the return value to be equal to StringWithinTransport(), and it is desirable but
// not required/guaranteed that it will be a valid input to Transport().ParseReference().
// Returns "" if configuration identities for these references are not supported.
func (ref openshiftReference) PolicyConfigurationIdentity() string {
res, err := policyconfiguration.DockerReferenceIdentity(ref.dockerReference)
if res == "" || err != nil { // Coverage: Should never happen, NewReference constructs a valid tagged reference.
panic(fmt.Sprintf("Internal inconsistency: policyconfiguration.DockerReferenceIdentity returned %#v, %v", res, err))
}
return res
}
// PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search
// for if explicit configuration for PolicyConfigurationIdentity() is not set. The list will be processed
// in order, terminating on first match, and an implicit "" is always checked at the end.
// It is STRONGLY recommended for the first element, if any, to be a prefix of PolicyConfigurationIdentity(),
// and each following element to be a prefix of the element preceding it.
func (ref openshiftReference) PolicyConfigurationNamespaces() []string {
return policyconfiguration.DockerReferenceNamespaces(ref.dockerReference)
}
// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport.
// The caller must call .Close() on the returned ImageCloser.
// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource,
// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details.
func (ref openshiftReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) {
return genericImage.FromReference(ctx, sys, ref)
}
// NewImageSource returns a types.ImageSource for this reference.
// The caller must call .Close() on the returned ImageSource.
func (ref openshiftReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) {
return newImageSource(sys, ref)
}
// NewImageDestination returns a types.ImageDestination for this reference.
// The caller must call .Close() on the returned ImageDestination.
func (ref openshiftReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) {
return newImageDestination(ctx, sys, ref)
}
// DeleteImage deletes the named image from the registry, if supported.
func (ref openshiftReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error {
return errors.New("Deleting images not implemented for atomic: images")
}

View file

@ -0,0 +1,517 @@
//go:build containers_image_ostree
// +build containers_image_ostree
package ostree
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"syscall"
"time"
"unsafe"
"github.com/containers/image/v5/internal/imagedestination/impl"
"github.com/containers/image/v5/internal/imagedestination/stubs"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/internal/putblobdigest"
"github.com/containers/image/v5/internal/signature"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/types"
"github.com/containers/storage/pkg/archive"
"github.com/klauspost/pgzip"
"github.com/opencontainers/go-digest"
selinux "github.com/opencontainers/selinux/go-selinux"
"github.com/ostreedev/ostree-go/pkg/otbuiltin"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)
// #cgo pkg-config: glib-2.0 gobject-2.0 ostree-1 libselinux
// #include <glib.h>
// #include <glib-object.h>
// #include <gio/gio.h>
// #include <stdlib.h>
// #include <ostree.h>
// #include <gio/ginputstream.h>
// #include <selinux/selinux.h>
// #include <selinux/label.h>
import "C"
type blobToImport struct {
Size int64
Digest digest.Digest
BlobPath string
}
type descriptor struct {
Size int64 `json:"size"`
Digest digest.Digest `json:"digest"`
}
type fsLayersSchema1 struct {
BlobSum digest.Digest `json:"blobSum"`
}
type manifestSchema struct {
LayersDescriptors []descriptor `json:"layers"`
FSLayers []fsLayersSchema1 `json:"fsLayers"`
}
type ostreeImageDestination struct {
impl.Compat
impl.PropertyMethodsInitialize
stubs.NoPutBlobPartialInitialize
stubs.AlwaysSupportsSignatures
ref ostreeReference
manifest string
schema manifestSchema
tmpDirPath string
blobs map[string]*blobToImport
digest digest.Digest
signaturesLen int
repo *C.struct_OstreeRepo
}
// newImageDestination returns an ImageDestination for writing to an existing ostree.
func newImageDestination(ref ostreeReference, tmpDirPath string) (private.ImageDestination, error) {
tmpDirPath = filepath.Join(tmpDirPath, ref.branchName)
if err := ensureDirectoryExists(tmpDirPath); err != nil {
return nil, err
}
d := &ostreeImageDestination{
PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{
SupportedManifestMIMETypes: []string{manifest.DockerV2Schema2MediaType},
DesiredLayerCompression: types.PreserveOriginal,
AcceptsForeignLayerURLs: false,
MustMatchRuntimeOS: true,
IgnoresEmbeddedDockerReference: false, // N/A, DockerReference() returns nil.
HasThreadSafePutBlob: false,
}),
NoPutBlobPartialInitialize: stubs.NoPutBlobPartial(ref),
ref: ref,
manifest: "",
schema: manifestSchema{},
tmpDirPath: tmpDirPath,
blobs: map[string]*blobToImport{},
digest: "",
signaturesLen: 0,
repo: nil,
}
d.Compat = impl.AddCompat(d)
return d, nil
}
// Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent,
// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects.
func (d *ostreeImageDestination) Reference() types.ImageReference {
return d.ref
}
// Close removes resources associated with an initialized ImageDestination, if any.
func (d *ostreeImageDestination) Close() error {
if d.repo != nil {
C.g_object_unref(C.gpointer(d.repo))
}
return os.RemoveAll(d.tmpDirPath)
}
// PutBlobWithOptions writes contents of stream and returns data representing the result.
// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents.
// inputInfo.Size is the expected length of stream, if known.
// inputInfo.MediaType describes the blob format, if known.
// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
// to any other readers for download using the supplied digest.
// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
func (d *ostreeImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options private.PutBlobOptions) (private.UploadedBlob, error) {
tmpDir, err := os.MkdirTemp(d.tmpDirPath, "blob")
if err != nil {
return private.UploadedBlob{}, err
}
blobPath := filepath.Join(tmpDir, "content")
blobFile, err := os.Create(blobPath)
if err != nil {
return private.UploadedBlob{}, err
}
defer blobFile.Close()
digester, stream := putblobdigest.DigestIfCanonicalUnknown(stream, inputInfo)
// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
size, err := io.Copy(blobFile, stream)
if err != nil {
return private.UploadedBlob{}, err
}
blobDigest := digester.Digest()
if inputInfo.Size != -1 && size != inputInfo.Size {
return private.UploadedBlob{}, fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", blobDigest, inputInfo.Size, size)
}
if err := blobFile.Sync(); err != nil {
return private.UploadedBlob{}, err
}
hash := blobDigest.Hex()
d.blobs[hash] = &blobToImport{Size: size, Digest: blobDigest, BlobPath: blobPath}
return private.UploadedBlob{Digest: blobDigest, Size: size}, nil
}
func fixFiles(selinuxHnd *C.struct_selabel_handle, root string, dir string, usermode bool) error {
entries, err := os.ReadDir(dir)
if err != nil {
return err
}
for _, entry := range entries {
fullpath := filepath.Join(dir, entry.Name())
if entry.Type()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
if err := os.Remove(fullpath); err != nil {
return err
}
continue
}
info, err := entry.Info()
if err != nil {
return err
}
if selinuxHnd != nil {
relPath, err := filepath.Rel(root, fullpath)
if err != nil {
return err
}
// Handle /exports/hostfs as a special case. Files under this directory are copied to the host,
// thus we benefit from maintaining the same SELinux label they would have on the host as we could
// use hard links instead of copying the files.
relPath = fmt.Sprintf("/%s", strings.TrimPrefix(relPath, "exports/hostfs/"))
relPathC := C.CString(relPath)
defer C.free(unsafe.Pointer(relPathC))
var context *C.char
res, err := C.selabel_lookup_raw(selinuxHnd, &context, relPathC, C.int(info.Mode()&os.ModePerm))
if int(res) < 0 && err != syscall.ENOENT {
return fmt.Errorf("cannot selabel_lookup_raw %s: %w", relPath, err)
}
if int(res) == 0 {
defer C.freecon(context)
fullpathC := C.CString(fullpath)
defer C.free(unsafe.Pointer(fullpathC))
res, err = C.lsetfilecon_raw(fullpathC, context)
if int(res) < 0 {
return fmt.Errorf("cannot setfilecon_raw %s to %s: %w", fullpath, C.GoString(context), err)
}
}
}
if entry.IsDir() {
if usermode {
if err := os.Chmod(fullpath, info.Mode()|0700); err != nil {
return err
}
}
err = fixFiles(selinuxHnd, root, fullpath, usermode)
if err != nil {
return err
}
} else if usermode && (entry.Type().IsRegular()) {
if err := os.Chmod(fullpath, info.Mode()|0600); err != nil {
return err
}
}
}
return nil
}
func (d *ostreeImageDestination) ostreeCommit(repo *otbuiltin.Repo, branch string, root string, metadata []string) error {
opts := otbuiltin.NewCommitOptions()
opts.AddMetadataString = metadata
opts.Timestamp = time.Now()
// OCI layers have no parent OSTree commit
opts.Parent = "0000000000000000000000000000000000000000000000000000000000000000"
_, err := repo.Commit(root, branch, opts)
return err
}
func generateTarSplitMetadata(output *bytes.Buffer, file string) (digest.Digest, int64, error) {
mfz := pgzip.NewWriter(output)
defer mfz.Close()
metaPacker := storage.NewJSONPacker(mfz)
stream, err := os.OpenFile(file, os.O_RDONLY, 0)
if err != nil {
return "", -1, err
}
defer stream.Close()
gzReader, err := archive.DecompressStream(stream)
if err != nil {
return "", -1, err
}
defer gzReader.Close()
its, err := asm.NewInputTarStream(gzReader, metaPacker, nil)
if err != nil {
return "", -1, err
}
digester := digest.Canonical.Digester()
written, err := io.Copy(digester.Hash(), its)
if err != nil {
return "", -1, err
}
return digester.Digest(), written, nil
}
func (d *ostreeImageDestination) importBlob(selinuxHnd *C.struct_selabel_handle, repo *otbuiltin.Repo, blob *blobToImport) error {
// TODO: This can take quite some time, and should ideally be cancellable using a context.Context.
ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex())
destinationPath := filepath.Join(d.tmpDirPath, blob.Digest.Hex(), "root")
if err := ensureDirectoryExists(destinationPath); err != nil {
return err
}
defer func() {
os.Remove(blob.BlobPath)
os.RemoveAll(destinationPath)
}()
var tarSplitOutput bytes.Buffer
uncompressedDigest, uncompressedSize, err := generateTarSplitMetadata(&tarSplitOutput, blob.BlobPath)
if err != nil {
return err
}
if os.Getuid() == 0 {
if err := archive.UntarPath(blob.BlobPath, destinationPath); err != nil {
return err
}
if err := fixFiles(selinuxHnd, destinationPath, destinationPath, false); err != nil {
return err
}
} else {
os.MkdirAll(destinationPath, 0755)
if err := exec.Command("tar", "-C", destinationPath, "--no-same-owner", "--no-same-permissions", "--delay-directory-restore", "-xf", blob.BlobPath).Run(); err != nil {
return err
}
if err := fixFiles(selinuxHnd, destinationPath, destinationPath, true); err != nil {
return err
}
}
return d.ostreeCommit(repo, ostreeBranch, destinationPath, []string{fmt.Sprintf("docker.size=%d", blob.Size),
fmt.Sprintf("docker.uncompressed_size=%d", uncompressedSize),
fmt.Sprintf("docker.uncompressed_digest=%s", uncompressedDigest.String()),
fmt.Sprintf("tarsplit.output=%s", base64.StdEncoding.EncodeToString(tarSplitOutput.Bytes()))})
}
func (d *ostreeImageDestination) importConfig(repo *otbuiltin.Repo, blob *blobToImport) error {
ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex())
destinationPath := filepath.Dir(blob.BlobPath)
return d.ostreeCommit(repo, ostreeBranch, destinationPath, []string{fmt.Sprintf("docker.size=%d", blob.Size)})
}
// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree).
// info.Digest must not be empty.
// If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may
// include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be
// reflected in the manifest that will be written.
// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
func (d *ostreeImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) {
if !impl.OriginalBlobMatchesRequiredCompression(options) {
return false, private.ReusedBlob{}, nil
}
if d.repo == nil {
repo, err := openRepo(d.ref.repo)
if err != nil {
return false, private.ReusedBlob{}, err
}
d.repo = repo
}
branch := fmt.Sprintf("ociimage/%s", info.Digest.Hex())
found, data, err := readMetadata(d.repo, branch, "docker.uncompressed_digest")
if err != nil || !found {
return found, private.ReusedBlob{}, err
}
found, data, err = readMetadata(d.repo, branch, "docker.uncompressed_size")
if err != nil || !found {
return found, private.ReusedBlob{}, err
}
found, data, err = readMetadata(d.repo, branch, "docker.size")
if err != nil || !found {
return found, private.ReusedBlob{}, err
}
size, err := strconv.ParseInt(data, 10, 64)
if err != nil {
return false, private.ReusedBlob{}, err
}
return true, private.ReusedBlob{Digest: info.Digest, Size: size}, nil
}
// PutManifest writes manifest to the destination.
// The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so
// there can be no secondary manifests.
// FIXME? This should also receive a MIME type if known, to differentiate between schema versions.
// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema),
// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
func (d *ostreeImageDestination) PutManifest(ctx context.Context, manifestBlob []byte, instanceDigest *digest.Digest) error {
if instanceDigest != nil {
return errors.New(`Manifest lists are not supported by "ostree:"`)
}
d.manifest = string(manifestBlob)
if err := json.Unmarshal(manifestBlob, &d.schema); err != nil {
return err
}
manifestPath := filepath.Join(d.tmpDirPath, d.ref.manifestPath())
if err := ensureParentDirectoryExists(manifestPath); err != nil {
return err
}
digest, err := manifest.Digest(manifestBlob)
if err != nil {
return err
}
d.digest = digest
return os.WriteFile(manifestPath, manifestBlob, 0644)
}
// PutSignaturesWithFormat writes a set of signatures to the destination.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for
// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list.
// MUST be called after PutManifest (signatures may reference manifest contents).
func (d *ostreeImageDestination) PutSignaturesWithFormat(ctx context.Context, signatures []signature.Signature, instanceDigest *digest.Digest) error {
if instanceDigest != nil {
return errors.New(`Manifest lists are not supported by "ostree:"`)
}
path := filepath.Join(d.tmpDirPath, d.ref.signaturePath(0))
if err := ensureParentDirectoryExists(path); err != nil {
return err
}
for i, sig := range signatures {
signaturePath := filepath.Join(d.tmpDirPath, d.ref.signaturePath(i))
blob, err := signature.Blob(sig)
if err != nil {
return err
}
if err := os.WriteFile(signaturePath, blob, 0644); err != nil {
return err
}
}
d.signaturesLen = len(signatures)
return nil
}
func (d *ostreeImageDestination) Commit(context.Context, types.UnparsedImage) error {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
repo, err := otbuiltin.OpenRepo(d.ref.repo)
if err != nil {
return err
}
_, err = repo.PrepareTransaction()
if err != nil {
return err
}
var selinuxHnd *C.struct_selabel_handle
if os.Getuid() == 0 && selinux.GetEnabled() {
selinuxHnd, err = C.selabel_open(C.SELABEL_CTX_FILE, nil, 0)
if selinuxHnd == nil {
return fmt.Errorf("cannot open the SELinux DB: %w", err)
}
defer C.selabel_close(selinuxHnd)
}
checkLayer := func(hash string) error {
blob := d.blobs[hash]
// if the blob is not present in d.blobs then it is already stored in OSTree,
// and we don't need to import it.
if blob == nil {
return nil
}
err := d.importBlob(selinuxHnd, repo, blob)
if err != nil {
return err
}
delete(d.blobs, hash)
return nil
}
for _, layer := range d.schema.LayersDescriptors {
hash := layer.Digest.Hex()
if err = checkLayer(hash); err != nil {
return err
}
}
for _, layer := range d.schema.FSLayers {
hash := layer.BlobSum.Hex()
if err = checkLayer(hash); err != nil {
return err
}
}
// Import the other blobs that are not layers
for _, blob := range d.blobs {
err := d.importConfig(repo, blob)
if err != nil {
return err
}
}
manifestPath := filepath.Join(d.tmpDirPath, "manifest")
metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest)),
fmt.Sprintf("signatures=%d", d.signaturesLen),
fmt.Sprintf("docker.digest=%s", string(d.digest))}
if err := d.ostreeCommit(repo, fmt.Sprintf("ociimage/%s", d.ref.branchName), manifestPath, metadata); err != nil {
return err
}
_, err = repo.CommitTransaction()
return err
}
func ensureDirectoryExists(path string) error {
if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
if err := os.MkdirAll(path, 0755); err != nil {
return err
}
}
return nil
}
func ensureParentDirectoryExists(path string) error {
return ensureDirectoryExists(filepath.Dir(path))
}

View file

@ -0,0 +1,450 @@
//go:build containers_image_ostree
// +build containers_image_ostree
package ostree
import (
"bytes"
"context"
"encoding/base64"
"errors"
"fmt"
"io"
"strconv"
"strings"
"unsafe"
"github.com/containers/image/v5/internal/imagesource/impl"
"github.com/containers/image/v5/internal/imagesource/stubs"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/internal/signature"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/types"
"github.com/containers/storage/pkg/ioutils"
"github.com/klauspost/pgzip"
digest "github.com/opencontainers/go-digest"
glib "github.com/ostreedev/ostree-go/pkg/glibobject"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)
// #cgo pkg-config: glib-2.0 gobject-2.0 ostree-1
// #include <glib.h>
// #include <glib-object.h>
// #include <gio/gio.h>
// #include <stdlib.h>
// #include <ostree.h>
// #include <gio/ginputstream.h>
import "C"
type ostreeImageSource struct {
impl.Compat
impl.PropertyMethodsInitialize
stubs.NoGetBlobAtInitialize
ref ostreeReference
tmpDir string
repo *C.struct_OstreeRepo
// get the compressed layer by its uncompressed checksum
compressed map[digest.Digest]digest.Digest
}
// newImageSource returns an ImageSource for reading from an existing directory.
func newImageSource(tmpDir string, ref ostreeReference) (private.ImageSource, error) {
s := &ostreeImageSource{
PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{
HasThreadSafeGetBlob: false,
}),
NoGetBlobAtInitialize: stubs.NoGetBlobAt(ref),
ref: ref,
tmpDir: tmpDir,
compressed: nil,
}
s.Compat = impl.AddCompat(s)
return s, nil
}
// Reference returns the reference used to set up this source.
func (s *ostreeImageSource) Reference() types.ImageReference {
return s.ref
}
// Close removes resources associated with an initialized ImageSource, if any.
func (s *ostreeImageSource) Close() error {
if s.repo != nil {
C.g_object_unref(C.gpointer(s.repo))
}
return nil
}
func (s *ostreeImageSource) getBlobUncompressedSize(blob string, isCompressed bool) (int64, error) {
var metadataKey string
if isCompressed {
metadataKey = "docker.uncompressed_size"
} else {
metadataKey = "docker.size"
}
b := fmt.Sprintf("ociimage/%s", blob)
found, data, err := readMetadata(s.repo, b, metadataKey)
if err != nil || !found {
return 0, err
}
return strconv.ParseInt(data, 10, 64)
}
func (s *ostreeImageSource) getLenSignatures() (int64, error) {
b := fmt.Sprintf("ociimage/%s", s.ref.branchName)
found, data, err := readMetadata(s.repo, b, "signatures")
if err != nil {
return -1, err
}
if !found {
// if 'signatures' is not present, just return 0 signatures.
return 0, nil
}
return strconv.ParseInt(data, 10, 64)
}
func (s *ostreeImageSource) getTarSplitData(blob string) ([]byte, error) {
b := fmt.Sprintf("ociimage/%s", blob)
found, out, err := readMetadata(s.repo, b, "tarsplit.output")
if err != nil || !found {
return nil, err
}
return base64.StdEncoding.DecodeString(out)
}
// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
// It may use a remote (= slow) service.
// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil,
// as the primary manifest can not be a list, so there can be non-default instances.
func (s *ostreeImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
if instanceDigest != nil {
return nil, "", errors.New(`Manifest lists are not supported by "ostree:"`)
}
if s.repo == nil {
repo, err := openRepo(s.ref.repo)
if err != nil {
return nil, "", err
}
s.repo = repo
}
b := fmt.Sprintf("ociimage/%s", s.ref.branchName)
found, out, err := readMetadata(s.repo, b, "docker.manifest")
if err != nil {
return nil, "", err
}
if !found {
return nil, "", errors.New("manifest not found")
}
m := []byte(out)
return m, manifest.GuessMIMEType(m), nil
}
func (s *ostreeImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) {
return nil, "", errors.New("manifest lists are not supported by this transport")
}
func openRepo(path string) (*C.struct_OstreeRepo, error) {
var cerr *C.GError
cpath := C.CString(path)
defer C.free(unsafe.Pointer(cpath))
pathc := C.g_file_new_for_path(cpath)
defer C.g_object_unref(C.gpointer(pathc))
repo := C.ostree_repo_new(pathc)
r := glib.GoBool(glib.GBoolean(C.ostree_repo_open(repo, nil, &cerr)))
if !r {
C.g_object_unref(C.gpointer(repo))
return nil, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
}
return repo, nil
}
type ostreePathFileGetter struct {
repo *C.struct_OstreeRepo
parentRoot *C.GFile
}
type ostreeReader struct {
stream *C.GFileInputStream
}
func (o ostreeReader) Close() error {
C.g_object_unref(C.gpointer(o.stream))
return nil
}
func (o ostreeReader) Read(p []byte) (int, error) {
var cerr *C.GError
instanceCast := C.g_type_check_instance_cast((*C.GTypeInstance)(unsafe.Pointer(o.stream)), C.g_input_stream_get_type())
stream := (*C.GInputStream)(unsafe.Pointer(instanceCast))
b := C.g_input_stream_read_bytes(stream, (C.gsize)(cap(p)), nil, &cerr)
if b == nil {
return 0, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
}
defer C.g_bytes_unref(b)
count := int(C.g_bytes_get_size(b))
if count == 0 {
return 0, io.EOF
}
data := (*[1 << 30]byte)(unsafe.Pointer(C.g_bytes_get_data(b, nil)))[:count:count]
copy(p, data)
return count, nil
}
func readMetadata(repo *C.struct_OstreeRepo, commit, key string) (bool, string, error) {
var cerr *C.GError
var ref *C.char
defer C.free(unsafe.Pointer(ref))
cCommit := C.CString(commit)
defer C.free(unsafe.Pointer(cCommit))
if !glib.GoBool(glib.GBoolean(C.ostree_repo_resolve_rev(repo, cCommit, C.gboolean(1), &ref, &cerr))) {
return false, "", glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
}
if ref == nil {
return false, "", nil
}
var variant *C.GVariant
if !glib.GoBool(glib.GBoolean(C.ostree_repo_load_variant(repo, C.OSTREE_OBJECT_TYPE_COMMIT, ref, &variant, &cerr))) {
return false, "", glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
}
defer C.g_variant_unref(variant)
if variant != nil {
cKey := C.CString(key)
defer C.free(unsafe.Pointer(cKey))
metadata := C.g_variant_get_child_value(variant, 0)
defer C.g_variant_unref(metadata)
data := C.g_variant_lookup_value(metadata, (*C.gchar)(cKey), nil)
if data != nil {
defer C.g_variant_unref(data)
ptr := (*C.char)(C.g_variant_get_string(data, nil))
val := C.GoString(ptr)
return true, val, nil
}
}
return false, "", nil
}
func newOSTreePathFileGetter(repo *C.struct_OstreeRepo, commit string) (*ostreePathFileGetter, error) {
var cerr *C.GError
var parentRoot *C.GFile
cCommit := C.CString(commit)
defer C.free(unsafe.Pointer(cCommit))
if !glib.GoBool(glib.GBoolean(C.ostree_repo_read_commit(repo, cCommit, &parentRoot, nil, nil, &cerr))) {
return &ostreePathFileGetter{}, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
}
C.g_object_ref(C.gpointer(repo))
return &ostreePathFileGetter{repo: repo, parentRoot: parentRoot}, nil
}
func (o ostreePathFileGetter) Get(filename string) (io.ReadCloser, error) {
var file *C.GFile
if strings.HasPrefix(filename, "./") {
filename = filename[2:]
}
cfilename := C.CString(filename)
defer C.free(unsafe.Pointer(cfilename))
file = (*C.GFile)(C.g_file_resolve_relative_path(o.parentRoot, cfilename))
var cerr *C.GError
stream := C.g_file_read(file, nil, &cerr)
if stream == nil {
return nil, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
}
return &ostreeReader{stream: stream}, nil
}
func (o ostreePathFileGetter) Close() {
C.g_object_unref(C.gpointer(o.repo))
C.g_object_unref(C.gpointer(o.parentRoot))
}
func (s *ostreeImageSource) readSingleFile(commit, path string) (io.ReadCloser, error) {
getter, err := newOSTreePathFileGetter(s.repo, commit)
if err != nil {
return nil, err
}
defer getter.Close()
return getter.Get(path)
}
// GetBlob returns a stream for the specified blob, and the blobs size (or -1 if unknown).
// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
func (s *ostreeImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
blob := info.Digest.Hex()
// Ensure s.compressed is initialized. It is build by LayerInfosForCopy.
if s.compressed == nil {
_, err := s.LayerInfosForCopy(ctx, nil)
if err != nil {
return nil, -1, err
}
}
compressedBlob, isCompressed := s.compressed[info.Digest]
if isCompressed {
blob = compressedBlob.Hex()
}
branch := fmt.Sprintf("ociimage/%s", blob)
if s.repo == nil {
repo, err := openRepo(s.ref.repo)
if err != nil {
return nil, 0, err
}
s.repo = repo
}
layerSize, err := s.getBlobUncompressedSize(blob, isCompressed)
if err != nil {
return nil, 0, err
}
tarsplit, err := s.getTarSplitData(blob)
if err != nil {
return nil, 0, err
}
// if tarsplit is nil we are looking at the manifest. Return directly the file in /content
if tarsplit == nil {
file, err := s.readSingleFile(branch, "/content")
if err != nil {
return nil, 0, err
}
return file, layerSize, nil
}
mf := bytes.NewReader(tarsplit)
mfz, err := pgzip.NewReader(mf)
if err != nil {
return nil, 0, err
}
metaUnpacker := storage.NewJSONUnpacker(mfz)
getter, err := newOSTreePathFileGetter(s.repo, branch)
if err != nil {
mfz.Close()
return nil, 0, err
}
ots := asm.NewOutputTarStream(getter, metaUnpacker)
rc := ioutils.NewReadCloserWrapper(ots, func() error {
getter.Close()
mfz.Close()
return ots.Close()
})
return rc, layerSize, nil
}
// GetSignaturesWithFormat returns the image's signatures. It may use a remote (= slow) service.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for
// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list
// (e.g. if the source never returns manifest lists).
func (s *ostreeImageSource) GetSignaturesWithFormat(ctx context.Context, instanceDigest *digest.Digest) ([]signature.Signature, error) {
if instanceDigest != nil {
return nil, errors.New(`Manifest lists are not supported by "ostree:"`)
}
lenSignatures, err := s.getLenSignatures()
if err != nil {
return nil, err
}
branch := fmt.Sprintf("ociimage/%s", s.ref.branchName)
if s.repo == nil {
repo, err := openRepo(s.ref.repo)
if err != nil {
return nil, err
}
s.repo = repo
}
signatures := []signature.Signature{}
for i := int64(1); i <= lenSignatures; i++ {
path := fmt.Sprintf("/signature-%d", i)
sigReader, err := s.readSingleFile(branch, path)
if err != nil {
return nil, err
}
defer sigReader.Close()
sigBlob, err := io.ReadAll(sigReader)
if err != nil {
return nil, err
}
sig, err := signature.FromBlob(sigBlob)
if err != nil {
return nil, fmt.Errorf("parsing signature %q: %w", path, err)
}
signatures = append(signatures, sig)
}
return signatures, nil
}
// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer
// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob()
// to read the image's layers.
// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil,
// as the primary manifest can not be a list, so there can be secondary manifests.
// The Digest field is guaranteed to be provided; Size may be -1.
// WARNING: The list may contain duplicates, and they are semantically relevant.
func (s *ostreeImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) {
if instanceDigest != nil {
return nil, errors.New(`Manifest lists are not supported by "ostree:"`)
}
updatedBlobInfos := []types.BlobInfo{}
manifestBlob, manifestType, err := s.GetManifest(ctx, nil)
if err != nil {
return nil, err
}
man, err := manifest.FromBlob(manifestBlob, manifestType)
s.compressed = make(map[digest.Digest]digest.Digest)
layerBlobs := man.LayerInfos()
for _, layerBlob := range layerBlobs {
branch := fmt.Sprintf("ociimage/%s", layerBlob.Digest.Hex())
found, uncompressedDigestStr, err := readMetadata(s.repo, branch, "docker.uncompressed_digest")
if err != nil || !found {
return nil, err
}
found, uncompressedSizeStr, err := readMetadata(s.repo, branch, "docker.uncompressed_size")
if err != nil || !found {
return nil, err
}
uncompressedSize, err := strconv.ParseInt(uncompressedSizeStr, 10, 64)
if err != nil {
return nil, err
}
uncompressedDigest := digest.Digest(uncompressedDigestStr)
blobInfo := types.BlobInfo{
Digest: uncompressedDigest,
Size: uncompressedSize,
MediaType: layerBlob.MediaType,
}
s.compressed[uncompressedDigest] = layerBlob.Digest
updatedBlobInfos = append(updatedBlobInfos, blobInfo)
}
return updatedBlobInfos, nil
}

View file

@ -0,0 +1,242 @@
//go:build containers_image_ostree
// +build containers_image_ostree
package ostree
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/containers/image/v5/directory/explicitfilepath"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/image"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
"github.com/containers/storage/pkg/regexp"
)
const defaultOSTreeRepo = "/ostree/repo"
// Transport is an ImageTransport for ostree paths.
var Transport = ostreeTransport{}
type ostreeTransport struct{}
func (t ostreeTransport) Name() string {
return "ostree"
}
func init() {
transports.Register(Transport)
}
// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys
// (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value).
// It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion.
// scope passed to this function will not be "", that value is always allowed.
func (t ostreeTransport) ValidatePolicyConfigurationScope(scope string) error {
sep := strings.Index(scope, ":")
if sep < 0 {
return fmt.Errorf("Invalid ostree: scope %s: Must include a repo", scope)
}
repo := scope[:sep]
if !strings.HasPrefix(repo, "/") {
return fmt.Errorf("Invalid ostree: scope %s: repository must be an absolute path", scope)
}
cleaned := filepath.Clean(repo)
if cleaned != repo {
return fmt.Errorf(`Invalid ostree: scope %s: Uses non-canonical path format, perhaps try with path %s`, scope, cleaned)
}
// FIXME? In the namespaces within a repo,
// we could be verifying the various character set and length restrictions
// from docker/distribution/reference.regexp.go, but other than that there
// are few semantically invalid strings.
return nil
}
// ostreeReference is an ImageReference for ostree paths.
type ostreeReference struct {
image string
branchName string
repo string
}
type ostreeImageCloser struct {
types.ImageCloser
size int64
}
func (t ostreeTransport) ParseReference(ref string) (types.ImageReference, error) {
var repo = ""
image, repoPart, gotRepoPart := strings.Cut(ref, "@/")
if !gotRepoPart {
repo = defaultOSTreeRepo
} else {
repo = "/" + repoPart
}
return NewReference(image, repo)
}
// NewReference returns an OSTree reference for a specified repo and image.
func NewReference(image string, repo string) (types.ImageReference, error) {
// image is not _really_ in a containers/image/docker/reference format;
// as far as the libOSTree ociimage/* namespace is concerned, it is more or
// less an arbitrary string with an implied tag.
// Parse the image using reference.ParseNormalizedNamed so that we can
// check whether the images has a tag specified and we can add ":latest" if needed
ostreeImage, err := reference.ParseNormalizedNamed(image)
if err != nil {
return nil, err
}
if reference.IsNameOnly(ostreeImage) {
image = image + ":latest"
}
resolved, err := explicitfilepath.ResolvePathToFullyExplicit(repo)
if err != nil {
// With os.IsNotExist(err), the parent directory of repo is also not existent;
// that should ordinarily not happen, but it would be a bit weird to reject
// references which do not specify a repo just because the implicit defaultOSTreeRepo
// does not exist.
if os.IsNotExist(err) && repo == defaultOSTreeRepo {
resolved = repo
} else {
return nil, err
}
}
// This is necessary to prevent directory paths returned by PolicyConfigurationNamespaces
// from being ambiguous with values of PolicyConfigurationIdentity.
if strings.Contains(resolved, ":") {
return nil, fmt.Errorf("Invalid OSTree reference %s@%s: path %s contains a colon", image, repo, resolved)
}
return ostreeReference{
image: image,
branchName: encodeOStreeRef(image),
repo: resolved,
}, nil
}
func (ref ostreeReference) Transport() types.ImageTransport {
return Transport
}
// StringWithinTransport returns a string representation of the reference, which MUST be such that
// reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference.
// NOTE: The returned string is not promised to be equal to the original input to ParseReference;
// e.g. default attribute values omitted by the user may be filled in the return value, or vice versa.
// WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix.
func (ref ostreeReference) StringWithinTransport() string {
return fmt.Sprintf("%s@%s", ref.image, ref.repo)
}
// DockerReference returns a Docker reference associated with this reference
// (fully explicit, i.e. !reference.IsNameOnly, but reflecting user intent,
// not e.g. after redirect or alias processing), or nil if unknown/not applicable.
func (ref ostreeReference) DockerReference() reference.Named {
return nil
}
func (ref ostreeReference) PolicyConfigurationIdentity() string {
return fmt.Sprintf("%s:%s", ref.repo, ref.image)
}
// PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search
// for if explicit configuration for PolicyConfigurationIdentity() is not set. The list will be processed
// in order, terminating on first match, and an implicit "" is always checked at the end.
// It is STRONGLY recommended for the first element, if any, to be a prefix of PolicyConfigurationIdentity(),
// and each following element to be a prefix of the element preceding it.
func (ref ostreeReference) PolicyConfigurationNamespaces() []string {
repo, _, gotTag := strings.Cut(ref.image, ":")
if !gotTag { // Coverage: Should never happen, NewReference above ensures ref.image has a :tag.
panic(fmt.Sprintf("Internal inconsistency: ref.image value %q does not have a :tag", ref.image))
}
name := repo
res := []string{}
for {
res = append(res, fmt.Sprintf("%s:%s", ref.repo, name))
lastSlash := strings.LastIndex(name, "/")
if lastSlash == -1 {
break
}
name = name[:lastSlash]
}
return res
}
func (s *ostreeImageCloser) Size() (int64, error) {
return s.size, nil
}
// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport.
// The caller must call .Close() on the returned ImageCloser.
// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource,
// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
func (ref ostreeReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) {
return image.FromReference(ctx, sys, ref)
}
// NewImageSource returns a types.ImageSource for this reference.
// The caller must call .Close() on the returned ImageSource.
func (ref ostreeReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) {
var tmpDir string
if sys == nil || sys.OSTreeTmpDirPath == "" {
tmpDir = os.TempDir()
} else {
tmpDir = sys.OSTreeTmpDirPath
}
return newImageSource(tmpDir, ref)
}
// NewImageDestination returns a types.ImageDestination for this reference.
// The caller must call .Close() on the returned ImageDestination.
func (ref ostreeReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) {
var tmpDir string
if sys == nil || sys.OSTreeTmpDirPath == "" {
tmpDir = os.TempDir()
} else {
tmpDir = sys.OSTreeTmpDirPath
}
return newImageDestination(ref, tmpDir)
}
// DeleteImage deletes the named image from the registry, if supported.
func (ref ostreeReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error {
return errors.New("Deleting images not implemented for ostree: images")
}
var ostreeRefRegexp = regexp.Delayed(`^[A-Za-z0-9.-]$`)
func encodeOStreeRef(in string) string {
var buffer bytes.Buffer
for i := range in {
sub := in[i : i+1]
if ostreeRefRegexp.MatchString(sub) {
buffer.WriteString(sub)
} else {
buffer.WriteString(fmt.Sprintf("_%02X", sub[0]))
}
}
return buffer.String()
}
// manifestPath returns a path for the manifest within a ostree using our conventions.
func (ref ostreeReference) manifestPath() string {
return filepath.Join("manifest", "manifest.json")
}
// signaturePath returns a path for a signature within a ostree using our conventions.
func (ref ostreeReference) signaturePath(index int) string {
return filepath.Join("manifest", fmt.Sprintf("signature-%d", index+1))
}

210
vendor/github.com/containers/image/v5/sif/load.go generated vendored Normal file
View file

@ -0,0 +1,210 @@
package sif
import (
"bufio"
"context"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/sirupsen/logrus"
"github.com/sylabs/sif/v2/pkg/sif"
)
// injectedScriptTargetPath is the path injectedScript should be written to in the created image.
const injectedScriptTargetPath = "/podman/runscript"
// parseDefFile parses a SIF definition file from reader,
// and returns non-trivial contents of the %environment and %runscript sections.
func parseDefFile(reader io.Reader) ([]string, []string, error) {
type parserState int
const (
parsingOther parserState = iota
parsingEnvironment
parsingRunscript
)
environment := []string{}
runscript := []string{}
state := parsingOther
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
s := strings.TrimSpace(scanner.Text())
switch {
case s == `%environment`:
state = parsingEnvironment
case s == `%runscript`:
state = parsingRunscript
case strings.HasPrefix(s, "%"):
state = parsingOther
case state == parsingEnvironment:
if s != "" && !strings.HasPrefix(s, "#") {
environment = append(environment, s)
}
case state == parsingRunscript:
runscript = append(runscript, s)
default: // parsingOther: ignore the line
}
}
if err := scanner.Err(); err != nil {
return nil, nil, fmt.Errorf("reading lines from SIF definition file object: %w", err)
}
return environment, runscript, nil
}
// generateInjectedScript generates a shell script based on
// SIF definition file %environment and %runscript data, and returns it.
func generateInjectedScript(environment []string, runscript []string) []byte {
script := fmt.Sprintf("#!/bin/bash\n"+
"%s\n"+
"%s\n", strings.Join(environment, "\n"), strings.Join(runscript, "\n"))
return []byte(script)
}
// processDefFile finds sif.DataDeffile in sifImage, if any,
// and returns:
// - the command to run
// - contents of a script to inject as injectedScriptTargetPath, or nil
func processDefFile(sifImage *sif.FileImage) (string, []byte, error) {
var environment, runscript []string
desc, err := sifImage.GetDescriptor(sif.WithDataType(sif.DataDeffile))
if err == nil {
environment, runscript, err = parseDefFile(desc.GetReader())
if err != nil {
return "", nil, err
}
}
var command string
var injectedScript []byte
if len(environment) == 0 && len(runscript) == 0 {
command = "bash"
injectedScript = nil
} else {
injectedScript = generateInjectedScript(environment, runscript)
command = injectedScriptTargetPath
}
return command, injectedScript, nil
}
func writeInjectedScript(extractedRootPath string, injectedScript []byte) error {
if injectedScript == nil {
return nil
}
filePath := filepath.Join(extractedRootPath, injectedScriptTargetPath)
parentDirPath := filepath.Dir(filePath)
if err := os.MkdirAll(parentDirPath, 0755); err != nil {
return fmt.Errorf("creating %s: %w", parentDirPath, err)
}
if err := os.WriteFile(filePath, injectedScript, 0755); err != nil {
return fmt.Errorf("writing %s to %s: %w", injectedScriptTargetPath, filePath, err)
}
return nil
}
// createTarFromSIFInputs creates a tar file at tarPath, using a squashfs image at squashFSPath.
// It can also use extractedRootPath and scriptPath, which are allocated for its exclusive use,
// if necessary.
func createTarFromSIFInputs(ctx context.Context, tarPath, squashFSPath string, injectedScript []byte, extractedRootPath, scriptPath string) error {
// It's safe for the Remove calls to happen even before we create the files, because tempDir is exclusive
// for our use.
defer os.RemoveAll(extractedRootPath)
// Almost everything in extractedRootPath comes from squashFSPath.
conversionCommand := fmt.Sprintf("unsquashfs -d %s -f %s && tar --acls --xattrs -C %s -cpf %s ./",
extractedRootPath, squashFSPath, extractedRootPath, tarPath)
script := "#!/bin/sh\n" + conversionCommand + "\n"
if err := os.WriteFile(scriptPath, []byte(script), 0755); err != nil {
return err
}
defer os.Remove(scriptPath)
// On top of squashFSPath, we only add injectedScript, if necessary.
if err := writeInjectedScript(extractedRootPath, injectedScript); err != nil {
return err
}
logrus.Debugf("Converting squashfs to tar, command: %s ...", conversionCommand)
cmd := exec.CommandContext(ctx, "fakeroot", "--", scriptPath)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("converting image: %w, output: %s", err, string(output))
}
logrus.Debugf("... finished converting squashfs to tar")
return nil
}
// convertSIFToElements processes sifImage and creates/returns
// the relevant elements for constructing an OCI-like image:
// - A path to a tar file containing a root filesystem,
// - A command to run.
// The returned tar file path is inside tempDir, which can be assumed to be empty
// at start, and is exclusively used by the current process (i.e. it is safe
// to use hard-coded relative paths within it).
func convertSIFToElements(ctx context.Context, sifImage *sif.FileImage, tempDir string) (string, []string, error) {
// We could allocate unique names for all of these using os.{CreateTemp,MkdirTemp}, but tempDir is exclusive,
// so we can just hard-code a set of unique values here.
// We create and/or manage cleanup of these two paths.
squashFSPath := filepath.Join(tempDir, "rootfs.squashfs")
tarPath := filepath.Join(tempDir, "rootfs.tar")
// We only allocate these paths, the user is responsible for cleaning them up.
extractedRootPath := filepath.Join(tempDir, "rootfs")
scriptPath := filepath.Join(tempDir, "script")
succeeded := false
// It's safe for the Remove calls to happen even before we create the files, because tempDir is exclusive
// for our use.
// Ideally we would remove squashFSPath immediately after creating extractedRootPath, but we need
// to run both creation and consumption of extractedRootPath in the same fakeroot context.
// So, overall, this process requires at least 2 compressed copies (SIF and squashFSPath) and 2
// uncompressed copies (extractedRootPath and tarPath) of the data, all using up space at the same time.
// That's rather unsatisfactory, ideally we would be streaming the data directly from a squashfs parser
// reading from the SIF file to a tarball, for 1 compressed and 1 uncompressed copy.
defer os.Remove(squashFSPath)
defer func() {
if !succeeded {
os.Remove(tarPath)
}
}()
command, injectedScript, err := processDefFile(sifImage)
if err != nil {
return "", nil, err
}
rootFS, err := sifImage.GetDescriptor(sif.WithPartitionType(sif.PartPrimSys))
if err != nil {
return "", nil, fmt.Errorf("looking up rootfs from SIF file: %w", err)
}
// TODO: We'd prefer not to make a full copy of the file here; unsquashfs ≥ 4.4
// has an -o option that allows extracting a squashfs from the SIF file directly,
// but that version is not currently available in RHEL 8.
logrus.Debugf("Creating a temporary squashfs image %s ...", squashFSPath)
if err := func() error { // A scope for defer
f, err := os.Create(squashFSPath)
if err != nil {
return err
}
defer f.Close()
// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
if _, err := io.CopyN(f, rootFS.GetReader(), rootFS.Size()); err != nil {
return err
}
return nil
}(); err != nil {
return "", nil, err
}
logrus.Debugf("... finished creating a temporary squashfs image")
if err := createTarFromSIFInputs(ctx, tarPath, squashFSPath, injectedScript, extractedRootPath, scriptPath); err != nil {
return "", nil, err
}
succeeded = true
return tarPath, []string{command}, nil
}

206
vendor/github.com/containers/image/v5/sif/src.go generated vendored Normal file
View file

@ -0,0 +1,206 @@
package sif
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"github.com/containers/image/v5/internal/imagesource/impl"
"github.com/containers/image/v5/internal/imagesource/stubs"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/internal/tmpdir"
"github.com/containers/image/v5/types"
"github.com/opencontainers/go-digest"
imgspecs "github.com/opencontainers/image-spec/specs-go"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"github.com/sylabs/sif/v2/pkg/sif"
)
type sifImageSource struct {
impl.Compat
impl.PropertyMethodsInitialize
impl.NoSignatures
impl.DoesNotAffectLayerInfosForCopy
stubs.NoGetBlobAtInitialize
ref sifReference
workDir string
layerDigest digest.Digest
layerSize int64
layerFile string
config []byte
configDigest digest.Digest
manifest []byte
}
// getBlobInfo returns the digest, and size of the provided file.
func getBlobInfo(path string) (digest.Digest, int64, error) {
f, err := os.Open(path)
if err != nil {
return "", -1, fmt.Errorf("opening %q for reading: %w", path, err)
}
defer f.Close()
// TODO: Instead of writing the tar file to disk, and reading
// it here again, stream the tar file to a pipe and
// compute the digest while writing it to disk.
logrus.Debugf("Computing a digest of the SIF conversion output...")
digester := digest.Canonical.Digester()
// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
size, err := io.Copy(digester.Hash(), f)
if err != nil {
return "", -1, fmt.Errorf("reading %q: %w", path, err)
}
digest := digester.Digest()
logrus.Debugf("... finished computing the digest of the SIF conversion output")
return digest, size, nil
}
// newImageSource returns an ImageSource for reading from an existing directory.
// newImageSource extracts SIF objects and saves them in a temp directory.
func newImageSource(ctx context.Context, sys *types.SystemContext, ref sifReference) (private.ImageSource, error) {
sifImg, err := sif.LoadContainerFromPath(ref.file, sif.OptLoadWithFlag(os.O_RDONLY))
if err != nil {
return nil, fmt.Errorf("loading SIF file: %w", err)
}
defer func() {
_ = sifImg.UnloadContainer()
}()
workDir, err := tmpdir.MkDirBigFileTemp(sys, "sif")
if err != nil {
return nil, fmt.Errorf("creating temp directory: %w", err)
}
succeeded := false
defer func() {
if !succeeded {
os.RemoveAll(workDir)
}
}()
layerPath, commandLine, err := convertSIFToElements(ctx, sifImg, workDir)
if err != nil {
return nil, fmt.Errorf("converting rootfs from SquashFS to Tarball: %w", err)
}
layerDigest, layerSize, err := getBlobInfo(layerPath)
if err != nil {
return nil, fmt.Errorf("gathering blob information: %w", err)
}
created := sifImg.ModifiedAt()
config := imgspecv1.Image{
Created: &created,
Platform: imgspecv1.Platform{
Architecture: sifImg.PrimaryArch(),
OS: "linux",
},
Config: imgspecv1.ImageConfig{
Cmd: commandLine,
},
RootFS: imgspecv1.RootFS{
Type: "layers",
DiffIDs: []digest.Digest{layerDigest},
},
History: []imgspecv1.History{
{
Created: &created,
CreatedBy: fmt.Sprintf("/bin/sh -c #(nop) ADD file:%s in %c", layerDigest.Hex(), os.PathSeparator),
Comment: "imported from SIF, uuid: " + sifImg.ID(),
},
{
Created: &created,
CreatedBy: "/bin/sh -c #(nop) CMD [\"bash\"]",
EmptyLayer: true,
},
},
}
configBytes, err := json.Marshal(&config)
if err != nil {
return nil, fmt.Errorf("generating configuration blob for %q: %w", ref.resolvedFile, err)
}
configDigest := digest.Canonical.FromBytes(configBytes)
manifest := imgspecv1.Manifest{
Versioned: imgspecs.Versioned{SchemaVersion: 2},
MediaType: imgspecv1.MediaTypeImageManifest,
Config: imgspecv1.Descriptor{
Digest: configDigest,
Size: int64(len(configBytes)),
MediaType: imgspecv1.MediaTypeImageConfig,
},
Layers: []imgspecv1.Descriptor{{
Digest: layerDigest,
Size: layerSize,
MediaType: imgspecv1.MediaTypeImageLayer,
}},
}
manifestBytes, err := json.Marshal(&manifest)
if err != nil {
return nil, fmt.Errorf("generating manifest for %q: %w", ref.resolvedFile, err)
}
succeeded = true
s := &sifImageSource{
PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{
HasThreadSafeGetBlob: true,
}),
NoGetBlobAtInitialize: stubs.NoGetBlobAt(ref),
ref: ref,
workDir: workDir,
layerDigest: layerDigest,
layerSize: layerSize,
layerFile: layerPath,
config: configBytes,
configDigest: configDigest,
manifest: manifestBytes,
}
s.Compat = impl.AddCompat(s)
return s, nil
}
// Reference returns the reference used to set up this source.
func (s *sifImageSource) Reference() types.ImageReference {
return s.ref
}
// Close removes resources associated with an initialized ImageSource, if any.
func (s *sifImageSource) Close() error {
return os.RemoveAll(s.workDir)
}
// GetBlob returns a stream for the specified blob, and the blobs size (or -1 if unknown).
// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
func (s *sifImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
switch info.Digest {
case s.configDigest:
return io.NopCloser(bytes.NewReader(s.config)), int64(len(s.config)), nil
case s.layerDigest:
reader, err := os.Open(s.layerFile)
if err != nil {
return nil, -1, fmt.Errorf("opening %q: %w", s.layerFile, err)
}
return reader, s.layerSize, nil
default:
return nil, -1, fmt.Errorf("no blob with digest %q found", info.Digest.String())
}
}
// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
// It may use a remote (= slow) service.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list);
// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists).
func (s *sifImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
if instanceDigest != nil {
return nil, "", errors.New("manifest lists are not supported by the sif transport")
}
return s.manifest, imgspecv1.MediaTypeImageManifest, nil
}

160
vendor/github.com/containers/image/v5/sif/transport.go generated vendored Normal file
View file

@ -0,0 +1,160 @@
package sif
import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"github.com/containers/image/v5/directory/explicitfilepath"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/image"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
)
func init() {
transports.Register(Transport)
}
// Transport is an ImageTransport for SIF images.
var Transport = sifTransport{}
type sifTransport struct{}
func (t sifTransport) Name() string {
return "sif"
}
// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an ImageReference.
func (t sifTransport) ParseReference(reference string) (types.ImageReference, error) {
return NewReference(reference)
}
// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys
// (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value).
// It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion.
// scope passed to this function will not be "", that value is always allowed.
func (t sifTransport) ValidatePolicyConfigurationScope(scope string) error {
if !strings.HasPrefix(scope, "/") {
return fmt.Errorf("Invalid scope %s: Must be an absolute path", scope)
}
// Refuse also "/", otherwise "/" and "" would have the same semantics,
// and "" could be unexpectedly shadowed by the "/" entry.
if scope == "/" {
return errors.New(`Invalid scope "/": Use the generic default scope ""`)
}
cleaned := filepath.Clean(scope)
if cleaned != scope {
return fmt.Errorf(`Invalid scope %s: Uses non-canonical format, perhaps try %s`, scope, cleaned)
}
return nil
}
// sifReference is an ImageReference for SIF images.
type sifReference struct {
// Note that the interpretation of paths below depends on the underlying filesystem state, which may change under us at any time!
// Either of the paths may point to a different, or no, inode over time. resolvedFile may contain symbolic links, and so on.
// Generally we follow the intent of the user, and use the "file" member for filesystem operations (e.g. the user can use a relative path to avoid
// being exposed to symlinks and renames in the parent directories to the working directory).
// (But in general, we make no attempt to be completely safe against concurrent hostile filesystem modifications.)
file string // As specified by the user. May be relative, contain symlinks, etc.
resolvedFile string // Absolute file path with no symlinks, at least at the time of its creation. Primarily used for policy namespaces.
}
// There is no sif.ParseReference because it is rather pointless.
// Callers who need a transport-independent interface will go through
// sifTransport.ParseReference; callers who intentionally deal with SIF files
// can use sif.NewReference.
// NewReference returns an image file reference for a specified path.
func NewReference(file string) (types.ImageReference, error) {
// We do not expose an API supplying the resolvedFile; we could, but recomputing it
// is generally cheap enough that we prefer being confident about the properties of resolvedFile.
resolved, err := explicitfilepath.ResolvePathToFullyExplicit(file)
if err != nil {
return nil, err
}
return sifReference{file: file, resolvedFile: resolved}, nil
}
func (ref sifReference) Transport() types.ImageTransport {
return Transport
}
// StringWithinTransport returns a string representation of the reference, which MUST be such that
// reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference.
// NOTE: The returned string is not promised to be equal to the original input to ParseReference;
// e.g. default attribute values omitted by the user may be filled in the return value, or vice versa.
// WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix;
// instead, see transports.ImageName().
func (ref sifReference) StringWithinTransport() string {
return ref.file
}
// DockerReference returns a Docker reference associated with this reference
// (fully explicit, i.e. !reference.IsNameOnly, but reflecting user intent,
// not e.g. after redirect or alias processing), or nil if unknown/not applicable.
func (ref sifReference) DockerReference() reference.Named {
return nil
}
// PolicyConfigurationIdentity returns a string representation of the reference, suitable for policy lookup.
// This MUST reflect user intent, not e.g. after processing of third-party redirects or aliases;
// The value SHOULD be fully explicit about its semantics, with no hidden defaults, AND canonical
// (i.e. various references with exactly the same semantics should return the same configuration identity)
// It is fine for the return value to be equal to StringWithinTransport(), and it is desirable but
// not required/guaranteed that it will be a valid input to Transport().ParseReference().
// Returns "" if configuration identities for these references are not supported.
func (ref sifReference) PolicyConfigurationIdentity() string {
return ref.resolvedFile
}
// PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search
// for if explicit configuration for PolicyConfigurationIdentity() is not set. The list will be processed
// in order, terminating on first match, and an implicit "" is always checked at the end.
// It is STRONGLY recommended for the first element, if any, to be a prefix of PolicyConfigurationIdentity(),
// and each following element to be a prefix of the element preceding it.
func (ref sifReference) PolicyConfigurationNamespaces() []string {
res := []string{}
path := ref.resolvedFile
for {
lastSlash := strings.LastIndex(path, "/")
if lastSlash == -1 || lastSlash == 0 {
break
}
path = path[:lastSlash]
res = append(res, path)
}
// Note that we do not include "/"; it is redundant with the default "" global default,
// and rejected by sifTransport.ValidatePolicyConfigurationScope above.
return res
}
// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport.
// The caller must call .Close() on the returned ImageCloser.
// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource,
// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details.
func (ref sifReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) {
return image.FromReference(ctx, sys, ref)
}
// NewImageSource returns a types.ImageSource for this reference.
// The caller must call .Close() on the returned ImageSource.
func (ref sifReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) {
return newImageSource(ctx, sys, ref)
}
// NewImageDestination returns a types.ImageDestination for this reference.
// The caller must call .Close() on the returned ImageDestination.
func (ref sifReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) {
return nil, errors.New(`"sif:" locations can only be read from, not written to`)
}
// DeleteImage deletes the named image from the registry, if supported.
func (ref sifReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error {
return errors.New("Deleting images not implemented for sif: images")
}

View file

@ -0,0 +1,958 @@
//go:build !containers_image_storage_stub
// +build !containers_image_storage_stub
package storage
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sync"
"sync/atomic"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/blobinfocache"
"github.com/containers/image/v5/internal/imagedestination/impl"
"github.com/containers/image/v5/internal/imagedestination/stubs"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/internal/putblobdigest"
"github.com/containers/image/v5/internal/set"
"github.com/containers/image/v5/internal/signature"
"github.com/containers/image/v5/internal/tmpdir"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/blobinfocache/none"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
graphdriver "github.com/containers/storage/drivers"
"github.com/containers/storage/pkg/archive"
"github.com/containers/storage/pkg/chunked"
"github.com/containers/storage/pkg/ioutils"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
)
var (
// ErrBlobDigestMismatch could potentially be returned when PutBlob() is given a blob
// with a digest-based name that doesn't match its contents.
// Deprecated: PutBlob() doesn't do this any more (it just accepts the callers value),
// and there is no known user of this error.
ErrBlobDigestMismatch = errors.New("blob digest mismatch")
// ErrBlobSizeMismatch is returned when PutBlob() is given a blob
// with an expected size that doesn't match the reader.
ErrBlobSizeMismatch = errors.New("blob size mismatch")
)
type storageImageDestination struct {
impl.Compat
impl.PropertyMethodsInitialize
stubs.ImplementsPutBlobPartial
stubs.AlwaysSupportsSignatures
imageRef storageReference
directory string // Temporary directory where we store blobs until Commit() time
nextTempFileID atomic.Int32 // A counter that we use for computing filenames to assign to blobs
manifest []byte // Manifest contents, temporary
manifestDigest digest.Digest // Valid if len(manifest) != 0
signatures []byte // Signature contents, temporary
signatureses map[digest.Digest][]byte // Instance signature contents, temporary
SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice
SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // Sizes of each manifest's signature slice
// A storage destination may be used concurrently. Accesses are
// serialized via a mutex. Please refer to the individual comments
// below for details.
lock sync.Mutex
// Mapping from layer (by index) to the associated ID in the storage.
// It's protected *implicitly* since `commitLayer()`, at any given
// time, can only be executed by *one* goroutine. Please refer to
// `queueOrCommit()` for further details on how the single-caller
// guarantee is implemented.
indexToStorageID map[int]*string
// All accesses to below data are protected by `lock` which is made
// *explicit* in the code.
blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs
fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes
filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them
currentIndex int // The index of the layer to be committed (i.e., lower indices have already been committed)
indexToAddedLayerInfo map[int]addedLayerInfo // Mapping from layer (by index) to blob to add to the image
blobAdditionalLayer map[digest.Digest]storage.AdditionalLayer // Mapping from layer blobsums to their corresponding additional layer
diffOutputs map[digest.Digest]*graphdriver.DriverWithDifferOutput // Mapping from digest to differ output
}
// addedLayerInfo records data about a layer to use in this image.
type addedLayerInfo struct {
digest digest.Digest
emptyLayer bool // The layer is an “empty”/“throwaway” one, and may or may not be physically represented in various transport / storage systems. false if the manifest type does not have the concept.
}
// newImageDestination sets us up to write a new image, caching blobs in a temporary directory until
// it's time to Commit() the image
func newImageDestination(sys *types.SystemContext, imageRef storageReference) (*storageImageDestination, error) {
directory, err := tmpdir.MkDirBigFileTemp(sys, "storage")
if err != nil {
return nil, fmt.Errorf("creating a temporary directory: %w", err)
}
dest := &storageImageDestination{
PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{
SupportedManifestMIMETypes: []string{
imgspecv1.MediaTypeImageManifest,
manifest.DockerV2Schema2MediaType,
manifest.DockerV2Schema1SignedMediaType,
manifest.DockerV2Schema1MediaType,
},
// We ultimately have to decompress layers to populate trees on disk
// and need to explicitly ask for it here, so that the layers' MIME
// types can be set accordingly.
DesiredLayerCompression: types.PreserveOriginal,
AcceptsForeignLayerURLs: false,
MustMatchRuntimeOS: true,
IgnoresEmbeddedDockerReference: true, // Yes, we want the unmodified manifest
HasThreadSafePutBlob: true,
}),
imageRef: imageRef,
directory: directory,
signatureses: make(map[digest.Digest][]byte),
blobDiffIDs: make(map[digest.Digest]digest.Digest),
blobAdditionalLayer: make(map[digest.Digest]storage.AdditionalLayer),
fileSizes: make(map[digest.Digest]int64),
filenames: make(map[digest.Digest]string),
SignatureSizes: []int{},
SignaturesSizes: make(map[digest.Digest][]int),
indexToStorageID: make(map[int]*string),
indexToAddedLayerInfo: make(map[int]addedLayerInfo),
diffOutputs: make(map[digest.Digest]*graphdriver.DriverWithDifferOutput),
}
dest.Compat = impl.AddCompat(dest)
return dest, nil
}
// Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent,
// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects.
func (s *storageImageDestination) Reference() types.ImageReference {
return s.imageRef
}
// Close cleans up the temporary directory and additional layer store handlers.
func (s *storageImageDestination) Close() error {
for _, al := range s.blobAdditionalLayer {
al.Release()
}
for _, v := range s.diffOutputs {
if v.Target != "" {
_ = s.imageRef.transport.store.CleanupStagingDirectory(v.Target)
}
}
return os.RemoveAll(s.directory)
}
func (s *storageImageDestination) computeNextBlobCacheFile() string {
return filepath.Join(s.directory, fmt.Sprintf("%d", s.nextTempFileID.Add(1)))
}
// PutBlobWithOptions writes contents of stream and returns data representing the result.
// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents.
// inputInfo.Size is the expected length of stream, if known.
// inputInfo.MediaType describes the blob format, if known.
// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
// to any other readers for download using the supplied digest.
// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
func (s *storageImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, options private.PutBlobOptions) (private.UploadedBlob, error) {
info, err := s.putBlobToPendingFile(stream, blobinfo, &options)
if err != nil {
return info, err
}
if options.IsConfig || options.LayerIndex == nil {
return info, nil
}
return info, s.queueOrCommit(*options.LayerIndex, addedLayerInfo{
digest: info.Digest,
emptyLayer: options.EmptyLayer,
})
}
// putBlobToPendingFile implements ImageDestination.PutBlobWithOptions, storing stream into an on-disk file.
// The caller must arrange the blob to be eventually committed using s.commitLayer().
func (s *storageImageDestination) putBlobToPendingFile(stream io.Reader, blobinfo types.BlobInfo, options *private.PutBlobOptions) (private.UploadedBlob, error) {
// Stores a layer or data blob in our temporary directory, checking that any information
// in the blobinfo matches the incoming data.
if blobinfo.Digest != "" {
if err := blobinfo.Digest.Validate(); err != nil {
return private.UploadedBlob{}, fmt.Errorf("invalid digest %#v: %w", blobinfo.Digest.String(), err)
}
}
// Set up to digest the blob if necessary, and count its size while saving it to a file.
filename := s.computeNextBlobCacheFile()
file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600)
if err != nil {
return private.UploadedBlob{}, fmt.Errorf("creating temporary file %q: %w", filename, err)
}
defer file.Close()
counter := ioutils.NewWriteCounter(file)
stream = io.TeeReader(stream, counter)
digester, stream := putblobdigest.DigestIfUnknown(stream, blobinfo)
decompressed, err := archive.DecompressStream(stream)
if err != nil {
return private.UploadedBlob{}, fmt.Errorf("setting up to decompress blob: %w", err)
}
diffID := digest.Canonical.Digester()
// Copy the data to the file.
// TODO: This can take quite some time, and should ideally be cancellable using context.Context.
_, err = io.Copy(diffID.Hash(), decompressed)
decompressed.Close()
if err != nil {
return private.UploadedBlob{}, fmt.Errorf("storing blob to file %q: %w", filename, err)
}
// Determine blob properties, and fail if information that we were given about the blob
// is known to be incorrect.
blobDigest := digester.Digest()
blobSize := blobinfo.Size
if blobSize < 0 {
blobSize = counter.Count
} else if blobinfo.Size != counter.Count {
return private.UploadedBlob{}, ErrBlobSizeMismatch
}
// Record information about the blob.
s.lock.Lock()
s.blobDiffIDs[blobDigest] = diffID.Digest()
s.fileSizes[blobDigest] = counter.Count
s.filenames[blobDigest] = filename
s.lock.Unlock()
// This is safe because we have just computed diffID, and blobDigest was either computed
// by us, or validated by the caller (usually copy.digestingReader).
options.Cache.RecordDigestUncompressedPair(blobDigest, diffID.Digest())
return private.UploadedBlob{
Digest: blobDigest,
Size: blobSize,
}, nil
}
type zstdFetcher struct {
chunkAccessor private.BlobChunkAccessor
ctx context.Context
blobInfo types.BlobInfo
}
// GetBlobAt converts from chunked.GetBlobAt to BlobChunkAccessor.GetBlobAt.
func (f *zstdFetcher) GetBlobAt(chunks []chunked.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
newChunks := make([]private.ImageSourceChunk, 0, len(chunks))
for _, v := range chunks {
i := private.ImageSourceChunk{
Offset: v.Offset,
Length: v.Length,
}
newChunks = append(newChunks, i)
}
rc, errs, err := f.chunkAccessor.GetBlobAt(f.ctx, f.blobInfo, newChunks)
if _, ok := err.(private.BadPartialRequestError); ok {
err = chunked.ErrBadRequest{}
}
return rc, errs, err
}
// PutBlobPartial attempts to create a blob using the data that is already present
// at the destination. chunkAccessor is accessed in a non-sequential way to retrieve the missing chunks.
// It is available only if SupportsPutBlobPartial().
// Even if SupportsPutBlobPartial() returns true, the call can fail, in which case the caller
// should fall back to PutBlobWithOptions.
func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, cache blobinfocache.BlobInfoCache2) (private.UploadedBlob, error) {
fetcher := zstdFetcher{
chunkAccessor: chunkAccessor,
ctx: ctx,
blobInfo: srcInfo,
}
differ, err := chunked.GetDiffer(ctx, s.imageRef.transport.store, srcInfo.Size, srcInfo.Annotations, &fetcher)
if err != nil {
return private.UploadedBlob{}, err
}
out, err := s.imageRef.transport.store.ApplyDiffWithDiffer("", nil, differ)
if err != nil {
return private.UploadedBlob{}, err
}
blobDigest := srcInfo.Digest
s.lock.Lock()
s.blobDiffIDs[blobDigest] = blobDigest
s.fileSizes[blobDigest] = 0
s.filenames[blobDigest] = ""
s.diffOutputs[blobDigest] = out
s.lock.Unlock()
return private.UploadedBlob{
Digest: blobDigest,
Size: srcInfo.Size,
}, nil
}
// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree).
// info.Digest must not be empty.
// If the blob has been successfully reused, returns (true, info, nil).
// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context, blobinfo types.BlobInfo, options private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) {
if !impl.OriginalBlobMatchesRequiredCompression(options) {
return false, private.ReusedBlob{}, nil
}
reused, info, err := s.tryReusingBlobAsPending(blobinfo.Digest, blobinfo.Size, &options)
if err != nil || !reused || options.LayerIndex == nil {
return reused, info, err
}
return reused, info, s.queueOrCommit(*options.LayerIndex, addedLayerInfo{
digest: info.Digest,
emptyLayer: options.EmptyLayer,
})
}
// tryReusingBlobAsPending implements TryReusingBlobWithOptions for (digest, size or -1), filling s.blobDiffIDs and other metadata.
// The caller must arrange the blob to be eventually committed using s.commitLayer().
func (s *storageImageDestination) tryReusingBlobAsPending(digest digest.Digest, size int64, options *private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) {
// lock the entire method as it executes fairly quickly
s.lock.Lock()
defer s.lock.Unlock()
if options.SrcRef != nil {
// Check if we have the layer in the underlying additional layer store.
aLayer, err := s.imageRef.transport.store.LookupAdditionalLayer(digest, options.SrcRef.String())
if err != nil && !errors.Is(err, storage.ErrLayerUnknown) {
return false, private.ReusedBlob{}, fmt.Errorf(`looking for compressed layers with digest %q and labels: %w`, digest, err)
} else if err == nil {
// Record the uncompressed value so that we can use it to calculate layer IDs.
s.blobDiffIDs[digest] = aLayer.UncompressedDigest()
s.blobAdditionalLayer[digest] = aLayer
return true, private.ReusedBlob{
Digest: digest,
Size: aLayer.CompressedSize(),
}, nil
}
}
if digest == "" {
return false, private.ReusedBlob{}, errors.New(`Can not check for a blob with unknown digest`)
}
if err := digest.Validate(); err != nil {
return false, private.ReusedBlob{}, fmt.Errorf("Can not check for a blob with invalid digest: %w", err)
}
// Check if we've already cached it in a file.
if size, ok := s.fileSizes[digest]; ok {
return true, private.ReusedBlob{
Digest: digest,
Size: size,
}, nil
}
// Check if we have a wasn't-compressed layer in storage that's based on that blob.
layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(digest)
if err != nil && !errors.Is(err, storage.ErrLayerUnknown) {
return false, private.ReusedBlob{}, fmt.Errorf(`looking for layers with digest %q: %w`, digest, err)
}
if len(layers) > 0 {
// Save this for completeness.
s.blobDiffIDs[digest] = layers[0].UncompressedDigest
return true, private.ReusedBlob{
Digest: digest,
Size: layers[0].UncompressedSize,
}, nil
}
// Check if we have a was-compressed layer in storage that's based on that blob.
layers, err = s.imageRef.transport.store.LayersByCompressedDigest(digest)
if err != nil && !errors.Is(err, storage.ErrLayerUnknown) {
return false, private.ReusedBlob{}, fmt.Errorf(`looking for compressed layers with digest %q: %w`, digest, err)
}
if len(layers) > 0 {
// Record the uncompressed value so that we can use it to calculate layer IDs.
s.blobDiffIDs[digest] = layers[0].UncompressedDigest
return true, private.ReusedBlob{
Digest: digest,
Size: layers[0].CompressedSize,
}, nil
}
// Does the blob correspond to a known DiffID which we already have available?
// Because we must return the size, which is unknown for unavailable compressed blobs, the returned BlobInfo refers to the
// uncompressed layer, and that can happen only if options.CanSubstitute, or if the incoming manifest already specifies the size.
if options.CanSubstitute || size != -1 {
if uncompressedDigest := options.Cache.UncompressedDigest(digest); uncompressedDigest != "" && uncompressedDigest != digest {
layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(uncompressedDigest)
if err != nil && !errors.Is(err, storage.ErrLayerUnknown) {
return false, private.ReusedBlob{}, fmt.Errorf(`looking for layers with digest %q: %w`, uncompressedDigest, err)
}
if len(layers) > 0 {
if size != -1 {
s.blobDiffIDs[digest] = layers[0].UncompressedDigest
return true, private.ReusedBlob{
Digest: digest,
Size: size,
}, nil
}
if !options.CanSubstitute {
return false, private.ReusedBlob{}, fmt.Errorf("Internal error: options.CanSubstitute was expected to be true for blob with digest %s", digest)
}
s.blobDiffIDs[uncompressedDigest] = layers[0].UncompressedDigest
return true, private.ReusedBlob{
Digest: uncompressedDigest,
Size: layers[0].UncompressedSize,
}, nil
}
}
}
// Nope, we don't have it.
return false, private.ReusedBlob{}, nil
}
// computeID computes a recommended image ID based on information we have so far. If
// the manifest is not of a type that we recognize, we return an empty value, indicating
// that since we don't have a recommendation, a random ID should be used if one needs
// to be allocated.
func (s *storageImageDestination) computeID(m manifest.Manifest) string {
// Build the diffID list. We need the decompressed sums that we've been calculating to
// fill in the DiffIDs. It's expected (but not enforced by us) that the number of
// diffIDs corresponds to the number of non-EmptyLayer entries in the history.
var diffIDs []digest.Digest
switch m := m.(type) {
case *manifest.Schema1:
// Build a list of the diffIDs we've generated for the non-throwaway FS layers,
// in reverse of the order in which they were originally listed.
for i, compat := range m.ExtractedV1Compatibility {
if compat.ThrowAway {
continue
}
blobSum := m.FSLayers[i].BlobSum
diffID, ok := s.blobDiffIDs[blobSum]
if !ok {
logrus.Infof("error looking up diffID for layer %q", blobSum.String())
return ""
}
diffIDs = append([]digest.Digest{diffID}, diffIDs...)
}
case *manifest.Schema2, *manifest.OCI1:
// We know the ID calculation for these formats doesn't actually use the diffIDs,
// so we don't need to populate the diffID list.
default:
return ""
}
id, err := m.ImageID(diffIDs)
if err != nil {
return ""
}
return id
}
// getConfigBlob exists only to let us retrieve the configuration blob so that the manifest package can dig
// information out of it for Inspect().
func (s *storageImageDestination) getConfigBlob(info types.BlobInfo) ([]byte, error) {
if info.Digest == "" {
return nil, errors.New(`no digest supplied when reading blob`)
}
if err := info.Digest.Validate(); err != nil {
return nil, fmt.Errorf("invalid digest supplied when reading blob: %w", err)
}
// Assume it's a file, since we're only calling this from a place that expects to read files.
if filename, ok := s.filenames[info.Digest]; ok {
contents, err2 := os.ReadFile(filename)
if err2 != nil {
return nil, fmt.Errorf(`reading blob from file %q: %w`, filename, err2)
}
return contents, nil
}
// If it's not a file, it's a bug, because we're not expecting to be asked for a layer.
return nil, errors.New("blob not found")
}
// queueOrCommit queues the specified layer to be committed to the storage.
// If no other goroutine is already committing layers, the layer and all
// subsequent layers (if already queued) will be committed to the storage.
func (s *storageImageDestination) queueOrCommit(index int, info addedLayerInfo) error {
// NOTE: whenever the code below is touched, make sure that all code
// paths unlock the lock and to unlock it exactly once.
//
// Conceptually, the code is divided in two stages:
//
// 1) Queue in work by marking the layer as ready to be committed.
// If at least one previous/parent layer with a lower index has
// not yet been committed, return early.
//
// 2) Process the queued-in work by committing the "ready" layers
// in sequence. Make sure that more items can be queued-in
// during the comparatively I/O expensive task of committing a
// layer.
//
// The conceptual benefit of this design is that caller can continue
// pulling layers after an early return. At any given time, only one
// caller is the "worker" routine committing layers. All other routines
// can continue pulling and queuing in layers.
s.lock.Lock()
s.indexToAddedLayerInfo[index] = info
// We're still waiting for at least one previous/parent layer to be
// committed, so there's nothing to do.
if index != s.currentIndex {
s.lock.Unlock()
return nil
}
for {
info, ok := s.indexToAddedLayerInfo[index]
if !ok {
break
}
s.lock.Unlock()
// Note: commitLayer locks on-demand.
if err := s.commitLayer(index, info, -1); err != nil {
return err
}
s.lock.Lock()
index++
}
// Set the index at the very end to make sure that only one routine
// enters stage 2).
s.currentIndex = index
s.lock.Unlock()
return nil
}
// commitLayer commits the specified layer with the given index to the storage.
// size can usually be -1; it can be provided if the layer is not known to be already present in blobDiffIDs.
//
// Note that the previous layer is expected to already be committed.
//
// Caution: this function must be called without holding `s.lock`. Callers
// must guarantee that, at any given time, at most one goroutine may execute
// `commitLayer()`.
func (s *storageImageDestination) commitLayer(index int, info addedLayerInfo, size int64) error {
// Already committed? Return early.
if _, alreadyCommitted := s.indexToStorageID[index]; alreadyCommitted {
return nil
}
// Start with an empty string or the previous layer ID. Note that
// `s.indexToStorageID` can only be accessed by *one* goroutine at any
// given time. Hence, we don't need to lock accesses.
var lastLayer string
if prev := s.indexToStorageID[index-1]; prev != nil {
lastLayer = *prev
}
// Carry over the previous ID for empty non-base layers.
if info.emptyLayer {
s.indexToStorageID[index] = &lastLayer
return nil
}
// Check if there's already a layer with the ID that we'd give to the result of applying
// this layer blob to its parent, if it has one, or the blob's hex value otherwise.
s.lock.Lock()
diffID, haveDiffID := s.blobDiffIDs[info.digest]
s.lock.Unlock()
if !haveDiffID {
// Check if it's elsewhere and the caller just forgot to pass it to us in a PutBlob(),
// or to even check if we had it.
// Use none.NoCache to avoid a repeated DiffID lookup in the BlobInfoCache; a caller
// that relies on using a blob digest that has never been seen by the store had better call
// TryReusingBlob; not calling PutBlob already violates the documented API, so theres only
// so far we are going to accommodate that (if we should be doing that at all).
logrus.Debugf("looking for diffID for blob %+v", info.digest)
// Use tryReusingBlobAsPending, not the top-level TryReusingBlobWithOptions, to prevent recursion via queueOrCommit.
has, _, err := s.tryReusingBlobAsPending(info.digest, size, &private.TryReusingBlobOptions{
Cache: none.NoCache,
CanSubstitute: false,
})
if err != nil {
return fmt.Errorf("checking for a layer based on blob %q: %w", info.digest.String(), err)
}
if !has {
return fmt.Errorf("error determining uncompressed digest for blob %q", info.digest.String())
}
diffID, haveDiffID = s.blobDiffIDs[info.digest]
if !haveDiffID {
return fmt.Errorf("we have blob %q, but don't know its uncompressed digest", info.digest.String())
}
}
id := diffID.Hex()
if lastLayer != "" {
id = digest.Canonical.FromBytes([]byte(lastLayer + "+" + diffID.Hex())).Hex()
}
if layer, err2 := s.imageRef.transport.store.Layer(id); layer != nil && err2 == nil {
// There's already a layer that should have the right contents, just reuse it.
lastLayer = layer.ID
s.indexToStorageID[index] = &lastLayer
return nil
}
s.lock.Lock()
diffOutput, ok := s.diffOutputs[info.digest]
s.lock.Unlock()
if ok {
layer, err := s.imageRef.transport.store.CreateLayer(id, lastLayer, nil, "", false, nil)
if err != nil {
return err
}
// FIXME: what to do with the uncompressed digest?
diffOutput.UncompressedDigest = info.digest
if err := s.imageRef.transport.store.ApplyDiffFromStagingDirectory(layer.ID, diffOutput.Target, diffOutput, nil); err != nil {
_ = s.imageRef.transport.store.Delete(layer.ID)
return err
}
s.indexToStorageID[index] = &layer.ID
return nil
}
s.lock.Lock()
al, ok := s.blobAdditionalLayer[info.digest]
s.lock.Unlock()
if ok {
layer, err := al.PutAs(id, lastLayer, nil)
if err != nil && !errors.Is(err, storage.ErrDuplicateID) {
return fmt.Errorf("failed to put layer from digest and labels: %w", err)
}
lastLayer = layer.ID
s.indexToStorageID[index] = &lastLayer
return nil
}
// Check if we previously cached a file with that blob's contents. If we didn't,
// then we need to read the desired contents from a layer.
s.lock.Lock()
filename, ok := s.filenames[info.digest]
s.lock.Unlock()
if !ok {
// Try to find the layer with contents matching that blobsum.
layer := ""
layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(diffID)
if err2 == nil && len(layers) > 0 {
layer = layers[0].ID
} else {
layers, err2 = s.imageRef.transport.store.LayersByCompressedDigest(info.digest)
if err2 == nil && len(layers) > 0 {
layer = layers[0].ID
}
}
if layer == "" {
return fmt.Errorf("locating layer for blob %q: %w", info.digest, err2)
}
// Read the layer's contents.
noCompression := archive.Uncompressed
diffOptions := &storage.DiffOptions{
Compression: &noCompression,
}
diff, err2 := s.imageRef.transport.store.Diff("", layer, diffOptions)
if err2 != nil {
return fmt.Errorf("reading layer %q for blob %q: %w", layer, info.digest, err2)
}
// Copy the layer diff to a file. Diff() takes a lock that it holds
// until the ReadCloser that it returns is closed, and PutLayer() wants
// the same lock, so the diff can't just be directly streamed from one
// to the other.
filename = s.computeNextBlobCacheFile()
file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600)
if err != nil {
diff.Close()
return fmt.Errorf("creating temporary file %q: %w", filename, err)
}
// Copy the data to the file.
// TODO: This can take quite some time, and should ideally be cancellable using
// ctx.Done().
_, err = io.Copy(file, diff)
diff.Close()
file.Close()
if err != nil {
return fmt.Errorf("storing blob to file %q: %w", filename, err)
}
// Make sure that we can find this file later, should we need the layer's
// contents again.
s.lock.Lock()
s.filenames[info.digest] = filename
s.lock.Unlock()
}
// Read the cached blob and use it as a diff.
file, err := os.Open(filename)
if err != nil {
return fmt.Errorf("opening file %q: %w", filename, err)
}
defer file.Close()
// Build the new layer using the diff, regardless of where it came from.
// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
layer, _, err := s.imageRef.transport.store.PutLayer(id, lastLayer, nil, "", false, &storage.LayerOptions{
OriginalDigest: info.digest,
UncompressedDigest: diffID,
}, file)
if err != nil && !errors.Is(err, storage.ErrDuplicateID) {
return fmt.Errorf("adding layer with blob %q: %w", info.digest, err)
}
s.indexToStorageID[index] = &layer.ID
return nil
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
if len(s.manifest) == 0 {
return errors.New("Internal error: storageImageDestination.Commit() called without PutManifest()")
}
toplevelManifest, _, err := unparsedToplevel.Manifest(ctx)
if err != nil {
return fmt.Errorf("retrieving top-level manifest: %w", err)
}
// If the name we're saving to includes a digest, then check that the
// manifests that we're about to save all either match the one from the
// unparsedToplevel, or match the digest in the name that we're using.
if s.imageRef.named != nil {
if digested, ok := s.imageRef.named.(reference.Digested); ok {
matches, err := manifest.MatchesDigest(s.manifest, digested.Digest())
if err != nil {
return err
}
if !matches {
matches, err = manifest.MatchesDigest(toplevelManifest, digested.Digest())
if err != nil {
return err
}
}
if !matches {
return fmt.Errorf("Manifest to be saved does not match expected digest %s", digested.Digest())
}
}
}
// Find the list of layer blobs.
man, err := manifest.FromBlob(s.manifest, manifest.GuessMIMEType(s.manifest))
if err != nil {
return fmt.Errorf("parsing manifest: %w", err)
}
layerBlobs := man.LayerInfos()
// Extract, commit, or find the layers.
for i, blob := range layerBlobs {
if err := s.commitLayer(i, addedLayerInfo{
digest: blob.Digest,
emptyLayer: blob.EmptyLayer,
}, blob.Size); err != nil {
return err
}
}
var lastLayer string
if len(layerBlobs) > 0 { // Can happen when using caches
prev := s.indexToStorageID[len(layerBlobs)-1]
if prev == nil {
return fmt.Errorf("Internal error: storageImageDestination.Commit(): previous layer %d hasn't been committed (lastLayer == nil)", len(layerBlobs)-1)
}
lastLayer = *prev
}
// If one of those blobs was a configuration blob, then we can try to dig out the date when the image
// was originally created, in case we're just copying it. If not, no harm done.
options := &storage.ImageOptions{}
if inspect, err := man.Inspect(s.getConfigBlob); err == nil && inspect.Created != nil {
logrus.Debugf("setting image creation date to %s", inspect.Created)
options.CreationDate = *inspect.Created
}
// Set up to save the non-layer blobs as data items. Since we only share layers, they should all be in files, so
// we just need to screen out the ones that are actually layers to get the list of non-layers.
dataBlobs := set.New[digest.Digest]()
for blob := range s.filenames {
dataBlobs.Add(blob)
}
for _, layerBlob := range layerBlobs {
dataBlobs.Delete(layerBlob.Digest)
}
for _, blob := range dataBlobs.Values() {
v, err := os.ReadFile(s.filenames[blob])
if err != nil {
return fmt.Errorf("copying non-layer blob %q to image: %w", blob, err)
}
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: blob.String(),
Data: v,
Digest: digest.Canonical.FromBytes(v),
})
}
// Set up to save the unparsedToplevel's manifest if it differs from
// the per-platform one, which is saved below.
if len(toplevelManifest) != 0 && !bytes.Equal(toplevelManifest, s.manifest) {
manifestDigest, err := manifest.Digest(toplevelManifest)
if err != nil {
return fmt.Errorf("digesting top-level manifest: %w", err)
}
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: manifestBigDataKey(manifestDigest),
Data: toplevelManifest,
Digest: manifestDigest,
})
}
// Set up to save the image's manifest. Allow looking it up by digest by using the key convention defined by the Store.
// Record the manifest twice: using a digest-specific key to allow references to that specific digest instance,
// and using storage.ImageDigestBigDataKey for future users that dont specify any digest and for compatibility with older readers.
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: manifestBigDataKey(s.manifestDigest),
Data: s.manifest,
Digest: s.manifestDigest,
})
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: storage.ImageDigestBigDataKey,
Data: s.manifest,
Digest: s.manifestDigest,
})
// Set up to save the signatures, if we have any.
if len(s.signatures) > 0 {
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: "signatures",
Data: s.signatures,
Digest: digest.Canonical.FromBytes(s.signatures),
})
}
for instanceDigest, signatures := range s.signatureses {
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: signatureBigDataKey(instanceDigest),
Data: signatures,
Digest: digest.Canonical.FromBytes(signatures),
})
}
// Set up to save our metadata.
metadata, err := json.Marshal(s)
if err != nil {
return fmt.Errorf("encoding metadata for image: %w", err)
}
if len(metadata) != 0 {
options.Metadata = string(metadata)
}
// Create the image record, pointing to the most-recently added layer.
intendedID := s.imageRef.id
if intendedID == "" {
intendedID = s.computeID(man)
}
oldNames := []string{}
img, err := s.imageRef.transport.store.CreateImage(intendedID, nil, lastLayer, "", options)
if err != nil {
if !errors.Is(err, storage.ErrDuplicateID) {
logrus.Debugf("error creating image: %q", err)
return fmt.Errorf("creating image %q: %w", intendedID, err)
}
img, err = s.imageRef.transport.store.Image(intendedID)
if err != nil {
return fmt.Errorf("reading image %q: %w", intendedID, err)
}
if img.TopLayer != lastLayer {
logrus.Debugf("error creating image: image with ID %q exists, but uses different layers", intendedID)
return fmt.Errorf("image with ID %q already exists, but uses a different top layer: %w", intendedID, storage.ErrDuplicateID)
}
logrus.Debugf("reusing image ID %q", img.ID)
oldNames = append(oldNames, img.Names...)
// set the data items and metadata on the already-present image
// FIXME: this _replaces_ any "signatures" blobs and their
// sizes (tracked in the metadata) which might have already
// been present with new values, when ideally we'd find a way
// to merge them since they all apply to the same image
for _, data := range options.BigData {
if err := s.imageRef.transport.store.SetImageBigData(img.ID, data.Key, data.Data, manifest.Digest); err != nil {
logrus.Debugf("error saving big data %q for image %q: %v", data.Key, img.ID, err)
return fmt.Errorf("saving big data %q for image %q: %w", data.Key, img.ID, err)
}
}
if options.Metadata != "" {
if err := s.imageRef.transport.store.SetMetadata(img.ID, options.Metadata); err != nil {
logrus.Debugf("error saving metadata for image %q: %v", img.ID, err)
return fmt.Errorf("saving metadata for image %q: %w", img.ID, err)
}
logrus.Debugf("saved image metadata %q", options.Metadata)
}
} else {
logrus.Debugf("created new image ID %q with metadata %q", img.ID, options.Metadata)
}
// Clean up the unfinished image on any error.
// (Is this the right thing to do if the image has existed before?)
commitSucceeded := false
defer func() {
if !commitSucceeded {
logrus.Errorf("Updating image %q (old names %v) failed, deleting it", img.ID, oldNames)
if _, err := s.imageRef.transport.store.DeleteImage(img.ID, true); err != nil {
logrus.Errorf("Error deleting incomplete image %q: %v", img.ID, err)
}
}
}()
// Add the reference's name on the image. We don't need to worry about avoiding duplicate
// values because AddNames() will deduplicate the list that we pass to it.
if name := s.imageRef.DockerReference(); name != nil {
if err := s.imageRef.transport.store.AddNames(img.ID, []string{name.String()}); err != nil {
return fmt.Errorf("adding names %v to image %q: %w", name, img.ID, err)
}
logrus.Debugf("added name %q to image %q", name, img.ID)
}
commitSucceeded = true
return nil
}
// PutManifest writes the manifest to the destination.
func (s *storageImageDestination) PutManifest(ctx context.Context, manifestBlob []byte, instanceDigest *digest.Digest) error {
digest, err := manifest.Digest(manifestBlob)
if err != nil {
return err
}
s.manifest = slices.Clone(manifestBlob)
s.manifestDigest = digest
return nil
}
// PutSignaturesWithFormat writes a set of signatures to the destination.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for
// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list.
// MUST be called after PutManifest (signatures may reference manifest contents).
func (s *storageImageDestination) PutSignaturesWithFormat(ctx context.Context, signatures []signature.Signature, instanceDigest *digest.Digest) error {
sizes := []int{}
sigblob := []byte{}
for _, sigWithFormat := range signatures {
sig, err := signature.Blob(sigWithFormat)
if err != nil {
return err
}
sizes = append(sizes, len(sig))
sigblob = append(sigblob, sig...)
}
if instanceDigest == nil {
s.signatures = sigblob
s.SignatureSizes = sizes
if len(s.manifest) > 0 {
manifestDigest := s.manifestDigest
instanceDigest = &manifestDigest
}
}
if instanceDigest != nil {
s.signatureses[*instanceDigest] = sigblob
s.SignaturesSizes[*instanceDigest] = sizes
}
return nil
}

View file

@ -0,0 +1,59 @@
//go:build !containers_image_storage_stub
// +build !containers_image_storage_stub
package storage
import (
"context"
"github.com/containers/image/v5/internal/image"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
digest "github.com/opencontainers/go-digest"
)
var (
// ErrNoSuchImage is returned when we attempt to access an image which
// doesn't exist in the storage area.
ErrNoSuchImage = storage.ErrNotAnImage
)
type storageImageCloser struct {
types.ImageCloser
size int64
}
// manifestBigDataKey returns a key suitable for recording a manifest with the specified digest using storage.Store.ImageBigData and related functions.
// If a specific manifest digest is explicitly requested by the user, the key returned by this function should be used preferably;
// for compatibility, if a manifest is not available under this key, check also storage.ImageDigestBigDataKey
func manifestBigDataKey(digest digest.Digest) string {
return storage.ImageDigestManifestBigDataNamePrefix + "-" + digest.String()
}
// signatureBigDataKey returns a key suitable for recording the signatures associated with the manifest with the specified digest using storage.Store.ImageBigData and related functions.
// If a specific manifest digest is explicitly requested by the user, the key returned by this function should be used preferably;
func signatureBigDataKey(digest digest.Digest) string {
return "signature-" + digest.Encoded()
}
// Size() returns the previously-computed size of the image, with no error.
func (s *storageImageCloser) Size() (int64, error) {
return s.size, nil
}
// newImage creates an image that also knows its size
func newImage(ctx context.Context, sys *types.SystemContext, s storageReference) (types.ImageCloser, error) {
src, err := newImageSource(sys, s)
if err != nil {
return nil, err
}
img, err := image.FromSource(ctx, sys, src)
if err != nil {
return nil, err
}
size, err := src.getSize()
if err != nil {
return nil, err
}
return &storageImageCloser{ImageCloser: img, size: size}, nil
}

View file

@ -0,0 +1,316 @@
//go:build !containers_image_storage_stub
// +build !containers_image_storage_stub
package storage
import (
"context"
"fmt"
"strings"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
digest "github.com/opencontainers/go-digest"
"github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
)
// A storageReference holds an arbitrary name and/or an ID, which is a 32-byte
// value hex-encoded into a 64-character string, and a reference to a Store
// where an image is, or would be, kept.
// Either "named" or "id" must be set.
type storageReference struct {
transport storageTransport
named reference.Named // may include a tag and/or a digest
id string
}
func newReference(transport storageTransport, named reference.Named, id string) (*storageReference, error) {
if named == nil && id == "" {
return nil, ErrInvalidReference
}
if named != nil && reference.IsNameOnly(named) {
return nil, fmt.Errorf("reference %s has neither a tag nor a digest: %w", named.String(), ErrInvalidReference)
}
if id != "" {
if err := validateImageID(id); err != nil {
return nil, fmt.Errorf("invalid ID value %q: %v: %w", id, err, ErrInvalidReference)
}
}
// We take a copy of the transport, which contains a pointer to the
// store that it used for resolving this reference, so that the
// transport that we'll return from Transport() won't be affected by
// further calls to the original transport's SetStore() method.
return &storageReference{
transport: transport,
named: named,
id: id,
}, nil
}
// imageMatchesRepo returns true iff image.Names contains an element with the same repo as ref
func imageMatchesRepo(image *storage.Image, ref reference.Named) bool {
repo := ref.Name()
return slices.ContainsFunc(image.Names, func(name string) bool {
if named, err := reference.ParseNormalizedNamed(name); err == nil && named.Name() == repo {
return true
}
return false
})
}
// multiArchImageMatchesSystemContext returns true if the passed-in image both contains a
// multi-arch manifest that matches the passed-in digest, and the image is the per-platform
// image instance that matches sys.
//
// See the comment in storageReference.ResolveImage explaining why
// this check is necessary.
func multiArchImageMatchesSystemContext(store storage.Store, img *storage.Image, manifestDigest digest.Digest, sys *types.SystemContext) bool {
// Load the manifest that matches the specified digest.
// We don't need to care about storage.ImageDigestBigDataKey because
// manifests lists are only stored into storage by c/image versions
// that know about manifestBigDataKey, and only using that key.
key := manifestBigDataKey(manifestDigest)
manifestBytes, err := store.ImageBigData(img.ID, key)
if err != nil {
return false
}
// The manifest is either a list, or not a list. If it's a list, find
// the digest of the instance that matches the current system, and try
// to load that manifest from the image record, and use it.
manifestType := manifest.GuessMIMEType(manifestBytes)
if !manifest.MIMETypeIsMultiImage(manifestType) {
// manifestDigest directly specifies a per-platform image, so we aren't
// choosing among different variants.
return false
}
list, err := manifest.ListFromBlob(manifestBytes, manifestType)
if err != nil {
return false
}
chosenInstance, err := list.ChooseInstance(sys)
if err != nil {
return false
}
key = manifestBigDataKey(chosenInstance)
_, err = store.ImageBigData(img.ID, key)
return err == nil // true if img.ID is based on chosenInstance.
}
// Resolve the reference's name to an image ID in the store, if there's already
// one present with the same name or ID, and return the image.
//
// Returns an error matching ErrNoSuchImage if an image matching ref was not found.
func (s *storageReference) resolveImage(sys *types.SystemContext) (*storage.Image, error) {
var loadedImage *storage.Image
if s.id == "" && s.named != nil {
// Look for an image that has the expanded reference name as an explicit Name value.
image, err := s.transport.store.Image(s.named.String())
if image != nil && err == nil {
loadedImage = image
s.id = image.ID
}
}
if s.id == "" && s.named != nil {
if digested, ok := s.named.(reference.Digested); ok {
// Look for an image with the specified digest that has the same name,
// though possibly with a different tag or digest, as a Name value, so
// that the canonical reference can be implicitly resolved to the image.
//
// Typically there should be at most one such image, because the same
// manifest digest implies the same config, and we choose the storage ID
// based on the config (deduplicating images), except:
// - the user can explicitly specify an ID when creating the image.
// In this case we don't have a preference among the alternatives.
// - when pulling an image from a multi-platform manifest list, we also
// store the manifest list in the image; this allows referencing a
// per-platform image using the manifest list digest, but that also
// means that we can have multiple genuinely different images in the
// storage matching the same manifest list digest (if pulled using different
// SystemContext.{OS,Architecture,Variant}Choice to the same storage).
// In this case we prefer the image matching the current SystemContext.
images, err := s.transport.store.ImagesByDigest(digested.Digest())
if err == nil && len(images) > 0 {
for _, image := range images {
if imageMatchesRepo(image, s.named) {
if loadedImage == nil || multiArchImageMatchesSystemContext(s.transport.store, image, digested.Digest(), sys) {
loadedImage = image
s.id = image.ID
}
}
}
}
}
}
if s.id == "" {
logrus.Debugf("reference %q does not resolve to an image ID", s.StringWithinTransport())
return nil, fmt.Errorf("reference %q does not resolve to an image ID: %w", s.StringWithinTransport(), ErrNoSuchImage)
}
if loadedImage == nil {
img, err := s.transport.store.Image(s.id)
if err != nil {
return nil, fmt.Errorf("reading image %q: %w", s.id, err)
}
loadedImage = img
}
if s.named != nil {
if !imageMatchesRepo(loadedImage, s.named) {
logrus.Errorf("no image matching reference %q found", s.StringWithinTransport())
return nil, ErrNoSuchImage
}
}
// Default to having the image digest that we hand back match the most recently
// added manifest...
if digest, ok := loadedImage.BigDataDigests[storage.ImageDigestBigDataKey]; ok {
loadedImage.Digest = digest
}
// ... unless the named reference says otherwise, and it matches one of the digests
// in the image. For those cases, set the Digest field to that value, for the
// sake of older consumers that don't know there's a whole list in there now.
if s.named != nil {
if digested, ok := s.named.(reference.Digested); ok {
digest := digested.Digest()
if slices.Contains(loadedImage.Digests, digest) {
loadedImage.Digest = digest
}
}
}
return loadedImage, nil
}
// Return a Transport object that defaults to using the same store that we used
// to build this reference object.
func (s storageReference) Transport() types.ImageTransport {
return &storageTransport{
store: s.transport.store,
defaultUIDMap: s.transport.defaultUIDMap,
defaultGIDMap: s.transport.defaultGIDMap,
}
}
// Return a name with a tag or digest, if we have either, else return it bare.
func (s storageReference) DockerReference() reference.Named {
return s.named
}
// Return a name with a tag, prefixed with the graph root and driver name, to
// disambiguate between images which may be present in multiple stores and
// share only their names.
func (s storageReference) StringWithinTransport() string {
optionsList := ""
options := s.transport.store.GraphOptions()
if len(options) > 0 {
optionsList = ":" + strings.Join(options, ",")
}
res := "[" + s.transport.store.GraphDriverName() + "@" + s.transport.store.GraphRoot() + "+" + s.transport.store.RunRoot() + optionsList + "]"
if s.named != nil {
res += s.named.String()
}
if s.id != "" {
res += "@" + s.id
}
return res
}
func (s storageReference) PolicyConfigurationIdentity() string {
res := "[" + s.transport.store.GraphDriverName() + "@" + s.transport.store.GraphRoot() + "]"
if s.named != nil {
res += s.named.String()
}
if s.id != "" {
res += "@" + s.id
}
return res
}
// Also accept policy that's tied to the combination of the graph root and
// driver name, to apply to all images stored in the Store, and to just the
// graph root, in case we're using multiple drivers in the same directory for
// some reason.
func (s storageReference) PolicyConfigurationNamespaces() []string {
storeSpec := "[" + s.transport.store.GraphDriverName() + "@" + s.transport.store.GraphRoot() + "]"
driverlessStoreSpec := "[" + s.transport.store.GraphRoot() + "]"
namespaces := []string{}
if s.named != nil {
if s.id != "" {
// The reference without the ID is also a valid namespace.
namespaces = append(namespaces, storeSpec+s.named.String())
}
tagged, isTagged := s.named.(reference.Tagged)
_, isDigested := s.named.(reference.Digested)
if isTagged && isDigested { // s.named is "name:tag@digest"; add a "name:tag" parent namespace.
namespaces = append(namespaces, storeSpec+s.named.Name()+":"+tagged.Tag())
}
components := strings.Split(s.named.Name(), "/")
for len(components) > 0 {
namespaces = append(namespaces, storeSpec+strings.Join(components, "/"))
components = components[:len(components)-1]
}
}
namespaces = append(namespaces, storeSpec)
namespaces = append(namespaces, driverlessStoreSpec)
return namespaces
}
// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport.
// The caller must call .Close() on the returned ImageCloser.
// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource,
// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details.
func (s storageReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) {
return newImage(ctx, sys, s)
}
func (s storageReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error {
img, err := s.resolveImage(sys)
if err != nil {
return err
}
layers, err := s.transport.store.DeleteImage(img.ID, true)
if err == nil {
logrus.Debugf("deleted image %q", img.ID)
for _, layer := range layers {
logrus.Debugf("deleted layer %q", layer)
}
}
return err
}
func (s storageReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) {
return newImageSource(sys, s)
}
func (s storageReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) {
return newImageDestination(sys, s)
}
// ResolveReference finds the underlying storage image for a storage.Transport reference.
// It returns that image, and an updated reference which can be used to refer back to the _same_
// image again.
//
// This matters if the input reference contains a tagged name; the destination of the tag can
// move in local storage. The updated reference returned by this function contains the resolved
// image ID, so later uses of that updated reference will either continue to refer to the same
// image, or fail.
//
// Note that it _is_ possible for the later uses to fail, either because the image was removed
// completely, or because the name used in the reference was untaged (even if the underlying image
// ID still exists in local storage).
//
// Returns an error matching ErrNoSuchImage if an image matching ref was not found.
func ResolveReference(ref types.ImageReference) (types.ImageReference, *storage.Image, error) {
sref, ok := ref.(*storageReference)
if !ok {
return nil, nil, fmt.Errorf("trying to resolve a non-%s: reference %q", Transport.Name(),
transports.ImageName(ref))
}
clone := *sref // A shallow copy we can update
img, err := clone.resolveImage(nil)
if err != nil {
return nil, nil, err
}
return clone, img, nil
}

View file

@ -0,0 +1,403 @@
//go:build !containers_image_storage_stub
// +build !containers_image_storage_stub
package storage
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"sync"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/image"
"github.com/containers/image/v5/internal/imagesource/impl"
"github.com/containers/image/v5/internal/imagesource/stubs"
"github.com/containers/image/v5/internal/signature"
"github.com/containers/image/v5/internal/tmpdir"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
"github.com/containers/storage/pkg/archive"
"github.com/containers/storage/pkg/ioutils"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
)
type storageImageSource struct {
impl.Compat
impl.PropertyMethodsInitialize
stubs.NoGetBlobAtInitialize
imageRef storageReference
image *storage.Image
systemContext *types.SystemContext // SystemContext used in GetBlob() to create temporary files
layerPosition map[digest.Digest]int // Where we are in reading a blob's layers
cachedManifest []byte // A cached copy of the manifest, if already known, or nil
getBlobMutex sync.Mutex // Mutex to sync state for parallel GetBlob executions
SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice
SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // List of sizes of each signature slice
}
// newImageSource sets up an image for reading.
func newImageSource(sys *types.SystemContext, imageRef storageReference) (*storageImageSource, error) {
// First, locate the image.
img, err := imageRef.resolveImage(sys)
if err != nil {
return nil, err
}
// Build the reader object.
image := &storageImageSource{
PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{
HasThreadSafeGetBlob: true,
}),
NoGetBlobAtInitialize: stubs.NoGetBlobAt(imageRef),
imageRef: imageRef,
systemContext: sys,
image: img,
layerPosition: make(map[digest.Digest]int),
SignatureSizes: []int{},
SignaturesSizes: make(map[digest.Digest][]int),
}
image.Compat = impl.AddCompat(image)
if img.Metadata != "" {
if err := json.Unmarshal([]byte(img.Metadata), image); err != nil {
return nil, fmt.Errorf("decoding metadata for source image: %w", err)
}
}
return image, nil
}
// Reference returns the image reference that we used to find this image.
func (s *storageImageSource) Reference() types.ImageReference {
return s.imageRef
}
// Close cleans up any resources we tied up while reading the image.
func (s *storageImageSource) Close() error {
return nil
}
// GetBlob returns a stream for the specified blob, and the blobs size (or -1 if unknown).
// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
func (s *storageImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (rc io.ReadCloser, n int64, err error) {
// We need a valid digest value.
digest := info.Digest
err = digest.Validate()
if err != nil {
return nil, 0, err
}
if digest == image.GzippedEmptyLayerDigest {
return io.NopCloser(bytes.NewReader(image.GzippedEmptyLayer)), int64(len(image.GzippedEmptyLayer)), nil
}
// Check if the blob corresponds to a diff that was used to initialize any layers. Our
// callers should try to retrieve layers using their uncompressed digests, so no need to
// check if they're using one of the compressed digests, which we can't reproduce anyway.
layers, _ := s.imageRef.transport.store.LayersByUncompressedDigest(digest)
// If it's not a layer, then it must be a data item.
if len(layers) == 0 {
b, err := s.imageRef.transport.store.ImageBigData(s.image.ID, digest.String())
if err != nil {
return nil, 0, err
}
r := bytes.NewReader(b)
logrus.Debugf("exporting opaque data as blob %q", digest.String())
return io.NopCloser(r), int64(r.Len()), nil
}
// NOTE: the blob is first written to a temporary file and subsequently
// closed. The intention is to keep the time we own the storage lock
// as short as possible to allow other processes to access the storage.
rc, n, _, err = s.getBlobAndLayerID(digest, layers)
if err != nil {
return nil, 0, err
}
defer rc.Close()
tmpFile, err := tmpdir.CreateBigFileTemp(s.systemContext, "")
if err != nil {
return nil, 0, err
}
success := false
tmpFileRemovePending := true
defer func() {
if !success {
tmpFile.Close()
if tmpFileRemovePending {
os.Remove(tmpFile.Name())
}
}
}()
// On Unix and modern Windows (2022 at least) we can eagerly unlink the file to ensure it's automatically
// cleaned up on process termination (or if the caller forgets to invoke Close())
// On older versions of Windows we will have to fallback to relying on the caller to invoke Close()
if err := os.Remove(tmpFile.Name()); err != nil {
tmpFileRemovePending = false
}
if _, err := io.Copy(tmpFile, rc); err != nil {
return nil, 0, err
}
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
return nil, 0, err
}
success = true
if tmpFileRemovePending {
return ioutils.NewReadCloserWrapper(tmpFile, func() error {
tmpFile.Close()
return os.Remove(tmpFile.Name())
}), n, nil
}
return tmpFile, n, nil
}
// getBlobAndLayer reads the data blob or filesystem layer which matches the digest and size, if given.
func (s *storageImageSource) getBlobAndLayerID(digest digest.Digest, layers []storage.Layer) (rc io.ReadCloser, n int64, layerID string, err error) {
var layer storage.Layer
var diffOptions *storage.DiffOptions
// Step through the list of matching layers. Tests may want to verify that if we have multiple layers
// which claim to have the same contents, that we actually do have multiple layers, otherwise we could
// just go ahead and use the first one every time.
s.getBlobMutex.Lock()
i := s.layerPosition[digest]
s.layerPosition[digest] = i + 1
s.getBlobMutex.Unlock()
if len(layers) > 0 {
layer = layers[i%len(layers)]
}
// Force the storage layer to not try to match any compression that was used when the layer was first
// handed to it.
noCompression := archive.Uncompressed
diffOptions = &storage.DiffOptions{
Compression: &noCompression,
}
if layer.UncompressedSize < 0 {
n = -1
} else {
n = layer.UncompressedSize
}
logrus.Debugf("exporting filesystem layer %q without compression for blob %q", layer.ID, digest)
rc, err = s.imageRef.transport.store.Diff("", layer.ID, diffOptions)
if err != nil {
return nil, -1, "", err
}
return rc, n, layer.ID, err
}
// GetManifest() reads the image's manifest.
func (s *storageImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) (manifestBlob []byte, mimeType string, err error) {
if instanceDigest != nil {
key := manifestBigDataKey(*instanceDigest)
blob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key)
if err != nil {
return nil, "", fmt.Errorf("reading manifest for image instance %q: %w", *instanceDigest, err)
}
return blob, manifest.GuessMIMEType(blob), err
}
if len(s.cachedManifest) == 0 {
// The manifest is stored as a big data item.
// Prefer the manifest corresponding to the user-specified digest, if available.
if s.imageRef.named != nil {
if digested, ok := s.imageRef.named.(reference.Digested); ok {
key := manifestBigDataKey(digested.Digest())
blob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key)
if err != nil && !os.IsNotExist(err) { // os.IsNotExist is true if the image exists but there is no data corresponding to key
return nil, "", err
}
if err == nil {
s.cachedManifest = blob
}
}
}
// If the user did not specify a digest, or this is an old image stored before manifestBigDataKey was introduced, use the default manifest.
// Note that the manifest may not match the expected digest, and that is likely to fail eventually, e.g. in c/image/image/UnparsedImage.Manifest().
if len(s.cachedManifest) == 0 {
cachedBlob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, storage.ImageDigestBigDataKey)
if err != nil {
return nil, "", err
}
s.cachedManifest = cachedBlob
}
}
return s.cachedManifest, manifest.GuessMIMEType(s.cachedManifest), err
}
// LayerInfosForCopy() returns the list of layer blobs that make up the root filesystem of
// the image, after they've been decompressed.
func (s *storageImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) {
manifestBlob, manifestType, err := s.GetManifest(ctx, instanceDigest)
if err != nil {
return nil, fmt.Errorf("reading image manifest for %q: %w", s.image.ID, err)
}
if manifest.MIMETypeIsMultiImage(manifestType) {
return nil, errors.New("can't copy layers for a manifest list (shouldn't be attempted)")
}
man, err := manifest.FromBlob(manifestBlob, manifestType)
if err != nil {
return nil, fmt.Errorf("parsing image manifest for %q: %w", s.image.ID, err)
}
uncompressedLayerType := ""
switch manifestType {
case imgspecv1.MediaTypeImageManifest:
uncompressedLayerType = imgspecv1.MediaTypeImageLayer
case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema2MediaType:
uncompressedLayerType = manifest.DockerV2SchemaLayerMediaTypeUncompressed
}
physicalBlobInfos := []types.BlobInfo{}
layerID := s.image.TopLayer
for layerID != "" {
layer, err := s.imageRef.transport.store.Layer(layerID)
if err != nil {
return nil, fmt.Errorf("reading layer %q in image %q: %w", layerID, s.image.ID, err)
}
if layer.UncompressedDigest == "" {
return nil, fmt.Errorf("uncompressed digest for layer %q is unknown", layerID)
}
if layer.UncompressedSize < 0 {
return nil, fmt.Errorf("uncompressed size for layer %q is unknown", layerID)
}
blobInfo := types.BlobInfo{
Digest: layer.UncompressedDigest,
Size: layer.UncompressedSize,
MediaType: uncompressedLayerType,
}
physicalBlobInfos = append([]types.BlobInfo{blobInfo}, physicalBlobInfos...)
layerID = layer.Parent
}
res, err := buildLayerInfosForCopy(man.LayerInfos(), physicalBlobInfos)
if err != nil {
return nil, fmt.Errorf("creating LayerInfosForCopy of image %q: %w", s.image.ID, err)
}
return res, nil
}
// buildLayerInfosForCopy builds a LayerInfosForCopy return value based on manifestInfos from the original manifest,
// but using layer data which we can actually produce — physicalInfos for non-empty layers,
// and image.GzippedEmptyLayer for empty ones.
// (This is split basically only to allow easily unit-testing the part that has no dependencies on the external environment.)
func buildLayerInfosForCopy(manifestInfos []manifest.LayerInfo, physicalInfos []types.BlobInfo) ([]types.BlobInfo, error) {
nextPhysical := 0
res := make([]types.BlobInfo, len(manifestInfos))
for i, mi := range manifestInfos {
if mi.EmptyLayer {
res[i] = types.BlobInfo{
Digest: image.GzippedEmptyLayerDigest,
Size: int64(len(image.GzippedEmptyLayer)),
MediaType: mi.MediaType,
}
} else {
if nextPhysical >= len(physicalInfos) {
return nil, fmt.Errorf("expected more than %d physical layers to exist", len(physicalInfos))
}
res[i] = physicalInfos[nextPhysical] // FIXME? Should we preserve more data in manifestInfos? Notably the current approach correctly removes zstd:chunked metadata annotations.
nextPhysical++
}
}
if nextPhysical != len(physicalInfos) {
return nil, fmt.Errorf("used only %d out of %d physical layers", nextPhysical, len(physicalInfos))
}
return res, nil
}
// GetSignaturesWithFormat returns the image's signatures. It may use a remote (= slow) service.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for
// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list
// (e.g. if the source never returns manifest lists).
func (s *storageImageSource) GetSignaturesWithFormat(ctx context.Context, instanceDigest *digest.Digest) ([]signature.Signature, error) {
var offset int
signatureBlobs := []byte{}
signatureSizes := s.SignatureSizes
key := "signatures"
instance := "default instance"
if instanceDigest != nil {
signatureSizes = s.SignaturesSizes[*instanceDigest]
key = signatureBigDataKey(*instanceDigest)
instance = instanceDigest.Encoded()
}
if len(signatureSizes) > 0 {
data, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key)
if err != nil {
return nil, fmt.Errorf("looking up signatures data for image %q (%s): %w", s.image.ID, instance, err)
}
signatureBlobs = data
}
res := []signature.Signature{}
for _, length := range signatureSizes {
if offset+length > len(signatureBlobs) {
return nil, fmt.Errorf("looking up signatures data for image %q (%s): expected at least %d bytes, only found %d", s.image.ID, instance, len(signatureBlobs), offset+length)
}
sig, err := signature.FromBlob(signatureBlobs[offset : offset+length])
if err != nil {
return nil, fmt.Errorf("parsing signature at (%d, %d): %w", offset, length, err)
}
res = append(res, sig)
offset += length
}
if offset != len(signatureBlobs) {
return nil, fmt.Errorf("signatures data (%s) contained %d extra bytes", instance, len(signatureBlobs)-offset)
}
return res, nil
}
// getSize() adds up the sizes of the image's data blobs (which includes the configuration blob), the
// signatures, and the uncompressed sizes of all of the image's layers.
func (s *storageImageSource) getSize() (int64, error) {
var sum int64
// Size up the data blobs.
dataNames, err := s.imageRef.transport.store.ListImageBigData(s.image.ID)
if err != nil {
return -1, fmt.Errorf("reading image %q: %w", s.image.ID, err)
}
for _, dataName := range dataNames {
bigSize, err := s.imageRef.transport.store.ImageBigDataSize(s.image.ID, dataName)
if err != nil {
return -1, fmt.Errorf("reading data blob size %q for %q: %w", dataName, s.image.ID, err)
}
sum += bigSize
}
// Add the signature sizes.
for _, sigSize := range s.SignatureSizes {
sum += int64(sigSize)
}
// Walk the layer list.
layerID := s.image.TopLayer
for layerID != "" {
layer, err := s.imageRef.transport.store.Layer(layerID)
if err != nil {
return -1, err
}
if layer.UncompressedDigest == "" || layer.UncompressedSize < 0 {
return -1, fmt.Errorf("size for layer %q is unknown, failing getSize()", layerID)
}
sum += layer.UncompressedSize
if layer.Parent == "" {
break
}
layerID = layer.Parent
}
return sum, nil
}
// Size() adds up the sizes of the image's data blobs (which includes the configuration blob), the
// signatures, and the uncompressed sizes of all of the image's layers.
func (s *storageImageSource) Size() (int64, error) {
return s.getSize()
}

View file

@ -0,0 +1,416 @@
//go:build !containers_image_storage_stub
// +build !containers_image_storage_stub
package storage
import (
"errors"
"fmt"
"path/filepath"
"strings"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
"github.com/containers/storage/pkg/idtools"
digest "github.com/opencontainers/go-digest"
"github.com/sirupsen/logrus"
)
const (
minimumTruncatedIDLength = 3
)
func init() {
transports.Register(Transport)
}
var (
// Transport is an ImageTransport that uses either a default
// storage.Store or one that's it's explicitly told to use.
Transport StoreTransport = &storageTransport{}
// ErrInvalidReference is returned when ParseReference() is passed an
// empty reference.
ErrInvalidReference = errors.New("invalid reference")
// ErrPathNotAbsolute is returned when a graph root is not an absolute
// path name.
ErrPathNotAbsolute = errors.New("path name is not absolute")
)
// StoreTransport is an ImageTransport that uses a storage.Store to parse
// references, either its own default or one that it's told to use.
type StoreTransport interface {
types.ImageTransport
// SetStore sets the default store for this transport.
SetStore(storage.Store)
// GetStoreIfSet returns the default store for this transport, or nil if not set/determined yet.
GetStoreIfSet() storage.Store
// GetImage retrieves the image from the transport's store that's named
// by the reference.
// Deprecated: Surprisingly, with a StoreTransport reference which contains an ID,
// this ignores that ID; and repeated calls of GetStoreImage with the same named reference
// can return different images, with no way for the caller to "freeze" the storage.Image identity
// without discarding the name entirely.
//
// Use storage.ResolveReference instead; note that if the image is not found, ResolveReference returns
// c/image/v5/storage.ErrNoSuchImage, not c/storage.ErrImageUnknown.
GetImage(types.ImageReference) (*storage.Image, error)
// GetStoreImage retrieves the image from a specified store that's named
// by the reference.
//
// Deprecated: Surprisingly, with a StoreTransport reference which contains an ID,
// this ignores that ID; and repeated calls of GetStoreImage with the same named reference
// can return different images, with no way for the caller to "freeze" the storage.Image identity
// without discarding the name entirely.
//
// Also, a StoreTransport reference already contains a store, so providing another one is redundant.
//
// Use storage.ResolveReference instead; note that if the image is not found, ResolveReference returns
// c/image/v5/storage.ErrNoSuchImage, not c/storage.ErrImageUnknown.
GetStoreImage(storage.Store, types.ImageReference) (*storage.Image, error)
// ParseStoreReference parses a reference, overriding any store
// specification that it may contain.
ParseStoreReference(store storage.Store, reference string) (*storageReference, error)
// NewStoreReference creates a reference for (named@ID) in store.
// either of name or ID can be unset; named must not be a reference.IsNameOnly.
NewStoreReference(store storage.Store, named reference.Named, id string) (*storageReference, error)
// SetDefaultUIDMap sets the default UID map to use when opening stores.
SetDefaultUIDMap(idmap []idtools.IDMap)
// SetDefaultGIDMap sets the default GID map to use when opening stores.
SetDefaultGIDMap(idmap []idtools.IDMap)
// DefaultUIDMap returns the default UID map used when opening stores.
DefaultUIDMap() []idtools.IDMap
// DefaultGIDMap returns the default GID map used when opening stores.
DefaultGIDMap() []idtools.IDMap
}
type storageTransport struct {
store storage.Store
defaultUIDMap []idtools.IDMap
defaultGIDMap []idtools.IDMap
}
func (s *storageTransport) Name() string {
// Still haven't really settled on a name.
return "containers-storage"
}
// SetStore sets the Store object which the Transport will use for parsing
// references when information about a Store is not directly specified as part
// of the reference. If one is not set, the library will attempt to initialize
// one with default settings when a reference needs to be parsed. Calling
// SetStore does not affect previously parsed references.
func (s *storageTransport) SetStore(store storage.Store) {
s.store = store
}
// GetStoreIfSet returns the default store for this transport, as set using SetStore() or initialized by default, or nil if not set/determined yet.
func (s *storageTransport) GetStoreIfSet() storage.Store {
return s.store
}
// SetDefaultUIDMap sets the default UID map to use when opening stores.
func (s *storageTransport) SetDefaultUIDMap(idmap []idtools.IDMap) {
s.defaultUIDMap = idmap
}
// SetDefaultGIDMap sets the default GID map to use when opening stores.
func (s *storageTransport) SetDefaultGIDMap(idmap []idtools.IDMap) {
s.defaultGIDMap = idmap
}
// DefaultUIDMap returns the default UID map used when opening stores.
func (s *storageTransport) DefaultUIDMap() []idtools.IDMap {
return s.defaultUIDMap
}
// DefaultGIDMap returns the default GID map used when opening stores.
func (s *storageTransport) DefaultGIDMap() []idtools.IDMap {
return s.defaultGIDMap
}
// ParseStoreReference takes a name or an ID, tries to figure out which it is
// relative to the given store, and returns it in a reference object.
func (s storageTransport) ParseStoreReference(store storage.Store, ref string) (*storageReference, error) {
if ref == "" {
return nil, fmt.Errorf("%q is an empty reference: %w", ref, ErrInvalidReference)
}
if ref[0] == '[' {
// Ignore the store specifier.
closeIndex := strings.IndexRune(ref, ']')
if closeIndex < 1 {
return nil, fmt.Errorf("store specifier in %q did not end: %w", ref, ErrInvalidReference)
}
ref = ref[closeIndex+1:]
}
// The reference may end with an image ID. Image IDs and digests use the same "@" separator;
// here we only peel away an image ID, and leave digests alone.
split := strings.LastIndex(ref, "@")
id := ""
if split != -1 {
possibleID := ref[split+1:]
if possibleID == "" {
return nil, fmt.Errorf("empty trailing digest or ID in %q: %w", ref, ErrInvalidReference)
}
// If it looks like a digest, leave it alone for now.
if _, err := digest.Parse(possibleID); err != nil {
// Otherwise…
if err := validateImageID(possibleID); err == nil {
id = possibleID // … it is a full ID
} else if img, err := store.Image(possibleID); err == nil && img != nil && len(possibleID) >= minimumTruncatedIDLength && strings.HasPrefix(img.ID, possibleID) {
// … it is a truncated version of the ID of an image that's present in local storage,
// so we might as well use the expanded value.
id = img.ID
} else {
return nil, fmt.Errorf("%q does not look like an image ID or digest: %w", possibleID, ErrInvalidReference)
}
// We have recognized an image ID; peel it off.
ref = ref[:split]
}
}
// If we only have one @-delimited portion, then _maybe_ it's a truncated image ID. Only check on that if it's
// at least of what we guess is a reasonable minimum length, because we don't want a really short value
// like "a" matching an image by ID prefix when the input was actually meant to specify an image name.
if id == "" && len(ref) >= minimumTruncatedIDLength && !strings.ContainsAny(ref, "@:") {
if img, err := store.Image(ref); err == nil && img != nil && strings.HasPrefix(img.ID, ref) {
// It's a truncated version of the ID of an image that's present in local storage;
// we need to expand it.
id = img.ID
ref = ""
}
}
var named reference.Named
// Unless we have an un-named "ID" or "@ID" reference (where ID might only have been a prefix), which has been
// completely parsed above, the initial portion should be a name, possibly with a tag and/or a digest..
if ref != "" {
var err error
named, err = reference.ParseNormalizedNamed(ref)
if err != nil {
return nil, fmt.Errorf("parsing named reference %q: %w", ref, err)
}
named = reference.TagNameOnly(named)
}
result, err := s.NewStoreReference(store, named, id)
if err != nil {
return nil, err
}
logrus.Debugf("parsed reference into %q", result.StringWithinTransport())
return result, nil
}
// NewStoreReference creates a reference for (named@ID) in store.
// either of name or ID can be unset; named must not be a reference.IsNameOnly.
func (s *storageTransport) NewStoreReference(store storage.Store, named reference.Named, id string) (*storageReference, error) {
return newReference(storageTransport{store: store, defaultUIDMap: s.defaultUIDMap, defaultGIDMap: s.defaultGIDMap}, named, id)
}
func (s *storageTransport) GetStore() (storage.Store, error) {
// Return the transport's previously-set store. If we don't have one
// of those, initialize one now.
if s.store == nil {
options, err := storage.DefaultStoreOptionsAutoDetectUID()
if err != nil {
return nil, err
}
options.UIDMap = s.defaultUIDMap
options.GIDMap = s.defaultGIDMap
store, err := storage.GetStore(options)
if err != nil {
return nil, err
}
s.store = store
}
return s.store, nil
}
// ParseReference takes a name and a tag or digest and/or ID
// ("_name_"/"@_id_"/"_name_:_tag_"/"_name_:_tag_@_id_"/"_name_@_digest_"/"_name_@_digest_@_id_"/"_name_:_tag_@_digest_"/"_name_:_tag_@_digest_@_id_"),
// possibly prefixed with a store specifier in the form "[_graphroot_]" or
// "[_driver_@_graphroot_]" or "[_driver_@_graphroot_+_runroot_]" or
// "[_driver_@_graphroot_:_options_]" or "[_driver_@_graphroot_+_runroot_:_options_]",
// tries to figure out which it is, and returns it in a reference object.
// If _id_ is the ID of an image that's present in local storage, it can be truncated, and
// even be specified as if it were a _name_, value.
func (s *storageTransport) ParseReference(reference string) (types.ImageReference, error) {
var store storage.Store
// Check if there's a store location prefix. If there is, then it
// needs to match a store that was previously initialized using
// storage.GetStore(), or be enough to let the storage library fill out
// the rest using knowledge that it has from elsewhere.
if len(reference) > 0 && reference[0] == '[' {
closeIndex := strings.IndexRune(reference, ']')
if closeIndex < 1 {
return nil, ErrInvalidReference
}
storeSpec := reference[1:closeIndex]
reference = reference[closeIndex+1:]
// Peel off a "driver@" from the start.
driverInfo := ""
driverPart1, driverPart2, gotDriver := strings.Cut(storeSpec, "@")
if !gotDriver {
storeSpec = driverPart1
if storeSpec == "" {
return nil, ErrInvalidReference
}
} else {
driverInfo = driverPart1
if driverInfo == "" {
return nil, ErrInvalidReference
}
storeSpec = driverPart2
if storeSpec == "" {
return nil, ErrInvalidReference
}
}
// Peel off a ":options" from the end.
var options []string
storeSpec, optionsPart, gotOptions := strings.Cut(storeSpec, ":")
if gotOptions {
options = strings.Split(optionsPart, ",")
}
// Peel off a "+runroot" from the new end.
storeSpec, runRootInfo, _ := strings.Cut(storeSpec, "+") // runRootInfo is "" if there is no "+"
// The rest is our graph root.
rootInfo := storeSpec
// Check that any paths are absolute paths.
if rootInfo != "" && !filepath.IsAbs(rootInfo) {
return nil, ErrPathNotAbsolute
}
if runRootInfo != "" && !filepath.IsAbs(runRootInfo) {
return nil, ErrPathNotAbsolute
}
store2, err := storage.GetStore(storage.StoreOptions{
GraphDriverName: driverInfo,
GraphRoot: rootInfo,
RunRoot: runRootInfo,
GraphDriverOptions: options,
UIDMap: s.defaultUIDMap,
GIDMap: s.defaultGIDMap,
})
if err != nil {
return nil, err
}
store = store2
} else {
// We didn't have a store spec, so use the default.
store2, err := s.GetStore()
if err != nil {
return nil, err
}
store = store2
}
return s.ParseStoreReference(store, reference)
}
// Deprecated: Surprisingly, with a StoreTransport reference which contains an ID,
// this ignores that ID; and repeated calls of GetStoreImage with the same named reference
// can return different images, with no way for the caller to "freeze" the storage.Image identity
// without discarding the name entirely.
//
// Also, a StoreTransport reference already contains a store, so providing another one is redundant.
//
// Use storage.ResolveReference instead; note that if the image is not found, ResolveReference returns
// c/image/v5/storage.ErrNoSuchImage, not c/storage.ErrImageUnknown.
func (s storageTransport) GetStoreImage(store storage.Store, ref types.ImageReference) (*storage.Image, error) {
dref := ref.DockerReference()
if dref != nil {
if img, err := store.Image(dref.String()); err == nil {
return img, nil
}
}
if sref, ok := ref.(*storageReference); ok {
tmpRef := *sref
if img, err := tmpRef.resolveImage(nil); err == nil {
return img, nil
}
}
return nil, storage.ErrImageUnknown
}
// Deprecated: Surprisingly, with a StoreTransport reference which contains an ID,
// this ignores that ID; and repeated calls of GetStoreImage with the same named reference
// can return different images, with no way for the caller to "freeze" the storage.Image identity
// without discarding the name entirely.
//
// Use storage.ResolveReference instead; note that if the image is not found, ResolveReference returns
// c/image/v5/storage.ErrNoSuchImage, not c/storage.ErrImageUnknown.
func (s *storageTransport) GetImage(ref types.ImageReference) (*storage.Image, error) {
store, err := s.GetStore()
if err != nil {
return nil, err
}
return s.GetStoreImage(store, ref)
}
func (s storageTransport) ValidatePolicyConfigurationScope(scope string) error {
// Check that there's a store location prefix. Values we're passed are
// expected to come from PolicyConfigurationIdentity or
// PolicyConfigurationNamespaces, so if there's no store location,
// something's wrong.
if scope[0] != '[' {
return ErrInvalidReference
}
// Parse the store location prefix.
closeIndex := strings.IndexRune(scope, ']')
if closeIndex < 1 {
return ErrInvalidReference
}
storeSpec := scope[1:closeIndex]
scope = scope[closeIndex+1:]
storeInfo := strings.SplitN(storeSpec, "@", 2)
if len(storeInfo) == 1 && storeInfo[0] != "" {
// One component: the graph root.
if !filepath.IsAbs(storeInfo[0]) {
return ErrPathNotAbsolute
}
} else if len(storeInfo) == 2 && storeInfo[0] != "" && storeInfo[1] != "" {
// Two components: the driver type and the graph root.
if !filepath.IsAbs(storeInfo[1]) {
return ErrPathNotAbsolute
}
} else {
// Anything else: scope specified in a form we don't
// recognize.
return ErrInvalidReference
}
// That might be all of it, and that's okay.
if scope == "" {
return nil
}
fields := strings.SplitN(scope, "@", 3)
switch len(fields) {
case 1: // name only
case 2: // name:tag@ID or name[:tag]@digest
if idErr := validateImageID(fields[1]); idErr != nil {
if _, digestErr := digest.Parse(fields[1]); digestErr != nil {
return fmt.Errorf("%v is neither a valid digest(%s) nor a valid ID(%s)", fields[1], digestErr.Error(), idErr.Error())
}
}
case 3: // name[:tag]@digest@ID
if _, err := digest.Parse(fields[1]); err != nil {
return err
}
if err := validateImageID(fields[2]); err != nil {
return err
}
default: // Coverage: This should never happen
return errors.New("Internal error: unexpected number of fields form strings.SplitN")
}
// As for field[0], if it is non-empty at all:
// FIXME? We could be verifying the various character set and length restrictions
// from docker/distribution/reference.regexp.go, but other than that there
// are few semantically invalid strings.
return nil
}
// validateImageID returns nil if id is a valid (full) image ID, or an error
func validateImageID(id string) error {
_, err := digest.Parse("sha256:" + id)
return err
}

61
vendor/github.com/containers/image/v5/tarball/doc.go generated vendored Normal file
View file

@ -0,0 +1,61 @@
// Package tarball provides a way to generate images using one or more layer
// tarballs and an optional template configuration.
//
// An example:
//
// package main
//
// import (
// "context"
//
// cp "github.com/containers/image/v5/copy"
// "github.com/containers/image/v5/signature"
// "github.com/containers/image/v5/tarball"
// "github.com/containers/image/v5/transports/alltransports"
// "github.com/containers/image/v5/types"
// imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
// )
//
// func imageFromTarball() {
// src, err := alltransports.ParseImageName("tarball:/var/cache/mock/fedora-26-x86_64/root_cache/cache.tar.gz")
// // - or -
// // src, err := tarball.Transport.ParseReference("/var/cache/mock/fedora-26-x86_64/root_cache/cache.tar.gz")
// if err != nil {
// panic(err)
// }
// updater, ok := src.(tarball.ConfigUpdater)
// if !ok {
// panic("unexpected: a tarball reference should implement tarball.ConfigUpdater")
// }
// config := imgspecv1.Image{
// Config: imgspecv1.ImageConfig{
// Cmd: []string{"/bin/bash"},
// },
// }
// annotations := make(map[string]string)
// annotations[imgspecv1.AnnotationDescription] = "test image built from a mock root cache"
// err = updater.ConfigUpdate(config, annotations)
// if err != nil {
// panic(err)
// }
// dest, err := alltransports.ParseImageName("docker-daemon:mock:latest")
// if err != nil {
// panic(err)
// }
//
// policy, err := signature.DefaultPolicy(nil)
// if err != nil {
// panic(err)
// }
//
// pc, err := signature.NewPolicyContext(policy)
// if err != nil {
// panic(err)
// }
// defer pc.Destroy()
// _, err = cp.Image(context.TODO(), pc, dest, src, nil)
// if err != nil {
// panic(err)
// }
// }
package tarball

View file

@ -0,0 +1,82 @@
package tarball
import (
"context"
"fmt"
"os"
"strings"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/image"
"github.com/containers/image/v5/types"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/exp/maps"
)
// ConfigUpdater is an interface that ImageReferences for "tarball" images also
// implement. It can be used to set values for a configuration, and to set
// image annotations which will be present in the images returned by the
// reference's NewImage() or NewImageSource() methods.
type ConfigUpdater interface {
ConfigUpdate(config imgspecv1.Image, annotations map[string]string) error
}
type tarballReference struct {
config imgspecv1.Image
annotations map[string]string
filenames []string
stdin []byte
}
// ConfigUpdate updates the image's default configuration and adds annotations
// which will be visible in source images created using this reference.
func (r *tarballReference) ConfigUpdate(config imgspecv1.Image, annotations map[string]string) error {
r.config = config
if r.annotations == nil {
r.annotations = make(map[string]string)
}
maps.Copy(r.annotations, annotations)
return nil
}
func (r *tarballReference) Transport() types.ImageTransport {
return Transport
}
func (r *tarballReference) StringWithinTransport() string {
return strings.Join(r.filenames, ":")
}
func (r *tarballReference) DockerReference() reference.Named {
return nil
}
func (r *tarballReference) PolicyConfigurationIdentity() string {
return ""
}
func (r *tarballReference) PolicyConfigurationNamespaces() []string {
return nil
}
// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport.
// The caller must call .Close() on the returned ImageCloser.
// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource,
// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details.
func (r *tarballReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) {
return image.FromReference(ctx, sys, r)
}
func (r *tarballReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error {
for _, filename := range r.filenames {
if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("error removing %q: %w", filename, err)
}
}
return nil
}
func (r *tarballReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) {
return nil, fmt.Errorf(`"tarball:" locations can only be read from, not written to`)
}

View file

@ -0,0 +1,234 @@
package tarball
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"runtime"
"strings"
"time"
"github.com/containers/image/v5/internal/imagesource/impl"
"github.com/containers/image/v5/internal/imagesource/stubs"
"github.com/containers/image/v5/types"
"github.com/klauspost/pgzip"
digest "github.com/opencontainers/go-digest"
imgspecs "github.com/opencontainers/image-spec/specs-go"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/exp/maps"
)
type tarballImageSource struct {
impl.Compat
impl.PropertyMethodsInitialize
impl.NoSignatures
impl.DoesNotAffectLayerInfosForCopy
stubs.NoGetBlobAtInitialize
reference tarballReference
blobs map[digest.Digest]tarballBlob
manifest []byte
}
// tarballBlob is a blob that tarballImagSource can return by GetBlob.
type tarballBlob struct {
contents []byte // or nil to read from filename below
filename string // valid if contents == nil
size int64
}
func (r *tarballReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) {
// Pick up the layer comment from the configuration's history list, if one is set.
comment := "imported from tarball"
if len(r.config.History) > 0 && r.config.History[0].Comment != "" {
comment = r.config.History[0].Comment
}
// Gather up the digests, sizes, and history information for all of the files.
blobs := map[digest.Digest]tarballBlob{}
diffIDs := []digest.Digest{}
created := time.Time{}
history := []imgspecv1.History{}
layerDescriptors := []imgspecv1.Descriptor{}
for _, filename := range r.filenames {
var reader io.Reader
var blobTime time.Time
var blob tarballBlob
if filename == "-" {
reader = bytes.NewReader(r.stdin)
blobTime = time.Now()
blob = tarballBlob{
contents: r.stdin,
size: int64(len(r.stdin)),
}
} else {
file, err := os.Open(filename)
if err != nil {
return nil, err
}
defer file.Close()
reader = file
fileinfo, err := file.Stat()
if err != nil {
return nil, fmt.Errorf("error reading size of %q: %w", filename, err)
}
blobTime = fileinfo.ModTime()
blob = tarballBlob{
filename: filename,
size: fileinfo.Size(),
}
}
// Default to assuming the layer is compressed.
layerType := imgspecv1.MediaTypeImageLayerGzip
// Set up to digest the file as it is.
blobIDdigester := digest.Canonical.Digester()
reader = io.TeeReader(reader, blobIDdigester.Hash())
// Set up to digest the file after we maybe decompress it.
diffIDdigester := digest.Canonical.Digester()
uncompressed, err := pgzip.NewReader(reader)
if err == nil {
// It is compressed, so the diffID is the digest of the uncompressed version
reader = io.TeeReader(uncompressed, diffIDdigester.Hash())
} else {
// It is not compressed, so the diffID and the blobID are going to be the same
diffIDdigester = blobIDdigester
layerType = imgspecv1.MediaTypeImageLayer
uncompressed = nil
}
// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
if _, err := io.Copy(io.Discard, reader); err != nil {
return nil, fmt.Errorf("error reading %q: %v", filename, err)
}
if uncompressed != nil {
uncompressed.Close()
}
// Grab our uncompressed and possibly-compressed digests and sizes.
diffID := diffIDdigester.Digest()
blobID := blobIDdigester.Digest()
diffIDs = append(diffIDs, diffID)
blobs[blobID] = blob
history = append(history, imgspecv1.History{
Created: &blobTime,
CreatedBy: fmt.Sprintf("/bin/sh -c #(nop) ADD file:%s in %c", diffID.Hex(), os.PathSeparator),
Comment: comment,
})
// Use the mtime of the most recently modified file as the image's creation time.
if created.Before(blobTime) {
created = blobTime
}
layerDescriptors = append(layerDescriptors, imgspecv1.Descriptor{
Digest: blobID,
Size: blob.size,
MediaType: layerType,
})
}
// Pick up other defaults from the config in the reference.
config := r.config
if config.Created == nil {
config.Created = &created
}
if config.Architecture == "" {
config.Architecture = runtime.GOARCH
}
if config.OS == "" {
config.OS = runtime.GOOS
}
config.RootFS = imgspecv1.RootFS{
Type: "layers",
DiffIDs: diffIDs,
}
config.History = history
// Encode and digest the image configuration blob.
configBytes, err := json.Marshal(&config)
if err != nil {
return nil, fmt.Errorf("error generating configuration blob for %q: %v", strings.Join(r.filenames, separator), err)
}
configID := digest.Canonical.FromBytes(configBytes)
blobs[configID] = tarballBlob{
contents: configBytes,
size: int64(len(configBytes)),
}
// Populate a manifest with the configuration blob and the layers.
manifest := imgspecv1.Manifest{
Versioned: imgspecs.Versioned{
SchemaVersion: 2,
},
Config: imgspecv1.Descriptor{
Digest: configID,
Size: int64(len(configBytes)),
MediaType: imgspecv1.MediaTypeImageConfig,
},
Layers: layerDescriptors,
Annotations: maps.Clone(r.annotations),
}
// Encode the manifest.
manifestBytes, err := json.Marshal(&manifest)
if err != nil {
return nil, fmt.Errorf("error generating manifest for %q: %v", strings.Join(r.filenames, separator), err)
}
// Return the image.
src := &tarballImageSource{
PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{
HasThreadSafeGetBlob: false,
}),
NoGetBlobAtInitialize: stubs.NoGetBlobAt(r),
reference: *r,
blobs: blobs,
manifest: manifestBytes,
}
src.Compat = impl.AddCompat(src)
return src, nil
}
func (is *tarballImageSource) Close() error {
return nil
}
// GetBlob returns a stream for the specified blob, and the blobs size (or -1 if unknown).
// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
func (is *tarballImageSource) GetBlob(ctx context.Context, blobinfo types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
blob, ok := is.blobs[blobinfo.Digest]
if !ok {
return nil, -1, fmt.Errorf("no blob with digest %q found", blobinfo.Digest.String())
}
if blob.contents != nil {
return io.NopCloser(bytes.NewReader(blob.contents)), int64(len(blob.contents)), nil
}
reader, err := os.Open(blob.filename)
if err != nil {
return nil, -1, err
}
return reader, blob.size, nil
}
// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
// It may use a remote (= slow) service.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list);
// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists).
func (is *tarballImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
if instanceDigest != nil {
return nil, "", fmt.Errorf("manifest lists are not supported by the %q transport", transportName)
}
return is.manifest, imgspecv1.MediaTypeImageManifest, nil
}
func (is *tarballImageSource) Reference() types.ImageReference {
return &is.reference
}

View file

@ -0,0 +1,75 @@
package tarball
import (
"errors"
"fmt"
"io"
"os"
"strings"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
)
const (
transportName = "tarball"
separator = ":"
)
var (
// Transport implements the types.ImageTransport interface for "tarball:" images,
// which are makeshift images constructed using one or more possibly-compressed tar
// archives.
Transport = &tarballTransport{}
)
type tarballTransport struct {
}
func (t *tarballTransport) Name() string {
return transportName
}
func (t *tarballTransport) ParseReference(reference string) (types.ImageReference, error) {
var stdin []byte
var err error
filenames := strings.Split(reference, separator)
for _, filename := range filenames {
if filename == "-" {
stdin, err = io.ReadAll(os.Stdin)
if err != nil {
return nil, fmt.Errorf("error buffering stdin: %v", err)
}
continue
}
f, err := os.Open(filename)
if err != nil {
return nil, fmt.Errorf("error opening %q: %v", filename, err)
}
f.Close()
}
return NewReference(filenames, stdin)
}
// NewReference creates a new "tarball:" reference for the listed fileNames.
// If any of the fileNames is "-", the contents of stdin are used instead.
func NewReference(fileNames []string, stdin []byte) (types.ImageReference, error) {
for _, path := range fileNames {
if strings.Contains(path, separator) {
return nil, fmt.Errorf("Invalid path %q: paths including the separator %q are not supported", path, separator)
}
}
return &tarballReference{
filenames: fileNames,
stdin: stdin,
}, nil
}
func (t *tarballTransport) ValidatePolicyConfigurationScope(scope string) error {
// See the explanation in daemonReference.PolicyConfigurationIdentity.
return errors.New(`tarball: does not support any scopes except the default "" one`)
}
func init() {
transports.Register(Transport)
}

View file

@ -0,0 +1,49 @@
package alltransports
import (
"fmt"
"strings"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
// Register all known transports.
// NOTE: Make sure docs/containers-transports.5.md and docs/containers-policy.json.5.md are updated when adding or updating
// a transport.
_ "github.com/containers/image/v5/directory"
_ "github.com/containers/image/v5/docker"
_ "github.com/containers/image/v5/docker/archive"
_ "github.com/containers/image/v5/oci/archive"
_ "github.com/containers/image/v5/oci/layout"
_ "github.com/containers/image/v5/openshift"
_ "github.com/containers/image/v5/sif"
_ "github.com/containers/image/v5/tarball"
// The docker-daemon transport is registeredy by docker_daemon*.go
// The ostree transport is registered by ostree*.go
// The storage transport is registered by storage*.go
)
// ParseImageName converts a URL-like image name to a types.ImageReference.
func ParseImageName(imgName string) (types.ImageReference, error) {
// Keep this in sync with TransportFromImageName!
transportName, withinTransport, valid := strings.Cut(imgName, ":")
if !valid {
return nil, fmt.Errorf(`Invalid image name "%s", expected colon-separated transport:reference`, imgName)
}
transport := transports.Get(transportName)
if transport == nil {
return nil, fmt.Errorf(`Invalid image name "%s", unknown transport "%s"`, imgName, transportName)
}
return transport.ParseReference(withinTransport)
}
// TransportFromImageName converts an URL-like name to a types.ImageTransport or nil when
// the transport is unknown or when the input is invalid.
func TransportFromImageName(imageName string) types.ImageTransport {
// Keep this in sync with ParseImageName!
transportName, _, valid := strings.Cut(imageName, ":")
if valid {
return transports.Get(transportName)
}
return nil
}

View file

@ -0,0 +1,9 @@
//go:build !containers_image_docker_daemon_stub
// +build !containers_image_docker_daemon_stub
package alltransports
import (
// Register the docker-daemon transport
_ "github.com/containers/image/v5/docker/daemon"
)

View file

@ -0,0 +1,10 @@
//go:build containers_image_docker_daemon_stub
// +build containers_image_docker_daemon_stub
package alltransports
import "github.com/containers/image/v5/transports"
func init() {
transports.Register(transports.NewStubTransport("docker-daemon"))
}

View file

@ -0,0 +1,9 @@
//go:build containers_image_ostree && linux
// +build containers_image_ostree,linux
package alltransports
import (
// Register the ostree transport
_ "github.com/containers/image/v5/ostree"
)

View file

@ -0,0 +1,10 @@
//go:build !containers_image_ostree || !linux
// +build !containers_image_ostree !linux
package alltransports
import "github.com/containers/image/v5/transports"
func init() {
transports.Register(transports.NewStubTransport("ostree"))
}

View file

@ -0,0 +1,9 @@
//go:build !containers_image_storage_stub
// +build !containers_image_storage_stub
package alltransports
import (
// Register the storage transport
_ "github.com/containers/image/v5/storage"
)

View file

@ -0,0 +1,10 @@
//go:build containers_image_storage_stub
// +build containers_image_storage_stub
package alltransports
import "github.com/containers/image/v5/transports"
func init() {
transports.Register(transports.NewStubTransport("containers-storage"))
}

View file

@ -8,7 +8,7 @@ const (
// VersionMinor is for functionality in a backwards-compatible manner
VersionMinor = 29
// VersionPatch is for backwards-compatible bug fixes
VersionPatch = 0
VersionPatch = 1
// VersionDev indicates development branch. Releases will be empty string.
VersionDev = ""