go.mod: update to images@v0.117.0

This commit updates to images v0.117.0 so that the cross-distro.sh
test works again (images removed fedora-39.json in main but the
uses the previous version of images that includes fedora-39 so
there is a mismatch (we should look into if there is a way to
get github.com/osbuild/images@latest instead of main in the
cross-arch test).

It also updates all the vendor stuff that got pulled via the
new images release (which is giantonormous).

This update requires updating the Go version to 1.22.8

vendor/github.com/containers/storage/.cirrus.yml

@@ -17,13 +17,13 @@ env:
     ####
     #### Cache-image names to test with (double-quotes around names are critical)
     ###
-    FEDORA_NAME: "fedora-39"
+    FEDORA_NAME: "fedora-41"
     DEBIAN_NAME: "debian-13"
 
     # GCE project where images live
     IMAGE_PROJECT: "libpod-218412"
     # VM Image built in containers/automation_images
-    IMAGE_SUFFIX: "c20241010t105554z-f40f39d13"
+    IMAGE_SUFFIX: "c20250107t132430z-f41f40d13"
     FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
     DEBIAN_CACHE_IMAGE_NAME: "debian-${IMAGE_SUFFIX}"
 

vendor/github.com/containers/storage/Makefile

@@ -35,7 +35,7 @@ TESTFLAGS := $(shell $(GO) test -race $(BUILDFLAGS) ./pkg/stringutils 2>&1 > /de
 # N/B: This value is managed by Renovate, manual changes are
 # possible, as long as they don't disturb the formatting
 # (i.e. DO NOT ADD A 'v' prefix!)
-GOLANGCI_LINT_VERSION := 1.61.0
+GOLANGCI_LINT_VERSION := 1.63.4
 
 default all: local-binary docs local-validate local-cross ## validate all checks, build and cross-build\nbinaries and docs
 

vendor/github.com/containers/storage/VERSION

@@ -1 +1 @@
-1.56.1
+1.57.1

vendor/github.com/containers/storage/check.go

@@ -80,7 +80,7 @@ type CheckOptions struct {
 // layer to the contents that we'd expect it to have to ignore certain
 // discrepancies
 type checkIgnore struct {
-	ownership, timestamps, permissions bool
+	ownership, timestamps, permissions, filetype bool
 }
 
 // CheckMost returns a CheckOptions with mostly just "quick" checks enabled.
@@ -139,8 +139,10 @@ func (s *store) Check(options *CheckOptions) (CheckReport, error) {
 		if strings.Contains(o, "ignore_chown_errors=true") {
 			ignore.ownership = true
 		}
-		if strings.HasPrefix(o, "force_mask=") {
+		if strings.Contains(o, "force_mask=") {
+			ignore.ownership = true
 			ignore.permissions = true
+			ignore.filetype = true
 		}
 	}
 	for o := range s.pullOptions {
@@ -833,7 +835,7 @@ func (s *store) Repair(report CheckReport, options *RepairOptions) []error {
 // compareFileInfo returns a string summarizing what's different between the two checkFileInfos
 func compareFileInfo(a, b checkFileInfo, idmap *idtools.IDMappings, ignore checkIgnore) string {
 	var comparison []string
-	if a.typeflag != b.typeflag {
+	if a.typeflag != b.typeflag && !ignore.filetype {
 		comparison = append(comparison, fmt.Sprintf("filetype:%v￫%v", a.typeflag, b.typeflag))
 	}
 	if idmap != nil && !idmap.Empty() {

vendor/github.com/containers/storage/drivers/aufs/aufs.go

@@ -776,3 +776,8 @@ func (a *Driver) UpdateLayerIDMap(id string, toContainer, toHost *idtools.IDMapp
 func (a *Driver) SupportsShifting() bool {
 	return false
 }
+
+// Dedup performs deduplication of the driver's storage.
+func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) {
+	return graphdriver.DedupResult{}, nil
+}

vendor/github.com/containers/storage/drivers/btrfs/btrfs.go

@@ -673,3 +673,8 @@ func (d *Driver) ListLayers() ([]string, error) {
 func (d *Driver) AdditionalImageStores() []string {
 	return nil
 }
+
+// Dedup performs deduplication of the driver's storage.
+func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) {
+	return graphdriver.DedupResult{}, nil
+}

vendor/github.com/containers/storage/drivers/chown_darwin.go

@@ -83,7 +83,7 @@ func (c *platformChowner) LChown(path string, info os.FileInfo, toHost, toContai
 	}
 	if uid != int(st.Uid) || gid != int(st.Gid) {
 		capability, err := system.Lgetxattr(path, "security.capability")
-		if err != nil && !errors.Is(err, system.EOPNOTSUPP) && err != system.ErrNotSupportedPlatform {
+		if err != nil && !errors.Is(err, system.ENOTSUP) && err != system.ErrNotSupportedPlatform {
 			return fmt.Errorf("%s: %w", os.Args[0], err)
 		}
 

vendor/github.com/containers/storage/drivers/chown_unix.go

@@ -101,7 +101,7 @@ func (c *platformChowner) LChown(path string, info os.FileInfo, toHost, toContai
 	}
 	if uid != int(st.Uid) || gid != int(st.Gid) {
 		cap, err := system.Lgetxattr(path, "security.capability")
-		if err != nil && !errors.Is(err, system.EOPNOTSUPP) && !errors.Is(err, system.EOVERFLOW) && err != system.ErrNotSupportedPlatform {
+		if err != nil && !errors.Is(err, system.ENOTSUP) && !errors.Is(err, system.EOVERFLOW) && err != system.ErrNotSupportedPlatform {
 			return fmt.Errorf("%s: %w", os.Args[0], err)
 		}
 

vendor/github.com/containers/storage/drivers/copy/copy_linux.go

@@ -106,7 +106,7 @@ func legacyCopy(srcFile io.Reader, dstFile io.Writer) error {
 
 func copyXattr(srcPath, dstPath, attr string) error {
 	data, err := system.Lgetxattr(srcPath, attr)
-	if err != nil && !errors.Is(err, unix.EOPNOTSUPP) {
+	if err != nil && !errors.Is(err, system.ENOTSUP) {
 		return err
 	}
 	if data != nil {
@@ -279,7 +279,7 @@ func doCopyXattrs(srcPath, dstPath string) error {
 	}
 
 	xattrs, err := system.Llistxattr(srcPath)
-	if err != nil && !errors.Is(err, unix.EOPNOTSUPP) {
+	if err != nil && !errors.Is(err, system.ENOTSUP) {
 		return err
 	}
 

vendor/github.com/containers/storage/drivers/driver.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/containers/storage/internal/dedup"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/directory"
 	"github.com/containers/storage/pkg/fileutils"
@@ -81,6 +82,23 @@ type ApplyDiffWithDifferOpts struct {
 	Flags map[string]interface{}
 }
 
+// DedupArgs contains the information to perform storage deduplication.
+type DedupArgs struct {
+	// Layers is the list of layers to deduplicate.
+	Layers []string
+
+	// Options that are passed directly to the pkg/dedup.DedupDirs function.
+	Options dedup.DedupOptions
+}
+
+// DedupResult contains the result of the Dedup() call.
+type DedupResult struct {
+	// Deduped represents the total number of bytes saved by deduplication.
+	// This value accounts also for all previously deduplicated data, not only the savings
+	// from the last run.
+	Deduped uint64
+}
+
 // InitFunc initializes the storage driver.
 type InitFunc func(homedir string, options Options) (Driver, error)
 
@@ -139,6 +157,8 @@ type ProtoDriver interface {
 	// AdditionalImageStores returns additional image stores supported by the driver
 	// This API is experimental and can be changed without bumping the major version number.
 	AdditionalImageStores() []string
+	// Dedup performs deduplication of the driver's storage.
+	Dedup(DedupArgs) (DedupResult, error)
 }
 
 // DiffDriver is the interface to use to implement graph diffs
@@ -211,8 +231,8 @@ const (
 	// DifferOutputFormatDir means the output is a directory and it will
 	// keep the original layout.
 	DifferOutputFormatDir = iota
-	// DifferOutputFormatFlat will store the files by their checksum, in the form
-	// checksum[0:2]/checksum[2:]
+	// DifferOutputFormatFlat will store the files by their checksum, per
+	// pkg/chunked/internal/composefs.RegularFilePathForValidatedDigest.
 	DifferOutputFormatFlat
 )
 

vendor/github.com/containers/storage/drivers/overlay/check_116.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/system"
-	"golang.org/x/sys/unix"
 )
 
 func scanForMountProgramIndicators(home string) (detected bool, err error) {
@@ -28,7 +27,7 @@ func scanForMountProgramIndicators(home string) (detected bool, err error) {
 		}
 		if d.IsDir() {
 			xattrs, err := system.Llistxattr(path)
-			if err != nil && !errors.Is(err, unix.EOPNOTSUPP) {
+			if err != nil && !errors.Is(err, system.ENOTSUP) {
 				return err
 			}
 			for _, xattr := range xattrs {

vendor/github.com/containers/storage/drivers/overlay/composefs.go

@@ -1,4 +1,4 @@
-//go:build linux && cgo
+//go:build linux
 
 package overlay
 
@@ -27,7 +27,7 @@ var (
 	composeFsHelperErr  error
 
 	// skipMountViaFile is used to avoid trying to mount EROFS directly via the file if we already know the current kernel
-	// does not support it.  Mounting directly via a file will be supported in kernel 6.12.
+	// does not support it.  Mounting directly via a file is supported from Linux 6.12.
 	skipMountViaFile atomic.Bool
 )
 

vendor/github.com/containers/storage/drivers/overlay/overlay.go

@@ -22,6 +22,7 @@ import (
 	graphdriver "github.com/containers/storage/drivers"
 	"github.com/containers/storage/drivers/overlayutils"
 	"github.com/containers/storage/drivers/quota"
+	"github.com/containers/storage/internal/dedup"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/chrootarchive"
 	"github.com/containers/storage/pkg/directory"
@@ -1096,6 +1097,7 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, readOnl
 	}
 
 	if d.options.forceMask != nil {
+		st.Mode |= os.ModeDir
 		if err := idtools.SetContainersOverrideXattr(diff, st); err != nil {
 			return err
 		}
@@ -2740,3 +2742,22 @@ func getMappedMountRoot(path string) string {
 	}
 	return dirName
 }
+
+// Dedup performs deduplication of the driver's storage.
+func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) {
+	var dirs []string
+	for _, layer := range req.Layers {
+		dir, _, inAdditionalStore := d.dir2(layer, false)
+		if inAdditionalStore {
+			continue
+		}
+		if err := fileutils.Exists(dir); err == nil {
+			dirs = append(dirs, filepath.Join(dir, "diff"))
+		}
+	}
+	r, err := dedup.DedupDirs(dirs, req.Options)
+	if err != nil {
+		return graphdriver.DedupResult{}, err
+	}
+	return graphdriver.DedupResult{Deduped: r.Deduped}, nil
+}

vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go

@@ -1,23 +0,0 @@
-//go:build linux && !cgo
-
-package overlay
-
-import (
-	"fmt"
-)
-
-func openComposefsMount(dataDir string) (int, error) {
-	return 0, fmt.Errorf("composefs not supported on this build")
-}
-
-func getComposeFsHelper() (string, error) {
-	return "", fmt.Errorf("composefs not supported on this build")
-}
-
-func mountComposefsBlob(dataDir, mountPoint string) error {
-	return fmt.Errorf("composefs not supported on this build")
-}
-
-func generateComposeFsBlob(verityDigests map[string]string, toc interface{}, composefsDir string) error {
-	return fmt.Errorf("composefs not supported on this build")
-}

vendor/github.com/containers/storage/drivers/register/register_overlay.go

@@ -1,4 +1,4 @@
-//go:build !exclude_graphdriver_overlay && linux && cgo
+//go:build !exclude_graphdriver_overlay && linux
 
 package register
 

vendor/github.com/containers/storage/drivers/vfs/driver.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	graphdriver "github.com/containers/storage/drivers"
+	"github.com/containers/storage/internal/dedup"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/directory"
 	"github.com/containers/storage/pkg/fileutils"
@@ -348,3 +349,19 @@ func (d *Driver) Diff(id string, idMappings *idtools.IDMappings, parent string,
 func (d *Driver) DiffSize(id string, idMappings *idtools.IDMappings, parent string, parentMappings *idtools.IDMappings, mountLabel string) (size int64, err error) {
 	return d.naiveDiff.DiffSize(id, idMappings, parent, parentMappings, mountLabel)
 }
+
+// Dedup performs deduplication of the driver's storage.
+func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) {
+	var dirs []string
+	for _, layer := range req.Layers {
+		dir := d.dir2(layer, false)
+		if err := fileutils.Exists(dir); err == nil {
+			dirs = append(dirs, dir)
+		}
+	}
+	r, err := dedup.DedupDirs(dirs, req.Options)
+	if err != nil {
+		return graphdriver.DedupResult{}, err
+	}
+	return graphdriver.DedupResult{Deduped: r.Deduped}, nil
+}

vendor/github.com/containers/storage/drivers/windows/windows.go

@@ -975,6 +975,11 @@ func (d *Driver) AdditionalImageStores() []string {
 	return nil
 }
 
+// Dedup performs deduplication of the driver's storage.
+func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) {
+	return graphdriver.DedupResult{}, nil
+}
+
 // UpdateLayerIDMap changes ownerships in the layer's filesystem tree from
 // matching those in toContainer to matching those in toHost.
 func (d *Driver) UpdateLayerIDMap(id string, toContainer, toHost *idtools.IDMappings, mountLabel string) error {

vendor/github.com/containers/storage/drivers/zfs/zfs.go

@@ -511,3 +511,8 @@ func (d *Driver) ListLayers() ([]string, error) {
 func (d *Driver) AdditionalImageStores() []string {
 	return nil
 }
+
+// Dedup performs deduplication of the driver's storage.
+func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) {
+	return graphdriver.DedupResult{}, nil
+}

vendor/github.com/containers/storage/internal/dedup/dedup.go

@@ -0,0 +1,163 @@
+package dedup
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash/crc64"
+	"io/fs"
+	"sync"
+
+	"github.com/opencontainers/selinux/pkg/pwalkdir"
+	"github.com/sirupsen/logrus"
+)
+
+var notSupported = errors.New("reflinks are not supported on this platform")
+
+const (
+	DedupHashInvalid DedupHashMethod = iota
+	DedupHashCRC
+	DedupHashFileSize
+	DedupHashSHA256
+)
+
+type DedupHashMethod int
+
+type DedupOptions struct {
+	// HashMethod is the hash function to use to find identical files
+	HashMethod DedupHashMethod
+}
+
+type DedupResult struct {
+	// Deduped represents the total number of bytes saved by deduplication.
+	// This value accounts also for all previously deduplicated data, not only the savings
+	// from the last run.
+	Deduped uint64
+}
+
+func getFileChecksum(hashMethod DedupHashMethod, path string, info fs.FileInfo) (string, error) {
+	switch hashMethod {
+	case DedupHashInvalid:
+		return "", fmt.Errorf("invalid hash method: %v", hashMethod)
+	case DedupHashFileSize:
+		return fmt.Sprintf("%v", info.Size()), nil
+	case DedupHashSHA256:
+		return readAllFile(path, info, func(buf []byte) (string, error) {
+			h := sha256.New()
+			if _, err := h.Write(buf); err != nil {
+				return "", err
+			}
+			return string(h.Sum(nil)), nil
+		})
+	case DedupHashCRC:
+		return readAllFile(path, info, func(buf []byte) (string, error) {
+			c := crc64.New(crc64.MakeTable(crc64.ECMA))
+			if _, err := c.Write(buf); err != nil {
+				return "", err
+			}
+			bufRet := make([]byte, 8)
+			binary.BigEndian.PutUint64(bufRet, c.Sum64())
+			return string(bufRet), nil
+		})
+	default:
+		return "", fmt.Errorf("unknown hash method: %v", hashMethod)
+	}
+}
+
+type pathsLocked struct {
+	paths []string
+	lock  sync.Mutex
+}
+
+func DedupDirs(dirs []string, options DedupOptions) (DedupResult, error) {
+	res := DedupResult{}
+	hashToPaths := make(map[string]*pathsLocked)
+	lock := sync.Mutex{} // protects `hashToPaths` and `res`
+
+	dedup, err := newDedupFiles()
+	if err != nil {
+		return res, err
+	}
+
+	for _, dir := range dirs {
+		logrus.Debugf("Deduping directory %s", dir)
+		if err := pwalkdir.Walk(dir, func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if !d.Type().IsRegular() {
+				return nil
+			}
+			info, err := d.Info()
+			if err != nil {
+				return err
+			}
+			size := uint64(info.Size())
+			if size == 0 {
+				// do not bother with empty files
+				return nil
+			}
+
+			// the file was already deduplicated
+			if visited, err := dedup.isFirstVisitOf(info); err != nil {
+				return err
+			} else if visited {
+				return nil
+			}
+
+			h, err := getFileChecksum(options.HashMethod, path, info)
+			if err != nil {
+				return err
+			}
+
+			lock.Lock()
+			item, foundItem := hashToPaths[h]
+			if !foundItem {
+				item = &pathsLocked{paths: []string{path}}
+				hashToPaths[h] = item
+				lock.Unlock()
+				return nil
+			}
+			item.lock.Lock()
+			lock.Unlock()
+
+			dedupBytes, err := func() (uint64, error) { // function to have a scope for the defer statement
+				defer item.lock.Unlock()
+
+				var dedupBytes uint64
+				for _, src := range item.paths {
+					deduped, err := dedup.dedup(src, path, info)
+					if err == nil && deduped > 0 {
+						logrus.Debugf("Deduped %q -> %q (%d bytes)", src, path, deduped)
+						dedupBytes += deduped
+						break
+					}
+					logrus.Debugf("Failed to deduplicate: %v", err)
+					if errors.Is(err, notSupported) {
+						return dedupBytes, err
+					}
+				}
+				if dedupBytes == 0 {
+					item.paths = append(item.paths, path)
+				}
+				return dedupBytes, nil
+			}()
+			if err != nil {
+				return err
+			}
+
+			lock.Lock()
+			res.Deduped += dedupBytes
+			lock.Unlock()
+			return nil
+		}); err != nil {
+			// if reflinks are not supported, return immediately without errors
+			if errors.Is(err, notSupported) {
+				return res, nil
+			}
+			return res, err
+		}
+	}
+	return res, nil
+}

vendor/github.com/containers/storage/internal/dedup/dedup_linux.go

@@ -0,0 +1,139 @@
+package dedup
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"sync"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+type deviceInodePair struct {
+	dev uint64
+	ino uint64
+}
+
+type dedupFiles struct {
+	lock          sync.Mutex
+	visitedInodes map[deviceInodePair]struct{}
+}
+
+func newDedupFiles() (*dedupFiles, error) {
+	return &dedupFiles{
+		visitedInodes: make(map[deviceInodePair]struct{}),
+	}, nil
+}
+
+func (d *dedupFiles) recordInode(dev, ino uint64) (bool, error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	di := deviceInodePair{
+		dev: dev,
+		ino: ino,
+	}
+
+	_, visited := d.visitedInodes[di]
+	d.visitedInodes[di] = struct{}{}
+	return visited, nil
+}
+
+// isFirstVisitOf records that the file is being processed.  Returns true if the file was already visited.
+func (d *dedupFiles) isFirstVisitOf(fi fs.FileInfo) (bool, error) {
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return false, fmt.Errorf("unable to get raw syscall.Stat_t data")
+	}
+	return d.recordInode(uint64(st.Dev), st.Ino)
+}
+
+// dedup deduplicates the file at src path to dst path
+func (d *dedupFiles) dedup(src, dst string, fiDst fs.FileInfo) (uint64, error) {
+	srcFile, err := os.OpenFile(src, os.O_RDONLY, 0)
+	if err != nil {
+		return 0, fmt.Errorf("failed to open source file: %w", err)
+	}
+	defer srcFile.Close()
+
+	dstFile, err := os.OpenFile(dst, os.O_WRONLY, 0)
+	if err != nil {
+		return 0, fmt.Errorf("failed to open destination file: %w", err)
+	}
+	defer dstFile.Close()
+
+	stSrc, err := srcFile.Stat()
+	if err != nil {
+		return 0, fmt.Errorf("failed to stat source file: %w", err)
+	}
+	sSrc, ok := stSrc.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, fmt.Errorf("unable to get raw syscall.Stat_t data")
+	}
+	sDest, ok := fiDst.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, fmt.Errorf("unable to get raw syscall.Stat_t data")
+	}
+	if sSrc.Dev == sDest.Dev && sSrc.Ino == sDest.Ino {
+		// same inode, we are dealing with a hard link, no need to deduplicate
+		return 0, nil
+	}
+
+	value := unix.FileDedupeRange{
+		Src_offset: 0,
+		Src_length: uint64(stSrc.Size()),
+		Info: []unix.FileDedupeRangeInfo{
+			{
+				Dest_fd:     int64(dstFile.Fd()),
+				Dest_offset: 0,
+			},
+		},
+	}
+	err = unix.IoctlFileDedupeRange(int(srcFile.Fd()), &value)
+	if err == nil {
+		return uint64(value.Info[0].Bytes_deduped), nil
+	}
+
+	if errors.Is(err, unix.ENOTSUP) {
+		return 0, notSupported
+	}
+	return 0, fmt.Errorf("failed to clone file %q: %w", src, err)
+}
+
+func readAllFile(path string, info fs.FileInfo, fn func([]byte) (string, error)) (string, error) {
+	size := info.Size()
+	if size == 0 {
+		return fn(nil)
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	if size < 4096 {
+		// small file, read it all
+		data := make([]byte, size)
+		_, err = io.ReadFull(file, data)
+		if err != nil {
+			return "", err
+		}
+		return fn(data)
+	}
+
+	mmap, err := unix.Mmap(int(file.Fd()), 0, int(size), unix.PROT_READ, unix.MAP_PRIVATE)
+	if err != nil {
+		return "", fmt.Errorf("failed to mmap file: %w", err)
+	}
+	defer func() {
+		_ = unix.Munmap(mmap)
+	}()
+
+	_ = unix.Madvise(mmap, unix.MADV_SEQUENTIAL)
+
+	return fn(mmap)
+}

vendor/github.com/containers/storage/internal/dedup/dedup_unsupported.go

@@ -0,0 +1,27 @@
+//go:build !linux
+
+package dedup
+
+import (
+	"io/fs"
+)
+
+type dedupFiles struct{}
+
+func newDedupFiles() (*dedupFiles, error) {
+	return nil, notSupported
+}
+
+// isFirstVisitOf records that the file is being processed.  Returns true if the file was already visited.
+func (d *dedupFiles) isFirstVisitOf(fi fs.FileInfo) (bool, error) {
+	return false, notSupported
+}
+
+// dedup deduplicates the file at src path to dst path
+func (d *dedupFiles) dedup(src, dst string, fiDst fs.FileInfo) (uint64, error) {
+	return 0, notSupported
+}
+
+func readAllFile(path string, info fs.FileInfo, fn func([]byte) (string, error)) (string, error) {
+	return "", notSupported
+}

vendor/github.com/containers/storage/layers.go

@@ -336,6 +336,9 @@ type rwLayerStore interface {
 
 	// Clean up unreferenced layers
 	GarbageCollect() error
+
+	// Dedup deduplicates layers in the store.
+	dedup(drivers.DedupArgs) (drivers.DedupResult, error)
 }
 
 type multipleLockFile struct {
@@ -2601,6 +2604,11 @@ func (r *layerStore) LayersByTOCDigest(d digest.Digest) ([]Layer, error) {
 	return r.layersByDigestMap(r.bytocsum, d)
 }
 
+// Requires startWriting.
+func (r *layerStore) dedup(req drivers.DedupArgs) (drivers.DedupResult, error) {
+	return r.driver.Dedup(req)
+}
+
 func closeAll(closes ...func() error) (rErr error) {
 	for _, f := range closes {
 		if err := f(); err != nil {

vendor/github.com/containers/storage/pkg/archive/archive.go

@@ -78,6 +78,7 @@ const (
 	windows = "windows"
 	darwin  = "darwin"
 	freebsd = "freebsd"
+	linux   = "linux"
 )
 
 var xattrsToIgnore = map[string]interface{}{
@@ -427,7 +428,7 @@ func readSecurityXattrToTarHeader(path string, hdr *tar.Header) error {
 	}
 	for _, xattr := range []string{"security.capability", "security.ima"} {
 		capability, err := system.Lgetxattr(path, xattr)
-		if err != nil && !errors.Is(err, system.EOPNOTSUPP) && err != system.ErrNotSupportedPlatform {
+		if err != nil && !errors.Is(err, system.ENOTSUP) && err != system.ErrNotSupportedPlatform {
 			return fmt.Errorf("failed to read %q attribute from %q: %w", xattr, path, err)
 		}
 		if capability != nil {
@@ -440,7 +441,7 @@ func readSecurityXattrToTarHeader(path string, hdr *tar.Header) error {
 // readUserXattrToTarHeader reads user.* xattr from filesystem to a tar header
 func readUserXattrToTarHeader(path string, hdr *tar.Header) error {
 	xattrs, err := system.Llistxattr(path)
-	if err != nil && !errors.Is(err, system.EOPNOTSUPP) && err != system.ErrNotSupportedPlatform {
+	if err != nil && !errors.Is(err, system.ENOTSUP) && err != system.ErrNotSupportedPlatform {
 		return err
 	}
 	for _, key := range xattrs {
@@ -655,12 +656,20 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L
 	// so use hdrInfo.Mode() (they differ for e.g. setuid bits)
 	hdrInfo := hdr.FileInfo()
 
+	typeFlag := hdr.Typeflag
 	mask := hdrInfo.Mode()
+
+	// update also the implementation of ForceMask in pkg/chunked
 	if forceMask != nil {
 		mask = *forceMask
+		// If we have a forceMask, force the real type to either be a directory,
+		// a link, or a regular file.
+		if typeFlag != tar.TypeDir && typeFlag != tar.TypeSymlink && typeFlag != tar.TypeLink {
+			typeFlag = tar.TypeReg
+		}
 	}
 
-	switch hdr.Typeflag {
+	switch typeFlag {
 	case tar.TypeDir:
 		// Create directory unless it exists as a directory already.
 		// In that case we just want to merge the two
@@ -728,16 +737,6 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L
 		return fmt.Errorf("unhandled tar header type %d", hdr.Typeflag)
 	}
 
-	if forceMask != nil && (hdr.Typeflag != tar.TypeSymlink || runtime.GOOS == "darwin") {
-		value := idtools.Stat{
-			IDs:  idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid},
-			Mode: hdrInfo.Mode() & 0o7777,
-		}
-		if err := idtools.SetContainersOverrideXattr(path, value); err != nil {
-			return err
-		}
-	}
-
 	// Lchown is not supported on Windows.
 	if Lchown && runtime.GOOS != windows {
 		if chownOpts == nil {
@@ -793,18 +792,30 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L
 			continue
 		}
 		if err := system.Lsetxattr(path, xattrKey, []byte(value), 0); err != nil {
-			if errors.Is(err, syscall.ENOTSUP) || (inUserns && errors.Is(err, syscall.EPERM)) {
-				// We ignore errors here because not all graphdrivers support
-				// xattrs *cough* old versions of AUFS *cough*. However only
-				// ENOTSUP should be emitted in that case, otherwise we still
-				// bail.  We also ignore EPERM errors if we are running in a
-				// user namespace.
+			if errors.Is(err, system.ENOTSUP) || (inUserns && errors.Is(err, syscall.EPERM)) {
+				// Ignore specific error cases:
+				// - ENOTSUP: Expected for graphdrivers lacking extended attribute support:
+				//   - Legacy AUFS versions
+				//   - FreeBSD with unsupported namespaces (trusted, security)
+				// - EPERM: Expected when operating within a user namespace
+				// All other errors will cause a failure.
 				errs = append(errs, err.Error())
 				continue
 			}
 			return err
 		}
+	}
 
+	if forceMask != nil && (typeFlag == tar.TypeReg || typeFlag == tar.TypeDir || runtime.GOOS == "darwin") {
+		value := idtools.Stat{
+			IDs:   idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid},
+			Mode:  hdrInfo.Mode(),
+			Major: int(hdr.Devmajor),
+			Minor: int(hdr.Devminor),
+		}
+		if err := idtools.SetContainersOverrideXattr(path, value); err != nil {
+			return err
+		}
 	}
 
 	// We defer setting flags on directories until the end of
@@ -1149,11 +1160,11 @@ loop:
 	}
 
 	if options.ForceMask != nil {
-		value := idtools.Stat{Mode: 0o755}
+		value := idtools.Stat{Mode: os.ModeDir | os.FileMode(0o755)}
 		if rootHdr != nil {
 			value.IDs.UID = rootHdr.Uid
 			value.IDs.GID = rootHdr.Gid
-			value.Mode = os.FileMode(rootHdr.Mode)
+			value.Mode = os.ModeDir | os.FileMode(rootHdr.Mode)
 		}
 		if err := idtools.SetContainersOverrideXattr(dest, value); err != nil {
 			return err
@@ -1379,7 +1390,7 @@ func remapIDs(readIDMappings, writeIDMappings *idtools.IDMappings, chownOpts *id
 			uid, gid = hdr.Uid, hdr.Gid
 			if xstat, ok := hdr.PAXRecords[PaxSchilyXattr+idtools.ContainersOverrideXattr]; ok {
 				attrs := strings.Split(string(xstat), ":")
-				if len(attrs) == 3 {
+				if len(attrs) >= 3 {
 					val, err := strconv.ParseUint(attrs[0], 10, 32)
 					if err != nil {
 						uid = int(val)

vendor/github.com/containers/storage/pkg/archive/changes.go

@@ -270,6 +270,7 @@ type FileInfo struct {
 	capability []byte
 	added      bool
 	xattrs     map[string]string
+	target     string
 }
 
 // LookUp looks up the file information of a file.
@@ -336,6 +337,7 @@ func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
 			// back mtime
 			if statDifferent(oldStat, oldInfo, newStat, info) ||
 				!bytes.Equal(oldChild.capability, newChild.capability) ||
+				oldChild.target != newChild.target ||
 				!reflect.DeepEqual(oldChild.xattrs, newChild.xattrs) {
 				change := Change{
 					Path: newChild.path(),
@@ -390,6 +392,7 @@ func newRootFileInfo(idMappings *idtools.IDMappings) *FileInfo {
 		name:       string(os.PathSeparator),
 		idMappings: idMappings,
 		children:   make(map[string]*FileInfo),
+		target:     "",
 	}
 	return root
 }

vendor/github.com/containers/storage/pkg/archive/changes_linux.go

@@ -79,6 +79,7 @@ func walkchunk(path string, fi os.FileInfo, dir string, root *FileInfo) error {
 		children:   make(map[string]*FileInfo),
 		parent:     parent,
 		idMappings: root.idMappings,
+		target:     "",
 	}
 	cpath := filepath.Join(dir, path)
 	stat, err := system.FromStatT(fi.Sys().(*syscall.Stat_t))
@@ -87,11 +88,11 @@ func walkchunk(path string, fi os.FileInfo, dir string, root *FileInfo) error {
 	}
 	info.stat = stat
 	info.capability, err = system.Lgetxattr(cpath, "security.capability") // lgetxattr(2): fs access
-	if err != nil && !errors.Is(err, system.EOPNOTSUPP) {
+	if err != nil && !errors.Is(err, system.ENOTSUP) {
 		return err
 	}
 	xattrs, err := system.Llistxattr(cpath)
-	if err != nil && !errors.Is(err, system.EOPNOTSUPP) {
+	if err != nil && !errors.Is(err, system.ENOTSUP) {
 		return err
 	}
 	for _, key := range xattrs {
@@ -110,6 +111,12 @@ func walkchunk(path string, fi os.FileInfo, dir string, root *FileInfo) error {
 			info.xattrs[key] = string(value)
 		}
 	}
+	if fi.Mode()&os.ModeSymlink != 0 {
+		info.target, err = os.Readlink(cpath)
+		if err != nil {
+			return err
+		}
+	}
 	parent.children[info.name] = info
 	return nil
 }

vendor/github.com/containers/storage/pkg/chrootarchive/archive.go

@@ -47,7 +47,7 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error
 // This should be used to prevent a potential attacker from manipulating `dest`
 // such that it would provide access to files outside of `dest` through things
 // like symlinks. Normally `ResolveSymlinksInScope` would handle this, however
-// sanitizing symlinks in this manner is inherrently racey:
+// sanitizing symlinks in this manner is inherently racey:
 // ref: CVE-2018-15664
 func UntarWithRoot(tarArchive io.Reader, dest string, options *archive.TarOptions, root string) error {
 	return untarHandler(tarArchive, dest, options, true, root)

vendor/github.com/containers/storage/pkg/chunked/cache_linux.go

@@ -16,7 +16,7 @@ import (
 
 	storage "github.com/containers/storage"
 	graphdriver "github.com/containers/storage/drivers"
-	"github.com/containers/storage/pkg/chunked/internal"
+	"github.com/containers/storage/pkg/chunked/internal/minimal"
 	"github.com/containers/storage/pkg/ioutils"
 	"github.com/docker/go-units"
 	jsoniter "github.com/json-iterator/go"
@@ -710,7 +710,7 @@ func prepareCacheFile(manifest []byte, format graphdriver.DifferOutputFormat) ([
 	switch format {
 	case graphdriver.DifferOutputFormatDir:
 	case graphdriver.DifferOutputFormatFlat:
-		entries, err = makeEntriesFlat(entries)
+		entries, err = makeEntriesFlat(entries, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -848,12 +848,12 @@ func (c *layersCache) findFileInOtherLayers(file *fileMetadata, useHardLinks boo
 	return "", "", nil
 }
 
-func (c *layersCache) findChunkInOtherLayers(chunk *internal.FileMetadata) (string, string, int64, error) {
+func (c *layersCache) findChunkInOtherLayers(chunk *minimal.FileMetadata) (string, string, int64, error) {
 	return c.findDigestInternal(chunk.ChunkDigest)
 }
 
-func unmarshalToc(manifest []byte) (*internal.TOC, error) {
-	var toc internal.TOC
+func unmarshalToc(manifest []byte) (*minimal.TOC, error) {
+	var toc minimal.TOC
 
 	iter := jsoniter.ParseBytes(jsoniter.ConfigFastest, manifest)
 
@@ -864,7 +864,7 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) {
 
 		case "entries":
 			for iter.ReadArray() {
-				var m internal.FileMetadata
+				var m minimal.FileMetadata
 				for field := iter.ReadObject(); field != ""; field = iter.ReadObject() {
 					switch strings.ToLower(field) {
 					case "type":

vendor/github.com/containers/storage/pkg/chunked/compression.go

@@ -4,18 +4,18 @@ import (
 	"io"
 
 	"github.com/containers/storage/pkg/chunked/compressor"
-	"github.com/containers/storage/pkg/chunked/internal"
+	"github.com/containers/storage/pkg/chunked/internal/minimal"
 )
 
 const (
-	TypeReg     = internal.TypeReg
-	TypeChunk   = internal.TypeChunk
-	TypeLink    = internal.TypeLink
-	TypeChar    = internal.TypeChar
-	TypeBlock   = internal.TypeBlock
-	TypeDir     = internal.TypeDir
-	TypeFifo    = internal.TypeFifo
-	TypeSymlink = internal.TypeSymlink
+	TypeReg     = minimal.TypeReg
+	TypeChunk   = minimal.TypeChunk
+	TypeLink    = minimal.TypeLink
+	TypeChar    = minimal.TypeChar
+	TypeBlock   = minimal.TypeBlock
+	TypeDir     = minimal.TypeDir
+	TypeFifo    = minimal.TypeFifo
+	TypeSymlink = minimal.TypeSymlink
 )
 
 // ZstdCompressor is a CompressorFunc for the zstd compression algorithm.

vendor/github.com/containers/storage/pkg/chunked/compression_linux.go

@@ -10,7 +10,7 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/containers/storage/pkg/chunked/internal"
+	"github.com/containers/storage/pkg/chunked/internal/minimal"
 	"github.com/klauspost/compress/zstd"
 	"github.com/klauspost/pgzip"
 	digest "github.com/opencontainers/go-digest"
@@ -20,6 +20,12 @@ import (
 	expMaps "golang.org/x/exp/maps"
 )
 
+const (
+	// maxTocSize is the maximum size of a blob that we will attempt to process.
+	// It is used to prevent DoS attacks from layers that embed a very large TOC file.
+	maxTocSize = (1 << 20) * 150
+)
+
 var typesToTar = map[string]byte{
 	TypeReg:     tar.TypeReg,
 	TypeLink:    tar.TypeLink,
@@ -38,33 +44,35 @@ func typeToTarType(t string) (byte, error) {
 	return r, nil
 }
 
+// readEstargzChunkedManifest reads the estargz manifest from the seekable stream blobStream.
+// It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert.
 func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, tocDigest digest.Digest) ([]byte, int64, error) {
 	// information on the format here https://github.com/containerd/stargz-snapshotter/blob/main/docs/stargz-estargz.md
 	footerSize := int64(51)
 	if blobSize <= footerSize {
 		return nil, 0, errors.New("blob too small")
 	}
-	chunk := ImageSourceChunk{
-		Offset: uint64(blobSize - footerSize),
-		Length: uint64(footerSize),
-	}
-	parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk})
-	if err != nil {
-		return nil, 0, err
-	}
-	var reader io.ReadCloser
-	select {
-	case r := <-parts:
-		reader = r
-	case err := <-errs:
-		return nil, 0, err
-	}
-	defer reader.Close()
+
 	footer := make([]byte, footerSize)
-	if _, err := io.ReadFull(reader, footer); err != nil {
+	streamsOrErrors, err := getBlobAt(blobStream, ImageSourceChunk{Offset: uint64(blobSize - footerSize), Length: uint64(footerSize)})
+	if err != nil {
+		var badRequestErr ErrBadRequest
+		if errors.As(err, &badRequestErr) {
+			err = errFallbackCanConvert{newErrFallbackToOrdinaryLayerDownload(err)}
+		}
 		return nil, 0, err
 	}
 
+	for soe := range streamsOrErrors {
+		if soe.stream != nil {
+			_, err = io.ReadFull(soe.stream, footer)
+			_ = soe.stream.Close()
+		}
+		if soe.err != nil && err == nil {
+			err = soe.err
+		}
+	}
+
 	/* Read the ToC offset:
 	   - 10 bytes  gzip header
 	   - 2  bytes  XLEN (length of Extra field) = 26 (4 bytes header + 16 hex digits + len("STARGZ"))
@@ -81,48 +89,59 @@ func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64,
 
 	size := int64(blobSize - footerSize - tocOffset)
 	// set a reasonable limit
-	if size > (1<<20)*50 {
-		return nil, 0, errors.New("manifest too big")
+	if size > maxTocSize {
+		// Not errFallbackCanConvert: we would still use too much memory.
+		return nil, 0, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("estargz manifest too big to process in memory (%d bytes)", size))
 	}
 
-	chunk = ImageSourceChunk{
-		Offset: uint64(tocOffset),
-		Length: uint64(size),
-	}
-	parts, errs, err = blobStream.GetBlobAt([]ImageSourceChunk{chunk})
+	streamsOrErrors, err = getBlobAt(blobStream, ImageSourceChunk{Offset: uint64(tocOffset), Length: uint64(size)})
 	if err != nil {
+		var badRequestErr ErrBadRequest
+		if errors.As(err, &badRequestErr) {
+			err = errFallbackCanConvert{newErrFallbackToOrdinaryLayerDownload(err)}
+		}
 		return nil, 0, err
 	}
 
-	var tocReader io.ReadCloser
-	select {
-	case r := <-parts:
-		tocReader = r
-	case err := <-errs:
-		return nil, 0, err
-	}
-	defer tocReader.Close()
+	var manifestUncompressed []byte
 
-	r, err := pgzip.NewReader(tocReader)
-	if err != nil {
-		return nil, 0, err
-	}
-	defer r.Close()
+	for soe := range streamsOrErrors {
+		if soe.stream != nil {
+			err1 := func() error {
+				defer soe.stream.Close()
 
-	aTar := archivetar.NewReader(r)
+				r, err := pgzip.NewReader(soe.stream)
+				if err != nil {
+					return err
+				}
+				defer r.Close()
 
-	header, err := aTar.Next()
-	if err != nil {
-		return nil, 0, err
-	}
-	// set a reasonable limit
-	if header.Size > (1<<20)*50 {
-		return nil, 0, errors.New("manifest too big")
-	}
+				aTar := archivetar.NewReader(r)
 
-	manifestUncompressed := make([]byte, header.Size)
-	if _, err := io.ReadFull(aTar, manifestUncompressed); err != nil {
-		return nil, 0, err
+				header, err := aTar.Next()
+				if err != nil {
+					return err
+				}
+				// set a reasonable limit
+				if header.Size > maxTocSize {
+					return errors.New("manifest too big")
+				}
+
+				manifestUncompressed = make([]byte, header.Size)
+				if _, err := io.ReadFull(aTar, manifestUncompressed); err != nil {
+					return err
+				}
+				return nil
+			}()
+			if err == nil {
+				err = err1
+			}
+		} else if err == nil {
+			err = soe.err
+		}
+	}
+	if manifestUncompressed == nil {
+		return nil, 0, errors.New("manifest not found")
 	}
 
 	manifestDigester := digest.Canonical.Digester()
@@ -140,10 +159,11 @@ func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64,
 
 // readZstdChunkedManifest reads the zstd:chunked manifest from the seekable stream blobStream.
 // Returns (manifest blob, parsed manifest, tar-split blob or nil, manifest offset).
-func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Digest, annotations map[string]string) ([]byte, *internal.TOC, []byte, int64, error) {
-	offsetMetadata := annotations[internal.ManifestInfoKey]
+// It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert.
+func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Digest, annotations map[string]string) (_ []byte, _ *minimal.TOC, _ []byte, _ int64, retErr error) {
+	offsetMetadata := annotations[minimal.ManifestInfoKey]
 	if offsetMetadata == "" {
-		return nil, nil, nil, 0, fmt.Errorf("%q annotation missing", internal.ManifestInfoKey)
+		return nil, nil, nil, 0, fmt.Errorf("%q annotation missing", minimal.ManifestInfoKey)
 	}
 	var manifestChunk ImageSourceChunk
 	var manifestLengthUncompressed, manifestType uint64
@@ -153,48 +173,59 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di
 	// The tarSplit… values are valid if tarSplitChunk.Offset > 0
 	var tarSplitChunk ImageSourceChunk
 	var tarSplitLengthUncompressed uint64
-	if tarSplitInfoKeyAnnotation, found := annotations[internal.TarSplitInfoKey]; found {
+	if tarSplitInfoKeyAnnotation, found := annotations[minimal.TarSplitInfoKey]; found {
 		if _, err := fmt.Sscanf(tarSplitInfoKeyAnnotation, "%d:%d:%d", &tarSplitChunk.Offset, &tarSplitChunk.Length, &tarSplitLengthUncompressed); err != nil {
 			return nil, nil, nil, 0, err
 		}
 	}
 
-	if manifestType != internal.ManifestTypeCRFS {
+	if manifestType != minimal.ManifestTypeCRFS {
 		return nil, nil, nil, 0, errors.New("invalid manifest type")
 	}
 
 	// set a reasonable limit
-	if manifestChunk.Length > (1<<20)*50 {
-		return nil, nil, nil, 0, errors.New("manifest too big")
+	if manifestChunk.Length > maxTocSize {
+		// Not errFallbackCanConvert: we would still use too much memory.
+		return nil, nil, nil, 0, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("zstd:chunked manifest too big to process in memory (%d bytes compressed)", manifestChunk.Length))
 	}
-	if manifestLengthUncompressed > (1<<20)*50 {
-		return nil, nil, nil, 0, errors.New("manifest too big")
+	if manifestLengthUncompressed > maxTocSize {
+		// Not errFallbackCanConvert: we would still use too much memory.
+		return nil, nil, nil, 0, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("zstd:chunked manifest too big to process in memory (%d bytes uncompressed)", manifestLengthUncompressed))
 	}
 
 	chunks := []ImageSourceChunk{manifestChunk}
 	if tarSplitChunk.Offset > 0 {
 		chunks = append(chunks, tarSplitChunk)
 	}
-	parts, errs, err := blobStream.GetBlobAt(chunks)
+
+	streamsOrErrors, err := getBlobAt(blobStream, chunks...)
 	if err != nil {
+		var badRequestErr ErrBadRequest
+		if errors.As(err, &badRequestErr) {
+			err = errFallbackCanConvert{newErrFallbackToOrdinaryLayerDownload(err)}
+		}
 		return nil, nil, nil, 0, err
 	}
 
-	readBlob := func(len uint64) ([]byte, error) {
-		var reader io.ReadCloser
-		select {
-		case r := <-parts:
-			reader = r
-		case err := <-errs:
-			return nil, err
+	defer func() {
+		err := ensureAllBlobsDone(streamsOrErrors)
+		if retErr == nil {
+			retErr = err
 		}
+	}()
+
+	readBlob := func(len uint64) ([]byte, error) {
+		soe, ok := <-streamsOrErrors
+		if !ok {
+			return nil, errors.New("stream closed")
+		}
+		if soe.err != nil {
+			return nil, soe.err
+		}
+		defer soe.stream.Close()
 
 		blob := make([]byte, len)
-		if _, err := io.ReadFull(reader, blob); err != nil {
-			reader.Close()
-			return nil, err
-		}
-		if err := reader.Close(); err != nil {
+		if _, err := io.ReadFull(soe.stream, blob); err != nil {
 			return nil, err
 		}
 		return blob, nil
@@ -217,7 +248,7 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di
 	var decodedTarSplit []byte = nil
 	if toc.TarSplitDigest != "" {
 		if tarSplitChunk.Offset <= 0 {
-			return nil, nil, nil, 0, fmt.Errorf("TOC requires a tar-split, but the %s annotation does not describe a position", internal.TarSplitInfoKey)
+			return nil, nil, nil, 0, fmt.Errorf("TOC requires a tar-split, but the %s annotation does not describe a position", minimal.TarSplitInfoKey)
 		}
 		tarSplit, err := readBlob(tarSplitChunk.Length)
 		if err != nil {
@@ -247,11 +278,11 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di
 }
 
 // ensureTOCMatchesTarSplit validates that toc and tarSplit contain _exactly_ the same entries.
-func ensureTOCMatchesTarSplit(toc *internal.TOC, tarSplit []byte) error {
-	pendingFiles := map[string]*internal.FileMetadata{} // Name -> an entry in toc.Entries
+func ensureTOCMatchesTarSplit(toc *minimal.TOC, tarSplit []byte) error {
+	pendingFiles := map[string]*minimal.FileMetadata{} // Name -> an entry in toc.Entries
 	for i := range toc.Entries {
 		e := &toc.Entries[i]
-		if e.Type != internal.TypeChunk {
+		if e.Type != minimal.TypeChunk {
 			if _, ok := pendingFiles[e.Name]; ok {
 				return fmt.Errorf("TOC contains duplicate entries for path %q", e.Name)
 			}
@@ -266,7 +297,7 @@ func ensureTOCMatchesTarSplit(toc *internal.TOC, tarSplit []byte) error {
 			return fmt.Errorf("tar-split contains an entry for %q missing in TOC", hdr.Name)
 		}
 		delete(pendingFiles, hdr.Name)
-		expected, err := internal.NewFileMetadata(hdr)
+		expected, err := minimal.NewFileMetadata(hdr)
 		if err != nil {
 			return fmt.Errorf("determining expected metadata for %q: %w", hdr.Name, err)
 		}
@@ -347,8 +378,8 @@ func ensureTimePointersMatch(a, b *time.Time) error {
 
 // ensureFileMetadataAttributesMatch ensures that a and b match in file attributes (it ignores entries relevant to locating data
 // in the tar stream or matching contents)
-func ensureFileMetadataAttributesMatch(a, b *internal.FileMetadata) error {
-	// Keep this in sync with internal.FileMetadata!
+func ensureFileMetadataAttributesMatch(a, b *minimal.FileMetadata) error {
+	// Keep this in sync with minimal.FileMetadata!
 
 	if a.Type != b.Type {
 		return fmt.Errorf("mismatch of Type: %q != %q", a.Type, b.Type)

vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go

@@ -9,7 +9,7 @@ import (
 	"bytes"
 	"io"
 
-	"github.com/containers/storage/pkg/chunked/internal"
+	"github.com/containers/storage/pkg/chunked/internal/minimal"
 	"github.com/containers/storage/pkg/ioutils"
 	"github.com/klauspost/compress/zstd"
 	"github.com/opencontainers/go-digest"
@@ -213,7 +213,7 @@ func newTarSplitData(level int) (*tarSplitData, error) {
 	compressed := bytes.NewBuffer(nil)
 	digester := digest.Canonical.Digester()
 
-	zstdWriter, err := internal.ZstdWriterWithLevel(io.MultiWriter(compressed, digester.Hash()), level)
+	zstdWriter, err := minimal.ZstdWriterWithLevel(io.MultiWriter(compressed, digester.Hash()), level)
 	if err != nil {
 		return nil, err
 	}
@@ -254,7 +254,7 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r
 
 	buf := make([]byte, 4096)
 
-	zstdWriter, err := internal.ZstdWriterWithLevel(dest, level)
+	zstdWriter, err := minimal.ZstdWriterWithLevel(dest, level)
 	if err != nil {
 		return err
 	}
@@ -276,7 +276,7 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r
 		return offset, nil
 	}
 
-	var metadata []internal.FileMetadata
+	var metadata []minimal.FileMetadata
 	for {
 		hdr, err := tr.Next()
 		if err != nil {
@@ -341,9 +341,9 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r
 
 				chunkSize := rcReader.WrittenOut - lastChunkOffset
 				if chunkSize > 0 {
-					chunkType := internal.ChunkTypeData
+					chunkType := minimal.ChunkTypeData
 					if rcReader.IsLastChunkZeros {
-						chunkType = internal.ChunkTypeZeros
+						chunkType = minimal.ChunkTypeZeros
 					}
 
 					chunks = append(chunks, chunk{
@@ -368,17 +368,17 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r
 			}
 		}
 
-		mainEntry, err := internal.NewFileMetadata(hdr)
+		mainEntry, err := minimal.NewFileMetadata(hdr)
 		if err != nil {
 			return err
 		}
 		mainEntry.Digest = checksum
 		mainEntry.Offset = startOffset
 		mainEntry.EndOffset = lastOffset
-		entries := []internal.FileMetadata{mainEntry}
+		entries := []minimal.FileMetadata{mainEntry}
 		for i := 1; i < len(chunks); i++ {
-			entries = append(entries, internal.FileMetadata{
-				Type:        internal.TypeChunk,
+			entries = append(entries, minimal.FileMetadata{
+				Type:        minimal.TypeChunk,
 				Name:        hdr.Name,
 				ChunkOffset: chunks[i].ChunkOffset,
 			})
@@ -424,13 +424,13 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r
 	}
 	tarSplitData.zstd = nil
 
-	ts := internal.TarSplitData{
+	ts := minimal.TarSplitData{
 		Data:             tarSplitData.compressed.Bytes(),
 		Digest:           tarSplitData.digester.Digest(),
 		UncompressedSize: tarSplitData.uncompressedCounter.Count,
 	}
 
-	return internal.WriteZstdChunkedManifest(dest, outMetadata, uint64(dest.Count), &ts, metadata, level)
+	return minimal.WriteZstdChunkedManifest(dest, outMetadata, uint64(dest.Count), &ts, metadata, level)
 }
 
 type zstdChunkedWriter struct {

vendor/github.com/containers/storage/pkg/chunked/dump/dump.go

@@ -9,10 +9,11 @@ import (
 	"io"
 	"path/filepath"
 	"reflect"
-	"strings"
 	"time"
 
-	"github.com/containers/storage/pkg/chunked/internal"
+	"github.com/containers/storage/pkg/chunked/internal/minimal"
+	storagePath "github.com/containers/storage/pkg/chunked/internal/path"
+	"github.com/opencontainers/go-digest"
 	"golang.org/x/sys/unix"
 )
 
@@ -85,17 +86,17 @@ func escapedOptional(val []byte, escape int) string {
 
 func getStMode(mode uint32, typ string) (uint32, error) {
 	switch typ {
-	case internal.TypeReg, internal.TypeLink:
+	case minimal.TypeReg, minimal.TypeLink:
 		mode |= unix.S_IFREG
-	case internal.TypeChar:
+	case minimal.TypeChar:
 		mode |= unix.S_IFCHR
-	case internal.TypeBlock:
+	case minimal.TypeBlock:
 		mode |= unix.S_IFBLK
-	case internal.TypeDir:
+	case minimal.TypeDir:
 		mode |= unix.S_IFDIR
-	case internal.TypeFifo:
+	case minimal.TypeFifo:
 		mode |= unix.S_IFIFO
-	case internal.TypeSymlink:
+	case minimal.TypeSymlink:
 		mode |= unix.S_IFLNK
 	default:
 		return 0, fmt.Errorf("unknown type %s", typ)
@@ -103,24 +104,14 @@ func getStMode(mode uint32, typ string) (uint32, error) {
 	return mode, nil
 }
 
-func sanitizeName(name string) string {
-	path := filepath.Clean(name)
-	if path == "." {
-		path = "/"
-	} else if path[0] != '/' {
-		path = "/" + path
-	}
-	return path
-}
-
-func dumpNode(out io.Writer, added map[string]*internal.FileMetadata, links map[string]int, verityDigests map[string]string, entry *internal.FileMetadata) error {
-	path := sanitizeName(entry.Name)
+func dumpNode(out io.Writer, added map[string]*minimal.FileMetadata, links map[string]int, verityDigests map[string]string, entry *minimal.FileMetadata) error {
+	path := storagePath.CleanAbsPath(entry.Name)
 
 	parent := filepath.Dir(path)
 	if _, found := added[parent]; !found && path != "/" {
-		parentEntry := &internal.FileMetadata{
+		parentEntry := &minimal.FileMetadata{
 			Name: parent,
-			Type: internal.TypeDir,
+			Type: minimal.TypeDir,
 			Mode: 0o755,
 		}
 		if err := dumpNode(out, added, links, verityDigests, parentEntry); err != nil {
@@ -143,7 +134,7 @@ func dumpNode(out io.Writer, added map[string]*internal.FileMetadata, links map[
 
 	nlinks := links[entry.Name] + links[entry.Linkname] + 1
 	link := ""
-	if entry.Type == internal.TypeLink {
+	if entry.Type == minimal.TypeLink {
 		link = "@"
 	}
 
@@ -169,16 +160,21 @@ func dumpNode(out io.Writer, added map[string]*internal.FileMetadata, links map[
 
 	var payload string
 	if entry.Linkname != "" {
-		if entry.Type == internal.TypeSymlink {
+		if entry.Type == minimal.TypeSymlink {
 			payload = entry.Linkname
 		} else {
-			payload = sanitizeName(entry.Linkname)
+			payload = storagePath.CleanAbsPath(entry.Linkname)
 		}
-	} else {
-		if len(entry.Digest) > 10 {
-			d := strings.Replace(entry.Digest, "sha256:", "", 1)
-			payload = d[:2] + "/" + d[2:]
+	} else if entry.Digest != "" {
+		d, err := digest.Parse(entry.Digest)
+		if err != nil {
+			return fmt.Errorf("invalid digest %q for %q: %w", entry.Digest, entry.Name, err)
 		}
+		path, err := storagePath.RegularFilePathForValidatedDigest(d)
+		if err != nil {
+			return fmt.Errorf("determining physical file path for %q: %w", entry.Name, err)
+		}
+		payload = path
 	}
 
 	if _, err := fmt.Fprint(out, escapedOptional([]byte(payload), ESCAPE_LONE_DASH)); err != nil {
@@ -219,7 +215,7 @@ func dumpNode(out io.Writer, added map[string]*internal.FileMetadata, links map[
 
 // GenerateDump generates a dump of the TOC in the same format as `composefs-info dump`
 func GenerateDump(tocI interface{}, verityDigests map[string]string) (io.Reader, error) {
-	toc, ok := tocI.(*internal.TOC)
+	toc, ok := tocI.(*minimal.TOC)
 	if !ok {
 		return nil, fmt.Errorf("invalid TOC type")
 	}
@@ -235,21 +231,21 @@ func GenerateDump(tocI interface{}, verityDigests map[string]string) (io.Reader,
 		}()
 
 		links := make(map[string]int)
-		added := make(map[string]*internal.FileMetadata)
+		added := make(map[string]*minimal.FileMetadata)
 		for _, e := range toc.Entries {
 			if e.Linkname == "" {
 				continue
 			}
-			if e.Type == internal.TypeSymlink {
+			if e.Type == minimal.TypeSymlink {
 				continue
 			}
 			links[e.Linkname] = links[e.Linkname] + 1
 		}
 
 		if len(toc.Entries) == 0 {
-			root := &internal.FileMetadata{
+			root := &minimal.FileMetadata{
 				Name: "/",
-				Type: internal.TypeDir,
+				Type: minimal.TypeDir,
 				Mode: 0o755,
 			}
 
@@ -261,7 +257,7 @@ func GenerateDump(tocI interface{}, verityDigests map[string]string) (io.Reader,
 		}
 
 		for _, e := range toc.Entries {
-			if e.Type == internal.TypeChunk {
+			if e.Type == minimal.TypeChunk {
 				continue
 			}
 			if err := dumpNode(w, added, links, verityDigests, &e); err != nil {

vendor/github.com/containers/storage/pkg/chunked/filesystem_linux.go

@@ -15,7 +15,8 @@ import (
 
 	driversCopy "github.com/containers/storage/drivers/copy"
 	"github.com/containers/storage/pkg/archive"
-	"github.com/containers/storage/pkg/chunked/internal"
+	"github.com/containers/storage/pkg/chunked/internal/minimal"
+	storagePath "github.com/containers/storage/pkg/chunked/internal/path"
 	securejoin "github.com/cyphar/filepath-securejoin"
 	"github.com/vbatts/tar-split/archive/tar"
 	"golang.org/x/sys/unix"
@@ -34,14 +35,14 @@ func procPathForFd(fd int) string {
 	return fmt.Sprintf("/proc/self/fd/%d", fd)
 }
 
-// fileMetadata is a wrapper around internal.FileMetadata with additional private fields that
+// fileMetadata is a wrapper around minimal.FileMetadata with additional private fields that
 // are not part of the TOC document.
 // Type: TypeChunk entries are stored in Chunks, the primary [fileMetadata] entries never use TypeChunk.
 type fileMetadata struct {
-	internal.FileMetadata
+	minimal.FileMetadata
 
 	// chunks stores the TypeChunk entries relevant to this entry when FileMetadata.Type == TypeReg.
-	chunks []*internal.FileMetadata
+	chunks []*minimal.FileMetadata
 
 	// skipSetAttrs is set when the file attributes must not be
 	// modified, e.g. it is a hard link from a different source,
@@ -49,10 +50,37 @@ type fileMetadata struct {
 	skipSetAttrs bool
 }
 
+// splitPath takes a file path as input and returns two components: dir and base.
+// Differently than filepath.Split(), this function handles some edge cases.
+// If the path refers to a file in the root directory, the returned dir is "/".
+// The returned base value is never empty, it never contains any slash and the
+// value "..".
+func splitPath(path string) (string, string, error) {
+	path = storagePath.CleanAbsPath(path)
+	dir, base := filepath.Split(path)
+	if base == "" {
+		base = "."
+	}
+	// Remove trailing slashes from dir, but make sure that "/" is preserved.
+	dir = strings.TrimSuffix(dir, "/")
+	if dir == "" {
+		dir = "/"
+	}
+
+	if strings.Contains(base, "/") {
+		// This should never happen, but be safe as the base is passed to *at syscalls.
+		return "", "", fmt.Errorf("internal error: splitPath(%q) contains a slash", path)
+	}
+	return dir, base, nil
+}
+
 func doHardLink(dirfd, srcFd int, destFile string) error {
-	destDir, destBase := filepath.Split(destFile)
+	destDir, destBase, err := splitPath(destFile)
+	if err != nil {
+		return err
+	}
 	destDirFd := dirfd
-	if destDir != "" && destDir != "." {
+	if destDir != "/" {
 		f, err := openOrCreateDirUnderRoot(dirfd, destDir, 0)
 		if err != nil {
 			return err
@@ -72,7 +100,7 @@ func doHardLink(dirfd, srcFd int, destFile string) error {
 		return nil
 	}
 
-	err := doLink()
+	err = doLink()
 
 	// if the destination exists, unlink it first and try again
 	if err != nil && os.IsExist(err) {
@@ -281,8 +309,11 @@ func openFileUnderRootFallback(dirfd int, name string, flags uint64, mode os.Fil
 	// If O_NOFOLLOW is specified in the flags, then resolve only the parent directory and use the
 	// last component as the path to openat().
 	if hasNoFollow {
-		dirName, baseName := filepath.Split(name)
-		if dirName != "" && dirName != "." {
+		dirName, baseName, err := splitPath(name)
+		if err != nil {
+			return -1, err
+		}
+		if dirName != "/" {
 			newRoot, err := securejoin.SecureJoin(root, dirName)
 			if err != nil {
 				return -1, err
@@ -409,7 +440,8 @@ func openOrCreateDirUnderRoot(dirfd int, name string, mode os.FileMode) (*os.Fil
 
 	if errors.Is(err, unix.ENOENT) {
 		parent := filepath.Dir(name)
-		if parent != "" {
+		// do not create the root directory, it should always exist
+		if parent != name {
 			pDir, err2 := openOrCreateDirUnderRoot(dirfd, parent, mode)
 			if err2 != nil {
 				return nil, err
@@ -448,9 +480,12 @@ func appendHole(fd int, name string, size int64) error {
 }
 
 func safeMkdir(dirfd int, mode os.FileMode, name string, metadata *fileMetadata, options *archive.TarOptions) error {
-	parent, base := filepath.Split(name)
+	parent, base, err := splitPath(name)
+	if err != nil {
+		return err
+	}
 	parentFd := dirfd
-	if parent != "" && parent != "." {
+	if parent != "/" {
 		parentFile, err := openOrCreateDirUnderRoot(dirfd, parent, 0)
 		if err != nil {
 			return err
@@ -506,9 +541,12 @@ func safeLink(dirfd int, mode os.FileMode, metadata *fileMetadata, options *arch
 }
 
 func safeSymlink(dirfd int, metadata *fileMetadata) error {
-	destDir, destBase := filepath.Split(metadata.Name)
+	destDir, destBase, err := splitPath(metadata.Name)
+	if err != nil {
+		return err
+	}
 	destDirFd := dirfd
-	if destDir != "" && destDir != "." {
+	if destDir != "/" {
 		f, err := openOrCreateDirUnderRoot(dirfd, destDir, 0)
 		if err != nil {
 			return err
@@ -542,9 +580,12 @@ func (d whiteoutHandler) Setxattr(path, name string, value []byte) error {
 }
 
 func (d whiteoutHandler) Mknod(path string, mode uint32, dev int) error {
-	dir, base := filepath.Split(path)
+	dir, base, err := splitPath(path)
+	if err != nil {
+		return err
+	}
 	dirfd := d.Dirfd
-	if dir != "" && dir != "." {
+	if dir != "/" {
 		dir, err := openOrCreateDirUnderRoot(d.Dirfd, dir, 0)
 		if err != nil {
 			return err

vendor/github.com/containers/storage/pkg/chunked/internal/minimal/compression.go

@@ -1,4 +1,4 @@
-package internal
+package minimal
 
 // NOTE: This is used from github.com/containers/image by callers that
 // don't otherwise use containers/storage, so don't make this depend on any

vendor/github.com/containers/storage/pkg/chunked/internal/path/path.go

@@ -0,0 +1,27 @@
+package path
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/opencontainers/go-digest"
+)
+
+// CleanAbsPath removes any ".." and "." from the path
+// and ensures it starts with a "/".  If the path refers to the root
+// directory, it returns "/".
+func CleanAbsPath(path string) string {
+	return filepath.Clean("/" + path)
+}
+
+// RegularFilePath returns the path used in the composefs backing store for a
+// regular file with the provided content digest.
+//
+// The caller MUST ensure d is a valid digest (in particular, that it contains no path separators or .. entries)
+func RegularFilePathForValidatedDigest(d digest.Digest) (string, error) {
+	if algo := d.Algorithm(); algo != digest.SHA256 {
+		return "", fmt.Errorf("unexpected digest algorithm %q", algo)
+	}
+	e := d.Encoded()
+	return e[0:2] + "/" + e[2:], nil
+}

vendor/github.com/containers/storage/pkg/chunked/storage_linux.go

@@ -2,6 +2,7 @@ package chunked
 
 import (
 	archivetar "archive/tar"
+	"bytes"
 	"context"
 	"encoding/base64"
 	"errors"
@@ -22,17 +23,21 @@ import (
 	graphdriver "github.com/containers/storage/drivers"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/chunked/compressor"
-	"github.com/containers/storage/pkg/chunked/internal"
+	"github.com/containers/storage/pkg/chunked/internal/minimal"
+	path "github.com/containers/storage/pkg/chunked/internal/path"
 	"github.com/containers/storage/pkg/chunked/toc"
 	"github.com/containers/storage/pkg/fsverity"
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/containers/storage/pkg/system"
+	securejoin "github.com/cyphar/filepath-securejoin"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/compress/zstd"
 	"github.com/klauspost/pgzip"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/sirupsen/logrus"
 	"github.com/vbatts/tar-split/archive/tar"
+	"github.com/vbatts/tar-split/tar/asm"
+	tsStorage "github.com/vbatts/tar-split/tar/storage"
 	"golang.org/x/sys/unix"
 )
 
@@ -57,46 +62,53 @@ const (
 type compressedFileType int
 
 type chunkedDiffer struct {
+	// Initial parameters, used throughout and never modified
+	// ==========
+	pullOptions pullOptions
 	stream      ImageSourceSeekable
-	manifest    []byte
-	toc         *internal.TOC // The parsed contents of manifest, or nil if not yet available
-	tarSplit    []byte
-	layersCache *layersCache
-	tocOffset   int64
-	fileType    compressedFileType
-
-	copyBuffer []byte
-
-	gzipReader *pgzip.Reader
-	zstdReader *zstd.Decoder
-	rawReader  io.Reader
-
-	// tocDigest is the digest of the TOC document when the layer
-	// is partially pulled.
-	tocDigest digest.Digest
+	// blobDigest is the digest of the whole compressed layer.  It is used if
+	// convertToZstdChunked to validate a layer when it is converted since there
+	// is no TOC referenced by the manifest.
+	blobDigest digest.Digest
+	blobSize   int64
 
+	// Input format
+	// ==========
+	fileType compressedFileType
 	// convertedToZstdChunked is set to true if the layer needs to
 	// be converted to the zstd:chunked format before it can be
 	// handled.
 	convertToZstdChunked bool
 
+	// Chunked metadata
+	// This is usually set in GetDiffer, but if convertToZstdChunked, it is only computed in chunkedDiffer.ApplyDiff
+	// ==========
+	// tocDigest is the digest of the TOC document when the layer
+	// is partially pulled, or "" if not relevant to consumers.
+	tocDigest           digest.Digest
+	tocOffset           int64
+	manifest            []byte
+	toc                 *minimal.TOC // The parsed contents of manifest, or nil if not yet available
+	tarSplit            []byte
+	uncompressedTarSize int64 // -1 if unknown
 	// skipValidation is set to true if the individual files in
 	// the layer are trusted and should not be validated.
 	skipValidation bool
 
-	// blobDigest is the digest of the whole compressed layer.  It is used if
-	// convertToZstdChunked to validate a layer when it is converted since there
-	// is no TOC referenced by the manifest.
-	blobDigest digest.Digest
-
-	blobSize            int64
-	uncompressedTarSize int64 // -1 if unknown
-
-	pullOptions map[string]string
-
-	useFsVerity     graphdriver.DifferFsVerity
+	// Long-term caches
+	// This is set in GetDiffer, when the caller must not hold any storage locks, and later consumed in .ApplyDiff()
+	// ==========
+	layersCache     *layersCache
+	copyBuffer      []byte
+	fsVerityMutex   sync.Mutex // protects fsVerityDigests
 	fsVerityDigests map[string]string
-	fsVerityMutex   sync.Mutex
+
+	// Private state of .ApplyDiff
+	// ==========
+	gzipReader  *pgzip.Reader
+	zstdReader  *zstd.Decoder
+	rawReader   io.Reader
+	useFsVerity graphdriver.DifferFsVerity
 }
 
 var xattrsToIgnore = map[string]interface{}{
@@ -108,6 +120,42 @@ type chunkedLayerData struct {
 	Format graphdriver.DifferOutputFormat `json:"format"`
 }
 
+// pullOptions contains parsed data from storage.Store.PullOptions.
+// TO DO: ideally this should be parsed along with the rest of the config file into StoreOptions directly
+// (and then storage.Store.PullOptions would need to be somehow simulated).
+type pullOptions struct {
+	enablePartialImages                     bool     // enable_partial_images
+	convertImages                           bool     // convert_images
+	useHardLinks                            bool     // use_hard_links
+	insecureAllowUnpredictableImageContents bool     // insecure_allow_unpredictable_image_contents
+	ostreeRepos                             []string // ostree_repos
+}
+
+func parsePullOptions(store storage.Store) pullOptions {
+	options := store.PullOptions()
+
+	res := pullOptions{}
+	for _, e := range []struct {
+		dest         *bool
+		name         string
+		defaultValue bool
+	}{
+		{&res.enablePartialImages, "enable_partial_images", false},
+		{&res.convertImages, "convert_images", false},
+		{&res.useHardLinks, "use_hard_links", false},
+		{&res.insecureAllowUnpredictableImageContents, "insecure_allow_unpredictable_image_contents", false},
+	} {
+		if value, ok := options[e.name]; ok {
+			*e.dest = strings.ToLower(value) == "true"
+		} else {
+			*e.dest = e.defaultValue
+		}
+	}
+	res.ostreeRepos = strings.Split(options["ostree_repos"], ":")
+
+	return res
+}
+
 func (c *chunkedDiffer) convertTarToZstdChunked(destDirectory string, payload *os.File) (int64, *seekableFile, digest.Digest, map[string]string, error) {
 	diff, err := archive.DecompressStream(payload)
 	if err != nil {
@@ -144,127 +192,160 @@ func (c *chunkedDiffer) convertTarToZstdChunked(destDirectory string, payload *o
 }
 
 // GetDiffer returns a differ than can be used with ApplyDiffWithDiffer.
-// If it returns an error that implements IsErrFallbackToOrdinaryLayerDownload, the caller can
+// If it returns an error that matches ErrFallbackToOrdinaryLayerDownload, the caller can
 // retry the operation with a different method.
 func GetDiffer(ctx context.Context, store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) {
-	pullOptions := store.PullOptions()
+	pullOptions := parsePullOptions(store)
 
-	if !parseBooleanPullOption(pullOptions, "enable_partial_images", false) {
-		// If convertImages is set, the two options disagree whether fallback is permissible.
+	if !pullOptions.enablePartialImages {
+		// If pullOptions.convertImages is set, the two options disagree whether fallback is permissible.
 		// Right now, we enable it, but that’s not a promise; rather, such a configuration should ideally be rejected.
 		return nil, newErrFallbackToOrdinaryLayerDownload(errors.New("partial images are disabled"))
 	}
-	// convertImages also serves as a “must not fallback to non-partial pull” option (?!)
-	convertImages := parseBooleanPullOption(pullOptions, "convert_images", false)
+	// pullOptions.convertImages also serves as a “must not fallback to non-partial pull” option (?!)
 
 	graphDriver, err := store.GraphDriver()
 	if err != nil {
 		return nil, err
 	}
 	if _, partialSupported := graphDriver.(graphdriver.DriverWithDiffer); !partialSupported {
-		if convertImages {
+		if pullOptions.convertImages {
 			return nil, fmt.Errorf("graph driver %s does not support partial pull but convert_images requires that", graphDriver.String())
 		}
 		return nil, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("graph driver %s does not support partial pull", graphDriver.String()))
 	}
 
-	differ, canFallback, err := getProperDiffer(store, blobDigest, blobSize, annotations, iss, pullOptions)
+	differ, err := getProperDiffer(store, blobDigest, blobSize, annotations, iss, pullOptions)
 	if err != nil {
-		if !canFallback {
+		var fallbackErr ErrFallbackToOrdinaryLayerDownload
+		if !errors.As(err, &fallbackErr) {
 			return nil, err
 		}
 		// If convert_images is enabled, always attempt to convert it instead of returning an error or falling back to a different method.
-		if convertImages {
-			logrus.Debugf("Created differ to convert blob %q", blobDigest)
-			return makeConvertFromRawDiffer(store, blobDigest, blobSize, iss, pullOptions)
+		if !pullOptions.convertImages {
+			return nil, err
 		}
-		return nil, newErrFallbackToOrdinaryLayerDownload(err)
+		var canConvertErr errFallbackCanConvert
+		if !errors.As(err, &canConvertErr) {
+			// We are supposed to use makeConvertFromRawDiffer, but that would not work.
+			// Fail, and make sure the error does _not_ match ErrFallbackToOrdinaryLayerDownload: use only the error text,
+			// discard all type information.
+			return nil, fmt.Errorf("neither a partial pull nor convert_images is possible: %s", err.Error())
+		}
+		logrus.Debugf("Created differ to convert blob %q", blobDigest)
+		return makeConvertFromRawDiffer(store, blobDigest, blobSize, iss, pullOptions)
 	}
 
 	return differ, nil
 }
 
+// errFallbackCanConvert is an an error type _accompanying_ ErrFallbackToOrdinaryLayerDownload
+// within getProperDiffer, to mark that using makeConvertFromRawDiffer makes sense.
+// This is used to distinguish between cases where the environment does not support partial pulls
+// (e.g. a registry does not support range requests) and convert_images is still possible,
+// from cases where the image content is unacceptable for partial pulls (e.g. exceeds memory limits)
+// and convert_images would not help.
+type errFallbackCanConvert struct {
+	err error
+}
+
+func (e errFallbackCanConvert) Error() string {
+	return e.err.Error()
+}
+
+func (e errFallbackCanConvert) Unwrap() error {
+	return e.err
+}
+
 // getProperDiffer is an implementation detail of GetDiffer.
 // It returns a “proper” differ (not a convert_images one) if possible.
-// On error, the second parameter is true if a fallback to an alternative (either the makeConverToRaw differ, or a non-partial pull)
-// is permissible.
-func getProperDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable, pullOptions map[string]string) (graphdriver.Differ, bool, error) {
-	zstdChunkedTOCDigestString, hasZstdChunkedTOC := annotations[internal.ManifestChecksumKey]
+// May return an error matching ErrFallbackToOrdinaryLayerDownload if a fallback to an alternative
+// (either makeConvertFromRawDiffer, or a non-partial pull) is permissible.
+func getProperDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable, pullOptions pullOptions) (graphdriver.Differ, error) {
+	zstdChunkedTOCDigestString, hasZstdChunkedTOC := annotations[minimal.ManifestChecksumKey]
 	estargzTOCDigestString, hasEstargzTOC := annotations[estargz.TOCJSONDigestAnnotation]
 
 	switch {
 	case hasZstdChunkedTOC && hasEstargzTOC:
-		return nil, false, errors.New("both zstd:chunked and eStargz TOC found")
+		return nil, errors.New("both zstd:chunked and eStargz TOC found")
 
 	case hasZstdChunkedTOC:
 		zstdChunkedTOCDigest, err := digest.Parse(zstdChunkedTOCDigestString)
 		if err != nil {
-			return nil, false, err
+			return nil, err
 		}
 		differ, err := makeZstdChunkedDiffer(store, blobSize, zstdChunkedTOCDigest, annotations, iss, pullOptions)
 		if err != nil {
 			logrus.Debugf("Could not create zstd:chunked differ for blob %q: %v", blobDigest, err)
-			// If the error is a bad request to the server, then signal to the caller that it can try a different method.
-			var badRequestErr ErrBadRequest
-			return nil, errors.As(err, &badRequestErr), err
+			return nil, err
 		}
 		logrus.Debugf("Created zstd:chunked differ for blob %q", blobDigest)
-		return differ, false, nil
+		return differ, nil
 
 	case hasEstargzTOC:
 		estargzTOCDigest, err := digest.Parse(estargzTOCDigestString)
 		if err != nil {
-			return nil, false, err
+			return nil, err
 		}
 		differ, err := makeEstargzChunkedDiffer(store, blobSize, estargzTOCDigest, iss, pullOptions)
 		if err != nil {
 			logrus.Debugf("Could not create estargz differ for blob %q: %v", blobDigest, err)
-			// If the error is a bad request to the server, then signal to the caller that it can try a different method.
-			var badRequestErr ErrBadRequest
-			return nil, errors.As(err, &badRequestErr), err
+			return nil, err
 		}
 		logrus.Debugf("Created eStargz differ for blob %q", blobDigest)
-		return differ, false, nil
+		return differ, nil
 
 	default: // no TOC
-		convertImages := parseBooleanPullOption(pullOptions, "convert_images", false)
-		if !convertImages {
-			return nil, true, errors.New("no TOC found and convert_images is not configured")
+		message := "no TOC found"
+		if !pullOptions.convertImages {
+			message = "no TOC found and convert_images is not configured"
+		}
+		return nil, errFallbackCanConvert{
+			newErrFallbackToOrdinaryLayerDownload(errors.New(message)),
 		}
-		return nil, true, errors.New("no TOC found")
 	}
 }
 
-func makeConvertFromRawDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, iss ImageSourceSeekable, pullOptions map[string]string) (*chunkedDiffer, error) {
+func makeConvertFromRawDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, iss ImageSourceSeekable, pullOptions pullOptions) (*chunkedDiffer, error) {
 	layersCache, err := getLayersCache(store)
 	if err != nil {
 		return nil, err
 	}
 
 	return &chunkedDiffer{
-		fsVerityDigests:      make(map[string]string),
-		blobDigest:           blobDigest,
-		blobSize:             blobSize,
-		uncompressedTarSize:  -1, // Will be computed later
+		pullOptions: pullOptions,
+		stream:      iss,
+		blobDigest:  blobDigest,
+		blobSize:    blobSize,
+
 		convertToZstdChunked: true,
-		copyBuffer:           makeCopyBuffer(),
-		layersCache:          layersCache,
-		pullOptions:          pullOptions,
-		stream:               iss,
+
+		uncompressedTarSize: -1, // Will be computed later
+
+		layersCache:     layersCache,
+		copyBuffer:      makeCopyBuffer(),
+		fsVerityDigests: make(map[string]string),
 	}, nil
 }
 
-func makeZstdChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, annotations map[string]string, iss ImageSourceSeekable, pullOptions map[string]string) (*chunkedDiffer, error) {
+// makeZstdChunkedDiffer sets up a chunkedDiffer for a zstd:chunked layer.
+// It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert.
+func makeZstdChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, annotations map[string]string, iss ImageSourceSeekable, pullOptions pullOptions) (*chunkedDiffer, error) {
 	manifest, toc, tarSplit, tocOffset, err := readZstdChunkedManifest(iss, tocDigest, annotations)
-	if err != nil {
+	if err != nil { // May be ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert
 		return nil, fmt.Errorf("read zstd:chunked manifest: %w", err)
 	}
+
 	var uncompressedTarSize int64 = -1
 	if tarSplit != nil {
 		uncompressedTarSize, err = tarSizeFromTarSplit(tarSplit)
 		if err != nil {
 			return nil, fmt.Errorf("computing size from tar-split: %w", err)
 		}
+	} else if !pullOptions.insecureAllowUnpredictableImageContents { // With no tar-split, we can't compute the traditional UncompressedDigest.
+		return nil, errFallbackCanConvert{
+			newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("zstd:chunked layers without tar-split data don't support partial pulls with guaranteed consistency with non-partial pulls")),
+		}
 	}
 
 	layersCache, err := getLayersCache(store)
@@ -273,25 +354,36 @@ func makeZstdChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest
 	}
 
 	return &chunkedDiffer{
-		fsVerityDigests:     make(map[string]string),
-		blobSize:            blobSize,
-		uncompressedTarSize: uncompressedTarSize,
+		pullOptions: pullOptions,
+		stream:      iss,
+		blobSize:    blobSize,
+
+		fileType: fileTypeZstdChunked,
+
 		tocDigest:           tocDigest,
-		copyBuffer:          makeCopyBuffer(),
-		fileType:            fileTypeZstdChunked,
-		layersCache:         layersCache,
+		tocOffset:           tocOffset,
 		manifest:            manifest,
 		toc:                 toc,
-		pullOptions:         pullOptions,
-		stream:              iss,
 		tarSplit:            tarSplit,
-		tocOffset:           tocOffset,
+		uncompressedTarSize: uncompressedTarSize,
+
+		layersCache:     layersCache,
+		copyBuffer:      makeCopyBuffer(),
+		fsVerityDigests: make(map[string]string),
 	}, nil
 }
 
-func makeEstargzChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, iss ImageSourceSeekable, pullOptions map[string]string) (*chunkedDiffer, error) {
+// makeEstargzChunkedDiffer sets up a chunkedDiffer for an estargz layer.
+// It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert.
+func makeEstargzChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, iss ImageSourceSeekable, pullOptions pullOptions) (*chunkedDiffer, error) {
+	if !pullOptions.insecureAllowUnpredictableImageContents { // With no tar-split, we can't compute the traditional UncompressedDigest.
+		return nil, errFallbackCanConvert{
+			newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("estargz layers don't support partial pulls with guaranteed consistency with non-partial pulls")),
+		}
+	}
+
 	manifest, tocOffset, err := readEstargzChunkedManifest(iss, blobSize, tocDigest)
-	if err != nil {
+	if err != nil { // May be ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert
 		return nil, fmt.Errorf("read zstd:chunked manifest: %w", err)
 	}
 	layersCache, err := getLayersCache(store)
@@ -300,17 +392,20 @@ func makeEstargzChunkedDiffer(store storage.Store, blobSize int64, tocDigest dig
 	}
 
 	return &chunkedDiffer{
-		fsVerityDigests:     make(map[string]string),
-		blobSize:            blobSize,
-		uncompressedTarSize: -1, // We would have to read and decompress the whole layer
+		pullOptions: pullOptions,
+		stream:      iss,
+		blobSize:    blobSize,
+
+		fileType: fileTypeEstargz,
+
 		tocDigest:           tocDigest,
-		copyBuffer:          makeCopyBuffer(),
-		fileType:            fileTypeEstargz,
-		layersCache:         layersCache,
-		manifest:            manifest,
-		pullOptions:         pullOptions,
-		stream:              iss,
 		tocOffset:           tocOffset,
+		manifest:            manifest,
+		uncompressedTarSize: -1, // We would have to read and decompress the whole layer
+
+		layersCache:     layersCache,
+		copyBuffer:      makeCopyBuffer(),
+		fsVerityDigests: make(map[string]string),
 	}, nil
 }
 
@@ -391,7 +486,7 @@ func canDedupFileWithHardLink(file *fileMetadata, fd int, s os.FileInfo) bool {
 	}
 	// fill only the attributes used by canDedupMetadataWithHardLink.
 	otherFile := fileMetadata{
-		FileMetadata: internal.FileMetadata{
+		FileMetadata: minimal.FileMetadata{
 			UID:    int(st.Uid),
 			GID:    int(st.Gid),
 			Mode:   int64(st.Mode),
@@ -735,7 +830,12 @@ func (d *destinationFile) Close() (Err error) {
 		}
 	}
 
-	return setFileAttrs(d.dirfd, d.file, os.FileMode(d.metadata.Mode), d.metadata, d.options, false)
+	mode := os.FileMode(d.metadata.Mode)
+	if d.options.ForceMask != nil {
+		mode = *d.options.ForceMask
+	}
+
+	return setFileAttrs(d.dirfd, d.file, mode, d.metadata, d.options, false)
 }
 
 func closeDestinationFiles(files chan *destinationFile, errors chan error) {
@@ -1038,13 +1138,6 @@ type hardLinkToCreate struct {
 	metadata *fileMetadata
 }
 
-func parseBooleanPullOption(pullOptions map[string]string, name string, def bool) bool {
-	if value, ok := pullOptions[name]; ok {
-		return strings.ToLower(value) == "true"
-	}
-	return def
-}
-
 type findAndCopyFileOptions struct {
 	useHardLinks bool
 	ostreeRepos  []string
@@ -1111,10 +1204,13 @@ func (c *chunkedDiffer) findAndCopyFile(dirfd int, r *fileMetadata, copyOptions
 	return false, nil
 }
 
-func makeEntriesFlat(mergedEntries []fileMetadata) ([]fileMetadata, error) {
+// makeEntriesFlat collects regular-file entries from mergedEntries, and produces a new list
+// where each file content is only represented once, and uses composefs.RegularFilePathForValidatedDigest for its name.
+// If flatPathNameMap is not nil, this function writes to it a mapping from filepath.Clean(originalName) to the composefs name.
+func makeEntriesFlat(mergedEntries []fileMetadata, flatPathNameMap map[string]string) ([]fileMetadata, error) {
 	var new []fileMetadata
 
-	hashes := make(map[string]string)
+	knownFlatPaths := make(map[string]struct{})
 	for i := range mergedEntries {
 		if mergedEntries[i].Type != TypeReg {
 			continue
@@ -1124,16 +1220,22 @@ func makeEntriesFlat(mergedEntries []fileMetadata) ([]fileMetadata, error) {
 		}
 		digest, err := digest.Parse(mergedEntries[i].Digest)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("invalid digest %q for %q: %w", mergedEntries[i].Digest, mergedEntries[i].Name, err)
+		}
+		path, err := path.RegularFilePathForValidatedDigest(digest)
+		if err != nil {
+			return nil, fmt.Errorf("determining physical file path for %q: %w", mergedEntries[i].Name, err)
+		}
+		if flatPathNameMap != nil {
+			flatPathNameMap[filepath.Clean(mergedEntries[i].Name)] = path
 		}
-		d := digest.Encoded()
 
-		if hashes[d] != "" {
+		if _, known := knownFlatPaths[path]; known {
 			continue
 		}
-		hashes[d] = d
+		knownFlatPaths[path] = struct{}{}
 
-		mergedEntries[i].Name = fmt.Sprintf("%s/%s", d[0:2], d[2:])
+		mergedEntries[i].Name = path
 		mergedEntries[i].skipSetAttrs = true
 
 		new = append(new, mergedEntries[i])
@@ -1141,44 +1243,140 @@ func makeEntriesFlat(mergedEntries []fileMetadata) ([]fileMetadata, error) {
 	return new, nil
 }
 
-func (c *chunkedDiffer) copyAllBlobToFile(destination *os.File) (digest.Digest, error) {
-	var payload io.ReadCloser
-	var streams chan io.ReadCloser
-	var errs chan error
-	var err error
+type streamOrErr struct {
+	stream io.ReadCloser
+	err    error
+}
 
-	chunksToRequest := []ImageSourceChunk{
-		{
-			Offset: 0,
-			Length: uint64(c.blobSize),
-		},
+// ensureAllBlobsDone ensures that all blobs are closed and returns the first error encountered.
+func ensureAllBlobsDone(streamsOrErrors chan streamOrErr) (retErr error) {
+	for soe := range streamsOrErrors {
+		if soe.stream != nil {
+			_ = soe.stream.Close()
+		} else if retErr == nil {
+			retErr = soe.err
+		}
 	}
+	return
+}
 
-	streams, errs, err = c.stream.GetBlobAt(chunksToRequest)
+// getBlobAtConverterGoroutine reads from the streams and errs channels, then sends
+// either a stream or an error to the stream channel.  The streams channel is closed when
+// there are no more streams and errors to read.
+// It ensures that no more than maxStreams streams are returned, and that every item from the
+// streams and errs channels is consumed.
+func getBlobAtConverterGoroutine(stream chan streamOrErr, streams chan io.ReadCloser, errs chan error, maxStreams int) {
+	tooManyStreams := false
+	streamsSoFar := 0
+
+	err := errors.New("Unexpected error in getBlobAtGoroutine")
+
+	defer func() {
+		if err != nil {
+			stream <- streamOrErr{err: err}
+		}
+		close(stream)
+	}()
+
+loop:
+	for {
+		select {
+		case p, ok := <-streams:
+			if !ok {
+				streams = nil
+				break loop
+			}
+			if streamsSoFar >= maxStreams {
+				tooManyStreams = true
+				_ = p.Close()
+				continue
+			}
+			streamsSoFar++
+			stream <- streamOrErr{stream: p}
+		case err, ok := <-errs:
+			if !ok {
+				errs = nil
+				break loop
+			}
+			stream <- streamOrErr{err: err}
+		}
+	}
+	if streams != nil {
+		for p := range streams {
+			if streamsSoFar >= maxStreams {
+				tooManyStreams = true
+				_ = p.Close()
+				continue
+			}
+			streamsSoFar++
+			stream <- streamOrErr{stream: p}
+		}
+	}
+	if errs != nil {
+		for err := range errs {
+			stream <- streamOrErr{err: err}
+		}
+	}
+	if tooManyStreams {
+		stream <- streamOrErr{err: fmt.Errorf("too many streams returned, got more than %d", maxStreams)}
+	}
+	err = nil
+}
+
+// getBlobAt provides a much more convenient way to consume data returned by ImageSourceSeekable.GetBlobAt.
+// GetBlobAt returns two channels, forcing a caller to `select` on both of them — and in Go, reading a closed channel
+// always succeeds in select.
+// Instead, getBlobAt provides a single channel with all events, which can be consumed conveniently using `range`.
+func getBlobAt(is ImageSourceSeekable, chunksToRequest ...ImageSourceChunk) (chan streamOrErr, error) {
+	streams, errs, err := is.GetBlobAt(chunksToRequest)
+	if err != nil {
+		return nil, err
+	}
+	stream := make(chan streamOrErr)
+	go getBlobAtConverterGoroutine(stream, streams, errs, len(chunksToRequest))
+	return stream, nil
+}
+
+func (c *chunkedDiffer) copyAllBlobToFile(destination *os.File) (digest.Digest, error) {
+	streamsOrErrors, err := getBlobAt(c.stream, ImageSourceChunk{Offset: 0, Length: uint64(c.blobSize)})
 	if err != nil {
 		return "", err
 	}
-	select {
-	case p := <-streams:
-		payload = p
-	case err := <-errs:
-		return "", err
-	}
-	if payload == nil {
-		return "", errors.New("invalid stream returned")
-	}
-	defer payload.Close()
 
 	originalRawDigester := digest.Canonical.Digester()
+	for soe := range streamsOrErrors {
+		if soe.stream != nil {
+			r := io.TeeReader(soe.stream, originalRawDigester.Hash())
 
-	r := io.TeeReader(payload, originalRawDigester.Hash())
-
-	// copy the entire tarball and compute its digest
-	_, err = io.CopyBuffer(destination, r, c.copyBuffer)
-
+			// copy the entire tarball and compute its digest
+			_, err = io.CopyBuffer(destination, r, c.copyBuffer)
+			_ = soe.stream.Close()
+		}
+		if soe.err != nil && err == nil {
+			err = soe.err
+		}
+	}
 	return originalRawDigester.Digest(), err
 }
 
+func typeToOsMode(typ string) (os.FileMode, error) {
+	switch typ {
+	case TypeReg, TypeLink:
+		return 0, nil
+	case TypeSymlink:
+		return os.ModeSymlink, nil
+	case TypeDir:
+		return os.ModeDir, nil
+	case TypeChar:
+		return os.ModeDevice | os.ModeCharDevice, nil
+	case TypeBlock:
+		return os.ModeDevice, nil
+	case TypeFifo:
+		return os.ModeNamedPipe, nil
+	}
+	return 0, fmt.Errorf("unknown file type %q", typ)
+}
+
 func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, differOpts *graphdriver.DifferOptions) (graphdriver.DriverWithDifferOutput, error) {
 	defer c.layersCache.release()
 	defer func() {
@@ -1298,13 +1496,6 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 		Size:               c.uncompressedTarSize,
 	}
 
-	// When the hard links deduplication is used, file attributes are ignored because setting them
-	// modifies the source file as well.
-	useHardLinks := parseBooleanPullOption(c.pullOptions, "use_hard_links", false)
-
-	// List of OSTree repositories to use for deduplication
-	ostreeRepos := strings.Split(c.pullOptions["ostree_repos"], ":")
-
 	whiteoutConverter := archive.GetWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData)
 
 	var missingParts []missingPart
@@ -1325,7 +1516,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 		if err == nil {
 			value := idtools.Stat{
 				IDs:  idtools.IDPair{UID: int(uid), GID: int(gid)},
-				Mode: os.FileMode(mode),
+				Mode: os.ModeDir | os.FileMode(mode),
 			}
 			if err := idtools.SetContainersOverrideXattr(dest, value); err != nil {
 				return output, err
@@ -1337,16 +1528,20 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 	if err != nil {
 		return output, &fs.PathError{Op: "open", Path: dest, Err: err}
 	}
-	defer unix.Close(dirfd)
+	dirFile := os.NewFile(uintptr(dirfd), dest)
+	defer dirFile.Close()
 
+	var flatPathNameMap map[string]string // = nil
 	if differOpts != nil && differOpts.Format == graphdriver.DifferOutputFormatFlat {
-		mergedEntries, err = makeEntriesFlat(mergedEntries)
+		flatPathNameMap = map[string]string{}
+		mergedEntries, err = makeEntriesFlat(mergedEntries, flatPathNameMap)
 		if err != nil {
 			return output, err
 		}
 		createdDirs := make(map[string]struct{})
 		for _, e := range mergedEntries {
-			d := e.Name[0:2]
+			// This hard-codes an assumption that RegularFilePathForValidatedDigest creates paths with exactly one directory component.
+			d := filepath.Dir(e.Name)
 			if _, found := createdDirs[d]; !found {
 				if err := unix.Mkdirat(dirfd, d, 0o755); err != nil {
 					return output, &fs.PathError{Op: "mkdirat", Path: d, Err: err}
@@ -1363,8 +1558,10 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 	missingPartsSize, totalChunksSize := int64(0), int64(0)
 
 	copyOptions := findAndCopyFileOptions{
-		useHardLinks: useHardLinks,
-		ostreeRepos:  ostreeRepos,
+		// When the hard links deduplication is used, file attributes are ignored because setting them
+		// modifies the source file as well.
+		useHardLinks: c.pullOptions.useHardLinks,
+		ostreeRepos:  c.pullOptions.ostreeRepos, // List of OSTree repositories to use for deduplication
 		options:      options,
 	}
 
@@ -1408,13 +1605,6 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 	filesToWaitFor := 0
 	for i := range mergedEntries {
 		r := &mergedEntries[i]
-		if options.ForceMask != nil {
-			value := idtools.FormatContainersOverrideXattr(r.UID, r.GID, int(r.Mode))
-			if r.Xattrs == nil {
-				r.Xattrs = make(map[string]string)
-			}
-			r.Xattrs[idtools.ContainersOverrideXattr] = base64.StdEncoding.EncodeToString([]byte(value))
-		}
 
 		mode := os.FileMode(r.Mode)
 
@@ -1423,10 +1613,37 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 			return output, err
 		}
 
-		r.Name = filepath.Clean(r.Name)
+		size := r.Size
+
+		// update also the implementation of ForceMask in pkg/archive
+		if options.ForceMask != nil {
+			mode = *options.ForceMask
+
+			// special files will be stored as regular files
+			if t != tar.TypeDir && t != tar.TypeSymlink && t != tar.TypeReg && t != tar.TypeLink {
+				t = tar.TypeReg
+				size = 0
+			}
+
+			// if the entry will be stored as a directory or a regular file, store in a xattr the original
+			// owner and mode.
+			if t == tar.TypeDir || t == tar.TypeReg {
+				typeMode, err := typeToOsMode(r.Type)
+				if err != nil {
+					return output, err
+				}
+				value := idtools.FormatContainersOverrideXattrDevice(r.UID, r.GID, typeMode|fs.FileMode(r.Mode), int(r.Devmajor), int(r.Devminor))
+				if r.Xattrs == nil {
+					r.Xattrs = make(map[string]string)
+				}
+				r.Xattrs[idtools.ContainersOverrideXattr] = base64.StdEncoding.EncodeToString([]byte(value))
+			}
+		}
+
+		r.Name = path.CleanAbsPath(r.Name)
 		// do not modify the value of symlinks
 		if r.Linkname != "" && t != tar.TypeSymlink {
-			r.Linkname = filepath.Clean(r.Linkname)
+			r.Linkname = path.CleanAbsPath(r.Linkname)
 		}
 
 		if whiteoutConverter != nil {
@@ -1434,8 +1651,8 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 				Typeflag: t,
 				Name:     r.Name,
 				Linkname: r.Linkname,
-				Size:     r.Size,
-				Mode:     r.Mode,
+				Size:     size,
+				Mode:     int64(mode),
 				Uid:      r.UID,
 				Gid:      r.GID,
 			}
@@ -1454,7 +1671,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 		switch t {
 		case tar.TypeReg:
 			// Create directly empty files.
-			if r.Size == 0 {
+			if size == 0 {
 				// Used to have a scope for cleanup.
 				createEmptyFile := func() error {
 					file, err := openFileUnderRoot(dirfd, r.Name, newFileFlags, 0)
@@ -1474,7 +1691,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 			}
 
 		case tar.TypeDir:
-			if r.Name == "" || r.Name == "." {
+			if r.Name == "/" {
 				output.RootDirMode = &mode
 			}
 			if err := safeMkdir(dirfd, mode, r.Name, r, options); err != nil {
@@ -1509,7 +1726,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 			return output, fmt.Errorf("invalid type %q", t)
 		}
 
-		totalChunksSize += r.Size
+		totalChunksSize += size
 
 		if t == tar.TypeReg {
 			index := i
@@ -1572,7 +1789,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 			}
 
 			switch chunk.ChunkType {
-			case internal.ChunkTypeData:
+			case minimal.ChunkTypeData:
 				root, path, offset, err := c.layersCache.findChunkInOtherLayers(chunk)
 				if err != nil {
 					return output, err
@@ -1585,7 +1802,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 						Offset: offset,
 					}
 				}
-			case internal.ChunkTypeZeros:
+			case minimal.ChunkTypeZeros:
 				missingPartsSize -= size
 				mp.Hole = true
 				// Mark all chunks belonging to the missing part as holes
@@ -1609,6 +1826,39 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 		}
 	}
 
+	// To ensure that consumers of the layer who decompress and read the full tar stream,
+	// and consumers who consume the data via the TOC, both see exactly the same data and metadata,
+	// compute the UncompressedDigest.
+	// c/image will then ensure that this value matches the value in the image config’s RootFS.DiffID, i.e. the image must commit
+	// to one UncompressedDigest value for each layer, and that will avoid the ambiguity (in consumers who validate layers against DiffID).
+	//
+	// c/image also uses the UncompressedDigest as a layer ID, allowing it to use the traditional layer and image IDs.
+	//
+	// This is, sadly, quite costly: Up to now we might have only have had to write, and digest, only the new/modified files.
+	// Here we need to read, and digest, the whole layer, even if almost all of it was already present locally previously.
+	// So, really specialized (EXTREMELY RARE) users can opt out of this check using insecureAllowUnpredictableImageContents .
+	//
+	// Layers without a tar-split (estargz layers and old zstd:chunked layers) can't produce an UncompressedDigest that
+	// matches the expected RootFS.DiffID; we always fall back to full pulls, again unless the user opts out
+	// via insecureAllowUnpredictableImageContents .
+	if output.UncompressedDigest == "" {
+		switch {
+		case c.pullOptions.insecureAllowUnpredictableImageContents:
+			// Oh well.  Skip the costly digest computation.
+		case output.TarSplit != nil:
+			metadata := tsStorage.NewJSONUnpacker(bytes.NewReader(output.TarSplit))
+			fg := newStagedFileGetter(dirFile, flatPathNameMap)
+			digester := digest.Canonical.Digester()
+			if err := asm.WriteOutputTarStream(fg, metadata, digester.Hash()); err != nil {
+				return output, fmt.Errorf("digesting staged uncompressed stream: %w", err)
+			}
+			output.UncompressedDigest = digester.Digest()
+		default:
+			// We are checking for this earlier in GetDiffer, so this should not be reachable.
+			return output, fmt.Errorf(`internal error: layer's UncompressedDigest is unknown and "insecure_allow_unpredictable_image_contents" is not set`)
+		}
+	}
+
 	if totalChunksSize > 0 {
 		logrus.Debugf("Missing %d bytes out of %d (%.2f %%)", missingPartsSize, totalChunksSize, float32(missingPartsSize*100.0)/float32(totalChunksSize))
 	}
@@ -1618,7 +1868,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 	return output, nil
 }
 
-func mustSkipFile(fileType compressedFileType, e internal.FileMetadata) bool {
+func mustSkipFile(fileType compressedFileType, e minimal.FileMetadata) bool {
 	// ignore the metadata files for the estargz format.
 	if fileType != fileTypeEstargz {
 		return false
@@ -1631,7 +1881,7 @@ func mustSkipFile(fileType compressedFileType, e internal.FileMetadata) bool {
 	return false
 }
 
-func (c *chunkedDiffer) mergeTocEntries(fileType compressedFileType, entries []internal.FileMetadata) ([]fileMetadata, error) {
+func (c *chunkedDiffer) mergeTocEntries(fileType compressedFileType, entries []minimal.FileMetadata) ([]fileMetadata, error) {
 	countNextChunks := func(start int) int {
 		count := 0
 		for _, e := range entries[start:] {
@@ -1668,7 +1918,7 @@ func (c *chunkedDiffer) mergeTocEntries(fileType compressedFileType, entries []i
 		if e.Type == TypeReg {
 			nChunks := countNextChunks(i + 1)
 
-			e.chunks = make([]*internal.FileMetadata, nChunks+1)
+			e.chunks = make([]*minimal.FileMetadata, nChunks+1)
 			for j := 0; j <= nChunks; j++ {
 				// we need a copy here, otherwise we override the
 				// .Size later
@@ -1703,7 +1953,7 @@ func (c *chunkedDiffer) mergeTocEntries(fileType compressedFileType, entries []i
 
 // validateChunkChecksum checks if the file at $root/$path[offset:chunk.ChunkSize] has the
 // same digest as chunk.ChunkDigest
-func validateChunkChecksum(chunk *internal.FileMetadata, root, path string, offset int64, copyBuffer []byte) bool {
+func validateChunkChecksum(chunk *minimal.FileMetadata, root, path string, offset int64, copyBuffer []byte) bool {
 	parentDirfd, err := unix.Open(root, unix.O_PATH|unix.O_CLOEXEC, 0)
 	if err != nil {
 		return false
@@ -1734,3 +1984,33 @@ func validateChunkChecksum(chunk *internal.FileMetadata, root, path string, offs
 
 	return digester.Digest() == digest
 }
+
+// newStagedFileGetter returns an object usable as storage.FileGetter for rootDir.
+// if flatPathNameMap is not nil, it must be used to map logical file names into the backing file paths.
+func newStagedFileGetter(rootDir *os.File, flatPathNameMap map[string]string) *stagedFileGetter {
+	return &stagedFileGetter{
+		rootDir:         rootDir,
+		flatPathNameMap: flatPathNameMap,
+	}
+}
+
+type stagedFileGetter struct {
+	rootDir         *os.File
+	flatPathNameMap map[string]string // nil, or a map from filepath.Clean()ed tar file names to expected on-filesystem names
+}
+
+func (fg *stagedFileGetter) Get(filename string) (io.ReadCloser, error) {
+	if fg.flatPathNameMap != nil {
+		path, ok := fg.flatPathNameMap[filepath.Clean(filename)]
+		if !ok {
+			return nil, fmt.Errorf("no path mapping exists for tar entry %q", filename)
+		}
+		filename = path
+	}
+	pathFD, err := securejoin.OpenatInRoot(fg.rootDir, filename)
+	if err != nil {
+		return nil, err
+	}
+	defer pathFD.Close()
+	return securejoin.Reopen(pathFD, unix.O_RDONLY)
+}

vendor/github.com/containers/storage/pkg/chunked/storage_unsupported.go

@@ -13,5 +13,5 @@ import (
 
 // GetDiffer returns a differ than can be used with ApplyDiffWithDiffer.
 func GetDiffer(ctx context.Context, store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) {
-	return nil, errors.New("format not supported on this system")
+	return nil, newErrFallbackToOrdinaryLayerDownload(errors.New("format not supported on this system"))
 }

vendor/github.com/containers/storage/pkg/chunked/toc/toc.go

@@ -3,7 +3,7 @@ package toc
 import (
 	"errors"
 
-	"github.com/containers/storage/pkg/chunked/internal"
+	"github.com/containers/storage/pkg/chunked/internal/minimal"
 	digest "github.com/opencontainers/go-digest"
 )
 
@@ -19,7 +19,7 @@ const tocJSONDigestAnnotation = "containerd.io/snapshot/stargz/toc.digest"
 // This is an experimental feature and may be changed/removed in the future.
 func GetTOCDigest(annotations map[string]string) (*digest.Digest, error) {
 	d1, ok1 := annotations[tocJSONDigestAnnotation]
-	d2, ok2 := annotations[internal.ManifestChecksumKey]
+	d2, ok2 := annotations[minimal.ManifestChecksumKey]
 	switch {
 	case ok1 && ok2:
 		return nil, errors.New("both zstd:chunked and eStargz TOC found")

vendor/github.com/containers/storage/pkg/idtools/idtools.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"os/user"
 	"runtime"
@@ -369,27 +370,66 @@ func checkChownErr(err error, name string, uid, gid int) error {
 
 // Stat contains file states that can be overridden with ContainersOverrideXattr.
 type Stat struct {
-	IDs  IDPair
-	Mode os.FileMode
+	IDs   IDPair
+	Mode  os.FileMode
+	Major int
+	Minor int
 }
 
 // FormatContainersOverrideXattr will format the given uid, gid, and mode into a string
 // that can be used as the value for the ContainersOverrideXattr xattr.
 func FormatContainersOverrideXattr(uid, gid, mode int) string {
-	return fmt.Sprintf("%d:%d:0%o", uid, gid, mode&0o7777)
+	return FormatContainersOverrideXattrDevice(uid, gid, fs.FileMode(mode), 0, 0)
+}
+
+// FormatContainersOverrideXattrDevice will format the given uid, gid, and mode into a string
+// that can be used as the value for the ContainersOverrideXattr xattr.  For devices, it also
+// needs the major and minor numbers.
+func FormatContainersOverrideXattrDevice(uid, gid int, mode fs.FileMode, major, minor int) string {
+	typ := ""
+	switch mode & os.ModeType {
+	case os.ModeDir:
+		typ = "dir"
+	case os.ModeSymlink:
+		typ = "symlink"
+	case os.ModeNamedPipe:
+		typ = "pipe"
+	case os.ModeSocket:
+		typ = "socket"
+	case os.ModeDevice:
+		typ = fmt.Sprintf("block-%d-%d", major, minor)
+	case os.ModeDevice | os.ModeCharDevice:
+		typ = fmt.Sprintf("char-%d-%d", major, minor)
+	default:
+		typ = "file"
+	}
+	unixMode := mode & os.ModePerm
+	if mode&os.ModeSetuid != 0 {
+		unixMode |= 0o4000
+	}
+	if mode&os.ModeSetgid != 0 {
+		unixMode |= 0o2000
+	}
+	if mode&os.ModeSticky != 0 {
+		unixMode |= 0o1000
+	}
+	return fmt.Sprintf("%d:%d:%04o:%s", uid, gid, unixMode, typ)
 }
 
 // GetContainersOverrideXattr will get and decode ContainersOverrideXattr.
 func GetContainersOverrideXattr(path string) (Stat, error) {
-	var stat Stat
 	xstat, err := system.Lgetxattr(path, ContainersOverrideXattr)
 	if err != nil {
-		return stat, err
+		return Stat{}, err
 	}
+	return parseOverrideXattr(xstat) // This will fail if (xstat, err) == (nil, nil), i.e. the xattr does not exist.
+}
 
+func parseOverrideXattr(xstat []byte) (Stat, error) {
+	var stat Stat
 	attrs := strings.Split(string(xstat), ":")
-	if len(attrs) != 3 {
-		return stat, fmt.Errorf("The number of clons in %s does not equal to 3",
+	if len(attrs) < 3 {
+		return stat, fmt.Errorf("The number of parts in %s is less than 3",
 			ContainersOverrideXattr)
 	}
 
@@ -397,47 +437,105 @@ func GetContainersOverrideXattr(path string) (Stat, error) {
 	if err != nil {
 		return stat, fmt.Errorf("Failed to parse UID: %w", err)
 	}
-
 	stat.IDs.UID = int(value)
 
-	value, err = strconv.ParseUint(attrs[0], 10, 32)
+	value, err = strconv.ParseUint(attrs[1], 10, 32)
 	if err != nil {
 		return stat, fmt.Errorf("Failed to parse GID: %w", err)
 	}
-
 	stat.IDs.GID = int(value)
 
 	value, err = strconv.ParseUint(attrs[2], 8, 32)
 	if err != nil {
 		return stat, fmt.Errorf("Failed to parse mode: %w", err)
 	}
+	stat.Mode = os.FileMode(value) & os.ModePerm
+	if value&0o1000 != 0 {
+		stat.Mode |= os.ModeSticky
+	}
+	if value&0o2000 != 0 {
+		stat.Mode |= os.ModeSetgid
+	}
+	if value&0o4000 != 0 {
+		stat.Mode |= os.ModeSetuid
+	}
 
-	stat.Mode = os.FileMode(value)
-
+	if len(attrs) > 3 {
+		typ := attrs[3]
+		if strings.HasPrefix(typ, "file") {
+		} else if strings.HasPrefix(typ, "dir") {
+			stat.Mode |= os.ModeDir
+		} else if strings.HasPrefix(typ, "symlink") {
+			stat.Mode |= os.ModeSymlink
+		} else if strings.HasPrefix(typ, "pipe") {
+			stat.Mode |= os.ModeNamedPipe
+		} else if strings.HasPrefix(typ, "socket") {
+			stat.Mode |= os.ModeSocket
+		} else if strings.HasPrefix(typ, "block") {
+			stat.Mode |= os.ModeDevice
+			stat.Major, stat.Minor, err = parseDevice(typ)
+			if err != nil {
+				return stat, err
+			}
+		} else if strings.HasPrefix(typ, "char") {
+			stat.Mode |= os.ModeDevice | os.ModeCharDevice
+			stat.Major, stat.Minor, err = parseDevice(typ)
+			if err != nil {
+				return stat, err
+			}
+		} else {
+			return stat, fmt.Errorf("Invalid file type %s", typ)
+		}
+	}
 	return stat, nil
 }
 
+func parseDevice(typ string) (int, int, error) {
+	parts := strings.Split(typ, "-")
+	// If there are more than 3 parts, just ignore them to be forward compatible
+	if len(parts) < 3 {
+		return 0, 0, fmt.Errorf("Invalid device type %s", typ)
+	}
+	if parts[0] != "block" && parts[0] != "char" {
+		return 0, 0, fmt.Errorf("Invalid device type %s", typ)
+	}
+	major, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return 0, 0, fmt.Errorf("Failed to parse major number: %w", err)
+	}
+	minor, err := strconv.Atoi(parts[2])
+	if err != nil {
+		return 0, 0, fmt.Errorf("Failed to parse minor number: %w", err)
+	}
+	return major, minor, nil
+}
+
 // SetContainersOverrideXattr will encode and set ContainersOverrideXattr.
 func SetContainersOverrideXattr(path string, stat Stat) error {
-	value := FormatContainersOverrideXattr(stat.IDs.UID, stat.IDs.GID, int(stat.Mode))
+	value := FormatContainersOverrideXattrDevice(stat.IDs.UID, stat.IDs.GID, stat.Mode, stat.Major, stat.Minor)
 	return system.Lsetxattr(path, ContainersOverrideXattr, []byte(value), 0)
 }
 
 func SafeChown(name string, uid, gid int) error {
 	if runtime.GOOS == "darwin" {
-		var mode os.FileMode = 0o0700
-		xstat, err := system.Lgetxattr(name, ContainersOverrideXattr)
-		if err == nil {
-			attrs := strings.Split(string(xstat), ":")
-			if len(attrs) == 3 {
-				val, err := strconv.ParseUint(attrs[2], 8, 32)
-				if err == nil {
-					mode = os.FileMode(val)
-				}
-			}
+		stat := Stat{
+			Mode: os.FileMode(0o0700),
 		}
-		value := Stat{IDPair{uid, gid}, mode}
-		if err = SetContainersOverrideXattr(name, value); err != nil {
+		xstat, err := system.Lgetxattr(name, ContainersOverrideXattr)
+		if err == nil && xstat != nil {
+			stat, err = parseOverrideXattr(xstat)
+			if err != nil {
+				return err
+			}
+		} else {
+			st, err := os.Stat(name) // Ideally we would share this with system.Stat below, but then we would need to convert Mode.
+			if err != nil {
+				return err
+			}
+			stat.Mode = st.Mode()
+		}
+		stat.IDs = IDPair{UID: uid, GID: gid}
+		if err = SetContainersOverrideXattr(name, stat); err != nil {
 			return err
 		}
 		uid = os.Getuid()
@@ -453,19 +551,24 @@ func SafeChown(name string, uid, gid int) error {
 
 func SafeLchown(name string, uid, gid int) error {
 	if runtime.GOOS == "darwin" {
-		var mode os.FileMode = 0o0700
-		xstat, err := system.Lgetxattr(name, ContainersOverrideXattr)
-		if err == nil {
-			attrs := strings.Split(string(xstat), ":")
-			if len(attrs) == 3 {
-				val, err := strconv.ParseUint(attrs[2], 8, 32)
-				if err == nil {
-					mode = os.FileMode(val)
-				}
-			}
+		stat := Stat{
+			Mode: os.FileMode(0o0700),
 		}
-		value := Stat{IDPair{uid, gid}, mode}
-		if err = SetContainersOverrideXattr(name, value); err != nil {
+		xstat, err := system.Lgetxattr(name, ContainersOverrideXattr)
+		if err == nil && xstat != nil {
+			stat, err = parseOverrideXattr(xstat)
+			if err != nil {
+				return err
+			}
+		} else {
+			st, err := os.Lstat(name) // Ideally we would share this with system.Stat below, but then we would need to convert Mode.
+			if err != nil {
+				return err
+			}
+			stat.Mode = st.Mode()
+		}
+		stat.IDs = IDPair{UID: uid, GID: gid}
+		if err = SetContainersOverrideXattr(name, stat); err != nil {
 			return err
 		}
 		uid = os.Getuid()

vendor/github.com/containers/storage/pkg/ioutils/writers.go

@@ -36,9 +36,9 @@ func (r *writeCloserWrapper) Close() error {
 }
 
 // NewWriteCloserWrapper returns a new io.WriteCloser.
-func NewWriteCloserWrapper(r io.Writer, closer func() error) io.WriteCloser {
+func NewWriteCloserWrapper(w io.Writer, closer func() error) io.WriteCloser {
 	return &writeCloserWrapper{
-		Writer: r,
+		Writer: w,
 		closer: closer,
 	}
 }

vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go

@@ -1,4 +1,4 @@
-//go:build linux && cgo
+//go:build linux
 
 package loopback
 

vendor/github.com/containers/storage/pkg/loopback/ioctl.go

@@ -1,4 +1,4 @@
-//go:build linux && cgo
+//go:build linux
 
 package loopback
 

vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go

@@ -1,21 +1,7 @@
-//go:build linux && cgo
+//go:build linux
 
 package loopback
 
-/*
-#include <linux/loop.h> // FIXME: present only for defines, maybe we can remove it?
-
-#ifndef LOOP_CTL_GET_FREE
-  #define LOOP_CTL_GET_FREE 0x4C82
-#endif
-
-#ifndef LO_FLAGS_PARTSCAN
-  #define LO_FLAGS_PARTSCAN 8
-#endif
-
-*/
-import "C"
-
 type loopInfo64 struct {
 	loDevice         uint64 /* ioctl r/o */
 	loInode          uint64 /* ioctl r/o */
@@ -34,19 +20,19 @@ type loopInfo64 struct {
 
 // IOCTL consts
 const (
-	LoopSetFd       = C.LOOP_SET_FD
-	LoopCtlGetFree  = C.LOOP_CTL_GET_FREE
-	LoopGetStatus64 = C.LOOP_GET_STATUS64
-	LoopSetStatus64 = C.LOOP_SET_STATUS64
-	LoopClrFd       = C.LOOP_CLR_FD
-	LoopSetCapacity = C.LOOP_SET_CAPACITY
+	LoopSetFd       = 0x4C00
+	LoopCtlGetFree  = 0x4C82
+	LoopGetStatus64 = 0x4C05
+	LoopSetStatus64 = 0x4C04
+	LoopClrFd       = 0x4C01
+	LoopSetCapacity = 0x4C07
 )
 
 // LOOP consts.
 const (
-	LoFlagsAutoClear = C.LO_FLAGS_AUTOCLEAR
-	LoFlagsReadOnly  = C.LO_FLAGS_READ_ONLY
-	LoFlagsPartScan  = C.LO_FLAGS_PARTSCAN
-	LoKeySize        = C.LO_KEY_SIZE
-	LoNameSize       = C.LO_NAME_SIZE
+	LoFlagsAutoClear = 0x4C07
+	LoFlagsReadOnly  = 1
+	LoFlagsPartScan  = 8
+	LoKeySize        = 32
+	LoNameSize       = 64
 )

vendor/github.com/containers/storage/pkg/loopback/loopback.go

@@ -1,4 +1,4 @@
-//go:build linux && cgo
+//go:build linux
 
 package loopback
 

vendor/github.com/containers/storage/pkg/system/extattr_freebsd.go

@@ -0,0 +1,93 @@
+//go:build freebsd
+
+package system
+
+import (
+	"os"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+const (
+	EXTATTR_NAMESPACE_EMPTY  = unix.EXTATTR_NAMESPACE_EMPTY
+	EXTATTR_NAMESPACE_USER   = unix.EXTATTR_NAMESPACE_USER
+	EXTATTR_NAMESPACE_SYSTEM = unix.EXTATTR_NAMESPACE_SYSTEM
+)
+
+// ExtattrGetLink retrieves the value of the extended attribute identified by attrname
+// in the given namespace and associated with the given path in the file system.
+// If the path is a symbolic link, the extended attribute is retrieved from the link itself.
+// Returns a []byte slice if the extattr is set and nil otherwise.
+func ExtattrGetLink(path string, attrnamespace int, attrname string) ([]byte, error) {
+	size, errno := unix.ExtattrGetLink(path, attrnamespace, attrname,
+		uintptr(unsafe.Pointer(nil)), 0)
+	if errno != nil {
+		if errno == unix.ENOATTR {
+			return nil, nil
+		}
+		return nil, &os.PathError{Op: "extattr_get_link", Path: path, Err: errno}
+	}
+	if size == 0 {
+		return []byte{}, nil
+	}
+
+	dest := make([]byte, size)
+	size, errno = unix.ExtattrGetLink(path, attrnamespace, attrname,
+		uintptr(unsafe.Pointer(&dest[0])), size)
+	if errno != nil {
+		return nil, &os.PathError{Op: "extattr_get_link", Path: path, Err: errno}
+	}
+
+	return dest[:size], nil
+}
+
+// ExtattrSetLink sets the value of extended attribute identified by attrname
+// in the given namespace and associated with the given path in the file system.
+// If the path is a symbolic link, the extended attribute is set on the link itself.
+func ExtattrSetLink(path string, attrnamespace int, attrname string, data []byte) error {
+	if len(data) == 0 {
+		data = []byte{} // ensure non-nil for empty data
+	}
+	if _, errno := unix.ExtattrSetLink(path, attrnamespace, attrname,
+		uintptr(unsafe.Pointer(&data[0])), len(data)); errno != nil {
+		return &os.PathError{Op: "extattr_set_link", Path: path, Err: errno}
+	}
+
+	return nil
+}
+
+// ExtattrListLink lists extended attributes associated with the given path
+// in the specified namespace. If the path is a symbolic link, the attributes
+// are listed from the link itself.
+func ExtattrListLink(path string, attrnamespace int) ([]string, error) {
+	size, errno := unix.ExtattrListLink(path, attrnamespace,
+		uintptr(unsafe.Pointer(nil)), 0)
+	if errno != nil {
+		return nil, &os.PathError{Op: "extattr_list_link", Path: path, Err: errno}
+	}
+	if size == 0 {
+		return []string{}, nil
+	}
+
+	dest := make([]byte, size)
+	size, errno = unix.ExtattrListLink(path, attrnamespace,
+		uintptr(unsafe.Pointer(&dest[0])), size)
+	if errno != nil {
+		return nil, &os.PathError{Op: "extattr_list_link", Path: path, Err: errno}
+	}
+
+	var attrs []string
+	for i := 0; i < size; {
+		// Each attribute is preceded by a single byte length
+		length := int(dest[i])
+		i++
+		if i+length > size {
+			break
+		}
+		attrs = append(attrs, string(dest[i:i+length]))
+		i += length
+	}
+
+	return attrs, nil
+}

vendor/github.com/containers/storage/pkg/system/extattr_unsupported.go

@@ -0,0 +1,24 @@
+//go:build !freebsd
+
+package system
+
+const (
+	EXTATTR_NAMESPACE_EMPTY  = 0
+	EXTATTR_NAMESPACE_USER   = 0
+	EXTATTR_NAMESPACE_SYSTEM = 0
+)
+
+// ExtattrGetLink is not supported on platforms other than FreeBSD.
+func ExtattrGetLink(path string, attrnamespace int, attrname string) ([]byte, error) {
+	return nil, ErrNotSupportedPlatform
+}
+
+// ExtattrSetLink is not supported on platforms other than FreeBSD.
+func ExtattrSetLink(path string, attrnamespace int, attrname string, data []byte) error {
+	return ErrNotSupportedPlatform
+}
+
+// ExtattrListLink is not supported on platforms other than FreeBSD.
+func ExtattrListLink(path string, attrnamespace int) ([]string, error) {
+	return nil, ErrNotSupportedPlatform
+}

vendor/github.com/containers/storage/pkg/system/stat_netbsd.go

@@ -0,0 +1,13 @@
+package system
+
+import "syscall"
+
+// fromStatT converts a syscall.Stat_t type to a system.Stat_t type
+func fromStatT(s *syscall.Stat_t) (*StatT, error) {
+	return &StatT{size: s.Size,
+		mode: uint32(s.Mode),
+		uid:  s.Uid,
+		gid:  s.Gid,
+		rdev: uint64(s.Rdev),
+		mtim: s.Mtimespec}, nil
+}

vendor/github.com/containers/storage/pkg/system/xattrs_darwin.go

@@ -12,7 +12,7 @@ const (
 	E2BIG unix.Errno = unix.E2BIG
 
 	// Operation not supported
-	EOPNOTSUPP unix.Errno = unix.EOPNOTSUPP
+	ENOTSUP unix.Errno = unix.ENOTSUP
 )
 
 // Lgetxattr retrieves the value of the extended attribute identified by attr

vendor/github.com/containers/storage/pkg/system/xattrs_freebsd.go

@@ -0,0 +1,85 @@
+package system
+
+import (
+	"strings"
+
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// Value is larger than the maximum size allowed
+	E2BIG unix.Errno = unix.E2BIG
+
+	// Operation not supported
+	ENOTSUP unix.Errno = unix.ENOTSUP
+
+	// Value is too small or too large for maximum size allowed
+	EOVERFLOW unix.Errno = unix.EOVERFLOW
+)
+
+var (
+	namespaceMap = map[string]int{
+		"user":   EXTATTR_NAMESPACE_USER,
+		"system": EXTATTR_NAMESPACE_SYSTEM,
+	}
+)
+
+func xattrToExtattr(xattr string) (namespace int, extattr string, err error) {
+	namespaceName, extattr, found := strings.Cut(xattr, ".")
+	if !found {
+		return -1, "", ENOTSUP
+	}
+
+	namespace, ok := namespaceMap[namespaceName]
+	if !ok {
+		return -1, "", ENOTSUP
+	}
+	return namespace, extattr, nil
+}
+
+// Lgetxattr retrieves the value of the extended attribute identified by attr
+// and associated with the given path in the file system.
+// Returns a []byte slice if the xattr is set and nil otherwise.
+func Lgetxattr(path string, attr string) ([]byte, error) {
+	namespace, extattr, err := xattrToExtattr(attr)
+	if err != nil {
+		return nil, err
+	}
+	return ExtattrGetLink(path, namespace, extattr)
+}
+
+// Lsetxattr sets the value of the extended attribute identified by attr
+// and associated with the given path in the file system.
+func Lsetxattr(path string, attr string, value []byte, flags int) error {
+	if flags != 0 {
+		// FIXME: Flags are not supported on FreeBSD, but we can implement
+		// them mimicking the behavior of the Linux implementation.
+		// See lsetxattr(2) on Linux for more information.
+		return ENOTSUP
+	}
+
+	namespace, extattr, err := xattrToExtattr(attr)
+	if err != nil {
+		return err
+	}
+	return ExtattrSetLink(path, namespace, extattr, value)
+}
+
+// Llistxattr lists extended attributes associated with the given path
+// in the file system.
+func Llistxattr(path string) ([]string, error) {
+	attrs := []string{}
+
+	for namespaceName, namespace := range namespaceMap {
+		namespaceAttrs, err := ExtattrListLink(path, namespace)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, attr := range namespaceAttrs {
+			attrs = append(attrs, namespaceName+"."+attr)
+		}
+	}
+
+	return attrs, nil
+}

vendor/github.com/containers/storage/pkg/system/xattrs_linux.go

@@ -12,7 +12,7 @@ const (
 	E2BIG unix.Errno = unix.E2BIG
 
 	// Operation not supported
-	EOPNOTSUPP unix.Errno = unix.EOPNOTSUPP
+	ENOTSUP unix.Errno = unix.ENOTSUP
 
 	// Value is too small or too large for maximum size allowed
 	EOVERFLOW unix.Errno = unix.EOVERFLOW

vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go

@@ -1,4 +1,4 @@
-//go:build !linux && !darwin
+//go:build !linux && !darwin && !freebsd
 
 package system
 
@@ -9,7 +9,7 @@ const (
 	E2BIG syscall.Errno = syscall.Errno(0)
 
 	// Operation not supported
-	EOPNOTSUPP syscall.Errno = syscall.Errno(0)
+	ENOTSUP syscall.Errno = syscall.Errno(0)
 
 	// Value is too small or too large for maximum size allowed
 	EOVERFLOW syscall.Errno = syscall.Errno(0)

vendor/github.com/containers/storage/storage.conf

@@ -80,6 +80,25 @@ additionalimagestores = [
 # This is a "string bool": "false" | "true" (cannot be native TOML boolean)
 # convert_images = "false"
 
+# This should ALMOST NEVER be set.
+# It allows partial pulls of images without guaranteeing that "partial
+# pulls" and non-partial pulls both result in consistent image contents.
+# This allows pulling estargz images and early versions of zstd:chunked images;
+# otherwise, these layers always use the traditional non-partial pull path.
+#
+# This option should be enabled EXTREMELY rarely, only if ALL images that could
+# EVER be conceivably pulled on this system are GUARANTEED (e.g. using a signature policy)
+# to come from a build system trusted to never attack image integrity.
+#
+# If this consistency enforcement were disabled, malicious images could be built
+# in a way designed to evade other audit mechanisms, so presence of most other audit
+# mechanisms is not a replacement for the above-mentioned need for all images to come
+# from a trusted build system.
+#
+# As a side effect, enabling this option will also make image IDs unpredictable
+# (usually not equal to the traditional value matching the config digest).
+# insecure_allow_unpredictable_image_contents = "false"
+
 # Root-auto-userns-user is a user name which can be used to look up one or more UID/GID
 # ranges in the /etc/subuid and /etc/subgid file.  These ranges will be partitioned
 # to containers configured to create automatically a user namespace.  Containers

vendor/github.com/containers/storage/store.go

@@ -20,6 +20,7 @@ import (
 	_ "github.com/containers/storage/drivers/register"
 
 	drivers "github.com/containers/storage/drivers"
+	"github.com/containers/storage/internal/dedup"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/directory"
 	"github.com/containers/storage/pkg/idtools"
@@ -166,6 +167,26 @@ type flaggableStore interface {
 
 type StoreOptions = types.StoreOptions
 
+type DedupHashMethod = dedup.DedupHashMethod
+
+const (
+	DedupHashInvalid  = dedup.DedupHashInvalid
+	DedupHashCRC      = dedup.DedupHashCRC
+	DedupHashFileSize = dedup.DedupHashFileSize
+	DedupHashSHA256   = dedup.DedupHashSHA256
+)
+
+type (
+	DedupOptions = dedup.DedupOptions
+	DedupResult  = dedup.DedupResult
+)
+
+// DedupArgs is used to pass arguments to the Dedup command.
+type DedupArgs struct {
+	// Options that are passed directly to the internal/dedup.DedupDirs function.
+	Options DedupOptions
+}
+
 // Store wraps up the various types of file-based stores that we use into a
 // singleton object that initializes and manages them all together.
 type Store interface {
@@ -589,6 +610,9 @@ type Store interface {
 	// MultiList returns consistent values as of a single point in time.
 	// WARNING: The values may already be out of date by the time they are returned to the caller.
 	MultiList(MultiListOptions) (MultiListResult, error)
+
+	// Dedup deduplicates layers in the store.
+	Dedup(DedupArgs) (drivers.DedupResult, error)
 }
 
 // AdditionalLayer represents a layer that is contained in the additional layer store
@@ -3843,3 +3867,43 @@ func (s *store) MultiList(options MultiListOptions) (MultiListResult, error) {
 	}
 	return out, nil
 }
+
+// Dedup deduplicates layers in the store.
+func (s *store) Dedup(req DedupArgs) (drivers.DedupResult, error) {
+	imgs, err := s.Images()
+	if err != nil {
+		return drivers.DedupResult{}, err
+	}
+	var topLayers []string
+	for _, i := range imgs {
+		topLayers = append(topLayers, i.TopLayer)
+		topLayers = append(topLayers, i.MappedTopLayers...)
+	}
+	return writeToLayerStore(s, func(rlstore rwLayerStore) (drivers.DedupResult, error) {
+		layers := make(map[string]struct{})
+		for _, i := range topLayers {
+			cur := i
+			for cur != "" {
+				if _, visited := layers[cur]; visited {
+					break
+				}
+				l, err := rlstore.Get(cur)
+				if err != nil {
+					if err == ErrLayerUnknown {
+						break
+					}
+					return drivers.DedupResult{}, err
+				}
+				layers[cur] = struct{}{}
+				cur = l.Parent
+			}
+		}
+		r := drivers.DedupArgs{
+			Options: req.Options,
+		}
+		for l := range layers {
+			r.Layers = append(r.Layers, l)
+		}
+		return rlstore.dedup(r)
+	})
+}