Version 5.22 introduced a new option to /etc/containers/policy.json called
keyPaths, see
https://github.com/containers/image/pull/1609
EL9 immediately took advantage of this new feature and started using it, see
04645c4a84
This quickly became an issue in our code: The go library (containers/image)
parses the configuration file very strictly and refuses to create a client
when policy.json with an unknown key is present on the filesystem. As we
used 5.21.1 that doesn't know the new key, our unit tests started to
failing when containers-common was present.
Reproducer:
podman run --pull=always --rm -it centos:stream9
dnf install -y dnf-plugins-core
dnf config-manager --set-enabled crb
dnf install -y gpgme-devel libassuan-devel krb5-devel golang git-core
git clone https://github.com/osbuild/osbuild-composer
cd osbuild-composer
# install the new containers-common and run the test
dnf install -y https://kojihub.stream.centos.org/kojifiles/packages/containers-common/1/44.el9/x86_64/containers-common-1-44.el9.x86_64.rpm
go test -count 1 ./...
# this returns:
--- FAIL: TestClientResolve (0.00s)
client_test.go:31:
Error Trace: client_test.go:31
Error: Received unexpected error:
Unknown key "keyPaths"
invalid policy in "/etc/containers/policy.json"
github.com/containers/image/v5/signature.NewPolicyFromFile
/osbuild-composer/vendor/github.com/containers/image/v5/signature/policy_config.go:88
github.com/osbuild/osbuild-composer/internal/container.NewClient
/osbuild-composer/internal/container/client.go:123
github.com/osbuild/osbuild-composer/internal/container_test.TestClientResolve
/osbuild-composer/internal/container/client_test.go:29
testing.tRunner
/usr/lib/golang/src/testing/testing.go:1439
runtime.goexit
/usr/lib/golang/src/runtime/asm_amd64.s:1571
Test: TestClientResolve
client_test.go:32:
Error Trace: client_test.go:32
Error: Expected value not to be nil.
Test: TestClientResolve
When run with an older containers-common, it succeeds:
dnf install -y https://kojihub.stream.centos.org/kojifiles/packages/containers-common/1/40.el9/x86_64/containers-common-1-40.el9.x86_64.rpm
go test -count 1 ./...
PASS
To sum it up, I had to upgrade github.com/containers/image/v5 to v5.22.0.
Unfortunately, this wasn't so simple, see
go get github.com/containers/image/v5@latest
go: github.com/containers/image/v5@v5.22.0 requires
github.com/letsencrypt/boulder@v0.0.0-20220331220046-b23ab962616e requires
github.com/honeycombio/beeline-go@v1.1.1 requires
github.com/gobuffalo/pop/v5@v5.3.1 requires
github.com/mattn/go-sqlite3@v2.0.3+incompatible: reading github.com/mattn/go-sqlite3/go.mod at revision v2.0.3: unknown revision v2.0.3
It turns out that github.com/mattn/go-sqlite3@v2.0.3+incompatible has been
recently retracted https://github.com/mattn/go-sqlite3/pull/998 and this
broke a ton of packages depending on it. I was able to fix it by adding
exclude github.com/mattn/go-sqlite3 v2.0.3+incompatible
to our go.mod, see
https://github.com/mattn/go-sqlite3/issues/975#issuecomment-955661657
After adding it,
go get github.com/containers/image/v5@latest
succeeded and tools/prepare-source.sh took care of the rest.
Signed-off-by: Ondřej Budai <ondrej@budai.cz>
230 lines
10 KiB
Go
230 lines
10 KiB
Go
package manifest
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
|
||
compressiontypes "github.com/containers/image/v5/pkg/compression/types"
|
||
"github.com/containers/image/v5/types"
|
||
"github.com/sirupsen/logrus"
|
||
)
|
||
|
||
// dupStringSlice returns a deep copy of a slice of strings, or nil if the
|
||
// source slice is empty.
|
||
func dupStringSlice(list []string) []string {
|
||
if len(list) == 0 {
|
||
return nil
|
||
}
|
||
dup := make([]string, len(list))
|
||
copy(dup, list)
|
||
return dup
|
||
}
|
||
|
||
// dupStringStringMap returns a deep copy of a map[string]string, or nil if the
|
||
// passed-in map is nil or has no keys.
|
||
func dupStringStringMap(m map[string]string) map[string]string {
|
||
if len(m) == 0 {
|
||
return nil
|
||
}
|
||
result := make(map[string]string)
|
||
for k, v := range m {
|
||
result[k] = v
|
||
}
|
||
return result
|
||
}
|
||
|
||
// allowedManifestFields is a bit mask of “essential” manifest fields that validateUnambiguousManifestFormat
|
||
// can expect to be present.
|
||
type allowedManifestFields int
|
||
|
||
const (
|
||
allowedFieldConfig allowedManifestFields = 1 << iota
|
||
allowedFieldFSLayers
|
||
allowedFieldHistory
|
||
allowedFieldLayers
|
||
allowedFieldManifests
|
||
allowedFieldFirstUnusedBit // Keep this at the end!
|
||
)
|
||
|
||
// validateUnambiguousManifestFormat rejects manifests (incl. multi-arch) that look like more than
|
||
// one kind we currently recognize, i.e. if they contain any of the known “essential” format fields
|
||
// other than the ones the caller specifically allows.
|
||
// expectedMIMEType is used only for diagnostics.
|
||
// NOTE: The caller should do the non-heuristic validations (e.g. check for any specified format
|
||
// identification/version, or other “magic numbers”) before calling this, to cleanly reject unambiguous
|
||
// data that just isn’t what was expected, as opposed to actually ambiguous data.
|
||
func validateUnambiguousManifestFormat(manifest []byte, expectedMIMEType string,
|
||
allowed allowedManifestFields) error {
|
||
if allowed >= allowedFieldFirstUnusedBit {
|
||
return fmt.Errorf("internal error: invalid allowedManifestFields value %#v", allowed)
|
||
}
|
||
// Use a private type to decode, not just a map[string]interface{}, because we want
|
||
// to also reject case-insensitive matches (which would be used by Go when really decoding
|
||
// the manifest).
|
||
// (It is expected that as manifest formats are added or extended over time, more fields will be added
|
||
// here.)
|
||
detectedFields := struct {
|
||
Config interface{} `json:"config"`
|
||
FSLayers interface{} `json:"fsLayers"`
|
||
History interface{} `json:"history"`
|
||
Layers interface{} `json:"layers"`
|
||
Manifests interface{} `json:"manifests"`
|
||
}{}
|
||
if err := json.Unmarshal(manifest, &detectedFields); err != nil {
|
||
// The caller was supposed to already validate version numbers, so this should not happen;
|
||
// let’s not bother with making this error “nice”.
|
||
return err
|
||
}
|
||
unexpected := []string{}
|
||
// Sadly this isn’t easy to automate in Go, without reflection. So, copy&paste.
|
||
if detectedFields.Config != nil && (allowed&allowedFieldConfig) == 0 {
|
||
unexpected = append(unexpected, "config")
|
||
}
|
||
if detectedFields.FSLayers != nil && (allowed&allowedFieldFSLayers) == 0 {
|
||
unexpected = append(unexpected, "fsLayers")
|
||
}
|
||
if detectedFields.History != nil && (allowed&allowedFieldHistory) == 0 {
|
||
unexpected = append(unexpected, "history")
|
||
}
|
||
if detectedFields.Layers != nil && (allowed&allowedFieldLayers) == 0 {
|
||
unexpected = append(unexpected, "layers")
|
||
}
|
||
if detectedFields.Manifests != nil && (allowed&allowedFieldManifests) == 0 {
|
||
unexpected = append(unexpected, "manifests")
|
||
}
|
||
if len(unexpected) != 0 {
|
||
return fmt.Errorf(`rejecting ambiguous manifest, unexpected fields %#v in supposedly %s`,
|
||
unexpected, expectedMIMEType)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// layerInfosToStrings converts a list of layer infos, presumably obtained from a Manifest.LayerInfos()
|
||
// method call, into a format suitable for inclusion in a types.ImageInspectInfo structure.
|
||
func layerInfosToStrings(infos []LayerInfo) []string {
|
||
layers := make([]string, len(infos))
|
||
for i, info := range infos {
|
||
layers[i] = info.Digest.String()
|
||
}
|
||
return layers
|
||
}
|
||
|
||
// compressionMIMETypeSet describes a set of MIME type “variants” that represent differently-compressed
|
||
// versions of “the same kind of content”.
|
||
// The map key is the return value of compressiontypes.Algorithm.Name(), or mtsUncompressed;
|
||
// the map value is a MIME type, or mtsUnsupportedMIMEType to mean "recognized but unsupported".
|
||
type compressionMIMETypeSet map[string]string
|
||
|
||
const mtsUncompressed = "" // A key in compressionMIMETypeSet for the uncompressed variant
|
||
const mtsUnsupportedMIMEType = "" // A value in compressionMIMETypeSet that means “recognized but unsupported”
|
||
|
||
// findCompressionMIMETypeSet returns a pointer to a compressionMIMETypeSet in variantTable that contains a value of mimeType, or nil if not found
|
||
func findCompressionMIMETypeSet(variantTable []compressionMIMETypeSet, mimeType string) compressionMIMETypeSet {
|
||
for _, variants := range variantTable {
|
||
for _, mt := range variants {
|
||
if mt == mimeType {
|
||
return variants
|
||
}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// compressionVariantMIMEType returns a variant of mimeType for the specified algorithm (which may be nil
|
||
// to mean "no compression"), based on variantTable.
|
||
// The returned error will be a ManifestLayerCompressionIncompatibilityError if mimeType has variants
|
||
// that differ only in what type of compression is applied, but it can't be combined with this
|
||
// algorithm to produce an updated MIME type that complies with the standard that defines mimeType.
|
||
// If the compression algorithm is unrecognized, or mimeType is not known to have variants that
|
||
// differ from it only in what type of compression has been applied, the returned error will not be
|
||
// a ManifestLayerCompressionIncompatibilityError.
|
||
func compressionVariantMIMEType(variantTable []compressionMIMETypeSet, mimeType string, algorithm *compressiontypes.Algorithm) (string, error) {
|
||
if mimeType == mtsUnsupportedMIMEType { // Prevent matching against the {algo:mtsUnsupportedMIMEType} entries
|
||
return "", fmt.Errorf("cannot update unknown MIME type")
|
||
}
|
||
variants := findCompressionMIMETypeSet(variantTable, mimeType)
|
||
if variants != nil {
|
||
name := mtsUncompressed
|
||
if algorithm != nil {
|
||
name = algorithm.InternalUnstableUndocumentedMIMEQuestionMark()
|
||
}
|
||
if res, ok := variants[name]; ok {
|
||
if res != mtsUnsupportedMIMEType {
|
||
return res, nil
|
||
}
|
||
if name != mtsUncompressed {
|
||
return "", ManifestLayerCompressionIncompatibilityError{fmt.Sprintf("%s compression is not supported for type %q", name, mimeType)}
|
||
}
|
||
return "", ManifestLayerCompressionIncompatibilityError{fmt.Sprintf("uncompressed variant is not supported for type %q", mimeType)}
|
||
}
|
||
if name != mtsUncompressed {
|
||
return "", ManifestLayerCompressionIncompatibilityError{fmt.Sprintf("unknown compressed with algorithm %s variant for type %s", name, mimeType)}
|
||
}
|
||
// We can't very well say “the idea of no compression is unknown”
|
||
return "", ManifestLayerCompressionIncompatibilityError{fmt.Sprintf("uncompressed variant is not supported for type %q", mimeType)}
|
||
}
|
||
if algorithm != nil {
|
||
return "", fmt.Errorf("unsupported MIME type for compression: %s", mimeType)
|
||
}
|
||
return "", fmt.Errorf("unsupported MIME type for decompression: %s", mimeType)
|
||
}
|
||
|
||
// updatedMIMEType returns the result of applying edits in updated (MediaType, CompressionOperation) to
|
||
// mimeType, based on variantTable. It may use updated.Digest for error messages.
|
||
// The returned error will be a ManifestLayerCompressionIncompatibilityError if mimeType has variants
|
||
// that differ only in what type of compression is applied, but applying updated.CompressionOperation
|
||
// and updated.CompressionAlgorithm to it won't produce an updated MIME type that complies with the
|
||
// standard that defines mimeType.
|
||
func updatedMIMEType(variantTable []compressionMIMETypeSet, mimeType string, updated types.BlobInfo) (string, error) {
|
||
// Note that manifests in containers-storage might be reporting the
|
||
// wrong media type since the original manifests are stored while layers
|
||
// are decompressed in storage. Hence, we need to consider the case
|
||
// that an already {de}compressed layer should be {de}compressed;
|
||
// compressionVariantMIMEType does that by not caring whether the original is
|
||
// {de}compressed.
|
||
switch updated.CompressionOperation {
|
||
case types.PreserveOriginal:
|
||
// Force a change to the media type if we're being told to use a particular compressor,
|
||
// since it might be different from the one associated with the media type. Otherwise,
|
||
// try to keep the original media type.
|
||
if updated.CompressionAlgorithm != nil {
|
||
return compressionVariantMIMEType(variantTable, mimeType, updated.CompressionAlgorithm)
|
||
}
|
||
// Keep the original media type.
|
||
return mimeType, nil
|
||
|
||
case types.Decompress:
|
||
return compressionVariantMIMEType(variantTable, mimeType, nil)
|
||
|
||
case types.Compress:
|
||
if updated.CompressionAlgorithm == nil {
|
||
logrus.Debugf("Error preparing updated manifest: blob %q was compressed but does not specify by which algorithm: falling back to use the original blob", updated.Digest)
|
||
return mimeType, nil
|
||
}
|
||
return compressionVariantMIMEType(variantTable, mimeType, updated.CompressionAlgorithm)
|
||
|
||
default:
|
||
return "", fmt.Errorf("unknown compression operation (%d)", updated.CompressionOperation)
|
||
}
|
||
}
|
||
|
||
// ManifestLayerCompressionIncompatibilityError indicates that a specified compression algorithm
|
||
// could not be applied to a layer MIME type. A caller that receives this should either retry
|
||
// the call with a different compression algorithm, or attempt to use a different manifest type.
|
||
type ManifestLayerCompressionIncompatibilityError struct {
|
||
text string
|
||
}
|
||
|
||
func (m ManifestLayerCompressionIncompatibilityError) Error() string {
|
||
return m.text
|
||
}
|
||
|
||
// compressionVariantsRecognizeMIMEType returns true if variantTable contains data about compressing/decompressing layers with mimeType
|
||
// Note that the caller still needs to worry about a specific algorithm not being supported.
|
||
func compressionVariantsRecognizeMIMEType(variantTable []compressionMIMETypeSet, mimeType string) bool {
|
||
if mimeType == mtsUnsupportedMIMEType { // Prevent matching against the {algo:mtsUnsupportedMIMEType} entries
|
||
return false
|
||
}
|
||
variants := findCompressionMIMETypeSet(variantTable, mimeType)
|
||
return variants != nil // Alternatively, this could be len(variants) > 1, but really the caller should ask about a specific algorithm.
|
||
}
|