Version 5.22 introduced a new option to /etc/containers/policy.json called
keyPaths, see
https://github.com/containers/image/pull/1609
EL9 immediately took advantage of this new feature and started using it, see
04645c4a84
This quickly became an issue in our code: The go library (containers/image)
parses the configuration file very strictly and refuses to create a client
when policy.json with an unknown key is present on the filesystem. As we
used 5.21.1 that doesn't know the new key, our unit tests started to
failing when containers-common was present.
Reproducer:
podman run --pull=always --rm -it centos:stream9
dnf install -y dnf-plugins-core
dnf config-manager --set-enabled crb
dnf install -y gpgme-devel libassuan-devel krb5-devel golang git-core
git clone https://github.com/osbuild/osbuild-composer
cd osbuild-composer
# install the new containers-common and run the test
dnf install -y https://kojihub.stream.centos.org/kojifiles/packages/containers-common/1/44.el9/x86_64/containers-common-1-44.el9.x86_64.rpm
go test -count 1 ./...
# this returns:
--- FAIL: TestClientResolve (0.00s)
client_test.go:31:
Error Trace: client_test.go:31
Error: Received unexpected error:
Unknown key "keyPaths"
invalid policy in "/etc/containers/policy.json"
github.com/containers/image/v5/signature.NewPolicyFromFile
/osbuild-composer/vendor/github.com/containers/image/v5/signature/policy_config.go:88
github.com/osbuild/osbuild-composer/internal/container.NewClient
/osbuild-composer/internal/container/client.go:123
github.com/osbuild/osbuild-composer/internal/container_test.TestClientResolve
/osbuild-composer/internal/container/client_test.go:29
testing.tRunner
/usr/lib/golang/src/testing/testing.go:1439
runtime.goexit
/usr/lib/golang/src/runtime/asm_amd64.s:1571
Test: TestClientResolve
client_test.go:32:
Error Trace: client_test.go:32
Error: Expected value not to be nil.
Test: TestClientResolve
When run with an older containers-common, it succeeds:
dnf install -y https://kojihub.stream.centos.org/kojifiles/packages/containers-common/1/40.el9/x86_64/containers-common-1-40.el9.x86_64.rpm
go test -count 1 ./...
PASS
To sum it up, I had to upgrade github.com/containers/image/v5 to v5.22.0.
Unfortunately, this wasn't so simple, see
go get github.com/containers/image/v5@latest
go: github.com/containers/image/v5@v5.22.0 requires
github.com/letsencrypt/boulder@v0.0.0-20220331220046-b23ab962616e requires
github.com/honeycombio/beeline-go@v1.1.1 requires
github.com/gobuffalo/pop/v5@v5.3.1 requires
github.com/mattn/go-sqlite3@v2.0.3+incompatible: reading github.com/mattn/go-sqlite3/go.mod at revision v2.0.3: unknown revision v2.0.3
It turns out that github.com/mattn/go-sqlite3@v2.0.3+incompatible has been
recently retracted https://github.com/mattn/go-sqlite3/pull/998 and this
broke a ton of packages depending on it. I was able to fix it by adding
exclude github.com/mattn/go-sqlite3 v2.0.3+incompatible
to our go.mod, see
https://github.com/mattn/go-sqlite3/issues/975#issuecomment-955661657
After adding it,
go get github.com/containers/image/v5@latest
succeeded and tools/prepare-source.sh took care of the rest.
Signed-off-by: Ondřej Budai <ondrej@budai.cz>
233 lines
5.7 KiB
Go
233 lines
5.7 KiB
Go
// Copyright 2018 Klaus Post. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
// Based on work Copyright (c) 2013, Yann Collet, released under BSD License.
|
|
|
|
package huff0
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
)
|
|
|
|
// bitReader reads a bitstream in reverse.
|
|
// The last set bit indicates the start of the stream and is used
|
|
// for aligning the input.
|
|
type bitReaderBytes struct {
|
|
in []byte
|
|
off uint // next byte to read is at in[off - 1]
|
|
value uint64
|
|
bitsRead uint8
|
|
}
|
|
|
|
// init initializes and resets the bit reader.
|
|
func (b *bitReaderBytes) init(in []byte) error {
|
|
if len(in) < 1 {
|
|
return errors.New("corrupt stream: too short")
|
|
}
|
|
b.in = in
|
|
b.off = uint(len(in))
|
|
// The highest bit of the last byte indicates where to start
|
|
v := in[len(in)-1]
|
|
if v == 0 {
|
|
return errors.New("corrupt stream, did not find end of stream")
|
|
}
|
|
b.bitsRead = 64
|
|
b.value = 0
|
|
if len(in) >= 8 {
|
|
b.fillFastStart()
|
|
} else {
|
|
b.fill()
|
|
b.fill()
|
|
}
|
|
b.advance(8 - uint8(highBit32(uint32(v))))
|
|
return nil
|
|
}
|
|
|
|
// peekBitsFast requires that at least one bit is requested every time.
|
|
// There are no checks if the buffer is filled.
|
|
func (b *bitReaderBytes) peekByteFast() uint8 {
|
|
got := uint8(b.value >> 56)
|
|
return got
|
|
}
|
|
|
|
func (b *bitReaderBytes) advance(n uint8) {
|
|
b.bitsRead += n
|
|
b.value <<= n & 63
|
|
}
|
|
|
|
// fillFast() will make sure at least 32 bits are available.
|
|
// There must be at least 4 bytes available.
|
|
func (b *bitReaderBytes) fillFast() {
|
|
if b.bitsRead < 32 {
|
|
return
|
|
}
|
|
|
|
// 2 bounds checks.
|
|
v := b.in[b.off-4 : b.off]
|
|
v = v[:4]
|
|
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
|
b.value |= uint64(low) << (b.bitsRead - 32)
|
|
b.bitsRead -= 32
|
|
b.off -= 4
|
|
}
|
|
|
|
// fillFastStart() assumes the bitReaderBytes is empty and there is at least 8 bytes to read.
|
|
func (b *bitReaderBytes) fillFastStart() {
|
|
// Do single re-slice to avoid bounds checks.
|
|
b.value = binary.LittleEndian.Uint64(b.in[b.off-8:])
|
|
b.bitsRead = 0
|
|
b.off -= 8
|
|
}
|
|
|
|
// fill() will make sure at least 32 bits are available.
|
|
func (b *bitReaderBytes) fill() {
|
|
if b.bitsRead < 32 {
|
|
return
|
|
}
|
|
if b.off > 4 {
|
|
v := b.in[b.off-4:]
|
|
v = v[:4]
|
|
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
|
b.value |= uint64(low) << (b.bitsRead - 32)
|
|
b.bitsRead -= 32
|
|
b.off -= 4
|
|
return
|
|
}
|
|
for b.off > 0 {
|
|
b.value |= uint64(b.in[b.off-1]) << (b.bitsRead - 8)
|
|
b.bitsRead -= 8
|
|
b.off--
|
|
}
|
|
}
|
|
|
|
// finished returns true if all bits have been read from the bit stream.
|
|
func (b *bitReaderBytes) finished() bool {
|
|
return b.off == 0 && b.bitsRead >= 64
|
|
}
|
|
|
|
func (b *bitReaderBytes) remaining() uint {
|
|
return b.off*8 + uint(64-b.bitsRead)
|
|
}
|
|
|
|
// close the bitstream and returns an error if out-of-buffer reads occurred.
|
|
func (b *bitReaderBytes) close() error {
|
|
// Release reference.
|
|
b.in = nil
|
|
if b.remaining() > 0 {
|
|
return fmt.Errorf("corrupt input: %d bits remain on stream", b.remaining())
|
|
}
|
|
if b.bitsRead > 64 {
|
|
return io.ErrUnexpectedEOF
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// bitReaderShifted reads a bitstream in reverse.
|
|
// The last set bit indicates the start of the stream and is used
|
|
// for aligning the input.
|
|
type bitReaderShifted struct {
|
|
in []byte
|
|
off uint // next byte to read is at in[off - 1]
|
|
value uint64
|
|
bitsRead uint8
|
|
}
|
|
|
|
// init initializes and resets the bit reader.
|
|
func (b *bitReaderShifted) init(in []byte) error {
|
|
if len(in) < 1 {
|
|
return errors.New("corrupt stream: too short")
|
|
}
|
|
b.in = in
|
|
b.off = uint(len(in))
|
|
// The highest bit of the last byte indicates where to start
|
|
v := in[len(in)-1]
|
|
if v == 0 {
|
|
return errors.New("corrupt stream, did not find end of stream")
|
|
}
|
|
b.bitsRead = 64
|
|
b.value = 0
|
|
if len(in) >= 8 {
|
|
b.fillFastStart()
|
|
} else {
|
|
b.fill()
|
|
b.fill()
|
|
}
|
|
b.advance(8 - uint8(highBit32(uint32(v))))
|
|
return nil
|
|
}
|
|
|
|
// peekBitsFast requires that at least one bit is requested every time.
|
|
// There are no checks if the buffer is filled.
|
|
func (b *bitReaderShifted) peekBitsFast(n uint8) uint16 {
|
|
return uint16(b.value >> ((64 - n) & 63))
|
|
}
|
|
|
|
func (b *bitReaderShifted) advance(n uint8) {
|
|
b.bitsRead += n
|
|
b.value <<= n & 63
|
|
}
|
|
|
|
// fillFast() will make sure at least 32 bits are available.
|
|
// There must be at least 4 bytes available.
|
|
func (b *bitReaderShifted) fillFast() {
|
|
if b.bitsRead < 32 {
|
|
return
|
|
}
|
|
|
|
// 2 bounds checks.
|
|
v := b.in[b.off-4 : b.off]
|
|
v = v[:4]
|
|
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
|
b.value |= uint64(low) << ((b.bitsRead - 32) & 63)
|
|
b.bitsRead -= 32
|
|
b.off -= 4
|
|
}
|
|
|
|
// fillFastStart() assumes the bitReaderShifted is empty and there is at least 8 bytes to read.
|
|
func (b *bitReaderShifted) fillFastStart() {
|
|
// Do single re-slice to avoid bounds checks.
|
|
b.value = binary.LittleEndian.Uint64(b.in[b.off-8:])
|
|
b.bitsRead = 0
|
|
b.off -= 8
|
|
}
|
|
|
|
// fill() will make sure at least 32 bits are available.
|
|
func (b *bitReaderShifted) fill() {
|
|
if b.bitsRead < 32 {
|
|
return
|
|
}
|
|
if b.off > 4 {
|
|
v := b.in[b.off-4:]
|
|
v = v[:4]
|
|
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
|
b.value |= uint64(low) << ((b.bitsRead - 32) & 63)
|
|
b.bitsRead -= 32
|
|
b.off -= 4
|
|
return
|
|
}
|
|
for b.off > 0 {
|
|
b.value |= uint64(b.in[b.off-1]) << ((b.bitsRead - 8) & 63)
|
|
b.bitsRead -= 8
|
|
b.off--
|
|
}
|
|
}
|
|
|
|
func (b *bitReaderShifted) remaining() uint {
|
|
return b.off*8 + uint(64-b.bitsRead)
|
|
}
|
|
|
|
// close the bitstream and returns an error if out-of-buffer reads occurred.
|
|
func (b *bitReaderShifted) close() error {
|
|
// Release reference.
|
|
b.in = nil
|
|
if b.remaining() > 0 {
|
|
return fmt.Errorf("corrupt input: %d bits remain on stream", b.remaining())
|
|
}
|
|
if b.bitsRead > 64 {
|
|
return io.ErrUnexpectedEOF
|
|
}
|
|
return nil
|
|
}
|