From 64f479092dbacddf218f0c82cb3718046a171c78 Mon Sep 17 00:00:00 2001 From: Lukas Zapletal Date: Thu, 31 Oct 2024 15:31:16 +0100 Subject: [PATCH] osbuild-worker: use the new ostree resolver API --- cmd/osbuild-koji-tests/main_test.go | 4 +- cmd/osbuild-koji/main.go | 4 +- cmd/osbuild-worker/jobimpl-osbuild.go | 5 + cmd/osbuild-worker/jobimpl-ostree-resolve.go | 42 ++++++- cmd/osbuild-worker/main.go | 9 +- go.mod | 2 +- go.sum | 4 +- internal/cloudapi/v2/imagerequest.go | 7 +- internal/cloudapi/v2/server.go | 13 ++- internal/weldr/api.go | 1 + internal/weldr/upload.go | 7 +- .../osbuild/images/pkg/disk/disk.go | 70 ++++++++++++ .../github.com/osbuild/images/pkg/disk/lvm.go | 50 +++++---- .../images/pkg/disk/partition_table.go | 2 +- .../osbuild/images/pkg/distro/distro.go | 24 +--- .../images/pkg/distro/fedora/imagetype.go | 10 +- .../images/pkg/distro/rhel/imagetype.go | 10 +- .../images/pkg/distro/rhel/rhel8/azure.go | 1 - .../images/pkg/distro/test_distro/distro.go | 5 +- .../osbuild/images/pkg/image/bootc_disk.go | 12 +- .../osbuild/images/pkg/image/disk.go | 9 +- .../osbuild/images/pkg/image/gce.go | 24 ++++ .../osbuild/images/pkg/manifest/build.go | 4 +- .../pkg/osbuild/container_deploy_stage.go | 3 +- .../osbuild/images/pkg/ostree/ostree.go | 105 +++++++++++------- .../osbuild/images/pkg/platform/bootmode.go | 25 +++++ vendor/modules.txt | 2 +- 27 files changed, 318 insertions(+), 136 deletions(-) create mode 100644 vendor/github.com/osbuild/images/pkg/image/gce.go create mode 100644 vendor/github.com/osbuild/images/pkg/platform/bootmode.go diff --git a/cmd/osbuild-koji-tests/main_test.go b/cmd/osbuild-koji-tests/main_test.go index 16012b467..3fd0fed0b 100644 --- a/cmd/osbuild-koji-tests/main_test.go +++ b/cmd/osbuild-koji-tests/main_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/platform" "github.com/osbuild/images/pkg/rpmmd" "github.com/osbuild/osbuild-composer/internal/upload/koji" ) @@ -182,7 +182,7 @@ func TestKojiImport(t *testing.T) { Extra: &koji.BuildOutputExtra{ ImageOutput: koji.ImageExtraInfo{ Arch: "noarch", - BootMode: distro.BOOT_LEGACY.String(), + BootMode: platform.BOOT_LEGACY.String(), }, }, }, diff --git a/cmd/osbuild-koji/main.go b/cmd/osbuild-koji/main.go index 26dbc84e7..fe4dc01c5 100644 --- a/cmd/osbuild-koji/main.go +++ b/cmd/osbuild-koji/main.go @@ -8,7 +8,7 @@ import ( "time" "github.com/google/uuid" - "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/platform" "github.com/osbuild/images/pkg/rpmmd" "github.com/osbuild/osbuild-composer/internal/upload/koji" "github.com/sirupsen/logrus" @@ -101,7 +101,7 @@ func main() { Extra: &koji.BuildOutputExtra{ ImageOutput: koji.ImageExtraInfo{ Arch: arch, - BootMode: distro.BOOT_NONE.String(), // TODO: put the correct boot mode here + BootMode: platform.BOOT_NONE.String(), // TODO: put the correct boot mode here }, }, }, diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go index e73706b1f..c5e6db7ce 100644 --- a/cmd/osbuild-worker/jobimpl-osbuild.go +++ b/cmd/osbuild-worker/jobimpl-osbuild.go @@ -516,14 +516,19 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { } } + // Both curl and ostree input share the same MTLS config if impl.RepositoryMTLSConfig != nil { if impl.RepositoryMTLSConfig.CA != "" { extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_CURL_SSL_CA_CERT=%s", impl.RepositoryMTLSConfig.CA)) + extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_OSTREE_SSL_CA_CERT=%s", impl.RepositoryMTLSConfig.CA)) } extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_CURL_SSL_CLIENT_KEY=%s", impl.RepositoryMTLSConfig.MTLSClientKey)) extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_CURL_SSL_CLIENT_CERT=%s", impl.RepositoryMTLSConfig.MTLSClientCert)) + extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_OSTREE_SSL_CLIENT_KEY=%s", impl.RepositoryMTLSConfig.MTLSClientKey)) + extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_OSTREE_SSL_CLIENT_CERT=%s", impl.RepositoryMTLSConfig.MTLSClientCert)) if impl.RepositoryMTLSConfig.Proxy != nil { extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_CURL_PROXY=%s", impl.RepositoryMTLSConfig.Proxy.String())) + extraEnv = append(extraEnv, fmt.Sprintf("OSBUILD_SOURCES_OSTREE_PROXY=%s", impl.RepositoryMTLSConfig.Proxy.String())) } } diff --git a/cmd/osbuild-worker/jobimpl-ostree-resolve.go b/cmd/osbuild-worker/jobimpl-ostree-resolve.go index 775d217d2..3dd230105 100644 --- a/cmd/osbuild-worker/jobimpl-ostree-resolve.go +++ b/cmd/osbuild-worker/jobimpl-ostree-resolve.go @@ -2,6 +2,8 @@ package main import ( "fmt" + "net/url" + "strings" "github.com/sirupsen/logrus" @@ -11,6 +13,26 @@ import ( ) type OSTreeResolveJobImpl struct { + RepositoryMTLSConfig *RepositoryMTLSConfig +} + +func (job *OSTreeResolveJobImpl) CompareBaseURL(baseURLStr string) (bool, error) { + baseURL, err := url.Parse(baseURLStr) + if err != nil { + return false, err + } + + if baseURL.Scheme != job.RepositoryMTLSConfig.BaseURL.Scheme { + return false, nil + } + if baseURL.Host != job.RepositoryMTLSConfig.BaseURL.Host { + return false, nil + } + if !strings.HasPrefix(baseURL.Path, job.RepositoryMTLSConfig.BaseURL.Path) { + return false, nil + } + + return true, nil } func setError(err error, result *worker.OSTreeResolveJobResult) { @@ -51,7 +73,25 @@ func (impl *OSTreeResolveJobImpl) Run(job worker.Job) error { logWithId.Infof("Resolving (%d) ostree commits", len(args.Specs)) for i, s := range args.Specs { - reqParams := ostree.SourceSpec(s) + reqParams := ostree.SourceSpec{} + reqParams.URL = s.URL + reqParams.Ref = s.Ref + if match, err := impl.CompareBaseURL(s.URL); match && err == nil { + reqParams.Proxy = impl.RepositoryMTLSConfig.Proxy.String() + reqParams.MTLS = &ostree.MTLS{ + CA: impl.RepositoryMTLSConfig.CA, + ClientCert: impl.RepositoryMTLSConfig.MTLSClientCert, + ClientKey: impl.RepositoryMTLSConfig.MTLSClientKey, + } + } else if err != nil { + logWithId.Errorf("Error comparing base URL: %v", err) + result.JobError = clienterrors.New( + clienterrors.ErrorInvalidRepositoryURL, + "Repository URL is malformed", + err.Error(), + ) + break + } commitSpec, err := ostree.Resolve(reqParams) if err != nil { logWithId.Infof("Resolving ostree params failed: %v", err) diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go index d81c75460..dc4eb6958 100644 --- a/cmd/osbuild-worker/main.go +++ b/cmd/osbuild-worker/main.go @@ -7,13 +7,14 @@ import ( "errors" "flag" "fmt" - slogger "github.com/osbuild/osbuild-composer/pkg/splunk_logger" "net/url" "os" "path" "strings" "time" + slogger "github.com/osbuild/osbuild-composer/pkg/splunk_logger" + "github.com/BurntSushi/toml" "github.com/sirupsen/logrus" @@ -508,8 +509,10 @@ func main() { worker.JobTypeContainerResolve: &ContainerResolveJobImpl{ AuthFilePath: containersAuthFilePath, }, - worker.JobTypeOSTreeResolve: &OSTreeResolveJobImpl{}, - worker.JobTypeFileResolve: &FileResolveJobImpl{}, + worker.JobTypeOSTreeResolve: &OSTreeResolveJobImpl{ + RepositoryMTLSConfig: repositoryMTLSConfig, + }, + worker.JobTypeFileResolve: &FileResolveJobImpl{}, worker.JobTypeAWSEC2Copy: &AWSEC2CopyJobImpl{ AWSCreds: awsCredentials, }, diff --git a/go.mod b/go.mod index f13ff632b..4ac9aa6c7 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/labstack/gommon v0.4.2 github.com/openshift-online/ocm-sdk-go v0.1.438 github.com/oracle/oci-go-sdk/v54 v54.0.0 - github.com/osbuild/images v0.95.0 + github.com/osbuild/images v0.96.0 github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d github.com/osbuild/pulp-client v0.1.0 github.com/prometheus/client_golang v1.20.2 diff --git a/go.sum b/go.sum index b34567e15..115947df4 100644 --- a/go.sum +++ b/go.sum @@ -534,8 +534,8 @@ github.com/openshift-online/ocm-sdk-go v0.1.438 h1:tsLCCUzbLCTL4RZG02y9RuopmGCXp github.com/openshift-online/ocm-sdk-go v0.1.438/go.mod h1:CiAu2jwl3ITKOxkeV0Qnhzv4gs35AmpIzVABQLtcI2Y= github.com/oracle/oci-go-sdk/v54 v54.0.0 h1:CDLjeSejv2aDpElAJrhKpi6zvT/zhZCZuXchUUZ+LS4= github.com/oracle/oci-go-sdk/v54 v54.0.0/go.mod h1:+t+yvcFGVp+3ZnztnyxqXfQDsMlq8U25faBLa+mqCMc= -github.com/osbuild/images v0.95.0 h1:WWxYEQKD9wFGs/zkWF4wd3IDwNColZwzKsQh/+dwvUw= -github.com/osbuild/images v0.95.0/go.mod h1:4bNmMQOVadIKVC1q8zsLO8tdEQFH90zIp+MQBQUnCiE= +github.com/osbuild/images v0.96.0 h1:ZieK4i5pyKTdLaA/EwxeNEQsWBLEkX3FsZVyIaYCJKI= +github.com/osbuild/images v0.96.0/go.mod h1:4bNmMQOVadIKVC1q8zsLO8tdEQFH90zIp+MQBQUnCiE= github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d h1:r9BFPDv0uuA9k1947Jybcxs36c/pTywWS1gjeizvtcQ= github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d/go.mod h1:zR1iu/hOuf+OQNJlk70tju9IqzzM4ycq0ectkFBm94U= github.com/osbuild/pulp-client v0.1.0 h1:L0C4ezBJGTamN3BKdv+rKLuq/WxXJbsFwz/Hj7aEmJ8= diff --git a/internal/cloudapi/v2/imagerequest.go b/internal/cloudapi/v2/imagerequest.go index e666ff3ae..8f919e4b9 100644 --- a/internal/cloudapi/v2/imagerequest.go +++ b/internal/cloudapi/v2/imagerequest.go @@ -10,6 +10,7 @@ import ( "github.com/osbuild/images/pkg/disk" "github.com/osbuild/images/pkg/distro" "github.com/osbuild/images/pkg/ostree" + "github.com/osbuild/images/pkg/platform" "github.com/osbuild/osbuild-composer/internal/blueprint" "github.com/osbuild/osbuild-composer/internal/cloud/gcp" "github.com/osbuild/osbuild-composer/internal/common" @@ -56,11 +57,11 @@ func newAWSTarget(options UploadOptions, imageType distro.ImageType) (*target.Ta var amiBootMode *string switch imageType.BootMode() { - case distro.BOOT_HYBRID: + case platform.BOOT_HYBRID: amiBootMode = common.ToPtr(string(ec2types.BootModeValuesUefiPreferred)) - case distro.BOOT_UEFI: + case platform.BOOT_UEFI: amiBootMode = common.ToPtr(string(ec2types.BootModeValuesUefi)) - case distro.BOOT_LEGACY: + case platform.BOOT_LEGACY: amiBootMode = common.ToPtr(string(ec2types.BootModeValuesLegacyBios)) } diff --git a/internal/cloudapi/v2/server.go b/internal/cloudapi/v2/server.go index 741bea64f..2d702de8a 100644 --- a/internal/cloudapi/v2/server.go +++ b/internal/cloudapi/v2/server.go @@ -228,7 +228,12 @@ func (s *Server) enqueueCompose(irs []imageRequest, channel string) (uuid.UUID, workerResolveSpecs := make([]worker.OSTreeResolveSpec, len(sources)) for idx, source := range sources { // ostree.SourceSpec is directly convertible to worker.OSTreeResolveSpec - workerResolveSpecs[idx] = worker.OSTreeResolveSpec(source) + workerResolveSpecs[idx] = worker.OSTreeResolveSpec{ + URL: source.URL, + Ref: source.Ref, + RHSM: source.RHSM, + } + } jobID, err := s.workers.EnqueueOSTreeResolveJob(&worker.OSTreeResolveJob{Specs: workerResolveSpecs}, channel) if err != nil { @@ -356,7 +361,11 @@ func (s *Server) enqueueKojiCompose(taskID uint64, server, name, version, releas workerResolveSpecs := make([]worker.OSTreeResolveSpec, len(sources)) for idx, source := range sources { // ostree.SourceSpec is directly convertible to worker.OSTreeResolveSpec - workerResolveSpecs[idx] = worker.OSTreeResolveSpec(source) + workerResolveSpecs[idx] = worker.OSTreeResolveSpec{ + URL: source.URL, + Ref: source.Ref, + RHSM: source.RHSM, + } } jobID, err := s.workers.EnqueueOSTreeResolveJob(&worker.OSTreeResolveJob{Specs: workerResolveSpecs}, channel) if err != nil { diff --git a/internal/weldr/api.go b/internal/weldr/api.go index 7f9f4c5ec..b4f766918 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -2393,6 +2393,7 @@ func (api *API) resolveOSTreeCommits(sourceSpecs map[string][]ostree.SourceSpec, Checksum: checksum, } } else { + // MTLS not supported on-prem commit, err := ostree.Resolve(source) if err != nil { return nil, err diff --git a/internal/weldr/upload.go b/internal/weldr/upload.go index c67116a8d..847ae5857 100644 --- a/internal/weldr/upload.go +++ b/internal/weldr/upload.go @@ -10,6 +10,7 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/platform" "github.com/osbuild/osbuild-composer/internal/cloud/gcp" "github.com/osbuild/osbuild-composer/internal/common" "github.com/sirupsen/logrus" @@ -283,11 +284,11 @@ func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target. var amiBootMode *string switch imageType.BootMode() { - case distro.BOOT_HYBRID: + case platform.BOOT_HYBRID: amiBootMode = common.ToPtr(string(ec2types.BootModeValuesUefiPreferred)) - case distro.BOOT_UEFI: + case platform.BOOT_UEFI: amiBootMode = common.ToPtr(string(ec2types.BootModeValuesUefi)) - case distro.BOOT_LEGACY: + case platform.BOOT_LEGACY: amiBootMode = common.ToPtr(string(ec2types.BootModeValuesLegacyBios)) } diff --git a/vendor/github.com/osbuild/images/pkg/disk/disk.go b/vendor/github.com/osbuild/images/pkg/disk/disk.go index 6aeab82d4..e92a8fee6 100644 --- a/vendor/github.com/osbuild/images/pkg/disk/disk.go +++ b/vendor/github.com/osbuild/images/pkg/disk/disk.go @@ -19,6 +19,7 @@ package disk import ( "encoding/hex" + "fmt" "io" "math/rand" "reflect" @@ -61,6 +62,54 @@ const ( DosFat16B = "06" ) +// FSType is the filesystem type enum. +// +// There should always be one value for each filesystem type supported by +// osbuild stages (stages/org.osbuild.mkfs.*) and the unset/none value. +type FSType uint64 + +const ( + FS_NONE FSType = iota + FS_VFAT + FS_EXT4 + FS_XFS + FS_BTRFS +) + +func (f FSType) String() string { + switch f { + case FS_NONE: + return "" + case FS_VFAT: + return "vfat" + case FS_EXT4: + return "ext4" + case FS_XFS: + return "xfs" + case FS_BTRFS: + return "btrfs" + default: + panic(fmt.Sprintf("unknown or unsupported filesystem type with enum value %d", f)) + } +} + +func NewFSType(s string) (FSType, error) { + switch s { + case "": + return FS_NONE, nil + case "vfat": + return FS_VFAT, nil + case "ext4": + return FS_EXT4, nil + case "xfs": + return FS_XFS, nil + case "btrfs": + return FS_BTRFS, nil + default: + return FS_NONE, fmt.Errorf("unknown or unsupported filesystem type name: %s", s) + } +} + // Entity is the base interface for all disk-related entities. type Entity interface { // Clone returns a deep copy of the entity. @@ -205,3 +254,24 @@ func NewVolIDFromRand(r *rand.Rand) string { } return hex.EncodeToString(volid) } + +// genUniqueString returns a string based on base that does does not exist in +// the existing set. If the base itself does not exist, it is returned as is, +// otherwise a two digit number is added and incremented until a unique string +// is found. +// This function is mimicking what blivet does for avoiding name collisions. +// See blivet/blivet.py#L1060 commit 2eb4bd4 +func genUniqueString(base string, existing map[string]bool) (string, error) { + if !existing[base] { + return base, nil + } + + for i := 0; i < 100; i++ { + uniq := fmt.Sprintf("%s%02d", base, i) + if !existing[uniq] { + return uniq, nil + } + } + + return "", fmt.Errorf("name collision: could not generate unique version of %q", base) +} diff --git a/vendor/github.com/osbuild/images/pkg/disk/lvm.go b/vendor/github.com/osbuild/images/pkg/disk/lvm.go index a6fb546b2..54fc2523d 100644 --- a/vendor/github.com/osbuild/images/pkg/disk/lvm.go +++ b/vendor/github.com/osbuild/images/pkg/disk/lvm.go @@ -81,41 +81,49 @@ func (vg *LVMVolumeGroup) CreateMountpoint(mountpoint string, size uint64) (Enti FSTabPassNo: 0, } - return vg.CreateLogicalVolume(mountpoint, size, &filesystem) + // leave lv name empty to autogenerate based on mountpoint + return vg.CreateLogicalVolume("", size, &filesystem) } -func (vg *LVMVolumeGroup) CreateLogicalVolume(lvName string, size uint64, payload Entity) (Entity, error) { - if vg == nil { - panic("LVMVolumeGroup.CreateLogicalVolume: nil entity") - } - +// genLVName generates a valid logical volume name from a mountpoint or base +// that does not conflict with existing ones. +func (vg *LVMVolumeGroup) genLVName(base string) (string, error) { names := make(map[string]bool, len(vg.LogicalVolumes)) for _, lv := range vg.LogicalVolumes { names[lv.Name] = true } - base := lvname(lvName) - var exists bool - name := base + base = lvname(base) // if the mountpoint is used (i.e. if the base contains /), sanitize it and append 'lv' - // Make sure that we don't collide with an existing volume, e.g. 'home/test' - // and /home/test_test would collide. We try 100 times and then give up. This - // is mimicking what blivet does. See blivet/blivet.py#L1060 commit 2eb4bd4 - for i := 0; i < 100; i++ { - exists = names[name] - if !exists { - break - } + // Make sure that we don't collide with an existing volume, e.g. + // 'home/test' and /home_test would collide. + return genUniqueString(base, names) +} - name = fmt.Sprintf("%s%02d", base, i) +// CreateLogicalVolume creates a new logical volume on the volume group. If a +// name is not provided, a valid one is generated based on the payload +// mountpoint. If a name is provided, it is used directly without validating. +func (vg *LVMVolumeGroup) CreateLogicalVolume(lvName string, size uint64, payload Entity) (*LVMLogicalVolume, error) { + if vg == nil { + panic("LVMVolumeGroup.CreateLogicalVolume: nil entity") } - if exists { - return nil, fmt.Errorf("could not create logical volume: name collision") + if lvName == "" { + // generate a name based on the payload's mountpoint + mntble, ok := payload.(Mountable) + if !ok { + return nil, fmt.Errorf("could not create logical volume: no name provided and payload is not mountable") + } + mountpoint := mntble.GetMountpoint() + autoName, err := vg.genLVName(mountpoint) + if err != nil { + return nil, err + } + lvName = autoName } lv := LVMLogicalVolume{ - Name: name, + Name: lvName, Size: vg.AlignUp(size), Payload: payload, } diff --git a/vendor/github.com/osbuild/images/pkg/disk/partition_table.go b/vendor/github.com/osbuild/images/pkg/disk/partition_table.go index 41d9c1195..407074edb 100644 --- a/vendor/github.com/osbuild/images/pkg/disk/partition_table.go +++ b/vendor/github.com/osbuild/images/pkg/disk/partition_table.go @@ -715,7 +715,7 @@ func (pt *PartitionTable) ensureLVM() error { // create root logical volume on the new volume group with the same // size and filesystem as the previous root partition - _, err := vg.CreateLogicalVolume("root", part.Size, filesystem) + _, err := vg.CreateLogicalVolume("rootlv", part.Size, filesystem) if err != nil { panic(fmt.Sprintf("Could not create LV: %v", err)) } diff --git a/vendor/github.com/osbuild/images/pkg/distro/distro.go b/vendor/github.com/osbuild/images/pkg/distro/distro.go index a4f8bfd8f..4b18b81b7 100644 --- a/vendor/github.com/osbuild/images/pkg/distro/distro.go +++ b/vendor/github.com/osbuild/images/pkg/distro/distro.go @@ -6,36 +6,16 @@ import ( "github.com/osbuild/images/pkg/disk" "github.com/osbuild/images/pkg/manifest" "github.com/osbuild/images/pkg/ostree" + "github.com/osbuild/images/pkg/platform" "github.com/osbuild/images/pkg/rhsm/facts" "github.com/osbuild/images/pkg/rpmmd" ) -type BootMode uint64 - const ( - BOOT_NONE BootMode = iota - BOOT_LEGACY - BOOT_UEFI - BOOT_HYBRID UnsupportedCustomizationError = "unsupported blueprint customizations found for image type %q: (allowed: %s)" NoCustomizationsAllowedError = "image type %q does not support customizations" ) -func (m BootMode) String() string { - switch m { - case BOOT_NONE: - return "none" - case BOOT_LEGACY: - return "legacy" - case BOOT_UEFI: - return "uefi" - case BOOT_HYBRID: - return "hybrid" - default: - panic("invalid boot mode") - } -} - // A Distro represents composer's notion of what a given distribution is. type Distro interface { // Returns the name of the distro. @@ -121,7 +101,7 @@ type ImageType interface { PartitionType() string // Returns the corresponding boot mode ("legacy", "uefi", "hybrid") or "none" - BootMode() BootMode + BootMode() platform.BootMode // Returns the names of the pipelines that set up the build environment (buildroot). BuildPipelines() []string diff --git a/vendor/github.com/osbuild/images/pkg/distro/fedora/imagetype.go b/vendor/github.com/osbuild/images/pkg/distro/fedora/imagetype.go index 076715399..a8a20162e 100644 --- a/vendor/github.com/osbuild/images/pkg/distro/fedora/imagetype.go +++ b/vendor/github.com/osbuild/images/pkg/distro/fedora/imagetype.go @@ -126,15 +126,15 @@ func (t *imageType) Exports() []string { return []string{"assembler"} } -func (t *imageType) BootMode() distro.BootMode { +func (t *imageType) BootMode() platform.BootMode { if t.platform.GetUEFIVendor() != "" && t.platform.GetBIOSPlatform() != "" { - return distro.BOOT_HYBRID + return platform.BOOT_HYBRID } else if t.platform.GetUEFIVendor() != "" { - return distro.BOOT_UEFI + return platform.BOOT_UEFI } else if t.platform.GetBIOSPlatform() != "" || t.platform.GetZiplSupport() { - return distro.BOOT_LEGACY + return platform.BOOT_LEGACY } - return distro.BOOT_NONE + return platform.BOOT_NONE } func (t *imageType) getPartitionTable( diff --git a/vendor/github.com/osbuild/images/pkg/distro/rhel/imagetype.go b/vendor/github.com/osbuild/images/pkg/distro/rhel/imagetype.go index 7a0d92679..35168f6d3 100644 --- a/vendor/github.com/osbuild/images/pkg/distro/rhel/imagetype.go +++ b/vendor/github.com/osbuild/images/pkg/distro/rhel/imagetype.go @@ -168,15 +168,15 @@ func (t *ImageType) Exports() []string { return []string{"assembler"} } -func (t *ImageType) BootMode() distro.BootMode { +func (t *ImageType) BootMode() platform.BootMode { if t.platform.GetUEFIVendor() != "" && t.platform.GetBIOSPlatform() != "" { - return distro.BOOT_HYBRID + return platform.BOOT_HYBRID } else if t.platform.GetUEFIVendor() != "" { - return distro.BOOT_UEFI + return platform.BOOT_UEFI } else if t.platform.GetBIOSPlatform() != "" || t.platform.GetZiplSupport() { - return distro.BOOT_LEGACY + return platform.BOOT_LEGACY } - return distro.BOOT_NONE + return platform.BOOT_NONE } func (t *ImageType) GetPartitionTable( diff --git a/vendor/github.com/osbuild/images/pkg/distro/rhel/rhel8/azure.go b/vendor/github.com/osbuild/images/pkg/distro/rhel/rhel8/azure.go index e65ace67c..589b70258 100644 --- a/vendor/github.com/osbuild/images/pkg/distro/rhel/rhel8/azure.go +++ b/vendor/github.com/osbuild/images/pkg/distro/rhel/rhel8/azure.go @@ -529,7 +529,6 @@ var defaultAzureImageConfig = &distro.ImageConfig{ "nm-cloud-setup.service", "nm-cloud-setup.timer", "sshd", - "systemd-resolved", "waagent", }, SshdConfig: &osbuild.SshdConfigStageOptions{ diff --git a/vendor/github.com/osbuild/images/pkg/distro/test_distro/distro.go b/vendor/github.com/osbuild/images/pkg/distro/test_distro/distro.go index 90bcf7531..4c6c193df 100644 --- a/vendor/github.com/osbuild/images/pkg/distro/test_distro/distro.go +++ b/vendor/github.com/osbuild/images/pkg/distro/test_distro/distro.go @@ -9,6 +9,7 @@ import ( "github.com/osbuild/images/pkg/distro" "github.com/osbuild/images/pkg/manifest" "github.com/osbuild/images/pkg/ostree" + "github.com/osbuild/images/pkg/platform" "github.com/osbuild/images/pkg/policies" "github.com/osbuild/images/pkg/rpmmd" ) @@ -210,8 +211,8 @@ func (t *TestImageType) PartitionType() string { return "" } -func (t *TestImageType) BootMode() distro.BootMode { - return distro.BOOT_HYBRID +func (t *TestImageType) BootMode() platform.BootMode { + return platform.BOOT_HYBRID } func (t *TestImageType) BuildPipelines() []string { diff --git a/vendor/github.com/osbuild/images/pkg/image/bootc_disk.go b/vendor/github.com/osbuild/images/pkg/image/bootc_disk.go index 5e5cd6bfb..b5e4624f2 100644 --- a/vendor/github.com/osbuild/images/pkg/image/bootc_disk.go +++ b/vendor/github.com/osbuild/images/pkg/image/bootc_disk.go @@ -3,9 +3,7 @@ package image import ( "fmt" "math/rand" - "regexp" - "github.com/osbuild/images/internal/common" "github.com/osbuild/images/pkg/container" "github.com/osbuild/images/pkg/customizations/users" "github.com/osbuild/images/pkg/disk" @@ -92,15 +90,7 @@ func (img *BootcDiskImage) InstantiateManifestFromContainers(m *manifest.Manifes fmt.Sprintf("%s.vhd", fileBasename), } - // XXX: copied from https://github.com/osbuild/images/blob/v0.85.0/pkg/image/disk.go#L102 - gcePipeline := manifest.NewTar(buildPipeline, rawImage, "gce") - gcePipeline.Format = osbuild.TarArchiveFormatOldgnu - gcePipeline.RootNode = osbuild.TarRootNodeOmit - // these are required to successfully import the image to GCP - gcePipeline.ACLs = common.ToPtr(false) - gcePipeline.SELinux = common.ToPtr(false) - gcePipeline.Xattrs = common.ToPtr(false) - gcePipeline.Transform = fmt.Sprintf(`s/%s/disk.raw/`, regexp.QuoteMeta(rawImage.Filename())) + gcePipeline := newGCETarPipelineForImg(buildPipeline, rawImage, "gce") gcePipeline.SetFilename("image.tar.gz") return nil diff --git a/vendor/github.com/osbuild/images/pkg/image/disk.go b/vendor/github.com/osbuild/images/pkg/image/disk.go index 3d7d34c5c..49a22c24b 100644 --- a/vendor/github.com/osbuild/images/pkg/image/disk.go +++ b/vendor/github.com/osbuild/images/pkg/image/disk.go @@ -6,7 +6,6 @@ import ( "path/filepath" "strings" - "github.com/osbuild/images/internal/common" "github.com/osbuild/images/internal/environment" "github.com/osbuild/images/internal/workload" "github.com/osbuild/images/pkg/artifact" @@ -103,13 +102,7 @@ func (img *DiskImage) InstantiateManifest(m *manifest.Manifest, // NOTE(akoutsou): temporary workaround; filename required for GCP // TODO: define internal raw filename on image type rawImagePipeline.SetFilename("disk.raw") - tarPipeline := manifest.NewTar(buildPipeline, rawImagePipeline, "archive") - tarPipeline.Format = osbuild.TarArchiveFormatOldgnu - tarPipeline.RootNode = osbuild.TarRootNodeOmit - // these are required to successfully import the image to GCP - tarPipeline.ACLs = common.ToPtr(false) - tarPipeline.SELinux = common.ToPtr(false) - tarPipeline.Xattrs = common.ToPtr(false) + tarPipeline := newGCETarPipelineForImg(buildPipeline, rawImagePipeline, "archive") tarPipeline.SetFilename(img.Filename) // filename extension will determine compression imagePipeline = tarPipeline default: diff --git a/vendor/github.com/osbuild/images/pkg/image/gce.go b/vendor/github.com/osbuild/images/pkg/image/gce.go new file mode 100644 index 000000000..b41b59872 --- /dev/null +++ b/vendor/github.com/osbuild/images/pkg/image/gce.go @@ -0,0 +1,24 @@ +package image + +import ( + "fmt" + "regexp" + + "github.com/osbuild/images/internal/common" + "github.com/osbuild/images/pkg/manifest" + "github.com/osbuild/images/pkg/osbuild" +) + +func newGCETarPipelineForImg(buildPipeline manifest.Build, inputPipeline manifest.FilePipeline, pipelinename string) *manifest.Tar { + tarPipeline := manifest.NewTar(buildPipeline, inputPipeline, pipelinename) + tarPipeline.Format = osbuild.TarArchiveFormatOldgnu + tarPipeline.RootNode = osbuild.TarRootNodeOmit + // these are required to successfully import the image to GCP + tarPipeline.ACLs = common.ToPtr(false) + tarPipeline.SELinux = common.ToPtr(false) + tarPipeline.Xattrs = common.ToPtr(false) + if inputPipeline.Filename() != "disk.raw" { + tarPipeline.Transform = fmt.Sprintf(`s/%s/disk.raw/`, regexp.QuoteMeta(inputPipeline.Filename())) + } + return tarPipeline +} diff --git a/vendor/github.com/osbuild/images/pkg/manifest/build.go b/vendor/github.com/osbuild/images/pkg/manifest/build.go index eca633d44..715771ba8 100644 --- a/vendor/github.com/osbuild/images/pkg/manifest/build.go +++ b/vendor/github.com/osbuild/images/pkg/manifest/build.go @@ -236,7 +236,9 @@ func (p *BuildrootFromContainer) serialize() osbuild.Pipeline { pipeline.Runner = p.runner.String() image := osbuild.NewContainersInputForSingleSource(p.containerSpecs[0]) - stage, err := osbuild.NewContainerDeployStage(image, &osbuild.ContainerDeployOptions{}) + // Make skopeo copy to remove the signatures of signed containers by default to workaround + // build failures until https://github.com/containers/image/issues/2599 is implemented + stage, err := osbuild.NewContainerDeployStage(image, &osbuild.ContainerDeployOptions{RemoveSignatures: true}) if err != nil { panic(err) } diff --git a/vendor/github.com/osbuild/images/pkg/osbuild/container_deploy_stage.go b/vendor/github.com/osbuild/images/pkg/osbuild/container_deploy_stage.go index be7606b2f..5d4d1dc1b 100644 --- a/vendor/github.com/osbuild/images/pkg/osbuild/container_deploy_stage.go +++ b/vendor/github.com/osbuild/images/pkg/osbuild/container_deploy_stage.go @@ -9,7 +9,8 @@ type ContainerDeployInputs struct { func (ContainerDeployInputs) isStageInputs() {} type ContainerDeployOptions struct { - Exclude []string `json:"exclude,omitempty"` + Exclude []string `json:"exclude,omitempty"` + RemoveSignatures bool `json:"remove-signatures,omitempty"` } func (ContainerDeployOptions) isStageOptions() {} diff --git a/vendor/github.com/osbuild/images/pkg/ostree/ostree.go b/vendor/github.com/osbuild/images/pkg/ostree/ostree.go index 8898f3370..1be6e9c28 100644 --- a/vendor/github.com/osbuild/images/pkg/ostree/ostree.go +++ b/vendor/github.com/osbuild/images/pkg/ostree/ostree.go @@ -25,9 +25,21 @@ var ( // SourceSpec serves as input for ResolveParams, and contains all necessary // variables to resolve a ref, which can then be turned into a CommitSpec. type SourceSpec struct { - URL string - Ref string + URL string + Ref string + // RHSM indicates to use RHSM secrets when pulling from the remote. Alternatively, you can use MTLS with plain certs. RHSM bool + // MTLS information. Will be ignored if RHSM is set. + MTLS *MTLS + // Proxy as HTTP proxy to use when fetching the ref. + Proxy string +} + +// MTLS contains the options for resolving an ostree source. +type MTLS struct { + CA string + ClientCert string + ClientKey string } // CommitSpec specifies an ostree commit using any combination of Ref (branch), URL (source), and Checksum (commit ID). @@ -138,59 +150,53 @@ func verifyChecksum(commit string) bool { return len(commit) > 0 && ostreeCommitRE.MatchString(commit) } -// ResolveRef resolves the URL path specified by the location and ref +// resolveRef resolves the URL path specified by the location and ref // (location+"refs/heads/"+ref) and returns the commit ID for the named ref. If // there is an error, it will be of type ResolveRefError. -func ResolveRef(location, ref string, consumerCerts bool, subs *rhsm.Subscriptions, ca *string) (string, error) { - u, err := url.Parse(location) +func resolveRef(ss SourceSpec) (string, error) { + u, err := url.Parse(ss.URL) if err != nil { return "", NewResolveRefError("error parsing ostree repository location: %v", err) } - u.Path = path.Join(u.Path, "refs/heads/", ref) - - var client *http.Client - if consumerCerts { - if subs == nil { - subs, err = rhsm.LoadSystemSubscriptions() - if err != nil { - return "", NewResolveRefError("error adding rhsm certificates when resolving ref: %s", err) - } - if subs.Consumer == nil { - return "", NewResolveRefError("error adding rhsm certificates when resolving ref") - } - } + u.Path = path.Join(u.Path, "refs/heads/", ss.Ref) + transport := http.DefaultTransport.(*http.Transport).Clone() + client := &http.Client{ + Transport: transport, + Timeout: 300 * time.Second, + } + if u.Scheme == "https" { tlsConf := &tls.Config{ MinVersion: tls.VersionTLS12, } - if ca != nil { - caCertPEM, err := os.ReadFile(*ca) + // If CA is set, load the CA certificate and add it to the TLS configuration. Otherwise, use the system CA. + if ss.MTLS.CA != "" { + caCertPEM, err := os.ReadFile(ss.MTLS.CA) if err != nil { - return "", NewResolveRefError("error adding rhsm certificates when resolving ref: %s", err) + return "", NewResolveRefError("error adding ca certificate when resolving ref: %s", err) } - roots := x509.NewCertPool() - ok := roots.AppendCertsFromPEM(caCertPEM) - if !ok { - return "", NewResolveRefError("error adding rhsm certificates when resolving ref") + tlsConf.RootCAs = x509.NewCertPool() + if ok := tlsConf.RootCAs.AppendCertsFromPEM(caCertPEM); !ok { + return "", NewResolveRefError("error adding ca certificate when resolving ref") } - tlsConf.RootCAs = roots } - cert, err := tls.LoadX509KeyPair(subs.Consumer.ConsumerCert, subs.Consumer.ConsumerKey) - if err != nil { - return "", NewResolveRefError("error adding rhsm certificates when resolving ref: %s", err) + if ss.MTLS.ClientCert != "" && ss.MTLS.ClientKey != "" { + cert, err := tls.LoadX509KeyPair(ss.MTLS.ClientCert, ss.MTLS.ClientKey) + if err != nil { + return "", NewResolveRefError("error adding client certificate when resolving ref: %s", err) + } + tlsConf.Certificates = []tls.Certificate{cert} } - tlsConf.Certificates = []tls.Certificate{cert} - client = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: tlsConf, - }, - Timeout: 300 * time.Second, + transport.TLSClientConfig = tlsConf + } + + if ss.Proxy != "" { + transport.Proxy = func(request *http.Request) (*url.URL, error) { + return url.Parse(ss.Proxy) } - } else { - client = &http.Client{} } req, err := http.NewRequest(http.MethodGet, u.String(), nil) @@ -234,8 +240,31 @@ func Resolve(source SourceSpec) (CommitSpec, error) { URL: source.URL, } + if source.RHSM && source.MTLS != nil { + return commit, NewResolveRefError("cannot use both RHSM and MTLS when resolving ref") + } + if source.RHSM { + var subs *rhsm.Subscriptions + var err error + commit.Secrets = "org.osbuild.rhsm.consumer" + subs, err = rhsm.LoadSystemSubscriptions() + + if err != nil { + return commit, NewResolveRefError("error adding rhsm certificates when resolving ref: %s", err) + } + + if subs.Consumer == nil { + return commit, NewResolveRefError("error adding rhsm certificates when resolving ref") + } + + source.MTLS = &MTLS{ + ClientCert: subs.Consumer.ConsumerCert, + ClientKey: subs.Consumer.ConsumerKey, + } + } else if source.MTLS != nil { + commit.Secrets = "org.osbuild.mtls" } if verifyChecksum(source.Ref) { @@ -252,7 +281,7 @@ func Resolve(source SourceSpec) (CommitSpec, error) { // URL set: Resolve checksum if source.URL != "" { // If a URL is specified, we need to fetch the commit at the URL. - checksum, err := ResolveRef(source.URL, source.Ref, source.RHSM, nil, nil) + checksum, err := resolveRef(source) if err != nil { return CommitSpec{}, err // ResolveRefError } diff --git a/vendor/github.com/osbuild/images/pkg/platform/bootmode.go b/vendor/github.com/osbuild/images/pkg/platform/bootmode.go new file mode 100644 index 000000000..a6d85245a --- /dev/null +++ b/vendor/github.com/osbuild/images/pkg/platform/bootmode.go @@ -0,0 +1,25 @@ +package platform + +type BootMode uint64 + +const ( + BOOT_NONE BootMode = iota + BOOT_LEGACY + BOOT_UEFI + BOOT_HYBRID +) + +func (m BootMode) String() string { + switch m { + case BOOT_NONE: + return "none" + case BOOT_LEGACY: + return "legacy" + case BOOT_UEFI: + return "uefi" + case BOOT_HYBRID: + return "hybrid" + default: + panic("invalid boot mode") + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index e65d18b11..e6a41b6af 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1022,7 +1022,7 @@ github.com/oracle/oci-go-sdk/v54/identity github.com/oracle/oci-go-sdk/v54/objectstorage github.com/oracle/oci-go-sdk/v54/objectstorage/transfer github.com/oracle/oci-go-sdk/v54/workrequests -# github.com/osbuild/images v0.95.0 +# github.com/osbuild/images v0.96.0 ## explicit; go 1.21.0 github.com/osbuild/images/internal/common github.com/osbuild/images/internal/environment