osbuild-worker: use the new ostree resolver API

vendor/github.com/osbuild/images/pkg/disk/disk.go

@@ -19,6 +19,7 @@ package disk
 
 import (
 	"encoding/hex"
+	"fmt"
 	"io"
 	"math/rand"
 	"reflect"
@@ -61,6 +62,54 @@ const (
 	DosFat16B = "06"
 )
 
+// FSType is the filesystem type enum.
+//
+// There should always be one value for each filesystem type supported by
+// osbuild stages (stages/org.osbuild.mkfs.*) and the unset/none value.
+type FSType uint64
+
+const (
+	FS_NONE FSType = iota
+	FS_VFAT
+	FS_EXT4
+	FS_XFS
+	FS_BTRFS
+)
+
+func (f FSType) String() string {
+	switch f {
+	case FS_NONE:
+		return ""
+	case FS_VFAT:
+		return "vfat"
+	case FS_EXT4:
+		return "ext4"
+	case FS_XFS:
+		return "xfs"
+	case FS_BTRFS:
+		return "btrfs"
+	default:
+		panic(fmt.Sprintf("unknown or unsupported filesystem type with enum value %d", f))
+	}
+}
+
+func NewFSType(s string) (FSType, error) {
+	switch s {
+	case "":
+		return FS_NONE, nil
+	case "vfat":
+		return FS_VFAT, nil
+	case "ext4":
+		return FS_EXT4, nil
+	case "xfs":
+		return FS_XFS, nil
+	case "btrfs":
+		return FS_BTRFS, nil
+	default:
+		return FS_NONE, fmt.Errorf("unknown or unsupported filesystem type name: %s", s)
+	}
+}
+
 // Entity is the base interface for all disk-related entities.
 type Entity interface {
 	// Clone returns a deep copy of the entity.
@@ -205,3 +254,24 @@ func NewVolIDFromRand(r *rand.Rand) string {
 	}
 	return hex.EncodeToString(volid)
 }
+
+// genUniqueString returns a string based on base that does does not exist in
+// the existing set. If the base itself does not exist, it is returned as is,
+// otherwise a two digit number is added and incremented until a unique string
+// is found.
+// This function is mimicking what blivet does for avoiding name collisions.
+// See blivet/blivet.py#L1060 commit 2eb4bd4
+func genUniqueString(base string, existing map[string]bool) (string, error) {
+	if !existing[base] {
+		return base, nil
+	}
+
+	for i := 0; i < 100; i++ {
+		uniq := fmt.Sprintf("%s%02d", base, i)
+		if !existing[uniq] {
+			return uniq, nil
+		}
+	}
+
+	return "", fmt.Errorf("name collision: could not generate unique version of %q", base)
+}

vendor/github.com/osbuild/images/pkg/disk/lvm.go

@@ -81,41 +81,49 @@ func (vg *LVMVolumeGroup) CreateMountpoint(mountpoint string, size uint64) (Enti
 		FSTabPassNo:  0,
 	}
 
-	return vg.CreateLogicalVolume(mountpoint, size, &filesystem)
+	// leave lv name empty to autogenerate based on mountpoint
+	return vg.CreateLogicalVolume("", size, &filesystem)
 }
 
-func (vg *LVMVolumeGroup) CreateLogicalVolume(lvName string, size uint64, payload Entity) (Entity, error) {
-	if vg == nil {
-		panic("LVMVolumeGroup.CreateLogicalVolume: nil entity")
-	}
-
+// genLVName generates a valid logical volume name from a mountpoint or base
+// that does not conflict with existing ones.
+func (vg *LVMVolumeGroup) genLVName(base string) (string, error) {
 	names := make(map[string]bool, len(vg.LogicalVolumes))
 	for _, lv := range vg.LogicalVolumes {
 		names[lv.Name] = true
 	}
 
-	base := lvname(lvName)
-	var exists bool
-	name := base
+	base = lvname(base) // if the mountpoint is used (i.e. if the base contains /), sanitize it and append 'lv'
 
-	// Make sure that we don't collide with an existing volume, e.g. 'home/test'
-	// and /home/test_test would collide. We try 100 times and then give up. This
-	// is mimicking what blivet does. See blivet/blivet.py#L1060 commit 2eb4bd4
-	for i := 0; i < 100; i++ {
-		exists = names[name]
-		if !exists {
-			break
-		}
+	// Make sure that we don't collide with an existing volume, e.g.
+	// 'home/test' and /home_test would collide.
+	return genUniqueString(base, names)
+}
 
-		name = fmt.Sprintf("%s%02d", base, i)
+// CreateLogicalVolume creates a new logical volume on the volume group. If a
+// name is not provided, a valid one is generated based on the payload
+// mountpoint. If a name is provided, it is used directly without validating.
+func (vg *LVMVolumeGroup) CreateLogicalVolume(lvName string, size uint64, payload Entity) (*LVMLogicalVolume, error) {
+	if vg == nil {
+		panic("LVMVolumeGroup.CreateLogicalVolume: nil entity")
 	}
 
-	if exists {
-		return nil, fmt.Errorf("could not create logical volume: name collision")
+	if lvName == "" {
+		// generate a name based on the payload's mountpoint
+		mntble, ok := payload.(Mountable)
+		if !ok {
+			return nil, fmt.Errorf("could not create logical volume: no name provided and payload is not mountable")
+		}
+		mountpoint := mntble.GetMountpoint()
+		autoName, err := vg.genLVName(mountpoint)
+		if err != nil {
+			return nil, err
+		}
+		lvName = autoName
 	}
 
 	lv := LVMLogicalVolume{
-		Name:    name,
+		Name:    lvName,
 		Size:    vg.AlignUp(size),
 		Payload: payload,
 	}

vendor/github.com/osbuild/images/pkg/disk/partition_table.go

@@ -715,7 +715,7 @@ func (pt *PartitionTable) ensureLVM() error {
 
 		// create root logical volume on the new volume group with the same
 		// size and filesystem as the previous root partition
-		_, err := vg.CreateLogicalVolume("root", part.Size, filesystem)
+		_, err := vg.CreateLogicalVolume("rootlv", part.Size, filesystem)
 		if err != nil {
 			panic(fmt.Sprintf("Could not create LV: %v", err))
 		}

vendor/github.com/osbuild/images/pkg/distro/distro.go

@@ -6,36 +6,16 @@ import (
 	"github.com/osbuild/images/pkg/disk"
 	"github.com/osbuild/images/pkg/manifest"
 	"github.com/osbuild/images/pkg/ostree"
+	"github.com/osbuild/images/pkg/platform"
 	"github.com/osbuild/images/pkg/rhsm/facts"
 	"github.com/osbuild/images/pkg/rpmmd"
 )
 
-type BootMode uint64
-
 const (
-	BOOT_NONE BootMode = iota
-	BOOT_LEGACY
-	BOOT_UEFI
-	BOOT_HYBRID
 	UnsupportedCustomizationError = "unsupported blueprint customizations found for image type %q: (allowed: %s)"
 	NoCustomizationsAllowedError  = "image type %q does not support customizations"
 )
 
-func (m BootMode) String() string {
-	switch m {
-	case BOOT_NONE:
-		return "none"
-	case BOOT_LEGACY:
-		return "legacy"
-	case BOOT_UEFI:
-		return "uefi"
-	case BOOT_HYBRID:
-		return "hybrid"
-	default:
-		panic("invalid boot mode")
-	}
-}
-
 // A Distro represents composer's notion of what a given distribution is.
 type Distro interface {
 	// Returns the name of the distro.
@@ -121,7 +101,7 @@ type ImageType interface {
 	PartitionType() string
 
 	// Returns the corresponding boot mode ("legacy", "uefi", "hybrid") or "none"
-	BootMode() BootMode
+	BootMode() platform.BootMode
 
 	// Returns the names of the pipelines that set up the build environment (buildroot).
 	BuildPipelines() []string

vendor/github.com/osbuild/images/pkg/distro/fedora/imagetype.go

@@ -126,15 +126,15 @@ func (t *imageType) Exports() []string {
 	return []string{"assembler"}
 }
 
-func (t *imageType) BootMode() distro.BootMode {
+func (t *imageType) BootMode() platform.BootMode {
 	if t.platform.GetUEFIVendor() != "" && t.platform.GetBIOSPlatform() != "" {
-		return distro.BOOT_HYBRID
+		return platform.BOOT_HYBRID
 	} else if t.platform.GetUEFIVendor() != "" {
-		return distro.BOOT_UEFI
+		return platform.BOOT_UEFI
 	} else if t.platform.GetBIOSPlatform() != "" || t.platform.GetZiplSupport() {
-		return distro.BOOT_LEGACY
+		return platform.BOOT_LEGACY
 	}
-	return distro.BOOT_NONE
+	return platform.BOOT_NONE
 }
 
 func (t *imageType) getPartitionTable(

vendor/github.com/osbuild/images/pkg/distro/rhel/imagetype.go

@@ -168,15 +168,15 @@ func (t *ImageType) Exports() []string {
 	return []string{"assembler"}
 }
 
-func (t *ImageType) BootMode() distro.BootMode {
+func (t *ImageType) BootMode() platform.BootMode {
 	if t.platform.GetUEFIVendor() != "" && t.platform.GetBIOSPlatform() != "" {
-		return distro.BOOT_HYBRID
+		return platform.BOOT_HYBRID
 	} else if t.platform.GetUEFIVendor() != "" {
-		return distro.BOOT_UEFI
+		return platform.BOOT_UEFI
 	} else if t.platform.GetBIOSPlatform() != "" || t.platform.GetZiplSupport() {
-		return distro.BOOT_LEGACY
+		return platform.BOOT_LEGACY
 	}
-	return distro.BOOT_NONE
+	return platform.BOOT_NONE
 }
 
 func (t *ImageType) GetPartitionTable(

vendor/github.com/osbuild/images/pkg/distro/rhel/rhel8/azure.go

@@ -529,7 +529,6 @@ var defaultAzureImageConfig = &distro.ImageConfig{
 		"nm-cloud-setup.service",
 		"nm-cloud-setup.timer",
 		"sshd",
-		"systemd-resolved",
 		"waagent",
 	},
 	SshdConfig: &osbuild.SshdConfigStageOptions{

vendor/github.com/osbuild/images/pkg/distro/test_distro/distro.go

@@ -9,6 +9,7 @@ import (
 	"github.com/osbuild/images/pkg/distro"
 	"github.com/osbuild/images/pkg/manifest"
 	"github.com/osbuild/images/pkg/ostree"
+	"github.com/osbuild/images/pkg/platform"
 	"github.com/osbuild/images/pkg/policies"
 	"github.com/osbuild/images/pkg/rpmmd"
 )
@@ -210,8 +211,8 @@ func (t *TestImageType) PartitionType() string {
 	return ""
 }
 
-func (t *TestImageType) BootMode() distro.BootMode {
-	return distro.BOOT_HYBRID
+func (t *TestImageType) BootMode() platform.BootMode {
+	return platform.BOOT_HYBRID
 }
 
 func (t *TestImageType) BuildPipelines() []string {

vendor/github.com/osbuild/images/pkg/image/bootc_disk.go

@@ -3,9 +3,7 @@ package image
 import (
 	"fmt"
 	"math/rand"
-	"regexp"
 
-	"github.com/osbuild/images/internal/common"
 	"github.com/osbuild/images/pkg/container"
 	"github.com/osbuild/images/pkg/customizations/users"
 	"github.com/osbuild/images/pkg/disk"
@@ -92,15 +90,7 @@ func (img *BootcDiskImage) InstantiateManifestFromContainers(m *manifest.Manifes
 		fmt.Sprintf("%s.vhd", fileBasename),
 	}
 
-	// XXX: copied from https://github.com/osbuild/images/blob/v0.85.0/pkg/image/disk.go#L102
-	gcePipeline := manifest.NewTar(buildPipeline, rawImage, "gce")
-	gcePipeline.Format = osbuild.TarArchiveFormatOldgnu
-	gcePipeline.RootNode = osbuild.TarRootNodeOmit
-	// these are required to successfully import the image to GCP
-	gcePipeline.ACLs = common.ToPtr(false)
-	gcePipeline.SELinux = common.ToPtr(false)
-	gcePipeline.Xattrs = common.ToPtr(false)
-	gcePipeline.Transform = fmt.Sprintf(`s/%s/disk.raw/`, regexp.QuoteMeta(rawImage.Filename()))
+	gcePipeline := newGCETarPipelineForImg(buildPipeline, rawImage, "gce")
 	gcePipeline.SetFilename("image.tar.gz")
 
 	return nil

vendor/github.com/osbuild/images/pkg/image/disk.go

@@ -6,7 +6,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/osbuild/images/internal/common"
 	"github.com/osbuild/images/internal/environment"
 	"github.com/osbuild/images/internal/workload"
 	"github.com/osbuild/images/pkg/artifact"
@@ -103,13 +102,7 @@ func (img *DiskImage) InstantiateManifest(m *manifest.Manifest,
 		// NOTE(akoutsou): temporary workaround; filename required for GCP
 		// TODO: define internal raw filename on image type
 		rawImagePipeline.SetFilename("disk.raw")
-		tarPipeline := manifest.NewTar(buildPipeline, rawImagePipeline, "archive")
-		tarPipeline.Format = osbuild.TarArchiveFormatOldgnu
-		tarPipeline.RootNode = osbuild.TarRootNodeOmit
-		// these are required to successfully import the image to GCP
-		tarPipeline.ACLs = common.ToPtr(false)
-		tarPipeline.SELinux = common.ToPtr(false)
-		tarPipeline.Xattrs = common.ToPtr(false)
+		tarPipeline := newGCETarPipelineForImg(buildPipeline, rawImagePipeline, "archive")
 		tarPipeline.SetFilename(img.Filename) // filename extension will determine compression
 		imagePipeline = tarPipeline
 	default:

vendor/github.com/osbuild/images/pkg/image/gce.go

@@ -0,0 +1,24 @@
+package image
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/osbuild/images/internal/common"
+	"github.com/osbuild/images/pkg/manifest"
+	"github.com/osbuild/images/pkg/osbuild"
+)
+
+func newGCETarPipelineForImg(buildPipeline manifest.Build, inputPipeline manifest.FilePipeline, pipelinename string) *manifest.Tar {
+	tarPipeline := manifest.NewTar(buildPipeline, inputPipeline, pipelinename)
+	tarPipeline.Format = osbuild.TarArchiveFormatOldgnu
+	tarPipeline.RootNode = osbuild.TarRootNodeOmit
+	// these are required to successfully import the image to GCP
+	tarPipeline.ACLs = common.ToPtr(false)
+	tarPipeline.SELinux = common.ToPtr(false)
+	tarPipeline.Xattrs = common.ToPtr(false)
+	if inputPipeline.Filename() != "disk.raw" {
+		tarPipeline.Transform = fmt.Sprintf(`s/%s/disk.raw/`, regexp.QuoteMeta(inputPipeline.Filename()))
+	}
+	return tarPipeline
+}

vendor/github.com/osbuild/images/pkg/manifest/build.go

@@ -236,7 +236,9 @@ func (p *BuildrootFromContainer) serialize() osbuild.Pipeline {
 	pipeline.Runner = p.runner.String()
 
 	image := osbuild.NewContainersInputForSingleSource(p.containerSpecs[0])
-	stage, err := osbuild.NewContainerDeployStage(image, &osbuild.ContainerDeployOptions{})
+	// Make skopeo copy to remove the signatures of signed containers by default to workaround
+	// build failures until https://github.com/containers/image/issues/2599 is implemented
+	stage, err := osbuild.NewContainerDeployStage(image, &osbuild.ContainerDeployOptions{RemoveSignatures: true})
 	if err != nil {
 		panic(err)
 	}

vendor/github.com/osbuild/images/pkg/osbuild/container_deploy_stage.go

@@ -9,7 +9,8 @@ type ContainerDeployInputs struct {
 func (ContainerDeployInputs) isStageInputs() {}
 
 type ContainerDeployOptions struct {
-	Exclude []string `json:"exclude,omitempty"`
+	Exclude          []string `json:"exclude,omitempty"`
+	RemoveSignatures bool     `json:"remove-signatures,omitempty"`
 }
 
 func (ContainerDeployOptions) isStageOptions() {}

vendor/github.com/osbuild/images/pkg/ostree/ostree.go

@@ -25,9 +25,21 @@ var (
 // SourceSpec serves as input for ResolveParams, and contains all necessary
 // variables to resolve a ref, which can then be turned into a CommitSpec.
 type SourceSpec struct {
-	URL  string
-	Ref  string
+	URL string
+	Ref string
+	// RHSM indicates to use RHSM secrets when pulling from the remote. Alternatively, you can use MTLS with plain certs.
 	RHSM bool
+	// MTLS information. Will be ignored if RHSM is set.
+	MTLS *MTLS
+	// Proxy as HTTP proxy to use when fetching the ref.
+	Proxy string
+}
+
+// MTLS contains the options for resolving an ostree source.
+type MTLS struct {
+	CA         string
+	ClientCert string
+	ClientKey  string
 }
 
 // CommitSpec specifies an ostree commit using any combination of Ref (branch), URL (source), and Checksum (commit ID).
@@ -138,59 +150,53 @@ func verifyChecksum(commit string) bool {
 	return len(commit) > 0 && ostreeCommitRE.MatchString(commit)
 }
 
-// ResolveRef resolves the URL path specified by the location and ref
+// resolveRef resolves the URL path specified by the location and ref
 // (location+"refs/heads/"+ref) and returns the commit ID for the named ref. If
 // there is an error, it will be of type ResolveRefError.
-func ResolveRef(location, ref string, consumerCerts bool, subs *rhsm.Subscriptions, ca *string) (string, error) {
-	u, err := url.Parse(location)
+func resolveRef(ss SourceSpec) (string, error) {
+	u, err := url.Parse(ss.URL)
 	if err != nil {
 		return "", NewResolveRefError("error parsing ostree repository location: %v", err)
 	}
-	u.Path = path.Join(u.Path, "refs/heads/", ref)
-
-	var client *http.Client
-	if consumerCerts {
-		if subs == nil {
-			subs, err = rhsm.LoadSystemSubscriptions()
-			if err != nil {
-				return "", NewResolveRefError("error adding rhsm certificates when resolving ref: %s", err)
-			}
-			if subs.Consumer == nil {
-				return "", NewResolveRefError("error adding rhsm certificates when resolving ref")
-			}
-		}
+	u.Path = path.Join(u.Path, "refs/heads/", ss.Ref)
 
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	client := &http.Client{
+		Transport: transport,
+		Timeout:   300 * time.Second,
+	}
+	if u.Scheme == "https" {
 		tlsConf := &tls.Config{
 			MinVersion: tls.VersionTLS12,
 		}
 
-		if ca != nil {
-			caCertPEM, err := os.ReadFile(*ca)
+		// If CA is set, load the CA certificate and add it to the TLS configuration. Otherwise, use the system CA.
+		if ss.MTLS.CA != "" {
+			caCertPEM, err := os.ReadFile(ss.MTLS.CA)
 			if err != nil {
-				return "", NewResolveRefError("error adding rhsm certificates when resolving ref: %s", err)
+				return "", NewResolveRefError("error adding ca certificate when resolving ref: %s", err)
 			}
-			roots := x509.NewCertPool()
-			ok := roots.AppendCertsFromPEM(caCertPEM)
-			if !ok {
-				return "", NewResolveRefError("error adding rhsm certificates when resolving ref")
+			tlsConf.RootCAs = x509.NewCertPool()
+			if ok := tlsConf.RootCAs.AppendCertsFromPEM(caCertPEM); !ok {
+				return "", NewResolveRefError("error adding ca certificate when resolving ref")
 			}
-			tlsConf.RootCAs = roots
 		}
 
-		cert, err := tls.LoadX509KeyPair(subs.Consumer.ConsumerCert, subs.Consumer.ConsumerKey)
-		if err != nil {
-			return "", NewResolveRefError("error adding rhsm certificates when resolving ref: %s", err)
+		if ss.MTLS.ClientCert != "" && ss.MTLS.ClientKey != "" {
+			cert, err := tls.LoadX509KeyPair(ss.MTLS.ClientCert, ss.MTLS.ClientKey)
+			if err != nil {
+				return "", NewResolveRefError("error adding client certificate when resolving ref: %s", err)
+			}
+			tlsConf.Certificates = []tls.Certificate{cert}
 		}
-		tlsConf.Certificates = []tls.Certificate{cert}
 
-		client = &http.Client{
-			Transport: &http.Transport{
-				TLSClientConfig: tlsConf,
-			},
-			Timeout: 300 * time.Second,
+		transport.TLSClientConfig = tlsConf
+	}
+
+	if ss.Proxy != "" {
+		transport.Proxy = func(request *http.Request) (*url.URL, error) {
+			return url.Parse(ss.Proxy)
 		}
-	} else {
-		client = &http.Client{}
 	}
 
 	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
@@ -234,8 +240,31 @@ func Resolve(source SourceSpec) (CommitSpec, error) {
 		URL: source.URL,
 	}
 
+	if source.RHSM && source.MTLS != nil {
+		return commit, NewResolveRefError("cannot use both RHSM and MTLS when resolving ref")
+	}
+
 	if source.RHSM {
+		var subs *rhsm.Subscriptions
+		var err error
+
 		commit.Secrets = "org.osbuild.rhsm.consumer"
+		subs, err = rhsm.LoadSystemSubscriptions()
+
+		if err != nil {
+			return commit, NewResolveRefError("error adding rhsm certificates when resolving ref: %s", err)
+		}
+
+		if subs.Consumer == nil {
+			return commit, NewResolveRefError("error adding rhsm certificates when resolving ref")
+		}
+
+		source.MTLS = &MTLS{
+			ClientCert: subs.Consumer.ConsumerCert,
+			ClientKey:  subs.Consumer.ConsumerKey,
+		}
+	} else if source.MTLS != nil {
+		commit.Secrets = "org.osbuild.mtls"
 	}
 
 	if verifyChecksum(source.Ref) {
@@ -252,7 +281,7 @@ func Resolve(source SourceSpec) (CommitSpec, error) {
 	// URL set: Resolve checksum
 	if source.URL != "" {
 		// If a URL is specified, we need to fetch the commit at the URL.
-		checksum, err := ResolveRef(source.URL, source.Ref, source.RHSM, nil, nil)
+		checksum, err := resolveRef(source)
 		if err != nil {
 			return CommitSpec{}, err // ResolveRefError
 		}

vendor/github.com/osbuild/images/pkg/platform/bootmode.go

@@ -0,0 +1,25 @@
+package platform
+
+type BootMode uint64
+
+const (
+	BOOT_NONE BootMode = iota
+	BOOT_LEGACY
+	BOOT_UEFI
+	BOOT_HYBRID
+)
+
+func (m BootMode) String() string {
+	switch m {
+	case BOOT_NONE:
+		return "none"
+	case BOOT_LEGACY:
+		return "legacy"
+	case BOOT_UEFI:
+		return "uefi"
+	case BOOT_HYBRID:
+		return "hybrid"
+	default:
+		panic("invalid boot mode")
+	}
+}