From f14dc2fb635eaad306bbf6603717f0c2c470c353 Mon Sep 17 00:00:00 2001 From: Jordi Gil Date: Mon, 11 Apr 2022 13:45:08 -0400 Subject: [PATCH] distro/fedora: refactor based on RHEL 9.0 code --- cmd/osbuild-dnf-json-tests/main_test.go | 4 +- cmd/osbuild-store-dump/main.go | 2 +- internal/distro/fedora/distro.go | 873 +++++++++++++++++++++ internal/distro/fedora/distro_test.go | 734 +++++++++++++++++ internal/distro/fedora/package_sets.go | 800 +++++++++++++++++++ internal/distro/fedora/partition_tables.go | 107 +++ internal/distro/fedora/pipelines.go | 679 ++++++++++++++++ internal/distro/fedora/stage_options.go | 324 ++++++++ internal/distroregistry/distroregistry.go | 2 +- internal/store/json.go | 5 + internal/store/json_test.go | 2 +- 11 files changed, 3527 insertions(+), 5 deletions(-) create mode 100644 internal/distro/fedora/distro.go create mode 100644 internal/distro/fedora/distro_test.go create mode 100644 internal/distro/fedora/package_sets.go create mode 100644 internal/distro/fedora/partition_tables.go create mode 100644 internal/distro/fedora/pipelines.go create mode 100644 internal/distro/fedora/stage_options.go diff --git a/cmd/osbuild-dnf-json-tests/main_test.go b/cmd/osbuild-dnf-json-tests/main_test.go index 885677a9b..525d0c5b9 100644 --- a/cmd/osbuild-dnf-json-tests/main_test.go +++ b/cmd/osbuild-dnf-json-tests/main_test.go @@ -14,7 +14,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/blueprint" "github.com/osbuild/osbuild-composer/internal/distro" - fedora "github.com/osbuild/osbuild-composer/internal/distro/fedora33" + "github.com/osbuild/osbuild-composer/internal/distro/fedora" rhel "github.com/osbuild/osbuild-composer/internal/distro/rhel86" "github.com/osbuild/osbuild-composer/internal/rpmmd" "github.com/osbuild/osbuild-composer/internal/test" @@ -83,7 +83,7 @@ func TestCrossArchDepsolve(t *testing.T) { packages := imgType.PackageSets(blueprint.Blueprint{}) - _, _, err = rpm.Depsolve(packages["build-packages"], repos[archStr], distroStruct.ModulePlatformID(), archStr, distroStruct.Releasever()) + _, _, err = rpm.Depsolve(packages["build"], repos[archStr], distroStruct.ModulePlatformID(), archStr, distroStruct.Releasever()) assert.NoError(t, err) _, _, err = rpm.Depsolve(packages["packages"], repos[archStr], distroStruct.ModulePlatformID(), archStr, distroStruct.Releasever()) diff --git a/cmd/osbuild-store-dump/main.go b/cmd/osbuild-store-dump/main.go index ba5a9a315..145061f8c 100644 --- a/cmd/osbuild-store-dump/main.go +++ b/cmd/osbuild-store-dump/main.go @@ -11,7 +11,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/blueprint" "github.com/osbuild/osbuild-composer/internal/distro" - fedora "github.com/osbuild/osbuild-composer/internal/distro/fedora33" + "github.com/osbuild/osbuild-composer/internal/distro/fedora" "github.com/osbuild/osbuild-composer/internal/rpmmd" "github.com/osbuild/osbuild-composer/internal/store" "github.com/osbuild/osbuild-composer/internal/target" diff --git a/internal/distro/fedora/distro.go b/internal/distro/fedora/distro.go new file mode 100644 index 000000000..42948f56a --- /dev/null +++ b/internal/distro/fedora/distro.go @@ -0,0 +1,873 @@ +package fedora + +import ( + "encoding/json" + "errors" + "fmt" + "math/rand" + "path" + "sort" + "strings" + + "github.com/osbuild/osbuild-composer/internal/blueprint" + "github.com/osbuild/osbuild-composer/internal/disk" + "github.com/osbuild/osbuild-composer/internal/distro" + osbuild "github.com/osbuild/osbuild-composer/internal/osbuild2" + "github.com/osbuild/osbuild-composer/internal/rpmmd" +) + +const ( + GigaByte = 1024 * 1024 * 1024 + // package set names + + // build package set name + buildPkgsKey = "build" + + // main/common os image package set name + osPkgsKey = "packages" + + // container package set name + containerPkgsKey = "container" + + // installer package set name + installerPkgsKey = "installer" + + // blueprint package set name + blueprintPkgsKey = "blueprint" + + // Fedora distribution + fedora34Distribution = "fedora-34" + fedora35Distribution = "fedora-35" + fedora36Distribution = "fedora-36" +) + +var ( + mountpointAllowList = []string{ + "/", "/var", "/opt", "/srv", "/usr", "/app", "/data", "/home", "/tmp", + } + + // Services + iotServices = []string{ + "NetworkManager.service", + "firewalld.service", + "rngd.service", + "sshd.service", + "zezere_ignition.timer", + "zezere_ignition_banner.service", + "greenboot-grub2-set-counter", + "greenboot-grub2-set-success", + "greenboot-healthcheck", + "greenboot-rpm-ostree-grub2-check-fallback", + "greenboot-status", + "greenboot-task-runner", + "redboot-auto-reboot", + "redboot-task-runner", + "parsec", + "dbus-parsec", + } + + // Image Definitions + iotCommitImgType = imageType{ + name: "fedora-iot-commit", + nameAliases: []string{"iot-commit"}, + filename: "commit.tar", + mimeType: "application/x-tar", + packageSets: map[string]packageSetFunc{ + buildPkgsKey: iotBuildPackageSet, + osPkgsKey: iotCommitPackageSet, + }, + defaultImageConfig: &distro.ImageConfig{ + EnabledServices: iotServices, + }, + rpmOstree: true, + pipelines: edgeCommitPipelines, + buildPipelines: []string{"build"}, + payloadPipelines: []string{"ostree-tree", "ostree-commit", "commit-archive"}, + exports: []string{"commit-archive"}, + } + + iotOCIImgType = imageType{ + name: "fedora-iot-container", + nameAliases: []string{"iot-container"}, + filename: "container.tar", + mimeType: "application/x-tar", + packageSets: map[string]packageSetFunc{ + buildPkgsKey: iotBuildPackageSet, + osPkgsKey: iotCommitPackageSet, + containerPkgsKey: func(t *imageType) rpmmd.PackageSet { + return rpmmd.PackageSet{ + Include: []string{"nginx"}, + } + }, + }, + defaultImageConfig: &distro.ImageConfig{ + EnabledServices: iotServices, + }, + rpmOstree: true, + bootISO: false, + pipelines: iotContainerPipelines, + buildPipelines: []string{"build"}, + payloadPipelines: []string{"ostree-tree", "ostree-commit", "container-tree", "container"}, + exports: []string{"container"}, + } + + iotInstallerImgType = imageType{ + name: "fedora-iot-installer", + nameAliases: []string{"iot-installer"}, + filename: "installer.iso", + mimeType: "application/x-iso9660-image", + packageSets: map[string]packageSetFunc{ + buildPkgsKey: iotInstallerBuildPackageSet, + osPkgsKey: iotCommitPackageSet, + installerPkgsKey: iotInstallerPackageSet, + }, + defaultImageConfig: &distro.ImageConfig{ + Locale: "en_US.UTF-8", + EnabledServices: iotServices, + }, + rpmOstree: true, + bootISO: true, + pipelines: iotInstallerPipelines, + buildPipelines: []string{"build"}, + payloadPipelines: []string{"anaconda-tree", "bootiso-tree", "bootiso"}, + exports: []string{"bootiso"}, + } + + qcow2ImgType = imageType{ + name: "qcow2", + filename: "disk.qcow2", + mimeType: "application/x-qemu-disk", + packageSets: map[string]packageSetFunc{ + buildPkgsKey: distroBuildPackageSet, + osPkgsKey: qcow2CommonPackageSet, + }, + defaultImageConfig: &distro.ImageConfig{ + DefaultTarget: "graphical.target", + EnabledServices: []string{ + "cloud-init.service", + "cloud-config.service", + "cloud-final.service", + "cloud-init-local.service", + }, + }, + bootable: true, + defaultSize: 2 * GigaByte, + pipelines: qcow2Pipelines, + buildPipelines: []string{"build"}, + payloadPipelines: []string{"os", "image", "qcow2"}, + exports: []string{"qcow2"}, + basePartitionTables: defaultBasePartitionTables, + } + + vhdImgType = imageType{ + name: "vhd", + filename: "disk.vhd", + mimeType: "application/x-vhd", + packageSets: map[string]packageSetFunc{ + buildPkgsKey: distroBuildPackageSet, + osPkgsKey: vhdCommonPackageSet, + }, + defaultImageConfig: &distro.ImageConfig{ + Locale: "en_US.UTF-8", + EnabledServices: []string{ + "sshd", + "waagent", + }, + DefaultTarget: "graphical.target", + DisabledServices: []string{ + "proc-sys-fs-binfmt_misc.mount", + "loadmodules.service", + }, + }, + kernelOptions: "ro biosdevname=0 rootdelay=300 console=ttyS0 earlyprintk=ttyS0 net.ifnames=0", + bootable: true, + defaultSize: 2 * GigaByte, + pipelines: vhdPipelines, + buildPipelines: []string{"build"}, + payloadPipelines: []string{"os", "image", "vpc"}, + exports: []string{"vpc"}, + basePartitionTables: defaultBasePartitionTables, + } + + vmdkImgType = imageType{ + name: "vmdk", + filename: "disk.vmdk", + mimeType: "application/x-vmdk", + packageSets: map[string]packageSetFunc{ + buildPkgsKey: distroBuildPackageSet, + osPkgsKey: vmdkCommonPackageSet, + }, + defaultImageConfig: &distro.ImageConfig{ + Locale: "en_US.UTF-8", + EnabledServices: []string{ + "cloud-init.service", + "cloud-config.service", + "cloud-final.service", + "cloud-init-local.service", + }, + }, + bootable: true, + defaultSize: 2 * GigaByte, + pipelines: vmdkPipelines, + buildPipelines: []string{"build"}, + payloadPipelines: []string{"os", "image", "vmdk"}, + exports: []string{"vmdk"}, + basePartitionTables: defaultBasePartitionTables, + } + + openstackImgType = imageType{ + name: "openstack", + filename: "disk.qcow2", + mimeType: "application/x-qemu-disk", + packageSets: map[string]packageSetFunc{ + buildPkgsKey: distroBuildPackageSet, + osPkgsKey: openstackCommonPackageSet, + }, + defaultImageConfig: &distro.ImageConfig{ + Locale: "en_US.UTF-8", + EnabledServices: []string{ + "cloud-init.service", + "cloud-config.service", + "cloud-final.service", + "cloud-init-local.service", + }, + }, + bootable: true, + defaultSize: 2 * GigaByte, + pipelines: openstackPipelines, + buildPipelines: []string{"build"}, + payloadPipelines: []string{"os", "image", "qcow2"}, + exports: []string{"qcow2"}, + basePartitionTables: defaultBasePartitionTables, + } + + // default EC2 images config (common for all architectures) + defaultEc2ImageConfig = &distro.ImageConfig{ + Locale: "en_US", + Timezone: "UTC", + EnabledServices: []string{ + "cloud-init.service", + }, + } + + amiImgType = imageType{ + name: "ami", + filename: "image.raw", + mimeType: "application/octet-stream", + packageSets: map[string]packageSetFunc{ + buildPkgsKey: ec2BuildPackageSet, + osPkgsKey: ec2CommonPackageSet, + }, + defaultImageConfig: defaultEc2ImageConfig, + kernelOptions: "ro no_timer_check net.ifnames=0 console=tty1 console=ttyS0,115200n8", + bootable: true, + bootType: distro.LegacyBootType, + defaultSize: 6 * GigaByte, + pipelines: ec2Pipelines, + buildPipelines: []string{"build"}, + payloadPipelines: []string{"os", "image"}, + exports: []string{"image"}, + basePartitionTables: defaultBasePartitionTables, + } +) + +type distribution struct { + name string + product string + osVersion string + releaseVersion string + modulePlatformID string + vendor string + ostreeRefTmpl string + isolabelTmpl string + runner string + arches map[string]distro.Arch + defaultImageConfig *distro.ImageConfig +} + +// RHEL-based OS image configuration defaults +var defaultDistroImageConfig = &distro.ImageConfig{ + Timezone: "UTC", + Locale: "en_US", +} + +// distribution objects without the arches > image types +var distroMap = map[string]distribution{ + fedora34Distribution: { + name: fedora34Distribution, + product: "Fedora", + osVersion: "34", + releaseVersion: "34", + modulePlatformID: "platform:f34", + vendor: "fedora", + ostreeRefTmpl: "fedora/34/%s/iot", + isolabelTmpl: "Fedora-34-BaseOS-%s", + runner: "org.osbuild.fedora34", + defaultImageConfig: defaultDistroImageConfig, + }, + fedora35Distribution: { + name: fedora35Distribution, + product: "Fedora", + osVersion: "35", + releaseVersion: "35", + modulePlatformID: "platform:f35", + vendor: "fedora", + ostreeRefTmpl: "fedora/35/%s/iot", + isolabelTmpl: "Fedora-35-BaseOS-%s", + runner: "org.osbuild.fedora35", + defaultImageConfig: defaultDistroImageConfig, + }, + fedora36Distribution: { + name: fedora36Distribution, + product: "Fedora", + osVersion: "36", + releaseVersion: "36", + modulePlatformID: "platform:f36", + vendor: "fedora", + ostreeRefTmpl: "fedora/36/%s/iot", + isolabelTmpl: "Fedora-36-BaseOS-%s", + runner: "org.osbuild.fedora36", + defaultImageConfig: defaultDistroImageConfig, + }, +} + +func (d *distribution) Name() string { + return d.name +} + +func (d *distribution) Releasever() string { + return d.releaseVersion +} + +func (d *distribution) ModulePlatformID() string { + return d.modulePlatformID +} + +func (d *distribution) OSTreeRef() string { + return d.ostreeRefTmpl +} + +func (d *distribution) ListArches() []string { + archNames := make([]string, 0, len(d.arches)) + for name := range d.arches { + archNames = append(archNames, name) + } + sort.Strings(archNames) + return archNames +} + +func (d *distribution) GetArch(name string) (distro.Arch, error) { + arch, exists := d.arches[name] + if !exists { + return nil, errors.New("invalid architecture: " + name) + } + return arch, nil +} + +func (d *distribution) addArches(arches ...architecture) { + if d.arches == nil { + d.arches = map[string]distro.Arch{} + } + + // Do not make copies of architectures, as opposed to image types, + // because architecture definitions are not used by more than a single + // distro definition. + for idx := range arches { + d.arches[arches[idx].name] = &arches[idx] + } +} + +func (d *distribution) getDefaultImageConfig() *distro.ImageConfig { + return d.defaultImageConfig +} + +type architecture struct { + distro *distribution + name string + imageTypes map[string]distro.ImageType + imageTypeAliases map[string]string + legacy string + bootType distro.BootType +} + +func (a *architecture) Name() string { + return a.name +} + +func (a *architecture) ListImageTypes() []string { + itNames := make([]string, 0, len(a.imageTypes)) + for name := range a.imageTypes { + itNames = append(itNames, name) + } + sort.Strings(itNames) + return itNames +} + +func (a *architecture) GetImageType(name string) (distro.ImageType, error) { + t, exists := a.imageTypes[name] + if !exists { + aliasForName, exists := a.imageTypeAliases[name] + if !exists { + return nil, errors.New("invalid image type: " + name) + } + t, exists = a.imageTypes[aliasForName] + if !exists { + panic(fmt.Sprintf("image type '%s' is an alias to a non-existing image type '%s'", name, aliasForName)) + } + } + return t, nil +} + +func (a *architecture) addImageTypes(imageTypes ...imageType) { + if a.imageTypes == nil { + a.imageTypes = map[string]distro.ImageType{} + } + for idx := range imageTypes { + it := imageTypes[idx] + it.arch = a + a.imageTypes[it.name] = &it + for _, alias := range it.nameAliases { + if a.imageTypeAliases == nil { + a.imageTypeAliases = map[string]string{} + } + if existingAliasFor, exists := a.imageTypeAliases[alias]; exists { + panic(fmt.Sprintf("image type alias '%s' for '%s' is already defined for another image type '%s'", alias, it.name, existingAliasFor)) + } + a.imageTypeAliases[alias] = it.name + } + } +} + +func (a *architecture) Distro() distro.Distro { + return a.distro +} + +type pipelinesFunc func(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]osbuild.Pipeline, error) + +type packageSetFunc func(t *imageType) rpmmd.PackageSet + +type imageType struct { + arch *architecture + name string + nameAliases []string + filename string + mimeType string + packageSets map[string]packageSetFunc + defaultImageConfig *distro.ImageConfig + kernelOptions string + defaultSize uint64 + buildPipelines []string + payloadPipelines []string + exports []string + pipelines pipelinesFunc + + // bootISO: installable ISO + bootISO bool + // rpmOstree: edge/ostree + rpmOstree bool + // bootable image + bootable bool + // If set to a value, it is preferred over the architecture value + bootType distro.BootType + // List of valid arches for the image type + basePartitionTables distro.BasePartitionTableMap +} + +func (t *imageType) Name() string { + return t.name +} + +func (t *imageType) Arch() distro.Arch { + return t.arch +} + +func (t *imageType) Filename() string { + return t.filename +} + +func (t *imageType) MIMEType() string { + return t.mimeType +} + +func (t *imageType) OSTreeRef() string { + d := t.arch.distro + if t.rpmOstree { + return fmt.Sprintf(d.ostreeRefTmpl, t.arch.Name()) + } + return "" +} + +func (t *imageType) Size(size uint64) uint64 { + const MegaByte = 1024 * 1024 + // Microsoft Azure requires vhd images to be rounded up to the nearest MB + if t.name == "vhd" && size%MegaByte != 0 { + size = (size/MegaByte + 1) * MegaByte + } + if size == 0 { + size = t.defaultSize + } + return size +} + +func (t *imageType) getPackages(name string) rpmmd.PackageSet { + getter := t.packageSets[name] + if getter == nil { + return rpmmd.PackageSet{} + } + + return getter(t) +} + +func (t *imageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet { + // merge package sets that appear in the image type with the package sets + // of the same name from the distro and arch + mergedSets := make(map[string]rpmmd.PackageSet) + + imageSets := t.packageSets + + for name := range imageSets { + mergedSets[name] = t.getPackages(name) + } + + if _, hasPackages := imageSets[osPkgsKey]; !hasPackages { + // should this be possible?? + mergedSets[osPkgsKey] = rpmmd.PackageSet{} + } + + // every image type must define a 'build' package set + if _, hasBuild := imageSets[buildPkgsKey]; !hasBuild { + panic(fmt.Sprintf("'%s' image type has no '%s' package set defined", t.name, buildPkgsKey)) + } + + // blueprint packages + bpPackages := bp.GetPackages() + timezone, _ := bp.Customizations.GetTimezoneSettings() + if timezone != nil { + bpPackages = append(bpPackages, "chrony") + } + + // if we have file system customization that will need to a new mount point + // the layout is converted to LVM so we need to corresponding packages + if !t.rpmOstree { + + pt, exists := t.basePartitionTables[t.arch.Name()] + if !exists { + panic(fmt.Sprintf("unknown architecture with boot type: %s %s", t.arch.Name(), t.bootType)) + } + haveNewMountpoint := false + + if fs := bp.Customizations.GetFilesystems(); fs != nil { + for i := 0; !haveNewMountpoint && i < len(fs); i++ { + haveNewMountpoint = !pt.ContainsMountpoint(fs[i].Mountpoint) + } + } + + if haveNewMountpoint { + bpPackages = append(bpPackages, "lvm2") + } + } + + // depsolve bp packages separately + // bp packages aren't restricted by exclude lists + mergedSets[blueprintPkgsKey] = rpmmd.PackageSet{Include: bpPackages} + kernel := bp.Customizations.GetKernel().Name + + // add bp kernel to main OS package set to avoid duplicate kernels + mergedSets[osPkgsKey] = mergedSets[osPkgsKey].Append(rpmmd.PackageSet{Include: []string{kernel}}) + return mergedSets + +} + +func (t *imageType) BuildPipelines() []string { + return t.buildPipelines +} + +func (t *imageType) PayloadPipelines() []string { + return t.payloadPipelines +} + +func (t *imageType) PayloadPackageSets() []string { + return []string{blueprintPkgsKey} +} + +func (t *imageType) Exports() []string { + if len(t.exports) > 0 { + return t.exports + } + return []string{"assembler"} +} + +// getBootType returns the BootType which should be used for this particular +// combination of architecture and image type. +func (t *imageType) getBootType() distro.BootType { + bootType := t.arch.bootType + if t.bootType != distro.UnsetBootType { + bootType = t.bootType + } + return bootType +} + +func (t *imageType) supportsUEFI() bool { + bootType := t.getBootType() + return bootType == distro.HybridBootType || bootType == distro.UEFIBootType +} + +func (t *imageType) getPartitionTable( + mountpoints []blueprint.FilesystemCustomization, + options distro.ImageOptions, + rng *rand.Rand, +) (*disk.PartitionTable, error) { + basePartitionTable, exists := t.basePartitionTables[t.arch.Name()] + if !exists { + return nil, fmt.Errorf("unknown arch: " + t.arch.Name()) + } + + imageSize := t.Size(options.Size) + + lvmify := !t.rpmOstree + + return disk.NewPartitionTable(&basePartitionTable, mountpoints, imageSize, lvmify, rng) +} + +func (t *imageType) getDefaultImageConfig() *distro.ImageConfig { + // ensure that image always returns non-nil default config + imageConfig := t.defaultImageConfig + if imageConfig == nil { + imageConfig = &distro.ImageConfig{} + } + return imageConfig.InheritFrom(t.arch.distro.getDefaultImageConfig()) + +} + +func (t *imageType) PartitionType() string { + basePartitionTable, exists := t.basePartitionTables[t.arch.Name()] + if !exists { + return "" + } + + return basePartitionTable.Type +} + +// local type for ostree commit metadata used to define commit sources +type ostreeCommit struct { + Checksum string + URL string +} + +func (t *imageType) Manifest(customizations *blueprint.Customizations, + options distro.ImageOptions, + repos []rpmmd.RepoConfig, + packageSpecSets map[string][]rpmmd.PackageSpec, + seed int64) (distro.Manifest, error) { + + if err := t.checkOptions(customizations, options); err != nil { + return distro.Manifest{}, err + } + + source := rand.NewSource(seed) + // math/rand is good enough in this case + /* #nosec G404 */ + rng := rand.New(source) + + pipelines, err := t.pipelines(t, customizations, options, repos, packageSpecSets, rng) + if err != nil { + return distro.Manifest{}, err + } + + // flatten spec sets for sources + allPackageSpecs := make([]rpmmd.PackageSpec, 0) + for _, specs := range packageSpecSets { + allPackageSpecs = append(allPackageSpecs, specs...) + } + + // handle OSTree commit inputs + var commits []ostreeCommit + if options.OSTree.Parent != "" && options.OSTree.URL != "" { + commits = []ostreeCommit{{Checksum: options.OSTree.Parent, URL: options.OSTree.URL}} + } + + // handle inline sources + inlineData := []string{} + + // FDO root certs, if any, are transmitted via an inline source + if fdo := customizations.GetFDO(); fdo != nil && fdo.DiunPubKeyRootCerts != "" { + inlineData = append(inlineData, fdo.DiunPubKeyRootCerts) + } + + return json.Marshal( + osbuild.Manifest{ + Version: "2", + Pipelines: pipelines, + Sources: t.sources(allPackageSpecs, commits, inlineData), + }, + ) +} + +func (t *imageType) sources(packages []rpmmd.PackageSpec, ostreeCommits []ostreeCommit, inlineData []string) osbuild.Sources { + sources := osbuild.Sources{} + curl := &osbuild.CurlSource{ + Items: make(map[string]osbuild.CurlSourceItem), + } + for _, pkg := range packages { + item := new(osbuild.URLWithSecrets) + item.URL = pkg.RemoteLocation + if pkg.Secrets == "org.osbuild.rhsm" { + item.Secrets = &osbuild.URLSecrets{ + Name: "org.osbuild.rhsm", + } + } + curl.Items[pkg.Checksum] = item + } + if len(curl.Items) > 0 { + sources["org.osbuild.curl"] = curl + } + + ostree := &osbuild.OSTreeSource{ + Items: make(map[string]osbuild.OSTreeSourceItem), + } + for _, commit := range ostreeCommits { + item := new(osbuild.OSTreeSourceItem) + item.Remote.URL = commit.URL + ostree.Items[commit.Checksum] = *item + } + if len(ostree.Items) > 0 { + sources["org.osbuild.ostree"] = ostree + } + + if len(inlineData) > 0 { + ils := osbuild.NewInlineSource() + for _, data := range inlineData { + ils.AddItem(data) + } + + sources["org.osbuild.inline"] = ils + } + + return sources +} + +func isMountpointAllowed(mountpoint string) bool { + for _, allowed := range mountpointAllowList { + match, _ := path.Match(allowed, mountpoint) + if match { + return true + } + // ensure that only clean mountpoints + // are valid + if strings.Contains(mountpoint, "//") { + return false + } + match = strings.HasPrefix(mountpoint, allowed+"/") + if allowed != "/" && match { + return true + } + } + return false +} + +// checkOptions checks the validity and compatibility of options and customizations for the image type. +func (t *imageType) checkOptions(customizations *blueprint.Customizations, options distro.ImageOptions) error { + if t.bootISO && t.rpmOstree { + if options.OSTree.Parent == "" { + return fmt.Errorf("boot ISO image type %q requires specifying a URL from which to retrieve the OSTree commit", t.name) + } + + if customizations != nil { + return fmt.Errorf("boot ISO image type %q does not support blueprint customizations", t.name) + } + } + + if kernelOpts := customizations.GetKernel(); kernelOpts.Append != "" && t.rpmOstree { + return fmt.Errorf("kernel boot parameter customizations are not supported for ostree types") + } + + mountpoints := customizations.GetFilesystems() + + if mountpoints != nil && t.rpmOstree { + return fmt.Errorf("Custom mountpoints are not supported for ostree types") + } + + invalidMountpoints := []string{} + for _, m := range mountpoints { + if !isMountpointAllowed(m.Mountpoint) { + invalidMountpoints = append(invalidMountpoints, m.Mountpoint) + } + } + + if len(invalidMountpoints) > 0 { + return fmt.Errorf("The following custom mountpoints are not supported %+q", invalidMountpoints) + } + + return nil +} + +func NewHostDistro(name, modulePlatformID, ostreeRef string) distro.Distro { + return newDistro(name) +} + +// New creates a new distro object, defining the supported architectures and image types +func NewF34() distro.Distro { + return newDistro(fedora34Distribution) +} +func NewF35() distro.Distro { + return newDistro(fedora35Distribution) +} +func NewF36() distro.Distro { + return newDistro(fedora36Distribution) +} + +func newDistro(distroName string) distro.Distro { + + rd := distroMap[distroName] + + // Architecture definitions + x86_64 := architecture{ + name: distro.X86_64ArchName, + distro: &rd, + legacy: "i386-pc", + bootType: distro.LegacyBootType, + } + + aarch64 := architecture{ + name: distro.Aarch64ArchName, + distro: &rd, + bootType: distro.UEFIBootType, + } + + s390x := architecture{ + distro: &rd, + name: distro.S390xArchName, + bootType: distro.LegacyBootType, + } + + ociImgType := qcow2ImgType + ociImgType.name = "oci" + + x86_64.addImageTypes( + amiImgType, + qcow2ImgType, + openstackImgType, + vhdImgType, + vmdkImgType, + ociImgType, + iotOCIImgType, + iotCommitImgType, + iotInstallerImgType, + ) + aarch64.addImageTypes( + amiImgType, + qcow2ImgType, + openstackImgType, + ociImgType, + iotCommitImgType, + iotOCIImgType, + iotInstallerImgType, + ) + + s390x.addImageTypes() + + rd.addArches(x86_64, aarch64, s390x) + return &rd +} + +// Shared Services diff --git a/internal/distro/fedora/distro_test.go b/internal/distro/fedora/distro_test.go new file mode 100644 index 000000000..a15f39c70 --- /dev/null +++ b/internal/distro/fedora/distro_test.go @@ -0,0 +1,734 @@ +package fedora_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/osbuild/osbuild-composer/internal/blueprint" + "github.com/osbuild/osbuild-composer/internal/distro" + "github.com/osbuild/osbuild-composer/internal/distro/distro_test_common" + "github.com/osbuild/osbuild-composer/internal/distro/fedora" +) + +type fedoraFamilyDistro struct { + name string + distro distro.Distro +} + +var fedoraFamilyDistros = []fedoraFamilyDistro{ + { + name: "fedora", + distro: fedora.NewF35(), + }, +} + +func TestFilenameFromType(t *testing.T) { + type args struct { + outputFormat string + } + type wantResult struct { + filename string + mimeType string + wantErr bool + } + tests := []struct { + name string + args args + want wantResult + }{ + { + name: "ami", + args: args{"ami"}, + want: wantResult{ + filename: "image.raw", + mimeType: "application/octet-stream", + }, + }, + { + name: "qcow2", + args: args{"qcow2"}, + want: wantResult{ + filename: "disk.qcow2", + mimeType: "application/x-qemu-disk", + }, + }, + { + name: "openstack", + args: args{"openstack"}, + want: wantResult{ + filename: "disk.qcow2", + mimeType: "application/x-qemu-disk", + }, + }, + { + name: "vhd", + args: args{"vhd"}, + want: wantResult{ + filename: "disk.vhd", + mimeType: "application/x-vhd", + }, + }, + { + name: "vmdk", + args: args{"vmdk"}, + want: wantResult{ + filename: "disk.vmdk", + mimeType: "application/x-vmdk", + }, + }, + + { + name: "iot-commit", + args: args{"iot-commit"}, + want: wantResult{ + filename: "commit.tar", + mimeType: "application/x-tar", + }, + }, + // Alias + { + name: "fedora-iot-commit", + args: args{"fedora-iot-commit"}, + want: wantResult{ + filename: "commit.tar", + mimeType: "application/x-tar", + }, + }, + { + name: "fedora-iot-commit", + args: args{"fedora-iot-commit"}, + want: wantResult{ + filename: "commit.tar", + mimeType: "application/x-tar", + }, + }, + { + name: "fedora-iot-container", + args: args{"fedora-iot-container"}, + want: wantResult{ + filename: "container.tar", + mimeType: "application/x-tar", + }, + }, + // Alias + { + name: "iot-container", + args: args{"iot-container"}, + want: wantResult{ + filename: "container.tar", + mimeType: "application/x-tar", + }, + }, + { + name: "fedora-iot-container", + args: args{"fedora-iot-container"}, + want: wantResult{ + filename: "container.tar", + mimeType: "application/x-tar", + }, + }, + + { + name: "fedora-iot-installer", + args: args{"fedora-iot-installer"}, + want: wantResult{ + filename: "installer.iso", + mimeType: "application/x-iso9660-image", + }, + }, + // Alias + { + name: "iot-installer", + args: args{"iot-installer"}, + want: wantResult{ + filename: "installer.iso", + mimeType: "application/x-iso9660-image", + }, + }, + { + name: "fedora-iot-installer", + args: args{"fedora-iot-installer"}, + want: wantResult{ + filename: "installer.iso", + mimeType: "application/x-iso9660-image", + }, + }, + { + name: "invalid-output-type", + args: args{"foobar"}, + want: wantResult{wantErr: true}, + }, + } + for _, dist := range fedoraFamilyDistros { + t.Run(dist.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dist := dist.distro + arch, _ := dist.GetArch("x86_64") + imgType, err := arch.GetImageType(tt.args.outputFormat) + if (err != nil) != tt.want.wantErr { + t.Errorf("Arch.GetImageType() error = %v, wantErr %v", err, tt.want.wantErr) + return + } + if !tt.want.wantErr { + gotFilename := imgType.Filename() + gotMIMEType := imgType.MIMEType() + if gotFilename != tt.want.filename { + t.Errorf("ImageType.Filename() got = %v, want %v", gotFilename, tt.want.filename) + } + if gotMIMEType != tt.want.mimeType { + t.Errorf("ImageType.MIMEType() got1 = %v, want %v", gotMIMEType, tt.want.mimeType) + } + } + }) + } + }) + } +} + +func TestImageType_BuildPackages(t *testing.T) { + x8664BuildPackages := []string{ + "dnf", + "dosfstools", + "e2fsprogs", + "grub2-efi-x64", + "grub2-pc", + "policycoreutils", + "shim-x64", + "systemd", + "tar", + "qemu-img", + "xz", + } + aarch64BuildPackages := []string{ + "dnf", + "dosfstools", + "e2fsprogs", + "policycoreutils", + "qemu-img", + "systemd", + "tar", + "xz", + } + buildPackages := map[string][]string{ + "x86_64": x8664BuildPackages, + "aarch64": aarch64BuildPackages, + } + for _, dist := range fedoraFamilyDistros { + t.Run(dist.name, func(t *testing.T) { + d := dist.distro + for _, archLabel := range d.ListArches() { + archStruct, err := d.GetArch(archLabel) + if assert.NoErrorf(t, err, "d.GetArch(%v) returned err = %v; expected nil", archLabel, err) { + continue + } + for _, itLabel := range archStruct.ListImageTypes() { + itStruct, err := archStruct.GetImageType(itLabel) + if assert.NoErrorf(t, err, "d.GetArch(%v) returned err = %v; expected nil", archLabel, err) { + continue + } + buildPkgs := itStruct.PackageSets(blueprint.Blueprint{})["build"] + assert.NotNil(t, buildPkgs) + assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs.Include) + } + } + }) + } +} + +func TestImageType_Name(t *testing.T) { + imgMap := []struct { + arch string + imgNames []string + }{ + { + arch: "x86_64", + imgNames: []string{ + "qcow2", + "openstack", + "vhd", + "vmdk", + "ami", + "fedora-iot-commit", + "fedora-iot-container", + "fedora-iot-installer", + "oci", + }, + }, + { + arch: "aarch64", + imgNames: []string{ + "qcow2", + "openstack", + "ami", + "oci", + "fedora-iot-commit", + "fedora-iot-container", + "fedora-iot-installer", + }, + }, + } + + for _, dist := range fedoraFamilyDistros { + t.Run(dist.name, func(t *testing.T) { + for _, mapping := range imgMap { + if mapping.arch == "s390x" && dist.name == "centos" { + continue + } + arch, err := dist.distro.GetArch(mapping.arch) + if assert.NoError(t, err) { + for _, imgName := range mapping.imgNames { + if imgName == "fedora-iot-commit" && dist.name == "centos" { + continue + } + imgType, err := arch.GetImageType(imgName) + if assert.NoError(t, err) { + assert.Equalf(t, imgName, imgType.Name(), "arch: %s", mapping.arch) + } + } + } + } + }) + } +} + +func TestImageTypeAliases(t *testing.T) { + type args struct { + imageTypeAliases []string + } + type wantResult struct { + imageTypeName string + } + tests := []struct { + name string + args args + want wantResult + }{ + { + name: "fedora-iot-commit aliases", + args: args{ + imageTypeAliases: []string{"iot-commit"}, + }, + want: wantResult{ + imageTypeName: "fedora-iot-commit", + }, + }, + + { + name: "fedora-iot-container aliases", + args: args{ + imageTypeAliases: []string{"iot-container"}, + }, + want: wantResult{ + imageTypeName: "fedora-iot-container", + }, + }, + + { + name: "fedora-iot-installer aliases", + args: args{ + imageTypeAliases: []string{"iot-installer"}, + }, + want: wantResult{ + imageTypeName: "fedora-iot-installer", + }, + }, + } + for _, dist := range fedoraFamilyDistros { + t.Run(dist.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dist := dist.distro + for _, archName := range dist.ListArches() { + t.Run(archName, func(t *testing.T) { + arch, err := dist.GetArch(archName) + require.Nilf(t, err, + "failed to get architecture '%s', previously listed as supported for the distro '%s'", + archName, dist.Name()) + // Test image type aliases only if the aliased image type is supported for the arch + if _, err = arch.GetImageType(tt.want.imageTypeName); err != nil { + t.Skipf("aliased image type '%s' is not supported for architecture '%s'", + tt.want.imageTypeName, archName) + } + for _, alias := range tt.args.imageTypeAliases { + t.Run(fmt.Sprintf("'%s' alias for image type '%s'", alias, tt.want.imageTypeName), + func(t *testing.T) { + gotImage, err := arch.GetImageType(alias) + require.Nilf(t, err, "arch.GetImageType() for image type alias '%s' failed: %v", + alias, err) + assert.Equalf(t, tt.want.imageTypeName, gotImage.Name(), + "got unexpected image type name for alias '%s'. got = %s, want = %s", + alias, tt.want.imageTypeName, gotImage.Name()) + }) + } + }) + } + }) + } + }) + } +} + +// Check that Manifest() function returns an error for unsupported +// configurations. +func TestDistro_ManifestError(t *testing.T) { + // Currently, the only unsupported configuration is OSTree commit types + // with Kernel boot options + fedoraDistro := fedora.NewF35() + bp := blueprint.Blueprint{ + Customizations: &blueprint.Customizations{ + Kernel: &blueprint.KernelCustomization{ + Append: "debug", + }, + }, + } + + for _, archName := range fedoraDistro.ListArches() { + arch, _ := fedoraDistro.GetArch(archName) + for _, imgTypeName := range arch.ListImageTypes() { + imgType, _ := arch.GetImageType(imgTypeName) + imgOpts := distro.ImageOptions{ + Size: imgType.Size(0), + } + testPackageSpecSets := distro_test_common.GetTestingPackageSpecSets("kernel", arch.Name(), imgType.PayloadPackageSets()) + _, err := imgType.Manifest(bp.Customizations, imgOpts, nil, testPackageSpecSets, 0) + if imgTypeName == "fedora-iot-commit" || imgTypeName == "fedora-iot-container" { + assert.EqualError(t, err, "kernel boot parameter customizations are not supported for ostree types") + } else if imgTypeName == "fedora-iot-installer" { + assert.EqualError(t, err, fmt.Sprintf("boot ISO image type \"%s\" requires specifying a URL from which to retrieve the OSTree commit", imgTypeName)) + } else { + assert.NoError(t, err) + } + } + } +} + +func TestArchitecture_ListImageTypes(t *testing.T) { + imgMap := []struct { + arch string + imgNames []string + fedoraAdditionalImageTypes []string + }{ + { + arch: "x86_64", + imgNames: []string{ + "qcow2", + "openstack", + "vhd", + "vmdk", + "ami", + "fedora-iot-commit", + "fedora-iot-container", + "fedora-iot-installer", + "oci", + }, + }, + { + arch: "aarch64", + imgNames: []string{ + "qcow2", + "openstack", + "ami", + "fedora-iot-commit", + "fedora-iot-container", + "fedora-iot-installer", + "oci", + }, + }, + } + + for _, dist := range fedoraFamilyDistros { + t.Run(dist.name, func(t *testing.T) { + for _, mapping := range imgMap { + arch, err := dist.distro.GetArch(mapping.arch) + require.NoError(t, err) + imageTypes := arch.ListImageTypes() + + var expectedImageTypes []string + expectedImageTypes = append(expectedImageTypes, mapping.imgNames...) + if dist.name == "fedora" { + expectedImageTypes = append(expectedImageTypes, mapping.fedoraAdditionalImageTypes...) + } + + require.ElementsMatch(t, expectedImageTypes, imageTypes) + } + }) + } +} + +func TestFedora_ListArches(t *testing.T) { + arches := fedora.NewF35().ListArches() + assert.Equal(t, []string{"aarch64", "s390x", "x86_64"}, arches) +} + +func TestFedora35_GetArch(t *testing.T) { + arches := []struct { + name string + errorExpected bool + errorExpectedInCentos bool + }{ + { + name: "x86_64", + }, + { + name: "aarch64", + }, + { + name: "s390x", + }, + { + name: "foo-arch", + errorExpected: true, + }, + } + + for _, dist := range fedoraFamilyDistros { + t.Run(dist.name, func(t *testing.T) { + for _, a := range arches { + actualArch, err := dist.distro.GetArch(a.name) + if a.errorExpected { + assert.Nil(t, actualArch) + assert.Error(t, err) + } else { + assert.Equal(t, a.name, actualArch.Name()) + assert.NoError(t, err) + } + } + }) + } +} + +func TestFedora35_Name(t *testing.T) { + distro := fedora.NewF35() + assert.Equal(t, "fedora-35", distro.Name()) +} + +func TestFedora35_KernelOption(t *testing.T) { + distro_test_common.TestDistro_KernelOption(t, fedora.NewF35()) +} + +func TestDistro_CustomFileSystemManifestError(t *testing.T) { + fedoraDistro := fedora.NewF35() + bp := blueprint.Blueprint{ + Customizations: &blueprint.Customizations{ + Filesystem: []blueprint.FilesystemCustomization{ + { + MinSize: 1024, + Mountpoint: "/boot", + }, + }, + }, + } + for _, archName := range fedoraDistro.ListArches() { + arch, _ := fedoraDistro.GetArch(archName) + for _, imgTypeName := range arch.ListImageTypes() { + imgType, _ := arch.GetImageType(imgTypeName) + _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, 0) + if imgTypeName == "fedora-iot-commit" || imgTypeName == "fedora-iot-container" { + assert.EqualError(t, err, "Custom mountpoints are not supported for ostree types") + } else if imgTypeName == "fedora-iot-installer" || imgTypeName == "edge-simplified-installer" || imgTypeName == "edge-raw-image" { + continue + } else { + assert.EqualError(t, err, "The following custom mountpoints are not supported [\"/boot\"]") + } + } + } +} + +func TestDistro_TestRootMountPoint(t *testing.T) { + fedoraDistro := fedora.NewF35() + bp := blueprint.Blueprint{ + Customizations: &blueprint.Customizations{ + Filesystem: []blueprint.FilesystemCustomization{ + { + MinSize: 1024, + Mountpoint: "/", + }, + }, + }, + } + for _, archName := range fedoraDistro.ListArches() { + arch, _ := fedoraDistro.GetArch(archName) + for _, imgTypeName := range arch.ListImageTypes() { + imgType, _ := arch.GetImageType(imgTypeName) + testPackageSpecSets := distro_test_common.GetTestingPackageSpecSets("kernel", arch.Name(), imgType.PayloadPackageSets()) + _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, testPackageSpecSets, 0) + if imgTypeName == "fedora-iot-commit" || imgTypeName == "fedora-iot-container" { + assert.EqualError(t, err, "Custom mountpoints are not supported for ostree types") + } else if imgTypeName == "fedora-iot-installer" { + continue + } else { + assert.NoError(t, err) + } + } + } +} + +func TestDistro_CustomFileSystemSubDirectories(t *testing.T) { + fedoraDistro := fedora.NewF35() + bp := blueprint.Blueprint{ + Customizations: &blueprint.Customizations{ + Filesystem: []blueprint.FilesystemCustomization{ + { + MinSize: 1024, + Mountpoint: "/var/log", + }, + { + MinSize: 1024, + Mountpoint: "/var/log/audit", + }, + }, + }, + } + for _, archName := range fedoraDistro.ListArches() { + arch, _ := fedoraDistro.GetArch(archName) + for _, imgTypeName := range arch.ListImageTypes() { + imgType, _ := arch.GetImageType(imgTypeName) + testPackageSpecSets := distro_test_common.GetTestingPackageSpecSets("kernel", arch.Name(), imgType.PayloadPackageSets()) + _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, testPackageSpecSets, 0) + if strings.HasPrefix(imgTypeName, "fedora-iot-") { + continue + } else { + assert.NoError(t, err) + } + } + } +} + +func TestDistro_MountpointsWithArbitraryDepthAllowed(t *testing.T) { + fedoraDistro := fedora.NewF35() + bp := blueprint.Blueprint{ + Customizations: &blueprint.Customizations{ + Filesystem: []blueprint.FilesystemCustomization{ + { + MinSize: 1024, + Mountpoint: "/var/a", + }, + { + MinSize: 1024, + Mountpoint: "/var/a/b", + }, + { + MinSize: 1024, + Mountpoint: "/var/a/b/c", + }, + { + MinSize: 1024, + Mountpoint: "/var/a/b/c/d", + }, + }, + }, + } + for _, archName := range fedoraDistro.ListArches() { + arch, _ := fedoraDistro.GetArch(archName) + for _, imgTypeName := range arch.ListImageTypes() { + imgType, _ := arch.GetImageType(imgTypeName) + testPackageSpecSets := distro_test_common.GetTestingPackageSpecSets("kernel", arch.Name(), imgType.PayloadPackageSets()) + _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, testPackageSpecSets, 0) + if strings.HasPrefix(imgTypeName, "fedora-iot-") { + continue + } else { + assert.NoError(t, err) + } + } + } +} + +func TestDistro_DirtyMountpointsNotAllowed(t *testing.T) { + fedoraDistro := fedora.NewF35() + bp := blueprint.Blueprint{ + Customizations: &blueprint.Customizations{ + Filesystem: []blueprint.FilesystemCustomization{ + { + MinSize: 1024, + Mountpoint: "//", + }, + { + MinSize: 1024, + Mountpoint: "/var//", + }, + { + MinSize: 1024, + Mountpoint: "/var//log/audit/", + }, + }, + }, + } + for _, archName := range fedoraDistro.ListArches() { + arch, _ := fedoraDistro.GetArch(archName) + for _, imgTypeName := range arch.ListImageTypes() { + imgType, _ := arch.GetImageType(imgTypeName) + _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, 0) + if strings.HasPrefix(imgTypeName, "fedora-iot-") { + continue + } else { + assert.EqualError(t, err, "The following custom mountpoints are not supported [\"//\" \"/var//\" \"/var//log/audit/\"]") + } + } + } +} + +func TestDistro_CustomFileSystemPatternMatching(t *testing.T) { + fedoraDistro := fedora.NewF35() + bp := blueprint.Blueprint{ + Customizations: &blueprint.Customizations{ + Filesystem: []blueprint.FilesystemCustomization{ + { + MinSize: 1024, + Mountpoint: "/variable", + }, + { + MinSize: 1024, + Mountpoint: "/variable/log/audit", + }, + }, + }, + } + for _, archName := range fedoraDistro.ListArches() { + arch, _ := fedoraDistro.GetArch(archName) + for _, imgTypeName := range arch.ListImageTypes() { + imgType, _ := arch.GetImageType(imgTypeName) + _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, 0) + if imgTypeName == "fedora-iot-commit" || imgTypeName == "fedora-iot-container" { + assert.EqualError(t, err, "Custom mountpoints are not supported for ostree types") + } else if imgTypeName == "fedora-iot-installer" { + continue + } else { + assert.EqualError(t, err, "The following custom mountpoints are not supported [\"/variable\" \"/variable/log/audit\"]") + } + } + } +} + +func TestDistro_CustomUsrPartitionNotLargeEnough(t *testing.T) { + fedoraDistro := fedora.NewF35() + bp := blueprint.Blueprint{ + Customizations: &blueprint.Customizations{ + Filesystem: []blueprint.FilesystemCustomization{ + { + MinSize: 1024, + Mountpoint: "/usr", + }, + }, + }, + } + for _, archName := range fedoraDistro.ListArches() { + arch, _ := fedoraDistro.GetArch(archName) + for _, imgTypeName := range arch.ListImageTypes() { + imgType, _ := arch.GetImageType(imgTypeName) + testPackageSpecSets := distro_test_common.GetTestingPackageSpecSets("kernel", arch.Name(), imgType.PayloadPackageSets()) + _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, testPackageSpecSets, 0) + if imgTypeName == "fedora-iot-commit" || imgTypeName == "fedora-iot-container" { + assert.EqualError(t, err, "Custom mountpoints are not supported for ostree types") + } else if imgTypeName == "fedora-iot-installer" { + continue + } else { + assert.NoError(t, err) + } + } + } +} diff --git a/internal/distro/fedora/package_sets.go b/internal/distro/fedora/package_sets.go new file mode 100644 index 000000000..81de3d74e --- /dev/null +++ b/internal/distro/fedora/package_sets.go @@ -0,0 +1,800 @@ +package fedora + +// This file defines package sets that are used by more than one image type. + +import ( + "fmt" + "strconv" + + "github.com/labstack/gommon/log" + "github.com/osbuild/osbuild-composer/internal/distro" + "github.com/osbuild/osbuild-composer/internal/rpmmd" +) + +// BUILD PACKAGE SETS + +// distro-wide build package set +func distroBuildPackageSet(t *imageType) rpmmd.PackageSet { + ps := rpmmd.PackageSet{ + Include: []string{ + "dnf", + "dosfstools", + "e2fsprogs", + "policycoreutils", + "qemu-img", + "selinux-policy-targeted", + "systemd", + "tar", + "xz", + }, + } + + switch t.Arch().Name() { + + case distro.X86_64ArchName: + ps = ps.Append(x8664BuildPackageSet(t)) + } + + return ps +} + +// x86_64 build package set +func x8664BuildPackageSet(t *imageType) rpmmd.PackageSet { + return rpmmd.PackageSet{ + Include: []string{ + "grub2-pc", + }, + } +} + +// common ec2 image build package set +func ec2BuildPackageSet(t *imageType) rpmmd.PackageSet { + return distroBuildPackageSet(t) +} + +func ec2CommonPackageSet(t *imageType) rpmmd.PackageSet { + return rpmmd.PackageSet{ + Include: []string{ + "@core", + "chrony", + "selinux-policy-targeted", + "langpacks-en", + "libxcrypt-compat", + "xfsprogs", + "cloud-init", + "checkpolicy", + "net-tools", + }, + Exclude: []string{ + "dracut-config-rescue", + "geolite2-city", + "geolite2-country", + "zram-generator-defaults", + }, + }.Append(bootPackageSet(t)) +} + +// common edge image build package set +func iotBuildPackageSet(t *imageType) rpmmd.PackageSet { + return distroBuildPackageSet(t).Append( + rpmmd.PackageSet{ + Include: []string{ + "rpm-ostree", + }, + }) +} + +// installer boot package sets, needed for booting and +// also in the build host + +func anacondaBootPackageSet(t *imageType) rpmmd.PackageSet { + ps := rpmmd.PackageSet{} + + grubCommon := rpmmd.PackageSet{ + Include: []string{ + "grub2-tools", + "grub2-tools-extra", + "grub2-tools-minimal", + }, + } + + efiCommon := rpmmd.PackageSet{ + Include: []string{ + "efibootmgr", + }, + } + + switch t.Arch().Name() { + case distro.X86_64ArchName: + ps = ps.Append(grubCommon) + ps = ps.Append(efiCommon) + ps = ps.Append(rpmmd.PackageSet{ + Include: []string{ + "grub2-efi-x64", + "grub2-efi-x64-cdboot", + "grub2-pc", + "grub2-pc-modules", + "shim-x64", + "syslinux", + "syslinux-nonlinux", + }, + }) + case distro.Aarch64ArchName: + ps = ps.Append(grubCommon) + ps = ps.Append(efiCommon) + ps = ps.Append(rpmmd.PackageSet{ + Include: []string{ + "grub2-efi-aa64-cdboot", + "grub2-efi-aa64", + "shim-aa64", + }, + }) + + default: + panic(fmt.Sprintf("unsupported arch: %s", t.Arch().Name())) + } + + return ps +} + +func installerBuildPackageSet(t *imageType) rpmmd.PackageSet { + return distroBuildPackageSet(t).Append( + rpmmd.PackageSet{ + Include: []string{ + "isomd5sum", + "xorriso", + }, + }) +} + +func anacondaBuildPackageSet(t *imageType) rpmmd.PackageSet { + ps := rpmmd.PackageSet{ + Include: []string{ + "squashfs-tools", + "lorax-templates-generic", + }, + } + + ps = ps.Append(installerBuildPackageSet(t)) + ps = ps.Append(anacondaBootPackageSet(t)) + + return ps +} + +func iotInstallerBuildPackageSet(t *imageType) rpmmd.PackageSet { + return anacondaBuildPackageSet(t).Append( + iotBuildPackageSet(t), + ) +} + +// BOOT PACKAGE SETS + +func bootPackageSet(t *imageType) rpmmd.PackageSet { + if !t.bootable { + return rpmmd.PackageSet{} + } + + var addLegacyBootPkg bool + var addUEFIBootPkg bool + + switch bt := t.getBootType(); bt { + case distro.LegacyBootType: + addLegacyBootPkg = true + case distro.UEFIBootType: + addUEFIBootPkg = true + case distro.HybridBootType: + addLegacyBootPkg = true + addUEFIBootPkg = true + default: + panic(fmt.Sprintf("unsupported boot type: %q", bt)) + } + + ps := rpmmd.PackageSet{} + + switch t.Arch().Name() { + case distro.X86_64ArchName: + if addLegacyBootPkg { + ps = ps.Append(x8664LegacyBootPackageSet(t)) + } + if addUEFIBootPkg { + ps = ps.Append(x8664UEFIBootPackageSet(t)) + } + + case distro.Aarch64ArchName: + ps = ps.Append(aarch64UEFIBootPackageSet(t)) + + default: + panic(fmt.Sprintf("unsupported boot arch: %s", t.Arch().Name())) + } + + return ps +} + +// x86_64 Legacy arch-specific boot package set +func x8664LegacyBootPackageSet(t *imageType) rpmmd.PackageSet { + return rpmmd.PackageSet{ + Include: []string{ + "dracut-config-generic", + "grub2-pc", + }, + } +} + +// x86_64 UEFI arch-specific boot package set +func x8664UEFIBootPackageSet(t *imageType) rpmmd.PackageSet { + return rpmmd.PackageSet{ + Include: []string{ + "dracut-config-generic", + "efibootmgr", + "grub2-efi-x64", + "shim-x64", + }, + } +} + +// aarch64 UEFI arch-specific boot package set +func aarch64UEFIBootPackageSet(t *imageType) rpmmd.PackageSet { + return rpmmd.PackageSet{ + Include: []string{ + "dracut-config-generic", + "efibootmgr", + "grub2-efi-aa64", + "grub2-tools", + "shim-aa64", + }, + } +} + +// We need to unpack the 'fedora cloud server' group because of the dependency collision with fedora-release-identity +// and the packages coming from the blueprint +func fedoraCloudServerPackageSet(t *imageType) rpmmd.PackageSet { + + ps := rpmmd.PackageSet{ + Include: []string{ + // Mandatory packages + "@core", + "cloud-init", + "cloud-utils-growpart", + "dracut-config-generic", + "grubby", + "rsync", + "tar", + + // Default packages + "console-login-helper-messages-issuegen", + "console-login-helper-messages-motdgen", + "console-login-helper-messages-profile", + }, + Exclude: []string{ + "grubby-deprecated", + "extlinux-bootloader", + }, + } + + r, err := strconv.Atoi(t.Arch().Distro().Releasever()) + if err != nil { + log.Errorf("failed to convert fedora release %s to string: %s", t.Arch().Distro().Releasever(), err) + } + + switch t.Arch().Name() { + case distro.X86_64ArchName: + ps = ps.Append(rpmmd.PackageSet{ + Include: []string{ + "syslinux", + "syslinux-extlinux", + "syslinux-extlinux-nonlinux", + "syslinux-nonlinux", + "mtools", + }, + }) + } + if r > 34 { + ps = ps.Append(rpmmd.PackageSet{ + Include: []string{ + "@Bootloader tools for Cloud images", + "libpng", + "graphite2", + "harfbuzz", + "freetype", + "grub2-tools-extra", + }, + }) + switch t.Arch().Name() { + case distro.X86_64ArchName: + ps = ps.Append(rpmmd.PackageSet{ + Include: []string{ + "efi-filesystem", + "efibootmgr", + "shim-ia32", + "shim-x64", + "grub2-efi-ia32", + "grub2-efi-x64", + "grub2-tools-efi", + "mokutil", + "efivar-libs", + "mtools", + }, + }) + } + } + + return ps +} + +func qcow2CommonPackageSet(t *imageType) rpmmd.PackageSet { + ps := rpmmd.PackageSet{ + Include: []string{ + "chrony", + "systemd-udev", + "selinux-policy-targeted", + "langpacks-en", + }, + Exclude: []string{ + "dracut-config-rescue", + "etables", + "firewalld", + "geolite2-city", + "geolite2-country", + "gobject-introspection", + "plymouth", + "zram-generator-defaults", + }, + }.Append(bootPackageSet(t)).Append(fedoraCloudServerPackageSet(t)) + + return ps +} + +func vhdCommonPackageSet(t *imageType) rpmmd.PackageSet { + ps := rpmmd.PackageSet{ + Include: []string{ + "@core", + "chrony", + "selinux-policy-targeted", + "langpacks-en", + "net-tools", + "ntfsprogs", + "WALinuxAgent", + "libxcrypt-compat", + "initscripts", + "glibc-all-langpacks", + }, + Exclude: []string{ + "dracut-config-rescue", + "geolite2-city", + "geolite2-country", + "zram-generator-defaults", + }, + }.Append(bootPackageSet(t)) + + return ps +} + +func vmdkCommonPackageSet(t *imageType) rpmmd.PackageSet { + ps := rpmmd.PackageSet{ + Include: []string{ + "chrony", + "systemd-udev", + "selinux-policy-targeted", + "langpacks-en", + }, + Exclude: []string{ + "dracut-config-rescue", + "etables", + "firewalld", + "geolite2-city", + "geolite2-country", + "gobject-introspection", + "plymouth", + "zram-generator-defaults", + }, + }.Append(bootPackageSet(t)).Append(fedoraCloudServerPackageSet(t)) + + return ps +} + +func openstackCommonPackageSet(t *imageType) rpmmd.PackageSet { + ps := rpmmd.PackageSet{ + Include: []string{ + "@core", + "chrony", + "selinux-policy-targeted", + "spice-vdagent", + "qemu-guest-agent", + "xen-libs", + "langpacks-en", + "cloud-init", + "libdrm", + }, + Exclude: []string{ + "dracut-config-rescue", + "geolite2-city", + "geolite2-country", + "zram-generator-defaults", + }, + }.Append(bootPackageSet(t)) + + return ps +} + +// fedora iot commit OS package set +func iotCommitPackageSet(t *imageType) rpmmd.PackageSet { + ps := rpmmd.PackageSet{ + Include: []string{ + "glibc", + "glibc-minimal-langpack", + "nss-altfiles", + "sssd-client", + "libsss_sudo", + "shadow-utils", + "dracut-config-generic", + "dracut-network", + "polkit", + "lvm2", + "cryptsetup", + "pinentry", + "keyutils", + "cracklib-dicts", + "e2fsprogs", + "xfsprogs", + "dosfstools", + "gnupg2", + "basesystem", + "python3", + "bash", + "xz", + "gzip", + "coreutils", + "which", + "curl", + "firewalld", + "iptables", + "NetworkManager", + "NetworkManager-wifi", + "NetworkManager-wwan", + "wpa_supplicant", + "iwd", + "tpm2-pkcs11", + "dnsmasq", + "traceroute", + "hostname", + "iproute", + "iputils", + "openssh-clients", + "openssh-server", + "passwd", + "policycoreutils", + "procps-ng", + "rootfiles", + "rpm", + "selinux-policy-targeted", + "setup", + "shadow-utils", + "sudo", + "systemd", + "util-linux", + "vim-minimal", + "less", + "tar", + "fwupd", + "usbguard", + "greenboot", + "greenboot-grub2", + "greenboot-rpm-ostree-grub2", + "greenboot-reboot", + "greenboot-status", + "ignition", + "zezere-ignition", + "rsync", + "attr", + "ima-evm-utils", + "bash-completion", + "tmux", + "screen", + "policycoreutils-python-utils", + "setools-console", + "audit", + "rng-tools", + "chrony", + "bluez", + "bluez-libs", + "bluez-mesh", + "kernel-tools", + "libgpiod-utils", + "podman", + "container-selinux", + "skopeo", + "criu", + "slirp4netns", + "fuse-overlayfs", + "clevis", + "clevis-dracut", + "clevis-luks", + "clevis-pin-tpm2", + "parsec", + "dbus-parsec", + "iwl7260-firmware", + "iwlax2xx-firmware", + }, + } + switch t.Arch().Name() { + case distro.X86_64ArchName: + ps = ps.Append(x8664IOTCommitPackageSet()) + + case distro.Aarch64ArchName: + ps = ps.Append(aarch64IOTCommitPackageSet()) + } + + return ps + +} + +func x8664IOTCommitPackageSet() rpmmd.PackageSet { + return rpmmd.PackageSet{ + Include: []string{ + "grub2", + "grub2-efi-x64", + "efibootmgr", + "shim-x64", + "microcode_ctl", + "iwl1000-firmware", + "iwl100-firmware", + "iwl105-firmware", + "iwl135-firmware", + "iwl2000-firmware", + "iwl2030-firmware", + "iwl3160-firmware", + "iwl5000-firmware", + "iwl5150-firmware", + "iwl6000-firmware", + "iwl6050-firmware", + }, + } +} + +func aarch64IOTCommitPackageSet() rpmmd.PackageSet { + return rpmmd.PackageSet{ + Include: []string{ + "grub2", + "grub2-efi-aa64", + "efibootmgr", + "shim-aa64", + "uboot-images-armv8", + "bcm283x-firmware", + "arm-image-installer"}, + } +} + +// INSTALLER PACKAGE SET + +func installerPackageSet(t *imageType) rpmmd.PackageSet { + ps := rpmmd.PackageSet{ + Include: []string{ + "anaconda-dracut", + "curl", + "dracut-config-generic", + "dracut-network", + "hostname", + "iwl100-firmware", + "iwl1000-firmware", + "iwl105-firmware", + "iwl135-firmware", + "iwl2000-firmware", + "iwl2030-firmware", + "iwl3160-firmware", + "iwl5000-firmware", + "iwl5150-firmware", + "iwl6050-firmware", + "iwl7260-firmware", + "kernel", + "less", + "nfs-utils", + "openssh-clients", + "ostree", + "plymouth", + "rng-tools", + "rpcbind", + "selinux-policy-targeted", + "systemd", + "tar", + "xfsprogs", + "xz", + }, + } + + switch t.Arch().Name() { + case distro.X86_64ArchName: + ps = ps.Append(rpmmd.PackageSet{ + Include: []string{ + "biosdevname", + }, + }) + } + + return ps +} + +func anacondaPackageSet(t *imageType) rpmmd.PackageSet { + + // common installer packages + ps := installerPackageSet(t) + + ps = ps.Append(rpmmd.PackageSet{ + Include: []string{ + "aajohan-comfortaa-fonts", + "abattis-cantarell-fonts", + "alsa-firmware", + "alsa-tools-firmware", + "anaconda", + "anaconda-dracut", + "anaconda-install-env-deps", + "anaconda-widgets", + "audit", + "bind-utils", + "bitmap-fangsongti-fonts", + "bzip2", + "cryptsetup", + "curl", + "dbus-x11", + "dejavu-sans-fonts", + "dejavu-sans-mono-fonts", + "device-mapper-persistent-data", + "dmidecode", + "dnf", + "dracut-config-generic", + "dracut-network", + "efibootmgr", + "ethtool", + "fcoe-utils", + "ftp", + "gdb-gdbserver", + "gdisk", + "glibc-all-langpacks", + "gnome-kiosk", + "google-noto-sans-cjk-ttc-fonts", + "grub2-tools", + "grub2-tools-extra", + "grub2-tools-minimal", + "grubby", + "gsettings-desktop-schemas", + "hdparm", + "hexedit", + "hostname", + "initscripts", + "ipmitool", + "iwl1000-firmware", + "iwl100-firmware", + "iwl105-firmware", + "iwl135-firmware", + "iwl2000-firmware", + "iwl2030-firmware", + "iwl3160-firmware", + "iwl5000-firmware", + "iwl5150-firmware", + "iwl6000g2a-firmware", + "iwl6000g2b-firmware", + "iwl6050-firmware", + "iwl7260-firmware", + "jomolhari-fonts", + "kacst-farsi-fonts", + "kacst-qurn-fonts", + "kbd", + "kbd-misc", + "kdump-anaconda-addon", + "kernel", + "khmeros-base-fonts", + "less", + "libblockdev-lvm-dbus", + "libibverbs", + "libreport-plugin-bugzilla", + "libreport-plugin-reportuploader", + "librsvg2", + "linux-firmware", + "lklug-fonts", + "lldpad", + "lohit-assamese-fonts", + "lohit-bengali-fonts", + "lohit-devanagari-fonts", + "lohit-gujarati-fonts", + "lohit-gurmukhi-fonts", + "lohit-kannada-fonts", + "lohit-odia-fonts", + "lohit-tamil-fonts", + "lohit-telugu-fonts", + "lsof", + "madan-fonts", + "mtr", + "mt-st", + "net-tools", + "nfs-utils", + "nmap-ncat", + "nm-connection-editor", + "nss-tools", + "openssh-clients", + "openssh-server", + "oscap-anaconda-addon", + "ostree", + "pciutils", + "perl-interpreter", + "pigz", + "plymouth", + "python3-pyatspi", + "rdma-core", + "rng-tools", + "rpcbind", + "rpm-ostree", + "rsync", + "rsyslog", + "selinux-policy-targeted", + "sg3_utils", + "sil-abyssinica-fonts", + "sil-padauk-fonts", + "sil-scheherazade-fonts", + "smartmontools", + "smc-meera-fonts", + "spice-vdagent", + "strace", + "systemd", + "tar", + "thai-scalable-waree-fonts", + "tigervnc-server-minimal", + "tigervnc-server-module", + "udisks2", + "udisks2-iscsi", + "usbutils", + "vim-minimal", + "volume_key", + "wget", + "xfsdump", + "xfsprogs", + "xorg-x11-drivers", + "xorg-x11-fonts-misc", + "xorg-x11-server-Xorg", + "xorg-x11-xauth", + "metacity", + "xrdb", + "xz", + }, + }) + + ps = ps.Append(anacondaBootPackageSet(t)) + + r, err := strconv.Atoi(t.Arch().Distro().Releasever()) + if err != nil { + log.Errorf("failed to convert fedora release %s to string: %s", t.Arch().Distro().Releasever(), err) + } + + if r <= 34 { + ps = ps.Append(rpmmd.PackageSet{ + Include: []string{ + "xorg-x11-server-utils", + }, + }) + } + switch t.Arch().Name() { + case distro.X86_64ArchName: + ps = ps.Append(rpmmd.PackageSet{ + Include: []string{ + "biosdevname", + "dmidecode", + "grub2-tools-efi", + "memtest86+", + }, + }) + + case distro.Aarch64ArchName: + ps = ps.Append(rpmmd.PackageSet{ + Include: []string{ + "dmidecode", + }, + }) + + default: + panic(fmt.Sprintf("unsupported arch: %s", t.Arch().Name())) + } + + return ps +} + +func iotInstallerPackageSet(t *imageType) rpmmd.PackageSet { + return anacondaPackageSet(t) +} diff --git a/internal/distro/fedora/partition_tables.go b/internal/distro/fedora/partition_tables.go new file mode 100644 index 000000000..2575fe0ce --- /dev/null +++ b/internal/distro/fedora/partition_tables.go @@ -0,0 +1,107 @@ +package fedora + +import ( + "github.com/osbuild/osbuild-composer/internal/disk" + "github.com/osbuild/osbuild-composer/internal/distro" +) + +var defaultBasePartitionTables = distro.BasePartitionTableMap{ + distro.X86_64ArchName: disk.PartitionTable{ + UUID: "D209C89E-EA5E-4FBD-B161-B461CCE297E0", + Type: "gpt", + Partitions: []disk.Partition{ + { + Size: 1048576, // 1MB + Bootable: true, + Type: disk.BIOSBootPartitionGUID, + UUID: disk.BIOSBootPartitionUUID, + }, + { + Size: 209715200, // 200 MB + Type: disk.EFISystemPartitionGUID, + UUID: disk.EFISystemPartitionUUID, + Payload: &disk.Filesystem{ + Type: "vfat", + UUID: disk.EFIFilesystemUUID, + Mountpoint: "/boot/efi", + Label: "EFI-SYSTEM", + FSTabOptions: "defaults,uid=0,gid=0,umask=077,shortname=winnt", + FSTabFreq: 0, + FSTabPassNo: 2, + }, + }, + { + Size: 524288000, // 500 MB + Type: disk.FilesystemDataGUID, + UUID: disk.FilesystemDataUUID, + Payload: &disk.Filesystem{ + Type: "ext4", + Mountpoint: "/boot", + Label: "boot", + FSTabOptions: "defaults", + FSTabFreq: 0, + FSTabPassNo: 0, + }, + }, + { + Size: 2147483648, // 2GiB + Type: disk.FilesystemDataGUID, + UUID: disk.RootPartitionUUID, + Payload: &disk.Filesystem{ + Type: "ext4", + Label: "root", + Mountpoint: "/", + FSTabOptions: "defaults", + FSTabFreq: 0, + FSTabPassNo: 0, + }, + }, + }, + }, + distro.Aarch64ArchName: disk.PartitionTable{ + UUID: "D209C89E-EA5E-4FBD-B161-B461CCE297E0", + Type: "gpt", + Partitions: []disk.Partition{ + { + Size: 209715200, // 200 MB + Type: disk.EFISystemPartitionGUID, + UUID: disk.EFISystemPartitionUUID, + Payload: &disk.Filesystem{ + Type: "vfat", + UUID: disk.EFIFilesystemUUID, + Mountpoint: "/boot/efi", + Label: "EFI-SYSTEM", + FSTabOptions: "defaults,uid=0,gid=0,umask=077,shortname=winnt", + FSTabFreq: 0, + FSTabPassNo: 2, + }, + }, + { + Size: 524288000, // 500 MB + Type: disk.FilesystemDataGUID, + UUID: disk.FilesystemDataUUID, + Payload: &disk.Filesystem{ + Type: "ext4", + Mountpoint: "/boot", + Label: "boot", + FSTabOptions: "defaults", + FSTabFreq: 0, + FSTabPassNo: 0, + }, + }, + { + Size: 2147483648, // 2GiB + Type: disk.FilesystemDataGUID, + UUID: disk.RootPartitionUUID, + Payload: &disk.Filesystem{ + Type: "ext4", + Label: "root", + Mountpoint: "/", + FSTabOptions: "defaults", + FSTabFreq: 0, + FSTabPassNo: 0, + }, + }, + }, + }, +} diff --git a/internal/distro/fedora/pipelines.go b/internal/distro/fedora/pipelines.go new file mode 100644 index 000000000..bda1acc2f --- /dev/null +++ b/internal/distro/fedora/pipelines.go @@ -0,0 +1,679 @@ +package fedora + +import ( + "fmt" + "math/rand" + "path" + "path/filepath" + "strings" + + "github.com/osbuild/osbuild-composer/internal/blueprint" + "github.com/osbuild/osbuild-composer/internal/common" + "github.com/osbuild/osbuild-composer/internal/disk" + "github.com/osbuild/osbuild-composer/internal/distro" + osbuild "github.com/osbuild/osbuild-composer/internal/osbuild2" + "github.com/osbuild/osbuild-composer/internal/rpmmd" +) + +func qcow2Pipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]osbuild.Pipeline, error) { + pipelines := make([]osbuild.Pipeline, 0) + pipelines = append(pipelines, *buildPipeline(repos, packageSetSpecs[buildPkgsKey], t.arch.distro.runner)) + + partitionTable, err := t.getPartitionTable(customizations.GetFilesystems(), options, rng) + if err != nil { + return nil, err + } + + treePipeline, err := osPipeline(t, repos, packageSetSpecs[osPkgsKey], packageSetSpecs[blueprintPkgsKey], customizations, options, partitionTable) + if err != nil { + return nil, err + } + pipelines = append(pipelines, *treePipeline) + + diskfile := "disk.img" + kernelVer := rpmmd.GetVerStrFromPackageSpecListPanic(packageSetSpecs[blueprintPkgsKey], customizations.GetKernel().Name) + imagePipeline := liveImagePipeline(treePipeline.Name, diskfile, partitionTable, t.arch, kernelVer) + pipelines = append(pipelines, *imagePipeline) + + qemuPipeline := qemuPipeline(imagePipeline.Name, diskfile, t.filename, osbuild.QEMUFormatQCOW2, osbuild.QCOW2Options{Compat: "1.1"}) + pipelines = append(pipelines, *qemuPipeline) + + return pipelines, nil +} + +func prependKernelCmdlineStage(pipeline *osbuild.Pipeline, kernelOptions string, pt *disk.PartitionTable) *osbuild.Pipeline { + rootFs := pt.FindMountable("/") + if rootFs == nil { + panic("root filesystem must be defined for kernel-cmdline stage, this is a programming error") + } + rootFsUUID := rootFs.GetFSSpec().UUID + kernelStage := osbuild.NewKernelCmdlineStage(osbuild.NewKernelCmdlineStageOptions(rootFsUUID, kernelOptions)) + pipeline.Stages = append([]*osbuild.Stage{kernelStage}, pipeline.Stages...) + return pipeline +} + +func vhdPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]osbuild.Pipeline, error) { + pipelines := make([]osbuild.Pipeline, 0) + pipelines = append(pipelines, *buildPipeline(repos, packageSetSpecs[buildPkgsKey], t.arch.distro.runner)) + + partitionTable, err := t.getPartitionTable(customizations.GetFilesystems(), options, rng) + if err != nil { + return nil, err + } + + treePipeline, err := osPipeline(t, repos, packageSetSpecs[osPkgsKey], packageSetSpecs[blueprintPkgsKey], customizations, options, partitionTable) + if err != nil { + return nil, err + } + pipelines = append(pipelines, *treePipeline) + + diskfile := "disk.img" + kernelVer := rpmmd.GetVerStrFromPackageSpecListPanic(packageSetSpecs[blueprintPkgsKey], customizations.GetKernel().Name) + imagePipeline := liveImagePipeline(treePipeline.Name, diskfile, partitionTable, t.arch, kernelVer) + pipelines = append(pipelines, *imagePipeline) + + qemuPipeline := qemuPipeline(imagePipeline.Name, diskfile, t.filename, osbuild.QEMUFormatVPC, nil) + pipelines = append(pipelines, *qemuPipeline) + return pipelines, nil +} + +func vmdkPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]osbuild.Pipeline, error) { + pipelines := make([]osbuild.Pipeline, 0) + pipelines = append(pipelines, *buildPipeline(repos, packageSetSpecs[buildPkgsKey], t.arch.distro.runner)) + + partitionTable, err := t.getPartitionTable(customizations.GetFilesystems(), options, rng) + if err != nil { + return nil, err + } + + treePipeline, err := osPipeline(t, repos, packageSetSpecs[osPkgsKey], packageSetSpecs[blueprintPkgsKey], customizations, options, partitionTable) + if err != nil { + return nil, err + } + pipelines = append(pipelines, *treePipeline) + + diskfile := "disk.img" + kernelVer := rpmmd.GetVerStrFromPackageSpecListPanic(packageSetSpecs[blueprintPkgsKey], customizations.GetKernel().Name) + imagePipeline := liveImagePipeline(treePipeline.Name, diskfile, partitionTable, t.arch, kernelVer) + pipelines = append(pipelines, *imagePipeline) + + qemuPipeline := qemuPipeline(imagePipeline.Name, diskfile, t.filename, osbuild.QEMUFormatVMDK, nil) + pipelines = append(pipelines, *qemuPipeline) + return pipelines, nil +} + +func openstackPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]osbuild.Pipeline, error) { + pipelines := make([]osbuild.Pipeline, 0) + pipelines = append(pipelines, *buildPipeline(repos, packageSetSpecs[buildPkgsKey], t.arch.distro.runner)) + + partitionTable, err := t.getPartitionTable(customizations.GetFilesystems(), options, rng) + if err != nil { + return nil, err + } + + treePipeline, err := osPipeline(t, repos, packageSetSpecs[osPkgsKey], packageSetSpecs[blueprintPkgsKey], customizations, options, partitionTable) + if err != nil { + return nil, err + } + pipelines = append(pipelines, *treePipeline) + + diskfile := "disk.img" + kernelVer := rpmmd.GetVerStrFromPackageSpecListPanic(packageSetSpecs[blueprintPkgsKey], customizations.GetKernel().Name) + imagePipeline := liveImagePipeline(treePipeline.Name, diskfile, partitionTable, t.arch, kernelVer) + pipelines = append(pipelines, *imagePipeline) + + qemuPipeline := qemuPipeline(imagePipeline.Name, diskfile, t.filename, osbuild.QEMUFormatQCOW2, nil) + pipelines = append(pipelines, *qemuPipeline) + return pipelines, nil +} + +func ec2CommonPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, + repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, + rng *rand.Rand, diskfile string) ([]osbuild.Pipeline, error) { + pipelines := make([]osbuild.Pipeline, 0) + pipelines = append(pipelines, *buildPipeline(repos, packageSetSpecs[buildPkgsKey], t.arch.distro.runner)) + + partitionTable, err := t.getPartitionTable(customizations.GetFilesystems(), options, rng) + if err != nil { + return nil, err + } + + treePipeline, err := osPipeline(t, repos, packageSetSpecs[osPkgsKey], packageSetSpecs[blueprintPkgsKey], customizations, options, partitionTable) + if err != nil { + return nil, err + } + pipelines = append(pipelines, *treePipeline) + + kernelVer := rpmmd.GetVerStrFromPackageSpecListPanic(packageSetSpecs[blueprintPkgsKey], customizations.GetKernel().Name) + imagePipeline := liveImagePipeline(treePipeline.Name, diskfile, partitionTable, t.arch, kernelVer) + pipelines = append(pipelines, *imagePipeline) + return pipelines, nil +} + +// ec2Pipelines returns pipelines which produce uncompressed EC2 images which are expected to use RHSM for content +func ec2Pipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]osbuild.Pipeline, error) { + return ec2CommonPipelines(t, customizations, options, repos, packageSetSpecs, rng, t.Filename()) +} + +//makeISORootPath return a path that can be used to address files and folders in +//the root of the iso +func makeISORootPath(p string) string { + fullpath := path.Join("/run/install/repo", p) + return fmt.Sprintf("file://%s", fullpath) +} + +func iotInstallerPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]osbuild.Pipeline, error) { + pipelines := make([]osbuild.Pipeline, 0) + pipelines = append(pipelines, *buildPipeline(repos, packageSetSpecs[buildPkgsKey], t.arch.distro.runner)) + installerPackages := packageSetSpecs[installerPkgsKey] + d := t.arch.distro + archName := t.Arch().Name() + kernelVer := rpmmd.GetVerStrFromPackageSpecListPanic(installerPackages, "kernel") + ostreeRepoPath := "/ostree/repo" + payloadStages := ostreePayloadStages(options, ostreeRepoPath) + kickstartOptions := ostreeKickstartStageOptions(makeISORootPath(ostreeRepoPath), options.OSTree.Ref) + pipelines = append(pipelines, *anacondaTreePipeline(repos, installerPackages, kernelVer, archName, d.product, d.osVersion, "iot")) + isolabel := fmt.Sprintf(d.isolabelTmpl, archName) + pipelines = append(pipelines, *bootISOTreePipeline(kernelVer, archName, d.vendor, d.product, d.osVersion, isolabel, kickstartOptions, payloadStages)) + pipelines = append(pipelines, *bootISOPipeline(t.Filename(), d.isolabelTmpl, archName, false)) + return pipelines, nil +} + +func iotCorePipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec) ([]osbuild.Pipeline, error) { + pipelines := make([]osbuild.Pipeline, 0) + pipelines = append(pipelines, *buildPipeline(repos, packageSetSpecs[buildPkgsKey], t.arch.distro.runner)) + + treePipeline, err := osPipeline(t, repos, packageSetSpecs[osPkgsKey], packageSetSpecs[blueprintPkgsKey], customizations, options, nil) + if err != nil { + return nil, err + } + + pipelines = append(pipelines, *treePipeline) + pipelines = append(pipelines, *ostreeCommitPipeline(options, t.arch.distro.osVersion)) + + return pipelines, nil +} + +func edgeCommitPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]osbuild.Pipeline, error) { + pipelines, err := iotCorePipelines(t, customizations, options, repos, packageSetSpecs) + if err != nil { + return nil, err + } + tarPipeline := osbuild.Pipeline{ + Name: "commit-archive", + Build: "name:build", + } + tarPipeline.AddStage(tarStage("ostree-commit", t.Filename())) + pipelines = append(pipelines, tarPipeline) + return pipelines, nil +} + +func iotContainerPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]osbuild.Pipeline, error) { + pipelines, err := iotCorePipelines(t, customizations, options, repos, packageSetSpecs) + if err != nil { + return nil, err + } + + nginxConfigPath := "/etc/nginx.conf" + httpPort := "8080" + pipelines = append(pipelines, *containerTreePipeline(repos, packageSetSpecs[containerPkgsKey], options, customizations, nginxConfigPath, httpPort)) + pipelines = append(pipelines, *containerPipeline(t, nginxConfigPath, httpPort)) + return pipelines, nil +} + +func buildPipeline(repos []rpmmd.RepoConfig, buildPackageSpecs []rpmmd.PackageSpec, runner string) *osbuild.Pipeline { + p := new(osbuild.Pipeline) + p.Name = "build" + p.Runner = runner + p.AddStage(osbuild.NewRPMStage(rpmStageOptions(repos), osbuild.NewRpmStageSourceFilesInputs(buildPackageSpecs))) + p.AddStage(osbuild.NewSELinuxStage(selinuxStageOptions(true))) + return p +} + +func osPipeline(t *imageType, + repos []rpmmd.RepoConfig, + packages []rpmmd.PackageSpec, + bpPackages []rpmmd.PackageSpec, + c *blueprint.Customizations, + options distro.ImageOptions, + pt *disk.PartitionTable) (*osbuild.Pipeline, error) { + imageConfig := t.getDefaultImageConfig() + p := new(osbuild.Pipeline) + if t.rpmOstree { + p.Name = "ostree-tree" + } else { + p.Name = "os" + } + p.Build = "name:build" + packages = append(packages, bpPackages...) + + if t.rpmOstree && options.OSTree.Parent != "" && options.OSTree.URL != "" { + p.AddStage(osbuild.NewOSTreePasswdStage("org.osbuild.source", options.OSTree.Parent)) + } + + rpmOptions := rpmStageOptions(repos) + p.AddStage(osbuild.NewRPMStage(rpmOptions, osbuild.NewRpmStageSourceFilesInputs(packages))) + + // If the /boot is on a separate partition, the prefix for the BLS stage must be "" + if pt == nil || pt.FindMountable("/boot") == nil { + p.AddStage(osbuild.NewFixBLSStage(&osbuild.FixBLSStageOptions{})) + } else { + p.AddStage(osbuild.NewFixBLSStage(&osbuild.FixBLSStageOptions{Prefix: common.StringToPtr("")})) + } + + language, keyboard := c.GetPrimaryLocale() + if language != nil { + p.AddStage(osbuild.NewLocaleStage(&osbuild.LocaleStageOptions{Language: *language})) + } else { + p.AddStage(osbuild.NewLocaleStage(&osbuild.LocaleStageOptions{Language: imageConfig.Locale})) + } + if keyboard != nil { + p.AddStage(osbuild.NewKeymapStage(&osbuild.KeymapStageOptions{Keymap: *keyboard})) + } else if imageConfig.Keyboard != nil { + p.AddStage(osbuild.NewKeymapStage(imageConfig.Keyboard)) + } + + if hostname := c.GetHostname(); hostname != nil { + p.AddStage(osbuild.NewHostnameStage(&osbuild.HostnameStageOptions{Hostname: *hostname})) + } else { + p.AddStage(osbuild.NewHostnameStage(&osbuild.HostnameStageOptions{Hostname: "localhost.localdomain"})) + } + + timezone, ntpServers := c.GetTimezoneSettings() + if timezone != nil { + p.AddStage(osbuild.NewTimezoneStage(&osbuild.TimezoneStageOptions{Zone: *timezone})) + } else { + p.AddStage(osbuild.NewTimezoneStage(&osbuild.TimezoneStageOptions{Zone: imageConfig.Timezone})) + } + + if len(ntpServers) > 0 { + p.AddStage(osbuild.NewChronyStage(&osbuild.ChronyStageOptions{Timeservers: ntpServers})) + } else if imageConfig.TimeSynchronization != nil { + p.AddStage(osbuild.NewChronyStage(imageConfig.TimeSynchronization)) + } + + if groups := c.GetGroups(); len(groups) > 0 { + p.AddStage(osbuild.NewGroupsStage(osbuild.NewGroupsStageOptions(groups))) + } + + if users := c.GetUsers(); len(users) > 0 { + userOptions, err := userStageOptions(users) + if err != nil { + return nil, err + } + if t.rpmOstree { + // for ostree, writing the key during user creation is redundant + // and can cause issues so create users without keys and write them + // on first boot + userOptionsSansKeys := new(osbuild.UsersStageOptions) + userOptionsSansKeys.Users = make(map[string]osbuild.UsersStageOptionsUser, len(userOptions.Users)) + for name, options := range userOptions.Users { + userOptionsSansKeys.Users[name] = osbuild.UsersStageOptionsUser{ + UID: options.UID, + GID: options.GID, + Groups: options.Groups, + Description: options.Description, + Home: options.Home, + Shell: options.Shell, + Password: options.Password, + Key: nil, + } + } + p.AddStage(osbuild.NewUsersStage(userOptionsSansKeys)) + p.AddStage(osbuild.NewFirstBootStage(usersFirstBootOptions(userOptions))) + } else { + p.AddStage(osbuild.NewUsersStage(userOptions)) + } + } + + if services := c.GetServices(); services != nil || imageConfig.EnabledServices != nil || + imageConfig.DisabledServices != nil || imageConfig.DefaultTarget != "" { + p.AddStage(osbuild.NewSystemdStage(systemdStageOptions( + imageConfig.EnabledServices, + imageConfig.DisabledServices, + services, + imageConfig.DefaultTarget, + ))) + } + + if firewall := c.GetFirewall(); firewall != nil { + p.AddStage(osbuild.NewFirewallStage(firewallStageOptions(firewall))) + } + + for _, sysconfigConfig := range imageConfig.Sysconfig { + p.AddStage(osbuild.NewSysconfigStage(sysconfigConfig)) + } + + for _, systemdLogindConfig := range imageConfig.SystemdLogind { + p.AddStage(osbuild.NewSystemdLogindStage(systemdLogindConfig)) + } + + for _, cloudInitConfig := range imageConfig.CloudInit { + p.AddStage(osbuild.NewCloudInitStage(cloudInitConfig)) + } + + for _, modprobeConfig := range imageConfig.Modprobe { + p.AddStage(osbuild.NewModprobeStage(modprobeConfig)) + } + + for _, dracutConfConfig := range imageConfig.DracutConf { + p.AddStage(osbuild.NewDracutConfStage(dracutConfConfig)) + } + + for _, systemdUnitConfig := range imageConfig.SystemdUnit { + p.AddStage(osbuild.NewSystemdUnitStage(systemdUnitConfig)) + } + + if authselectConfig := imageConfig.Authselect; authselectConfig != nil { + p.AddStage(osbuild.NewAuthselectStage(authselectConfig)) + } + + if seLinuxConfig := imageConfig.SELinuxConfig; seLinuxConfig != nil { + p.AddStage(osbuild.NewSELinuxConfigStage(seLinuxConfig)) + } + + if tunedConfig := imageConfig.Tuned; tunedConfig != nil { + p.AddStage(osbuild.NewTunedStage(tunedConfig)) + } + + for _, tmpfilesdConfig := range imageConfig.Tmpfilesd { + p.AddStage(osbuild.NewTmpfilesdStage(tmpfilesdConfig)) + } + + for _, pamLimitsConfConfig := range imageConfig.PamLimitsConf { + p.AddStage(osbuild.NewPamLimitsConfStage(pamLimitsConfConfig)) + } + + for _, sysctldConfig := range imageConfig.Sysctld { + p.AddStage(osbuild.NewSysctldStage(sysctldConfig)) + } + + for _, dnfConfig := range imageConfig.DNFConfig { + p.AddStage(osbuild.NewDNFConfigStage(dnfConfig)) + } + + if sshdConfig := imageConfig.SshdConfig; sshdConfig != nil { + p.AddStage((osbuild.NewSshdConfigStage(sshdConfig))) + } + + if authConfig := imageConfig.Authconfig; authConfig != nil { + p.AddStage(osbuild.NewAuthconfigStage(authConfig)) + } + + if pwQuality := imageConfig.PwQuality; pwQuality != nil { + p.AddStage(osbuild.NewPwqualityConfStage(pwQuality)) + } + + if waConfig := imageConfig.WAAgentConfig; waConfig != nil { + p.AddStage(osbuild.NewWAAgentConfStage(waConfig)) + } + + if pt != nil { + kernelOptions := osbuild.GenImageKernelOptions(pt) + if t.kernelOptions != "" { + kernelOptions = append(kernelOptions, t.kernelOptions) + } + if bpKernel := c.GetKernel(); bpKernel.Append != "" { + kernelOptions = append(kernelOptions, bpKernel.Append) + } + p = prependKernelCmdlineStage(p, strings.Join(kernelOptions, " "), pt) + p.AddStage(osbuild.NewFSTabStage(osbuild.NewFSTabStageOptions(pt))) + kernelVer := rpmmd.GetVerStrFromPackageSpecListPanic(bpPackages, c.GetKernel().Name) + bootloader := bootloaderConfigStage(t, *pt, kernelVer, false, false) + + if cfg := imageConfig.Grub2Config; cfg != nil { + if grub2, ok := bootloader.Options.(*osbuild.GRUB2StageOptions); ok { + grub2.Config = cfg + } + } + + p.AddStage(bootloader) + } + + p.AddStage(osbuild.NewSELinuxStage(selinuxStageOptions(false))) + + if t.rpmOstree { + p.AddStage(osbuild.NewOSTreePrepTreeStage(&osbuild.OSTreePrepTreeStageOptions{ + EtcGroupMembers: []string{ + // NOTE: We may want to make this configurable. + "wheel", "docker", + }, + })) + } + + return p, nil +} + +func ostreeCommitPipeline(options distro.ImageOptions, osVersion string) *osbuild.Pipeline { + p := new(osbuild.Pipeline) + p.Name = "ostree-commit" + p.Build = "name:build" + p.AddStage(osbuild.NewOSTreeInitStage(&osbuild.OSTreeInitStageOptions{Path: "/repo"})) + + commitStageInput := new(osbuild.OSTreeCommitStageInput) + commitStageInput.Type = "org.osbuild.tree" + commitStageInput.Origin = "org.osbuild.pipeline" + commitStageInput.References = osbuild.OSTreeCommitStageReferences{"name:ostree-tree"} + + p.AddStage(osbuild.NewOSTreeCommitStage( + &osbuild.OSTreeCommitStageOptions{ + Ref: options.OSTree.Ref, + OSVersion: osVersion, + Parent: options.OSTree.Parent, + }, + &osbuild.OSTreeCommitStageInputs{Tree: commitStageInput}), + ) + return p +} + +func tarStage(source, filename string) *osbuild.Stage { + tree := new(osbuild.TarStageInput) + tree.Type = "org.osbuild.tree" + tree.Origin = "org.osbuild.pipeline" + tree.References = []string{"name:" + source} + return osbuild.NewTarStage(&osbuild.TarStageOptions{Filename: filename}, &osbuild.TarStageInputs{Tree: tree}) +} + +func containerTreePipeline(repos []rpmmd.RepoConfig, packages []rpmmd.PackageSpec, options distro.ImageOptions, c *blueprint.Customizations, nginxConfigPath, listenPort string) *osbuild.Pipeline { + p := new(osbuild.Pipeline) + p.Name = "container-tree" + p.Build = "name:build" + p.AddStage(osbuild.NewRPMStage(rpmStageOptions(repos), osbuild.NewRpmStageSourceFilesInputs(packages))) + language, _ := c.GetPrimaryLocale() + if language != nil { + p.AddStage(osbuild.NewLocaleStage(&osbuild.LocaleStageOptions{Language: *language})) + } else { + p.AddStage(osbuild.NewLocaleStage(&osbuild.LocaleStageOptions{Language: "en_US"})) + } + + htmlRoot := "/usr/share/nginx/html" + repoPath := filepath.Join(htmlRoot, "repo") + p.AddStage(osbuild.NewOSTreeInitStage(&osbuild.OSTreeInitStageOptions{Path: repoPath})) + + p.AddStage(osbuild.NewOSTreePullStage( + &osbuild.OSTreePullStageOptions{Repo: repoPath}, + osbuild.NewOstreePullStageInputs("org.osbuild.pipeline", "name:ostree-commit", options.OSTree.Ref), + )) + + // make nginx log and lib directories world writeable, otherwise nginx can't start in + // an unprivileged container + p.AddStage(osbuild.NewChmodStage(chmodStageOptions("/var/log/nginx", "a+rwX", true))) + p.AddStage(osbuild.NewChmodStage(chmodStageOptions("/var/lib/nginx", "a+rwX", true))) + + p.AddStage(osbuild.NewNginxConfigStage(nginxConfigStageOptions(nginxConfigPath, htmlRoot, listenPort))) + return p +} + +func containerPipeline(t *imageType, nginxConfigPath, listenPort string) *osbuild.Pipeline { + p := new(osbuild.Pipeline) + p.Name = "container" + p.Build = "name:build" + options := &osbuild.OCIArchiveStageOptions{ + Architecture: t.Arch().Name(), + Filename: t.Filename(), + Config: &osbuild.OCIArchiveConfig{ + Cmd: []string{"nginx", "-c", nginxConfigPath}, + ExposedPorts: []string{listenPort}, + }, + } + baseInput := new(osbuild.OCIArchiveStageInput) + baseInput.Type = "org.osbuild.tree" + baseInput.Origin = "org.osbuild.pipeline" + baseInput.References = []string{"name:container-tree"} + inputs := &osbuild.OCIArchiveStageInputs{Base: baseInput} + p.AddStage(osbuild.NewOCIArchiveStage(options, inputs)) + return p +} + +func ostreePayloadStages(options distro.ImageOptions, ostreeRepoPath string) []*osbuild.Stage { + stages := make([]*osbuild.Stage, 0) + + // ostree commit payload + stages = append(stages, osbuild.NewOSTreeInitStage(&osbuild.OSTreeInitStageOptions{Path: ostreeRepoPath})) + stages = append(stages, osbuild.NewOSTreePullStage( + &osbuild.OSTreePullStageOptions{Repo: ostreeRepoPath}, + osbuild.NewOstreePullStageInputs("org.osbuild.source", options.OSTree.Parent, options.OSTree.Ref), + )) + + return stages +} + +func anacondaTreePipeline(repos []rpmmd.RepoConfig, packages []rpmmd.PackageSpec, kernelVer, arch, product, osVersion, variant string) *osbuild.Pipeline { + p := new(osbuild.Pipeline) + p.Name = "anaconda-tree" + p.Build = "name:build" + p.AddStage(osbuild.NewRPMStage(rpmStageOptions(repos), osbuild.NewRpmStageSourceFilesInputs(packages))) + p.AddStage(osbuild.NewBuildstampStage(buildStampStageOptions(arch, product, osVersion, variant))) + p.AddStage(osbuild.NewLocaleStage(&osbuild.LocaleStageOptions{Language: "en_US.UTF-8"})) + + rootPassword := "" + rootUser := osbuild.UsersStageOptionsUser{ + Password: &rootPassword, + } + + installUID := 0 + installGID := 0 + installHome := "/root" + installShell := "/usr/libexec/anaconda/run-anaconda" + installPassword := "" + installUser := osbuild.UsersStageOptionsUser{ + UID: &installUID, + GID: &installGID, + Home: &installHome, + Shell: &installShell, + Password: &installPassword, + } + usersStageOptions := &osbuild.UsersStageOptions{ + Users: map[string]osbuild.UsersStageOptionsUser{ + "root": rootUser, + "install": installUser, + }, + } + + p.AddStage(osbuild.NewUsersStage(usersStageOptions)) + p.AddStage(osbuild.NewAnacondaStage(anacondaStageOptions())) + p.AddStage(osbuild.NewLoraxScriptStage(loraxScriptStageOptions(arch))) + p.AddStage(osbuild.NewDracutStage(dracutStageOptions(kernelVer, arch, []string{ + "anaconda", + "rdma", + "rngd", + "multipath", + "fcoe", + "fcoe-uefi", + "iscsi", + "lunmask", + "nfs", + }))) + p.AddStage(osbuild.NewSELinuxConfigStage(&osbuild.SELinuxConfigStageOptions{State: osbuild.SELinuxStatePermissive})) + + return p +} + +func bootISOTreePipeline(kernelVer, arch, vendor, product, osVersion, isolabel string, ksOptions *osbuild.KickstartStageOptions, payloadStages []*osbuild.Stage) *osbuild.Pipeline { + p := new(osbuild.Pipeline) + p.Name = "bootiso-tree" + p.Build = "name:build" + + p.AddStage(osbuild.NewBootISOMonoStage(bootISOMonoStageOptions(kernelVer, arch, vendor, product, osVersion, isolabel), osbuild.NewBootISOMonoStagePipelineTreeInputs("anaconda-tree"))) + p.AddStage(osbuild.NewKickstartStage(ksOptions)) + p.AddStage(osbuild.NewDiscinfoStage(discinfoStageOptions(arch))) + + for _, stage := range payloadStages { + p.AddStage(stage) + } + + return p +} +func bootISOPipeline(filename, isolabel, arch string, isolinux bool) *osbuild.Pipeline { + p := new(osbuild.Pipeline) + p.Name = "bootiso" + p.Build = "name:build" + + p.AddStage(osbuild.NewXorrisofsStage(xorrisofsStageOptions(filename, isolabel, arch, isolinux), osbuild.NewXorrisofsStagePipelineTreeInputs("bootiso-tree"))) + p.AddStage(osbuild.NewImplantisomd5Stage(&osbuild.Implantisomd5StageOptions{Filename: filename})) + + return p +} + +func liveImagePipeline(inputPipelineName string, outputFilename string, pt *disk.PartitionTable, arch *architecture, kernelVer string) *osbuild.Pipeline { + p := new(osbuild.Pipeline) + p.Name = "image" + p.Build = "name:build" + + for _, stage := range osbuild.GenImagePrepareStages(pt, outputFilename) { + p.AddStage(stage) + } + + inputName := "root-tree" + copyOptions, copyDevices, copyMounts := osbuild.GenCopyFSTreeOptions(inputName, inputPipelineName, outputFilename, pt) + copyInputs := osbuild.NewCopyStagePipelineTreeInputs(inputName, inputPipelineName) + p.AddStage(osbuild.NewCopyStage(copyOptions, copyInputs, copyDevices, copyMounts)) + + for _, stage := range osbuild.GenImageFinishStages(pt, outputFilename) { + p.AddStage(stage) + } + + loopback := osbuild.NewLoopbackDevice(&osbuild.LoopbackDeviceOptions{Filename: outputFilename}) + p.AddStage(bootloaderInstStage(outputFilename, pt, arch, kernelVer, copyDevices, copyMounts, loopback)) + return p +} + +func qemuPipeline(inputPipelineName, inputFilename, outputFilename string, format osbuild.QEMUFormat, formatOptions osbuild.QEMUFormatOptions) *osbuild.Pipeline { + p := new(osbuild.Pipeline) + p.Name = string(format) + p.Build = "name:build" + + qemuStage := osbuild.NewQEMUStage( + osbuild.NewQEMUStageOptions(outputFilename, format, formatOptions), + osbuild.NewQemuStagePipelineFilesInputs(inputPipelineName, inputFilename), + ) + p.AddStage(qemuStage) + return p +} + +func bootloaderConfigStage(t *imageType, partitionTable disk.PartitionTable, kernelVer string, install, greenboot bool) *osbuild.Stage { + if t.Arch().Name() == distro.S390xArchName { + return osbuild.NewZiplStage(new(osbuild.ZiplStageOptions)) + } + + uefi := t.supportsUEFI() + legacy := t.arch.legacy + + options := osbuild.NewGrub2StageOptionsUnified(&partitionTable, kernelVer, uefi, legacy, t.arch.distro.vendor, install) + options.Greenboot = greenboot + + return osbuild.NewGRUB2Stage(options) +} + +func bootloaderInstStage(filename string, pt *disk.PartitionTable, arch *architecture, kernelVer string, devices *osbuild.Devices, mounts *osbuild.Mounts, disk *osbuild.Device) *osbuild.Stage { + platform := arch.legacy + if platform != "" { + return osbuild.NewGrub2InstStage(osbuild.NewGrub2InstStageOption(filename, pt, platform)) + } + + if arch.name == distro.S390xArchName { + return osbuild.NewZiplInstStage(osbuild.NewZiplInstStageOptions(kernelVer, pt), disk, devices, mounts) + } + + return nil +} diff --git a/internal/distro/fedora/stage_options.go b/internal/distro/fedora/stage_options.go new file mode 100644 index 000000000..10016dac7 --- /dev/null +++ b/internal/distro/fedora/stage_options.go @@ -0,0 +1,324 @@ +package fedora + +import ( + "fmt" + "path/filepath" + + "github.com/osbuild/osbuild-composer/internal/blueprint" + "github.com/osbuild/osbuild-composer/internal/common" + "github.com/osbuild/osbuild-composer/internal/crypt" + "github.com/osbuild/osbuild-composer/internal/distro" + osbuild "github.com/osbuild/osbuild-composer/internal/osbuild2" + "github.com/osbuild/osbuild-composer/internal/rpmmd" +) + +const ( + kspath = "/osbuild.ks" +) + +func rpmStageOptions(repos []rpmmd.RepoConfig) *osbuild.RPMStageOptions { + var gpgKeys []string + for _, repo := range repos { + if repo.GPGKey == "" { + continue + } + gpgKeys = append(gpgKeys, repo.GPGKey) + } + + return &osbuild.RPMStageOptions{ + GPGKeys: gpgKeys, + } +} + +// selinuxStageOptions returns the options for the org.osbuild.selinux stage. +// Setting the argument to 'true' relabels the '/usr/bin/cp' +// binariy with 'install_exec_t'. This should be set in the build root. +func selinuxStageOptions(labelcp bool) *osbuild.SELinuxStageOptions { + options := &osbuild.SELinuxStageOptions{ + FileContexts: "etc/selinux/targeted/contexts/files/file_contexts", + } + if labelcp { + options.Labels = map[string]string{ + "/usr/bin/cp": "system_u:object_r:install_exec_t:s0", + } + } + return options +} + +func userStageOptions(users []blueprint.UserCustomization) (*osbuild.UsersStageOptions, error) { + options := osbuild.UsersStageOptions{ + Users: make(map[string]osbuild.UsersStageOptionsUser), + } + + for _, c := range users { + if c.Password != nil && !crypt.PasswordIsCrypted(*c.Password) { + cryptedPassword, err := crypt.CryptSHA512(*c.Password) + if err != nil { + return nil, err + } + + c.Password = &cryptedPassword + } + + user := osbuild.UsersStageOptionsUser{ + Groups: c.Groups, + Description: c.Description, + Home: c.Home, + Shell: c.Shell, + Password: c.Password, + Key: c.Key, + } + + user.UID = c.UID + user.GID = c.GID + + options.Users[c.Name] = user + } + + return &options, nil +} + +func usersFirstBootOptions(usersStageOptions *osbuild.UsersStageOptions) *osbuild.FirstBootStageOptions { + cmds := make([]string, 0, 3*len(usersStageOptions.Users)+2) + // workaround for creating authorized_keys file for user + // need to special case the root user, which has its home in a different place + varhome := filepath.Join("/var", "home") + roothome := filepath.Join("/var", "roothome") + + for name, user := range usersStageOptions.Users { + if user.Key != nil { + var home string + + if name == "root" { + home = roothome + } else { + home = filepath.Join(varhome, name) + } + + sshdir := filepath.Join(home, ".ssh") + + cmds = append(cmds, fmt.Sprintf("mkdir -p %s", sshdir)) + cmds = append(cmds, fmt.Sprintf("sh -c 'echo %q >> %q'", *user.Key, filepath.Join(sshdir, "authorized_keys"))) + cmds = append(cmds, fmt.Sprintf("chown %s:%s -Rc %s", name, name, sshdir)) + } + } + cmds = append(cmds, fmt.Sprintf("restorecon -rvF %s", varhome)) + cmds = append(cmds, fmt.Sprintf("restorecon -rvF %s", roothome)) + + options := &osbuild.FirstBootStageOptions{ + Commands: cmds, + WaitForNetwork: false, + } + + return options +} + +func firewallStageOptions(firewall *blueprint.FirewallCustomization) *osbuild.FirewallStageOptions { + options := osbuild.FirewallStageOptions{ + Ports: firewall.Ports, + } + + if firewall.Services != nil { + options.EnabledServices = firewall.Services.Enabled + options.DisabledServices = firewall.Services.Disabled + } + + return &options +} + +func systemdStageOptions(enabledServices, disabledServices []string, s *blueprint.ServicesCustomization, target string) *osbuild.SystemdStageOptions { + if s != nil { + enabledServices = append(enabledServices, s.Enabled...) + disabledServices = append(disabledServices, s.Disabled...) + } + return &osbuild.SystemdStageOptions{ + EnabledServices: enabledServices, + DisabledServices: disabledServices, + DefaultTarget: target, + } +} + +func buildStampStageOptions(arch, product, osVersion, variant string) *osbuild.BuildstampStageOptions { + return &osbuild.BuildstampStageOptions{ + Arch: arch, + Product: product, + Version: osVersion, + Variant: variant, + Final: true, + } +} + +func anacondaStageOptions() *osbuild.AnacondaStageOptions { + return &osbuild.AnacondaStageOptions{ + KickstartModules: []string{ + "org.fedoraproject.Anaconda.Modules.Network", + "org.fedoraproject.Anaconda.Modules.Payloads", + "org.fedoraproject.Anaconda.Modules.Storage", + }, + } +} + +func loraxScriptStageOptions(arch string) *osbuild.LoraxScriptStageOptions { + return &osbuild.LoraxScriptStageOptions{ + Path: "99-generic/runtime-postinstall.tmpl", + BaseArch: arch, + } +} + +func dracutStageOptions(kernelVer, arch string, additionalModules []string) *osbuild.DracutStageOptions { + kernel := []string{kernelVer} + modules := []string{ + "bash", + "systemd", + "fips", + "systemd-initrd", + "modsign", + "nss-softokn", + "i18n", + "convertfs", + "network-manager", + "network", + "ifcfg", + "url-lib", + "drm", + "plymouth", + "crypt", + "dm", + "dmsquash-live", + "kernel-modules", + "kernel-modules-extra", + "kernel-network-modules", + "livenet", + "lvm", + "mdraid", + "qemu", + "qemu-net", + "resume", + "rootfs-block", + "terminfo", + "udev-rules", + "dracut-systemd", + "pollcdrom", + "usrmount", + "base", + "fs-lib", + "img-lib", + "shutdown", + "uefi-lib", + } + + if arch == distro.X86_64ArchName { + modules = append(modules, "biosdevname") + } + + modules = append(modules, additionalModules...) + return &osbuild.DracutStageOptions{ + Kernel: kernel, + Modules: modules, + Install: []string{"/.buildstamp"}, + } +} + +func ostreeKickstartStageOptions(ostreeURL, ostreeRef string) *osbuild.KickstartStageOptions { + return &osbuild.KickstartStageOptions{ + Path: kspath, + OSTree: &osbuild.OSTreeOptions{ + OSName: "fedora", + URL: ostreeURL, + Ref: ostreeRef, + GPG: false, + }, + } +} + +func bootISOMonoStageOptions(kernelVer, arch, vendor, product, osVersion, isolabel string) *osbuild.BootISOMonoStageOptions { + comprOptions := new(osbuild.FSCompressionOptions) + if bcj := osbuild.BCJOption(arch); bcj != "" { + comprOptions.BCJ = bcj + } + var architectures []string + + if arch == distro.X86_64ArchName { + architectures = []string{"X64"} + } else if arch == distro.Aarch64ArchName { + architectures = []string{"AA64"} + } else { + panic("unsupported architecture") + } + + return &osbuild.BootISOMonoStageOptions{ + Product: osbuild.Product{ + Name: product, + Version: osVersion, + }, + ISOLabel: isolabel, + Kernel: kernelVer, + KernelOpts: fmt.Sprintf("inst.ks=hd:LABEL=%s:%s", isolabel, kspath), + EFI: osbuild.EFI{ + Architectures: architectures, + Vendor: vendor, + }, + ISOLinux: osbuild.ISOLinux{ + Enabled: arch == distro.X86_64ArchName, + Debug: false, + }, + Templates: "99-generic", + RootFS: osbuild.RootFS{ + Size: 9216, + Compression: osbuild.FSCompression{ + Method: "xz", + Options: comprOptions, + }, + }, + } +} + +func discinfoStageOptions(arch string) *osbuild.DiscinfoStageOptions { + return &osbuild.DiscinfoStageOptions{ + BaseArch: arch, + Release: "202010217.n.0", + } +} + +func xorrisofsStageOptions(filename, isolabel, arch string, isolinux bool) *osbuild.XorrisofsStageOptions { + options := &osbuild.XorrisofsStageOptions{ + Filename: filename, + VolID: fmt.Sprintf(isolabel, arch), + SysID: "LINUX", + EFI: "images/efiboot.img", + ISOLevel: 3, + } + + if isolinux { + options.Boot = &osbuild.XorrisofsBoot{ + Image: "isolinux/isolinux.bin", + Catalog: "isolinux/boot.cat", + } + + options.IsohybridMBR = "/usr/share/syslinux/isohdpfx.bin" + } + + return options +} + +func nginxConfigStageOptions(path, htmlRoot, listen string) *osbuild.NginxConfigStageOptions { + // configure nginx to work in an unprivileged container + cfg := &osbuild.NginxConfig{ + Listen: listen, + Root: htmlRoot, + Daemon: common.BoolToPtr(false), + PID: "/tmp/nginx.pid", + } + return &osbuild.NginxConfigStageOptions{ + Path: path, + Config: cfg, + } +} + +func chmodStageOptions(path, mode string, recursive bool) *osbuild.ChmodStageOptions { + return &osbuild.ChmodStageOptions{ + Items: map[string]osbuild.ChmodStagePathOptions{ + path: {Mode: mode, Recursive: recursive}, + }, + } +} diff --git a/internal/distroregistry/distroregistry.go b/internal/distroregistry/distroregistry.go index 0d67e57a7..6ddabc3e1 100644 --- a/internal/distroregistry/distroregistry.go +++ b/internal/distroregistry/distroregistry.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/osbuild/osbuild-composer/internal/distro" - fedora "github.com/osbuild/osbuild-composer/internal/distro/fedora33" + "github.com/osbuild/osbuild-composer/internal/distro/fedora" "github.com/osbuild/osbuild-composer/internal/distro/rhel8" "github.com/osbuild/osbuild-composer/internal/distro/rhel84" "github.com/osbuild/osbuild-composer/internal/distro/rhel85" diff --git a/internal/store/json.go b/internal/store/json.go index 497fc10d9..efb61df32 100644 --- a/internal/store/json.go +++ b/internal/store/json.go @@ -350,6 +350,11 @@ var imageTypeCompatMapping = map[string]string{ "partitioned-disk": "Partitioned-disk", "tar": "Tar", "fedora-iot-commit": "fedora-iot-commit", + "fedora-iot-container": "fedora-iot-container", + "fedora-iot-installer": "fedora-iot-installer", + "iot-commit": "iot-commit", + "iot-container": "iot-container", + "iot-installer": "iot-installer", "rhel-edge-commit": "rhel-edge-commit", "rhel-edge-container": "rhel-edge-container", "rhel-edge-installer": "rhel-edge-installer", diff --git a/internal/store/json_test.go b/internal/store/json_test.go index 9f2c0c308..19cd96cd8 100644 --- a/internal/store/json_test.go +++ b/internal/store/json_test.go @@ -13,7 +13,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/blueprint" "github.com/osbuild/osbuild-composer/internal/common" "github.com/osbuild/osbuild-composer/internal/distro" - fedora "github.com/osbuild/osbuild-composer/internal/distro/fedora33" + "github.com/osbuild/osbuild-composer/internal/distro/fedora" "github.com/osbuild/osbuild-composer/internal/distro/test_distro" "github.com/osbuild/osbuild-composer/internal/rpmmd" "github.com/osbuild/osbuild-composer/internal/target"