debian-forge-composer/internal/distro/fedora32/distro.go
Major Hayden 0f32f35756 📦 Use raw image format for AWS
Although the vhdx format is smaller than raw, AWS often has some issues
booting instances from an imported vhdx snapshot. The raw format seems
to boot consistently at AWS.

Fixes #754.

Signed-off-by: Major Hayden <major@redhat.com>
2020-07-02 13:11:11 -05:00

780 lines
20 KiB
Go

package fedora32
import (
"encoding/json"
"errors"
"fmt"
"sort"
"github.com/osbuild/osbuild-composer/internal/distro"
"github.com/osbuild/osbuild-composer/internal/osbuild"
"github.com/google/uuid"
"github.com/osbuild/osbuild-composer/internal/blueprint"
"github.com/osbuild/osbuild-composer/internal/crypt"
"github.com/osbuild/osbuild-composer/internal/rpmmd"
)
const name = "fedora-32"
const modulePlatformID = "platform:f32"
type distribution struct {
arches map[string]architecture
imageTypes map[string]imageType
buildPackages []string
}
type architecture struct {
distro *distribution
name string
bootloaderPackages []string
buildPackages []string
legacy string
uefi bool
imageTypes map[string]imageType
}
type imageType struct {
arch *architecture
name string
filename string
mimeType string
packages []string
excludedPackages []string
enabledServices []string
disabledServices []string
kernelOptions string
bootable bool
rpmOstree bool
defaultSize uint64
assembler func(uefi bool, options distro.ImageOptions, arch distro.Arch) *osbuild.Assembler
}
func (a *architecture) Distro() distro.Distro {
return a.distro
}
func (t *imageType) Arch() distro.Arch {
return t.arch
}
func (d *distribution) ListArches() []string {
archs := make([]string, 0, len(d.arches))
for name := range d.arches {
archs = append(archs, name)
}
sort.Strings(archs)
return archs
}
func (d *distribution) GetArch(arch string) (distro.Arch, error) {
a, exists := d.arches[arch]
if !exists {
return nil, errors.New("invalid architecture: " + arch)
}
return &a, nil
}
func (d *distribution) setArches(arches ...architecture) {
d.arches = map[string]architecture{}
for _, a := range arches {
d.arches[a.name] = architecture{
distro: d,
name: a.name,
bootloaderPackages: a.bootloaderPackages,
buildPackages: a.buildPackages,
uefi: a.uefi,
imageTypes: a.imageTypes,
}
}
}
func (a *architecture) Name() string {
return a.name
}
func (a *architecture) ListImageTypes() []string {
formats := make([]string, 0, len(a.imageTypes))
for name := range a.imageTypes {
formats = append(formats, name)
}
sort.Strings(formats)
return formats
}
func (a *architecture) GetImageType(imageType string) (distro.ImageType, error) {
t, exists := a.imageTypes[imageType]
if !exists {
return nil, errors.New("invalid image type: " + imageType)
}
return &t, nil
}
func (a *architecture) setImageTypes(imageTypes ...imageType) {
a.imageTypes = map[string]imageType{}
for _, it := range imageTypes {
a.imageTypes[it.name] = imageType{
arch: a,
name: it.name,
filename: it.filename,
mimeType: it.mimeType,
packages: it.packages,
excludedPackages: it.excludedPackages,
enabledServices: it.enabledServices,
disabledServices: it.disabledServices,
kernelOptions: it.kernelOptions,
bootable: it.bootable,
rpmOstree: it.rpmOstree,
defaultSize: it.defaultSize,
assembler: it.assembler,
}
}
}
func (t *imageType) Name() string {
return t.name
}
func (t *imageType) Filename() string {
return t.filename
}
func (t *imageType) MIMEType() string {
return t.mimeType
}
func (t *imageType) Size(size uint64) uint64 {
const MegaByte = 1024 * 1024
// Microsoft Azure requires vhd images to be rounded up to the nearest MB
if t.name == "vhd" && size%MegaByte != 0 {
size = (size/MegaByte + 1) * MegaByte
}
if size == 0 {
size = t.defaultSize
}
return size
}
func (t *imageType) Packages(bp blueprint.Blueprint) ([]string, []string) {
packages := append(t.packages, bp.GetPackages()...)
timezone, _ := bp.Customizations.GetTimezoneSettings()
if timezone != nil {
packages = append(packages, "chrony")
}
if t.bootable {
packages = append(packages, t.arch.bootloaderPackages...)
}
return packages, t.excludedPackages
}
func (t *imageType) BuildPackages() []string {
packages := append(t.arch.distro.buildPackages, t.arch.buildPackages...)
if t.rpmOstree {
packages = append(packages, "rpm-ostree")
}
return packages
}
func (t *imageType) Manifest(c *blueprint.Customizations,
options distro.ImageOptions,
repos []rpmmd.RepoConfig,
packageSpecs,
buildPackageSpecs []rpmmd.PackageSpec) (distro.Manifest, error) {
pipeline, err := t.pipeline(c, options, repos, packageSpecs, buildPackageSpecs)
if err != nil {
return distro.Manifest{}, err
}
return json.Marshal(
osbuild.Manifest{
Sources: *sources(append(packageSpecs, buildPackageSpecs...)),
Pipeline: *pipeline,
},
)
}
func (d *distribution) Name() string {
return name
}
func (d *distribution) ModulePlatformID() string {
return modulePlatformID
}
func sources(packages []rpmmd.PackageSpec) *osbuild.Sources {
files := &osbuild.FilesSource{
URLs: make(map[string]osbuild.FileSource),
}
for _, pkg := range packages {
fileSource := osbuild.FileSource{
URL: pkg.RemoteLocation,
}
files.URLs[pkg.Checksum] = fileSource
}
return &osbuild.Sources{
"org.osbuild.files": files,
}
}
func (t *imageType) pipeline(c *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec) (*osbuild.Pipeline, error) {
p := &osbuild.Pipeline{}
p.SetBuild(t.buildPipeline(repos, *t.arch, buildPackageSpecs), "org.osbuild.fedora32")
p.AddStage(osbuild.NewRPMStage(t.rpmStageOptions(*t.arch, repos, packageSpecs)))
p.AddStage(osbuild.NewFixBLSStage())
// TODO support setting all languages and install corresponding langpack-* package
language, keyboard := c.GetPrimaryLocale()
if language != nil {
p.AddStage(osbuild.NewLocaleStage(&osbuild.LocaleStageOptions{Language: *language}))
} else {
p.AddStage(osbuild.NewLocaleStage(&osbuild.LocaleStageOptions{Language: "en_US"}))
}
if keyboard != nil {
p.AddStage(osbuild.NewKeymapStage(&osbuild.KeymapStageOptions{Keymap: *keyboard}))
}
if hostname := c.GetHostname(); hostname != nil {
p.AddStage(osbuild.NewHostnameStage(&osbuild.HostnameStageOptions{Hostname: *hostname}))
}
timezone, ntpServers := c.GetTimezoneSettings()
if timezone != nil {
p.AddStage(osbuild.NewTimezoneStage(&osbuild.TimezoneStageOptions{Zone: *timezone}))
}
if len(ntpServers) > 0 {
p.AddStage(osbuild.NewChronyStage(&osbuild.ChronyStageOptions{Timeservers: ntpServers}))
}
if groups := c.GetGroups(); len(groups) > 0 {
p.AddStage(osbuild.NewGroupsStage(t.groupStageOptions(groups)))
}
if users := c.GetUsers(); len(users) > 0 {
options, err := t.userStageOptions(users)
if err != nil {
return nil, err
}
p.AddStage(osbuild.NewUsersStage(options))
}
if t.bootable {
p.AddStage(osbuild.NewFSTabStage(t.fsTabStageOptions(t.arch.uefi)))
p.AddStage(osbuild.NewGRUB2Stage(t.grub2StageOptions(t.kernelOptions, c.GetKernel(), t.arch.uefi)))
}
if services := c.GetServices(); services != nil || t.enabledServices != nil {
p.AddStage(osbuild.NewSystemdStage(t.systemdStageOptions(t.enabledServices, t.disabledServices, services)))
}
if firewall := c.GetFirewall(); firewall != nil {
p.AddStage(osbuild.NewFirewallStage(t.firewallStageOptions(firewall)))
}
p.AddStage(osbuild.NewSELinuxStage(t.selinuxStageOptions()))
if t.rpmOstree {
p.AddStage(osbuild.NewRPMOSTreeStage(&osbuild.RPMOSTreeStageOptions{
EtcGroupMembers: []string{
// NOTE: We may want to make this configurable.
"wheel", "docker",
},
}))
}
p.Assembler = t.assembler(t.arch.uefi, options, t.arch)
return p, nil
}
func (t *imageType) buildPipeline(repos []rpmmd.RepoConfig, arch architecture, buildPackageSpecs []rpmmd.PackageSpec) *osbuild.Pipeline {
p := &osbuild.Pipeline{}
p.AddStage(osbuild.NewRPMStage(t.rpmStageOptions(arch, repos, buildPackageSpecs)))
return p
}
func (t *imageType) rpmStageOptions(arch architecture, repos []rpmmd.RepoConfig, specs []rpmmd.PackageSpec) *osbuild.RPMStageOptions {
var gpgKeys []string
for _, repo := range repos {
if repo.GPGKey == "" {
continue
}
gpgKeys = append(gpgKeys, repo.GPGKey)
}
var packages []osbuild.RPMPackage
for _, spec := range specs {
pkg := osbuild.RPMPackage{
Checksum: spec.Checksum,
CheckGPG: spec.CheckGPG,
}
packages = append(packages, pkg)
}
return &osbuild.RPMStageOptions{
GPGKeys: gpgKeys,
Packages: packages,
}
}
func (t *imageType) userStageOptions(users []blueprint.UserCustomization) (*osbuild.UsersStageOptions, error) {
options := osbuild.UsersStageOptions{
Users: make(map[string]osbuild.UsersStageOptionsUser),
}
for _, c := range users {
if c.Password != nil && !crypt.PasswordIsCrypted(*c.Password) {
cryptedPassword, err := crypt.CryptSHA512(*c.Password)
if err != nil {
return nil, err
}
c.Password = &cryptedPassword
}
user := osbuild.UsersStageOptionsUser{
Groups: c.Groups,
Description: c.Description,
Home: c.Home,
Shell: c.Shell,
Password: c.Password,
Key: c.Key,
}
user.UID = c.UID
user.GID = c.GID
options.Users[c.Name] = user
}
return &options, nil
}
func (t *imageType) groupStageOptions(groups []blueprint.GroupCustomization) *osbuild.GroupsStageOptions {
options := osbuild.GroupsStageOptions{
Groups: map[string]osbuild.GroupsStageOptionsGroup{},
}
for _, group := range groups {
groupData := osbuild.GroupsStageOptionsGroup{
Name: group.Name,
}
groupData.GID = group.GID
options.Groups[group.Name] = groupData
}
return &options
}
func (t *imageType) firewallStageOptions(firewall *blueprint.FirewallCustomization) *osbuild.FirewallStageOptions {
options := osbuild.FirewallStageOptions{
Ports: firewall.Ports,
}
if firewall.Services != nil {
options.EnabledServices = firewall.Services.Enabled
options.DisabledServices = firewall.Services.Disabled
}
return &options
}
func (t *imageType) systemdStageOptions(enabledServices, disabledServices []string, s *blueprint.ServicesCustomization) *osbuild.SystemdStageOptions {
if s != nil {
enabledServices = append(enabledServices, s.Enabled...)
disabledServices = append(disabledServices, s.Disabled...)
}
return &osbuild.SystemdStageOptions{
EnabledServices: enabledServices,
DisabledServices: disabledServices,
}
}
func (t *imageType) fsTabStageOptions(uefi bool) *osbuild.FSTabStageOptions {
options := osbuild.FSTabStageOptions{}
options.AddFilesystem("76a22bf4-f153-4541-b6c7-0332c0dfaeac", "ext4", "/", "defaults", 1, 1)
if uefi {
options.AddFilesystem("46BB-8120", "vfat", "/boot/efi", "umask=0077,shortname=winnt", 0, 2)
}
return &options
}
func (t *imageType) grub2StageOptions(kernelOptions string, kernel *blueprint.KernelCustomization, uefi bool) *osbuild.GRUB2StageOptions {
id := uuid.MustParse("76a22bf4-f153-4541-b6c7-0332c0dfaeac")
if kernel != nil {
kernelOptions += " " + kernel.Append
}
var uefiOptions *osbuild.GRUB2UEFI
if uefi {
uefiOptions = &osbuild.GRUB2UEFI{
Vendor: "fedora",
}
}
var legacy string
if !uefi {
legacy = t.arch.legacy
}
return &osbuild.GRUB2StageOptions{
RootFilesystemUUID: id,
KernelOptions: kernelOptions,
Legacy: legacy,
UEFI: uefiOptions,
}
}
func (t *imageType) selinuxStageOptions() *osbuild.SELinuxStageOptions {
return &osbuild.SELinuxStageOptions{
FileContexts: "etc/selinux/targeted/contexts/files/file_contexts",
}
}
func qemuAssembler(format string, filename string, uefi bool, imageOptions distro.ImageOptions) *osbuild.Assembler {
var options osbuild.QEMUAssemblerOptions
if uefi {
options = osbuild.QEMUAssemblerOptions{
Format: format,
Filename: filename,
Size: imageOptions.Size,
PTUUID: "8DFDFF87-C96E-EA48-A3A6-9408F1F6B1EF",
PTType: "gpt",
Partitions: []osbuild.QEMUPartition{
{
Start: 2048,
Size: 972800,
Type: "C12A7328-F81F-11D2-BA4B-00A0C93EC93B",
UUID: "02C1E068-1D2F-4DA3-91FD-8DD76A955C9D",
Filesystem: &osbuild.QEMUFilesystem{
Type: "vfat",
UUID: "46BB-8120",
Label: "EFI System Partition",
Mountpoint: "/boot/efi",
},
},
{
Start: 976896,
UUID: "8D760010-FAAE-46D1-9E5B-4A2EAC5030CD",
Filesystem: &osbuild.QEMUFilesystem{
Type: "ext4",
UUID: "76a22bf4-f153-4541-b6c7-0332c0dfaeac",
Mountpoint: "/",
},
},
},
}
} else {
options = osbuild.QEMUAssemblerOptions{
Format: format,
Filename: filename,
Size: imageOptions.Size,
PTUUID: "0x14fc63d2",
PTType: "mbr",
Partitions: []osbuild.QEMUPartition{
{
Start: 2048,
Bootable: true,
Filesystem: &osbuild.QEMUFilesystem{
Type: "ext4",
UUID: "76a22bf4-f153-4541-b6c7-0332c0dfaeac",
Mountpoint: "/",
},
},
},
}
}
return osbuild.NewQEMUAssembler(&options)
}
func ostreeCommitAssembler(options distro.ImageOptions, arch distro.Arch) *osbuild.Assembler {
ref := options.OSTree.Ref
if ref == "" {
ref = fmt.Sprintf("fedora/32/%s/iot", arch.Name())
}
return osbuild.NewOSTreeCommitAssembler(
&osbuild.OSTreeCommitAssemblerOptions{
Ref: ref,
Parent: options.OSTree.Parent,
Tar: osbuild.OSTreeCommitAssemblerTarOptions{
Filename: "commit.tar",
},
},
)
}
// New creates a new distro object, defining the supported architectures and image types
func New() distro.Distro {
const GigaByte = 1024 * 1024 * 1024
iotImgType := imageType{
name: "fedora-iot-commit",
filename: "commit.tar",
mimeType: "application/x-tar",
packages: []string{
"fedora-release-iot",
"glibc", "glibc-minimal-langpack", "nss-altfiles",
"sssd-client", "libsss_sudo", "shadow-utils",
"kernel",
"dracut-config-generic", "dracut-network",
"rpm-ostree", "polkit", "lvm2",
"chrony", "zram",
"cryptsetup", "pinentry",
"keyutils",
"e2fsprogs", "dosfstools",
"gnupg2",
"basesystem", "python3", "bash",
"xz", "gzip",
"coreutils", "which", "curl",
"firewalld", "iptables",
"NetworkManager", "NetworkManager-wifi", "NetworkManager-wwan",
"wpa_supplicant", "iwd",
"dnsmasq", "traceroute",
"hostname", "iproute", "iputils",
"openssh-clients", "openssh-server", "passwd",
"policycoreutils", "procps-ng", "rootfiles", "rpm",
"selinux-policy-targeted", "setup", "shadow-utils",
"sudo", "systemd", "util-linux", "vim-minimal",
"less", "tar",
"fwupd", // "usbguard", BUG: this fails due to an SELinux policy issue
"greenboot", "greenboot-grub2", "greenboot-rpm-ostree-grub2", "greenboot-reboot", "greenboot-status",
"ignition",
"rsync",
"ima-evm-utils",
"bash-completion",
"tmux", "screen",
"policycoreutils-python-utils",
"setools-console",
"audit", "rng-tools",
"bluez", "bluez-libs", "bluez-mesh", "wpan-tools",
"kernel-tools", "libgpiod-utils",
"podman", "container-selinux", "skopeo", "criu",
"slirp4netns",
"clevis", "clevis-dracut", "clevis-luks",
"attr",
// x86 specific
"grub2", "grub2-efi-x64", "efibootmgr", "shim-x64", "microcode_ctl",
"iwl1000-firmware", "iwl100-firmware", "iwl105-firmware", "iwl135-firmware",
"iwl2000-firmware", "iwl2030-firmware", "iwl3160-firmware", "iwl5000-firmware",
"iwl5150-firmware", "iwl6000-firmware", "iwl6050-firmware", "iwl7260-firmware",
},
enabledServices: []string{
"NetworkManager.service", "firewalld.service", "rngd.service", "sshd.service", "zram-swap.service",
},
rpmOstree: true,
assembler: func(uefi bool, options distro.ImageOptions, arch distro.Arch) *osbuild.Assembler {
return ostreeCommitAssembler(options, arch)
},
}
amiImgType := imageType{
name: "ami",
filename: "image.raw",
mimeType: "application/octet-stream",
packages: []string{
"@Core",
"chrony",
"kernel",
"selinux-policy-targeted",
"langpacks-en",
"libxcrypt-compat",
"xfsprogs",
"cloud-init",
"checkpolicy",
"net-tools",
},
excludedPackages: []string{
"dracut-config-rescue",
},
enabledServices: []string{
"cloud-init.service",
},
kernelOptions: "ro no_timer_check console=ttyS0,115200n8 console=tty1 biosdevname=0 net.ifnames=0 console=ttyS0,115200",
bootable: true,
defaultSize: 6 * GigaByte,
assembler: func(uefi bool, options distro.ImageOptions, arch distro.Arch) *osbuild.Assembler {
return qemuAssembler("raw", "image.raw", uefi, options)
},
}
qcow2ImageType := imageType{
name: "qcow2",
filename: "disk.qcow2",
mimeType: "application/x-qemu-disk",
packages: []string{
"kernel-core",
"@Fedora Cloud Server",
"chrony",
"polkit",
"systemd-udev",
"selinux-policy-targeted",
"langpacks-en",
},
excludedPackages: []string{
"dracut-config-rescue",
"etables",
"firewalld",
"gobject-introspection",
"plymouth",
},
kernelOptions: "ro biosdevname=0 net.ifnames=0",
bootable: true,
defaultSize: 2 * GigaByte,
assembler: func(uefi bool, options distro.ImageOptions, arch distro.Arch) *osbuild.Assembler {
return qemuAssembler("qcow2", "disk.qcow2", uefi, options)
},
}
openstackImgType := imageType{
name: "openstack",
filename: "disk.qcow2",
mimeType: "application/x-qemu-disk",
packages: []string{
"@Core",
"chrony",
"kernel",
"selinux-policy-targeted",
"spice-vdagent",
"qemu-guest-agent",
"xen-libs",
"langpacks-en",
"cloud-init",
"libdrm",
},
excludedPackages: []string{
"dracut-config-rescue",
},
kernelOptions: "ro biosdevname=0 net.ifnames=0",
bootable: true,
defaultSize: 2 * GigaByte,
assembler: func(uefi bool, options distro.ImageOptions, arch distro.Arch) *osbuild.Assembler {
return qemuAssembler("qcow2", "disk.qcow2", uefi, options)
},
}
vhdImgType := imageType{
name: "vhd",
filename: "disk.vhd",
mimeType: "application/x-vhd",
packages: []string{
"@Core",
"chrony",
"kernel",
"selinux-policy-targeted",
"langpacks-en",
"net-tools",
"ntfsprogs",
"WALinuxAgent",
"libxcrypt-compat",
"initscripts",
"glibc-all-langpacks",
},
excludedPackages: []string{
"dracut-config-rescue",
},
enabledServices: []string{
"sshd",
"waagent", // needed to run in Azure
},
disabledServices: []string{
"proc-sys-fs-binfmt_misc.mount",
"loadmodules.service",
},
// These kernel parameters are required by Azure documentation
kernelOptions: "ro biosdevname=0 rootdelay=300 console=ttyS0 earlyprintk=ttyS0 net.ifnames=0",
bootable: true,
defaultSize: 2 * GigaByte,
assembler: func(uefi bool, options distro.ImageOptions, arch distro.Arch) *osbuild.Assembler {
return qemuAssembler("vpc", "disk.vhd", uefi, options)
},
}
vmdkImgType := imageType{
name: "vmdk",
filename: "disk.vmdk",
mimeType: "application/x-vmdk",
packages: []string{
"@core",
"chrony",
"firewalld",
"kernel",
"langpacks-en",
"open-vm-tools",
"selinux-policy-targeted",
},
excludedPackages: []string{
"dracut-config-rescue",
},
kernelOptions: "ro biosdevname=0 net.ifnames=0",
bootable: true,
defaultSize: 2 * GigaByte,
assembler: func(uefi bool, options distro.ImageOptions, arch distro.Arch) *osbuild.Assembler {
return qemuAssembler("vmdk", "disk.vmdk", uefi, options)
},
}
r := distribution{
imageTypes: map[string]imageType{},
buildPackages: []string{
"dnf",
"dosfstools",
"e2fsprogs",
"policycoreutils",
"qemu-img",
"systemd",
"tar",
"xz",
},
}
x8664 := architecture{
distro: &r,
name: "x86_64",
bootloaderPackages: []string{
"dracut-config-generic",
"grub2-pc",
},
buildPackages: []string{
"grub2-pc",
},
legacy: "i386-pc",
}
x8664.setImageTypes(
iotImgType,
amiImgType,
qcow2ImageType,
openstackImgType,
vhdImgType,
vmdkImgType,
)
aarch64 := architecture{
distro: &r,
name: "aarch64",
bootloaderPackages: []string{
"dracut-config-generic",
"efibootmgr",
"grub2-efi-aa64",
"grub2-tools",
"shim-aa64",
},
uefi: true,
}
aarch64.setImageTypes(
amiImgType,
qcow2ImageType,
openstackImgType,
)
r.setArches(x8664, aarch64)
return &r
}