diff --git a/cmd/osbuild-pipeline/main.go b/cmd/osbuild-pipeline/main.go index 579ff3461..c0299e6cf 100644 --- a/cmd/osbuild-pipeline/main.go +++ b/cmd/osbuild-pipeline/main.go @@ -10,13 +10,16 @@ import ( "github.com/osbuild/osbuild-composer/internal/distro" _ "github.com/osbuild/osbuild-composer/internal/distro/fedora30" + _ "github.com/osbuild/osbuild-composer/internal/distro/rhel82" ) func main() { var format string var blueprintArg string + var distroArg string flag.StringVar(&format, "output-format", "qcow2", "output format") flag.StringVar(&blueprintArg, "blueprint", "", "blueprint to translate") + flag.StringVar(&distroArg, "distro", "", "distribution to create") flag.Parse() blueprint := &blueprint.Blueprint{} @@ -31,7 +34,7 @@ func main() { } } - d := distro.New("") + d := distro.New(distroArg) pipeline, err := d.Pipeline(blueprint, format) if err != nil { panic(err.Error()) diff --git a/internal/blueprint/blueprint.go b/internal/blueprint/blueprint.go index b7edfe074..30b818687 100644 --- a/internal/blueprint/blueprint.go +++ b/internal/blueprint/blueprint.go @@ -41,7 +41,105 @@ type Group struct { Name string `json:"name"` } -func (b *Blueprint) GetKernelCustomization() *KernelCustomization { +// packages, modules, and groups all resolve to rpm packages right now. This +// function returns a combined list of "name-version" strings. +func (b *Blueprint) GetPackages() []string { + packages := []string{} + for _, pkg := range b.Packages { + packages = append(packages, pkg.ToNameVersion()) + } + for _, pkg := range b.Modules { + packages = append(packages, pkg.ToNameVersion()) + } + for _, group := range b.Groups { + packages = append(packages, group.Name) + } + return packages +} + +func (b *Blueprint) GetHostname() *string { + if b.Customizations == nil { + return nil + } + return b.Customizations.Hostname +} + +func (b *Blueprint) GetPrimaryLocale() (*string, *string) { + if b.Customizations == nil { + return nil, nil + } + if b.Customizations.Locale == nil { + return nil, nil + } + if len(b.Customizations.Locale.Languages) == 0 { + return nil, b.Customizations.Locale.Keyboard + } + return &b.Customizations.Locale.Languages[0], b.Customizations.Locale.Keyboard +} + +func (b *Blueprint) GetTimezoneSettings() (*string, []string) { + if b.Customizations == nil { + return nil, nil + } + if b.Customizations.Timezone == nil { + return nil, nil + } + return b.Customizations.Timezone.Timezone, b.Customizations.Timezone.NTPServers +} + +func (b *Blueprint) GetUsers() []UserCustomization { + if b.Customizations == nil { + return nil + } + + users := []UserCustomization{} + + // prepend sshkey for backwards compat (overridden by users) + if len(b.Customizations.SSHKey) > 0 { + for _, c := range b.Customizations.SSHKey { + users = append(users, UserCustomization{ + Name: c.User, + Key: &c.Key, + }) + } + } + + return append(users, b.Customizations.User...) +} + +func (b *Blueprint) GetGroups() []GroupCustomization { + if b.Customizations == nil { + return nil + } + + // This is for parity with lorax, which assumes that for each + // user, a group with that name already exists. Thus, filter groups + // named like an existing user. + + groups := []GroupCustomization{} + for _, group := range b.Customizations.Group { + exists := false + for _, user := range b.Customizations.User { + if user.Name == group.Name { + exists = true + break + } + } + for _, key := range b.Customizations.SSHKey { + if key.User == group.Name { + exists = true + break + } + } + if !exists { + groups = append(groups, group) + } + } + + return groups +} + +func (b *Blueprint) GetKernel() *KernelCustomization { if b.Customizations == nil { return nil } @@ -49,7 +147,7 @@ func (b *Blueprint) GetKernelCustomization() *KernelCustomization { return b.Customizations.Kernel } -func (b *Blueprint) GetFirewallCustomization() *FirewallCustomization { +func (b *Blueprint) GetFirewall() *FirewallCustomization { if b.Customizations == nil { return nil } @@ -57,7 +155,7 @@ func (b *Blueprint) GetFirewallCustomization() *FirewallCustomization { return b.Customizations.Firewall } -func (b *Blueprint) GetServicesCustomization() *ServicesCustomization { +func (b *Blueprint) GetServices() *ServicesCustomization { if b.Customizations == nil { return nil } diff --git a/internal/distro/fedora30/helpers.go b/internal/distro/fedora30/helpers.go index 98d3c926b..327968b98 100644 --- a/internal/distro/fedora30/helpers.go +++ b/internal/distro/fedora30/helpers.go @@ -103,7 +103,7 @@ func setBootloader(p *pipeline.Pipeline, kernelOptions string, blueprint *bluepr panic("invalid UUID") } - kernelCustomization := blueprint.GetKernelCustomization() + kernelCustomization := blueprint.GetKernel() if kernelCustomization != nil { kernelOptions += " " + kernelCustomization.Append } @@ -127,7 +127,7 @@ func setFilesystems(p *pipeline.Pipeline) { } func setFirewall(p *pipeline.Pipeline, enabledServices []string, disabledServices []string, b *blueprint.Blueprint) { - f := b.GetFirewallCustomization() + f := b.GetFirewall() ports := []string{} if f != nil { @@ -152,7 +152,7 @@ func setFirewall(p *pipeline.Pipeline, enabledServices []string, disabledService } func setServices(p *pipeline.Pipeline, enabledServices []string, disabledServices []string, b *blueprint.Blueprint) { - s := b.GetServicesCustomization() + s := b.GetServices() if s != nil { enabledServices = append(enabledServices, s.Enabled...) diff --git a/internal/distro/rhel82/distro.go b/internal/distro/rhel82/distro.go new file mode 100644 index 000000000..de8557b5c --- /dev/null +++ b/internal/distro/rhel82/distro.go @@ -0,0 +1,503 @@ +package rhel82 + +import ( + "sort" + "strconv" + + "github.com/google/uuid" + + "github.com/osbuild/osbuild-composer/internal/blueprint" + "github.com/osbuild/osbuild-composer/internal/crypt" + "github.com/osbuild/osbuild-composer/internal/distro" + "github.com/osbuild/osbuild-composer/internal/pipeline" + "github.com/osbuild/osbuild-composer/internal/rpmmd" +) + +type RHEL82 struct { + outputs map[string]output +} + +type output struct { + Name string + MimeType string + Packages []string + ExcludedPackages []string + IncludeFSTab bool + Assembler *pipeline.Assembler +} + +func init() { + r := RHEL82{ + outputs: map[string]output{}, + } + + r.outputs["ami"] = output{ + Name: "image.ami", + MimeType: "application/x-qemu-disk", + Packages: []string{ + "@Core", + "chrony", + "kernel", + "selinux-policy-targeted", + "grub2-pc", + "dracut-config-generic", + "cloud-init", + "checkpolicy", + "net-tools", + }, + ExcludedPackages: []string{ + "dracut-config-rescue", + + // TODO setfiles failes because of usr/sbin/timedatex. Exlude until + // https://errata.devel.redhat.com/advisory/47339 lands + "timedatex", + }, + IncludeFSTab: true, + Assembler: r.qemuAssembler("raw", "image.ami"), + } + + r.outputs["ext4-filesystem"] = output{ + Name: "filesystem.img", + MimeType: "application/octet-stream", + Packages: []string{ + "policycoreutils", + "selinux-policy-targeted", + "kernel", + "firewalld", + "chrony", + "langpacks-en", + }, + ExcludedPackages: []string{ + "dracut-config-rescue", + }, + IncludeFSTab: false, + Assembler: r.rawFSAssembler("filesystem.img"), + } + + r.outputs["partitioned-disk"] = output{ + Name: "disk.img", + MimeType: "application/octet-stream", + Packages: []string{ + "@core", + "chrony", + "firewalld", + "grub2-pc", + "kernel", + "langpacks-en", + "selinux-policy-targeted", + }, + ExcludedPackages: []string{ + "dracut-config-rescue", + }, + IncludeFSTab: true, + Assembler: r.qemuAssembler("raw", "disk.img"), + } + + r.outputs["qcow2"] = output{ + Name: "image.qcow2", + MimeType: "application/x-qemu-disk", + Packages: []string{ + "kernel-core", + "chrony", + "polkit", + "systemd-udev", + "selinux-policy-targeted", + "grub2-pc", + "langpacks-en", + }, + ExcludedPackages: []string{ + "dracut-config-rescue", + "etables", + "firewalld", + "gobject-introspection", + "plymouth", + }, + IncludeFSTab: true, + Assembler: r.qemuAssembler("qcow2", "image.qcow2"), + } + + r.outputs["openstack"] = output{ + Name: "image.qcow2", + MimeType: "application/x-qemu-disk", + Packages: []string{ + "@Core", + "chrony", + "kernel", + "selinux-policy-targeted", + "grub2-pc", + "spice-vdagent", + "qemu-guest-agent", + "xen-libs", + "langpacks-en", + "cloud-init", + "libdrm", + }, + ExcludedPackages: []string{ + "dracut-config-rescue", + }, + IncludeFSTab: true, + Assembler: r.qemuAssembler("qcow2", "image.qcow2"), + } + + r.outputs["tar"] = output{ + Name: "root.tar.xz", + MimeType: "application/x-tar", + Packages: []string{ + "policycoreutils", + "selinux-policy-targeted", + "kernel", + "firewalld", + "chrony", + "langpacks-en", + }, + ExcludedPackages: []string{ + "dracut-config-rescue", + }, + IncludeFSTab: false, + Assembler: r.tarAssembler("root.tar.xz", "xz"), + } + + r.outputs["vhd"] = output{ + Name: "image.vhd", + MimeType: "application/x-vhd", + Packages: []string{ + "@Core", + "chrony", + "kernel", + "selinux-policy-targeted", + "grub2-pc", + "langpacks-en", + "net-tools", + "ntfsprogs", + "WALinuxAgent", + "libxcrypt-compat", + }, + ExcludedPackages: []string{ + "dracut-config-rescue", + }, + IncludeFSTab: true, + Assembler: r.qemuAssembler("vhd", "image.vhd"), + } + + r.outputs["vmdk"] = output{ + Name: "disk.vmdk", + MimeType: "application/x-vmdk", + Packages: []string{ + "@core", + "chrony", + "firewalld", + "grub2-pc", + "kernel", + "langpacks-en", + "open-vm-tools", + "selinux-policy-targeted", + }, + ExcludedPackages: []string{ + "dracut-config-rescue", + }, + IncludeFSTab: true, + Assembler: r.qemuAssembler("vmdk", "disk.vmdk"), + } + + distro.Register("rhel-8.2", &r) +} + +func (r *RHEL82) Repositories() []rpmmd.RepoConfig { + return []rpmmd.RepoConfig{ + { + Id: "baseos", + Name: "BaseOS", + BaseURL: "http://download-ipv4.eng.brq.redhat.com/rhel-8/nightly/RHEL-8/RHEL-8.2.0-20191117.n.0/compose/BaseOS/x86_64/os", + Checksum: "sha256:4699a755326e5af71cd069dc9d9289e7d0433ab0acc42ee33b93054fd0e980e7", + }, + { + Id: "appstream", + Name: "AppStream", + BaseURL: "http://download-ipv4.eng.brq.redhat.com/rhel-8/nightly/RHEL-8/RHEL-8.2.0-20191117.n.0/compose/AppStream/x86_64/os", + Checksum: "sha256:212f10ee3fb8265f38837a1e867e4218556e9bc71fa1d38827a088413974a949", + }, + } +} + +func (r *RHEL82) ListOutputFormats() []string { + formats := make([]string, 0, len(r.outputs)) + for name := range r.outputs { + formats = append(formats, name) + } + sort.Strings(formats) + return formats +} + +func (r *RHEL82) FilenameFromType(outputFormat string) (string, string, error) { + if output, exists := r.outputs[outputFormat]; exists { + return output.Name, output.MimeType, nil + } + return "", "", &distro.InvalidOutputFormatError{outputFormat} +} + +func (r *RHEL82) Pipeline(b *blueprint.Blueprint, outputFormat string) (*pipeline.Pipeline, error) { + output, exists := r.outputs[outputFormat] + if !exists { + return nil, &distro.InvalidOutputFormatError{outputFormat} + } + + p := &pipeline.Pipeline{} + p.SetBuild(r.buildPipeline(), "org.osbuild.rhel82") + + packages := append(output.Packages, b.GetPackages()...) + p.AddStage(pipeline.NewDNFStage(r.dnfStageOptions(packages, output.ExcludedPackages))) + p.AddStage(pipeline.NewFixBLSStage()) + + if output.IncludeFSTab { + p.AddStage(pipeline.NewFSTabStage(r.fsTabStageOptions())) + } + p.AddStage(pipeline.NewGRUB2Stage(r.grub2StageOptions(b.GetKernel()))) + + // TODO support setting all languages and install corresponding langpack-* package + language, keyboard := b.GetPrimaryLocale() + + if language != nil { + p.AddStage(pipeline.NewLocaleStage(&pipeline.LocaleStageOptions{*language})) + } else { + p.AddStage(pipeline.NewLocaleStage(&pipeline.LocaleStageOptions{"en_US"})) + } + + if keyboard != nil { + p.AddStage(pipeline.NewKeymapStage(&pipeline.KeymapStageOptions{*keyboard})) + } + + if hostname := b.GetHostname(); hostname != nil { + p.AddStage(pipeline.NewHostnameStage(&pipeline.HostnameStageOptions{*hostname})) + } + + timezone, ntpServers := b.GetTimezoneSettings() + + // TODO install chrony when this is set? + if timezone != nil { + p.AddStage(pipeline.NewTimezoneStage(&pipeline.TimezoneStageOptions{*timezone})) + } + + if len(ntpServers) > 0 { + p.AddStage(pipeline.NewChronyStage(&pipeline.ChronyStageOptions{ntpServers})) + } + + if users := b.GetUsers(); len(users) > 0 { + options, err := r.userStageOptions(users) + if err != nil { + return nil, err + } + p.AddStage(pipeline.NewUsersStage(options)) + } + + if groups := b.GetGroups(); len(groups) > 0 { + p.AddStage(pipeline.NewGroupsStage(r.groupStageOptions(groups))) + } + + if services := b.GetServices(); services != nil { + p.AddStage(pipeline.NewSystemdStage(r.systemdStageOptions(services))) + } + + if firewall := b.GetFirewall(); firewall != nil { + p.AddStage(pipeline.NewFirewallStage(r.firewallStageOptions(firewall))) + } + + p.AddStage(pipeline.NewSELinuxStage(r.selinuxStageOptions())) + p.Assembler = output.Assembler + + return p, nil +} + +func (r *RHEL82) buildPipeline() *pipeline.Pipeline { + packages := []string{ + "dnf", + "dracut-config-generic", + "e2fsprogs", + "glibc", + "grub2-pc", + "policycoreutils", + "python36", + "qemu-img", + "systemd", + "tar", + "xfsprogs", + } + p := &pipeline.Pipeline{} + p.AddStage(pipeline.NewDNFStage(r.dnfStageOptions(packages, nil))) + return p +} + +func (r *RHEL82) dnfStageOptions(packages, excludedPackages []string) *pipeline.DNFStageOptions { + options := &pipeline.DNFStageOptions{ + ReleaseVersion: "8", + BaseArchitecture: "x86_64", + ModulePlatformId: "platform:el8", + } + for _, repo := range r.Repositories() { + options.AddRepository(&pipeline.DNFRepository{ + BaseURL: repo.BaseURL, + MetaLink: repo.Metalink, + MirrorList: repo.MirrorList, + Checksum: repo.Checksum, + }) + } + + for _, pkg := range packages { + options.AddPackage(pkg) + } + + for _, pkg := range excludedPackages { + options.ExcludePackage(pkg) + } + + return options +} + +func (r *RHEL82) userStageOptions(users []blueprint.UserCustomization) (*pipeline.UsersStageOptions, error) { + options := pipeline.UsersStageOptions{ + Users: make(map[string]pipeline.UsersStageOptionsUser), + } + + for _, c := range users { + if c.Password != nil && !crypt.PasswordIsCrypted(*c.Password) { + cryptedPassword, err := crypt.CryptSHA512(*c.Password) + if err != nil { + return nil, err + } + + c.Password = &cryptedPassword + } + + user := pipeline.UsersStageOptionsUser{ + Groups: c.Groups, + Description: c.Description, + Home: c.Home, + Shell: c.Shell, + Password: c.Password, + Key: c.Key, + } + + if c.UID != nil { + uid := strconv.Itoa(*c.UID) + user.UID = &uid + } + + if c.GID != nil { + gid := strconv.Itoa(*c.GID) + user.GID = &gid + } + + options.Users[c.Name] = user + } + + return &options, nil +} + +func (r *RHEL82) groupStageOptions(groups []blueprint.GroupCustomization) *pipeline.GroupsStageOptions { + options := pipeline.GroupsStageOptions{ + Groups: map[string]pipeline.GroupsStageOptionsGroup{}, + } + + for _, group := range groups { + groupData := pipeline.GroupsStageOptionsGroup{ + Name: group.Name, + } + if group.GID != nil { + gid := strconv.Itoa(*group.GID) + groupData.GID = &gid + } + + options.Groups[group.Name] = groupData + } + + return &options +} + +func (r *RHEL82) firewallStageOptions(firewall *blueprint.FirewallCustomization) *pipeline.FirewallStageOptions { + options := pipeline.FirewallStageOptions{ + Ports: firewall.Ports, + } + + if firewall.Services != nil { + options.EnabledServices = firewall.Services.Enabled + options.DisabledServices = firewall.Services.Disabled + } + + return &options +} + +func (r *RHEL82) systemdStageOptions(s *blueprint.ServicesCustomization) *pipeline.SystemdStageOptions { + return &pipeline.SystemdStageOptions{ + EnabledServices: s.Enabled, + DisabledServices: s.Disabled, + } +} + +func (r *RHEL82) fsTabStageOptions() *pipeline.FSTabStageOptions { + id, err := uuid.Parse("0bd700f8-090f-4556-b797-b340297ea1bd") + if err != nil { + panic("invalid UUID") + } + options := pipeline.FSTabStageOptions{} + options.AddFilesystem(id, "xfs", "/", "defaults", 1, 1) + return &options +} + +func (r *RHEL82) grub2StageOptions(kernel *blueprint.KernelCustomization) *pipeline.GRUB2StageOptions { + id, err := uuid.Parse("0bd700f8-090f-4556-b797-b340297ea1bd") + if err != nil { + panic("invalid UUID") + } + kernelOptions := "ro biosdevname=0 net.ifnames=0" + + if kernel != nil { + kernelOptions += " " + kernel.Append + } + + return &pipeline.GRUB2StageOptions{ + RootFilesystemUUID: id, + KernelOptions: kernelOptions, + } +} + +func (r *RHEL82) selinuxStageOptions() *pipeline.SELinuxStageOptions { + return &pipeline.SELinuxStageOptions{ + FileContexts: "etc/selinux/targeted/contexts/files/file_contexts", + } +} + +func (r *RHEL82) qemuAssembler(format string, filename string) *pipeline.Assembler { + id, err := uuid.Parse("0bd700f8-090f-4556-b797-b340297ea1bd") + if err != nil { + panic("invalid UUID") + } + return pipeline.NewQEMUAssembler( + &pipeline.QEMUAssemblerOptions{ + Format: format, + Filename: filename, + PTUUID: "0x14fc63d2", + RootFilesystemUUDI: id, + Size: 3221225472, + RootFilesystemType: "xfs", + }) +} + +func (r *RHEL82) tarAssembler(filename, compression string) *pipeline.Assembler { + return pipeline.NewTarAssembler( + &pipeline.TarAssemblerOptions{ + Filename: filename, + }) +} + +func (r *RHEL82) rawFSAssembler(filename string) *pipeline.Assembler { + id, err := uuid.Parse("0bd700f8-090f-4556-b797-b340297ea1bd") + if err != nil { + panic("invalid UUID") + } + return pipeline.NewRawFSAssembler( + &pipeline.RawFSAssemblerOptions{ + Filename: filename, + RootFilesystemUUDI: id, + Size: 3221225472, + FilesystemType: "xfs", + }) +} diff --git a/internal/distro/rhel82/distro_test.go b/internal/distro/rhel82/distro_test.go new file mode 100644 index 000000000..6c459dd32 --- /dev/null +++ b/internal/distro/rhel82/distro_test.go @@ -0,0 +1,112 @@ +package rhel82_test + +import ( + "reflect" + "testing" + + "github.com/osbuild/osbuild-composer/internal/distro" + _ "github.com/osbuild/osbuild-composer/internal/distro/rhel82" +) + +func TestListOutputFormats(t *testing.T) { + want := []string{ + "ami", + "ext4-filesystem", + "openstack", + "partitioned-disk", + "qcow2", + "tar", + "vhd", + "vmdk", + } + + f31 := distro.New("rhel-8.2") + if got := f31.ListOutputFormats(); !reflect.DeepEqual(got, want) { + t.Errorf("ListOutputFormats() = %v, want %v", got, want) + } +} + +func TestFilenameFromType(t *testing.T) { + type args struct { + outputFormat string + } + tests := []struct { + name string + args args + want string + want1 string + wantErr bool + }{ + { + name: "ami", + args: args{"ami"}, + want: "image.ami", + want1: "application/x-qemu-disk", + }, + { + name: "ext4", + args: args{"ext4-filesystem"}, + want: "filesystem.img", + want1: "application/octet-stream", + }, + { + name: "openstack", + args: args{"openstack"}, + want: "image.qcow2", + want1: "application/x-qemu-disk", + }, + { + name: "partitioned-disk", + args: args{"partitioned-disk"}, + want: "disk.img", + want1: "application/octet-stream", + }, + { + name: "qcow2", + args: args{"qcow2"}, + want: "image.qcow2", + want1: "application/x-qemu-disk", + }, + { + name: "tar", + args: args{"tar"}, + want: "root.tar.xz", + want1: "application/x-tar", + }, + { + name: "vhd", + args: args{"vhd"}, + want: "image.vhd", + want1: "application/x-vhd", + }, + { + name: "vmdk", + args: args{"vmdk"}, + want: "disk.vmdk", + want1: "application/x-vmdk", + }, + { + name: "invalid-output-type", + args: args{"foobar"}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f31 := distro.New("rhel-8.2") + got, got1, err := f31.FilenameFromType(tt.args.outputFormat) + if (err != nil) != tt.wantErr { + t.Errorf("FilenameFromType() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr { + if got != tt.want { + t.Errorf("FilenameFromType() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("FilenameFromType() got1 = %v, want %v", got1, tt.want1) + } + } + }) + } +} diff --git a/internal/pipeline/dnf_stage.go b/internal/pipeline/dnf_stage.go index 523339beb..aadeb3d9a 100644 --- a/internal/pipeline/dnf_stage.go +++ b/internal/pipeline/dnf_stage.go @@ -13,6 +13,7 @@ type DNFStageOptions struct { ExcludedPackages []string `json:"exclude_packages,omitempty"` ReleaseVersion string `json:"releasever"` BaseArchitecture string `json:"basearch"` + ModulePlatformId string `json:"module_platform_id,omitempty"` } func (DNFStageOptions) isStageOptions() {} diff --git a/internal/pipeline/qemu_assembler.go b/internal/pipeline/qemu_assembler.go index dcb7ef3d7..b399aa40e 100644 --- a/internal/pipeline/qemu_assembler.go +++ b/internal/pipeline/qemu_assembler.go @@ -14,6 +14,7 @@ type QEMUAssemblerOptions struct { Filename string `json:"filename"` PTUUID string `json:"ptuuid"` RootFilesystemUUDI uuid.UUID `json:"root_fs_uuid"` + RootFilesystemType string `json:"root_fs_type"` Size uint64 `json:"size"` } diff --git a/internal/pipeline/rawfs_assembler.go b/internal/pipeline/rawfs_assembler.go index 8d95412b3..b6408ca4c 100644 --- a/internal/pipeline/rawfs_assembler.go +++ b/internal/pipeline/rawfs_assembler.go @@ -8,6 +8,7 @@ type RawFSAssemblerOptions struct { Filename string `json:"filename"` RootFilesystemUUDI uuid.UUID `json:"root_fs_uuid"` Size uint64 `json:"size"` + FilesystemType string `json:"fs_type,omitempty"` } func (RawFSAssemblerOptions) isAssemblerOptions() {}