particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

distro: move from dnf-based to rpm-based pipelines for all distros

Browse source 
Conceptually, we used to insert the high-level packages and package
groups into the pipeline together with the expected repository
metadata checksum.

osbuild, using the dnf stage, would then fetch the metadata, verify
that its checksum is correct, compute the dependencies, and install
the packages.

Among the problems this has is that it made it impossible to cache
and share the resolved metadata as well as the rpms. Moreover,
as the checksum was at the repository-level, rather than at the
package level, it meant that we would refuse to build a pipeline
as soon as there were any changes at all to the repository, as we
could no longer guarantee the installed packages would be the same.

As of this patch, all repository and metadata handling is done by
composer, rather than osbuild. This means that the resolved metadata
can be cached between runs, which and it means that we can now
pin individual packages, rather than the entire repository. Meaning,
that as long as the rpms are still available, we are able to build
a pipeline.

The downloading of rpms is now done by a source helper in osbuild,
which means that they can be cached and shared between runs too.

One consequence of this change is that we resolve the location of
each rpm in composer, and pass that to the worker. As the worker
may not be in the same location, we do not want to use metalinks
in composer for this, as it would pin the repository closest to
composer, rather than the runner. Instead, we now manually select
a baseurl for each repository, which should be generally the
most useful one. Fedora helpfully provides such baseurls, so
this should work ok.

The most important thing to verify when checking this commit, is
that the image info in our test-cases remains unchanged.

Signed-off-by: Tom Gundersen <teg@jklm.no>
This commit is contained in:
Tom Gundersen 2020-03-14 16:29:23 +01:00
parent 4dda272d8b
commit 4c40faebe6
24 changed files with 15044 additions and 894 deletions
Download patch file Download diff file Expand all files Collapse all files

66
internal/distro/fedora30/distro.go
View file

@ -398,15 +398,9 @@ func (r *Fedora30) pipeline(b *blueprint.Blueprint, additionalRepos []rpmmd.Repo
}
p := &osbuild.Pipeline{}
p.SetBuild(r.buildPipeline(arch, checksums), "org.osbuild.fedora30")
p.SetBuild(r.buildPipeline(arch, buildPackageSpecs), "org.osbuild.fedora30")
packages, excludedPackages, err := r.BasePackages(outputFormat, outputArchitecture)
if err != nil {
return nil, err
}
packages = append(packages, b.GetPackages()...)
p.AddStage(osbuild.NewDNFStage(r.dnfStageOptions(arch, additionalRepos, checksums, packages, excludedPackages)))
p.AddStage(osbuild.NewRPMStage(r.rpmStageOptions(arch, additionalRepos, packageSpecs)))
p.AddStage(osbuild.NewFixBLSStage())
// TODO support setting all languages and install corresponding langpack-* package
@ -470,7 +464,15 @@ func (r *Fedora30) pipeline(b *blueprint.Blueprint, additionalRepos []rpmmd.Repo
}
func (r *Fedora30) sources(packages []rpmmd.PackageSpec) *osbuild.Sources {
return &osbuild.Sources{}
files := &osbuild.FilesSource{
URLs: make(map[string]string),
}
for _, pkg := range packages {
files.URLs[pkg.Checksum] = pkg.RemoteLocation
}
return &osbuild.Sources{
"org.osbuild.files": files,
}
}
func (r *Fedora30) Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error) {
@ -489,45 +491,31 @@ func (r *Fedora30) Runner() string {
return "org.osbuild.fedora30"
}
func (r *Fedora30) buildPipeline(arch arch, checksums map[string]string) *osbuild.Pipeline {
packages, err := r.BuildPackages(arch.Name)
if err != nil {
panic("impossibly invalid arch")
}
func (r *Fedora30) buildPipeline(arch arch, packageSpecs []rpmmd.PackageSpec) *osbuild.Pipeline {
p := &osbuild.Pipeline{}
p.AddStage(osbuild.NewDNFStage(r.dnfStageOptions(arch, nil, checksums, packages, nil)))
p.AddStage(osbuild.NewRPMStage(r.rpmStageOptions(arch, nil, packageSpecs)))
return p
}
func (r *Fedora30) dnfStageOptions(arch arch, additionalRepos []rpmmd.RepoConfig, checksums map[string]string, packages, excludedPackages []string) *osbuild.DNFStageOptions {
options := &osbuild.DNFStageOptions{
ReleaseVersion: "30",
BaseArchitecture: arch.Name,
ModulePlatformId: ModulePlatformID,
func (r *Fedora30) rpmStageOptions(arch arch, additionalRepos []rpmmd.RepoConfig, specs []rpmmd.PackageSpec) *osbuild.RPMStageOptions {
var gpgKeys []string
repos := append(arch.Repositories, additionalRepos...)
for _, repo := range repos {
if repo.GPGKey == "" {
continue
}
gpgKeys = append(gpgKeys, repo.GPGKey)
}
for _, repo := range append(arch.Repositories, additionalRepos...) {
options.AddRepository(&osbuild.DNFRepository{
BaseURL: repo.BaseURL,
MetaLink: repo.Metalink,
MirrorList: repo.MirrorList,
GPGKey: repo.GPGKey,
Checksum: checksums[repo.Id],
})
var packages []string
for _, spec := range specs {
packages = append(packages, spec.Checksum)
}
sort.Strings(packages)
for _, pkg := range packages {
options.AddPackage(pkg)
return &osbuild.RPMStageOptions{
GPGKeys: gpgKeys,
Packages: packages,
}
sort.Strings(excludedPackages)
for _, pkg := range excludedPackages {
options.ExcludePackage(pkg)
}
return options
}
func (r *Fedora30) userStageOptions(users []blueprint.UserCustomization) (*osbuild.UsersStageOptions, error) {

66
internal/distro/fedora31/distro.go
View file

@ -398,15 +398,9 @@ func (r *Fedora31) pipeline(b *blueprint.Blueprint, additionalRepos []rpmmd.Repo
}
p := &osbuild.Pipeline{}
p.SetBuild(r.buildPipeline(arch, checksums), "org.osbuild.fedora31")
p.SetBuild(r.buildPipeline(arch, buildPackageSpecs), "org.osbuild.fedora31")
packages, excludedPackages, err := r.BasePackages(outputFormat, outputArchitecture)
if err != nil {
return nil, err
}
packages = append(packages, b.GetPackages()...)
p.AddStage(osbuild.NewDNFStage(r.dnfStageOptions(arch, additionalRepos, checksums, packages, excludedPackages)))
p.AddStage(osbuild.NewRPMStage(r.rpmStageOptions(arch, additionalRepos, packageSpecs)))
p.AddStage(osbuild.NewFixBLSStage())
// TODO support setting all languages and install corresponding langpack-* package
@ -470,7 +464,15 @@ func (r *Fedora31) pipeline(b *blueprint.Blueprint, additionalRepos []rpmmd.Repo
}
func (r *Fedora31) sources(packages []rpmmd.PackageSpec) *osbuild.Sources {
return &osbuild.Sources{}
files := &osbuild.FilesSource{
URLs: make(map[string]string),
}
for _, pkg := range packages {
files.URLs[pkg.Checksum] = pkg.RemoteLocation
}
return &osbuild.Sources{
"org.osbuild.files": files,
}
}
func (r *Fedora31) Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error) {
@ -489,45 +491,31 @@ func (r *Fedora31) Runner() string {
return "org.osbuild.fedora31"
}
func (r *Fedora31) buildPipeline(arch arch, checksums map[string]string) *osbuild.Pipeline {
packages, err := r.BuildPackages(arch.Name)
if err != nil {
panic("impossibly invalid arch")
}
func (r *Fedora31) buildPipeline(arch arch, buildPackageSpecs []rpmmd.PackageSpec) *osbuild.Pipeline {
p := &osbuild.Pipeline{}
p.AddStage(osbuild.NewDNFStage(r.dnfStageOptions(arch, nil, checksums, packages, nil)))
p.AddStage(osbuild.NewRPMStage(r.rpmStageOptions(arch, nil, buildPackageSpecs)))
return p
}
func (r *Fedora31) dnfStageOptions(arch arch, additionalRepos []rpmmd.RepoConfig, checksums map[string]string, packages, excludedPackages []string) *osbuild.DNFStageOptions {
options := &osbuild.DNFStageOptions{
ReleaseVersion: "31",
BaseArchitecture: arch.Name,
ModulePlatformId: ModulePlatformID,
func (r *Fedora31) rpmStageOptions(arch arch, additionalRepos []rpmmd.RepoConfig, specs []rpmmd.PackageSpec) *osbuild.RPMStageOptions {
var gpgKeys []string
repos := append(arch.Repositories, additionalRepos...)
for _, repo := range repos {
if repo.GPGKey == "" {
continue
}
gpgKeys = append(gpgKeys, repo.GPGKey)
}
for _, repo := range append(arch.Repositories, additionalRepos...) {
options.AddRepository(&osbuild.DNFRepository{
BaseURL: repo.BaseURL,
MetaLink: repo.Metalink,
MirrorList: repo.MirrorList,
GPGKey: repo.GPGKey,
Checksum: checksums[repo.Id],
})
var packages []string
for _, spec := range specs {
packages = append(packages, spec.Checksum)
}
sort.Strings(packages)
for _, pkg := range packages {
options.AddPackage(pkg)
return &osbuild.RPMStageOptions{
GPGKeys: gpgKeys,
Packages: packages,
}
sort.Strings(excludedPackages)
for _, pkg := range excludedPackages {
options.ExcludePackage(pkg)
}
return options
}
func (r *Fedora31) userStageOptions(users []blueprint.UserCustomization) (*osbuild.UsersStageOptions, error) {

66
internal/distro/fedora32/distro.go
View file

@ -398,15 +398,9 @@ func (r *Fedora32) pipeline(b *blueprint.Blueprint, additionalRepos []rpmmd.Repo
}
p := &osbuild.Pipeline{}
p.SetBuild(r.buildPipeline(arch, checksums), "org.osbuild.fedora32")
p.SetBuild(r.buildPipeline(arch, buildPackageSpecs), "org.osbuild.fedora32")
packages, excludedPackages, err := r.BasePackages(outputFormat, outputArchitecture)
if err != nil {
return nil, err
}
packages = append(packages, b.GetPackages()...)
p.AddStage(osbuild.NewDNFStage(r.dnfStageOptions(arch, additionalRepos, checksums, packages, excludedPackages)))
p.AddStage(osbuild.NewRPMStage(r.rpmStageOptions(arch, additionalRepos, packageSpecs)))
p.AddStage(osbuild.NewFixBLSStage())
// TODO support setting all languages and install corresponding langpack-* package
@ -470,7 +464,15 @@ func (r *Fedora32) pipeline(b *blueprint.Blueprint, additionalRepos []rpmmd.Repo
}
func (r *Fedora32) sources(packages []rpmmd.PackageSpec) *osbuild.Sources {
return &osbuild.Sources{}
files := &osbuild.FilesSource{
URLs: make(map[string]string),
}
for _, pkg := range packages {
files.URLs[pkg.Checksum] = pkg.RemoteLocation
}
return &osbuild.Sources{
"org.osbuild.files": files,
}
}
func (r *Fedora32) Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error) {
@ -489,45 +491,31 @@ func (r *Fedora32) Runner() string {
return "org.osbuild.fedora32"
}
func (r *Fedora32) buildPipeline(arch arch, checksums map[string]string) *osbuild.Pipeline {
packages, err := r.BuildPackages(arch.Name)
if err != nil {
panic("impossibly invalid arch")
}
func (r *Fedora32) buildPipeline(arch arch, buildPackageSpecs []rpmmd.PackageSpec) *osbuild.Pipeline {
p := &osbuild.Pipeline{}
p.AddStage(osbuild.NewDNFStage(r.dnfStageOptions(arch, nil, checksums, packages, nil)))
p.AddStage(osbuild.NewRPMStage(r.rpmStageOptions(arch, nil, buildPackageSpecs)))
return p
}
func (r *Fedora32) dnfStageOptions(arch arch, additionalRepos []rpmmd.RepoConfig, checksums map[string]string, packages, excludedPackages []string) *osbuild.DNFStageOptions {
options := &osbuild.DNFStageOptions{
ReleaseVersion: "32",
BaseArchitecture: arch.Name,
ModulePlatformId: ModulePlatformID,
func (r *Fedora32) rpmStageOptions(arch arch, additionalRepos []rpmmd.RepoConfig, specs []rpmmd.PackageSpec) *osbuild.RPMStageOptions {
var gpgKeys []string
repos := append(arch.Repositories, additionalRepos...)
for _, repo := range repos {
if repo.GPGKey == "" {
continue
}
gpgKeys = append(gpgKeys, repo.GPGKey)
}
for _, repo := range append(arch.Repositories, additionalRepos...) {
options.AddRepository(&osbuild.DNFRepository{
BaseURL: repo.BaseURL,
MetaLink: repo.Metalink,
MirrorList: repo.MirrorList,
GPGKey: repo.GPGKey,
Checksum: checksums[repo.Id],
})
var packages []string
for _, spec := range specs {
packages = append(packages, spec.Checksum)
}
sort.Strings(packages)
for _, pkg := range packages {
options.AddPackage(pkg)
return &osbuild.RPMStageOptions{
GPGKeys: gpgKeys,
Packages: packages,
}
sort.Strings(excludedPackages)
for _, pkg := range excludedPackages {
options.ExcludePackage(pkg)
}
return options
}
func (r *Fedora32) userStageOptions(users []blueprint.UserCustomization) (*osbuild.UsersStageOptions, error) {

63
internal/distro/rhel81/distro.go
View file

@ -537,14 +537,9 @@ func (r *RHEL81) pipeline(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoCo
}
p := &osbuild.Pipeline{}
p.SetBuild(r.buildPipeline(arch, checksums), "org.osbuild.rhel81")
p.SetBuild(r.buildPipeline(arch, buildPackageSpecs), "org.osbuild.rhel81")
packages, excludedPackages, err := r.BasePackages(outputFormat, outputArchitecture)
if err != nil {
return nil, err
}
packages = append(packages, b.GetPackages()...)
p.AddStage(osbuild.NewDNFStage(r.dnfStageOptions(arch, additionalRepos, checksums, packages, excludedPackages)))
p.AddStage(osbuild.NewRPMStage(r.rpmStageOptions(arch, additionalRepos, packageSpecs)))
p.AddStage(osbuild.NewFixBLSStage())
if output.Bootable {
@ -613,7 +608,15 @@ func (r *RHEL81) pipeline(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoCo
}
func (r *RHEL81) sources(packages []rpmmd.PackageSpec) *osbuild.Sources {
return &osbuild.Sources{}
files := &osbuild.FilesSource{
URLs: make(map[string]string),
}
for _, pkg := range packages {
files.URLs[pkg.Checksum] = pkg.RemoteLocation
}
return &osbuild.Sources{
"org.osbuild.files": files,
}
}
func (r *RHEL81) Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error) {
@ -632,43 +635,31 @@ func (r *RHEL81) Runner() string {
return "org.osbuild.rhel81"
}
func (r *RHEL81) buildPipeline(arch arch, checksums map[string]string) *osbuild.Pipeline {
packages, err := r.BuildPackages(arch.Name)
if err != nil {
panic("impossibly invalid arch")
}
func (r *RHEL81) buildPipeline(arch arch, buildPackageSpecs []rpmmd.PackageSpec) *osbuild.Pipeline {
p := &osbuild.Pipeline{}
p.AddStage(osbuild.NewDNFStage(r.dnfStageOptions(arch, nil, checksums, packages, nil)))
p.AddStage(osbuild.NewRPMStage(r.rpmStageOptions(arch, nil, buildPackageSpecs)))
return p
}
func (r *RHEL81) dnfStageOptions(arch arch, additionalRepos []rpmmd.RepoConfig, checksums map[string]string, packages, excludedPackages []string) *osbuild.DNFStageOptions {
options := &osbuild.DNFStageOptions{
ReleaseVersion: "8",
BaseArchitecture: arch.Name,
ModulePlatformId: ModulePlatformID,
}
for _, repo := range append(arch.Repositories, additionalRepos...) {
options.AddRepository(&osbuild.DNFRepository{
BaseURL: repo.BaseURL,
MetaLink: repo.Metalink,
MirrorList: repo.MirrorList,
Checksum: checksums[repo.Id],
})
func (r *RHEL81) rpmStageOptions(arch arch, additionalRepos []rpmmd.RepoConfig, specs []rpmmd.PackageSpec) *osbuild.RPMStageOptions {
var gpgKeys []string
repos := append(arch.Repositories, additionalRepos...)
for _, repo := range repos {
if repo.GPGKey == "" {
continue
}
gpgKeys = append(gpgKeys, repo.GPGKey)
}
sort.Strings(packages)
for _, pkg := range packages {
options.AddPackage(pkg)
var packages []string
for _, spec := range specs {
packages = append(packages, spec.Checksum)
}
sort.Strings(excludedPackages)
for _, pkg := range excludedPackages {
options.ExcludePackage(pkg)
return &osbuild.RPMStageOptions{
GPGKeys: gpgKeys,
Packages: packages,
}
return options
}
func (r *RHEL81) userStageOptions(users []blueprint.UserCustomization) (*osbuild.UsersStageOptions, error) {

64
internal/distro/rhel82/distro.go
View file

@ -537,14 +537,9 @@ func (r *RHEL82) pipeline(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoCo
}
p := &osbuild.Pipeline{}
p.SetBuild(r.buildPipeline(arch, checksums), "org.osbuild.rhel82")
p.SetBuild(r.buildPipeline(arch, buildPackageSpecs), "org.osbuild.rhel82")
packages, excludedPackages, err := r.BasePackages(outputFormat, outputArchitecture)
if err != nil {
return nil, err
}
packages = append(packages, b.GetPackages()...)
p.AddStage(osbuild.NewDNFStage(r.dnfStageOptions(arch, additionalRepos, checksums, packages, excludedPackages)))
p.AddStage(osbuild.NewRPMStage(r.rpmStageOptions(arch, additionalRepos, packageSpecs)))
p.AddStage(osbuild.NewFixBLSStage())
if output.Bootable {
@ -613,7 +608,15 @@ func (r *RHEL82) pipeline(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoCo
}
func (r *RHEL82) sources(packages []rpmmd.PackageSpec) *osbuild.Sources {
return &osbuild.Sources{}
files := &osbuild.FilesSource{
URLs: make(map[string]string),
}
for _, pkg := range packages {
files.URLs[pkg.Checksum] = pkg.RemoteLocation
}
return &osbuild.Sources{
"org.osbuild.files": files,
}
}
func (r *RHEL82) Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error) {
@ -632,45 +635,32 @@ func (r *RHEL82) Runner() string {
return "org.osbuild.rhel82"
}
func (r *RHEL82) buildPipeline(arch arch, checksums map[string]string) *osbuild.Pipeline {
packages, err := r.BuildPackages(arch.Name)
if err != nil {
panic("impossibly invalid arch")
}
func (r *RHEL82) buildPipeline(arch arch, buildPackageSpecs []rpmmd.PackageSpec) *osbuild.Pipeline {
p := &osbuild.Pipeline{}
p.AddStage(osbuild.NewDNFStage(r.dnfStageOptions(arch, nil, checksums, packages, nil)))
p.AddStage(osbuild.NewRPMStage(r.rpmStageOptions(arch, nil, buildPackageSpecs)))
return p
}
func (r *RHEL82) dnfStageOptions(arch arch, additionalRepos []rpmmd.RepoConfig, checksums map[string]string, packages, excludedPackages []string) *osbuild.DNFStageOptions {
options := &osbuild.DNFStageOptions{
ReleaseVersion: "8",
BaseArchitecture: arch.Name,
ModulePlatformId: ModulePlatformID,
}
for _, repo := range append(arch.Repositories, additionalRepos...) {
options.AddRepository(&osbuild.DNFRepository{
BaseURL: repo.BaseURL,
MetaLink: repo.Metalink,
MirrorList: repo.MirrorList,
Checksum: checksums[repo.Id],
})
func (r *RHEL82) rpmStageOptions(arch arch, additionalRepos []rpmmd.RepoConfig, specs []rpmmd.PackageSpec) *osbuild.RPMStageOptions {
var gpgKeys []string
repos := append(arch.Repositories, additionalRepos...)
for _, repo := range repos {
if repo.GPGKey == "" {
continue
}
gpgKeys = append(gpgKeys, repo.GPGKey)
}
sort.Strings(packages)
for _, pkg := range packages {
options.AddPackage(pkg)
var packages []string
for _, spec := range specs {
packages = append(packages, spec.Checksum)
}
sort.Strings(excludedPackages)
for _, pkg := range excludedPackages {
options.ExcludePackage(pkg)
return &osbuild.RPMStageOptions{
GPGKeys: gpgKeys,
Packages: packages,
}
return options
}
func (r *RHEL82) userStageOptions(users []blueprint.UserCustomization) (*osbuild.UsersStageOptions, error) {
options := osbuild.UsersStageOptions{
Users: make(map[string]osbuild.UsersStageOptionsUser),