particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

repositories: install the base repositories in the filesystem

Browse source 
During development of a new distro, we need to test composer against
nightly or beta repositories, but we cannot ship composer itself
with the nightly repository information hardcoded in. At the same
time, we want to distinguish between the system repositories of the
host and the repositories we use to generate images (the host may not
use the same distro/version/architecture as the target, and it may
include custom repositories that the target should not).

We therefore ship per distro repository information that can be
overriden (typically in testing) by dropping files in /etc.

For now use the latest nightlies for RHEL-8.2, we may want to
replace these with the official mirrors for GA eventually.

Signed-off-by: Tom Gundersen <teg@jklm.no>
This commit is contained in:
Tom Gundersen 2019-12-18 00:08:41 +01:00
parent 2059f9500b
commit 4919ef1271
23 changed files with 204 additions and 135 deletions
Download patch file Download diff file Expand all files Collapse all files

6
internal/distro/distro.go
View file

@ -44,12 +44,12 @@ type Registry struct {
distros map[string]Distro
}
func NewRegistry() *Registry {
func NewRegistry(confPaths []string) *Registry {
distros := &Registry{
distros: make(map[string]Distro),
}
distros.register(fedora30.New())
distros.register(rhel82.New())
distros.register(fedora30.New(confPaths))
distros.register(rhel82.New(confPaths))
return distros
}

2
internal/distro/distro_test.go
View file

@ -42,7 +42,7 @@ func TestDistro_Pipeline(t *testing.T) {
continue
}
t.Run(tt.Compose.OutputFormat, func(t *testing.T) {
distros := distro.NewRegistry()
distros := distro.NewRegistry([]string{"../.."})
d := distros.GetDistro(tt.Compose.Distro)
if d == nil {
t.Errorf("unknown distro: %v", tt.Compose.Distro)

97
internal/distro/fedora30/distro.go
View file

@ -2,6 +2,7 @@ package fedora30
import (
"errors"
"log"
"sort"
"strconv"
@ -23,6 +24,7 @@ type arch struct {
BootloaderPackages []string
BuildPackages []string
UEFI bool
Repositories []rpmmd.RepoConfig
}
type output struct {
@ -39,32 +41,50 @@ type output struct {
const Name = "fedora-30"
func New() *Fedora30 {
func New(confPaths []string) *Fedora30 {
r := Fedora30{
arches: map[string]arch{},
outputs: map[string]output{},
}
r.arches["x86_64"] = arch{
Name: "x86_64",
BootloaderPackages: []string{
"grub2-pc",
},
BuildPackages: []string{
"grub2-pc",
},
repoMap, err := rpmmd.LoadRepositories(confPaths, Name)
if err != nil {
log.Printf("Could not load repository data for %s: %s", Name, err.Error())
return nil
}
r.arches["aarch64"] = arch{
Name: "aarch64",
BootloaderPackages: []string{
"dracut-config-generic",
"efibootmgr",
"grub2-efi-aa64",
"grub2-tools",
"shim-aa64",
},
UEFI: true,
repos, exists := repoMap["x86_64"]
if !exists {
log.Printf("Could not load architecture-specific repository data for x86_64 (%s): %s", Name, err.Error())
} else {
r.arches["x86_64"] = arch{
Name: "x86_64",
BootloaderPackages: []string{
"grub2-pc",
},
BuildPackages: []string{
"grub2-pc",
},
Repositories: repos,
}
}
repos, exists = repoMap["aarch64"]
if !exists {
log.Printf("Could not load architecture-specific repository data for x86_64 (%s): %s", Name, err.Error())
} else {
r.arches["aarch64"] = arch{
Name: "aarch64",
BootloaderPackages: []string{
"dracut-config-generic",
"efibootmgr",
"grub2-efi-aa64",
"grub2-tools",
"shim-aa64",
},
UEFI: true,
Repositories: repos,
}
}
r.outputs["ami"] = output{
@ -259,42 +279,7 @@ func (r *Fedora30) Name() string {
}
func (r *Fedora30) Repositories(arch string) []rpmmd.RepoConfig {
return []rpmmd.RepoConfig{
{
Id: "fedora",
Name: "Fedora 30",
Metalink: "https://mirrors.fedoraproject.org/metalink?repo=fedora-30&arch=" + arch,
GPGKey: `-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBFturGcBEACv0xBo91V2n0uEC2vh69ywCiSyvUgN/AQH8EZpCVtM7NyjKgKm
bbY4G3R0M3ir1xXmvUDvK0493/qOiFrjkplvzXFTGpPTi0ypqGgxc5d0ohRA1M75
L+0AIlXoOgHQ358/c4uO8X0JAA1NYxCkAW1KSJgFJ3RjukrfqSHWthS1d4o8fhHy
KJKEnirE5hHqB50dafXrBfgZdaOs3C6ppRIePFe2o4vUEapMTCHFw0woQR8Ah4/R
n7Z9G9Ln+0Cinmy0nbIDiZJ+pgLAXCOWBfDUzcOjDGKvcpoZharA07c0q1/5ojzO
4F0Fh4g/BUmtrASwHfcIbjHyCSr1j/3Iz883iy07gJY5Yhiuaqmp0o0f9fgHkG53
2xCU1owmACqaIBNQMukvXRDtB2GJMuKa/asTZDP6R5re+iXs7+s9ohcRRAKGyAyc
YKIQKcaA+6M8T7/G+TPHZX6HJWqJJiYB+EC2ERblpvq9TPlLguEWcmvjbVc31nyq
SDoO3ncFWKFmVsbQPTbP+pKUmlLfJwtb5XqxNR5GEXSwVv4I7IqBmJz1MmRafnBZ
g0FJUtH668GnldO20XbnSVBr820F5SISMXVwCXDXEvGwwiB8Lt8PvqzXnGIFDAu3
DlQI5sxSqpPVWSyw08ppKT2Tpmy8adiBotLfaCFl2VTHwOae48X2dMPBvQARAQAB
tDFGZWRvcmEgKDMwKSA8ZmVkb3JhLTMwLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v
cmc+iQI4BBMBAgAiBQJbbqxnAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK
CRDvPBEfz8ZZudTnD/9170LL3nyTVUCFmBjT9wZ4gYnpwtKVPa/pKnxbbS+Bmmac
g9TrT9pZbqOHrNJLiZ3Zx1Hp+8uxr3Lo6kbYwImLhkOEDrf4aP17HfQ6VYFbQZI8
f79OFxWJ7si9+3gfzeh9UYFEqOQfzIjLWFyfnas0OnV/P+RMQ1Zr+vPRqO7AR2va
N9wg+Xl7157dhXPCGYnGMNSoxCbpRs0JNlzvJMuAea5nTTznRaJZtK/xKsqLn51D
K07k9MHVFXakOH8QtMCUglbwfTfIpO5YRq5imxlWbqsYWVQy1WGJFyW6hWC0+RcJ
Ox5zGtOfi4/dN+xJ+ibnbyvy/il7Qm+vyFhCYqIPyS5m2UVJUuao3eApE38k78/o
8aQOTnFQZ+U1Sw+6woFTxjqRQBXlQm2+7Bt3bqGATg4sXXWPbmwdL87Ic+mxn/ml
SMfQux/5k6iAu1kQhwkO2YJn9eII6HIPkW+2m5N1JsUyJQe4cbtZE5Yh3TRA0dm7
+zoBRfCXkOW4krchbgww/ptVmzMMP7GINJdROrJnsGl5FVeid9qHzV7aZycWSma7
CxBYB1J8HCbty5NjtD6XMYRrMLxXugvX6Q4NPPH+2NKjzX4SIDejS6JjgrP3KA3O
pMuo7ZHMfveBngv8yP+ZD/1sS6l+dfExvdaJdOdgFCnp4p3gPbw5+Lv70HrMjA==
=BfZ/
-----END PGP PUBLIC KEY BLOCK-----
`,
},
}
return r.arches[arch].Repositories
}
func (r *Fedora30) ListOutputFormats() []string {
@ -420,7 +405,7 @@ func (r *Fedora30) dnfStageOptions(arch arch, additionalRepos []rpmmd.RepoConfig
ModulePlatformId: "platform:f30",
}
for _, repo := range append(r.Repositories(arch.Name), additionalRepos...) {
for _, repo := range append(arch.Repositories, additionalRepos...) {
options.AddRepository(&pipeline.DNFRepository{
BaseURL: repo.BaseURL,
MetaLink: repo.Metalink,

4
internal/distro/fedora30/distro_test.go
View file

@ -19,7 +19,7 @@ func TestListOutputFormats(t *testing.T) {
"vmdk",
}
distros := distro.NewRegistry()
distros := distro.NewRegistry([]string{"../../../"})
f30 := distros.GetDistro("fedora-30")
if got := f30.ListOutputFormats(); !reflect.DeepEqual(got, want) {
t.Errorf("ListOutputFormats() = %v, want %v", got, want)
@ -93,7 +93,7 @@ func TestFilenameFromType(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
distros := distro.NewRegistry()
distros := distro.NewRegistry([]string{"../../../"})
f30 := distros.GetDistro("fedora-30")
got, got1, err := f30.FilenameFromType(tt.args.outputFormat)
if (err != nil) != tt.wantErr {

73
internal/distro/rhel82/distro.go
View file

@ -2,6 +2,7 @@ package rhel82
import (
"errors"
"log"
"sort"
"strconv"
@ -23,6 +24,7 @@ type arch struct {
BootloaderPackages []string
BuildPackages []string
UEFI bool
Repositories []rpmmd.RepoConfig
}
type output struct {
@ -40,7 +42,7 @@ type output struct {
const Name = "rhel-8.2"
func New() *RHEL82 {
func New(confPaths []string) *RHEL82 {
const GigaByte = 1024 * 1024 * 1024
r := RHEL82{
@ -48,26 +50,44 @@ func New() *RHEL82 {
outputs: map[string]output{},
}
r.arches["x86_64"] = arch{
Name: "x86_64",
BootloaderPackages: []string{
"grub2-pc",
},
BuildPackages: []string{
"grub2-pc",
},
repoMap, err := rpmmd.LoadRepositories(confPaths, Name)
if err != nil {
log.Printf("Could not load repository data for %s: %s", Name, err.Error())
return nil
}
r.arches["aarch64"] = arch{
Name: "aarch64",
BootloaderPackages: []string{
"dracut-config-generic",
"efibootmgr",
"grub2-efi-aa64",
"grub2-tools",
"shim-aa64",
},
UEFI: true,
repos, exists := repoMap["x86_64"]
if !exists {
log.Printf("Could not load architecture-specific repository data for x86_64 (%s): %s", Name, err.Error())
} else {
r.arches["x86_64"] = arch{
Name: "x86_64",
BootloaderPackages: []string{
"grub2-pc",
},
BuildPackages: []string{
"grub2-pc",
},
Repositories: repos,
}
}
repos, exists = repoMap["aarch64"]
if !exists {
log.Printf("Could not load architecture-specific repository data for aarch64 (%s): %s", Name, err.Error())
} else {
r.arches["aarch64"] = arch{
Name: "aarch64",
BootloaderPackages: []string{
"dracut-config-generic",
"efibootmgr",
"grub2-efi-aa64",
"grub2-tools",
"shim-aa64",
},
UEFI: true,
Repositories: repos,
}
}
r.outputs["ami"] = output{
@ -399,18 +419,7 @@ func (r *RHEL82) Name() string {
}
func (r *RHEL82) Repositories(arch string) []rpmmd.RepoConfig {
return []rpmmd.RepoConfig{
{
Id: "baseos",
Name: "BaseOS",
BaseURL: "http://download-ipv4.eng.brq.redhat.com/rhel-8/nightly/RHEL-8/RHEL-8.2.0-20191213.n.1/compose/BaseOS/" + arch + "/os",
},
{
Id: "appstream",
Name: "AppStream",
BaseURL: "http://download-ipv4.eng.brq.redhat.com/rhel-8/nightly/RHEL-8/RHEL-8.2.0-20191213.n.1/compose/AppStream/" + arch + "/os",
},
}
return r.arches[arch].Repositories
}
func (r *RHEL82) ListOutputFormats() []string {
@ -544,7 +553,7 @@ func (r *RHEL82) dnfStageOptions(arch arch, additionalRepos []rpmmd.RepoConfig,
BaseArchitecture: arch.Name,
ModulePlatformId: "platform:el8",
}
for _, repo := range append(r.Repositories(arch.Name), additionalRepos...) {
for _, repo := range append(arch.Repositories, additionalRepos...) {
options.AddRepository(&pipeline.DNFRepository{
BaseURL: repo.BaseURL,
MetaLink: repo.Metalink,

4
internal/distro/rhel82/distro_test.go
View file

@ -19,7 +19,7 @@ func TestListOutputFormats(t *testing.T) {
"vmdk",
}
distros := distro.NewRegistry()
distros := distro.NewRegistry([]string{"../../../"})
rhel82 := distros.GetDistro("rhel-8.2")
if got := rhel82.ListOutputFormats(); !reflect.DeepEqual(got, want) {
t.Errorf("ListOutputFormats() = %v, want %v", got, want)
@ -93,7 +93,7 @@ func TestFilenameFromType(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
distros := distro.NewRegistry()
distros := distro.NewRegistry([]string{"../../../"})
rhel82 := distros.GetDistro("rhel-8.2")
got, got1, err := rhel82.FilenameFromType(tt.args.outputFormat)
if (err != nil) != tt.wantErr {

2
internal/jobqueue/job.go
View file

@ -30,7 +30,7 @@ type JobStatus struct {
}
func (job *Job) Run() (*store.Image, error, []error) {
distros := distro.NewRegistry()
distros := distro.NewRegistry([]string{"/etc/osbuild-composer", "/usr/share/osbuild-composer"})
d := distros.GetDistro(job.Distro)
if d == nil {
return nil, fmt.Errorf("unknown distro: %s", job.Distro), nil

25
internal/rpmmd/repository.go
View file

@ -106,6 +106,31 @@ func (err *DNFError) Error() string {
return fmt.Sprintf("DNF error occured: %s: %s", err.Kind, err.Reason)
}
func LoadRepositories(confPaths []string, distro string) (map[string][]RepoConfig, error) {
var f *os.File
var err error
path := "/repositories/" + distro + ".json"
for _, confPath := range confPaths {
f, err = os.Open(confPath + path)
if err == nil {
break
} else if !os.IsNotExist(err) {
return nil, err
}
}
defer f.Close()
var repos map[string][]RepoConfig
err = json.NewDecoder(f).Decode(&repos)
if err != nil {
return nil, err
}
return repos, nil
}
func runDNF(command string, arguments interface{}, result interface{}) error {
var call = struct {
Command string `json:"command"`