particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

internal/rpmmd: RepoConfig baseurl change

Browse source 
Update the internal RepoConfig object to
accept a slice of baseurls rather than a
single field. This change was needed to
align RepoConfig with the dnf spec [1].

Additionally, this change adds custom json
marshal and unmarshal functions to ensure
backwards compatibility with older workers.
Add json tags to the internal rpmmd config
since this is serialized in dnfjson.
Add unit tests to check the serialization
is okay.

[1] See dnf.config
This commit is contained in:
Gianluca Zuccarelli 2023-01-23 16:07:21 +00:00 • committed by Tomáš Hozza
parent 17d730593c
commit 4d42808b6a
23 changed files with 368 additions and 198 deletions
Download patch file Download diff file Expand all files Collapse all files

7
cmd/gen-manifests/main.go
View file

@ -187,6 +187,11 @@ func makeManifestJob(name string, imgType distro.ImageType, cr composeRequest, d
type DistroArchRepoMap map[string]map[string][]repository
func convertRepo(r repository) rpmmd.RepoConfig {
var urls []string
if r.BaseURL != "" {
urls = []string{r.BaseURL}
}
var keys []string
if r.GPGKey != "" {
keys = []string{r.GPGKey}
@ -194,7 +199,7 @@ func convertRepo(r repository) rpmmd.RepoConfig {
return rpmmd.RepoConfig{
Name: r.Name,
BaseURL: r.BaseURL,
BaseURLs: urls,
Metalink: r.Metalink,
MirrorList: r.MirrorList,
GPGKeys: keys,

8
cmd/osbuild-pipeline/main.go
View file

@ -140,15 +140,17 @@ func main() {
if repoName == "" {
repoName = fmt.Sprintf("repo-%d", i)
}
var urls []string
if repo.BaseURL != "" {
urls = []string{repo.BaseURL}
}
var keys []string
if repo.GPGKey != "" {
keys = []string{repo.GPGKey}
}
repos[i] = rpmmd.RepoConfig{
Name: repoName,
BaseURL: repo.BaseURL,
BaseURLs: urls,
Metalink: repo.Metalink,
MirrorList: repo.MirrorList,
GPGKeys: keys,

2
internal/client/unit_test.go
View file

@ -75,7 +75,7 @@ func executeTests(m *testing.M) int {
rr := reporegistry.NewFromDistrosRepoConfigs(rpmmd.DistrosRepoConfigs{
test_distro.TestDistroName: {
test_distro.TestArchName: {
{Name: "test-system-repo", BaseURL: "http://example.com/test/os/test_arch"},
{Name: "test-system-repo", BaseURLs: []string{"http://example.com/test/os/test_arch"}},
},
},
})

4
internal/cloudapi/v2/handler.go
View file

@ -1352,8 +1352,8 @@ func genRepoConfig(repo Repository) (*rpmmd.RepoConfig, error) {
repoConfig.RHSM = repo.Rhsm != nil && *repo.Rhsm
if repo.Baseurl != nil {
repoConfig.BaseURL = *repo.Baseurl
if repo.Baseurl != nil && *repo.Baseurl != "" {
repoConfig.BaseURLs = []string{*repo.Baseurl}
} else if repo.Mirrorlist != nil {
repoConfig.MirrorList = *repo.Mirrorlist
} else if repo.Metalink != nil {

21
internal/cloudapi/v2/v2_internal_test.go
View file

@ -65,13 +65,13 @@ func TestCollectRepos(t *testing.T) {
}
expectedRepos := []rpmmd.RepoConfig{
{BaseURL: "http://example.com/baseos", PackageSets: nil},
{BaseURL: "http://example.com/appstream", PackageSets: nil},
{BaseURL: "http://example.com/baseos-rhel7", PackageSets: []string{"build"}},
{BaseURL: "http://example.com/extra-tools", PackageSets: []string{"build", "archive"}},
{BaseURL: "http://example.com/custom-os-stuff", PackageSets: []string{"blueprint"}},
{BaseURL: "http://example.com/repoone", PackageSets: []string{"blueprint"}},
{BaseURL: "http://example.com/repotwo", PackageSets: []string{"blueprint"}},
{BaseURLs: []string{"http://example.com/baseos"}, PackageSets: nil},
{BaseURLs: []string{"http://example.com/appstream"}, PackageSets: nil},
{BaseURLs: []string{"http://example.com/baseos-rhel7"}, PackageSets: []string{"build"}},
{BaseURLs: []string{"http://example.com/extra-tools"}, PackageSets: []string{"build", "archive"}},
{BaseURLs: []string{"http://example.com/custom-os-stuff"}, PackageSets: []string{"blueprint"}},
{BaseURLs: []string{"http://example.com/repoone"}, PackageSets: []string{"blueprint"}},
{BaseURLs: []string{"http://example.com/repotwo"}, PackageSets: []string{"blueprint"}},
}
payloadPkgSets := []string{"blueprint"}
@ -104,7 +104,7 @@ func TestRepoConfigConversion(t *testing.T) {
},
repoConfig: rpmmd.RepoConfig{
Name: "",
BaseURL: "http://base.url",
BaseURLs: []string{"http://base.url"},
Metalink: "",
MirrorList: "",
GPGKeys: []string{"some-kind-of-key"},
@ -128,7 +128,7 @@ func TestRepoConfigConversion(t *testing.T) {
},
repoConfig: rpmmd.RepoConfig{
Name: "",
BaseURL: "http://base.url",
BaseURLs: []string{"http://base.url"},
Metalink: "", // since BaseURL is specified, MetaLink is not copied
MirrorList: "", // since BaseURL is specified, MirrorList is not copied
CheckGPG: false,
@ -151,7 +151,6 @@ func TestRepoConfigConversion(t *testing.T) {
},
repoConfig: rpmmd.RepoConfig{
Name: "",
BaseURL: "",
Metalink: "", // since MirrorList is specified, MetaLink is not copied
MirrorList: "http://example.org/mirrorlist",
CheckGPG: false,
@ -174,7 +173,6 @@ func TestRepoConfigConversion(t *testing.T) {
},
repoConfig: rpmmd.RepoConfig{
Name: "",
BaseURL: "",
Metalink: "http://example.org/metalink",
MirrorList: "",
CheckGPG: false,
@ -214,7 +212,6 @@ func TestRepoConfigConversion(t *testing.T) {
// check gpg required but no gpgkey given
{
repo: Repository{
Baseurl: nil,
CheckGpg: common.ToPtr(true),
Gpgkey: nil,
IgnoreSsl: common.ToPtr(true),

74
internal/distro/distro_test.go
View file

@ -200,7 +200,7 @@ func TestImageTypePipelineNames(t *testing.T) {
repos := []rpmmd.RepoConfig{
{
Name: "payload",
BaseURL: "http://payload.example.com",
BaseURLs: []string{"http://payload.example.com"},
PackageSets: imageType.PayloadPackageSets(),
GPGKeys: []string{"payload-gpg-key"},
CheckGPG: true,
@ -287,12 +287,12 @@ func TestPipelineRepositories(t *testing.T) {
"globalonly": { // only global repos: most common scenario
repos: []rpmmd.RepoConfig{
{
Name: "global-1",
BaseURL: "http://global-1.example.com",
Name: "global-1",
BaseURLs: []string{"http://global-1.example.com"},
},
{
Name: "global-2",
BaseURL: "http://global-2.example.com",
Name: "global-2",
BaseURLs: []string{"http://global-2.example.com"},
},
},
result: map[string][]stringSet{
@ -302,21 +302,21 @@ func TestPipelineRepositories(t *testing.T) {
"global+build": { // global repos with build-specific repos: secondary common scenario
repos: []rpmmd.RepoConfig{
{
Name: "global-11",
BaseURL: "http://global-11.example.com",
Name: "global-11",
BaseURLs: []string{"http://global-11.example.com"},
},
{
Name: "global-12",
BaseURL: "http://global-12.example.com",
Name: "global-12",
BaseURLs: []string{"http://global-12.example.com"},
},
{
Name: "build-1",
BaseURL: "http://build-1.example.com",
BaseURLs: []string{"http://build-1.example.com"},
PackageSets: []string{"build"},
},
{
Name: "build-2",
BaseURL: "http://build-2.example.com",
BaseURLs: []string{"http://build-2.example.com"},
PackageSets: []string{"build"},
},
},
@ -328,21 +328,21 @@ func TestPipelineRepositories(t *testing.T) {
"global+os": { // global repos with os-specific repos
repos: []rpmmd.RepoConfig{
{
Name: "global-21",
BaseURL: "http://global-11.example.com",
Name: "global-21",
BaseURLs: []string{"http://global-11.example.com"},
},
{
Name: "global-22",
BaseURL: "http://global-12.example.com",
Name: "global-22",
BaseURLs: []string{"http://global-12.example.com"},
},
{
Name: "os-1",
BaseURL: "http://os-1.example.com",
BaseURLs: []string{"http://os-1.example.com"},
PackageSets: []string{"os"},
},
{
Name: "os-2",
BaseURL: "http://os-2.example.com",
BaseURLs: []string{"http://os-2.example.com"},
PackageSets: []string{"os"},
},
},
@ -354,26 +354,26 @@ func TestPipelineRepositories(t *testing.T) {
"global+os+payload": { // global repos with os-specific repos and (user-defined) payload repositories
repos: []rpmmd.RepoConfig{
{
Name: "global-21",
BaseURL: "http://global-11.example.com",
Name: "global-21",
BaseURLs: []string{"http://global-11.example.com"},
},
{
Name: "global-22",
BaseURL: "http://global-12.example.com",
Name: "global-22",
BaseURLs: []string{"http://global-12.example.com"},
},
{
Name: "os-1",
BaseURL: "http://os-1.example.com",
BaseURLs: []string{"http://os-1.example.com"},
PackageSets: []string{"os"},
},
{
Name: "os-2",
BaseURL: "http://os-2.example.com",
BaseURLs: []string{"http://os-2.example.com"},
PackageSets: []string{"os"},
},
{
Name: "payload",
BaseURL: "http://payload.example.com",
Name: "payload",
BaseURLs: []string{"http://payload.example.com"},
// User-defined payload repositories automatically get the "blueprint" key.
// This is handled by the APIs.
PackageSets: []string{"blueprint"},
@ -391,37 +391,37 @@ func TestPipelineRepositories(t *testing.T) {
repos: []rpmmd.RepoConfig{
{
Name: "build-1",
BaseURL: "http://build-1.example.com",
BaseURLs: []string{"http://build-1.example.com"},
PackageSets: []string{"build"},
},
{
Name: "build-2",
BaseURL: "http://build-2.example.com",
BaseURLs: []string{"http://build-2.example.com"},
PackageSets: []string{"build"},
},
{
Name: "os-1",
BaseURL: "http://os-1.example.com",
BaseURLs: []string{"http://os-1.example.com"},
PackageSets: []string{"os"},
},
{
Name: "os-2",
BaseURL: "http://os-2.example.com",
BaseURLs: []string{"http://os-2.example.com"},
PackageSets: []string{"os"},
},
{
Name: "anaconda-1",
BaseURL: "http://anaconda-1.example.com",
BaseURLs: []string{"http://anaconda-1.example.com"},
PackageSets: []string{"anaconda-tree"},
},
{
Name: "container-1",
BaseURL: "http://container-1.example.com",
BaseURLs: []string{"http://container-1.example.com"},
PackageSets: []string{"container-tree"},
},
{
Name: "coi-1",
BaseURL: "http://coi-1.example.com",
BaseURLs: []string{"http://coi-1.example.com"},
PackageSets: []string{"coi-tree"},
},
},
@ -437,16 +437,16 @@ func TestPipelineRepositories(t *testing.T) {
"global+unknown": { // package set names that don't match a pipeline are ignored
repos: []rpmmd.RepoConfig{
{
Name: "global-1",
BaseURL: "http://global-1.example.com",
Name: "global-1",
BaseURLs: []string{"http://global-1.example.com"},
},
{
Name: "global-2",
BaseURL: "http://global-2.example.com",
Name: "global-2",
BaseURLs: []string{"http://global-2.example.com"},
},
{
Name: "custom-1",
BaseURL: "http://custom.example.com",
BaseURLs: []string{"http://custom.example.com"},
PackageSets: []string{"notapipeline"},
},
},

7
internal/distro/distro_test_common/distro_test_common.go
View file

@ -62,14 +62,17 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, regis
repos := make([]rpmmd.RepoConfig, len(tt.ComposeRequest.Repositories))
for i, repo := range tt.ComposeRequest.Repositories {
var urls []string
if repo.BaseURL != "" {
urls = []string{repo.BaseURL}
}
var keys []string
if repo.GPGKey != "" {
keys = []string{repo.GPGKey}
}
repos[i] = rpmmd.RepoConfig{
Name: fmt.Sprintf("repo-%d", i),
BaseURL: repo.BaseURL,
BaseURLs: urls,
Metalink: repo.Metalink,
MirrorList: repo.MirrorList,
GPGKeys: keys,

2
internal/distro/rhel8/gce.go
View file

@ -99,7 +99,7 @@ func defaultGceByosImageConfig(rd distribution) *distro.ImageConfig {
{
Id: "google-compute-engine",
Name: "Google Compute Engine",
BaseURL: []string{"https://packages.cloud.google.com/yum/repos/google-compute-engine-el8-x86_64-stable"},
BaseURLs: []string{"https://packages.cloud.google.com/yum/repos/google-compute-engine-el8-x86_64-stable"},
Enabled: common.ToPtr(true),
GPGCheck: common.ToPtr(true),
RepoGPGCheck: common.ToPtr(false),

8
internal/distro/rhel9/gce.go
View file

@ -104,10 +104,10 @@ func baseGCEImageConfig(rhsm bool) *distro.ImageConfig {
Filename: "google-cloud.repo",
Repos: []osbuild.YumRepository{
{
Id: "google-compute-engine",
Name: "Google Compute Engine",
BaseURL: []string{"https://packages.cloud.google.com/yum/repos/google-compute-engine-el9-x86_64-stable"},
Enabled: common.ToPtr(true),
Id: "google-compute-engine",
Name: "Google Compute Engine",
BaseURLs: []string{"https://packages.cloud.google.com/yum/repos/google-compute-engine-el9-x86_64-stable"},
Enabled: common.ToPtr(true),
// TODO: enable GPG check once Google stops using SHA-1 in their keys
// https://issuetracker.google.com/issues/223626963
GPGCheck: common.ToPtr(false),

14
internal/dnfjson/dnfjson.go
View file

@ -262,7 +262,7 @@ func (s *Solver) reposFromRPMMD(rpmRepos []rpmmd.RepoConfig) ([]repoConfig, erro
dr := repoConfig{
ID: rr.Hash(),
Name: rr.Name,
BaseURL: rr.BaseURL,
BaseURLs: rr.BaseURLs,
Metalink: rr.Metalink,
MirrorList: rr.MirrorList,
GPGKeys: rr.GPGKeys,
@ -275,9 +275,9 @@ func (s *Solver) reposFromRPMMD(rpmRepos []rpmmd.RepoConfig) ([]repoConfig, erro
if s.subscriptions == nil {
return nil, fmt.Errorf("This system does not have any valid subscriptions. Subscribe it before specifying rhsm: true in sources.")
}
secrets, err := s.subscriptions.GetSecretsForBaseurl(rr.BaseURL, s.arch, s.releaseVer)
secrets, err := s.subscriptions.GetSecretsForBaseurl(rr.BaseURLs, s.arch, s.releaseVer)
if err != nil {
return nil, fmt.Errorf("RHSM secrets not found on the host for this baseurl: %s", rr.BaseURL)
return nil, fmt.Errorf("RHSM secrets not found on the host for this baseurl: %s", rr.BaseURLs)
}
dr.SSLCACert = secrets.SSLCACert
dr.SSLClientKey = secrets.SSLClientKey
@ -293,7 +293,7 @@ func (s *Solver) reposFromRPMMD(rpmRepos []rpmmd.RepoConfig) ([]repoConfig, erro
type repoConfig struct {
ID string `json:"id"`
Name string `json:"name,omitempty"`
BaseURL string `json:"baseurl,omitempty"`
BaseURLs []string `json:"baseurl,omitempty"`
Metalink string `json:"metalink,omitempty"`
MirrorList string `json:"mirrorlist,omitempty"`
GPGKeys []string `json:"gpgkeys,omitempty"`
@ -317,7 +317,7 @@ func (r *repoConfig) Hash() string {
ats := func(s []string) string {
return strings.Join(s, "")
}
return fmt.Sprintf("%x", sha256.Sum256([]byte(r.BaseURL+
return fmt.Sprintf("%x", sha256.Sum256([]byte(ats(r.BaseURLs)+
r.Metalink+
r.MirrorList+
ats(r.GPGKeys)+
@ -577,8 +577,8 @@ func parseError(data []byte, repos []repoConfig) Error {
for _, repo := range repos {
idstr := fmt.Sprintf("'%s'", repo.ID)
var nameURL string
if len(repo.BaseURL) > 0 {
nameURL = repo.BaseURL
if len(repo.BaseURLs) > 0 {
nameURL = strings.Join(repo.BaseURLs, ",")
} else if len(repo.Metalink) > 0 {
nameURL = repo.Metalink
} else if len(repo.MirrorList) > 0 {

113
internal/dnfjson/dnfjson_test.go
View file

@ -5,6 +5,7 @@ import (
"fmt"
"os"
"os/exec"
"strings"
"testing"
"github.com/osbuild/osbuild-composer/internal/mocks/rpmrepo"
@ -69,20 +70,20 @@ func TestDepsolver(t *testing.T) {
func TestMakeDepsolveRequest(t *testing.T) {
baseOS := rpmmd.RepoConfig{
Name: "baseos",
BaseURL: "https://example.org/baseos",
Name: "baseos",
BaseURLs: []string{"https://example.org/baseos"},
}
appstream := rpmmd.RepoConfig{
Name: "appstream",
BaseURL: "https://example.org/appstream",
Name: "appstream",
BaseURLs: []string{"https://example.org/appstream"},
}
userRepo := rpmmd.RepoConfig{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
Name: "user-repo",
BaseURLs: []string{"https://example.org/user-repo"},
}
userRepo2 := rpmmd.RepoConfig{
Name: "user-repo-2",
BaseURL: "https://example.org/user-repo-2",
Name: "user-repo-2",
BaseURLs: []string{"https://example.org/user-repo-2"},
}
tests := []struct {
packageSets []rpmmd.PackageSet
@ -111,14 +112,14 @@ func TestMakeDepsolveRequest(t *testing.T) {
},
wantRepos: []repoConfig{
{
ID: baseOS.Hash(),
Name: "baseos",
BaseURL: "https://example.org/baseos",
ID: baseOS.Hash(),
Name: "baseos",
BaseURLs: []string{"https://example.org/baseos"},
},
{
ID: appstream.Hash(),
Name: "appstream",
BaseURL: "https://example.org/appstream",
ID: appstream.Hash(),
Name: "appstream",
BaseURLs: []string{"https://example.org/appstream"},
},
},
},
@ -148,19 +149,19 @@ func TestMakeDepsolveRequest(t *testing.T) {
},
wantRepos: []repoConfig{
{
ID: baseOS.Hash(),
Name: "baseos",
BaseURL: "https://example.org/baseos",
ID: baseOS.Hash(),
Name: "baseos",
BaseURLs: []string{"https://example.org/baseos"},
},
{
ID: appstream.Hash(),
Name: "appstream",
BaseURL: "https://example.org/appstream",
ID: appstream.Hash(),
Name: "appstream",
BaseURLs: []string{"https://example.org/appstream"},
},
{
ID: userRepo.Hash(),
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
ID: userRepo.Hash(),
Name: "user-repo",
BaseURLs: []string{"https://example.org/user-repo"},
},
},
},
@ -190,14 +191,14 @@ func TestMakeDepsolveRequest(t *testing.T) {
},
wantRepos: []repoConfig{
{
ID: baseOS.Hash(),
Name: "baseos",
BaseURL: "https://example.org/baseos",
ID: baseOS.Hash(),
Name: "baseos",
BaseURLs: []string{"https://example.org/baseos"},
},
{
ID: appstream.Hash(),
Name: "appstream",
BaseURL: "https://example.org/appstream",
ID: appstream.Hash(),
Name: "appstream",
BaseURLs: []string{"https://example.org/appstream"},
},
},
},
@ -235,19 +236,19 @@ func TestMakeDepsolveRequest(t *testing.T) {
},
wantRepos: []repoConfig{
{
ID: baseOS.Hash(),
Name: "baseos",
BaseURL: "https://example.org/baseos",
ID: baseOS.Hash(),
Name: "baseos",
BaseURLs: []string{"https://example.org/baseos"},
},
{
ID: appstream.Hash(),
Name: "appstream",
BaseURL: "https://example.org/appstream",
ID: appstream.Hash(),
Name: "appstream",
BaseURLs: []string{"https://example.org/appstream"},
},
{
ID: userRepo.Hash(),
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
ID: userRepo.Hash(),
Name: "user-repo",
BaseURLs: []string{"https://example.org/user-repo"},
},
},
},
@ -286,24 +287,24 @@ func TestMakeDepsolveRequest(t *testing.T) {
},
wantRepos: []repoConfig{
{
ID: baseOS.Hash(),
Name: "baseos",
BaseURL: "https://example.org/baseos",
ID: baseOS.Hash(),
Name: "baseos",
BaseURLs: []string{"https://example.org/baseos"},
},
{
ID: appstream.Hash(),
Name: "appstream",
BaseURL: "https://example.org/appstream",
ID: appstream.Hash(),
Name: "appstream",
BaseURLs: []string{"https://example.org/appstream"},
},
{
ID: userRepo.Hash(),
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
ID: userRepo.Hash(),
Name: "user-repo",
BaseURLs: []string{"https://example.org/user-repo"},
},
{
ID: userRepo2.Hash(),
Name: "user-repo-2",
BaseURL: "https://example.org/user-repo-2",
ID: userRepo2.Hash(),
Name: "user-repo-2",
BaseURLs: []string{"https://example.org/user-repo-2"},
},
},
},
@ -479,7 +480,7 @@ func expectedResult(repo rpmmd.RepoConfig) []rpmmd.PackageSpec {
exp := []rpmmd.PackageSpec(expectedTemplate)
for idx := range exp {
urlTemplate := exp[idx].RemoteLocation
exp[idx].RemoteLocation = fmt.Sprintf(urlTemplate, repo.BaseURL)
exp[idx].RemoteLocation = fmt.Sprintf(urlTemplate, strings.Join(repo.BaseURLs, ","))
}
return exp
}
@ -503,7 +504,7 @@ func TestErrorRepoInfo(t *testing.T) {
{
repo: rpmmd.RepoConfig{
Name: "",
BaseURL: "https://0.0.0.0/baseos/repo",
BaseURLs: []string{"https://0.0.0.0/baseos/repo"},
Metalink: "https://0.0.0.0/baseos/metalink",
},
expMsg: "[https://0.0.0.0/baseos/repo]",
@ -511,7 +512,7 @@ func TestErrorRepoInfo(t *testing.T) {
{
repo: rpmmd.RepoConfig{
Name: "baseos",
BaseURL: "https://0.0.0.0/baseos/repo",
BaseURLs: []string{"https://0.0.0.0/baseos/repo"},
Metalink: "https://0.0.0.0/baseos/metalink",
},
expMsg: "[baseos: https://0.0.0.0/baseos/repo]",
@ -553,14 +554,14 @@ func TestRepoConfigHash(t *testing.T) {
rc := repoConfig{
ID: "repoid-1",
Name: "A test repository",
BaseURL: "https://arepourl/",
BaseURLs: []string{"https://arepourl/"},
IgnoreSSL: false,
}
hash := rc.Hash()
assert.Equal(t, 64, len(hash))
rc.BaseURL = "https://adifferenturl/"
rc.BaseURLs = []string{"https://adifferenturl/"}
assert.NotEqual(t, hash, rc.Hash())
}
@ -569,7 +570,7 @@ func TestRequestHash(t *testing.T) {
repos := []rpmmd.RepoConfig{
rpmmd.RepoConfig{
Name: "A test repository",
BaseURL: "https://arepourl/",
BaseURLs: []string{"https://arepourl/"},
IgnoreSSL: false,
},
}

2
internal/mocks/rpmrepo/rpmrepo.go
View file

@ -18,7 +18,7 @@ func NewTestServer() *testRepoServer {
server := httptest.NewServer(http.FileServer(http.Dir("../../test/data/testrepo/")))
testrepo := rpmmd.RepoConfig{
Name: "cs9-baseos",
BaseURL: server.URL,
BaseURLs: []string{server.URL},
CheckGPG: false,
IgnoreSSL: true,
RHSM: false,

6
internal/osbuild/yum_repos_stage.go
View file

@ -11,7 +11,7 @@ const repoIDRegex = "^[\\w.\\-:]+$"
// YumRepository represents a single DNF / YUM repository.
type YumRepository struct {
Id string `json:"id"`
BaseURL []string `json:"baseurl,omitempty"`
BaseURLs []string `json:"baseurl,omitempty"`
Cost *int `json:"cost,omitempty"`
Enabled *bool `json:"enabled,omitempty"`
GPGKey []string `json:"gpgkey,omitempty"`
@ -38,11 +38,11 @@ func (r YumRepository) validate() error {
}
// at least one of baseurl, metalink or mirrorlist must be provided
if len(r.BaseURL) == 0 && r.Metalink == "" && r.Mirrorlist == "" {
if len(r.BaseURLs) == 0 && r.Metalink == "" && r.Mirrorlist == "" {
return fmt.Errorf("at least one of baseurl, metalink or mirrorlist values must be provided")
}
for idx, url := range r.BaseURL {
for idx, url := range r.BaseURLs {
if url == "" {
return fmt.Errorf("baseurl must not be an empty string (idx %d)", idx)
}

28
internal/osbuild/yum_repos_stage_test.go
View file

@ -11,8 +11,8 @@ import (
func TestNewYumReposStage(t *testing.T) {
stageOptions := NewYumReposStageOptions("testing.repo", []YumRepository{
{
Id: "cool-id",
BaseURL: []string{"http://example.org/repo"},
Id: "cool-id",
BaseURLs: []string{"http://example.org/repo"},
},
})
expectedStage := &Stage{
@ -48,8 +48,8 @@ func TestYumReposStageOptionsValidate(t *testing.T) {
Filename: "@#$%^&.rap",
Repos: []YumRepository{
{
Id: "cool-id",
BaseURL: []string{"http://example.org/repo"},
Id: "cool-id",
BaseURLs: []string{"http://example.org/repo"},
},
},
},
@ -60,8 +60,8 @@ func TestYumReposStageOptionsValidate(t *testing.T) {
options: YumReposStageOptions{
Repos: []YumRepository{
{
Id: "cool-id",
BaseURL: []string{"http://example.org/repo"},
Id: "cool-id",
BaseURLs: []string{"http://example.org/repo"},
},
},
},
@ -85,8 +85,8 @@ func TestYumReposStageOptionsValidate(t *testing.T) {
Filename: "test.repo",
Repos: []YumRepository{
{
Id: "cool-id",
BaseURL: []string{""},
Id: "cool-id",
BaseURLs: []string{""},
},
},
},
@ -98,9 +98,9 @@ func TestYumReposStageOptionsValidate(t *testing.T) {
Filename: "test.repo",
Repos: []YumRepository{
{
Id: "cool-id",
BaseURL: []string{"http://example.org/repo"},
GPGKey: []string{""},
Id: "cool-id",
BaseURLs: []string{"http://example.org/repo"},
GPGKey: []string{""},
},
},
},
@ -112,8 +112,8 @@ func TestYumReposStageOptionsValidate(t *testing.T) {
Filename: "test.repo",
Repos: []YumRepository{
{
Id: "c@@l-id",
BaseURL: []string{"http://example.org/repo"},
Id: "c@@l-id",
BaseURLs: []string{"http://example.org/repo"},
},
},
},
@ -132,7 +132,7 @@ func TestYumReposStageOptionsValidate(t *testing.T) {
Name: "c@@l-name",
GPGCheck: common.ToPtr(true),
RepoGPGCheck: common.ToPtr(true),
BaseURL: []string{"http://example.org/repo"},
BaseURLs: []string{"http://example.org/repo"},
GPGKey: []string{"secretkey"},
},
},

18
internal/reporegistry/reporegistry_test.go
View file

@ -16,32 +16,32 @@ func getTestingRepoRegistry() *RepoRegistry {
test_distro.TestDistroName: {
test_distro.TestArchName: {
{
Name: "baseos",
BaseURL: "https://cdn.redhat.com/content/dist/rhel8/8/x86_64/baseos/os",
Name: "baseos",
BaseURLs: []string{"https://cdn.redhat.com/content/dist/rhel8/8/x86_64/baseos/os"},
},
{
Name: "appstream",
BaseURL: "https://cdn.redhat.com/content/dist/rhel8/8/x86_64/appstream/os",
Name: "appstream",
BaseURLs: []string{"https://cdn.redhat.com/content/dist/rhel8/8/x86_64/appstream/os"},
},
},
test_distro.TestArch2Name: {
{
Name: "baseos",
BaseURL: "https://cdn.redhat.com/content/dist/rhel8/8/aarch64/baseos/os",
Name: "baseos",
BaseURLs: []string{"https://cdn.redhat.com/content/dist/rhel8/8/aarch64/baseos/os"},
},
{
Name: "appstream",
BaseURL: "https://cdn.redhat.com/content/dist/rhel8/8/aarch64/appstream/os",
BaseURLs: []string{"https://cdn.redhat.com/content/dist/rhel8/8/aarch64/appstream/os"},
ImageTypeTags: []string{},
},
{
Name: "google-compute-engine",
BaseURL: "https://packages.cloud.google.com/yum/repos/google-compute-engine-el8-x86_64-stable",
BaseURLs: []string{"https://packages.cloud.google.com/yum/repos/google-compute-engine-el8-x86_64-stable"},
ImageTypeTags: []string{test_distro.TestImageType2Name},
},
{
Name: "google-cloud-sdk",
BaseURL: "https://packages.cloud.google.com/yum/repos/cloud-sdk-el8-x86_64",
BaseURLs: []string{"https://packages.cloud.google.com/yum/repos/cloud-sdk-el8-x86_64"},
ImageTypeTags: []string{test_distro.TestImageType2Name},
},
},

20
internal/rhsm/secrets.go
View file

@ -168,16 +168,18 @@ func parseRepoFile(content []byte) ([]subscription, error) {
}
// GetSecretsForBaseurl queries the Subscriptions structure for a RHSMSecrets of a single repository.
func (s *Subscriptions) GetSecretsForBaseurl(baseurl string, arch, releasever string) (*RHSMSecrets, error) {
func (s *Subscriptions) GetSecretsForBaseurl(baseurls []string, arch, releasever string) (*RHSMSecrets, error) {
for _, subs := range s.available {
url := strings.Replace(subs.baseurl, "$basearch", arch, -1)
url = strings.Replace(url, "$releasever", releasever, -1)
if url == baseurl {
return &RHSMSecrets{
SSLCACert: subs.sslCACert,
SSLClientKey: subs.sslClientKey,
SSLClientCert: subs.sslClientCert,
}, nil
for _, baseurl := range baseurls {
url := strings.Replace(subs.baseurl, "$basearch", arch, -1)
url = strings.Replace(url, "$releasever", releasever, -1)
if url == baseurl {
return &RHSMSecrets{
SSLCACert: subs.sslCACert,
SSLClientKey: subs.sslClientKey,
SSLClientCert: subs.sslClientCert,
}, nil
}
}
}
// If there is no matching URL, fall back to the global secrets

2
internal/rhsm/secrets_test.go
View file

@ -41,7 +41,7 @@ func TestParseRepoFile(t *testing.T) {
subscriptions := Subscriptions{
available: repoFileContent,
}
secrets, err := subscriptions.GetSecretsForBaseurl("https://cdn.redhat.com/content/dist/middleware/jws/1.0/x86_64/os", "x86_64", "")
secrets, err := subscriptions.GetSecretsForBaseurl([]string{"https://cdn.redhat.com/content/dist/middleware/jws/1.0/x86_64/os"}, "x86_64", "")
require.NoError(t, err, "Failed to get secrets for a baseurl")
assert.Equal(t, secrets.SSLCACert, "/etc/rhsm/ca/redhat-uep.pem", "Unexpected path to the CA certificate")
assert.Equal(t, secrets.SSLClientCert, "/etc/pki/entitlement/456.pem", "Unexpected path to the client cert")

87
internal/rpmmd/repository.go
View file

@ -28,18 +28,18 @@ type repository struct {
}
type RepoConfig struct {
Name string
BaseURL string
Metalink string
MirrorList string
GPGKeys []string
CheckGPG bool
CheckRepoGPG bool
IgnoreSSL bool
MetadataExpire string
RHSM bool
ImageTypeTags []string
PackageSets []string
Name string `json:"name,omitempty"`
BaseURLs []string `json:"baseurls,omitempty"`
Metalink string `json:"metalink,omitempty"`
MirrorList string `json:"mirrorlist,omitempty"`
GPGKeys []string `json:"gpgkeys,omitempty"`
CheckGPG bool `json:"check_gpg,omitempty"`
CheckRepoGPG bool `json:"check_repo_gpg,omitempty"`
IgnoreSSL bool `json:"ignore_ssl,omitempty"`
MetadataExpire string `json:"metadata_expire,omitempty"`
RHSM bool `json:"rhsm,omitempty"`
ImageTypeTags []string `json:"image_type_tags,omitempty"`
PackageSets []string `json:"package_sets,omitempty"`
}
// Hash calculates an ID string that uniquely represents a repository
@ -52,7 +52,7 @@ func (r *RepoConfig) Hash() string {
ats := func(s []string) string {
return strings.Join(s, "")
}
return fmt.Sprintf("%x", sha256.Sum256([]byte(r.BaseURL+
return fmt.Sprintf("%x", sha256.Sum256([]byte(ats(r.BaseURLs)+
r.Metalink+
r.MirrorList+
ats(r.GPGKeys)+
@ -223,14 +223,17 @@ func loadRepositoriesFromFile(filename string) (map[string][]RepoConfig, error)
for arch, repos := range reposMap {
for _, repo := range repos {
var urls []string
if repo.BaseURL != "" {
urls = []string{repo.BaseURL}
}
var keys []string
if repo.GPGKey != "" {
keys = []string{repo.GPGKey}
}
config := RepoConfig{
Name: repo.Name,
BaseURL: repo.BaseURL,
BaseURLs: urls,
Metalink: repo.Metalink,
MirrorList: repo.MirrorList,
GPGKeys: keys,
@ -375,3 +378,57 @@ func (packages PackageList) ToPackageInfos() []PackageInfo {
return results
}
// Backwards compatibility for old workers:
// This was added since the custom repository
// PR changes the baseurl field to a list of baseurls.
// This can be removed after 3 releases since the
// old-worker-regression test tests the current
// osbuild-composer with a worker from 3 releases ago
func (r RepoConfig) MarshalJSON() ([]byte, error) {
type aliasType RepoConfig
type compatType struct {
aliasType
BaseURL string `json:"baseurl,omitempty"`
}
compatRepo := compatType{
aliasType: aliasType(r),
}
var baseUrl string
if len(r.BaseURLs) > 0 {
baseUrl = strings.Join(r.BaseURLs, ",")
}
compatRepo.BaseURL = baseUrl
return json.Marshal(compatRepo)
}
// Backwards compatibility for old workers:
// This was added since the custom repository
// PR changes the baseurl field to a list of baseurls.
// This can be removed after 3 releases since the
// old-worker-regression test tests the current
// osbuild-composer with a worker from 3 releases ago
func (r *RepoConfig) UnmarshalJSON(data []byte) error {
type aliasType RepoConfig
type compatType struct {
aliasType
BaseURL string `json:"baseurl,omitempty"`
}
var compatRepo compatType
if err := json.Unmarshal(data, &compatRepo); err != nil {
return err
}
if compatRepo.BaseURL != "" {
compatRepo.BaseURLs = strings.Split(compatRepo.BaseURL, ",")
}
*r = RepoConfig(compatRepo.aliasType)
return nil
}

96
internal/rpmmd/test/repository_test.go
View file

@ -1,6 +1,7 @@
package rpmmdtests
import (
"encoding/json"
"fmt"
"path/filepath"
"reflect"
@ -189,3 +190,98 @@ func TestPackageSetResolveConflictExclude(t *testing.T) {
})
}
}
func TestOldWorkerRepositoryCompatUnmarshal(t *testing.T) {
testCases := []struct {
repoJSON []byte
repo rpmmd.RepoConfig
}{
{
repoJSON: []byte(`{"name":"fedora","baseurl":"http://example.com/fedora"}`),
repo: rpmmd.RepoConfig{
Name: "fedora",
BaseURLs: []string{"http://example.com/fedora"},
},
},
{
repoJSON: []byte(`{"name":"multiple","baseurl":"http://example.com/one,http://example.com/two"}`),
repo: rpmmd.RepoConfig{
Name: "multiple",
BaseURLs: []string{"http://example.com/one", "http://example.com/two"},
},
},
{
repoJSON: []byte(`{"name":"all","baseurls":["http://example.com/all"],"metalink":"http://example.com/metalink","mirrorlist":"http://example.com/mirrorlist","gpgkeys":["key1","key2"],"check_gpg":true,"check_repo_gpg":true,"ignore_ssl":true,"metadata_expire":"test","rhsm":true,"image_type_tags":["one","two"],"package_sets":["1","2"],"baseurl":"http://example.com/all"}`),
repo: rpmmd.RepoConfig{
Name: "all",
BaseURLs: []string{"http://example.com/all"},
Metalink: "http://example.com/metalink",
MirrorList: "http://example.com/mirrorlist",
GPGKeys: []string{"key1", "key2"},
CheckGPG: true,
CheckRepoGPG: true,
IgnoreSSL: true,
MetadataExpire: "test",
RHSM: true,
ImageTypeTags: []string{"one", "two"},
PackageSets: []string{"1", "2"},
},
},
}
for _, tc := range testCases {
t.Run(tc.repo.Name, func(t *testing.T) {
var repo rpmmd.RepoConfig
err := json.Unmarshal(tc.repoJSON, &repo)
assert.Nil(t, err)
assert.Equal(t, tc.repo, repo)
})
}
}
func TestOldWorkerRepositoryCompatMarshal(t *testing.T) {
testCases := []struct {
repoJSON []byte
repo rpmmd.RepoConfig
}{
{
repoJSON: []byte(`{"name":"fedora","baseurls":["http://example.com/fedora"],"baseurl":"http://example.com/fedora"}`),
repo: rpmmd.RepoConfig{
Name: "fedora",
BaseURLs: []string{"http://example.com/fedora"},
},
},
{
repoJSON: []byte(`{"name":"multiple","baseurls":["http://example.com/one","http://example.com/two"],"baseurl":"http://example.com/one,http://example.com/two"}`),
repo: rpmmd.RepoConfig{
Name: "multiple",
BaseURLs: []string{"http://example.com/one", "http://example.com/two"},
},
},
{
repoJSON: []byte(`{"name":"all","baseurls":["http://example.com/all"],"metalink":"http://example.com/metalink","mirrorlist":"http://example.com/mirrorlist","gpgkeys":["key1","key2"],"check_gpg":true,"check_repo_gpg":true,"ignore_ssl":true,"metadata_expire":"test","rhsm":true,"image_type_tags":["one","two"],"package_sets":["1","2"],"baseurl":"http://example.com/all"}`),
repo: rpmmd.RepoConfig{
Name: "all",
BaseURLs: []string{"http://example.com/all"},
Metalink: "http://example.com/metalink",
MirrorList: "http://example.com/mirrorlist",
GPGKeys: []string{"key1", "key2"},
CheckGPG: true,
CheckRepoGPG: true,
IgnoreSSL: true,
MetadataExpire: "test",
RHSM: true,
ImageTypeTags: []string{"one", "two"},
PackageSets: []string{"1", "2"},
},
},
}
for _, tc := range testCases {
t.Run(tc.repo.Name, func(t *testing.T) {
gotJson, err := json.Marshal(tc.repo)
assert.Nil(t, err)
assert.Equal(t, tc.repoJSON, gotJson)
})
}
}

13
internal/store/store.go
View file

@ -4,6 +4,8 @@ package store
import (
"crypto/rand"
"strings"
// The use of SHA1 is valid here
/* #nosec G505 */
"crypto/sha1"
@ -589,8 +591,8 @@ func NewSourceConfig(repo rpmmd.RepoConfig, system bool) SourceConfig {
GPGKeys: repo.GPGKeys,
}
if repo.BaseURL != "" {
sc.URL = repo.BaseURL
if len(repo.BaseURLs) != 0 {
sc.URL = strings.Join(repo.BaseURLs, ",")
sc.Type = "yum-baseurl"
} else if repo.Metalink != "" {
sc.URL = repo.Metalink
@ -613,8 +615,13 @@ func (s *SourceConfig) RepoConfig(name string) rpmmd.RepoConfig {
repo.CheckRepoGPG = s.CheckRepoGPG
repo.GPGKeys = s.GPGKeys
var urls []string
if s.URL != "" {
urls = []string{s.URL}
}
if s.Type == "yum-baseurl" {
repo.BaseURL = s.URL
repo.BaseURLs = urls
} else if s.Type == "yum-metalink" {
repo.Metalink = s.URL
} else if s.Type == "yum-mirrorlist" {

10
internal/store/store_test.go
View file

@ -421,7 +421,7 @@ func (suite *storeTest) TestGetAllSourcesByID() {
func (suite *storeTest) TestNewSourceConfigWithBaseURL() {
myRepoConfig := rpmmd.RepoConfig{
Name: "testRepo",
BaseURL: "testURL",
BaseURLs: []string{"testURL"},
CheckGPG: true,
}
expectedSource := SourceConfig{Name: "testRepo", Type: "yum-baseurl", URL: "testURL", CheckGPG: true, CheckSSL: true, System: true}
@ -452,7 +452,7 @@ func (suite *storeTest) TestNewSourceConfigWithMirrorList() {
// Test converting a SourceConfig with GPGkeys to a RepoConfig
func (suite *storeTest) TestRepoConfigGPGKeys() {
expectedRepo := rpmmd.RepoConfig{Name: "testSourceConfig", BaseURL: "testURL", Metalink: "", MirrorList: "", IgnoreSSL: true, MetadataExpire: "", CheckRepoGPG: true, GPGKeys: []string{"http://path.to.gpgkeys/key.pub", "-----BEGIN PGP PUBLIC KEY BLOCK-----\nFULL GPG KEY HERE\n-----END PGP PUBLIC KEY BLOCK-----"}}
expectedRepo := rpmmd.RepoConfig{Name: "testSourceConfig", BaseURLs: []string{"testURL"}, Metalink: "", MirrorList: "", IgnoreSSL: true, MetadataExpire: "", CheckRepoGPG: true, GPGKeys: []string{"http://path.to.gpgkeys/key.pub", "-----BEGIN PGP PUBLIC KEY BLOCK-----\nFULL GPG KEY HERE\n-----END PGP PUBLIC KEY BLOCK-----"}}
mySourceConfig := suite.mySourceConfig
mySourceConfig.Type = "yum-baseurl"
mySourceConfig.URL = "testURL"
@ -463,7 +463,7 @@ func (suite *storeTest) TestRepoConfigGPGKeys() {
}
func (suite *storeTest) TestRepoConfigBaseURL() {
expectedRepo := rpmmd.RepoConfig{Name: "testSourceConfig", BaseURL: "testURL", Metalink: "", MirrorList: "", IgnoreSSL: true, MetadataExpire: ""}
expectedRepo := rpmmd.RepoConfig{Name: "testSourceConfig", BaseURLs: []string{"testURL"}, Metalink: "", MirrorList: "", IgnoreSSL: true, MetadataExpire: ""}
suite.mySourceConfig.Type = "yum-baseurl"
suite.mySourceConfig.URL = "testURL"
actualRepo := suite.mySourceConfig.RepoConfig("testSourceConfig")
@ -471,7 +471,7 @@ func (suite *storeTest) TestRepoConfigBaseURL() {
}
func (suite *storeTest) TestRepoConfigMetalink() {
expectedRepo := rpmmd.RepoConfig{Name: "testSourceConfig", BaseURL: "", Metalink: "testURL", MirrorList: "", IgnoreSSL: true, MetadataExpire: ""}
expectedRepo := rpmmd.RepoConfig{Name: "testSourceConfig", Metalink: "testURL", MirrorList: "", IgnoreSSL: true, MetadataExpire: ""}
suite.mySourceConfig.Type = "yum-metalink"
suite.mySourceConfig.URL = "testURL"
actualRepo := suite.mySourceConfig.RepoConfig("testSourceConfig")
@ -479,7 +479,7 @@ func (suite *storeTest) TestRepoConfigMetalink() {
}
func (suite *storeTest) TestRepoConfigMirrorlist() {
expectedRepo := rpmmd.RepoConfig{Name: "testSourceConfig", BaseURL: "", Metalink: "", MirrorList: "testURL", IgnoreSSL: true, MetadataExpire: ""}
expectedRepo := rpmmd.RepoConfig{Name: "testSourceConfig", Metalink: "", MirrorList: "testURL", IgnoreSSL: true, MetadataExpire: ""}
suite.mySourceConfig.Type = "yum-mirrorlist"
suite.mySourceConfig.URL = "testURL"
actualRepo := suite.mySourceConfig.RepoConfig("testSourceConfig")

8
internal/weldr/api_test.go
View file

@ -63,12 +63,12 @@ func createWeldrAPI(tempdir string, fixtureGenerator rpmmd_mock.FixtureGenerator
rr := reporegistry.NewFromDistrosRepoConfigs(rpmmd.DistrosRepoConfigs{
test_distro.TestDistroName: {
test_distro.TestArchName: {
{Name: "test-id", BaseURL: "http://example.com/test/os/x86_64", CheckGPG: true},
{Name: "test-id", BaseURLs: []string{"http://example.com/test/os/x86_64"}, CheckGPG: true},
},
},
test_distro.TestDistro2Name: {
test_distro.TestArchName: {
{Name: "test-id-2", BaseURL: "http://example.com/test-2/os/x86_64", CheckGPG: true},
{Name: "test-id-2", BaseURLs: []string{"http://example.com/test-2/os/x86_64"}, CheckGPG: true},
},
},
})
@ -112,12 +112,12 @@ func createWeldrAPI2(tempdir string, fixtureGenerator rpmmd_mock.FixtureGenerato
rr := reporegistry.NewFromDistrosRepoConfigs(rpmmd.DistrosRepoConfigs{
test_distro.TestDistroName: {
test_distro.TestArch2Name: {
{Name: "test-id", BaseURL: "http://example.com/test/os/x86_64", CheckGPG: true},
{Name: "test-id", BaseURLs: []string{"http://example.com/test/os/x86_64"}, CheckGPG: true},
},
},
test_distro.TestDistro2Name: {
test_distro.TestArch2Name: {
{Name: "test-id-2", BaseURL: "http://example.com/test-2/os/x86_64", CheckGPG: true},
{Name: "test-id-2", BaseURLs: []string{"http://example.com/test-2/os/x86_64"}, CheckGPG: true},
},
},
})

16
internal/worker/server_test.go
View file

@ -616,20 +616,20 @@ func TestDepsolveJobArgsCompat(t *testing.T) {
// common elements
baseos := rpmmd.RepoConfig{
Name: "baseos",
BaseURL: "https://example.com/baseos",
Name: "baseos",
BaseURLs: []string{"https://example.com/baseos"},
}
appstream := rpmmd.RepoConfig{
Name: "appstream",
BaseURL: "https://example.com/appstream",
Name: "appstream",
BaseURLs: []string{"https://example.com/appstream"},
}
user1 := rpmmd.RepoConfig{
Name: "user1",
BaseURL: "https://example.com/user/1",
Name: "user1",
BaseURLs: []string{"https://example.com/user/1"},
}
user2 := rpmmd.RepoConfig{
Name: "user2",
BaseURL: "https://example.com/user/2",
Name: "user2",
BaseURLs: []string{"https://example.com/user/2"},
}
osIncludes := []string{"os1", "os2", "os3"}