From c6aa7d88d25c0324c770a08b241ea8132eab4253 Mon Sep 17 00:00:00 2001 From: Sanne Raymaekers Date: Tue, 10 Oct 2023 12:53:18 +0200 Subject: [PATCH] internal/weldr: specify architecture of compose This is useful in environments with multi-arch remote workers. Defaults to the host architecture. --- go.mod | 2 +- go.sum | 3 +- internal/blueprint/blueprint.go | 1 + internal/client/unit_test.go | 4 + internal/store/fixtures.go | 13 +++ internal/weldr/api.go | 199 +++++++++++++++++++++----------- internal/weldr/api_test.go | 31 ++++- internal/weldr/compose_test.go | 4 + test/cases/cross-distro.sh | 2 +- vendor/modules.txt | 2 +- 10 files changed, 188 insertions(+), 73 deletions(-) diff --git a/go.mod b/go.mod index 4f80dc210..5cc62267e 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/aws/aws-sdk-go v1.48.13 github.com/coreos/go-semver v0.3.1 - github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f + github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/deepmap/oapi-codegen v1.8.2 github.com/getkin/kin-openapi v0.93.0 github.com/gobwas/glob v0.2.3 diff --git a/go.sum b/go.sum index 94b813f6d..41e3ce4da 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,9 @@ github.com/containers/storage v1.51.0/go.mod h1:ybl8a3j1PPtpyaEi/5A6TOFs+5TrEyOb github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= diff --git a/internal/blueprint/blueprint.go b/internal/blueprint/blueprint.go index 517f2b39d..3df6a5782 100644 --- a/internal/blueprint/blueprint.go +++ b/internal/blueprint/blueprint.go @@ -22,6 +22,7 @@ type Blueprint struct { Containers []Container `json:"containers,omitempty" toml:"containers,omitempty"` Customizations *Customizations `json:"customizations,omitempty" toml:"customizations"` Distro string `json:"distro" toml:"distro"` + Arch string `json:"architecture,omitempty" toml:"architecture,omitempty"` } type Change struct { diff --git a/internal/client/unit_test.go b/internal/client/unit_test.go index cb0b27751..c4b0f76fa 100644 --- a/internal/client/unit_test.go +++ b/internal/client/unit_test.go @@ -71,6 +71,10 @@ func executeTests(m *testing.M) int { panic(err) } distro2 := test_distro.NewTestDistro("test-distro-2", "platform:test-2", "2") + _, err = fixture.Workers.RegisterWorker(arch.Name()) + if err != nil { + panic(err) + } rr := reporegistry.NewFromDistrosRepoConfigs(rpmmd.DistrosRepoConfigs{ test_distro.TestDistroName: { diff --git a/internal/store/fixtures.go b/internal/store/fixtures.go index 59f76687c..e71783ff1 100644 --- a/internal/store/fixtures.go +++ b/internal/store/fixtures.go @@ -79,6 +79,7 @@ func FixtureBase() *Store { }} s.blueprints[bName] = b + s.composes = map[uuid.UUID]Compose{ uuid.MustParse("30000000-0000-0000-0000-000000000000"): { Blueprint: &b, @@ -309,6 +310,18 @@ func FixtureEmpty() *Store { b3.Distro = "fedora-1" s.blueprints[b3.Name] = b3 + // Bad arch blueprint + b4 := b + b4.Name = "test-badarch" + b4.Arch = "badarch" + s.blueprints[b4.Name] = b4 + + // Cross arch blueprint + b5 := b + b5.Name = "test-crossarch" + b5.Arch = test_distro.TestArch2Name + s.blueprints[b5.Name] = b5 + return s } diff --git a/internal/weldr/api.go b/internal/weldr/api.go index 379a7dbfd..0057dc12f 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -52,7 +52,7 @@ type API struct { workers *worker.Server solver *dnfjson.BaseSolver - archName string + hostArch string repoRegistry *reporegistry.RepoRegistry logger *log.Logger @@ -63,7 +63,6 @@ type API struct { hostDistroName string // Name of the host distro distroRegistry *distroregistry.Registry // Available distros - distros []string // Supported distro names // List of ImageType names, which should not be exposed by the API distrosImageTypeDenylist map[string][]string @@ -97,7 +96,7 @@ func (cs ComposeState) ToString() string { // systemRepoNames returns a list of the system repos // NOTE: The system repos have no concept of id vs. name so the id is returned func (api *API) systemRepoNames() (names []string) { - repos, err := api.repoRegistry.ReposByArchName(api.hostDistroName, api.archName, false) + repos, err := api.repoRegistry.ReposByArchName(api.hostDistroName, api.hostArch, false) if err == nil { for _, repo := range repos { names = append(names, repo.Name) @@ -107,14 +106,16 @@ func (api *API) systemRepoNames() (names []string) { } // validDistros returns a list of distributions that also have repositories -func validDistros(rr *reporegistry.RepoRegistry, dr *distroregistry.Registry, arch string, logger *log.Logger) []string { +func (api *API) validDistros(arch string) []string { distros := []string{} - for _, d := range dr.List() { - _, found := rr.DistroHasRepos(d, arch) + for _, d := range api.distroRegistry.List() { + _, found := api.repoRegistry.DistroHasRepos(d, arch) if found { distros = append(distros, d) } else { - logger.Printf("Distro %s has no repositories, skipping.", d) + if api.logger != nil { + api.logger.Printf("Distro %s has no repositories, skipping.", d) + } } } @@ -136,13 +137,12 @@ func NewTestAPI(solver *dnfjson.BaseSolver, arch distro.Arch, dr *distroregistry store: store, workers: workers, solver: solver, - archName: arch.Name(), + hostArch: arch.Name(), repoRegistry: rr, logger: logger, compatOutputDir: compatOutputDir, hostDistroName: hostDistro.Name(), distroRegistry: dr, - distros: validDistros(rr, dr, arch.Name(), logger), distrosImageTypeDenylist: distrosImageTypeDenylist, } return setupRouter(api) @@ -158,7 +158,7 @@ func New(repoPaths []string, stateDir string, solver *dnfjson.BaseSolver, dr *di if err != nil { return nil, fmt.Errorf("failed to read host distro information") } - archName := common.CurrentArch() + hostArch := common.CurrentArch() rr, err := reporegistry.New(repoPaths) if err != nil { @@ -170,13 +170,13 @@ func New(repoPaths []string, stateDir string, solver *dnfjson.BaseSolver, dr *di // get canonical distro name if the host distro is supported hostDistroName = hostDistro.Name() - _, err = hostDistro.GetArch(archName) + _, err = hostDistro.GetArch(hostArch) if err != nil { return nil, fmt.Errorf("Host distro does not support host architecture: %v", err) } // Check if repositories for the host distro and arch were loaded - _, err = rr.ReposByArchName(hostDistroName, archName, false) + _, err = rr.ReposByArchName(hostDistroName, hostArch, false) if err != nil { log.Printf("loaded repository definitions don't contain any for the host distro/arch: %v", err) } @@ -192,13 +192,12 @@ func New(repoPaths []string, stateDir string, solver *dnfjson.BaseSolver, dr *di store: store, workers: workers, solver: solver, - archName: archName, + hostArch: hostArch, repoRegistry: rr, logger: logger, compatOutputDir: compatOutputDir, hostDistroName: hostDistroName, distroRegistry: dr, - distros: validDistros(rr, dr, archName, logger), distrosImageTypeDenylist: distrosImageTypeDenylist, } return setupRouter(api), nil @@ -308,22 +307,22 @@ func (api *API) ServeHTTP(writer http.ResponseWriter, request *http.Request) { // metadata. func (api *API) PreloadMetadata() { log.Printf("Starting metadata preload goroutines") - for _, distro := range api.distros { + for _, distro := range api.validDistros(api.hostArch) { go func(distro string) { startTime := time.Now() - d := api.getDistro(distro) + d := api.getDistro(distro, api.hostArch) if d == nil { log.Printf("GetDistro - unknown distribution: %s", distro) return } - repos, err := api.allRepositories(distro) + repos, err := api.allRepositories(distro, api.hostArch) if err != nil { log.Printf("Error getting repositories for distro %s: %s", distro, err) return } - solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name()) + solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.hostArch, d.Name()) _, err = solver.Depsolve([]rpmmd.PackageSet{{Include: []string{"filesystem"}, Repositories: repos}}) if err != nil { log.Printf("Problem preloading distro metadata for %s: %s", distro, err) @@ -469,7 +468,7 @@ func (api *API) isImageTypeAllowed(distroName, imageType string) (bool, error) { // getImageType returns the ImageType for the selected distro // This is necessary because different distros support different image types, and the image // type may have a different package set than other distros. -func (api *API) getImageType(distroName, imageType string) (distro.ImageType, error) { +func (api *API) getImageType(distroName, imageType, archName string) (distro.ImageType, error) { imgAllowed, err := api.isImageTypeAllowed(distroName, imageType) if err != nil { return nil, fmt.Errorf("error while checking if image type is allowed: %v", err) @@ -478,20 +477,21 @@ func (api *API) getImageType(distroName, imageType string) (distro.ImageType, er return nil, fmt.Errorf("image type %q for distro %q is denied by configuration", imageType, distroName) } - distro := api.getDistro(distroName) + distro := api.getDistro(distroName, archName) if distro == nil { return nil, fmt.Errorf("GetDistro - unknown distribution: %s", distroName) } - arch, err := distro.GetArch(api.archName) + + arch, err := distro.GetArch(archName) if err != nil { return nil, err } return arch.GetImageType(imageType) } -func (api *API) parseDistro(query url.Values) (string, error) { +func (api *API) parseDistro(query url.Values, arch string) (string, error) { if distro := query.Get("distro"); distro != "" { - if common.IsStringInSortedSlice(api.distros, distro) { + if common.IsStringInSortedSlice(api.validDistros(arch), distro) { return distro, nil } return "", errors_package.New("Invalid distro: " + distro) @@ -501,8 +501,8 @@ func (api *API) parseDistro(query url.Values) (string, error) { // getDistro returns the named distro or nil // It excludes unsupported distros by first checking the api.distros list -func (api *API) getDistro(name string) distro.Distro { - if !common.IsStringInSortedSlice(api.distros, name) { +func (api *API) getDistro(name, arch string) distro.Distro { + if !common.IsStringInSortedSlice(api.validDistros(arch), name) { return nil } return api.distroRegistry.GetDistro(name) @@ -664,7 +664,7 @@ func (api *API) getSourceConfigs(params httprouter.Params) (map[string]store.Sou sources := map[string]store.SourceConfig{} errors := []responseError{} - repos, err := api.repoRegistry.ReposByArchName(api.hostDistroName, api.archName, false) + repos, err := api.repoRegistry.ReposByArchName(api.hostDistroName, api.hostArch, false) if err != nil { error := responseError{ ID: "InternalError", @@ -939,7 +939,7 @@ func (api *API) sourceNewHandler(writer http.ResponseWriter, request *http.Reque // If there is a list of distros, check to make sure they are valid invalid := []string{} for _, d := range source.SourceConfig().Distros { - if !common.IsStringInSortedSlice(api.distros, d) { + if !common.IsStringInSortedSlice(api.validDistros(api.hostArch), d) { invalid = append(invalid, d) } } @@ -1050,9 +1050,14 @@ func (api *API) modulesListHandler(writer http.ResponseWriter, request *http.Req modulesParam := params.ByName("modules") + archName := api.hostArch + if queryArch := request.URL.Query().Get("arch"); queryArch != "" { + archName = queryArch + } + // Optional distro parameter // If it is empty it will return api.hostDistroName - distroName, err := api.parseDistro(request.URL.Query()) + distroName, err := api.parseDistro(request.URL.Query(), archName) if err != nil { errors := responseError{ ID: "DistroError", @@ -1071,7 +1076,7 @@ func (api *API) modulesListHandler(writer http.ResponseWriter, request *http.Req names = strings.Split(modulesParam, ",") } - packages, err := api.fetchPackageList(distroName, names) + packages, err := api.fetchPackageList(distroName, archName, names) if err != nil { errors := responseError{ @@ -1133,9 +1138,14 @@ func (api *API) projectsListHandler(writer http.ResponseWriter, request *http.Re return } + archName := api.hostArch + if queryArch := request.URL.Query().Get("arch"); queryArch != "" { + archName = queryArch + } + // Optional distro parameter // If it is empty it will return api.hostDistroName - distroName, err := api.parseDistro(request.URL.Query()) + distroName, err := api.parseDistro(request.URL.Query(), archName) if err != nil { errors := responseError{ ID: "DistroError", @@ -1144,7 +1154,7 @@ func (api *API) projectsListHandler(writer http.ResponseWriter, request *http.Re statusResponseError(writer, http.StatusBadRequest, errors) return } - availablePackages, err := api.fetchPackageList(distroName, []string{}) + availablePackages, err := api.fetchPackageList(distroName, archName, []string{}) if err != nil { errors := responseError{ @@ -1216,9 +1226,14 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req names := strings.Split(modules, ",") + archName := api.hostArch + if queryArch := request.URL.Query().Get("arch"); queryArch != "" { + archName = queryArch + } + // Optional distro parameter // If it is empty it will return api.hostDistroName - distroName, err := api.parseDistro(request.URL.Query()) + distroName, err := api.parseDistro(request.URL.Query(), archName) if err != nil { errors := responseError{ ID: "DistroError", @@ -1227,7 +1242,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req statusResponseError(writer, http.StatusBadRequest, errors) return } - foundPackages, err := api.fetchPackageList(distroName, names) + foundPackages, err := api.fetchPackageList(distroName, archName, names) if err != nil { errors := responseError{ @@ -1250,7 +1265,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req packageInfos := foundPackages.ToPackageInfos() if modulesRequested { - repos, err := api.allRepositories(distroName) + repos, err := api.allRepositories(distroName, archName) if err != nil { errors := responseError{ ID: "InternalError", @@ -1259,7 +1274,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req statusResponseError(writer, http.StatusBadRequest, errors) return } - d := api.getDistro(distroName) + d := api.getDistro(distroName, archName) if d == nil { errors := responseError{ ID: "DistroError", @@ -1269,7 +1284,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req return } - solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name()) + solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), archName, d.Name()) for i := range packageInfos { pkgName := packageInfos[i].Name solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: []string{pkgName}, Repositories: repos}}) @@ -1321,9 +1336,14 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt projects = projects[1:] names := strings.Split(projects, ",") + archName := api.hostArch + if queryArch := request.URL.Query().Get("arch"); queryArch != "" { + archName = queryArch + } + // Optional distro parameter // If it is empty it will return api.hostDistroName - distroName, err := api.parseDistro(request.URL.Query()) + distroName, err := api.parseDistro(request.URL.Query(), archName) if err != nil { errors := responseError{ ID: "DistroError", @@ -1333,7 +1353,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt return } - d := api.getDistro(distroName) + d := api.getDistro(distroName, archName) if d == nil { errors := responseError{ ID: "DistroError", @@ -1343,7 +1363,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt return } - repos, err := api.allRepositories(distroName) + repos, err := api.allRepositories(distroName, archName) if err != nil { errors := responseError{ ID: "ProjectsError", @@ -1353,7 +1373,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt return } - solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name()) + solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), archName, d.Name()) deps, err := solver.Depsolve([]rpmmd.PackageSet{{Include: names, Repositories: repos}}) if err != nil { errors := responseError{ @@ -2034,10 +2054,14 @@ func (api *API) blueprintsNewHandler(writer http.ResponseWriter, request *http.R // Check the blueprint's distro to make sure it is valid if len(blueprint.Distro) > 0 { - if !common.IsStringInSortedSlice(api.distros, blueprint.Distro) { + arch := blueprint.Arch + if arch == "" { + arch = api.hostArch + } + if !common.IsStringInSortedSlice(api.validDistros(arch), blueprint.Distro) { errors := responseError{ ID: "BlueprintsError", - Msg: fmt.Sprintf("'%s' is not a valid distribution", blueprint.Distro), + Msg: fmt.Sprintf("'%s' is not a valid distribution (architecture '%s')", blueprint.Distro, arch), } statusResponseError(writer, http.StatusBadRequest, errors) return @@ -2243,12 +2267,13 @@ func (api *API) blueprintsTagHandler(writer http.ResponseWriter, request *http.R } // depsolve handles depsolving package sets required for serializing a manifest for a given distribution. -func (api *API) depsolve(packageSets map[string][]rpmmd.PackageSet, distro distro.Distro) (map[string][]rpmmd.PackageSpec, error) { +func (api *API) depsolve(packageSets map[string][]rpmmd.PackageSet, arch distro.Arch) (map[string][]rpmmd.PackageSpec, error) { + distro := arch.Distro() platformID := distro.ModulePlatformID() releasever := distro.Releasever() distroName := distro.Name() - solver := api.solver.NewWithConfig(platformID, releasever, api.archName, distroName) + solver := api.solver.NewWithConfig(platformID, releasever, arch.Name(), distroName) depsolvedSets := make(map[string][]rpmmd.PackageSpec, len(packageSets)) @@ -2266,7 +2291,7 @@ func (api *API) depsolve(packageSets map[string][]rpmmd.PackageSet, distro distr return depsolvedSets, nil } -func (api *API) resolveContainers(sourceSpecs map[string][]container.SourceSpec) (map[string][]container.Spec, error) { +func (api *API) resolveContainers(sourceSpecs map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { specs := make(map[string][]container.Spec, len(sourceSpecs)) @@ -2280,7 +2305,7 @@ func (api *API) resolveContainers(sourceSpecs map[string][]container.SourceSpec) // to multiple pipelines at any point, this should work. for name, sources := range sourceSpecs { job := worker.ContainerResolveJob{ - Arch: api.archName, + Arch: archName, Specs: make([]worker.ContainerSpec, len(sources)), } @@ -2423,17 +2448,23 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request if distroName == "" { distroName = api.hostDistroName } - if api.getDistro(distroName) == nil { + + archName := bp.Arch + if archName == "" { + archName = api.hostArch + } + + if api.getDistro(distroName, archName) == nil { errors := responseError{ ID: "DistroError", - Msg: fmt.Sprintf("Unknown distribution: %s", distroName), + Msg: fmt.Sprintf("Unknown distribution: %s for arch %s", distroName, archName), } statusResponseError(writer, http.StatusBadRequest, errors) return } // Get the imageType that corresponds to the distribution selected by the blueprint - imageType, err := api.getImageType(distroName, cr.ComposeType) + imageType, err := api.getImageType(distroName, cr.ComposeType, archName) if err != nil { errors := responseError{ ID: "ComposeError", @@ -2526,7 +2557,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request return } - packageSets, err := api.depsolve(manifest.GetPackageSetChains(), imageType.Arch().Distro()) + packageSets, err := api.depsolve(manifest.GetPackageSetChains(), imageType.Arch()) if err != nil { errors := responseError{ ID: "DepsolveError", @@ -2536,7 +2567,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request return } - containerSpecs, err := api.resolveContainers(manifest.GetContainerSourceSpecs()) + containerSpecs, err := api.resolveContainers(manifest.GetContainerSourceSpecs(), archName) if err != nil { errors := responseError{ ID: "ContainerResolveError", @@ -2577,6 +2608,25 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request } } + workerAvailable, err := api.workers.WorkerAvailableForArch(archName) + if err != nil { + log.Println("error when pushing new compose: ", err.Error()) + errors := responseError{ + ID: "ComposePushErrored", + Msg: err.Error(), + } + statusResponseError(writer, http.StatusInternalServerError, errors) + return + } + if !workerAvailable { + errors := responseError{ + ID: "ComposePushErrored", + Msg: fmt.Sprintf("No worker for arch '%s' available", archName), + } + statusResponseError(writer, http.StatusBadRequest, errors) + return + } + if testMode == "1" { // Create a failed compose err = api.store.PushTestCompose(composeID, mf, imageType, bp, size, targets, false, packages) @@ -2585,8 +2635,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request err = api.store.PushTestCompose(composeID, mf, imageType, bp, size, targets, true, packages) } else { var jobId uuid.UUID - - jobId, err = api.workers.EnqueueOSBuild(api.archName, &worker.OSBuildJob{ + jobId, err = api.workers.EnqueueOSBuild(archName, &worker.OSBuildJob{ Manifest: mf, Targets: targets, PipelineNames: &worker.PipelineNames{ @@ -2767,9 +2816,15 @@ func (api *API) composeTypesHandler(writer http.ResponseWriter, request *http.Re if !verifyRequestVersion(writer, params, 0) { return } + + archName := api.hostArch + if queryArch := request.URL.Query().Get("arch"); queryArch != "" { + archName = queryArch + } + // Optional distro parameter // If it is empty it will return api.hostDistroName - distroName, err := api.parseDistro(request.URL.Query()) + distroName, err := api.parseDistro(request.URL.Query(), archName) if err != nil { errors := responseError{ ID: "DistroError", @@ -2779,7 +2834,7 @@ func (api *API) composeTypesHandler(writer http.ResponseWriter, request *http.Re return } - d := api.getDistro(distroName) + d := api.getDistro(distroName, archName) if d == nil { errors := responseError{ ID: "DistroError", @@ -2790,11 +2845,11 @@ func (api *API) composeTypesHandler(writer http.ResponseWriter, request *http.Re } // Get the distro specific arch so that we can return the correct list of image types - arch, err := d.GetArch(api.archName) + arch, err := d.GetArch(archName) if err != nil { errors := responseError{ ID: "DistroError", - Msg: fmt.Sprintf("Unknown arch: %s", api.archName), + Msg: fmt.Sprintf("Unknown arch: %s", arch), } statusResponseError(writer, http.StatusBadRequest, errors) return @@ -3442,17 +3497,17 @@ func (api *API) composeFailedHandler(writer http.ResponseWriter, request *http.R } // fetchPackageList returns the package list or the selected distribution -func (api *API) fetchPackageList(distroName string, names []string) (packages rpmmd.PackageList, err error) { - d := api.getDistro(distroName) +func (api *API) fetchPackageList(distroName, arch string, names []string) (packages rpmmd.PackageList, err error) { + d := api.getDistro(distroName, arch) if d == nil { return nil, fmt.Errorf("GetDistro - unknown distribution: %s", distroName) } - repos, err := api.allRepositories(distroName) + repos, err := api.allRepositories(distroName, arch) if err != nil { return nil, err } - solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name()) + solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), arch, d.Name()) if len(names) == 0 { packages, err = solver.FetchMetadata(repos) } else { @@ -3503,8 +3558,8 @@ func (api *API) allRepositoriesByImageType(imageType distro.ImageType) ([]rpmmd. } // Returns all configured repositories (base + sources) as rpmmd.RepoConfig -func (api *API) allRepositories(distroName string) ([]rpmmd.RepoConfig, error) { - repos, err := api.repoRegistry.ReposByArchName(distroName, api.archName, false) +func (api *API) allRepositories(distroName, arch string) ([]rpmmd.RepoConfig, error) { + repos, err := api.repoRegistry.ReposByArchName(distroName, arch, false) if err != nil { return nil, err } @@ -3521,16 +3576,21 @@ func (api *API) depsolveBlueprint(bp blueprint.Blueprint) ([]rpmmd.PackageSpec, bp.Distro = api.hostDistroName } - d := api.getDistro(bp.Distro) + arch := bp.Arch + if arch == "" { + arch = api.hostArch + } + + d := api.getDistro(bp.Distro, arch) if d == nil { return nil, fmt.Errorf("GetDistro - unknown distribution: %s", bp.Distro) } - repos, err := api.allRepositories(bp.Distro) + repos, err := api.allRepositories(bp.Distro, arch) if err != nil { return nil, err } - solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name()) + solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), arch, d.Name()) solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: bp.GetPackages(), Repositories: repos}}) if err != nil { return nil, err @@ -3629,10 +3689,15 @@ func (api *API) distrosListHandler(writer http.ResponseWriter, request *http.Req return } + archName := api.hostArch + if arch := request.URL.Query().Get("arch"); arch != "" { + archName = arch + } + var reply struct { Distros []string `json:"distros"` } - reply.Distros = api.distros + reply.Distros = api.validDistros(archName) err := json.NewEncoder(writer).Encode(reply) common.PanicOnError(err) diff --git a/internal/weldr/api_test.go b/internal/weldr/api_test.go index aab08bbc2..99018e1d8 100644 --- a/internal/weldr/api_test.go +++ b/internal/weldr/api_test.go @@ -70,6 +70,9 @@ func createWeldrAPI(tempdir string, fixtureGenerator rpmmd_mock.FixtureGenerator test_distro.TestArchName: { {Name: "test-id", BaseURLs: []string{"http://example.com/test/os/x86_64"}, CheckGPG: common.ToPtr(true)}, }, + test_distro.TestArch2Name: { + {Name: "test-id", BaseURLs: []string{"http://example.com/test/os/aarch64"}, CheckGPG: common.ToPtr(true)}, + }, }, test_distro.TestDistro2Name: { test_distro.TestArchName: { @@ -236,7 +239,7 @@ func TestBlueprintsNew(t *testing.T) { {"POST", "/api/v0/blueprints/new", ``, http.StatusBadRequest, `{"status":false,"errors":[{"id":"BlueprintsError","msg":"Missing blueprint"}]}`}, {"POST", "/api/v0/blueprints/new", `{"name":"test","description":"Test","distro":"test-distro","packages":[],"version":""}`, http.StatusOK, `{"status":true}`}, {"POST", "/api/v0/blueprints/new", `{"name":"test2","description":"Test 2","distro":"test-distro-2","packages":[],"version":""}`, http.StatusOK, `{"status":true}`}, - {"POST", "/api/v0/blueprints/new", `{"name":"test","description":"Test","distro":"fedora-1","packages":[],"version":""}`, http.StatusBadRequest, `{"status":false,"errors":[{"id":"BlueprintsError","msg":"'fedora-1' is not a valid distribution"}]}`}, + {"POST", "/api/v0/blueprints/new", `{"name":"test","description":"Test","distro":"fedora-1","packages":[],"version":""}`, http.StatusBadRequest, `{"status":false,"errors":[{"id":"BlueprintsError","msg":"'fedora-1' is not a valid distribution (architecture 'test_arch')"}]}`}, } tempdir := t.TempDir() @@ -1191,7 +1194,27 @@ func TestCompose(t *testing.T) { "/api/v1/compose", fmt.Sprintf(`{"blueprint_name": "test-fedora-1","compose_type": "%s","branch": "master"}`, test_distro.TestImageTypeName), http.StatusBadRequest, - `{"status": false,"errors":[{"id":"DistroError", "msg":"Unknown distribution: fedora-1"}]}`, + `{"status": false,"errors":[{"id":"DistroError", "msg":"Unknown distribution: fedora-1 for arch test_arch"}]}`, + nil, + []string{"build_id", "warnings"}, + }, + "bad-arch": { + false, + "POST", + "/api/v1/compose", + fmt.Sprintf(`{"blueprint_name": "test-badarch","compose_type": "%s","branch": "master"}`, test_distro.TestImageTypeName), + http.StatusBadRequest, + `{"status": false,"errors":[{"id":"DistroError", "msg":"Unknown distribution: test-distro for arch badarch"}]}`, + nil, + []string{"build_id", "warnings"}, + }, + "cross-arch": { + false, + "POST", + "/api/v1/compose", + fmt.Sprintf(`{"blueprint_name": "test-crossarch","compose_type": "%s","branch": "master"}`, test_distro.TestImageTypeName), + http.StatusBadRequest, + `{"status": false,"errors":[{"id":"ComposePushErrored", "msg":"No worker for arch 'test_arch2' available"}]}`, nil, []string{"build_id", "warnings"}, }, @@ -1300,6 +1323,8 @@ func TestCompose(t *testing.T) { tempdir := t.TempDir() for name, c := range cases { api, s := createWeldrAPI(tempdir, rpmmd_mock.NoComposesFixture) + _, err = api.workers.RegisterWorker(arch.Name()) + require.NoError(t, err) test.TestRoute(t, api, c.External, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON, c.IgnoreFields...) if c.ExpectedStatus != http.StatusOK { @@ -2271,6 +2296,8 @@ func TestComposePOST_ImageTypeDenylist(t *testing.T) { for _, c := range cases { api, s := createWeldrAPI2(tempdir, rpmmd_mock.NoComposesFixture, c.imageTypeDenylist) + _, err = api.workers.RegisterWorker(arch.Name()) + require.NoError(t, err) test.TestRoute(t, api, true, "POST", c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON, c.IgnoreFields...) if c.ExpectedStatus != http.StatusOK { diff --git a/internal/weldr/compose_test.go b/internal/weldr/compose_test.go index 38907c270..b91905567 100644 --- a/internal/weldr/compose_test.go +++ b/internal/weldr/compose_test.go @@ -42,6 +42,8 @@ func TestComposeStatusFromLegacyError(t *testing.T) { t.Fatalf("error serializing osbuild manifest: %v", err) } + _, err = api.workers.RegisterWorker(arch.Name()) + require.NoError(t, err) jobId, err := api.workers.EnqueueOSBuild(arch.Name(), &worker.OSBuildJob{Manifest: mf}, "") require.NoError(t, err) @@ -95,6 +97,8 @@ func TestComposeStatusFromJobError(t *testing.T) { t.Fatalf("error serializing osbuild manifest: %v", err) } + _, err = api.workers.RegisterWorker(arch.Name()) + require.NoError(t, err) jobId, err := api.workers.EnqueueOSBuild(arch.Name(), &worker.OSBuildJob{Manifest: mf}, "") require.NoError(t, err) diff --git a/test/cases/cross-distro.sh b/test/cases/cross-distro.sh index 918177476..22f421c22 100755 --- a/test/cases/cross-distro.sh +++ b/test/cases/cross-distro.sh @@ -146,7 +146,7 @@ EOF # there is a different reponse if legacy composer-cli is used if rpm -q --quiet weldr-client; then - EXPECTED_RESPONSE="ERROR: BlueprintsError: '$REMAINING_DISTRO' is not a valid distribution" + EXPECTED_RESPONSE="ERROR: BlueprintsError: '$REMAINING_DISTRO' is not a valid distribution (architecture '$(uname -m)')" else EXPECTED_RESPONSE="'$REMAINING_DISTRO' is not a valid distribution" RESPONSE=${RESPONSE#*: } diff --git a/vendor/modules.txt b/vendor/modules.txt index 76c699a83..3605ad984 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -289,7 +289,7 @@ github.com/containers/storage/pkg/unshare # github.com/coreos/go-semver v0.3.1 ## explicit; go 1.8 github.com/coreos/go-semver/semver -# github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f +# github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf ## explicit github.com/coreos/go-systemd/activation # github.com/cyberphone/json-canonicalization v0.0.0-20231011164504-785e29786b46