particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

api: improve parity of /{modules,projects}/{list,info} routes

Browse source 
These endpoint are similar in many ways, therefore just one commit. Their
functionality is basically same as in lorax except for error messages and
weird edge cases when handling trailing slashes.

closes #64, closes #65
This commit is contained in:
Ondřej Budai 2019-11-14 14:46:03 +01:00 • committed by Tom Gundersen
parent 2ed070b218
commit dd9a815c40
6 changed files with 441 additions and 232 deletions
Download patch file Download diff file Expand all files Collapse all files

62
internal/mocks/rpmmd/fixtures.go
View file

@ -1,14 +1,47 @@
package rpmmd_mock
import (
"fmt"
"github.com/osbuild/osbuild-composer/internal/rpmmd"
"time"
)
func generatePackageList() rpmmd.PackageList {
baseTime, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
if err != nil {
panic(err)
}
var packageList rpmmd.PackageList
for i := 0; i < 22; i++ {
basePackage := rpmmd.Package{
Name: fmt.Sprintf("package%d", i),
Summary: fmt.Sprintf("pkg%d sum", i),
Description: fmt.Sprintf("pkg%d desc", i),
URL: fmt.Sprintf("https://pkg%d.example.com", i),
Epoch: 0,
Version: fmt.Sprintf("%d.0", i),
Release: fmt.Sprintf("%d.fc30", i),
Arch: "x86_64",
BuildTime: baseTime.AddDate(0, i, 0),
License: "MIT",
}
secondBuild := basePackage
secondBuild.Version = fmt.Sprintf("%d.1", i)
secondBuild.BuildTime = basePackage.BuildTime.AddDate(0, 0, 1)
packageList = append(packageList, basePackage, secondBuild)
}
return packageList
}
var basePackageList = fetchPackageList{
rpmmd.PackageList{
{Name: "package1"},
{Name: "package2"},
},
generatePackageList(),
nil,
}
@ -17,14 +50,14 @@ var BaseFixture = Fixture{
depsolve{
[]rpmmd.PackageSpec{
{
Name: "libgpg-error",
Name: "dep-package1",
Epoch: 0,
Version: "1.33",
Release: "2.fc30",
Arch: "x86_64",
},
{
Name: "libsemanage",
Name: "dep-package2",
Epoch: 0,
Version: "2.9",
Release: "1.fc30",
@ -56,3 +89,20 @@ var BadDepsolve = Fixture{
},
},
}
var BadFetch = Fixture{
fetchPackageList{
ret: nil,
err: &rpmmd.DNFError{
Kind: "FetchError",
Reason: "There was a problem when fetching packages.",
},
},
depsolve{
nil,
&rpmmd.DNFError{
Kind: "DepsolveError",
Reason: "There was a problem depsolving ['go2rpm']: \n Problem: conflicting requests\n - nothing provides askalono-cli needed by go2rpm-1-4.fc31.noarch",
},
},
}

109
internal/rpmmd/repository.go
View file

@ -9,6 +9,8 @@ import (
"sort"
"time"
"github.com/gobwas/glob"
"github.com/osbuild/osbuild-composer/internal/store"
)
@ -35,6 +37,30 @@ type Package struct {
License string
}
func (pkg Package) ToPackageBuild() PackageBuild {
return PackageBuild{
Arch: pkg.Arch,
BuildTime: pkg.BuildTime,
Epoch: pkg.Epoch,
Release: pkg.Release,
Source: PackageSource{
License: pkg.License,
Version: pkg.Version,
},
}
}
func (pkg Package) ToPackageInfo() PackageInfo {
return PackageInfo{
Name: pkg.Name,
Summary: pkg.Summary,
Description: pkg.Description,
Homepage: pkg.URL,
Builds: []PackageBuild{pkg.ToPackageBuild()},
Dependencies: nil,
}
}
type PackageSpec struct {
Name string `json:"name"`
Epoch uint `json:"epoch"`
@ -43,6 +69,29 @@ type PackageSpec struct {
Arch string `json:"arch,omitempty"`
}
type PackageSource struct {
License string `json:"license"`
Version string `json:"version"`
}
type PackageBuild struct {
Arch string `json:"arch"`
BuildTime time.Time `json:"build_time"`
Epoch uint `json:"epoch"`
Release string `json:"release"`
Source PackageSource `json:"source"`
}
type PackageInfo struct {
Name string `json:"name"`
Summary string `json:"summary"`
Description string `json:"description"`
Homepage string `json:"homepage"`
UpstreamVCS string `json:"upstream_vcs"`
Builds []PackageBuild `json:"builds"`
Dependencies []PackageSpec `json:"dependencies,omitempty"`
}
type RPMMD interface {
FetchPackageList(repos []RepoConfig) (PackageList, error)
Depsolve(specs []string, repos []RepoConfig) ([]PackageSpec, error)
@ -145,21 +194,61 @@ func (*rpmmdImpl) Depsolve(specs []string, repos []RepoConfig) ([]PackageSpec, e
return dependencies, err
}
func (packages PackageList) Search(name string) (int, int) {
first := sort.Search(len(packages), func(i int) bool {
return packages[i].Name >= name
func (packages PackageList) Search(globPatterns ...string) (PackageList, error) {
var globs []glob.Glob
for _, globPattern := range globPatterns {
g, err := glob.Compile(globPattern)
if err != nil {
return nil, err
}
globs = append(globs, g)
}
var foundPackages PackageList
for _, pkg := range packages {
for _, g := range globs {
if g.Match(pkg.Name) {
foundPackages = append(foundPackages, pkg)
break
}
}
}
sort.Slice(packages, func(i, j int) bool {
return packages[i].Name < packages[j].Name
})
if first == len(packages) || packages[first].Name != name {
return first, 0
return foundPackages, nil
}
func (packages PackageList) ToPackageInfos() []PackageInfo {
resultsNames := make(map[string]int)
var results []PackageInfo
for _, pkg := range packages {
if index, ok := resultsNames[pkg.Name]; ok {
foundPkg := &results[index]
foundPkg.Builds = append(foundPkg.Builds, pkg.ToPackageBuild())
} else {
newIndex := len(results)
resultsNames[pkg.Name] = newIndex
packageInfo := pkg.ToPackageInfo()
results = append(results, packageInfo)
}
}
last := first + 1
for last < len(packages) && packages[last].Name == name {
last++
}
return results
}
return first, last - first
func (pkg *PackageInfo) FillDependencies(rpmmd RPMMD, repos []RepoConfig) (err error) {
pkg.Dependencies, err = rpmmd.Depsolve([]string{pkg.Name}, repos)
return
}
func SourceToRepo(source store.SourceConfig) RepoConfig {

399
internal/weldr/api.go
View file

@ -8,7 +8,6 @@ import (
"net"
"net/http"
"strings"
"time"
"github.com/google/uuid"
"github.com/julienschmidt/httprouter"
@ -55,11 +54,15 @@ func New(rpmmd rpmmd.RPMMD, repo rpmmd.RepoConfig, logger *log.Logger, store *st
api.router.GET("/api/v0/projects/depsolve/:projects", api.projectsDepsolveHandler)
api.router.GET("/api/v0/modules/list", api.modulesListAllHandler)
api.router.GET("/api/v0/modules/list/:modules", api.modulesListHandler)
api.router.GET("/api/v0/modules/list", api.modulesListHandler)
api.router.GET("/api/v0/modules/list/*modules", api.modulesListHandler)
api.router.GET("/api/v0/projects/list", api.projectsListHandler)
api.router.GET("/api/v0/projects/list/", api.projectsListHandler)
// these are the same, except that modules/info also includes dependencies
api.router.GET("/api/v0/modules/info", api.modulesInfoHandler)
api.router.GET("/api/v0/modules/info/*modules", api.modulesInfoHandler)
api.router.GET("/api/v0/projects/info", api.modulesInfoHandler)
api.router.GET("/api/v0/projects/info/*modules", api.modulesInfoHandler)
api.router.GET("/api/v0/blueprints/list", api.blueprintsListHandler)
@ -288,67 +291,17 @@ func (api *API) sourceDeleteHandler(writer http.ResponseWriter, request *http.Re
statusResponseOK(writer)
}
type modulesListModule struct {
Name string `json:"name"`
GroupType string `json:"group_type"`
}
type modulesListReply struct {
Total uint `json:"total"`
Offset uint `json:"offset"`
Limit uint `json:"limit"`
Modules []modulesListModule `json:"modules"`
}
func (api *API) modulesListAllHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) {
offset, limit, err := parseOffsetAndLimit(request.URL.Query())
if err != nil {
errors := responseError{
ID: "BadLimitOrOffset",
Msg: fmt.Sprintf("BadRequest: %s", err.Error()),
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
var repos []rpmmd.RepoConfig
repos = append(repos, api.repo)
for _, source := range api.store.GetAllSources() {
repos = append(repos, rpmmd.SourceToRepo(source))
}
packages, err := api.rpmmd.FetchPackageList(repos)
if err != nil {
errors := responseError{
ID: "ModulesError",
Msg: fmt.Sprintf("msg: %s", err.Error()),
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
total := uint(len(packages))
start := min(offset, total)
n := min(limit, total-start)
modules := make([]modulesListModule, n)
for i := uint(0); i < n; i++ {
modules[i] = modulesListModule{packages[start+i].Name, "rpm"}
}
json.NewEncoder(writer).Encode(modulesListReply{
Total: total,
Offset: offset,
Limit: limit,
Modules: modules,
})
}
func (api *API) modulesListHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
names := strings.Split(params.ByName("modules"), ",")
if names[0] == "" || names[0] == "*" {
api.modulesListAllHandler(writer, request, params)
return
type module struct {
Name string `json:"name"`
GroupType string `json:"group_type"`
}
type reply struct {
Total uint `json:"total"`
Offset uint `json:"offset"`
Limit uint `json:"limit"`
Modules []module `json:"modules"`
}
offset, limit, err := parseOffsetAndLimit(request.URL.Query())
@ -362,13 +315,10 @@ func (api *API) modulesListHandler(writer http.ResponseWriter, request *http.Req
return
}
var repos []rpmmd.RepoConfig
repos = append(repos, api.repo)
for _, source := range api.store.GetAllSources() {
repos = append(repos, rpmmd.SourceToRepo(source))
}
modulesParam := params.ByName("modules")
availablePackages, err := api.fetchPackageList()
packages, err := api.rpmmd.FetchPackageList(repos)
if err != nil {
errors := responseError{
ID: "ModulesError",
@ -378,156 +328,193 @@ func (api *API) modulesListHandler(writer http.ResponseWriter, request *http.Req
return
}
// we don't support glob-matching, but cockpit-composer surrounds some
// queries with asterisks; this is crude, but solves that case
for i := range names {
names[i] = strings.ReplaceAll(names[i], "*", "")
}
var packages rpmmd.PackageList
if modulesParam != "" && modulesParam != "/" {
// we have modules for search
modules := make([]modulesListModule, 0)
total := uint(0)
end := offset + limit
for i, name := range names {
for _, pkg := range packages {
if strings.Contains(pkg.Name, name) {
total++
if total > offset && total < end {
modules = append(modules, modulesListModule{pkg.Name, "rpm"})
// this removes names that have been found from the list of names
if len(names) < i-1 {
names = append(names[:i], names[i+1:]...)
} else {
names = names[:i]
}
}
}
}
}
// remove leading /
modulesParam = modulesParam[1:]
// if a name remains in the list it was not found
if len(names) > 0 {
errors := responseError{
ID: "UnknownModule",
Msg: fmt.Sprintf("one of the requested modules does not exist: ['%s']", names[0]),
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
names := strings.Split(modulesParam, ",")
json.NewEncoder(writer).Encode(modulesListReply{
Total: total,
Offset: offset,
Limit: limit,
Modules: modules,
})
}
packages, err = availablePackages.Search(names...)
func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
type source struct {
License string `json:"license"`
Version string `json:"version"`
}
type build struct {
Arch string `json:"arch"`
BuildTime time.Time `json:"build_time"`
Epoch uint `json:"epoch"`
Release string `json:"release"`
Source source `json:"source"`
}
type project struct {
Name string `json:"name"`
Summary string `json:"summary"`
Description string `json:"description"`
Homepage string `json:"homepage"`
Builds []build `json:"builds"`
Dependencies []rpmmd.PackageSpec `json:"dependencies,omitempty"`
}
type projectsReply struct {
Projects []project `json:"projects"`
}
type modulesReply struct {
Modules []project `json:"modules"`
}
// handle both projects/info and modules/info, the latter includes dependencies
modulesRequested := strings.HasPrefix(request.URL.Path, "/api/v0/modules")
names := strings.Split(params.ByName("modules"), ",")
if names[0] == "/" {
errors := responseError{
Code: http.StatusNotFound,
ID: "HTTPError",
Msg: "Not Found",
}
statusResponseError(writer, http.StatusNotFound, errors)
return
}
var repos []rpmmd.RepoConfig
repos = append(repos, api.repo)
for _, source := range api.store.GetAllSources() {
repos = append(repos, rpmmd.SourceToRepo(source))
}
packages, err := api.rpmmd.FetchPackageList(repos)
if err != nil {
errors := responseError{
ID: "ModulesError",
Msg: fmt.Sprintf("msg: %s", err.Error()),
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
projects := make([]project, 0)
for i, name := range names {
// remove leading / from first name
if i == 0 {
name = name[1:]
}
first, n := packages.Search(name)
if n == 0 {
if err != nil {
errors := responseError{
ID: "UnknownModule",
Msg: fmt.Sprintf("one of the requested modules does not exist: %s", name),
ID: "ModulesError",
Msg: fmt.Sprintf("Wrong glob pattern: %s", err.Error()),
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
// get basic info from the first package, but collect build
// information from all that have the same name
pkg := packages[first]
project := project{
Name: pkg.Name,
Summary: pkg.Summary,
Description: pkg.Description,
Homepage: pkg.URL,
if len(packages) == 0 {
errors := responseError{
ID: "UnknownModule",
Msg: "No packages have been found.",
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
} else {
// just return all available packages
packages = availablePackages
}
project.Builds = make([]build, n)
for i, pkg := range packages[first : first+n] {
project.Builds[i] = build{
Arch: pkg.Arch,
BuildTime: pkg.BuildTime,
Epoch: pkg.Epoch,
Release: pkg.Release,
Source: source{pkg.License, pkg.Version},
packageInfos := packages.ToPackageInfos()
total := uint(len(packageInfos))
start := min(offset, total)
n := min(limit, total-start)
modules := make([]module, n)
for i := uint(0); i < n; i++ {
modules[i] = module{packageInfos[start+i].Name, "rpm"}
}
json.NewEncoder(writer).Encode(reply{
Total: total,
Offset: offset,
Limit: limit,
Modules: modules,
})
}
func (api *API) projectsListHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) {
type reply struct {
Total uint `json:"total"`
Offset uint `json:"offset"`
Limit uint `json:"limit"`
Projects []rpmmd.PackageInfo `json:"projects"`
}
offset, limit, err := parseOffsetAndLimit(request.URL.Query())
if err != nil {
errors := responseError{
ID: "BadLimitOrOffset",
Msg: fmt.Sprintf("BadRequest: %s", err.Error()),
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
availablePackages, err := api.fetchPackageList()
if err != nil {
errors := responseError{
ID: "ProjectsError",
Msg: fmt.Sprintf("msg: %s", err.Error()),
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
packageInfos := availablePackages.ToPackageInfos()
total := uint(len(packageInfos))
start := min(offset, total)
n := min(limit, total-start)
packages := make([]rpmmd.PackageInfo, n)
for i := uint(0); i < n; i++ {
packages[i] = packageInfos[start+i]
}
json.NewEncoder(writer).Encode(reply{
Total: total,
Offset: offset,
Limit: limit,
Projects: packages,
})
}
func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
type projectsReply struct {
Projects []rpmmd.PackageInfo `json:"projects"`
}
type modulesReply struct {
Modules []rpmmd.PackageInfo `json:"modules"`
}
// handle both projects/info and modules/info, the latter includes dependencies
modulesRequested := strings.HasPrefix(request.URL.Path, "/api/v0/modules")
var errorId, unknownErrorId string
if modulesRequested {
errorId = "ModulesError"
unknownErrorId = "UnknownModule"
} else {
errorId = "ProjectsError"
unknownErrorId = "UnknownProject"
}
modules := params.ByName("modules")
if modules == "" || modules == "/" {
errors := responseError{
ID: unknownErrorId,
Msg: "No packages specified.",
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
// remove leading /
modules = modules[1:]
names := strings.Split(modules, ",")
availablePackages, err := api.fetchPackageList()
if err != nil {
errors := responseError{
ID: "ModulesError",
Msg: fmt.Sprintf("msg: %s", err.Error()),
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
foundPackages, err := availablePackages.Search(names...)
if err != nil {
errors := responseError{
ID: errorId,
Msg: fmt.Sprintf("Wrong glob pattern: %s", err.Error()),
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
if len(foundPackages) == 0 {
errors := responseError{
ID: unknownErrorId,
Msg: "No packages have been found.",
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
packageInfos := foundPackages.ToPackageInfos()
if modulesRequested {
for i, _ := range packageInfos {
err := packageInfos[i].FillDependencies(api.rpmmd, []rpmmd.RepoConfig{api.repo})
if err != nil {
errors := responseError{
ID: errorId,
Msg: fmt.Sprintf("Cannot depsolve package %s: %s", packageInfos[i].Name, err.Error()),
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
}
if modulesRequested {
project.Dependencies, _ = api.rpmmd.Depsolve([]string{pkg.Name}, repos)
}
projects = append(projects, project)
}
if modulesRequested {
json.NewEncoder(writer).Encode(modulesReply{projects})
json.NewEncoder(writer).Encode(modulesReply{packageInfos})
} else {
json.NewEncoder(writer).Encode(projectsReply{projects})
json.NewEncoder(writer).Encode(projectsReply{packageInfos})
}
}
@ -1112,3 +1099,13 @@ func (api *API) composeFailedHandler(writer http.ResponseWriter, request *http.R
json.NewEncoder(writer).Encode(reply)
}
func (api *API) fetchPackageList() (rpmmd.PackageList, error) {
var repos []rpmmd.RepoConfig
repos = append(repos, api.repo)
for _, source := range api.store.GetAllSources() {
repos = append(repos, rpmmd.SourceToRepo(source))
}
return api.rpmmd.FetchPackageList(repos)
}

100
internal/weldr/api_test.go
View file

File diff suppressed because one or more lines are too long