From 1adbf1a4a6777aa5837f348c1e6f82243bf4d449 Mon Sep 17 00:00:00 2001 From: Lars Karlitski Date: Wed, 11 Sep 2019 11:45:06 +0200 Subject: [PATCH] osbuild-composer --- dnf-json | 72 ++++++ go.mod | 5 + go.sum | 2 + main.go | 57 +++++ rpmmd/repository.go | 90 ++++++++ weldr/api.go | 517 ++++++++++++++++++++++++++++++++++++++++++++ weldr/store.go | 59 +++++ weldr/util.go | 36 +++ 8 files changed, 838 insertions(+) create mode 100644 dnf-json create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 rpmmd/repository.go create mode 100644 weldr/api.go create mode 100644 weldr/store.go create mode 100644 weldr/util.go diff --git a/dnf-json b/dnf-json new file mode 100644 index 000000000..8cca954ea --- /dev/null +++ b/dnf-json @@ -0,0 +1,72 @@ +#!/usr/bin/python3 + +import datetime +import dnf +import json +import sys + + +def timestamp_to_rfc3339(timestamp): + d = datetime.datetime.utcfromtimestamp(package.buildtime) + return d.strftime('%Y-%m-%dT%H:%M:%SZ') + + +# base.sack.query().filter(provides=str(reldep)) + + +try: + command = sys.argv[1] + arguments = sys.argv[2:] +except IndexError: + command = "list" + arguments = [] + + +base = dnf.Base() + +base.conf.cachedir = "./dnf-cache" +base.conf.substitutions["releasever"] = "30" +base.conf.substitutions["basearch"] = "x86_64" + +repo = dnf.repo.Repo("fedora", base.conf) +repo.name = "Fedora" +repo.metalink = "https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch" + +base.repos.add(repo) +base.fill_sack(load_system_repo=False) + +if command == "list": + packages = [p.name for p in base.sack.query().available()] + json.dump(packages, sys.stdout) + +elif command == "dump": + packages = [] + for package in base.sack.query().available(): + packages.append({ + "name": package.name, + "summary": package.summary, + "description": package.description, + "url": package.url, + "epoch": package.epoch, + "version": package.version, + "release": package.release, + "arch": package.arch, + "buildtime": timestamp_to_rfc3339(package.buildtime), + "license": package.license + }) + json.dump(packages, sys.stdout) + +elif command == "depsolve": + for pkgspec in arguments: + base.install(pkgspec) + base.resolve() + packages = [] + for package in base.transaction.install_set: + packages.append({ + "name": package.name, + "epoch": package.epoch, + "version": package.version, + "release": package.release, + "arch": package.arch + }) + json.dump(packages, sys.stdout) diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..219899158 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module osbuild-composer + +go 1.12 + +require github.com/julienschmidt/httprouter v1.2.0 diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..0ead70e8c --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= diff --git a/main.go b/main.go new file mode 100644 index 000000000..4dce42323 --- /dev/null +++ b/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "net" + "net/http" + "os" + "os/signal" + + "osbuild-composer/rpmmd" + "osbuild-composer/weldr" +) + +type Composer struct{} + +func (c *Composer) Repositories() []string { + return []string{"fedora-30"} +} + +func (c *Composer) RepositoryConfig(name string) (rpmmd.RepoConfig, bool) { + if name != "fedora-30" { + return rpmmd.RepoConfig{}, false + } + + return rpmmd.RepoConfig{ + Id: "fedora-30", + Name: "Fedora 30", + Metalink: "https://mirrors.fedoraproject.org/metalink?repo=fedora-30&arch=x86_64", + }, true +} + +func main() { + listener, err := net.Listen("unix", "/run/weldr/api.socket") + if err != nil { + panic(err) + } + + composer := Composer{} + api := weldr.New(&composer) + server := http.Server{Handler: api} + + shutdownDone := make(chan struct{}, 1) + go func() { + channel := make(chan os.Signal, 1) + signal.Notify(channel, os.Interrupt) + <-channel + server.Shutdown(context.Background()) + close(shutdownDone) + }() + + err = server.Serve(listener) + if err != nil && err != http.ErrServerClosed { + panic(err) + } + + <-shutdownDone +} diff --git a/rpmmd/repository.go b/rpmmd/repository.go new file mode 100644 index 000000000..081eabd97 --- /dev/null +++ b/rpmmd/repository.go @@ -0,0 +1,90 @@ +package rpmmd + +import ( + "encoding/json" + "os/exec" + "sort" + "time" +) + +type RepoConfig struct { + Id string `json:"id"` + Name string `json:"name"` + BaseURL string `json:"baseurl,omitempty"` + Metalink string `json:"metalink,omitempty"` + MirrorList string `json:"mirrorlist,omitempty"` +} + +type PackageList []Package + +type Package struct { + Name string + Summary string + Description string + URL string + Epoch uint + Version string + Release string + Arch string + BuildTime time.Time + License string +} + +type PackageSpec struct { + Name string `json:"name"` + Epoch uint `json:"epoch,omitempty"` + Version string `json:"version,omitempty"` + Release string `json:"release,omitempty"` + Arch string `json:"arch,omitempty"` +} + +func runDNF(command string, arguments []string, result interface{}) error { + argv := append([]string{"dnf-json", command}, arguments...) + + cmd := exec.Command("python3", argv...) + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + + err = cmd.Start() + if err != nil { + return err + } + + err = json.NewDecoder(stdout).Decode(result) + if err != nil { + return err + } + + return cmd.Wait() +} + +func FetchPackageList(repo RepoConfig) (PackageList, error) { + var packages PackageList + err := runDNF("dump", nil, &packages) + return packages, err +} + +func Depsolve(specs ...string) ([]PackageSpec, error) { + var dependencies []PackageSpec + err := runDNF("depsolve", specs, &dependencies) + return dependencies, err +} + +func (packages PackageList) Search(name string) (int, int) { + first := sort.Search(len(packages), func(i int) bool { + return packages[i].Name >= name + }) + + if first == len(packages) || packages[first].Name != name { + return first, 0 + } + + last := first + 1 + for last < len(packages) && packages[last].Name == name { + last++ + } + + return first, last - first +} diff --git a/weldr/api.go b/weldr/api.go new file mode 100644 index 000000000..ca0b065c3 --- /dev/null +++ b/weldr/api.go @@ -0,0 +1,517 @@ +package weldr + +import ( + "encoding/json" + "log" + "net/http" + "strings" + "time" + + "github.com/julienschmidt/httprouter" + + "osbuild-composer/rpmmd" +) + +type API struct { + store *store + composer Composer + + baseRepo rpmmd.RepoConfig + packages rpmmd.PackageList + + router *httprouter.Router +} + +type Composer interface { + Repositories() []string + RepositoryConfig(name string) (rpmmd.RepoConfig, bool) +} + +func New(composer Composer) *API { + api := &API{ + composer: composer, + store: newStore(), + } + + // only one repository right now + api.baseRepo, _ = composer.RepositoryConfig(composer.Repositories()[0]) + + var err error + api.packages, err = rpmmd.FetchPackageList(api.baseRepo) + if err != nil { + panic(err) + } + + // sample blueprint + api.store.pushBlueprint(blueprint{ + Name: "example", + Description: "An Example", + Version: "1", + Packages: []blueprintPackage{{"httpd", "2.*"}}, + Modules: []blueprintPackage{}, + }) + + api.router = httprouter.New() + api.router.RedirectTrailingSlash = false + api.router.RedirectFixedPath = false + api.router.MethodNotAllowed = http.HandlerFunc(methodNotAllowedHandler) + api.router.NotFound = http.HandlerFunc(notFoundHandler) + + api.router.GET("/api/status", api.statusHandler) + api.router.GET("/api/v0/projects/source/list", api.sourceListHandler) + api.router.GET("/api/v0/projects/source/info/:sources", api.sourceInfoHandler) + + api.router.GET("/api/v0/modules/list", api.modulesListAllHandler) + api.router.GET("/api/v0/modules/list/:modules", api.modulesListHandler) + + // these are the same, except that modules/info also includes dependencies + api.router.GET("/api/v0/modules/info/:modules", api.modulesInfoHandler) + api.router.GET("/api/v0/projects/info/:modules", api.modulesInfoHandler) + + api.router.GET("/api/v0/blueprints/list", api.blueprintsListHandler) + api.router.GET("/api/v0/blueprints/info/:blueprints", api.blueprintsInfoHandler) + api.router.GET("/api/v0/blueprints/depsolve/:blueprints", api.blueprintsDepsolveHandler) + api.router.GET("/api/v0/blueprints/diff/:blueprint/:from/:to", api.blueprintsDiffHandler) + api.router.POST("/api/v0/blueprints/new", api.blueprintsNewHandler) + + api.router.GET("/api/v0/compose/queue", api.composeQueueHandler) + api.router.GET("/api/v0/compose/finished", api.composeFinishedHandler) + api.router.GET("/api/v0/compose/failed", api.composeFailedHandler) + + return api +} + +func (api *API) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + log.Println(request.URL.Path) + writer.Header().Set("Content-Type", "application/json; charset=utf-8") + api.router.ServeHTTP(writer, request) +} + +func methodNotAllowedHandler(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusMethodNotAllowed) +} + +func notFoundHandler(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusNotFound) +} + +func statusResponseOK(writer http.ResponseWriter) { + type reply struct { + Status bool `json:"status"` + } + + writer.WriteHeader(http.StatusOK) + json.NewEncoder(writer).Encode(reply{true}) +} + +func statusResponseError(writer http.ResponseWriter, code int, errors ...string) { + type reply struct { + Status bool `json:"status"` + Errors []string `json:"errors,omitempty"` + } + + writer.WriteHeader(code) + json.NewEncoder(writer).Encode(reply{false, errors}) +} + +func (api *API) statusHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) { + type reply struct { + Api uint `json:"api"` + DBSupported bool `json:"db_supported"` + DBVersion string `json:"db_version"` + SchemaVersion string `json:"schema_version"` + Backend string `json:"backend"` + Build string `json:"build"` + Messages []string `json:"messages"` + } + + json.NewEncoder(writer).Encode(reply{ + Api: 1, + DBSupported: true, + DBVersion: "0", + SchemaVersion: "0", + Backend: "osbuild-composer", + Build: "devel", + Messages: make([]string, 0), + }) +} + +func (api *API) sourceListHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) { + type reply struct { + Sources []string `json:"sources"` + } + + json.NewEncoder(writer).Encode(reply{ + Sources: []string{api.baseRepo.Name}, + }) +} + +func (api *API) sourceInfoHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { + // weldr uses a slightly different format than dnf to store repository + // configuration + type sourceConfig struct { + Id string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + URL string `json:"url"` + CheckGPG bool `json:"check_gpg"` + CheckSSL bool `json:"check_ssl"` + System bool `json:"system"` + } + type reply struct { + Sources map[string]sourceConfig `json:"sources"` + } + + // we only have one repository + names := strings.Split(params.ByName("sources"), ",") + if names[0] != api.baseRepo.Name && names[0] != "*" { + statusResponseError(writer, http.StatusBadRequest, "repository not found: "+names[0]) + return + } + + cfg := sourceConfig{ + Id: api.baseRepo.Id, + Name: api.baseRepo.Name, + CheckGPG: true, + CheckSSL: true, + System: true, + } + + if api.baseRepo.BaseURL != "" { + cfg.URL = api.baseRepo.BaseURL + cfg.Type = "yum-baseurl" + } else if api.baseRepo.Metalink != "" { + cfg.URL = api.baseRepo.Metalink + cfg.Type = "yum-metalink" + } else if api.baseRepo.MirrorList != "" { + cfg.URL = api.baseRepo.MirrorList + cfg.Type = "yum-mirrorlist" + } + + json.NewEncoder(writer).Encode(reply{ + Sources: map[string]sourceConfig{cfg.Name: cfg}, + }) +} + +type modulesListModule struct { + Name string `json:"name"` + GroupType string `json:"group_type"` +} + +type modulesListReply struct { + Total uint `json:"total"` + Offset uint `json:"offset"` + Limit uint `json:"limit"` + Modules []modulesListModule `json:"modules"` +} + +func (api *API) modulesListAllHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) { + offset, limit, err := parseOffsetAndLimit(request.URL.Query()) + if err != nil { + statusResponseError(writer, http.StatusBadRequest, "BadRequest: "+err.Error()) + return + } + + total := uint(len(api.packages)) + offset = min(offset, total) + limit = min(limit, total-offset) + + modules := make([]modulesListModule, limit) + for i := uint(0); i < limit; i++ { + modules[i] = modulesListModule{api.packages[offset+i].Name, "rpm"} + } + + json.NewEncoder(writer).Encode(modulesListReply{ + Total: total, + Offset: offset, + Limit: limit, + Modules: modules, + }) +} + +func (api *API) modulesListHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { + names := strings.Split(params.ByName("modules"), ",") + if names[0] == "" || names[0] == "*" { + api.modulesListAllHandler(writer, request, params) + return + } + + offset, limit, err := parseOffsetAndLimit(request.URL.Query()) + if err != nil { + statusResponseError(writer, http.StatusBadRequest, "BadRequest: "+err.Error()) + return + } + + // we don't support glob-matching, but cockpit-composer surrounds some + // queries with asterisks; this is crude, but solves that case + for i := range names { + names[i] = strings.ReplaceAll(names[i], "*", "") + } + + modules := make([]modulesListModule, 0) + total := uint(0) + end := offset + limit + + for _, pkg := range api.packages { + for _, name := range names { + if strings.Contains(pkg.Name, name) { + total += 1 + if total > offset && total < end { + modules = append(modules, modulesListModule{pkg.Name, "rpm"}) + } + } + } + } + + json.NewEncoder(writer).Encode(modulesListReply{ + Total: total, + Offset: min(offset, total), + Limit: min(limit, total-offset), + Modules: modules, + }) +} + +func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { + type source struct { + License string `json:"license"` + Version string `json:"version"` + } + type build struct { + Arch string `json:"arch"` + BuildTime time.Time `json:"build_time"` + Epoch uint `json:"epoch"` + Release string `json:"release"` + Source source `json:"source"` + } + type project struct { + Name string `json:"name"` + Summary string `json:"summary"` + Description string `json:"description"` + Homepage string `json:"homepage"` + Builds []build `json:"builds"` + Dependencies []rpmmd.PackageSpec `json:"dependencies,omitempty"` + } + type projectsReply struct { + Projects []project `json:"projects"` + } + type modulesReply struct { + Modules []project `json:"modules"` + } + + // handle both projects/info and modules/info, the latter includes dependencies + modulesRequested := strings.HasPrefix(request.URL.Path, "/api/v0/modules") + + names := strings.Split(params.ByName("modules"), ",") + if names[0] == "" { + statusResponseError(writer, http.StatusNotFound) + return + } + + projects := make([]project, 0) + for _, name := range names { + first, n := api.packages.Search(name) + if n == 0 { + statusResponseError(writer, http.StatusNotFound) + return + } + + // get basic info from the first package, but collect build + // information from all that have the same name + pkg := api.packages[first] + project := project{ + Name: pkg.Name, + Summary: pkg.Summary, + Description: pkg.Description, + Homepage: pkg.URL, + } + + project.Builds = make([]build, n) + for i, pkg := range api.packages[first : first+n] { + project.Builds[i] = build{ + Arch: pkg.Arch, + BuildTime: pkg.BuildTime, + Epoch: pkg.Epoch, + Release: pkg.Release, + Source: source{pkg.License, pkg.Version}, + } + } + + if modulesRequested { + project.Dependencies, _ = rpmmd.Depsolve(pkg.Name) + } + + projects = append(projects, project) + } + + if modulesRequested { + json.NewEncoder(writer).Encode(modulesReply{projects}) + } else { + json.NewEncoder(writer).Encode(projectsReply{projects}) + } +} + +func (api *API) blueprintsListHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) { + type reply struct { + Total uint `json:"total"` + Offset uint `json:"offset"` + Limit uint `json:"limit"` + Blueprints []string `json:"blueprints"` + } + + offset, limit, err := parseOffsetAndLimit(request.URL.Query()) + if err != nil { + statusResponseError(writer, http.StatusBadRequest, "BadRequest: "+err.Error()) + return + } + + names := api.store.listBlueprints() + total := uint(len(names)) + offset = min(offset, total) + limit = min(limit, total-offset) + + json.NewEncoder(writer).Encode(reply{ + Total: total, + Offset: offset, + Limit: limit, + Blueprints: names[offset : offset+limit], + }) +} + +func (api *API) blueprintsInfoHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { + type change struct { + Changed bool `json:"changed"` + Name string `json:"name"` + } + type reply struct { + Blueprints []blueprint `json:"blueprints"` + Changes []change `json:"changes"` + Errors []string `json:"errors"` + } + + names := strings.Split(params.ByName("blueprints"), ",") + if names[0] == "" { + statusResponseError(writer, http.StatusNotFound) + return + } + + blueprints := []blueprint{} + changes := []change{} + for _, name := range names { + blueprint, ok := api.store.getBlueprint(name) + if !ok { + statusResponseError(writer, http.StatusNotFound) + return + } + blueprints = append(blueprints, blueprint) + changes = append(changes, change{false, blueprint.Name}) + } + + json.NewEncoder(writer).Encode(reply{ + Blueprints: blueprints, + Changes: changes, + Errors: []string{}, + }) +} + +func (api *API) blueprintsDepsolveHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { + type entry struct { + Blueprint blueprint `json:"blueprint"` + Dependencies []rpmmd.PackageSpec `json:"dependencies"` + } + type reply struct { + Blueprints []entry `json:"blueprints"` + Errors []string `json:"errors"` + } + + names := strings.Split(params.ByName("blueprints"), ",") + if names[0] == "" { + statusResponseError(writer, http.StatusNotFound) + return + } + + blueprints := []entry{} + for _, name := range names { + blueprint, ok := api.store.getBlueprint(name) + if !ok { + statusResponseError(writer, http.StatusNotFound) + return + } + + specs := make([]string, len(blueprint.Packages)) + for i, pkg := range blueprint.Packages { + specs[i] = pkg.Name + if pkg.Version != "" { + specs[i] += "-" + pkg.Version + } + } + + dependencies, _ := rpmmd.Depsolve(specs...) + + blueprints = append(blueprints, entry{blueprint, dependencies}) + } + + json.NewEncoder(writer).Encode(reply{ + Blueprints: blueprints, + Errors: []string{}, + }) +} + +func (api *API) blueprintsDiffHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) { + var reply struct { + Diff []interface{} `json:"diff"` + } + + reply.Diff = make([]interface{}, 0) + json.NewEncoder(writer).Encode(reply) +} + +func (api *API) blueprintsNewHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) { + contentType := request.Header["Content-Type"] + if len(contentType) != 1 || contentType[0] != "application/json" { + statusResponseError(writer, http.StatusUnsupportedMediaType, "blueprint must be json") + return + } + + var blueprint blueprint + err := json.NewDecoder(request.Body).Decode(&blueprint) + if err != nil { + statusResponseError(writer, http.StatusBadRequest, "invalid blueprint: "+err.Error()) + return + } + + api.store.pushBlueprint(blueprint) + + statusResponseOK(writer) +} + +func (api *API) composeQueueHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) { + var reply struct { + New []interface{} `json:"new"` + Run []interface{} `json:"run"` + } + + reply.New = make([]interface{}, 0) + reply.Run = make([]interface{}, 0) + + json.NewEncoder(writer).Encode(reply) +} + +func (api *API) composeFinishedHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) { + var reply struct { + Finished []interface{} `json:"finished"` + } + + reply.Finished = make([]interface{}, 0) + + json.NewEncoder(writer).Encode(reply) +} + +func (api *API) composeFailedHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) { + var reply struct { + Failed []interface{} `json:"failed"` + } + + reply.Failed = make([]interface{}, 0) + + json.NewEncoder(writer).Encode(reply) +} diff --git a/weldr/store.go b/weldr/store.go new file mode 100644 index 000000000..ea7fd729f --- /dev/null +++ b/weldr/store.go @@ -0,0 +1,59 @@ +package weldr + +import ( + "sort" + "sync" +) + +type store struct { + Blueprints map[string]blueprint `json:"blueprints"` + + mu sync.RWMutex // protects all fields +} + +type blueprint struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Version string `json:"version,omitempty"` + Packages []blueprintPackage `json:"packages"` + Modules []blueprintPackage `json:"modules"` +} + +type blueprintPackage struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` +} + +func newStore() *store { + return &store{ + Blueprints: make(map[string]blueprint), + } +} + +func (s *store) listBlueprints() []string { + s.mu.RLock() + defer s.mu.RUnlock() + + names := make([]string, 0, len(s.Blueprints)) + for name := range s.Blueprints { + names = append(names, name) + } + sort.Strings(names) + + return names +} + +func (s *store) getBlueprint(name string) (blueprint, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + bp, ok := s.Blueprints[name] + return bp, ok +} + +func (s *store) pushBlueprint(bp blueprint) { + s.mu.Lock() + defer s.mu.Unlock() + + s.Blueprints[bp.Name] = bp +} diff --git a/weldr/util.go b/weldr/util.go new file mode 100644 index 000000000..4e9c3d666 --- /dev/null +++ b/weldr/util.go @@ -0,0 +1,36 @@ +package weldr + +import ( + "errors" + "net/url" + "strconv" +) + +func parseOffsetAndLimit(query url.Values) (uint, uint, error) { + var offset, limit uint64 = 0, 10 + var err error + + if v := query.Get("offset"); v != "" { + offset, err = strconv.ParseUint(v, 10, 64) + if err != nil { + return 0, 0, errors.New("invalid value for 'offset': " + err.Error()) + } + } + + if v := query.Get("limit"); v != "" { + limit, err = strconv.ParseUint(v, 10, 64) + if err != nil { + return 0, 0, errors.New("invalid value for 'limit': " + err.Error()) + } + } + + return uint(offset), uint(limit), nil +} + +func min(a, b uint) uint { + if a < b { + return a + } else { + return b + } +}