osbuild-composer

This commit is contained in:
Lars Karlitski 2019-09-11 11:45:06 +02:00
commit 1adbf1a4a6
8 changed files with 838 additions and 0 deletions

72
dnf-json Normal file
View file

@ -0,0 +1,72 @@
#!/usr/bin/python3
import datetime
import dnf
import json
import sys
def timestamp_to_rfc3339(timestamp):
d = datetime.datetime.utcfromtimestamp(package.buildtime)
return d.strftime('%Y-%m-%dT%H:%M:%SZ')
# base.sack.query().filter(provides=str(reldep))
try:
command = sys.argv[1]
arguments = sys.argv[2:]
except IndexError:
command = "list"
arguments = []
base = dnf.Base()
base.conf.cachedir = "./dnf-cache"
base.conf.substitutions["releasever"] = "30"
base.conf.substitutions["basearch"] = "x86_64"
repo = dnf.repo.Repo("fedora", base.conf)
repo.name = "Fedora"
repo.metalink = "https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch"
base.repos.add(repo)
base.fill_sack(load_system_repo=False)
if command == "list":
packages = [p.name for p in base.sack.query().available()]
json.dump(packages, sys.stdout)
elif command == "dump":
packages = []
for package in base.sack.query().available():
packages.append({
"name": package.name,
"summary": package.summary,
"description": package.description,
"url": package.url,
"epoch": package.epoch,
"version": package.version,
"release": package.release,
"arch": package.arch,
"buildtime": timestamp_to_rfc3339(package.buildtime),
"license": package.license
})
json.dump(packages, sys.stdout)
elif command == "depsolve":
for pkgspec in arguments:
base.install(pkgspec)
base.resolve()
packages = []
for package in base.transaction.install_set:
packages.append({
"name": package.name,
"epoch": package.epoch,
"version": package.version,
"release": package.release,
"arch": package.arch
})
json.dump(packages, sys.stdout)

5
go.mod Normal file
View file

@ -0,0 +1,5 @@
module osbuild-composer
go 1.12
require github.com/julienschmidt/httprouter v1.2.0

2
go.sum Normal file
View file

@ -0,0 +1,2 @@
github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=

57
main.go Normal file
View file

@ -0,0 +1,57 @@
package main
import (
"context"
"net"
"net/http"
"os"
"os/signal"
"osbuild-composer/rpmmd"
"osbuild-composer/weldr"
)
type Composer struct{}
func (c *Composer) Repositories() []string {
return []string{"fedora-30"}
}
func (c *Composer) RepositoryConfig(name string) (rpmmd.RepoConfig, bool) {
if name != "fedora-30" {
return rpmmd.RepoConfig{}, false
}
return rpmmd.RepoConfig{
Id: "fedora-30",
Name: "Fedora 30",
Metalink: "https://mirrors.fedoraproject.org/metalink?repo=fedora-30&arch=x86_64",
}, true
}
func main() {
listener, err := net.Listen("unix", "/run/weldr/api.socket")
if err != nil {
panic(err)
}
composer := Composer{}
api := weldr.New(&composer)
server := http.Server{Handler: api}
shutdownDone := make(chan struct{}, 1)
go func() {
channel := make(chan os.Signal, 1)
signal.Notify(channel, os.Interrupt)
<-channel
server.Shutdown(context.Background())
close(shutdownDone)
}()
err = server.Serve(listener)
if err != nil && err != http.ErrServerClosed {
panic(err)
}
<-shutdownDone
}

90
rpmmd/repository.go Normal file
View file

@ -0,0 +1,90 @@
package rpmmd
import (
"encoding/json"
"os/exec"
"sort"
"time"
)
type RepoConfig struct {
Id string `json:"id"`
Name string `json:"name"`
BaseURL string `json:"baseurl,omitempty"`
Metalink string `json:"metalink,omitempty"`
MirrorList string `json:"mirrorlist,omitempty"`
}
type PackageList []Package
type Package struct {
Name string
Summary string
Description string
URL string
Epoch uint
Version string
Release string
Arch string
BuildTime time.Time
License string
}
type PackageSpec struct {
Name string `json:"name"`
Epoch uint `json:"epoch,omitempty"`
Version string `json:"version,omitempty"`
Release string `json:"release,omitempty"`
Arch string `json:"arch,omitempty"`
}
func runDNF(command string, arguments []string, result interface{}) error {
argv := append([]string{"dnf-json", command}, arguments...)
cmd := exec.Command("python3", argv...)
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
return err
}
err = json.NewDecoder(stdout).Decode(result)
if err != nil {
return err
}
return cmd.Wait()
}
func FetchPackageList(repo RepoConfig) (PackageList, error) {
var packages PackageList
err := runDNF("dump", nil, &packages)
return packages, err
}
func Depsolve(specs ...string) ([]PackageSpec, error) {
var dependencies []PackageSpec
err := runDNF("depsolve", specs, &dependencies)
return dependencies, err
}
func (packages PackageList) Search(name string) (int, int) {
first := sort.Search(len(packages), func(i int) bool {
return packages[i].Name >= name
})
if first == len(packages) || packages[first].Name != name {
return first, 0
}
last := first + 1
for last < len(packages) && packages[last].Name == name {
last++
}
return first, last - first
}

517
weldr/api.go Normal file
View file

@ -0,0 +1,517 @@
package weldr
import (
"encoding/json"
"log"
"net/http"
"strings"
"time"
"github.com/julienschmidt/httprouter"
"osbuild-composer/rpmmd"
)
type API struct {
store *store
composer Composer
baseRepo rpmmd.RepoConfig
packages rpmmd.PackageList
router *httprouter.Router
}
type Composer interface {
Repositories() []string
RepositoryConfig(name string) (rpmmd.RepoConfig, bool)
}
func New(composer Composer) *API {
api := &API{
composer: composer,
store: newStore(),
}
// only one repository right now
api.baseRepo, _ = composer.RepositoryConfig(composer.Repositories()[0])
var err error
api.packages, err = rpmmd.FetchPackageList(api.baseRepo)
if err != nil {
panic(err)
}
// sample blueprint
api.store.pushBlueprint(blueprint{
Name: "example",
Description: "An Example",
Version: "1",
Packages: []blueprintPackage{{"httpd", "2.*"}},
Modules: []blueprintPackage{},
})
api.router = httprouter.New()
api.router.RedirectTrailingSlash = false
api.router.RedirectFixedPath = false
api.router.MethodNotAllowed = http.HandlerFunc(methodNotAllowedHandler)
api.router.NotFound = http.HandlerFunc(notFoundHandler)
api.router.GET("/api/status", api.statusHandler)
api.router.GET("/api/v0/projects/source/list", api.sourceListHandler)
api.router.GET("/api/v0/projects/source/info/:sources", api.sourceInfoHandler)
api.router.GET("/api/v0/modules/list", api.modulesListAllHandler)
api.router.GET("/api/v0/modules/list/:modules", api.modulesListHandler)
// these are the same, except that modules/info also includes dependencies
api.router.GET("/api/v0/modules/info/:modules", api.modulesInfoHandler)
api.router.GET("/api/v0/projects/info/:modules", api.modulesInfoHandler)
api.router.GET("/api/v0/blueprints/list", api.blueprintsListHandler)
api.router.GET("/api/v0/blueprints/info/:blueprints", api.blueprintsInfoHandler)
api.router.GET("/api/v0/blueprints/depsolve/:blueprints", api.blueprintsDepsolveHandler)
api.router.GET("/api/v0/blueprints/diff/:blueprint/:from/:to", api.blueprintsDiffHandler)
api.router.POST("/api/v0/blueprints/new", api.blueprintsNewHandler)
api.router.GET("/api/v0/compose/queue", api.composeQueueHandler)
api.router.GET("/api/v0/compose/finished", api.composeFinishedHandler)
api.router.GET("/api/v0/compose/failed", api.composeFailedHandler)
return api
}
func (api *API) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
log.Println(request.URL.Path)
writer.Header().Set("Content-Type", "application/json; charset=utf-8")
api.router.ServeHTTP(writer, request)
}
func methodNotAllowedHandler(writer http.ResponseWriter, request *http.Request) {
writer.WriteHeader(http.StatusMethodNotAllowed)
}
func notFoundHandler(writer http.ResponseWriter, request *http.Request) {
writer.WriteHeader(http.StatusNotFound)
}
func statusResponseOK(writer http.ResponseWriter) {
type reply struct {
Status bool `json:"status"`
}
writer.WriteHeader(http.StatusOK)
json.NewEncoder(writer).Encode(reply{true})
}
func statusResponseError(writer http.ResponseWriter, code int, errors ...string) {
type reply struct {
Status bool `json:"status"`
Errors []string `json:"errors,omitempty"`
}
writer.WriteHeader(code)
json.NewEncoder(writer).Encode(reply{false, errors})
}
func (api *API) statusHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) {
type reply struct {
Api uint `json:"api"`
DBSupported bool `json:"db_supported"`
DBVersion string `json:"db_version"`
SchemaVersion string `json:"schema_version"`
Backend string `json:"backend"`
Build string `json:"build"`
Messages []string `json:"messages"`
}
json.NewEncoder(writer).Encode(reply{
Api: 1,
DBSupported: true,
DBVersion: "0",
SchemaVersion: "0",
Backend: "osbuild-composer",
Build: "devel",
Messages: make([]string, 0),
})
}
func (api *API) sourceListHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) {
type reply struct {
Sources []string `json:"sources"`
}
json.NewEncoder(writer).Encode(reply{
Sources: []string{api.baseRepo.Name},
})
}
func (api *API) sourceInfoHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
// weldr uses a slightly different format than dnf to store repository
// configuration
type sourceConfig struct {
Id string `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
URL string `json:"url"`
CheckGPG bool `json:"check_gpg"`
CheckSSL bool `json:"check_ssl"`
System bool `json:"system"`
}
type reply struct {
Sources map[string]sourceConfig `json:"sources"`
}
// we only have one repository
names := strings.Split(params.ByName("sources"), ",")
if names[0] != api.baseRepo.Name && names[0] != "*" {
statusResponseError(writer, http.StatusBadRequest, "repository not found: "+names[0])
return
}
cfg := sourceConfig{
Id: api.baseRepo.Id,
Name: api.baseRepo.Name,
CheckGPG: true,
CheckSSL: true,
System: true,
}
if api.baseRepo.BaseURL != "" {
cfg.URL = api.baseRepo.BaseURL
cfg.Type = "yum-baseurl"
} else if api.baseRepo.Metalink != "" {
cfg.URL = api.baseRepo.Metalink
cfg.Type = "yum-metalink"
} else if api.baseRepo.MirrorList != "" {
cfg.URL = api.baseRepo.MirrorList
cfg.Type = "yum-mirrorlist"
}
json.NewEncoder(writer).Encode(reply{
Sources: map[string]sourceConfig{cfg.Name: cfg},
})
}
type modulesListModule struct {
Name string `json:"name"`
GroupType string `json:"group_type"`
}
type modulesListReply struct {
Total uint `json:"total"`
Offset uint `json:"offset"`
Limit uint `json:"limit"`
Modules []modulesListModule `json:"modules"`
}
func (api *API) modulesListAllHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) {
offset, limit, err := parseOffsetAndLimit(request.URL.Query())
if err != nil {
statusResponseError(writer, http.StatusBadRequest, "BadRequest: "+err.Error())
return
}
total := uint(len(api.packages))
offset = min(offset, total)
limit = min(limit, total-offset)
modules := make([]modulesListModule, limit)
for i := uint(0); i < limit; i++ {
modules[i] = modulesListModule{api.packages[offset+i].Name, "rpm"}
}
json.NewEncoder(writer).Encode(modulesListReply{
Total: total,
Offset: offset,
Limit: limit,
Modules: modules,
})
}
func (api *API) modulesListHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
names := strings.Split(params.ByName("modules"), ",")
if names[0] == "" || names[0] == "*" {
api.modulesListAllHandler(writer, request, params)
return
}
offset, limit, err := parseOffsetAndLimit(request.URL.Query())
if err != nil {
statusResponseError(writer, http.StatusBadRequest, "BadRequest: "+err.Error())
return
}
// we don't support glob-matching, but cockpit-composer surrounds some
// queries with asterisks; this is crude, but solves that case
for i := range names {
names[i] = strings.ReplaceAll(names[i], "*", "")
}
modules := make([]modulesListModule, 0)
total := uint(0)
end := offset + limit
for _, pkg := range api.packages {
for _, name := range names {
if strings.Contains(pkg.Name, name) {
total += 1
if total > offset && total < end {
modules = append(modules, modulesListModule{pkg.Name, "rpm"})
}
}
}
}
json.NewEncoder(writer).Encode(modulesListReply{
Total: total,
Offset: min(offset, total),
Limit: min(limit, total-offset),
Modules: modules,
})
}
func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
type source struct {
License string `json:"license"`
Version string `json:"version"`
}
type build struct {
Arch string `json:"arch"`
BuildTime time.Time `json:"build_time"`
Epoch uint `json:"epoch"`
Release string `json:"release"`
Source source `json:"source"`
}
type project struct {
Name string `json:"name"`
Summary string `json:"summary"`
Description string `json:"description"`
Homepage string `json:"homepage"`
Builds []build `json:"builds"`
Dependencies []rpmmd.PackageSpec `json:"dependencies,omitempty"`
}
type projectsReply struct {
Projects []project `json:"projects"`
}
type modulesReply struct {
Modules []project `json:"modules"`
}
// handle both projects/info and modules/info, the latter includes dependencies
modulesRequested := strings.HasPrefix(request.URL.Path, "/api/v0/modules")
names := strings.Split(params.ByName("modules"), ",")
if names[0] == "" {
statusResponseError(writer, http.StatusNotFound)
return
}
projects := make([]project, 0)
for _, name := range names {
first, n := api.packages.Search(name)
if n == 0 {
statusResponseError(writer, http.StatusNotFound)
return
}
// get basic info from the first package, but collect build
// information from all that have the same name
pkg := api.packages[first]
project := project{
Name: pkg.Name,
Summary: pkg.Summary,
Description: pkg.Description,
Homepage: pkg.URL,
}
project.Builds = make([]build, n)
for i, pkg := range api.packages[first : first+n] {
project.Builds[i] = build{
Arch: pkg.Arch,
BuildTime: pkg.BuildTime,
Epoch: pkg.Epoch,
Release: pkg.Release,
Source: source{pkg.License, pkg.Version},
}
}
if modulesRequested {
project.Dependencies, _ = rpmmd.Depsolve(pkg.Name)
}
projects = append(projects, project)
}
if modulesRequested {
json.NewEncoder(writer).Encode(modulesReply{projects})
} else {
json.NewEncoder(writer).Encode(projectsReply{projects})
}
}
func (api *API) blueprintsListHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) {
type reply struct {
Total uint `json:"total"`
Offset uint `json:"offset"`
Limit uint `json:"limit"`
Blueprints []string `json:"blueprints"`
}
offset, limit, err := parseOffsetAndLimit(request.URL.Query())
if err != nil {
statusResponseError(writer, http.StatusBadRequest, "BadRequest: "+err.Error())
return
}
names := api.store.listBlueprints()
total := uint(len(names))
offset = min(offset, total)
limit = min(limit, total-offset)
json.NewEncoder(writer).Encode(reply{
Total: total,
Offset: offset,
Limit: limit,
Blueprints: names[offset : offset+limit],
})
}
func (api *API) blueprintsInfoHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
type change struct {
Changed bool `json:"changed"`
Name string `json:"name"`
}
type reply struct {
Blueprints []blueprint `json:"blueprints"`
Changes []change `json:"changes"`
Errors []string `json:"errors"`
}
names := strings.Split(params.ByName("blueprints"), ",")
if names[0] == "" {
statusResponseError(writer, http.StatusNotFound)
return
}
blueprints := []blueprint{}
changes := []change{}
for _, name := range names {
blueprint, ok := api.store.getBlueprint(name)
if !ok {
statusResponseError(writer, http.StatusNotFound)
return
}
blueprints = append(blueprints, blueprint)
changes = append(changes, change{false, blueprint.Name})
}
json.NewEncoder(writer).Encode(reply{
Blueprints: blueprints,
Changes: changes,
Errors: []string{},
})
}
func (api *API) blueprintsDepsolveHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
type entry struct {
Blueprint blueprint `json:"blueprint"`
Dependencies []rpmmd.PackageSpec `json:"dependencies"`
}
type reply struct {
Blueprints []entry `json:"blueprints"`
Errors []string `json:"errors"`
}
names := strings.Split(params.ByName("blueprints"), ",")
if names[0] == "" {
statusResponseError(writer, http.StatusNotFound)
return
}
blueprints := []entry{}
for _, name := range names {
blueprint, ok := api.store.getBlueprint(name)
if !ok {
statusResponseError(writer, http.StatusNotFound)
return
}
specs := make([]string, len(blueprint.Packages))
for i, pkg := range blueprint.Packages {
specs[i] = pkg.Name
if pkg.Version != "" {
specs[i] += "-" + pkg.Version
}
}
dependencies, _ := rpmmd.Depsolve(specs...)
blueprints = append(blueprints, entry{blueprint, dependencies})
}
json.NewEncoder(writer).Encode(reply{
Blueprints: blueprints,
Errors: []string{},
})
}
func (api *API) blueprintsDiffHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) {
var reply struct {
Diff []interface{} `json:"diff"`
}
reply.Diff = make([]interface{}, 0)
json.NewEncoder(writer).Encode(reply)
}
func (api *API) blueprintsNewHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) {
contentType := request.Header["Content-Type"]
if len(contentType) != 1 || contentType[0] != "application/json" {
statusResponseError(writer, http.StatusUnsupportedMediaType, "blueprint must be json")
return
}
var blueprint blueprint
err := json.NewDecoder(request.Body).Decode(&blueprint)
if err != nil {
statusResponseError(writer, http.StatusBadRequest, "invalid blueprint: "+err.Error())
return
}
api.store.pushBlueprint(blueprint)
statusResponseOK(writer)
}
func (api *API) composeQueueHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) {
var reply struct {
New []interface{} `json:"new"`
Run []interface{} `json:"run"`
}
reply.New = make([]interface{}, 0)
reply.Run = make([]interface{}, 0)
json.NewEncoder(writer).Encode(reply)
}
func (api *API) composeFinishedHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) {
var reply struct {
Finished []interface{} `json:"finished"`
}
reply.Finished = make([]interface{}, 0)
json.NewEncoder(writer).Encode(reply)
}
func (api *API) composeFailedHandler(writer http.ResponseWriter, request *http.Request, _ httprouter.Params) {
var reply struct {
Failed []interface{} `json:"failed"`
}
reply.Failed = make([]interface{}, 0)
json.NewEncoder(writer).Encode(reply)
}

59
weldr/store.go Normal file
View file

@ -0,0 +1,59 @@
package weldr
import (
"sort"
"sync"
)
type store struct {
Blueprints map[string]blueprint `json:"blueprints"`
mu sync.RWMutex // protects all fields
}
type blueprint struct {
Name string `json:"name"`
Description string `json:"description,omitempty"`
Version string `json:"version,omitempty"`
Packages []blueprintPackage `json:"packages"`
Modules []blueprintPackage `json:"modules"`
}
type blueprintPackage struct {
Name string `json:"name"`
Version string `json:"version,omitempty"`
}
func newStore() *store {
return &store{
Blueprints: make(map[string]blueprint),
}
}
func (s *store) listBlueprints() []string {
s.mu.RLock()
defer s.mu.RUnlock()
names := make([]string, 0, len(s.Blueprints))
for name := range s.Blueprints {
names = append(names, name)
}
sort.Strings(names)
return names
}
func (s *store) getBlueprint(name string) (blueprint, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
bp, ok := s.Blueprints[name]
return bp, ok
}
func (s *store) pushBlueprint(bp blueprint) {
s.mu.Lock()
defer s.mu.Unlock()
s.Blueprints[bp.Name] = bp
}

36
weldr/util.go Normal file
View file

@ -0,0 +1,36 @@
package weldr
import (
"errors"
"net/url"
"strconv"
)
func parseOffsetAndLimit(query url.Values) (uint, uint, error) {
var offset, limit uint64 = 0, 10
var err error
if v := query.Get("offset"); v != "" {
offset, err = strconv.ParseUint(v, 10, 64)
if err != nil {
return 0, 0, errors.New("invalid value for 'offset': " + err.Error())
}
}
if v := query.Get("limit"); v != "" {
limit, err = strconv.ParseUint(v, 10, 64)
if err != nil {
return 0, 0, errors.New("invalid value for 'limit': " + err.Error())
}
}
return uint(offset), uint(limit), nil
}
func min(a, b uint) uint {
if a < b {
return a
} else {
return b
}
}