particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

osbuild-composer: merge cloud API into main binary

Browse source 
This removes the osbuild-composer-cloud package, binary, systemd units,
the (unused) test binary, and the (only-run-on-RHEL) test in aws.sh.

Instead, move the cloud API into the main package, using the same
socket as the koji API, osbuild-composer-api.socket. Expose it next to
the koji API on route `/api/composer/v1`.

This is a backwards incompatible change, but only of the -cloud parts,
which have been marked as subject to change.
This commit is contained in:
Lars Karlitski 2020-10-03 09:58:39 +02:00 • committed by Ondřej Budai
parent 835b556db7
commit b25a350502
14 changed files with 50 additions and 428 deletions
Download patch file Download diff file Expand all files Collapse all files

80
cmd/osbuild-composer-cloud-tests/main_test.go
View file

@ -1,80 +0,0 @@
// +build integration
package main
import (
"context"
"net/http"
"testing"
"github.com/osbuild/osbuild-composer/internal/cloudapi"
"github.com/stretchr/testify/require"
"github.com/google/uuid"
)
func TestCloud(t *testing.T) {
client, err := cloudapi.NewClientWithResponses("http://127.0.0.1:8703/")
if err != nil {
panic(err)
}
response, err := client.ComposeWithResponse(context.Background(), cloudapi.ComposeJSONRequestBody{
Distribution: "rhel-8",
ImageRequests: []cloudapi.ImageRequest{
{
Architecture: "x86_64",
ImageType: "qcow2",
Repositories: []cloudapi.Repository{
{
Baseurl: "https://cdn.redhat.com/content/dist/rhel8/8/x86_64/baseos/os",
},
{
Baseurl: "https://cdn.redhat.com/content/dist/rhel8/8/x86_64/appstream/os",
},
},
UploadRequests: []cloudapi.UploadRequest{
{
Options: cloudapi.AWSUploadRequestOptions{
Ec2: cloudapi.AWSUploadRequestOptionsEc2{
AccessKeyId: "access-key-id",
SecretAccessKey: "my-secret-key",
},
Region: "eu-central-1",
S3: cloudapi.AWSUploadRequestOptionsS3{
AccessKeyId: "access-key-id",
SecretAccessKey: "my-secret-key",
Bucket: "bucket",
},
},
Type: "aws",
},
},
},
},
Customizations: &cloudapi.Customizations{
Subscription: &cloudapi.Subscription {
ActivationKey: "somekey",
BaseUrl: "http://cdn.stage.redhat.com/",
ServerUrl: "subscription.rhsm.stage.redhat.com",
Organization: 00000,
Insights: true,
},
},
})
require.NoError(t, err)
require.Equalf(t, http.StatusCreated, response.StatusCode(), "Error: got non-201 status. Full response: %v", string(response.Body))
require.NotNil(t, response.JSON201)
response2, err := client.ComposeStatusWithResponse(context.Background(), response.JSON201.Id)
require.NoError(t, err)
require.Equalf(t, response2.StatusCode(), http.StatusOK, "Error: got non-200 status. Full response: %v", response2.Body)
response2, err = client.ComposeStatusWithResponse(context.Background(), "invalid-id")
require.NoError(t, err)
require.Equalf(t, response2.StatusCode(), http.StatusBadRequest, "Error: got non-400 status. Full response: %v", response2.Body)
response2, err = client.ComposeStatusWithResponse(context.Background(), uuid.New().String())
require.NoError(t, err)
require.Equalf(t, response2.StatusCode(), http.StatusNotFound, "Error: got non-404 status. Full response: %s", response2.Body)
}

149
cmd/osbuild-composer-cloud/main.go
View file

@ -1,149 +0,0 @@
package main
import (
"crypto/tls"
"crypto/x509"
"flag"
"fmt"
"io/ioutil"
"log"
"net"
"os"
"path"
"github.com/osbuild/osbuild-composer/internal/cloudapi"
"github.com/osbuild/osbuild-composer/internal/distro"
"github.com/osbuild/osbuild-composer/internal/distro/fedora31"
"github.com/osbuild/osbuild-composer/internal/distro/fedora32"
"github.com/osbuild/osbuild-composer/internal/distro/rhel8"
"github.com/osbuild/osbuild-composer/internal/jobqueue/fsjobqueue"
"github.com/osbuild/osbuild-composer/internal/rpmmd"
"github.com/osbuild/osbuild-composer/internal/worker"
"github.com/coreos/go-systemd/activation"
)
type connectionConfig struct {
CACertFile string
ServerKeyFile string
ServerCertFile string
}
func createTLSConfig(c *connectionConfig) (*tls.Config, error) {
caCertPEM, err := ioutil.ReadFile(c.CACertFile)
if err != nil {
panic(fmt.Sprintf("Failed to read root certificate %v", c.CACertFile))
}
roots := x509.NewCertPool()
ok := roots.AppendCertsFromPEM(caCertPEM)
if !ok {
panic(fmt.Sprintf("Failed to parse root certificate %v", c.CACertFile))
}
cert, err := tls.LoadX509KeyPair(c.ServerCertFile, c.ServerKeyFile)
if err != nil {
return nil, err
}
return &tls.Config{
Certificates: []tls.Certificate{cert},
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: roots,
}, nil
}
func main() {
var verbose bool
flag.BoolVar(&verbose, "v", false, "Print access log")
flag.Parse()
tlsConfig, err := createTLSConfig(&connectionConfig{
CACertFile: "/etc/osbuild-composer/ca-crt.pem",
ServerKeyFile: "/etc/osbuild-composer/composer-key.pem",
ServerCertFile: "/etc/osbuild-composer/composer-crt.pem",
})
if err != nil {
log.Fatalf("TLS configuration cannot be created: %v", err.Error())
}
stateDir, ok := os.LookupEnv("STATE_DIRECTORY")
if !ok {
log.Fatal("STATE_DIRECTORY is not set. Is the service file missing StateDirectory=?")
}
cacheDirectory, ok := os.LookupEnv("CACHE_DIRECTORY")
if !ok {
log.Fatal("CACHE_DIRECTORY is not set. Is the service file missing CacheDirectory=?")
}
listeners, err := activation.ListenersWithNames()
if err != nil {
log.Fatalf("Could not get listening sockets: " + err.Error())
}
var cloudListener net.Listener
var jobListener net.Listener
if composerListeners, exists := listeners["osbuild-composer-cloud.socket"]; exists {
if len(composerListeners) != 2 {
log.Fatalf("Unexpected number of listening sockets (%d), expected 2", len(composerListeners))
}
cloudListener = composerListeners[0]
jobListener = tls.NewListener(composerListeners[1], tlsConfig)
} else {
log.Fatalf("osbuild-composer-cloud.socket doesn't exist")
}
var logger *log.Logger
if verbose {
logger = log.New(os.Stdout, "", 0)
}
queueDir := path.Join(stateDir, "jobs")
err = os.Mkdir(queueDir, 0700)
if err != nil && !os.IsExist(err) {
log.Fatalf("cannot create queue directory: %v", err)
}
distros, err := distro.NewRegistry(fedora31.New(), fedora32.New(), rhel8.New())
if err != nil {
log.Fatalf("Error loading distros: %v", err)
}
// construct job types of the form osbuild:{arch} for all arches
jobTypes := []string{"osbuild"}
jobTypesMap := map[string]bool{}
for _, name := range distros.List() {
d := distros.GetDistro(name)
for _, arch := range d.ListArches() {
jt := "osbuild:" + arch
if !jobTypesMap[jt] {
jobTypesMap[jt] = true
jobTypes = append(jobTypes, jt)
}
}
}
jobs, err := fsjobqueue.New(queueDir, jobTypes)
if err != nil {
log.Fatalf("cannot create jobqueue: %v", err)
}
rpm := rpmmd.NewRPMMD(path.Join(cacheDirectory, "rpmmd"), "/usr/libexec/osbuild-composer/dnf-json")
workerServer := worker.NewServer(logger, jobs, "")
cloudServer := cloudapi.NewServer(workerServer, rpm, distros)
go func() {
err := workerServer.Serve(jobListener)
if err != nil {
panic(err)
}
}()
err = cloudServer.Serve(cloudListener)
if err != nil {
panic(err)
}
}

34
cmd/osbuild-composer/composer.go
View file

@ -8,9 +8,11 @@ import (
"io/ioutil"
"log"
"net"
"net/http"
"os"
"path"
"github.com/osbuild/osbuild-composer/internal/cloudapi"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/jobqueue/fsjobqueue"
"github.com/osbuild/osbuild-composer/internal/kojiapi"
@ -38,9 +40,10 @@ type Composer struct {
workers *worker.Server
weldr *weldr.API
api *cloudapi.Server
koji *kojiapi.Server
weldrListener, localWorkerListener, workerListener, kojiListener net.Listener
weldrListener, localWorkerListener, workerListener, apiListener net.Listener
}
func NewComposer(config *ComposerConfigFile, stateDir, cacheDir string, logger *log.Logger) (*Composer, error) {
@ -127,7 +130,9 @@ func (c *Composer) InitWeldr(repoPaths []string, weldrListener, localWorkerListe
return nil
}
func (c *Composer) InitKoji(cert, key string, l net.Listener) error {
func (c *Composer) InitAPI(cert, key string, l net.Listener) error {
c.api = cloudapi.NewServer(c.workers, c.rpm, c.distros)
servers := make(map[string]koji.GSSAPICredentials)
for name, creds := range c.config.Koji.Servers {
if creds.Kerberos != nil {
@ -137,7 +142,6 @@ func (c *Composer) InitKoji(cert, key string, l net.Listener) error {
}
}
}
c.koji = kojiapi.NewServer(c.logger, c.workers, c.rpm, c.distros, servers)
tlsConfig, err := createTLSConfig(&connectionConfig{
@ -147,10 +151,10 @@ func (c *Composer) InitKoji(cert, key string, l net.Listener) error {
AllowedDomains: c.config.Koji.AllowedDomains,
})
if err != nil {
return fmt.Errorf("Error creating TLS configuration for Koji API: %v", err)
return fmt.Errorf("Error creating TLS configuration: %v", err)
}
c.kojiListener = tls.NewListener(l, tlsConfig)
c.apiListener = tls.NewListener(l, tlsConfig)
return nil
}
@ -197,9 +201,25 @@ func (c *Composer) Start() error {
}()
}
if c.kojiListener != nil {
if c.apiListener != nil {
go func() {
err := c.koji.Serve(c.kojiListener)
const apiRoute = "/api/composer/v1"
const kojiRoute = "/api/composer-koji/v1"
mux := http.NewServeMux()
// Add a "/" here, because http.ServeMux expects the
// trailing slash for rooted subtrees, whereas the
// handler functions don't.
mux.Handle(apiRoute+"/", c.api.Handler(apiRoute))
mux.Handle(kojiRoute+"/", c.koji.Handler(kojiRoute))
s := &http.Server{
ErrorLog: c.logger,
Handler: mux,
}
err := s.Serve(c.apiListener)
if err != nil {
panic(err)
}

2
cmd/osbuild-composer/main.go
View file

@ -82,7 +82,7 @@ func main() {
log.Fatal("The osbuild-composer-api.socket unit is misconfigured. It should contain only one socket.")
}
err = composer.InitKoji(ServerCertFile, ServerKeyFile, l[0])
err = composer.InitAPI(ServerCertFile, ServerKeyFile, l[0])
if err != nil {
log.Fatalf("Error initializing koji API: %v", err)
}