From 74eb3860df1b02635170ad8f0efd80203f2cd23b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Budai?= Date: Tue, 12 Jul 2022 10:02:52 +0200 Subject: [PATCH] internal: remove kojiapi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We no longer use it, let's remove it. If you are wondering what to use instead, use Cloud API. It supports everything that Koji API supported and more. Signed-off-by: Ondřej Budai --- .gitlab-ci.yml | 7 - cmd/osbuild-auth-tests/main_test.go | 51 +- cmd/osbuild-composer/composer.go | 10 +- internal/cloudapi/v2/v2_koji_test.go | 1 - internal/kojiapi/api/api.gen.go | 242 ---------- internal/kojiapi/api/api.go | 3 - internal/kojiapi/api/openapi.yml | 290 ----------- internal/kojiapi/server.go | 453 ------------------ internal/kojiapi/server_internal_test.go | 24 - internal/kojiapi/server_test.go | 582 ----------------------- 10 files changed, 4 insertions(+), 1659 deletions(-) delete mode 100644 internal/kojiapi/api/api.gen.go delete mode 100644 internal/kojiapi/api/api.go delete mode 100644 internal/kojiapi/api/openapi.yml delete mode 100644 internal/kojiapi/server.go delete mode 100644 internal/kojiapi/server_internal_test.go delete mode 100644 internal/kojiapi/server_test.go diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e8031ccb6..833134319 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -432,13 +432,6 @@ koji.sh (cloudapi): variables: SCRIPT: koji.sh -# internal composer still uses kojiapi, so keep testing it for now -koji.sh (kojiapi): - extends: .integration - variables: - COMPOSER_API: "false" - SCRIPT: koji.sh - aws.sh: extends: .integration variables: diff --git a/cmd/osbuild-auth-tests/main_test.go b/cmd/osbuild-auth-tests/main_test.go index 97da2bddc..ca6ee3d1c 100644 --- a/cmd/osbuild-auth-tests/main_test.go +++ b/cmd/osbuild-auth-tests/main_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package main @@ -96,56 +97,6 @@ func TestWorkerAPIAuth(t *testing.T) { }) } -func TestKojiAPIAuth(t *testing.T) { - t.Run("certificate signed by a trusted CA", func(t *testing.T) { - cases := []struct { - caseDesc string - subj string - addext string - success bool - }{ - {"valid CN and SAN 1", "/CN=client.osbuild.org/emailAddress=osbuild@example.com", "subjectAltName=DNS:example.com,DNS:client.osbuild.org", true}, - {"valid CN and SAN 2", "/CN=localhost/emailAddress=osbuild@example.com", "subjectAltName=DNS:example.com,DNS:localhost", true}, - {"invalid CN and SAN", "/CN=example.com/emailAddress=osbuild@example.com", "subjectAltName=DNS:example.com", false}, - } - - authority := &ca{BaseDir: trustedCADir} - - for _, c := range cases { - t.Run(c.caseDesc, func(t *testing.T) { - ckp, err := authority.newCertificateKeyPair(c.subj, osbuildClientExt, c.addext) - require.NoError(t, err) - defer ckp.remove() - - testRoute(t, "https://localhost/api/composer-koji/v1/status", ckp, c.success) - }) - } - }) - - t.Run("certificate signed by an untrusted CA", func(t *testing.T) { - // generate a new CA - ca, err := newCA("/CN=osbuild.org") - require.NoError(t, err) - defer ca.remove() - - // create a new certificate and signed it with the new CA - ckp, err := ca.newCertificateKeyPair("/CN=localhost/emailAddress=osbuild@example.com", osbuildClientExt, "subjectAltName=DNS:localhost") - require.NoError(t, err) - defer ckp.remove() - - testRoute(t, "https://localhost/api/composer-koji/v1/status", ckp, false) - }) - - t.Run("self-signed certificate", func(t *testing.T) { - // generate a new self-signed certificate - ckp, err := newSelfSignedCertificateKeyPair("/CN=osbuild.org") - require.NoError(t, err) - defer ckp.remove() - - testRoute(t, "https://localhost/api/composer-koji/v1/status", ckp, false) - }) -} - func testRoute(t *testing.T, route string, ckp *certificateKeyPair, expectSuccess bool) { tlsConfig, err := createTLSConfig(&connectionConfig{ CACertFile: "/etc/osbuild-composer/ca-crt.pem", diff --git a/cmd/osbuild-composer/composer.go b/cmd/osbuild-composer/composer.go index b6f0df851..d0026ad0e 100644 --- a/cmd/osbuild-composer/composer.go +++ b/cmd/osbuild-composer/composer.go @@ -16,18 +16,18 @@ import ( "syscall" "time" - "github.com/osbuild/osbuild-composer/pkg/jobqueue" - "github.com/osbuild/osbuild-composer/pkg/jobqueue/dbjobqueue" "github.com/prometheus/client_golang/prometheus/promhttp" logrus "github.com/sirupsen/logrus" + "github.com/osbuild/osbuild-composer/pkg/jobqueue" + "github.com/osbuild/osbuild-composer/pkg/jobqueue/dbjobqueue" + "github.com/osbuild/osbuild-composer/internal/auth" "github.com/osbuild/osbuild-composer/internal/cloudapi" v2 "github.com/osbuild/osbuild-composer/internal/cloudapi/v2" "github.com/osbuild/osbuild-composer/internal/distroregistry" "github.com/osbuild/osbuild-composer/internal/dnfjson" "github.com/osbuild/osbuild-composer/internal/jobqueue/fsjobqueue" - "github.com/osbuild/osbuild-composer/internal/kojiapi" "github.com/osbuild/osbuild-composer/internal/weldr" "github.com/osbuild/osbuild-composer/internal/worker" ) @@ -44,7 +44,6 @@ type Composer struct { workers *worker.Server weldr *weldr.API api *cloudapi.Server - koji *kojiapi.Server weldrListener, localWorkerListener, workerListener, apiListener net.Listener } @@ -134,7 +133,6 @@ func (c *Composer) InitAPI(cert, key string, enableTLS bool, enableMTLS bool, en } c.api = cloudapi.NewServer(c.workers, c.distros, config) - c.koji = kojiapi.NewServer(c.logger, c.workers, c.solver, c.distros) if !enableTLS { c.apiListener = l @@ -265,7 +263,6 @@ func (c *Composer) Start() error { if c.apiListener != nil { const apiRouteV2 = "/api/image-builder-composer/v2" - const kojiRoute = "/api/composer-koji/v1" mux := http.NewServeMux() @@ -273,7 +270,6 @@ func (c *Composer) Start() error { // trailing slash for rooted subtrees, whereas the // handler functions don't. mux.Handle(apiRouteV2+"/", c.api.V2(apiRouteV2)) - mux.Handle(kojiRoute+"/", c.koji.Handler(kojiRoute)) // Metrics handler attached to api mux to avoid a // separate listener/socket diff --git a/internal/cloudapi/v2/v2_koji_test.go b/internal/cloudapi/v2/v2_koji_test.go index 8609d3f2e..6c6e5167b 100644 --- a/internal/cloudapi/v2/v2_koji_test.go +++ b/internal/cloudapi/v2/v2_koji_test.go @@ -13,7 +13,6 @@ import ( v2 "github.com/osbuild/osbuild-composer/internal/cloudapi/v2" "github.com/osbuild/osbuild-composer/internal/distro/test_distro" - "github.com/osbuild/osbuild-composer/internal/kojiapi/api" "github.com/osbuild/osbuild-composer/internal/osbuild" "github.com/osbuild/osbuild-composer/internal/target" "github.com/osbuild/osbuild-composer/internal/test" diff --git a/internal/kojiapi/api/api.gen.go b/internal/kojiapi/api/api.gen.go deleted file mode 100644 index 79979fdc1..000000000 --- a/internal/kojiapi/api/api.gen.go +++ /dev/null @@ -1,242 +0,0 @@ -// Package api provides primitives to interact with the openapi HTTP API. -// -// Code generated by github.com/deepmap/oapi-codegen version v1.8.2 DO NOT EDIT. -package api - -import ( - "fmt" - "net/http" - - "github.com/deepmap/oapi-codegen/pkg/runtime" - "github.com/labstack/echo/v4" -) - -// Defines values for ComposeStatusValue. -const ( - ComposeStatusValueFailure ComposeStatusValue = "failure" - - ComposeStatusValuePending ComposeStatusValue = "pending" - - ComposeStatusValueRegistering ComposeStatusValue = "registering" - - ComposeStatusValueSuccess ComposeStatusValue = "success" -) - -// Defines values for ImageStatusValue. -const ( - ImageStatusValueBuilding ImageStatusValue = "building" - - ImageStatusValueFailure ImageStatusValue = "failure" - - ImageStatusValuePending ImageStatusValue = "pending" - - ImageStatusValueSuccess ImageStatusValue = "success" - - ImageStatusValueUploading ImageStatusValue = "uploading" -) - -// Defines values for StatusStatus. -const ( - StatusStatusOK StatusStatus = "OK" -) - -// ComposeLogs defines model for ComposeLogs. -type ComposeLogs struct { - ImageLogs []interface{} `json:"image_logs"` - KojiImportLogs interface{} `json:"koji_import_logs"` - KojiInitLogs interface{} `json:"koji_init_logs"` -} - -// ComposeRequest defines model for ComposeRequest. -type ComposeRequest struct { - Distribution string `json:"distribution"` - ImageRequests []ImageRequest `json:"image_requests"` - Koji Koji `json:"koji"` - Name string `json:"name"` - Release string `json:"release"` - Version string `json:"version"` -} - -// ComposeResponse defines model for ComposeResponse. -type ComposeResponse struct { - Id string `json:"id"` - KojiBuildId int `json:"koji_build_id"` -} - -// ComposeStatus defines model for ComposeStatus. -type ComposeStatus struct { - ImageStatuses []ImageStatus `json:"image_statuses"` - KojiBuildId *int `json:"koji_build_id,omitempty"` - KojiTaskId int `json:"koji_task_id"` - Status ComposeStatusValue `json:"status"` -} - -// ComposeStatusValue defines model for ComposeStatusValue. -type ComposeStatusValue string - -// ImageRequest defines model for ImageRequest. -type ImageRequest struct { - Architecture string `json:"architecture"` - ImageType string `json:"image_type"` - Repositories []Repository `json:"repositories"` -} - -// ImageStatus defines model for ImageStatus. -type ImageStatus struct { - Status ImageStatusValue `json:"status"` -} - -// ImageStatusValue defines model for ImageStatusValue. -type ImageStatusValue string - -// Koji defines model for Koji. -type Koji struct { - Server string `json:"server"` - TaskId int `json:"task_id"` -} - -// Repository defines model for Repository. -type Repository struct { - Baseurl string `json:"baseurl"` - Gpgkey *string `json:"gpgkey,omitempty"` -} - -// Status defines model for Status. -type Status struct { - Status StatusStatus `json:"status"` -} - -// StatusStatus defines model for Status.Status. -type StatusStatus string - -// PostComposeJSONBody defines parameters for PostCompose. -type PostComposeJSONBody ComposeRequest - -// PostComposeJSONRequestBody defines body for PostCompose for application/json ContentType. -type PostComposeJSONRequestBody PostComposeJSONBody - -// ServerInterface represents all server handlers. -type ServerInterface interface { - // Create compose - // (POST /compose) - PostCompose(ctx echo.Context) error - // The status of a compose - // (GET /compose/{id}) - GetComposeId(ctx echo.Context, id string) error - // Get logs for a compose. - // (GET /compose/{id}/logs) - GetComposeIdLogs(ctx echo.Context, id string) error - // Get the manifests for a compose. - // (GET /compose/{id}/manifests) - GetComposeIdManifests(ctx echo.Context, id string) error - // status - // (GET /status) - GetStatus(ctx echo.Context) error -} - -// ServerInterfaceWrapper converts echo contexts to parameters. -type ServerInterfaceWrapper struct { - Handler ServerInterface -} - -// PostCompose converts echo context to params. -func (w *ServerInterfaceWrapper) PostCompose(ctx echo.Context) error { - var err error - - // Invoke the callback with all the unmarshalled arguments - err = w.Handler.PostCompose(ctx) - return err -} - -// GetComposeId converts echo context to params. -func (w *ServerInterfaceWrapper) GetComposeId(ctx echo.Context) error { - var err error - // ------------- Path parameter "id" ------------- - var id string - - err = runtime.BindStyledParameterWithLocation("simple", false, "id", runtime.ParamLocationPath, ctx.Param("id"), &id) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err)) - } - - // Invoke the callback with all the unmarshalled arguments - err = w.Handler.GetComposeId(ctx, id) - return err -} - -// GetComposeIdLogs converts echo context to params. -func (w *ServerInterfaceWrapper) GetComposeIdLogs(ctx echo.Context) error { - var err error - // ------------- Path parameter "id" ------------- - var id string - - err = runtime.BindStyledParameterWithLocation("simple", false, "id", runtime.ParamLocationPath, ctx.Param("id"), &id) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err)) - } - - // Invoke the callback with all the unmarshalled arguments - err = w.Handler.GetComposeIdLogs(ctx, id) - return err -} - -// GetComposeIdManifests converts echo context to params. -func (w *ServerInterfaceWrapper) GetComposeIdManifests(ctx echo.Context) error { - var err error - // ------------- Path parameter "id" ------------- - var id string - - err = runtime.BindStyledParameterWithLocation("simple", false, "id", runtime.ParamLocationPath, ctx.Param("id"), &id) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err)) - } - - // Invoke the callback with all the unmarshalled arguments - err = w.Handler.GetComposeIdManifests(ctx, id) - return err -} - -// GetStatus converts echo context to params. -func (w *ServerInterfaceWrapper) GetStatus(ctx echo.Context) error { - var err error - - // Invoke the callback with all the unmarshalled arguments - err = w.Handler.GetStatus(ctx) - return err -} - -// This is a simple interface which specifies echo.Route addition functions which -// are present on both echo.Echo and echo.Group, since we want to allow using -// either of them for path registration -type EchoRouter interface { - CONNECT(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route - DELETE(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route - GET(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route - HEAD(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route - OPTIONS(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route - PATCH(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route - POST(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route - PUT(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route - TRACE(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route -} - -// RegisterHandlers adds each server route to the EchoRouter. -func RegisterHandlers(router EchoRouter, si ServerInterface) { - RegisterHandlersWithBaseURL(router, si, "") -} - -// Registers handlers, and prepends BaseURL to the paths, so that the paths -// can be served under a prefix. -func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL string) { - - wrapper := ServerInterfaceWrapper{ - Handler: si, - } - - router.POST(baseURL+"/compose", wrapper.PostCompose) - router.GET(baseURL+"/compose/:id", wrapper.GetComposeId) - router.GET(baseURL+"/compose/:id/logs", wrapper.GetComposeIdLogs) - router.GET(baseURL+"/compose/:id/manifests", wrapper.GetComposeIdManifests) - router.GET(baseURL+"/status", wrapper.GetStatus) - -} diff --git a/internal/kojiapi/api/api.go b/internal/kojiapi/api/api.go deleted file mode 100644 index b5f2978c4..000000000 --- a/internal/kojiapi/api/api.go +++ /dev/null @@ -1,3 +0,0 @@ -//go:generate go run github.com/deepmap/oapi-codegen/cmd/oapi-codegen -package=api -generate types,server -o api.gen.go openapi.yml - -package api diff --git a/internal/kojiapi/api/openapi.yml b/internal/kojiapi/api/openapi.yml deleted file mode 100644 index 2a3ee9c93..000000000 --- a/internal/kojiapi/api/openapi.yml +++ /dev/null @@ -1,290 +0,0 @@ -openapi: 3.0.1 -info: - title: OSBuild Composer - Koji - version: '1' - description: Service to build and push images to Koji. - license: - name: Apache 2.0 - url: 'https://www.apache.org/licenses/LICENSE-2.0.html' -servers: - - url: /api/composer-koji/v1 -paths: - /status: - get: - summary: status - tags: [ ] - responses: - '200': - description: OK - headers: { } - content: - application/json: - schema: - $ref: '#/components/schemas/Status' - operationId: GetStatus - description: Simple status handler to check whether the service is up. - '/compose/{id}': - get: - summary: The status of a compose - parameters: - - in: path - name: id - schema: - type: string - format: uuid - example: 123e4567-e89b-12d3-a456-426655440000 - required: true - description: ID of compose status to get - description: 'Get the status of a running or finished compose. This includes whether or not it succeeded, and also meta information about the result.' - responses: - '200': - description: Compose status - content: - application/json: - schema: - $ref: '#/components/schemas/ComposeStatus' - '400': - description: Invalid compose id - content: - text/plain: - schema: - type: string - '404': - description: Unknown compose id - content: - text/plain: - schema: - type: string - '/compose/{id}/logs': - get: - summary: Get logs for a compose. - parameters: - - in: path - name: id - schema: - type: string - format: uuid - example: 123e4567-e89b-12d3-a456-426655440000 - required: true - description: ID of compose status to get - description: 'Get the status of a running or finished compose. This includes whether or not it succeeded, and also meta information about the result.' - responses: - '200': - description: The logs for the given compose, in no particular format (though valid JSON). - content: - application/json: - schema: - $ref: '#/components/schemas/ComposeLogs' - '400': - description: Invalid compose id - content: - text/plain: - schema: - type: string - '404': - description: Unknown compose id - content: - text/plain: - schema: - type: string - '/compose/{id}/manifests': - get: - summary: Get the manifests for a compose. - parameters: - - in: path - name: id - schema: - type: string - format: uuid - example: 123e4567-e89b-12d3-a456-426655440000 - required: true - description: ID of compose status to get - description: 'Get the manifests of a running or finished compose. Returns one manifest for each image in the request. Each manifest conforms to the format defined at https://www.osbuild.org/man/osbuild-manifest.5' - responses: - '200': - description: The manifest for the given compose. - content: - application/json: - '400': - description: Invalid compose id - content: - text/plain: - schema: - type: string - '404': - description: Unknown compose id - content: - text/plain: - schema: - type: string - /compose: - post: - summary: Create compose - description: 'Create a new compose, potentially consisting of several images and upload each to koji.' - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/ComposeRequest' - responses: - '201': - description: Compose has started - content: - application/json: - schema: - $ref: '#/components/schemas/ComposeResponse' - '400': - description: Invalid compose request - content: - text/plain: - schema: - type: string - '415': - description: The content type is not supported - content: - text/plain: - schema: - type: string -components: - schemas: - Status: - required: - - status - properties: - status: - type: string - enum: - - OK - ComposeStatus: - required: - - status - - image_statuses - - koji_task_id - properties: - status: - $ref: '#/components/schemas/ComposeStatusValue' - image_statuses: - type: array - items: - $ref: '#/components/schemas/ImageStatus' - koji_task_id: - type: integer - example: 203143 - koji_build_id: - type: integer - example: 42 - ComposeStatusValue: - type: string - enum: - - success - - failure - - pending - - registering - example: success - ComposeLogs: - required: - - koji_init_logs - - koji_import_logs - - image_logs - properties: - koji_init_logs: {} - koji_import_logs: {} - image_logs: - type: array - ImageStatus: - required: - - status - properties: - status: - $ref: '#/components/schemas/ImageStatusValue' - ImageStatusValue: - type: string - enum: - - success - - failure - - pending - - building - - uploading - example: success - ComposeRequest: - type: object - required: - - name - - version - - release - - distribution - - image_requests - - koji - properties: - name: - type: string - example: Fedora-Cloud-Base - version: - type: string - example: '31' - release: - type: string - example: '20200907.0' - distribution: - type: string - example: fedora-32 - image_requests: - type: array - items: - $ref: '#/components/schemas/ImageRequest' - koji: - $ref: '#/components/schemas/Koji' - ImageRequest: - required: - - architecture - - image_type - - repositories - properties: - architecture: - type: string - example: x86_64 - image_type: - type: string - example: ami - repositories: - type: array - items: - $ref: '#/components/schemas/Repository' - Repository: - type: object - required: - - baseurl - properties: - baseurl: - type: string - format: url - example: 'https://cdn.redhat.com/content/dist/rhel8/8/x86_64/baseos/os/' - gpgkey: - type: string - example: "-----BEGIN PGP PUBLIC KEY BLOCK-----\\n\\nmQINBErgSTsBEACh2A4b0O9t+vzC9VrVtL1AKvUWi9OPCjkvR7Xd8DtJxeeMZ5eF\\n0HtzIG58qDRybwUe89FZprB1ffuUKzdE+HcL3FbNWSSOXVjZIersdXyH3NvnLLLF\\n0DNRB2ix3bXG9Rh/RXpFsNxDp2CEMdUvbYCzE79K1EnUTVh1L0Of023FtPSZXX0c\\nu7Pb5DI5lX5YeoXO6RoodrIGYJsVBQWnrWw4xNTconUfNPk0EGZtEnzvH2zyPoJh\\nXGF+Ncu9XwbalnYde10OCvSWAZ5zTCpoLMTvQjWpbCdWXJzCm6G+/hx9upke546H\\n5IjtYm4dTIVTnc3wvDiODgBKRzOl9rEOCIgOuGtDxRxcQkjrC+xvg5Vkqn7vBUyW\\n9pHedOU+PoF3DGOM+dqv+eNKBvh9YF9ugFAQBkcG7viZgvGEMGGUpzNgN7XnS1gj\\n/DPo9mZESOYnKceve2tIC87p2hqjrxOHuI7fkZYeNIcAoa83rBltFXaBDYhWAKS1\\nPcXS1/7JzP0ky7d0L6Xbu/If5kqWQpKwUInXtySRkuraVfuK3Bpa+X1XecWi24JY\\nHVtlNX025xx1ewVzGNCTlWn1skQN2OOoQTV4C8/qFpTW6DTWYurd4+fE0OJFJZQF\\nbuhfXYwmRlVOgN5i77NTIJZJQfYFj38c/Iv5vZBPokO6mffrOTv3MHWVgQARAQAB\\ntDNSZWQgSGF0LCBJbmMuIChyZWxlYXNlIGtleSAyKSA8c2VjdXJpdHlAcmVkaGF0\\nLmNvbT6JAjYEEwECACAFAkrgSTsCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAK\\nCRAZni+R/UMdUWzpD/9s5SFR/ZF3yjY5VLUFLMXIKUztNN3oc45fyLdTI3+UClKC\\n2tEruzYjqNHhqAEXa2sN1fMrsuKec61Ll2NfvJjkLKDvgVIh7kM7aslNYVOP6BTf\\nC/JJ7/ufz3UZmyViH/WDl+AYdgk3JqCIO5w5ryrC9IyBzYv2m0HqYbWfphY3uHw5\\nun3ndLJcu8+BGP5F+ONQEGl+DRH58Il9Jp3HwbRa7dvkPgEhfFR+1hI+Btta2C7E\\n0/2NKzCxZw7Lx3PBRcU92YKyaEihfy/aQKZCAuyfKiMvsmzs+4poIX7I9NQCJpyE\\nIGfINoZ7VxqHwRn/d5mw2MZTJjbzSf+Um9YJyA0iEEyD6qjriWQRbuxpQXmlAJbh\\n8okZ4gbVFv1F8MzK+4R8VvWJ0XxgtikSo72fHjwha7MAjqFnOq6eo6fEC/75g3NL\\nGht5VdpGuHk0vbdENHMC8wS99e5qXGNDued3hlTavDMlEAHl34q2H9nakTGRF5Ki\\nJUfNh3DVRGhg8cMIti21njiRh7gyFI2OccATY7bBSr79JhuNwelHuxLrCFpY7V25\\nOFktl15jZJaMxuQBqYdBgSay2G0U6D1+7VsWufpzd/Abx1/c3oi9ZaJvW22kAggq\\ndzdA27UUYjWvx42w9menJwh/0jeQcTecIUd0d0rFcw/c1pvgMMl/Q73yzKgKYw==\\n=zbHE\\n-----END PGP PUBLIC KEY BLOCK-----\\n" - Koji: - type: object - required: - - server - - task_id - properties: - server: - type: string - format: url - example: 'https://koji.fedoraproject.org/kojihub' - task_id: - type: integer - example: 42 - ComposeResponse: - required: - - id - - koji_build_id - properties: - id: - type: string - format: uuid - example: 123e4567-e89b-12d3-a456-426655440000 - koji_build_id: - type: integer - example: 42 diff --git a/internal/kojiapi/server.go b/internal/kojiapi/server.go deleted file mode 100644 index 20ebbd499..000000000 --- a/internal/kojiapi/server.go +++ /dev/null @@ -1,453 +0,0 @@ -// Package kojiapi provides a REST API to build and push images to Koji -package kojiapi - -import ( - "crypto/rand" - "encoding/json" - "fmt" - "log" - "math" - "math/big" - "net/http" - "strings" - "time" - - "github.com/google/uuid" - "github.com/labstack/echo/v4" - - "github.com/osbuild/osbuild-composer/internal/blueprint" - "github.com/osbuild/osbuild-composer/internal/distro" - "github.com/osbuild/osbuild-composer/internal/distroregistry" - "github.com/osbuild/osbuild-composer/internal/dnfjson" - "github.com/osbuild/osbuild-composer/internal/kojiapi/api" - "github.com/osbuild/osbuild-composer/internal/rpmmd" - "github.com/osbuild/osbuild-composer/internal/worker" -) - -// Server represents the state of the koji Server -type Server struct { - logger *log.Logger - workers *worker.Server - solver *dnfjson.BaseSolver - distros *distroregistry.Registry -} - -// NewServer creates a new koji server -func NewServer(logger *log.Logger, workers *worker.Server, solver *dnfjson.BaseSolver, distros *distroregistry.Registry) *Server { - s := &Server{ - logger: logger, - workers: workers, - solver: solver, - distros: distros, - } - - return s -} - -// Create an http.Handler() for this server, that provides the koji API at the -// given path. -func (s *Server) Handler(path string) http.Handler { - e := echo.New() - e.Binder = binder{} - e.StdLogger = s.logger - - // log errors returned from handlers - e.HTTPErrorHandler = func(err error, c echo.Context) { - log.Println(c.Path(), c.QueryParams().Encode(), err.Error()) - e.DefaultHTTPErrorHandler(err, c) - } - - api.RegisterHandlers(e.Group(path), &apiHandlers{s}) - - return e -} - -// apiHandlers implements api.ServerInterface - the http api route handlers -// generated from api/openapi.yml. This is a separate object, because these -// handlers should not be exposed on the `Server` object. -type apiHandlers struct { - server *Server -} - -// PostCompose handles a new /compose POST request -func (h *apiHandlers) PostCompose(ctx echo.Context) error { - var request api.ComposeRequest - err := ctx.Bind(&request) - if err != nil { - return err - } - - d := h.server.distros.GetDistro(request.Distribution) - if d == nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unsupported distribution: %s", request.Distribution)) - } - - type imageRequest struct { - manifest distro.Manifest - arch string - filename string - exports []string - pipelineNames *worker.PipelineNames - } - - imageRequests := make([]imageRequest, len(request.ImageRequests)) - kojiFilenames := make([]string, len(request.ImageRequests)) - kojiDirectory := "osbuild-composer-koji-" + uuid.New().String() - - // use the same seed for all images so we get the same IDs - bigSeed, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64)) - if err != nil { - panic("cannot generate a manifest seed: " + err.Error()) - } - manifestSeed := bigSeed.Int64() - - for i, ir := range request.ImageRequests { - arch, err := d.GetArch(ir.Architecture) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unsupported architecture '%s' for distribution '%s'", ir.Architecture, request.Distribution)) - } - imageType, err := arch.GetImageType(ir.ImageType) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unsupported image type '%s' for %s/%s", ir.ImageType, ir.Architecture, request.Distribution)) - } - repositories := make([]rpmmd.RepoConfig, len(ir.Repositories)) - for j, repo := range ir.Repositories { - repositories[j].BaseURL = repo.Baseurl - if repo.Gpgkey != nil { - repositories[j].GPGKey = *repo.Gpgkey - } - } - bp := &blueprint.Blueprint{} - err = bp.Initialize() - if err != nil { - panic("Could not initialize empty blueprint.") - } - - options := distro.ImageOptions{Size: imageType.Size(0)} - - solver := h.server.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), arch.Name()) - packageSets := imageType.PackageSets(*bp, options, repositories) - depsolvedSets := make(map[string][]rpmmd.PackageSpec, len(packageSets)) - - for name, pkgSet := range packageSets { - res, err := solver.Depsolve(pkgSet) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Failed to depsolve base packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err)) - } - depsolvedSets[name] = res - } - - manifest, err := imageType.Manifest(nil, options, repositories, depsolvedSets, manifestSeed) - if err != nil { - return echo.NewHTTPError(http.StatusBadGateway, fmt.Sprintf("Failed to get manifest for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err)) - } - - imageRequests[i].manifest = manifest - imageRequests[i].arch = arch.Name() - imageRequests[i].filename = imageType.Filename() - imageRequests[i].exports = imageType.Exports() - imageRequests[i].pipelineNames = &worker.PipelineNames{ - Build: imageType.BuildPipelines(), - Payload: imageType.PayloadPipelines(), - } - - kojiFilenames[i] = fmt.Sprintf( - "%s-%s-%s.%s%s", - request.Name, - request.Version, - request.Release, - ir.Architecture, - splitExtension(imageType.Filename()), - ) - } - - initID, err := h.server.workers.EnqueueKojiInit(&worker.KojiInitJob{ - Server: request.Koji.Server, - Name: request.Name, - Version: request.Version, - Release: request.Release, - }, "") - if err != nil { - // This is a programming error. - panic(err) - } - - var buildIDs []uuid.UUID - for i, ir := range imageRequests { - id, err := h.server.workers.EnqueueOSBuildKoji(ir.arch, &worker.OSBuildKojiJob{ - Manifest: ir.manifest, - ImageName: ir.filename, - Exports: ir.exports, - PipelineNames: ir.pipelineNames, - KojiServer: request.Koji.Server, - KojiDirectory: kojiDirectory, - KojiFilename: kojiFilenames[i], - }, initID, "") - if err != nil { - // This is a programming error. - panic(err) - } - buildIDs = append(buildIDs, id) - } - - id, err := h.server.workers.EnqueueKojiFinalize(&worker.KojiFinalizeJob{ - Server: request.Koji.Server, - Name: request.Name, - Version: request.Version, - Release: request.Release, - KojiFilenames: kojiFilenames, - KojiDirectory: kojiDirectory, - TaskID: uint64(request.Koji.TaskId), - StartTime: uint64(time.Now().Unix()), - }, initID, buildIDs, "") - if err != nil { - // This is a programming error. - panic(err) - } - - // TODO: remove - // For backwards compatibility we must only return once the - // build ID is known. This logic should live in the client, - // and `JobStatus()` should have a way to block until it - // changes. - var initResult worker.KojiInitJobResult - for { - status, _, err := h.server.workers.KojiInitJobStatus(initID, &initResult) - if err != nil { - panic(err) - } - if !status.Finished.IsZero() || status.Canceled { - break - } - time.Sleep(500 * time.Millisecond) - } - - if initResult.JobError != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Could not initialize build with koji: %v", initResult.JobError.Reason)) - } - - if err := h.server.solver.CleanCache(); err != nil { - // log and ignore - log.Printf("Error during rpm repo cache cleanup: %s", err.Error()) - } - - return ctx.JSON(http.StatusCreated, &api.ComposeResponse{ - Id: id.String(), - KojiBuildId: int(initResult.BuildID), - }) -} - -// splitExtension returns the extension of the given file. If there's -// a multipart extension (e.g. file.tar.gz), it returns all parts (e.g. -// .tar.gz). If there's no extension in the input, it returns an empty -// string. If the filename starts with dot, the part before the second dot -// is not considered as an extension. -func splitExtension(filename string) string { - filenameParts := strings.Split(filename, ".") - - if len(filenameParts) > 0 && filenameParts[0] == "" { - filenameParts = filenameParts[1:] - } - - if len(filenameParts) <= 1 { - return "" - } - - return "." + strings.Join(filenameParts[1:], ".") -} - -func composeStatusFromJobStatus(js *worker.JobStatus, initResult *worker.KojiInitJobResult, buildResults []worker.OSBuildKojiJobResult, result *worker.KojiFinalizeJobResult) api.ComposeStatusValue { - if js.Canceled { - return api.ComposeStatusValueFailure - } - - if js.Finished.IsZero() { - return api.ComposeStatusValuePending - } - - if initResult.JobError != nil { - return api.ComposeStatusValueFailure - } - - for _, buildResult := range buildResults { - if buildResult.OSBuildOutput == nil || !buildResult.OSBuildOutput.Success { - return api.ComposeStatusValueFailure - } - if buildResult.JobError != nil { - return api.ComposeStatusValueFailure - } - } - - if result.JobError != nil { - return api.ComposeStatusValueFailure - } - - return api.ComposeStatusValueSuccess -} - -func imageStatusFromJobStatus(js *worker.JobStatus, initResult *worker.KojiInitJobResult, buildResult *worker.OSBuildKojiJobResult) api.ImageStatusValue { - if js.Canceled { - return api.ImageStatusValueFailure - } - - if initResult.JobError != nil { - return api.ImageStatusValueFailure - } - - if js.Started.IsZero() { - return api.ImageStatusValuePending - } - - if js.Finished.IsZero() { - return api.ImageStatusValueBuilding - } - - if buildResult.OSBuildOutput != nil && buildResult.OSBuildOutput.Success && buildResult.JobError == nil { - return api.ImageStatusValueSuccess - } - - return "failure" -} - -// GetComposeId handles a /compose/{id} GET request -func (h *apiHandlers) GetComposeId(ctx echo.Context, idstr string) error { - id, err := uuid.Parse(idstr) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err)) - } - - var finalizeResult worker.KojiFinalizeJobResult - finalizeStatus, deps, err := h.server.workers.KojiFinalizeJobStatus(id, &finalizeResult) - if err != nil { - return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Job %s not found: %s", idstr, err)) - } - - var initResult worker.KojiInitJobResult - _, _, err = h.server.workers.KojiInitJobStatus(deps[0], &initResult) - if err != nil { - // this is a programming error - panic(err) - } - - var buildResults []worker.OSBuildKojiJobResult - var imageStatuses []api.ImageStatus - for i := 1; i < len(deps); i++ { - var buildResult worker.OSBuildKojiJobResult - jobStatus, _, err := h.server.workers.OSBuildKojiJobStatus(deps[i], &buildResult) - if err != nil { - // this is a programming error - panic(err) - } - buildResults = append(buildResults, buildResult) - imageStatuses = append(imageStatuses, api.ImageStatus{ - Status: imageStatusFromJobStatus(jobStatus, &initResult, &buildResult), - }) - } - - response := api.ComposeStatus{ - Status: composeStatusFromJobStatus(finalizeStatus, &initResult, buildResults, &finalizeResult), - ImageStatuses: imageStatuses, - } - buildID := int(initResult.BuildID) - if buildID != 0 { - response.KojiBuildId = &buildID - } - return ctx.JSON(http.StatusOK, response) -} - -// GetStatus handles a /status GET request -func (h *apiHandlers) GetStatus(ctx echo.Context) error { - return ctx.JSON(http.StatusOK, &api.Status{ - Status: "OK", - }) -} - -// Get logs for a compose -func (h *apiHandlers) GetComposeIdLogs(ctx echo.Context, idstr string) error { - id, err := uuid.Parse(idstr) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err)) - } - - var finalizeResult worker.KojiFinalizeJobResult - _, deps, err := h.server.workers.KojiFinalizeJobStatus(id, &finalizeResult) - if err != nil { - return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Job %s not found: %s", idstr, err)) - } - - var initResult worker.KojiInitJobResult - _, _, err = h.server.workers.KojiInitJobStatus(deps[0], &initResult) - if err != nil { - // This is a programming error. - panic(err) - } - - var buildResults []interface{} - for i := 1; i < len(deps); i++ { - var buildResult worker.OSBuildKojiJobResult - _, _, err = h.server.workers.OSBuildKojiJobStatus(deps[i], &buildResult) - if err != nil { - // This is a programming error. - panic(err) - } - buildResults = append(buildResults, buildResult) - } - - // Return the OSBuildJobResults as-is for now. The contents of ImageLogs - // is not part of the API. It's meant for a human to be able to access - // the logs, which just happen to be in JSON. - response := api.ComposeLogs{ - KojiInitLogs: initResult, - KojiImportLogs: finalizeResult, - ImageLogs: buildResults, - } - - return ctx.JSON(http.StatusOK, response) -} - -// GetComposeIdManifests returns the Manifests for a given Compose (one for each image). -func (h *apiHandlers) GetComposeIdManifests(ctx echo.Context, idstr string) error { - id, err := uuid.Parse(idstr) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err)) - } - - var finalizeResult worker.KojiFinalizeJobResult - _, deps, err := h.server.workers.KojiFinalizeJobStatus(id, &finalizeResult) - if err != nil { - return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Job %s not found: %s", idstr, err)) - } - - var manifests []distro.Manifest - for _, id := range deps[1:] { - var buildJob worker.OSBuildKojiJob - err = h.server.workers.OSBuildKojiJob(id, &buildJob) - if err != nil { - return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Job %s could not be deserialized: %s", idstr, err)) - } - manifests = append(manifests, buildJob.Manifest) - } - - return ctx.JSON(http.StatusOK, manifests) -} - -// A simple echo.Binder(), which only accepts application/json, but is more -// strict than echo's DefaultBinder. It does not handle binding query -// parameters either. -type binder struct{} - -func (b binder) Bind(i interface{}, ctx echo.Context) error { - request := ctx.Request() - - contentType := request.Header["Content-Type"] - if len(contentType) != 1 || contentType[0] != "application/json" { - return echo.NewHTTPError(http.StatusUnsupportedMediaType, "request must be json-encoded") - } - - err := json.NewDecoder(request.Body).Decode(i) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("cannot parse request body: %v", err)) - } - - return nil -} diff --git a/internal/kojiapi/server_internal_test.go b/internal/kojiapi/server_internal_test.go deleted file mode 100644 index 1dc5f5aa7..000000000 --- a/internal/kojiapi/server_internal_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package kojiapi - -import ( - "github.com/stretchr/testify/require" - "testing" -) - -func TestSplitExtension(t *testing.T) { - tests := []struct { - filename string - extension string - }{ - {filename: "image.qcow2", extension: ".qcow2"}, - {filename: "image.tar.gz", extension: ".tar.gz"}, - {filename: "", extension: ""}, - {filename: ".htaccess", extension: ""}, - {filename: ".weirdfile.txt", extension: ".txt"}, - } - for _, tt := range tests { - t.Run(tt.filename, func(t *testing.T) { - require.Equal(t, tt.extension, splitExtension(tt.filename)) - }) - } -} diff --git a/internal/kojiapi/server_test.go b/internal/kojiapi/server_test.go deleted file mode 100644 index bcf1b03aa..000000000 --- a/internal/kojiapi/server_test.go +++ /dev/null @@ -1,582 +0,0 @@ -package kojiapi_test - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "os" - "os/exec" - "path/filepath" - "sync" - "testing" - "time" - - "github.com/google/uuid" - "github.com/stretchr/testify/require" - - "github.com/osbuild/osbuild-composer/internal/distro/test_distro" - "github.com/osbuild/osbuild-composer/internal/dnfjson" - "github.com/osbuild/osbuild-composer/internal/kojiapi" - "github.com/osbuild/osbuild-composer/internal/kojiapi/api" - distro_mock "github.com/osbuild/osbuild-composer/internal/mocks/distro" - rpmmd_mock "github.com/osbuild/osbuild-composer/internal/mocks/rpmmd" - "github.com/osbuild/osbuild-composer/internal/osbuild" - "github.com/osbuild/osbuild-composer/internal/test" - "github.com/osbuild/osbuild-composer/internal/worker" - "github.com/osbuild/osbuild-composer/internal/worker/clienterrors" -) - -var dnfjsonPath string - -func setupDNFJSON() { - // compile the mock-dnf-json binary to speed up tests - tmpdir, err := os.MkdirTemp("", "") - if err != nil { - panic(err) - } - dnfjsonPath = filepath.Join(tmpdir, "mock-dnf-json") - cmd := exec.Command("go", "build", "-o", dnfjsonPath, "../../cmd/mock-dnf-json") - if err := cmd.Run(); err != nil { - panic(err) - } -} - -func newTestKojiServer(t *testing.T, dir string) (*kojiapi.Server, *worker.Server) { - - // create tempdir subdirectory for store - dbpath, err := os.MkdirTemp(dir, "") - if err != nil { - panic(err) - } - rpm_fixture := rpmmd_mock.BaseFixture(dbpath) - - distros, err := distro_mock.NewDefaultRegistry() - require.NoError(t, err) - require.NotNil(t, distros) - - solver := dnfjson.NewBaseSolver("") // test solver doesn't need a cache dir - // create tempdir subdirectory for solver response file - dspath, err := os.MkdirTemp(dir, "") - if err != nil { - panic(err) - } - - respfile := rpm_fixture.ResponseGenerator(dspath) - solver.SetDNFJSONPath(dnfjsonPath, respfile) - kojiServer := kojiapi.NewServer(nil, rpm_fixture.Workers, solver, distros) - require.NotNil(t, kojiServer) - - return kojiServer, rpm_fixture.Workers -} - -func TestStatus(t *testing.T) { - kojiServer, _ := newTestKojiServer(t, t.TempDir()) - handler := kojiServer.Handler("/api/composer-koji/v1") - test.TestRoute(t, handler, false, "GET", "/api/composer-koji/v1/status", ``, http.StatusOK, `{"status":"OK"}`, "message") -} - -type jobResult struct { - Result interface{} `json:"result"` -} - -func TestCompose(t *testing.T) { - kojiServer, workerServer := newTestKojiServer(t, t.TempDir()) - handler := kojiServer.Handler("/api/composer-koji/v1") - workerHandler := workerServer.Handler() - - type kojiCase struct { - initResult worker.KojiInitJobResult - buildResult worker.OSBuildKojiJobResult - finalizeResult worker.KojiFinalizeJobResult - composeReplyCode int - composeReply string - composeStatus string - } - - var cases = []kojiCase{ - { - initResult: worker.KojiInitJobResult{ - BuildID: 42, - Token: `"foobar"`, - }, - buildResult: worker.OSBuildKojiJobResult{ - Arch: test_distro.TestArchName, - HostOS: test_distro.TestDistroName, - ImageHash: "browns", - ImageSize: 42, - OSBuildOutput: &osbuild.Result{ - Success: true, - }, - }, - composeReplyCode: http.StatusCreated, - composeReply: `{"koji_build_id":42}`, - composeStatus: `{ - "image_statuses": [ - { - "status": "success" - }, - { - "status": "success" - } - ], - "koji_build_id": 42, - "koji_task_id": 0, - "status": "success" - }`, - }, - { - initResult: worker.KojiInitJobResult{ - KojiError: "failure", - }, - buildResult: worker.OSBuildKojiJobResult{ - Arch: test_distro.TestArchName, - HostOS: test_distro.TestDistroName, - ImageHash: "browns", - ImageSize: 42, - OSBuildOutput: &osbuild.Result{ - Success: true, - }, - }, - composeReplyCode: http.StatusBadRequest, - composeReply: `{"message":"Could not initialize build with koji: failure"}`, - composeStatus: `{ - "image_statuses": [ - { - "status": "failure" - }, - { - "status": "failure" - } - ], - "koji_task_id": 0, - "status": "failure" - }`, - }, - { - initResult: worker.KojiInitJobResult{ - JobResult: worker.JobResult{ - JobError: clienterrors.WorkerClientError(clienterrors.ErrorKojiInit, "Koji init error"), - }, - }, - buildResult: worker.OSBuildKojiJobResult{ - Arch: test_distro.TestArchName, - HostOS: test_distro.TestDistroName, - ImageHash: "browns", - ImageSize: 42, - OSBuildOutput: &osbuild.Result{ - Success: true, - }, - }, - composeReplyCode: http.StatusBadRequest, - composeReply: `{"message":"Could not initialize build with koji: Koji init error"}`, - composeStatus: `{ - "image_statuses": [ - { - "status": "failure" - }, - { - "status": "failure" - } - ], - "koji_task_id": 0, - "status": "failure" - }`, - }, - { - initResult: worker.KojiInitJobResult{ - BuildID: 42, - Token: `"foobar"`, - }, - buildResult: worker.OSBuildKojiJobResult{ - Arch: test_distro.TestArchName, - HostOS: test_distro.TestDistroName, - ImageHash: "browns", - ImageSize: 42, - OSBuildOutput: &osbuild.Result{ - Success: false, - }, - }, - composeReplyCode: http.StatusCreated, - composeReply: `{"koji_build_id":42}`, - composeStatus: `{ - "image_statuses": [ - { - "status": "failure" - }, - { - "status": "success" - } - ], - "koji_build_id": 42, - "koji_task_id": 0, - "status": "failure" - }`, - }, - { - initResult: worker.KojiInitJobResult{ - BuildID: 42, - Token: `"foobar"`, - }, - buildResult: worker.OSBuildKojiJobResult{ - Arch: test_distro.TestArchName, - HostOS: test_distro.TestDistroName, - ImageHash: "browns", - ImageSize: 42, - OSBuildOutput: &osbuild.Result{ - Success: true, - }, - KojiError: "failure", - }, - composeReplyCode: http.StatusCreated, - composeReply: `{"koji_build_id":42}`, - composeStatus: `{ - "image_statuses": [ - { - "status": "failure" - }, - { - "status": "success" - } - ], - "koji_build_id": 42, - "koji_task_id": 0, - "status": "failure" - }`, - }, - { - initResult: worker.KojiInitJobResult{ - BuildID: 42, - Token: `"foobar"`, - }, - buildResult: worker.OSBuildKojiJobResult{ - Arch: test_distro.TestArchName, - HostOS: test_distro.TestDistroName, - ImageHash: "browns", - ImageSize: 42, - OSBuildOutput: &osbuild.Result{ - Success: true, - }, - JobResult: worker.JobResult{ - JobError: clienterrors.WorkerClientError(clienterrors.ErrorBuildJob, "Koji build error"), - }, - }, - composeReplyCode: http.StatusCreated, - composeReply: `{"koji_build_id":42}`, - composeStatus: `{ - "image_statuses": [ - { - "status": "failure" - }, - { - "status": "success" - } - ], - "koji_build_id": 42, - "koji_task_id": 0, - "status": "failure" - }`, - }, - { - initResult: worker.KojiInitJobResult{ - BuildID: 42, - Token: `"foobar"`, - }, - buildResult: worker.OSBuildKojiJobResult{ - Arch: test_distro.TestArchName, - HostOS: test_distro.TestDistroName, - ImageHash: "browns", - ImageSize: 42, - OSBuildOutput: &osbuild.Result{ - Success: true, - }, - }, - finalizeResult: worker.KojiFinalizeJobResult{ - KojiError: "failure", - }, - composeReplyCode: http.StatusCreated, - composeReply: `{"koji_build_id":42}`, - composeStatus: `{ - "image_statuses": [ - { - "status": "success" - }, - { - "status": "success" - } - ], - "koji_build_id": 42, - "koji_task_id": 0, - "status": "failure" - }`, - }, - { - initResult: worker.KojiInitJobResult{ - BuildID: 42, - Token: `"foobar"`, - }, - buildResult: worker.OSBuildKojiJobResult{ - Arch: test_distro.TestArchName, - HostOS: test_distro.TestDistroName, - ImageHash: "browns", - ImageSize: 42, - OSBuildOutput: &osbuild.Result{ - Success: true, - }, - }, - finalizeResult: worker.KojiFinalizeJobResult{ - JobResult: worker.JobResult{ - JobError: clienterrors.WorkerClientError(clienterrors.ErrorKojiFinalize, "Koji finalize error"), - }, - }, - composeReplyCode: http.StatusCreated, - composeReply: `{"koji_build_id":42}`, - composeStatus: `{ - "image_statuses": [ - { - "status": "success" - }, - { - "status": "success" - } - ], - "koji_build_id": 42, - "koji_task_id": 0, - "status": "failure" - }`, - }, - } - for _, c := range cases { - var wg sync.WaitGroup - wg.Add(1) - - go func(t *testing.T, result worker.KojiInitJobResult) { - _, token, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), test_distro.TestArchName, []string{worker.JobTypeKojiInit}, []string{""}) - require.NoError(t, err) - require.Equal(t, worker.JobTypeKojiInit, jobType) - - var initJob worker.KojiInitJob - err = json.Unmarshal(rawJob, &initJob) - require.NoError(t, err) - require.Equal(t, "koji.example.com", initJob.Server) - require.Equal(t, "foo", initJob.Name) - require.Equal(t, "1", initJob.Version) - require.Equal(t, "2", initJob.Release) - - initJobResult, err := json.Marshal(&jobResult{Result: result}) - require.NoError(t, err) - test.TestRoute(t, workerHandler, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), string(initJobResult), http.StatusOK, - fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token)) - - wg.Done() - }(t, c.initResult) - - test.TestRoute(t, handler, false, "POST", "/api/composer-koji/v1/compose", fmt.Sprintf(` - { - "name":"foo", - "version":"1", - "release":"2", - "distribution":"%[1]s", - "image_requests": [ - { - "architecture": "%[2]s", - "image_type": "%[3]s", - "repositories": [ - { - "baseurl": "https://repo.example.com/" - } - ] - }, - { - "architecture": "%[2]s", - "image_type": "%[3]s", - "repositories": [ - { - "baseurl": "https://repo.example.com/" - } - ] - } - ], - "koji": { - "server": "koji.example.com" - } - }`, test_distro.TestDistroName, test_distro.TestArchName, test_distro.TestImageTypeName), - c.composeReplyCode, c.composeReply, "id") - wg.Wait() - - _, token, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), test_distro.TestArchName, []string{worker.JobTypeOSBuildKoji}, []string{""}) - require.NoError(t, err) - require.Equal(t, worker.JobTypeOSBuildKoji, jobType) - - var osbuildJob worker.OSBuildKojiJob - err = json.Unmarshal(rawJob, &osbuildJob) - require.NoError(t, err) - require.Equal(t, "koji.example.com", osbuildJob.KojiServer) - require.Equal(t, "test.img", osbuildJob.ImageName) - require.NotEmpty(t, osbuildJob.KojiDirectory) - - buildJobResult, err := json.Marshal(&jobResult{Result: c.buildResult}) - require.NoError(t, err) - test.TestRoute(t, workerHandler, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), string(buildJobResult), http.StatusOK, - fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token)) - - _, token, jobType, rawJob, _, err = workerServer.RequestJob(context.Background(), test_distro.TestArchName, []string{worker.JobTypeOSBuildKoji}, []string{""}) - require.NoError(t, err) - require.Equal(t, worker.JobTypeOSBuildKoji, jobType) - - err = json.Unmarshal(rawJob, &osbuildJob) - require.NoError(t, err) - require.Equal(t, "koji.example.com", osbuildJob.KojiServer) - require.Equal(t, "test.img", osbuildJob.ImageName) - require.NotEmpty(t, osbuildJob.KojiDirectory) - - test.TestRoute(t, workerHandler, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), fmt.Sprintf(`{ - "result": { - "arch": "%s", - "host_os": "%s", - "image_hash": "browns", - "image_size": 42, - "osbuild_output": { - "success": true - } - } - }`, test_distro.TestArchName, test_distro.TestDistroName), http.StatusOK, - fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token)) - - finalizeID, token, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), test_distro.TestArchName, []string{worker.JobTypeKojiFinalize}, []string{""}) - require.NoError(t, err) - require.Equal(t, worker.JobTypeKojiFinalize, jobType) - - var kojiFinalizeJob worker.KojiFinalizeJob - err = json.Unmarshal(rawJob, &kojiFinalizeJob) - require.NoError(t, err) - require.Equal(t, "koji.example.com", kojiFinalizeJob.Server) - require.Equal(t, "1", kojiFinalizeJob.Version) - require.Equal(t, "2", kojiFinalizeJob.Release) - require.ElementsMatch(t, []string{ - fmt.Sprintf("foo-1-2.%s.img", test_distro.TestArchName), - fmt.Sprintf("foo-1-2.%s.img", test_distro.TestArchName), - }, kojiFinalizeJob.KojiFilenames) - require.NotEmpty(t, kojiFinalizeJob.KojiDirectory) - - finalizeResult, err := json.Marshal(&jobResult{Result: c.finalizeResult}) - require.NoError(t, err) - test.TestRoute(t, workerHandler, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), string(finalizeResult), http.StatusOK, - fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token)) - - test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/composer-koji/v1/compose/%v", finalizeID), ``, http.StatusOK, c.composeStatus) - - test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/composer-koji/v1/compose/%v/manifests", finalizeID), ``, http.StatusOK, `[{"version": "", "pipelines": [], "sources": {}}, {"version": "", "pipelines": [], "sources": {}}]`) - } -} - -func TestRequest(t *testing.T) { - server, _ := newTestKojiServer(t, t.TempDir()) - handler := server.Handler("/api/composer-koji/v1") - - // Make request to an invalid route - req := httptest.NewRequest("GET", "/invalidroute", nil) - - rec := httptest.NewRecorder() - handler.ServeHTTP(rec, req) - resp := rec.Result() - - var status api.Status - err := json.NewDecoder(resp.Body).Decode(&status) - require.NoError(t, err) - require.Equal(t, http.StatusNotFound, resp.StatusCode) - - // Trigger an error 400 code - req = httptest.NewRequest("GET", "/api/composer-koji/v1/compose/badid", nil) - - rec = httptest.NewRecorder() - handler.ServeHTTP(rec, req) - resp = rec.Result() - - err = json.NewDecoder(resp.Body).Decode(&status) - require.NoError(t, err) - require.Equal(t, http.StatusBadRequest, resp.StatusCode) -} - -func TestJobTypeValidation(t *testing.T) { - dir := t.TempDir() - - server, workers := newTestKojiServer(t, dir) - handler := server.Handler("/api/composer-koji/v1") - - // Enqueue a compose job with N images (+ an Init and a Finalize job) - // Enqueuing them manually gives us access to the job IDs to use in - // requests. - nImages := 4 - initJob := worker.KojiInitJob{ - Server: "test-server", - Name: "test-job", - Version: "42", - Release: "1", - } - initID, err := workers.EnqueueKojiInit(&initJob, "") - require.NoError(t, err) - - buildJobs := make([]worker.OSBuildKojiJob, nImages) - buildJobIDs := make([]uuid.UUID, nImages) - filenames := make([]string, nImages) - for idx := 0; idx < nImages; idx++ { - fname := fmt.Sprintf("image-file-%04d", idx) - buildJob := worker.OSBuildKojiJob{ - ImageName: fmt.Sprintf("build-job-%04d", idx), - KojiServer: "test-server", - KojiDirectory: "koji-server-test-dir", - KojiFilename: fname, - } - buildID, err := workers.EnqueueOSBuildKoji(fmt.Sprintf("fake-arch-%d", idx), &buildJob, initID, "") - require.NoError(t, err) - - buildJobs[idx] = buildJob - buildJobIDs[idx] = buildID - filenames[idx] = fname - } - - finalizeJob := worker.KojiFinalizeJob{ - Server: "test-server", - Name: "test-job", - Version: "42", - Release: "1", - KojiFilenames: filenames, - KojiDirectory: "koji-server-test-dir", - TaskID: 0, - StartTime: uint64(time.Now().Unix()), - } - finalizeID, err := workers.EnqueueKojiFinalize(&finalizeJob, initID, buildJobIDs, "") - require.NoError(t, err) - - // ----- Jobs queued - Test API endpoints (status, manifests, logs) ----- // - - for _, path := range []string{"", "/manifests", "/logs"} { - // should return OK - actual result should be tested elsewhere - test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/composer-koji/v1/compose/%s%s", finalizeID, path), ``, http.StatusOK, "*") - - // The other IDs should fail - msg := fmt.Sprintf("Job %s not found: expected \"koji-finalize\", found \"koji-init\" job instead", initID) - resp, _ := json.Marshal(map[string]string{"message": msg}) - test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/composer-koji/v1/compose/%s%s", initID, path), ``, http.StatusNotFound, string(resp)) - - for idx, buildID := range buildJobIDs { - msg := fmt.Sprintf("Job %s not found: expected \"koji-finalize\", found \"osbuild-koji:fake-arch-%d\" job instead", buildID, idx) - resp, _ := json.Marshal(map[string]string{"message": msg}) - test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/composer-koji/v1/compose/%s%s", buildID, path), ``, http.StatusNotFound, string(resp)) - } - - badID := uuid.New() - msg = fmt.Sprintf("Job %s not found: job does not exist", badID) - resp, _ = json.Marshal(map[string]string{"message": msg}) - test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/composer-koji/v1/compose/%s%s", badID, path), ``, http.StatusNotFound, string(resp)) - } -} - -func TestMain(m *testing.M) { - setupDNFJSON() - defer os.RemoveAll(dnfjsonPath) - code := m.Run() - os.Exit(code) -}