//go:generate go run github.com/deepmap/oapi-codegen/cmd/oapi-codegen --package=cloudapi --generate types,spec,client,server -o openapi.gen.go openapi.yml package cloudapi import ( "crypto/rand" "encoding/base64" "encoding/json" "fmt" "math" "math/big" "net/http" "strings" "github.com/google/uuid" "github.com/labstack/echo/v4" "github.com/osbuild/osbuild-composer/internal/blueprint" "github.com/osbuild/osbuild-composer/internal/distro" "github.com/osbuild/osbuild-composer/internal/distroregistry" "github.com/osbuild/osbuild-composer/internal/osbuild1" "github.com/osbuild/osbuild-composer/internal/ostree" "github.com/osbuild/osbuild-composer/internal/prometheus" "github.com/osbuild/osbuild-composer/internal/rpmmd" "github.com/osbuild/osbuild-composer/internal/target" "github.com/osbuild/osbuild-composer/internal/worker" ) // Server represents the state of the cloud Server type Server struct { workers *worker.Server rpmMetadata rpmmd.RPMMD distros *distroregistry.Registry identityFilter []string } type contextKey int type apiHandlers struct { server *Server } // NewServer creates a new cloud server func NewServer(workers *worker.Server, rpmMetadata rpmmd.RPMMD, distros *distroregistry.Registry) *Server { server := &Server{ workers: workers, rpmMetadata: rpmMetadata, distros: distros, } return server } // Create an http.Handler() for this server, that provides the composer API at // the given path. func (server *Server) Handler(path string, identityFilter []string) http.Handler { e := echo.New() if len(identityFilter) > 0 { server.identityFilter = identityFilter e.Use(server.VerifyIdentityHeader) } handler := apiHandlers{ server: server, } RegisterHandlers(e.Group(path, server.IncRequests), &handler) return e } func (server *Server) VerifyIdentityHeader(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { const identityHeaderKey contextKey = iota type identityHeader struct { Identity struct { AccountNumber string `json:"account_number"` } `json:"identity"` } idHeaderB64 := c.Request().Header.Get("X-Rh-Identity") if idHeaderB64 == "" { return echo.NewHTTPError(http.StatusNotFound, "Auth header is not present") } b64Result, err := base64.StdEncoding.DecodeString(idHeaderB64) if err != nil { return echo.NewHTTPError(http.StatusNotFound, "Auth header has incorrect format") } var idHeader identityHeader err = json.Unmarshal([]byte(strings.TrimSuffix(fmt.Sprintf("%s", b64Result), "\n")), &idHeader) if err != nil { return echo.NewHTTPError(http.StatusNotFound, "Auth header has incorrect format") } for _, i := range server.identityFilter { if idHeader.Identity.AccountNumber == i { c.Set("IdentityHeader", idHeader) c.Set("IdentityHeaderKey", identityHeaderKey) return next(c) } } return echo.NewHTTPError(http.StatusNotFound, "Account not allowed") } } func (s *Server) IncRequests(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { prometheus.TotalRequests.Inc() if strings.HasSuffix(c.Path(), "/compose") { prometheus.ComposeRequests.Inc() } return next(c) } } // Compose handles a new /compose POST request func (h *apiHandlers) Compose(ctx echo.Context) error { contentType := ctx.Request().Header["Content-Type"] if len(contentType) != 1 || contentType[0] != "application/json" { return echo.NewHTTPError(http.StatusUnsupportedMediaType, "Only 'application/json' content type is supported") } var request ComposeRequest err := json.NewDecoder(ctx.Request().Body).Decode(&request) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, "Could not parse request body") } distribution := h.server.distros.GetDistro(request.Distribution) if distribution == nil { return echo.NewHTTPError(http.StatusBadRequest, "Unsupported distribution: %s", request.Distribution) } var bp = blueprint.Blueprint{} err = bp.Initialize() if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Unable to initialize blueprint") } if request.Customizations != nil && request.Customizations.Packages != nil { for _, p := range *request.Customizations.Packages { bp.Packages = append(bp.Packages, blueprint.Package{ Name: p, }) } } type imageRequest struct { manifest distro.Manifest arch string exports []string } imageRequests := make([]imageRequest, len(request.ImageRequests)) var targets []*target.Target // use the same seed for all images so we get the same IDs bigSeed, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64)) if err != nil { panic("cannot generate a manifest seed: " + err.Error()) } manifestSeed := bigSeed.Int64() for i, ir := range request.ImageRequests { arch, err := distribution.GetArch(ir.Architecture) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, "Unsupported architecture '%s' for distribution '%s'", ir.Architecture, request.Distribution) } imageType, err := arch.GetImageType(ir.ImageType) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, "Unsupported image type '%s' for %s/%s", ir.ImageType, ir.Architecture, request.Distribution) } repositories := make([]rpmmd.RepoConfig, len(ir.Repositories)) for j, repo := range ir.Repositories { repositories[j].RHSM = repo.Rhsm if repo.Baseurl != nil { repositories[j].BaseURL = *repo.Baseurl } else if repo.Mirrorlist != nil { repositories[j].MirrorList = *repo.Mirrorlist } else if repo.Metalink != nil { repositories[j].Metalink = *repo.Metalink } else { return echo.NewHTTPError(http.StatusBadRequest, "Must specify baseurl, mirrorlist, or metalink") } } packageSets := imageType.PackageSets(bp) pkgSpecSets := make(map[string][]rpmmd.PackageSpec) for name, packages := range packageSets { pkgs, _, err := h.server.rpmMetadata.Depsolve(packages, repositories, distribution.ModulePlatformID(), arch.Name()) if err != nil { var error_type int switch err.(type) { // Known DNF errors falls under BadRequest case *rpmmd.DNFError: error_type = http.StatusBadRequest // All other kind of errors are internal server Errors. // (json marshalling issues for instance) case error: error_type = http.StatusInternalServerError } return echo.NewHTTPError(error_type, "Failed to depsolve base packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err) } pkgSpecSets[name] = pkgs } imageOptions := distro.ImageOptions{Size: imageType.Size(0)} if request.Customizations != nil && request.Customizations.Subscription != nil { imageOptions.Subscription = &distro.SubscriptionImageOptions{ Organization: request.Customizations.Subscription.Organization, ActivationKey: request.Customizations.Subscription.ActivationKey, ServerUrl: request.Customizations.Subscription.ServerUrl, BaseUrl: request.Customizations.Subscription.BaseUrl, Insights: request.Customizations.Subscription.Insights, } } // set default ostree ref, if one not provided ostreeOptions := ir.Ostree if ostreeOptions == nil || ostreeOptions.Ref == nil { imageOptions.OSTree = distro.OSTreeImageOptions{Ref: imageType.OSTreeRef()} } else if !ostree.VerifyRef(*ostreeOptions.Ref) { return echo.NewHTTPError(http.StatusBadRequest, "Invalid OSTree ref: %s", *ostreeOptions.Ref) } else { imageOptions.OSTree = distro.OSTreeImageOptions{Ref: *ostreeOptions.Ref} } var parent string if ostreeOptions != nil && ostreeOptions.Url != nil { imageOptions.OSTree.URL = *ostreeOptions.Url parent, err = ostree.ResolveRef(imageOptions.OSTree.URL, imageOptions.OSTree.Ref) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, "Error resolving OSTree repo %s: %s", imageOptions.OSTree.URL, err) } imageOptions.OSTree.Parent = parent } // Set the blueprint customisation to take care of the user var blueprintCustoms *blueprint.Customizations if request.Customizations != nil && request.Customizations.Users != nil { var userCustomizations []blueprint.UserCustomization for _, user := range *request.Customizations.Users { var groups []string if user.Groups != nil { groups = *user.Groups } else { groups = nil } userCustomizations = append(userCustomizations, blueprint.UserCustomization{ Name: user.Name, Key: user.Key, Groups: groups, }, ) } blueprintCustoms = &blueprint.Customizations{ User: userCustomizations, } } manifest, err := imageType.Manifest(blueprintCustoms, imageOptions, repositories, pkgSpecSets, manifestSeed) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, "Failed to get manifest for for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err) } imageRequests[i].manifest = manifest imageRequests[i].arch = arch.Name() imageRequests[i].exports = imageType.Exports() uploadRequest := ir.UploadRequest /* oneOf is not supported by the openapi generator so marshal and unmarshal the uploadrequest based on the type */ if uploadRequest.Type == UploadTypes_aws { var awsUploadOptions AWSUploadRequestOptions jsonUploadOptions, err := json.Marshal(uploadRequest.Options) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Unable to marshal aws upload request") } err = json.Unmarshal(jsonUploadOptions, &awsUploadOptions) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Unable to unmarshal aws upload request") } var share []string if awsUploadOptions.Ec2.ShareWithAccounts != nil { share = *awsUploadOptions.Ec2.ShareWithAccounts } key := fmt.Sprintf("composer-api-%s", uuid.New().String()) t := target.NewAWSTarget(&target.AWSTargetOptions{ Filename: imageType.Filename(), Region: awsUploadOptions.Region, AccessKeyID: awsUploadOptions.S3.AccessKeyId, SecretAccessKey: awsUploadOptions.S3.SecretAccessKey, Bucket: awsUploadOptions.S3.Bucket, Key: key, ShareWithAccounts: share, }) if awsUploadOptions.Ec2.SnapshotName != nil { t.ImageName = *awsUploadOptions.Ec2.SnapshotName } else { t.ImageName = key } targets = append(targets, t) } else if uploadRequest.Type == UploadTypes_aws_s3 { var awsS3UploadOptions AWSS3UploadRequestOptions jsonUploadOptions, err := json.Marshal(uploadRequest.Options) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Unable to unmarshal aws upload request") } err = json.Unmarshal(jsonUploadOptions, &awsS3UploadOptions) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Unable to unmarshal aws upload request") } key := fmt.Sprintf("composer-api-%s", uuid.New().String()) t := target.NewAWSS3Target(&target.AWSS3TargetOptions{ Filename: imageType.Filename(), Region: awsS3UploadOptions.Region, AccessKeyID: awsS3UploadOptions.S3.AccessKeyId, SecretAccessKey: awsS3UploadOptions.S3.SecretAccessKey, Bucket: awsS3UploadOptions.S3.Bucket, Key: key, }) t.ImageName = key targets = append(targets, t) } else if uploadRequest.Type == UploadTypes_gcp { var gcpUploadOptions GCPUploadRequestOptions jsonUploadOptions, err := json.Marshal(uploadRequest.Options) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Unable to marshal gcp upload request") } err = json.Unmarshal(jsonUploadOptions, &gcpUploadOptions) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Unable to unmarshal gcp upload request") } var share []string if gcpUploadOptions.ShareWithAccounts != nil { share = *gcpUploadOptions.ShareWithAccounts } var region string if gcpUploadOptions.Region != nil { region = *gcpUploadOptions.Region } object := fmt.Sprintf("composer-api-%s", uuid.New().String()) t := target.NewGCPTarget(&target.GCPTargetOptions{ Filename: imageType.Filename(), Region: region, Os: "", // not exposed in cloudapi for now Bucket: gcpUploadOptions.Bucket, Object: object, ShareWithAccounts: share, }) // Import will fail if an image with this name already exists if gcpUploadOptions.ImageName != nil { t.ImageName = *gcpUploadOptions.ImageName } else { t.ImageName = object } targets = append(targets, t) } else if uploadRequest.Type == UploadTypes_azure { var azureUploadOptions AzureUploadRequestOptions jsonUploadOptions, err := json.Marshal(uploadRequest.Options) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Unable to marshal azure upload request") } err = json.Unmarshal(jsonUploadOptions, &azureUploadOptions) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Unable to unmarshal azure upload request") } t := target.NewAzureImageTarget(&target.AzureImageTargetOptions{ Filename: imageType.Filename(), TenantID: azureUploadOptions.TenantId, Location: azureUploadOptions.Location, SubscriptionID: azureUploadOptions.SubscriptionId, ResourceGroup: azureUploadOptions.ResourceGroup, }) if azureUploadOptions.ImageName != nil { t.ImageName = *azureUploadOptions.ImageName } else { // if ImageName wasn't given, generate a random one t.ImageName = fmt.Sprintf("composer-api-%s", uuid.New().String()) } targets = append(targets, t) } else { return echo.NewHTTPError(http.StatusBadRequest, "Unknown upload request type, only 'aws', 'azure' and 'gcp' are supported") } } var ir imageRequest if len(imageRequests) == 1 { // NOTE: the store currently does not support multi-image composes ir = imageRequests[0] } else { return echo.NewHTTPError(http.StatusBadRequest, "Only single-image composes are currently supported") } id, err := h.server.workers.EnqueueOSBuild(ir.arch, &worker.OSBuildJob{ Manifest: ir.manifest, Targets: targets, Exports: ir.exports, }) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Failed to enqueue manifest") } var response ComposeResult response.Id = id.String() return ctx.JSON(http.StatusOK, response) } // ComposeStatus handles a /compose/{id} GET request func (h *apiHandlers) ComposeStatus(ctx echo.Context, id string) error { jobId, err := uuid.Parse(id) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, "Invalid format for parameter id: %s", err) } var result worker.OSBuildJobResult status, _, err := h.server.workers.JobStatus(jobId, &result) if err != nil { return echo.NewHTTPError(http.StatusNotFound, "Job %s not found: %s", id, err) } var us *UploadStatus if result.TargetResults != nil { // Only single upload target is allowed, therefore only a single upload target result is allowed as well if len(result.TargetResults) != 1 { return echo.NewHTTPError(http.StatusInternalServerError, "Job %s returned more upload target results than allowed", id) } tr := *result.TargetResults[0] var uploadType UploadTypes var uploadOptions interface{} switch tr.Name { case "org.osbuild.aws": uploadType = UploadTypes_aws awsOptions := tr.Options.(*target.AWSTargetResultOptions) uploadOptions = AWSUploadStatus{ Ami: awsOptions.Ami, Region: awsOptions.Region, } case "org.osbuild.aws.s3": uploadType = UploadTypes_aws_s3 awsOptions := tr.Options.(*target.AWSS3TargetResultOptions) uploadOptions = AWSS3UploadStatus{ Url: awsOptions.URL, } case "org.osbuild.gcp": uploadType = UploadTypes_gcp gcpOptions := tr.Options.(*target.GCPTargetResultOptions) uploadOptions = GCPUploadStatus{ ImageName: gcpOptions.ImageName, ProjectId: gcpOptions.ProjectID, } case "org.osbuild.azure.image": uploadType = UploadTypes_azure gcpOptions := tr.Options.(*target.AzureImageTargetResultOptions) uploadOptions = AzureUploadStatus{ ImageName: gcpOptions.ImageName, } default: return echo.NewHTTPError(http.StatusInternalServerError, "Job %s returned unknown upload target results %s", id, tr.Name) } us = &UploadStatus{ Status: result.UploadStatus, Type: uploadType, Options: uploadOptions, } } response := ComposeStatus{ ImageStatus: ImageStatus{ Status: composeStatusFromJobStatus(status, &result), UploadStatus: us, }, } return ctx.JSON(http.StatusOK, response) } func composeStatusFromJobStatus(js *worker.JobStatus, result *worker.OSBuildJobResult) ImageStatusValue { if js.Canceled { return ImageStatusValue_failure } if js.Started.IsZero() { return ImageStatusValue_pending } if js.Finished.IsZero() { // TODO: handle also ImageStatusValue_uploading // TODO: handle also ImageStatusValue_registering return ImageStatusValue_building } if result.Success { return ImageStatusValue_success } return ImageStatusValue_failure } // GetOpenapiJson handles a /openapi.json GET request func (h *apiHandlers) GetOpenapiJson(ctx echo.Context) error { spec, err := GetSwagger() if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Could not load openapi spec") } return ctx.JSON(http.StatusOK, spec) } // GetVersion handles a /version GET request func (h *apiHandlers) GetVersion(ctx echo.Context) error { spec, err := GetSwagger() if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Could not load version") } version := Version{spec.Info.Version} return ctx.JSON(http.StatusOK, version) } // ComposeMetadata handles a /compose/{id}/metadata GET request func (h *apiHandlers) ComposeMetadata(ctx echo.Context, id string) error { jobId, err := uuid.Parse(id) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, "Invalid format for parameter id: %s", err) } var result worker.OSBuildJobResult status, _, err := h.server.workers.JobStatus(jobId, &result) if err != nil { return echo.NewHTTPError(http.StatusNotFound, "Job %s not found: %s", id, err) } var job worker.OSBuildJob if _, _, _, err = h.server.workers.Job(jobId, &job); err != nil { return echo.NewHTTPError(http.StatusNotFound, "Job %s not found: %s", id, err) } if status.Finished.IsZero() { // job still running: empty response return ctx.JSON(200, ComposeMetadata{}) } manifestVer, err := job.Manifest.Version() if err != nil { panic("Failed to parse manifest version: " + err.Error()) } var rpms []rpmmd.RPM var ostreeCommitResult *osbuild1.StageResult var coreStages []osbuild1.StageResult switch manifestVer { case "1": coreStages = result.OSBuildOutput.Stages if assemblerResult := result.OSBuildOutput.Assembler; assemblerResult.Name == "org.osbuild.ostree.commit" { ostreeCommitResult = result.OSBuildOutput.Assembler } case "2": // v2 manifest results store all stage output in the main stages // here we filter out the build stages to collect only the RPMs for the // core stages // the filtering relies on two assumptions: // 1. the build pipeline is named "build" // 2. the stage results from v2 manifests when converted to v1 are // named by prefixing the pipeline name for _, stage := range result.OSBuildOutput.Stages { if !strings.HasPrefix(stage.Name, "build") { coreStages = append(coreStages, stage) } } // find the ostree.commit stage for idx, stage := range result.OSBuildOutput.Stages { if strings.HasSuffix(stage.Name, "org.osbuild.ostree.commit") { ostreeCommitResult = &result.OSBuildOutput.Stages[idx] break } } default: panic("Unknown manifest version: " + manifestVer) } rpms = rpmmd.OSBuildStagesToRPMs(coreStages) packages := make([]PackageMetadata, len(rpms)) for idx, rpm := range rpms { packages[idx] = PackageMetadata{ Type: rpm.Type, Name: rpm.Name, Version: rpm.Version, Release: rpm.Release, Epoch: rpm.Epoch, Arch: rpm.Arch, Sigmd5: rpm.Sigmd5, Signature: rpm.Signature, } } resp := new(ComposeMetadata) resp.Packages = &packages if ostreeCommitResult != nil && ostreeCommitResult.Metadata != nil { commitMetadata, ok := ostreeCommitResult.Metadata.(*osbuild1.OSTreeCommitStageMetadata) if !ok { panic("Failed to convert ostree commit stage metadata") } resp.OstreeCommit = &commitMetadata.Compose.OSTreeCommit } return ctx.JSON(200, resp) }