particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cloudapi/v2: use the ostree resolve job to resolve ostree refs

Browse source
This commit is contained in:
Sanne Raymaekers 2022-10-18 15:25:45 +02:00
parent ebeb339f96
commit 8fdd158799
5 changed files with 128 additions and 183 deletions
Download patch file Download diff file Expand all files Collapse all files

43
internal/cloudapi/v2/handler.go
View file

@ -112,6 +112,7 @@ type imageRequest struct {
repositories []rpmmd.RepoConfig
imageOptions distro.ImageOptions
target *target.Target
ostree *ostree.RequestParams
}
func (h *apiHandlers) PostCompose(ctx echo.Context) error {
@ -286,44 +287,23 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
}
}
var ostreeOptions *ostree.RequestParams
// assume it's an ostree image if the type has a default ostree ref
if imageType.OSTreeRef() != "" && ir.Ostree != nil {
ostreeOptions := ostree.RequestParams{}
if ir.Ostree != nil {
if ir.Ostree.Ref != nil {
ostreeOptions.Ref = *ir.Ostree.Ref
}
if ir.Ostree.Url != nil {
ostreeOptions.URL = *ir.Ostree.Url
}
if ir.Ostree.Parent != nil {
ostreeOptions.Parent = *ir.Ostree.Parent
}
ostreeOptions = &ostree.RequestParams{}
if ir.Ostree.Ref != nil {
ostreeOptions.Ref = *ir.Ostree.Ref
}
if ir.Ostree.Url != nil {
ostreeOptions.URL = *ir.Ostree.Url
}
if ir.Ostree.Parent != nil {
ostreeOptions.Parent = *ir.Ostree.Parent
}
if ostreeOptions.Ref == "" {
ostreeOptions.Ref = imageType.OSTreeRef()
}
ref, checksum, err := ostree.ResolveParams(ostreeOptions)
if err != nil {
switch v := err.(type) {
case ostree.RefError:
return HTTPError(ErrorInvalidOSTreeRef)
case ostree.ResolveRefError:
return HTTPErrorWithInternal(ErrorInvalidOSTreeRepo, v)
case ostree.ParameterComboError:
return HTTPError(ErrorInvalidOSTreeParams)
default:
// general case
return HTTPError(ErrorInvalidOSTreeParams)
}
}
imageOptions.OSTree = distro.OSTreeImageOptions{
ImageRef: ref,
FetchChecksum: checksum,
URL: ostreeOptions.URL,
}
}
var irTarget *target.Target
@ -518,6 +498,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
repositories: repos,
imageOptions: imageOptions,
target: irTarget,
ostree: ostreeOptions,
})
}

69
internal/cloudapi/v2/server.go
View file

@ -139,6 +139,25 @@ func (s *Server) enqueueCompose(distribution distro.Distro, bp blueprint.Bluepri
dependencies = append(dependencies, jobId)
}
var ostreeResolveJobID uuid.UUID
if ir.ostree != nil {
jobID, err := s.workers.EnqueueOSTreeResolveJob(&worker.OSTreeResolveJob{
Specs: []worker.OSTreeResolveSpec{
worker.OSTreeResolveSpec{
URL: ir.ostree.URL,
Ref: ir.ostree.Ref,
Parent: ir.ostree.Parent,
},
},
}, channel)
if err != nil {
return id, HTTPErrorWithInternal(ErrorEnqueueingJob, err)
}
ostreeResolveJobID = jobID
dependencies = append(dependencies, ostreeResolveJobID)
}
manifestJobID, err := s.workers.EnqueueManifestJobByID(&worker.ManifestJobByID{}, dependencies, channel)
if err != nil {
return id, HTTPErrorWithInternal(ErrorEnqueueingJob, err)
@ -157,7 +176,7 @@ func (s *Server) enqueueCompose(distribution distro.Distro, bp blueprint.Bluepri
s.goroutinesGroup.Add(1)
go func() {
generateManifest(s.goroutinesCtx, s.workers, depsolveJobID, containerResolveJob, manifestJobID, ir.imageType, ir.repositories, ir.imageOptions, manifestSeed, bp.Customizations)
generateManifest(s.goroutinesCtx, s.workers, depsolveJobID, containerResolveJob, ostreeResolveJobID, manifestJobID, ir.imageType, ir.repositories, ir.imageOptions, manifestSeed, bp.Customizations)
defer s.goroutinesGroup.Done()
}()
@ -217,6 +236,25 @@ func (s *Server) enqueueKojiCompose(taskID uint64, server, name, version, releas
dependencies = append(dependencies, jobId)
}
var ostreeResolveJobID uuid.UUID
if ir.ostree != nil {
jobID, err := s.workers.EnqueueOSTreeResolveJob(&worker.OSTreeResolveJob{
Specs: []worker.OSTreeResolveSpec{
worker.OSTreeResolveSpec{
URL: ir.ostree.URL,
Ref: ir.ostree.Ref,
Parent: ir.ostree.Parent,
},
},
}, channel)
if err != nil {
return id, HTTPErrorWithInternal(ErrorEnqueueingJob, err)
}
ostreeResolveJobID = jobID
dependencies = append(dependencies, ostreeResolveJobID)
}
manifestJobID, err := s.workers.EnqueueManifestJobByID(&worker.ManifestJobByID{}, dependencies, channel)
if err != nil {
return id, HTTPErrorWithInternal(ErrorEnqueueingJob, err)
@ -261,7 +299,7 @@ func (s *Server) enqueueKojiCompose(taskID uint64, server, name, version, releas
// copy the image request while passing it into the goroutine to prevent data races
s.goroutinesGroup.Add(1)
go func(ir imageRequest) {
generateManifest(s.goroutinesCtx, s.workers, depsolveJobID, containerResolveJob, manifestJobID, ir.imageType, ir.repositories, ir.imageOptions, manifestSeed, bp.Customizations)
generateManifest(s.goroutinesCtx, s.workers, depsolveJobID, containerResolveJob, ostreeResolveJobID, manifestJobID, ir.imageType, ir.repositories, ir.imageOptions, manifestSeed, bp.Customizations)
defer s.goroutinesGroup.Done()
}(ir)
}
@ -282,7 +320,7 @@ func (s *Server) enqueueKojiCompose(taskID uint64, server, name, version, releas
return id, nil
}
func generateManifest(ctx context.Context, workers *worker.Server, depsolveJobID, containerResolveJobID, manifestJobID uuid.UUID, imageType distro.ImageType, repos []rpmmd.RepoConfig, options distro.ImageOptions, seed int64, b *blueprint.Customizations) {
func generateManifest(ctx context.Context, workers *worker.Server, depsolveJobID, containerResolveJobID, ostreeResolveJobID, manifestJobID uuid.UUID, imageType distro.ImageType, repos []rpmmd.RepoConfig, options distro.ImageOptions, seed int64, b *blueprint.Customizations) {
ctx, cancel := context.WithTimeout(ctx, time.Minute*5)
defer cancel()
@ -367,7 +405,7 @@ func generateManifest(ctx context.Context, workers *worker.Server, depsolveJobID
}
var containerSpecs []container.Spec
if len(dynArgs) == 2 {
if containerResolveJobID != uuid.Nil {
// Container resolve job
var result worker.ContainerResolveJobResult
@ -395,6 +433,29 @@ func generateManifest(ctx context.Context, workers *worker.Server, depsolveJobID
}
}
if ostreeResolveJobID != uuid.Nil {
var result worker.OSTreeResolveJobResult
_, err := workers.OSTreeResolveJobInfo(ostreeResolveJobID, &result)
if err != nil {
reason := "Error reading ostree resolve job status"
logrus.Errorf("%s: %v", reason, err)
jobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorReadingJobStatus, reason, nil)
return
}
if jobErr := result.JobError; jobErr != nil {
jobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorOSTreeDependency, "Error in ostree resolve job dependency", nil)
return
}
options.OSTree = distro.OSTreeImageOptions{
ImageRef: result.Specs[0].Ref,
FetchChecksum: result.Specs[0].Checksum,
URL: result.Specs[0].URL,
}
}
manifest, err := imageType.Manifest(b, options, repos, depsolveResults.PackageSpecs, containerSpecs, seed)
if err != nil {
reason := "Error generating manifest"

190
internal/cloudapi/v2/v2_test.go
View file

@ -78,8 +78,47 @@ func newV2Server(t *testing.T, dir string, depsolveChannels []string, enableJWT
}
}()
ostreeResolveContext, cancelOstree := context.WithCancel(context.Background())
wg.Add(1)
go func() {
defer wg.Done()
for {
_, token, _, _, _, err := workerServer.RequestJob(ostreeResolveContext, test_distro.TestDistroName, []string{worker.JobTypeOSTreeResolve}, depsolveChannels)
select {
case <-ostreeResolveContext.Done():
return
default:
}
if err != nil {
continue
}
oJR := &worker.OSTreeResolveJobResult{
Specs: []worker.OSTreeResolveResultSpec{
worker.OSTreeResolveResultSpec{
URL: "",
Ref: "",
Checksum: "",
},
},
}
if failDepsolve {
oJR.JobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorOSTreeParamsInvalid, "ostree error", nil)
}
rawMsg, err := json.Marshal(oJR)
require.NoError(t, err)
err = workerServer.FinishJob(token, rawMsg)
if err != nil {
return
}
}
}()
cancelWithWait := func() {
cancel()
cancelOstree()
wg.Wait()
}
@ -417,147 +456,6 @@ func TestCompose(t *testing.T) {
"href": "/api/image-builder-composer/v2/compose",
"kind": "ComposeId"
}`, "id")
// ostree errors
// bad url
test.TestRoute(t, srv.Handler("/api/image-builder-composer/v2"), false, "POST", "/api/image-builder-composer/v2/compose", fmt.Sprintf(`
{
"distribution": "%s",
"image_request":{
"architecture": "%s",
"image_type": "edge-commit",
"repositories": [{
"baseurl": "somerepo.org",
"rhsm": false
}],
"upload_options": {
"region": "eu-central-1"
},
"ostree": {
"ref": "rhel/10/x86_64/edge",
"url": "not-a-URL"
}
}
}`, test_distro.TestDistroName, test_distro.TestArch3Name), http.StatusBadRequest, `
{
"href": "/api/image-builder-composer/v2/errors/10",
"id": "10",
"kind": "Error",
"code": "IMAGE-BUILDER-COMPOSER-10",
"reason": "Error resolving OSTree repo"
}`, "operation_id", "details")
// bad ref
test.TestRoute(t, srv.Handler("/api/image-builder-composer/v2"), false, "POST", "/api/image-builder-composer/v2/compose", fmt.Sprintf(`
{
"distribution": "%s",
"image_request":{
"architecture": "%s",
"image_type": "edge-commit",
"repositories": [{
"baseurl": "somerepo.org",
"rhsm": false
}],
"upload_options": {
"region": "eu-central-1"
},
"ostree": {
"ref": "/bad/ref"
}
}
}`, test_distro.TestDistroName, test_distro.TestArch3Name), http.StatusBadRequest, `
{
"href": "/api/image-builder-composer/v2/errors/9",
"id": "9",
"kind": "Error",
"code": "IMAGE-BUILDER-COMPOSER-9",
"reason": "Invalid OSTree ref"
}`, "operation_id", "details")
// bad parent ref
test.TestRoute(t, srv.Handler("/api/image-builder-composer/v2"), false, "POST", "/api/image-builder-composer/v2/compose", fmt.Sprintf(`
{
"distribution": "%s",
"image_request":{
"architecture": "%s",
"image_type": "edge-commit",
"repositories": [{
"baseurl": "somerepo.org",
"rhsm": false
}],
"upload_options": {
"region": "eu-central-1"
},
"ostree": {
"ref": "%s",
"url": "%s",
"parent": "/bad/ref/number/2"
}
}
}`, test_distro.TestDistroName, test_distro.TestArch3Name, ostreeRepoDefault.OSTreeRef, ostreeRepoDefault.Server.URL), http.StatusBadRequest, `
{
"href": "/api/image-builder-composer/v2/errors/9",
"id": "9",
"kind": "Error",
"code": "IMAGE-BUILDER-COMPOSER-9",
"reason": "Invalid OSTree ref"
}`, "operation_id", "details")
// incorrect ref for URL
test.TestRoute(t, srv.Handler("/api/image-builder-composer/v2"), false, "POST", "/api/image-builder-composer/v2/compose", fmt.Sprintf(`
{
"distribution": "%s",
"image_request":{
"architecture": "%s",
"image_type": "edge-commit",
"repositories": [{
"baseurl": "somerepo.org",
"rhsm": false
}],
"upload_options": {
"region": "eu-central-1"
},
"ostree": {
"url": "%s",
"parent": "incorrect/ref"
}
}
}`, test_distro.TestDistroName, test_distro.TestArch3Name, ostreeRepoOther.Server.URL), http.StatusBadRequest, `
{
"href": "/api/image-builder-composer/v2/errors/10",
"id": "10",
"kind": "Error",
"code": "IMAGE-BUILDER-COMPOSER-10",
"reason": "Error resolving OSTree repo"
}`, "operation_id", "details")
// parent ref without URL
test.TestRoute(t, srv.Handler("/api/image-builder-composer/v2"), false, "POST", "/api/image-builder-composer/v2/compose", fmt.Sprintf(`
{
"distribution": "%s",
"image_request":{
"architecture": "%s",
"image_type": "edge-commit",
"repositories": [{
"baseurl": "somerepo.org",
"rhsm": false
}],
"upload_options": {
"region": "eu-central-1"
},
"ostree": {
"parent": "some/ref"
}
}
}`, test_distro.TestDistroName, test_distro.TestArch3Name), http.StatusBadRequest, `
{
"href": "/api/image-builder-composer/v2/errors/27",
"id": "27",
"kind": "Error",
"code": "IMAGE-BUILDER-COMPOSER-27",
"reason": "Invalid OSTree parameters or parameter combination"
}`, "operation_id", "details")
}
func TestComposeStatusSuccess(t *testing.T) {
@ -816,6 +714,10 @@ func TestComposeDependencyError(t *testing.T) {
"image_request":{
"architecture": "%s",
"image_type": "aws",
"ostree": {
"url": "somerepo.org",
"ref": "test"
},
"repositories": [{
"baseurl": "somerepo.org",
"rhsm": false
@ -864,7 +766,11 @@ func TestComposeDependencyError(t *testing.T) {
"details": [{
"id": 22,
"reason": "DNF Error"
}]
},
{
"id": 34,
"reason": "ostree error"
}]
}],
"id": 9,
"reason": "Manifest dependency failed"