particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

osbuild-service-maintenance: Clean up expired images

Browse source
This commit is contained in:
sanne 2021-11-19 16:26:09 +01:00 • committed by Tom Gundersen
parent 742e0e6616
commit c43ad2b22a
23 changed files with 899 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

4
internal/boot/aws.go
View file

@ -14,8 +14,8 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/osbuild/osbuild-composer/internal/cloud/awscloud"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/upload/awsupload"
)
type awsCredentials struct {
@ -91,7 +91,7 @@ func wrapErrorf(innerError error, format string, a ...interface{}) error {
// The s3 key is never returned - the same thing is done in osbuild-composer,
// the user has no way of getting the s3 key.
func UploadImageToAWS(c *awsCredentials, imagePath string, imageName string) error {
uploader, err := awsupload.New(c.Region, c.AccessKeyId, c.SecretAccessKey, c.sessionToken)
uploader, err := awscloud.New(c.Region, c.AccessKeyId, c.SecretAccessKey, c.sessionToken)
if err != nil {
return fmt.Errorf("cannot create aws uploader: %v", err)
}

52
internal/upload/awsupload/awsupload.go → internal/cloud/awscloud/awscloud.go
View file

@ -1,4 +1,4 @@
package awsupload
package awscloud
import (
"fmt"
@ -301,6 +301,56 @@ func (a *AWS) Register(name, bucket, key string, shareWith []string, rpmArch str
return registerOutput.ImageId, nil
}
func (a *AWS) RemoveSnapshotAndDeregisterImage(image *ec2.Image) error {
if image == nil {
return fmt.Errorf("image is nil")
}
var snapshots []*string
for _, bdm := range image.BlockDeviceMappings {
snapshots = append(snapshots, bdm.Ebs.SnapshotId)
}
_, err := a.ec2.DeregisterImage(
&ec2.DeregisterImageInput{
ImageId: image.ImageId,
},
)
if err != nil {
return err
}
for _, s := range snapshots {
_, err = a.ec2.DeleteSnapshot(
&ec2.DeleteSnapshotInput{
SnapshotId: s,
},
)
if err != nil {
// TODO return err?
log.Println("Unable to remove snapshot", s)
}
}
return err
}
// For service maintenance images are discovered by the "Name:composer-api-*" tag filter. Currently
// all image names in the service are generated, so they're guaranteed to be unique as well. If
// users are ever allowed to name their images, an extra tag should be added.
func (a *AWS) DescribeImagesByTag(tagKey, tagValue string) ([]*ec2.Image, error) {
imgs, err := a.ec2.DescribeImages(
&ec2.DescribeImagesInput{
Filters: []*ec2.Filter{
{
Name: aws.String(fmt.Sprintf("tag:%s", tagKey)),
Values: []*string{aws.String(tagValue)},
},
},
},
)
return imgs.Images, err
}
func (a *AWS) S3ObjectPresignedURL(bucket, objectKey string) (string, error) {
log.Printf("[AWS] 📋 Generating Presigned URL for S3 object %s/%s", bucket, objectKey)
req, _ := a.s3.GetObjectRequest(&s3.GetObjectInput{

18
internal/cloud/gcp/compute.go
View file

@ -265,17 +265,31 @@ func (g *GCP) ComputeImageShare(ctx context.Context, imageName string, shareWith
//
// Uses:
// - Compute Engine API
func (g *GCP) ComputeImageDelete(ctx context.Context, image string) error {
func (g *GCP) ComputeImageDelete(ctx context.Context, resourceId string) error {
computeService, err := compute.NewService(ctx, option.WithCredentials(g.creds))
if err != nil {
return fmt.Errorf("failed to get Compute Engine client: %v", err)
}
_, err = computeService.Images.Delete(g.creds.ProjectID, image).Context(ctx).Do()
_, err = computeService.Images.Delete(g.creds.ProjectID, resourceId).Context(ctx).Do()
return err
}
// ComputeExecuteFunctionForImages will pass all the compute images in the account to a function,
// which is able to iterate over the images. Useful if something needs to be execute for each image.
// Uses:
// - Compute Engine API
func (g *GCP) ComputeExecuteFunctionForImages(ctx context.Context, f func(*compute.ImageList) error) error {
computeService, err := compute.NewService(ctx, option.WithCredentials(g.creds))
if err != nil {
return fmt.Errorf("failed to get Compute Engine client: %v", err)
}
imagesService := compute.NewImagesService(computeService)
return imagesService.List(g.creds.ProjectID).Pages(ctx, f)
}
// ComputeInstanceDelete deletes a Compute Engine instance with the given name and
// running in the given zone. If the instance existed and was successfully deleted,
// no error is returned.

4
internal/cloudapi/v2/v2.go
View file

@ -263,6 +263,10 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
return HTTPError(ErrorJSONUnMarshallingError)
}
// For service maintenance, images are discovered by the "Name:composer-api-*"
// tag filter. Currently all image names in the service are generated, so they're
// guaranteed to be unique as well. If users are ever allowed to name their images,
// an extra tag should be added.
key := fmt.Sprintf("composer-api-%s", uuid.New().String())
t := target.NewAWSTarget(&target.AWSTargetOptions{
Filename: imageType.Filename(),

143
internal/jobqueue/dbjobqueue/dbjobqueue.go
View file

@ -96,27 +96,50 @@ const (
sqlDeleteHeartbeat = `
DELETE FROM heartbeats
WHERE id = $1`
// Maintenance queries
sqlQueryJobsUptoByType = `
SELECT array_agg(id), type
FROM jobs
WHERE type = ANY($1) AND finished_at < $2
GROUP BY type`
sqlQueryDepedenciesRecursively = `
WITH RECURSIVE dependencies(d) AS (
SELECT dependency_id
FROM job_dependencies
WHERE job_id = $1
UNION ALL
SELECT dependency_id
FROM dependencies, job_dependencies
WHERE job_dependencies.job_id = d )
SELECT * FROM dependencies`
sqlDeleteJobDependencies = `
DELETE FROM job_dependencies
WHERE dependency_id = ANY($1)`
sqlDeleteJobs = `
DELETE FROM jobs
WHERE id = ANY($1)`
)
type dbJobQueue struct {
type DBJobQueue struct {
pool *pgxpool.Pool
}
// Create a new dbJobQueue object for `url`.
func New(url string) (*dbJobQueue, error) {
// Create a new DBJobQueue object for `url`.
func New(url string) (*DBJobQueue, error) {
pool, err := pgxpool.Connect(context.Background(), url)
if err != nil {
return nil, fmt.Errorf("error establishing connection: %v", err)
}
return &dbJobQueue{pool}, nil
return &DBJobQueue{pool}, nil
}
func (q *dbJobQueue) Close() {
func (q *DBJobQueue) Close() {
q.pool.Close()
}
func (q *dbJobQueue) Enqueue(jobType string, args interface{}, dependencies []uuid.UUID) (uuid.UUID, error) {
func (q *DBJobQueue) Enqueue(jobType string, args interface{}, dependencies []uuid.UUID) (uuid.UUID, error) {
conn, err := q.pool.Acquire(context.Background())
if err != nil {
return uuid.Nil, fmt.Errorf("error connecting to database: %v", err)
@ -162,7 +185,7 @@ func (q *dbJobQueue) Enqueue(jobType string, args interface{}, dependencies []uu
return id, nil
}
func (q *dbJobQueue) Dequeue(ctx context.Context, jobTypes []string) (uuid.UUID, uuid.UUID, []uuid.UUID, string, json.RawMessage, error) {
func (q *DBJobQueue) Dequeue(ctx context.Context, jobTypes []string) (uuid.UUID, uuid.UUID, []uuid.UUID, string, json.RawMessage, error) {
// Return early if the context is already canceled.
if err := ctx.Err(); err != nil {
return uuid.Nil, uuid.Nil, nil, "", nil, jobqueue.ErrDequeueTimeout
@ -221,7 +244,7 @@ func (q *dbJobQueue) Dequeue(ctx context.Context, jobTypes []string) (uuid.UUID,
return id, token, dependencies, jobType, args, nil
}
func (q *dbJobQueue) DequeueByID(ctx context.Context, id uuid.UUID) (uuid.UUID, []uuid.UUID, string, json.RawMessage, error) {
func (q *DBJobQueue) DequeueByID(ctx context.Context, id uuid.UUID) (uuid.UUID, []uuid.UUID, string, json.RawMessage, error) {
// Return early if the context is already canceled.
if err := ctx.Err(); err != nil {
return uuid.Nil, nil, "", nil, jobqueue.ErrDequeueTimeout
@ -260,7 +283,7 @@ func (q *dbJobQueue) DequeueByID(ctx context.Context, id uuid.UUID) (uuid.UUID,
return token, dependencies, jobType, args, nil
}
func (q *dbJobQueue) FinishJob(id uuid.UUID, result interface{}) error {
func (q *DBJobQueue) FinishJob(id uuid.UUID, result interface{}) error {
conn, err := q.pool.Acquire(context.Background())
if err != nil {
return fmt.Errorf("error connecting to database: %v", err)
@ -327,7 +350,7 @@ func (q *dbJobQueue) FinishJob(id uuid.UUID, result interface{}) error {
return nil
}
func (q *dbJobQueue) CancelJob(id uuid.UUID) error {
func (q *DBJobQueue) CancelJob(id uuid.UUID) error {
conn, err := q.pool.Acquire(context.Background())
if err != nil {
return fmt.Errorf("error connecting to database: %v", err)
@ -348,7 +371,7 @@ func (q *dbJobQueue) CancelJob(id uuid.UUID) error {
return nil
}
func (q *dbJobQueue) JobStatus(id uuid.UUID) (result json.RawMessage, queued, started, finished time.Time, canceled bool, deps []uuid.UUID, err error) {
func (q *DBJobQueue) JobStatus(id uuid.UUID) (result json.RawMessage, queued, started, finished time.Time, canceled bool, deps []uuid.UUID, err error) {
conn, err := q.pool.Acquire(context.Background())
if err != nil {
return
@ -380,7 +403,7 @@ func (q *dbJobQueue) JobStatus(id uuid.UUID) (result json.RawMessage, queued, st
}
// Job returns all the parameters that define a job (everything provided during Enqueue).
func (q *dbJobQueue) Job(id uuid.UUID) (jobType string, args json.RawMessage, dependencies []uuid.UUID, err error) {
func (q *DBJobQueue) Job(id uuid.UUID) (jobType string, args json.RawMessage, dependencies []uuid.UUID, err error) {
conn, err := q.pool.Acquire(context.Background())
if err != nil {
return
@ -400,7 +423,7 @@ func (q *dbJobQueue) Job(id uuid.UUID) (jobType string, args json.RawMessage, de
}
// Find job by token, this will return an error if the job hasn't been dequeued
func (q *dbJobQueue) IdFromToken(token uuid.UUID) (id uuid.UUID, err error) {
func (q *DBJobQueue) IdFromToken(token uuid.UUID) (id uuid.UUID, err error) {
conn, err := q.pool.Acquire(context.Background())
if err != nil {
return uuid.Nil, fmt.Errorf("error establishing connection: %v", err)
@ -418,7 +441,7 @@ func (q *dbJobQueue) IdFromToken(token uuid.UUID) (id uuid.UUID, err error) {
}
// Get a list of tokens which haven't been updated in the specified time frame
func (q *dbJobQueue) Heartbeats(olderThan time.Duration) (tokens []uuid.UUID) {
func (q *DBJobQueue) Heartbeats(olderThan time.Duration) (tokens []uuid.UUID) {
conn, err := q.pool.Acquire(context.Background())
if err != nil {
return
@ -449,7 +472,7 @@ func (q *dbJobQueue) Heartbeats(olderThan time.Duration) (tokens []uuid.UUID) {
}
// Reset the last heartbeat time to time.Now()
func (q *dbJobQueue) RefreshHeartbeat(token uuid.UUID) {
func (q *DBJobQueue) RefreshHeartbeat(token uuid.UUID) {
conn, err := q.pool.Acquire(context.Background())
if err != nil {
return
@ -465,7 +488,7 @@ func (q *dbJobQueue) RefreshHeartbeat(token uuid.UUID) {
}
}
func (q *dbJobQueue) jobDependencies(ctx context.Context, conn *pgxpool.Conn, id uuid.UUID) ([]uuid.UUID, error) {
func (q *DBJobQueue) jobDependencies(ctx context.Context, conn *pgxpool.Conn, id uuid.UUID) ([]uuid.UUID, error) {
rows, err := conn.Query(ctx, sqlQueryDependencies, id)
if err != nil {
return nil, err
@ -488,3 +511,91 @@ func (q *dbJobQueue) jobDependencies(ctx context.Context, conn *pgxpool.Conn, id
return dependencies, nil
}
// return map id -> jobtype ?
func (q *DBJobQueue) JobsUptoByType(jobTypes []string, upto time.Time) (result map[string][]uuid.UUID, err error) {
result = make(map[string][]uuid.UUID)
conn, err := q.pool.Acquire(context.Background())
if err != nil {
err = fmt.Errorf("error connecting to database: %v", err)
return
}
defer conn.Release()
rows, err := conn.Query(context.Background(), sqlQueryJobsUptoByType, jobTypes, upto)
if err != nil {
return
}
defer rows.Close()
for rows.Next() {
var ids []uuid.UUID
var jt string
err = rows.Scan(&ids, &jt)
if err != nil {
return
}
result[jt] = ids
}
err = rows.Err()
return
}
// Deletes single job and dependencies (recursively)
func (q *DBJobQueue) DeleteJobIncludingDependencies(jobId uuid.UUID) error {
conn, err := q.pool.Acquire(context.Background())
if err != nil {
return fmt.Errorf("error connecting to database: %v", err)
}
defer conn.Release()
tx, err := conn.Begin(context.Background())
if err != nil {
return fmt.Errorf("error starting database transaction: %v", err)
}
defer func() {
err := tx.Rollback(context.Background())
if err != nil && !errors.As(err, &pgx.ErrTxClosed) {
logrus.Error("error rolling back enqueue transaction: ", err)
}
}()
rows, err := conn.Query(context.Background(), sqlQueryDepedenciesRecursively, jobId)
if err != nil {
return fmt.Errorf("error querying the job's dependencies: %v", err)
}
var dependencies []uuid.UUID
for rows.Next() {
var dep uuid.UUID
err = rows.Scan(&dep)
if err != nil {
return err
}
dependencies = append(dependencies, dep)
}
depTag, err := conn.Exec(context.Background(), sqlDeleteJobDependencies, dependencies)
if err != nil {
return fmt.Errorf("Error removing from dependencies recursively for job %v: %v", jobId, err)
}
jobAndDependencies := append(dependencies, jobId)
jobsTag, err := conn.Exec(context.Background(), sqlDeleteJobs, jobAndDependencies)
if err != nil {
return fmt.Errorf("Error removing from jobs recursively for job %v: %v", jobId, err)
}
err = tx.Commit(context.Background())
if err != nil {
return fmt.Errorf("unable to commit database transaction: %v", err)
}
logrus.Infof("Removed %d rows from dependencies for job %v", depTag.RowsAffected(), jobId)
logrus.Infof("Removed %d rows from jobs for job %v, this includes dependencies", jobsTag.RowsAffected(), jobId)
return nil
}