osbuild-service-maintenance: Clean up expired images

cmd/osbuild-composer-dbjobqueue-tests/main_test.go

@@ -6,18 +6,21 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
+	"github.com/google/uuid"
 	"github.com/jackc/pgx/v4"
+	"github.com/stretchr/testify/require"
 
 	"github.com/osbuild/osbuild-composer/internal/jobqueue"
 	"github.com/osbuild/osbuild-composer/internal/jobqueue/dbjobqueue"
 	"github.com/osbuild/osbuild-composer/internal/jobqueue/jobqueuetest"
 )
 
-func TestJobQueueInterface(t *testing.T) {
-	jobqueuetest.TestJobQueue(t, func() (jobqueue.JobQueue, func(), error) {
-		url := "postgres://postgres:foobar@localhost:5432/osbuildcomposer"
+const url = "postgres://postgres:foobar@localhost:5432/osbuildcomposer"
 
+func TestJobQueueInterface(t *testing.T) {
+	makeJobQueue := func() (jobqueue.JobQueue, func(), error) {
 		// clear db before each run
 		conn, err := pgx.Connect(context.Background(), url)
 		if err != nil {
@@ -43,5 +46,176 @@ func TestJobQueueInterface(t *testing.T) {
 			q.Close()
 		}
 		return q, stop, nil
-	})
+	}
+
+	jobqueuetest.TestJobQueue(t, makeJobQueue)
+
+	wrap := func(f func(t *testing.T, q *dbjobqueue.DBJobQueue)) func(*testing.T) {
+		q, stop, err := makeJobQueue()
+		require.NoError(t, err)
+		return func(t *testing.T) {
+			defer stop() // use defer because f() might call testing.T.FailNow()
+			dbq, ok := q.(*dbjobqueue.DBJobQueue)
+			require.True(t, ok)
+			f(t, dbq)
+		}
+	}
+
+	t.Run("maintenance-query-jobs-before", wrap(testJobsUptoByType))
+	t.Run("maintenance-delete-job-and-dependencies", wrap(testDeleteJobAndDependencies))
 }
+
+func setFinishedAt(t *testing.T, q *dbjobqueue.DBJobQueue, id uuid.UUID, finished time.Time) {
+	conn, err := pgx.Connect(context.Background(), url)
+	require.NoError(t, err)
+	defer conn.Close(context.Background())
+
+	started := finished.Add(-time.Second)
+	queued := started.Add(-time.Second)
+
+	_, err = conn.Exec(context.Background(), "UPDATE jobs SET queued_at = $1, started_at = $2, finished_at = $3 WHERE id = $4", queued, started, finished, id)
+	require.NoError(t, err)
+}
+
+func testJobsUptoByType(t *testing.T, q *dbjobqueue.DBJobQueue) {
+	date80 := time.Date(1980, time.January, 1, 0, 0, 0, 0, time.UTC)
+	date85 := time.Date(1985, time.January, 1, 0, 0, 0, 0, time.UTC)
+	date90 := time.Date(1990, time.January, 1, 0, 0, 0, 0, time.UTC)
+
+	id80, err := q.Enqueue("octopus", nil, nil)
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id80)
+	_,_,_,_,_, err = q.Dequeue(context.Background(), []string{"octopus"})
+	require.NoError(t, err)
+	err = q.FinishJob(id80, nil)
+	require.NoError(t, err)
+	setFinishedAt(t, q, id80, date80)
+
+	id85, err := q.Enqueue("octopus", nil, nil)
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id85)
+	_,_,_,_,_, err = q.Dequeue(context.Background(), []string{"octopus"})
+	require.NoError(t, err)
+	err = q.FinishJob(id85, nil)
+	require.NoError(t, err)
+	setFinishedAt(t, q, id85, date85)
+
+	ids, err := q.JobsUptoByType([]string{"octopus"}, date85)
+	require.NoError(t, err)
+	require.ElementsMatch(t, []uuid.UUID{id80}, ids["octopus"])
+
+	ids, err = q.JobsUptoByType([]string{"octopus"}, date90)
+	require.NoError(t, err)
+	require.ElementsMatch(t, []uuid.UUID{id80, id85}, ids["octopus"])
+}
+
+func testDeleteJobAndDependencies(t *testing.T, q *dbjobqueue.DBJobQueue) {
+	// id1 -> id2 -> id3
+	id1, err := q.Enqueue("octopus", nil, nil)
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id1)
+	id2, err := q.Enqueue("octopus", nil, []uuid.UUID{id1})
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id2)
+	id3, err := q.Enqueue("octopus", nil, []uuid.UUID{id2})
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id3)
+
+	c1, err := q.Enqueue("octopus", nil, nil)
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, c1)
+	c2, err := q.Enqueue("octopus", nil, []uuid.UUID{c1})
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, c2)
+	c3, err := q.Enqueue("octopus", nil, []uuid.UUID{c2})
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, c3)
+	controls := []uuid.UUID{c1, c2, c3}
+
+	_,_,_, err = q.Job(c1)
+	require.NoError(t, err)
+
+	require.NoError(t, q.DeleteJobIncludingDependencies(id3))
+	for _, id := range []uuid.UUID{id1, id2, id3} {
+		_,_,_, err = q.Job(id)
+		require.ErrorIs(t, err, jobqueue.ErrNotExist)
+	}
+
+	// controls should still exist
+	for _, c := range controls {
+		_,_,_, err = q.Job(c)
+		require.NoError(t, err)
+	}
+
+	// id1 -> id2 -> id4 && id3 -> id4
+	id1, err = q.Enqueue("octopus", nil, nil)
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id1)
+	id2, err = q.Enqueue("octopus", nil, []uuid.UUID{id1})
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id2)
+	id3, err = q.Enqueue("octopus", nil, nil)
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id3)
+	id4, err := q.Enqueue("octopus", nil, []uuid.UUID{id2, id3})
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id4)
+
+	require.NoError(t, q.DeleteJobIncludingDependencies(id4))
+	for _, id := range []uuid.UUID{id1, id2, id3, id4} {
+		_,_,_, err = q.Job(id)
+		require.ErrorIs(t, err, jobqueue.ErrNotExist)
+	}
+
+	// controls should still exist
+	for _, c := range controls {
+		_,_,_, err = q.Job(c)
+		require.NoError(t, err)
+	}
+
+	// id1 has 2 dependants, and the maintenance queries currently do not account for this
+	// situation as it does not occur in the service.  This should be changed once we allow
+	// multiple build job per depsolve job, and the depsolve job should only be removed once all
+	// the build jobs have been dealt with.
+	id1, err = q.Enqueue("octopus", nil, nil)
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id1)
+	id2a, err := q.Enqueue("octopus", nil, []uuid.UUID{id1})
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id2a)
+	id2b, err := q.Enqueue("octopus", nil, []uuid.UUID{id1})
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id2b)
+	id3, err = q.Enqueue("octopus", nil, []uuid.UUID{id2a})
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.Nil, id3)
+
+	require.NoError(t, q.DeleteJobIncludingDependencies(id3))
+	for _, id := range []uuid.UUID{id1, id2a, id3} {
+		_,_,_, err = q.Job(id)
+		require.ErrorIs(t, err, jobqueue.ErrNotExist)
+	}
+
+	// id2b still exists
+	_,_,_, err = q.Job(id2b)
+	require.NoError(t, err)
+
+	// id2b can still be deleted with it's dependencies missing
+	require.NoError(t, q.DeleteJobIncludingDependencies(id2b))
+	_,_,_, err = q.Job(id2b)
+	require.ErrorIs(t, err, jobqueue.ErrNotExist)
+
+	// controls should still exist
+	for _, c := range controls {
+		_,_,_, err = q.Job(c)
+		require.NoError(t, err)
+	}
+
+	require.NoError(t, q.DeleteJobIncludingDependencies(uuid.Nil))
+	// controls should still exist
+	for _, c := range controls {
+		_,_,_, err = q.Job(c)
+		require.NoError(t, err)
+	}
+}
+

cmd/osbuild-service-maintenance/aws.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/semaphore"
+
+	"github.com/osbuild/osbuild-composer/internal/cloud/awscloud"
+)
+
+func AWSCleanup(maxConcurrentRequests int, dryRun bool, accessKeyID, accessKey, region string, cutoff time.Time) error {
+	a, err := awscloud.New(region, accessKeyID, accessKey, "")
+	if err != nil {
+		return err
+	}
+
+	var wg sync.WaitGroup
+	sem := semaphore.NewWeighted(int64(maxConcurrentRequests))
+	images, err := a.DescribeImagesByTag("Name", "composer-api-*")
+	if err != nil {
+		return err
+	}
+
+	for index, image := range images {
+		// TODO are these actual concerns?
+		if image.ImageId == nil {
+			logrus.Infof("ImageId is nil %v", image)
+			continue
+		}
+		if image.CreationDate == nil {
+			logrus.Infof("Image %v has nil creationdate", *image.ImageId)
+			continue
+		}
+
+		created, err := time.Parse(time.RFC3339, *image.CreationDate)
+		if err != nil {
+			logrus.Infof("Unable to parse date %s for image %s", *image.CreationDate, *image.ImageId)
+			continue
+		}
+
+		if !created.Before(cutoff) {
+			continue
+		}
+
+		if dryRun {
+			logrus.Infof("Dry run, aws image %s in region %s, with creation date %s would be removed", *image.ImageId, region, *image.CreationDate)
+			continue
+		}
+
+		if err = sem.Acquire(context.Background(), 1); err != nil {
+			logrus.Errorf("Error acquiring semaphore: %v", err)
+			continue
+		}
+		wg.Add(1)
+
+		go func(i int) {
+			defer sem.Release(1)
+			defer wg.Done()
+
+			err := a.RemoveSnapshotAndDeregisterImage(images[i])
+			if err != nil {
+				logrus.Errorf("Cleanup for image %s in region %s failed", *images[i].ImageId, region)
+			}
+		}(index)
+	}
+
+	wg.Wait()
+	return nil
+}

cmd/osbuild-service-maintenance/config.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+)
+
+// Do not write this config to logs or stdout, it contains secrets!
+type Config struct {
+	DryRun                 string `env:"DRY_RUN"`
+	MaxConcurrentRequests  string `env:"MAX_CONCURRENT_REQUESTS"`
+	PGHost                 string `env:"PGHOST"`
+	PGPort                 string `env:"PGPORT"`
+	PGDatabase             string `env:"PGDATABASE"`
+	PGUser                 string `env:"PGUSER"`
+	PGPassword             string `env:"PGPASSWORD"`
+	PGSSLMode              string `env:"PGSSLMODE"`
+	GoogleApplicationCreds string `env:"GOOGLE_APPLICATION_CREDENTIALS"`
+	AWSAccessKeyID         string `env:"AWS_ACCESS_KEY_ID"`
+	AWSSecretAccessKey     string `env:"AWS_SECRET_ACCESS_KEY"`
+}
+
+// *string means the value is not required
+// string means the value is required and should have a default value
+func LoadConfigFromEnv(intf interface{}) error {
+	t := reflect.TypeOf(intf).Elem()
+	v := reflect.ValueOf(intf).Elem()
+
+	for i := 0; i < v.NumField(); i++ {
+		fieldT := t.Field(i)
+		fieldV := v.Field(i)
+		key, ok := fieldT.Tag.Lookup("env")
+		if !ok {
+			return fmt.Errorf("No env tag in config field")
+		}
+
+		confV, ok := os.LookupEnv(key)
+		kind := fieldV.Kind()
+		if ok {
+			switch kind {
+			case reflect.Ptr:
+				if fieldT.Type.Elem().Kind() != reflect.String {
+					return fmt.Errorf("Unsupported type")
+				}
+				fieldV.Set(reflect.ValueOf(&confV))
+			case reflect.String:
+				fieldV.SetString(confV)
+			default:
+				return fmt.Errorf("Unsupported type")
+			}
+		}
+	}
+	return nil
+}

cmd/osbuild-service-maintenance/gcp.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/semaphore"
+	"google.golang.org/api/compute/v1"
+
+	"github.com/osbuild/osbuild-composer/internal/cloud/gcp"
+)
+
+func GCPCleanup(maxConcurrentRequests int, dryRun bool, cutoff time.Time) error {
+	g, err := gcp.New(nil)
+	if err != nil {
+		return err
+	}
+
+	sem := semaphore.NewWeighted(int64(maxConcurrentRequests))
+	var wg sync.WaitGroup
+	removeImageOlderThan := func(images *compute.ImageList) error {
+		for _, image := range images.Items {
+			created, err := time.Parse(time.RFC3339, image.CreationTimestamp)
+			if err != nil {
+				logrus.Errorf("Unable to parse image %s(%d)'s creation timestamp: %v", image.Name, image.Id, err)
+				continue
+			}
+
+			if !created.Before(cutoff) {
+				continue
+			}
+
+			if dryRun {
+				logrus.Infof("Dry run, gcp image %s(%d), with creation date %v would be removed", image.Name, image.Id, created)
+				continue
+			}
+
+			if err = sem.Acquire(context.Background(), 1); err != nil {
+				logrus.Errorf("Error acquiring semaphore: %v", err)
+				continue
+			}
+			wg.Add(1)
+
+			go func(id string) {
+				defer sem.Release(1)
+				defer wg.Done()
+
+				err = g.ComputeImageDelete(context.Background(), id)
+				if err != nil {
+					logrus.Errorf("Error deleting image %s created at %v", id, created)
+				}
+			}(fmt.Sprintf("%d", image.Id))
+		}
+		return nil
+	}
+
+	err = g.ComputeExecuteFunctionForImages(context.Background(), removeImageOlderThan)
+	if err != nil {
+		return err
+	}
+	wg.Wait()
+	return nil
+}

cmd/osbuild-service-maintenance/main.go

@@ -0,0 +1,108 @@
+package main
+
+import (
+	"fmt"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/osbuild/osbuild-composer/internal/jobqueue/dbjobqueue"
+)
+
+func main() {
+	logrus.SetReportCaller(true)
+
+	archs := []string{"x86_64"}
+	jobType := "osbuild"
+	// 14 days
+	cutoff := time.Now().Add(-(time.Hour * 24 * 14))
+	logrus.Infof("Cutoff date: %v", cutoff)
+
+	var conf Config
+	err := LoadConfigFromEnv(&conf)
+	if err != nil {
+		panic(err)
+	}
+	maxCReqs, err := strconv.Atoi(conf.MaxConcurrentRequests)
+	if err != nil {
+		panic(err)
+	}
+	dryRun, err := strconv.ParseBool(conf.DryRun)
+	if err != nil {
+		panic(err)
+	}
+
+	if dryRun {
+		logrus.Info("Dry run, no state will be changed")
+	}
+
+	dbURL := fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=%s",
+		conf.PGUser,
+		conf.PGPassword,
+		conf.PGHost,
+		conf.PGPort,
+		conf.PGDatabase,
+		conf.PGSSLMode,
+	)
+	jobs, err := dbjobqueue.New(dbURL)
+	if err != nil {
+		panic(err)
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		logrus.Info("Cleaning up AWS")
+		err := AWSCleanup(maxCReqs, dryRun, conf.AWSAccessKeyID, conf.AWSSecretAccessKey, "us-east-1", cutoff)
+		if err != nil {
+			logrus.Errorf("AWS cleanup failed: %v", err)
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		logrus.Info("Cleaning up GCP")
+		if conf.GoogleApplicationCreds == "" {
+			logrus.Error("GCP credentials not specified")
+			return
+		}
+		err = GCPCleanup(maxCReqs, dryRun, cutoff)
+		if err != nil {
+			logrus.Errorf("GCP Cleanup failed: %v", err)
+		}
+	}()
+
+	wg.Wait()
+	logrus.Info("🦀🦀🦀 cloud cleanup done 🦀🦀🦀")
+
+	var jobTypes []string
+	for _, a := range archs {
+		jobTypes = append(jobTypes, fmt.Sprintf("%s:%s", jobType, a))
+	}
+
+	jobsByType, err := jobs.JobsUptoByType(jobTypes, cutoff)
+	if err != nil {
+		logrus.Errorf("Error querying jobs: %v", err)
+		return
+	}
+
+	for k, v := range jobsByType {
+		logrus.Infof("Deleting jobs and their dependencies of type %v", k)
+		if dryRun {
+			logrus.Infof("Dry run, skipping deletion of jobs: %v", v)
+			continue
+		}
+
+		for _, jobId := range v {
+			err = jobs.DeleteJobIncludingDependencies(jobId)
+			if err != nil {
+				logrus.Errorf("Error deleting job: %v", jobId)
+			}
+		}
+	}
+	logrus.Info("🦀🦀🦀 dbqueue cleanup done 🦀🦀🦀")
+}

cmd/osbuild-upload-aws/main.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 
-	"github.com/osbuild/osbuild-composer/internal/upload/awsupload"
+	"github.com/osbuild/osbuild-composer/internal/cloud/awscloud"
 )
 
 func main() {
@@ -32,7 +32,7 @@ func main() {
 	flag.StringVar(&arch, "arch", "", "arch (x86_64 or aarch64)")
 	flag.Parse()
 
-	a, err := awsupload.New(region, accessKeyID, secretAccessKey, sessionToken)
+	a, err := awscloud.New(region, accessKeyID, secretAccessKey, sessionToken)
 	if err != nil {
 		println(err.Error())
 		return

cmd/osbuild-worker/jobimpl-osbuild.go

@@ -11,11 +11,11 @@ import (
 
 	"github.com/google/uuid"
 
+	"github.com/osbuild/osbuild-composer/internal/cloud/awscloud"
 	"github.com/osbuild/osbuild-composer/internal/cloud/gcp"
 	"github.com/osbuild/osbuild-composer/internal/common"
 	osbuild "github.com/osbuild/osbuild-composer/internal/osbuild2"
 	"github.com/osbuild/osbuild-composer/internal/target"
-	"github.com/osbuild/osbuild-composer/internal/upload/awsupload"
 	"github.com/osbuild/osbuild-composer/internal/upload/azure"
 	"github.com/osbuild/osbuild-composer/internal/upload/koji"
 	"github.com/osbuild/osbuild-composer/internal/upload/vmware"
@@ -37,14 +37,14 @@ func appendTargetError(res *worker.OSBuildJobResult, err error) {
 	res.TargetErrors = append(res.TargetErrors, errStr)
 }
 
-// Returns an *awsupload.AWS object with the credentials of the request. If they
+// Returns an *awscloud.AWS object with the credentials of the request. If they
 // are not accessible, then try to use the one obtained in the worker
 // configuration.
-func (impl *OSBuildJobImpl) getAWS(region string, accessId string, secret string, token string) (*awsupload.AWS, error) {
+func (impl *OSBuildJobImpl) getAWS(region string, accessId string, secret string, token string) (*awscloud.AWS, error) {
 	if accessId != "" && secret != "" {
-		return awsupload.New(region, accessId, secret, token)
+		return awscloud.New(region, accessId, secret, token)
 	} else {
-		return awsupload.NewFromFile(impl.AWSCreds, region)
+		return awscloud.NewFromFile(impl.AWSCreds, region)
 	}
 }