particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

osbuild-service-maintenance: clean up all regions

Browse source 
Since we started cloning images to different regions, the maintenance
script should clean up all of these regions.
This commit is contained in:
Sanne Raymaekers 2023-01-16 15:38:34 +01:00
parent dc1a770c0f
commit 2e3dd16220
3 changed files with 67 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

92
cmd/osbuild-service-maintenance/aws.go
View file

@ -11,62 +11,76 @@ import (
"github.com/osbuild/osbuild-composer/internal/cloud/awscloud"
)
func AWSCleanup(maxConcurrentRequests int, dryRun bool, accessKeyID, accessKey, region string, cutoff time.Time) error {
a, err := awscloud.New(region, accessKeyID, accessKey, "")
func AWSCleanup(maxConcurrentRequests int, dryRun bool, accessKeyID, accessKey string, cutoff time.Time) error {
a, err := awscloud.New("us-east-1", accessKeyID, accessKey, "")
if err != nil {
return err
}
var wg sync.WaitGroup
sem := semaphore.NewWeighted(int64(maxConcurrentRequests))
images, err := a.DescribeImagesByTag("Name", "composer-api-*")
regions, err := a.Regions()
if err != nil {
return err
}
for index, image := range images {
// TODO are these actual concerns?
if image.ImageId == nil {
logrus.Infof("ImageId is nil %v", image)
continue
}
if image.CreationDate == nil {
logrus.Infof("Image %v has nil creationdate", *image.ImageId)
continue
}
created, err := time.Parse(time.RFC3339, *image.CreationDate)
for _, region := range regions {
a, err := awscloud.New(region, accessKeyID, accessKey, "")
if err != nil {
logrus.Infof("Unable to parse date %s for image %s", *image.CreationDate, *image.ImageId)
logrus.Errorf("Unable to create new aws session for region %s: %v", region, err)
continue
}
if !created.Before(cutoff) {
var wg sync.WaitGroup
sem := semaphore.NewWeighted(int64(maxConcurrentRequests))
images, err := a.DescribeImagesByTag("Name", "composer-api-*")
if err != nil {
logrus.Errorf("Unable to describe images for region %s: %v", region, err)
continue
}
if dryRun {
logrus.Infof("Dry run, aws image %s in region %s, with creation date %s would be removed", *image.ImageId, region, *image.CreationDate)
continue
}
if err = sem.Acquire(context.Background(), 1); err != nil {
logrus.Errorf("Error acquiring semaphore: %v", err)
continue
}
wg.Add(1)
go func(i int) {
defer sem.Release(1)
defer wg.Done()
err := a.RemoveSnapshotAndDeregisterImage(images[i])
if err != nil {
logrus.Errorf("Cleanup for image %s in region %s failed: %v", *images[i].ImageId, region, err)
for index, image := range images {
// TODO are these actual concerns?
if image.ImageId == nil {
logrus.Infof("ImageId is nil %v", image)
continue
}
}(index)
if image.CreationDate == nil {
logrus.Infof("Image %v has nil creationdate", *image.ImageId)
continue
}
created, err := time.Parse(time.RFC3339, *image.CreationDate)
if err != nil {
logrus.Infof("Unable to parse date %s for image %s", *image.CreationDate, *image.ImageId)
continue
}
if !created.Before(cutoff) {
continue
}
if dryRun {
logrus.Infof("Dry run, aws image %s in region %s, with creation date %s would be removed", *image.ImageId, region, *image.CreationDate)
continue
}
if err = sem.Acquire(context.Background(), 1); err != nil {
logrus.Errorf("Error acquiring semaphore: %v", err)
continue
}
wg.Add(1)
go func(i int) {
defer sem.Release(1)
defer wg.Done()
err := a.RemoveSnapshotAndDeregisterImage(images[i])
if err != nil {
logrus.Errorf("Cleanup for image %s in region %s failed: %v", *images[i].ImageId, region, err)
}
}(index)
}
wg.Wait()
}
wg.Wait()
return nil
}

2
cmd/osbuild-service-maintenance/main.go
View file

@ -40,7 +40,7 @@ func main() {
}
logrus.Info("Cleaning up AWS")
err := AWSCleanup(conf.MaxConcurrentRequests, conf.DryRun, conf.AWSAccessKeyID, conf.AWSSecretAccessKey, "us-east-1", cutoff)
err := AWSCleanup(conf.MaxConcurrentRequests, conf.DryRun, conf.AWSAccessKeyID, conf.AWSSecretAccessKey, cutoff)
if err != nil {
logrus.Errorf("AWS cleanup failed: %v", err)
}