particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

internal/awscloud: retry CreateFleet

Browse source 
When receiving the "UnfillableCapacity" error from CreateFleet, retry
the request with an OnDemand instance.
This commit is contained in:
Sanne Raymaekers 2024-06-20 11:40:29 +02:00
parent 55c5602f91
commit ae4467ab0d
1 changed files with 33 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

48
internal/cloud/awscloud/secure-instance.go
View file

@ -104,7 +104,7 @@ func (a *AWS) RunSecureInstance(iamProfile, keyName, CloudWatchGroup string) (*S
return nil, fmt.Errorf("Expected at least 1 subnet in the VPC, got 0")
}
createFleetOutput, err := a.ec2.CreateFleet(&ec2.CreateFleetInput{
createFleetOutput, err := a.createFleet(&ec2.CreateFleetInput{
LaunchTemplateConfigs: []*ec2.FleetLaunchTemplateConfigRequest{
&ec2.FleetLaunchTemplateConfigRequest{
LaunchTemplateSpecification: &ec2.FleetLaunchTemplateSpecificationRequest{
@ -141,22 +141,8 @@ func (a *AWS) RunSecureInstance(iamProfile, keyName, CloudWatchGroup string) (*S
if err != nil {
return nil, err
}
if len(createFleetOutput.Errors) > 0 {
fleetErrs := []string{}
for _, fleetErr := range createFleetOutput.Errors {
fleetErrs = append(fleetErrs, *fleetErr.ErrorMessage)
}
return nil, fmt.Errorf("Unable to create fleet: %v", strings.Join(fleetErrs, "; "))
}
secureInstance.FleetID = *createFleetOutput.FleetId
if len(createFleetOutput.Instances) != 1 {
return nil, fmt.Errorf("Unable to create fleet with exactly one instance, got %d instances", len(createFleetOutput.Instances))
}
if len(createFleetOutput.Instances[0].InstanceIds) != 1 {
return nil, fmt.Errorf("Expected exactly one instance ID on fleet %v, got %d", secureInstance.FleetID, len(createFleetOutput.Instances[0].InstanceIds))
}
instanceID := createFleetOutput.Instances[0].InstanceIds[0]
err = a.ec2.WaitUntilInstanceStatusOk(&ec2.DescribeInstanceStatusInput{
InstanceIds: []*string{
@ -436,3 +422,35 @@ func (a *AWS) deleteSGIfExists(si *SecureInstance) error {
}
return err
}
func (a *AWS) createFleet(input *ec2.CreateFleetInput) (*ec2.CreateFleetOutput, error) {
createFleetOutput, err := a.ec2.CreateFleet(input)
if err != nil {
return nil, fmt.Errorf("Unable to create spot fleet: %w", err)
}
if len(createFleetOutput.Errors) > 0 && createFleetOutput.Errors[0].ErrorCode == aws.String("UnfillableCapacity") {
logrus.Warn("Received UnfillableCapacity from CreateFleet, retrying CreateFleet with OnDemand instance")
input.SpotOptions = nil
createFleetOutput, err = a.ec2.CreateFleet(input)
}
if err != nil {
return nil, fmt.Errorf("Unable to create on-demand fleet: %w", err)
}
if len(createFleetOutput.Errors) > 0 {
fleetErrs := []string{}
for _, fleetErr := range createFleetOutput.Errors {
fleetErrs = append(fleetErrs, *fleetErr.ErrorMessage)
}
return nil, fmt.Errorf("Unable to create fleet: %v", strings.Join(fleetErrs, "; "))
}
if len(createFleetOutput.Instances) != 1 {
return nil, fmt.Errorf("Unable to create fleet with exactly one instance, got %d instances", len(createFleetOutput.Instances))
}
if len(createFleetOutput.Instances[0].InstanceIds) != 1 {
return nil, fmt.Errorf("Expected exactly one instance ID on fleet, got %d", len(createFleetOutput.Instances[0].InstanceIds))
}
return createFleetOutput, nil
}