internal/boot: introduce package for booting images

The package takes the existing code from /cmd/osbuild-image-tests and
makes it available for other executables.
This commit is contained in:
Martin Sehnoutka 2020-09-01 10:14:52 +02:00 committed by Tom Gundersen
parent 7e0711b805
commit ec6ce8387d
9 changed files with 8 additions and 8 deletions

281
internal/boot/aws.go Normal file
View file

@ -0,0 +1,281 @@
// +build integration
package boot
import (
"encoding/base64"
"errors"
"fmt"
"io/ioutil"
"os"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/osbuild/osbuild-composer/internal/upload/awsupload"
)
type awsCredentials struct {
AccessKeyId string
SecretAccessKey string
Region string
Bucket string
}
// getAWSCredentialsFromEnv gets the credentials from environment variables
// If none of the environment variables is set, it returns nil.
// If some but not all environment variables are set, it returns an error.
func getAWSCredentialsFromEnv() (*awsCredentials, error) {
accessKeyId, akExists := os.LookupEnv("AWS_ACCESS_KEY_ID")
secretAccessKey, sakExists := os.LookupEnv("AWS_SECRET_ACCESS_KEY")
region, regionExists := os.LookupEnv("AWS_REGION")
bucket, bucketExists := os.LookupEnv("AWS_BUCKET")
// If non of the variables is set, just ignore the test
if !akExists && !sakExists && !bucketExists && !regionExists {
return nil, nil
}
// If only one/two of them are not set, then fail
if !akExists || !sakExists || !bucketExists || !regionExists {
return nil, errors.New("not all required env variables were set")
}
return &awsCredentials{
AccessKeyId: accessKeyId,
SecretAccessKey: secretAccessKey,
Region: region,
Bucket: bucket,
}, nil
}
// encodeBase64 encodes string to base64-encoded string
func encodeBase64(input string) string {
return base64.StdEncoding.EncodeToString([]byte(input))
}
// createUserData creates cloud-init's user-data that contains user redhat with
// the specified public key
func createUserData(publicKeyFile string) (string, error) {
publicKey, err := ioutil.ReadFile(publicKeyFile)
if err != nil {
return "", fmt.Errorf("cannot read the public key: %#v", err)
}
userData := fmt.Sprintf(`#cloud-config
user: redhat
ssh_authorized_keys:
- %s
`, string(publicKey))
return userData, nil
}
// wrapErrorf returns error constructed using fmt.Errorf from format and any
// other args. If innerError != nil, it's appended at the end of the new
// error.
func wrapErrorf(innerError error, format string, a ...interface{}) error {
if innerError != nil {
a = append(a, innerError)
return fmt.Errorf(format+"\n\ninner error: %#s", a...)
}
return fmt.Errorf(format, a...)
}
// uploadImageToAWS mimics the upload feature of osbuild-composer.
// It takes an image and an image name and creates an ec2 instance from them.
// The s3 key is never returned - the same thing is done in osbuild-composer,
// the user has no way of getting the s3 key.
func uploadImageToAWS(c *awsCredentials, imagePath string, imageName string) error {
uploader, err := awsupload.New(c.Region, c.AccessKeyId, c.SecretAccessKey)
if err != nil {
return fmt.Errorf("cannot create aws uploader: %#v", err)
}
_, err = uploader.Upload(imagePath, c.Bucket, imageName)
if err != nil {
return fmt.Errorf("cannot upload the image: %#v", err)
}
_, err = uploader.Register(imageName, c.Bucket, imageName)
if err != nil {
return fmt.Errorf("cannot register the image: %#v", err)
}
return nil
}
// newEC2 creates EC2 struct from given credentials
func newEC2(c *awsCredentials) (*ec2.EC2, error) {
creds := credentials.NewStaticCredentials(c.AccessKeyId, c.SecretAccessKey, "")
sess, err := session.NewSession(&aws.Config{
Credentials: creds,
Region: aws.String(c.Region),
})
if err != nil {
return nil, fmt.Errorf("cannot create aws session: %#v", err)
}
return ec2.New(sess), nil
}
type imageDescription struct {
Id *string
SnapshotId *string
// this doesn't support multiple snapshots per one image,
// because this feature is not supported in composer
}
// describeEC2Image searches for EC2 image by its name and returns
// its id and snapshot id
func describeEC2Image(e *ec2.EC2, imageName string) (*imageDescription, error) {
imageDescriptions, err := e.DescribeImages(&ec2.DescribeImagesInput{
Filters: []*ec2.Filter{
{
Name: aws.String("name"),
Values: []*string{
aws.String(imageName),
},
},
},
})
if err != nil {
return nil, fmt.Errorf("cannot describe the image: %#v", err)
}
imageId := imageDescriptions.Images[0].ImageId
snapshotId := imageDescriptions.Images[0].BlockDeviceMappings[0].Ebs.SnapshotId
return &imageDescription{
Id: imageId,
SnapshotId: snapshotId,
}, nil
}
// deleteEC2Image deletes the specified image and its associated snapshot
func deleteEC2Image(e *ec2.EC2, imageDesc *imageDescription) error {
var retErr error
// firstly, deregister the image
_, err := e.DeregisterImage(&ec2.DeregisterImageInput{
ImageId: imageDesc.Id,
})
if err != nil {
retErr = wrapErrorf(retErr, "cannot deregister the image: %#v", err)
}
// now it's possible to delete the snapshot
_, err = e.DeleteSnapshot(&ec2.DeleteSnapshotInput{
SnapshotId: imageDesc.SnapshotId,
})
if err != nil {
retErr = wrapErrorf(retErr, "cannot delete the snapshot: %#v", err)
}
return retErr
}
// withBootedImageInEC2 runs the function f in the context of booted
// image in AWS EC2
func withBootedImageInEC2(e *ec2.EC2, imageDesc *imageDescription, publicKey string, f func(address string) error) (retErr error) {
// generate user data with given public key
userData, err := createUserData(publicKey)
if err != nil {
return err
}
// Security group must be now generated, because by default
// all traffic to EC2 instance is filtered.
securityGroupName, err := generateRandomString("osbuild-image-tests-security-group-")
if err != nil {
return fmt.Errorf("cannot generate a random name for the image: %#v", err)
}
// Firstly create a security group
securityGroup, err := e.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{
GroupName: aws.String(securityGroupName),
Description: aws.String("image-tests-security-group"),
})
if err != nil {
return fmt.Errorf("cannot create a new security group: %#v", err)
}
defer func() {
_, err = e.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{
GroupId: securityGroup.GroupId,
})
if err != nil {
retErr = wrapErrorf(retErr, "cannot delete the security group: %#v", err)
}
}()
// Authorize incoming SSH connections.
_, err = e.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
CidrIp: aws.String("0.0.0.0/0"),
GroupId: securityGroup.GroupId,
FromPort: aws.Int64(22),
ToPort: aws.Int64(22),
IpProtocol: aws.String("tcp"),
})
if err != nil {
return fmt.Errorf("canot add a rule to the security group: %#v", err)
}
// Finally, run the instance from the given image and with the created security group
res, err := e.RunInstances(&ec2.RunInstancesInput{
MaxCount: aws.Int64(1),
MinCount: aws.Int64(1),
ImageId: imageDesc.Id,
InstanceType: aws.String("t3.micro"),
SecurityGroupIds: []*string{securityGroup.GroupId},
UserData: aws.String(encodeBase64(userData)),
})
if err != nil {
return fmt.Errorf("cannot create a new instance: %#v", err)
}
describeInstanceInput := &ec2.DescribeInstancesInput{
InstanceIds: []*string{
res.Instances[0].InstanceId,
},
}
defer func() {
// We need to terminate the instance now and wait until the termination is done.
// Otherwise, it wouldn't be possible to delete the image.
_, err = e.TerminateInstances(&ec2.TerminateInstancesInput{
InstanceIds: []*string{
res.Instances[0].InstanceId,
},
})
if err != nil {
retErr = wrapErrorf(retErr, "cannot terminate the instance: %#v", err)
return
}
err = e.WaitUntilInstanceTerminated(describeInstanceInput)
if err != nil {
retErr = wrapErrorf(retErr, "waiting for the instance termination failed: %#v", err)
}
}()
// The instance has no IP address yet. It's assigned when the instance
// is in the state "EXISTS". However, in this state the instance is not
// much usable, therefore wait until "RUNNING" state, in which the instance
// actually can do something useful for us.
err = e.WaitUntilInstanceRunning(describeInstanceInput)
if err != nil {
return fmt.Errorf("waiting for the instance to be running failed: %#v", err)
}
// By describing the instance, we can get the ip address.
out, err := e.DescribeInstances(describeInstanceInput)
if err != nil {
return fmt.Errorf("cannot describe the instance: %#v", err)
}
return f(*out.Reservations[0].Instances[0].PublicIpAddress)
}

View file

@ -0,0 +1,325 @@
// +build integration
package azuretest
import (
"context"
"errors"
"fmt"
"io/ioutil"
"log"
"net/url"
"os"
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-09-01/network"
"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-05-01/resources"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/Azure/go-autorest/autorest/azure/auth"
"github.com/osbuild/osbuild-composer/internal/upload/azure"
)
// wrapErrorf returns error constructed using fmt.Errorf from format and any
// other args. If innerError != nil, it's appended at the end of the new
// error.
func wrapErrorf(innerError error, format string, a ...interface{}) error {
if innerError != nil {
a = append(a, innerError)
return fmt.Errorf(format+"\n\ninner error: %#s", a...)
}
return fmt.Errorf(format, a...)
}
type azureCredentials struct {
azure.Credentials
ContainerName string
SubscriptionID string
ClientID string
ClientSecret string
TenantID string
Location string
ResourceGroup string
}
// getAzureCredentialsFromEnv gets the credentials from environment variables
// If none of the environment variables is set, it returns nil.
// If some but not all environment variables are set, it returns an error.
func GetAzureCredentialsFromEnv() (*azureCredentials, error) {
storageAccount, saExists := os.LookupEnv("AZURE_STORAGE_ACCOUNT")
storageAccessKey, sakExists := os.LookupEnv("AZURE_STORAGE_ACCESS_KEY")
containerName, cExists := os.LookupEnv("AZURE_CONTAINER_NAME")
subscriptionId, siExists := os.LookupEnv("AZURE_SUBSCRIPTION_ID")
clientId, ciExists := os.LookupEnv("AZURE_CLIENT_ID")
clientSecret, csExists := os.LookupEnv("AZURE_CLIENT_SECRET")
tenantId, tiExists := os.LookupEnv("AZURE_TENANT_ID")
location, lExists := os.LookupEnv("AZURE_LOCATION")
resourceGroup, rgExists := os.LookupEnv("AZURE_RESOURCE_GROUP")
// If non of the variables is set, just ignore the test
if !saExists && !sakExists && !cExists && !siExists && !ciExists && !csExists && !tiExists && !lExists && !rgExists {
return nil, nil
}
// If only one/two of them are not set, then fail
if !saExists || !sakExists || !cExists || !siExists || !ciExists || !csExists || !tiExists || !lExists || !rgExists {
return nil, errors.New("not all required env variables were set")
}
return &azureCredentials{
Credentials: azure.Credentials{
StorageAccount: storageAccount,
StorageAccessKey: storageAccessKey,
},
ContainerName: containerName,
SubscriptionID: subscriptionId,
ClientID: clientId,
ClientSecret: clientSecret,
TenantID: tenantId,
Location: location,
ResourceGroup: resourceGroup,
}, nil
}
// UploadImageToAzure mimics the upload feature of osbuild-composer.
func UploadImageToAzure(c *azureCredentials, imagePath string, imageName string) error {
metadata := azure.ImageMetadata{
ContainerName: c.ContainerName,
ImageName: imageName,
}
err := azure.UploadImage(c.Credentials, metadata, imagePath, 16)
if err != nil {
return fmt.Errorf("upload to azure failed: %v", err)
}
return nil
}
// DeleteImageFromAzure deletes the image uploaded by osbuild-composer
// (or UpluadImageToAzure method).
func DeleteImageFromAzure(c *azureCredentials, imageName string) error {
// Create a default request pipeline using your storage account name and account key.
credential, err := azblob.NewSharedKeyCredential(c.StorageAccount, c.StorageAccessKey)
if err != nil {
return err
}
p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
// get storage account blob service URL endpoint.
URL, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s", c.StorageAccount, c.ContainerName))
// Create a ContainerURL object that wraps the container URL and a request
// pipeline to make requests.
containerURL := azblob.NewContainerURL(*URL, p)
// Create the container, use a never-expiring context
ctx := context.Background()
blobURL := containerURL.NewPageBlobURL(imageName)
_, err = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{})
if err != nil {
return fmt.Errorf("cannot delete the image: %v", err)
}
return nil
}
// readPublicKey reads the public key from a file and returns it as a string
func readPublicKey(publicKeyFile string) (string, error) {
publicKey, err := ioutil.ReadFile(publicKeyFile)
if err != nil {
return "", fmt.Errorf("cannot read the public key file: %v", err)
}
return string(publicKey), nil
}
// deleteResource is a convenient wrapper around Azure SDK to delete a resource
func deleteResource(client resources.Client, id string, apiVersion string) error {
deleteFuture, err := client.DeleteByID(context.Background(), id, apiVersion)
if err != nil {
return fmt.Errorf("cannot delete the resourceType %s: %v", id, err)
}
err = deleteFuture.WaitForCompletionRef(context.Background(), client.BaseClient.Client)
if err != nil {
return fmt.Errorf("waiting for the resourceType %s deletion failed: %v", id, err)
}
_, err = deleteFuture.Result(client)
if err != nil {
return fmt.Errorf("cannot retrieve the result of %s deletion: %v", id, err)
}
return nil
}
// withBootedImageInAzure runs the function f in the context of booted
// image in Azure
func WithBootedImageInAzure(creds *azureCredentials, imageName, testId, publicKeyFile string, f func(address string) error) (retErr error) {
publicKey, err := readPublicKey(publicKeyFile)
if err != nil {
return err
}
clientCredentialsConfig := auth.NewClientCredentialsConfig(creds.ClientID, creds.ClientSecret, creds.TenantID)
authorizer, err := clientCredentialsConfig.Authorizer()
if err != nil {
return fmt.Errorf("cannot create the authorizer: %v", err)
}
template, err := loadDeploymentTemplate()
if err != nil {
return err
}
// Azure requires a lot of names - for a virtual machine, a virtual network,
// a virtual interface and so on and so forth.
// Let's create all of them here from the test id so we can delete them
// later.
deploymentName := testId
imagePath := fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", creds.StorageAccount, creds.ContainerName, imageName)
parameters := deploymentParameters{
NetworkInterfaceName: newDeploymentParameter("iface-" + testId),
NetworkSecurityGroupName: newDeploymentParameter("nsg-" + testId),
VirtualNetworkName: newDeploymentParameter("vnet-" + testId),
PublicIPAddressName: newDeploymentParameter("ip-" + testId),
VirtualMachineName: newDeploymentParameter("vm-" + testId),
DiskName: newDeploymentParameter("disk-" + testId),
ImageName: newDeploymentParameter("image-" + testId),
Location: newDeploymentParameter(creds.Location),
ImagePath: newDeploymentParameter(imagePath),
AdminUsername: newDeploymentParameter("redhat"),
AdminPublicKey: newDeploymentParameter(publicKey),
}
deploymentsClient := resources.NewDeploymentsClient(creds.SubscriptionID)
deploymentsClient.Authorizer = authorizer
deploymentFuture, err := deploymentsClient.CreateOrUpdate(context.Background(), creds.ResourceGroup, deploymentName, resources.Deployment{
Properties: &resources.DeploymentProperties{
Mode: resources.Incremental,
Template: template,
Parameters: parameters,
},
})
// Let's registed the clean-up function as soon as possible.
defer func() {
resourcesClient := resources.NewClient(creds.SubscriptionID)
resourcesClient.Authorizer = authorizer
// This array specifies all the resources we need to delete. The
// order is important, e.g. one cannot delete a network interface
// that is still attached to a virtual machine.
resourcesToDelete := []struct {
resType string
name string
apiVersion string
}{
{
resType: "Microsoft.Compute/virtualMachines",
name: parameters.VirtualMachineName.Value,
apiVersion: "2019-07-01",
},
{
resType: "Microsoft.Network/networkInterfaces",
name: parameters.NetworkInterfaceName.Value,
apiVersion: "2019-09-01",
},
{
resType: "Microsoft.Network/publicIPAddresses",
name: parameters.PublicIPAddressName.Value,
apiVersion: "2019-09-01",
},
{
resType: "Microsoft.Network/networkSecurityGroups",
name: parameters.NetworkSecurityGroupName.Value,
apiVersion: "2019-09-01",
},
{
resType: "Microsoft.Network/virtualNetworks",
name: parameters.VirtualNetworkName.Value,
apiVersion: "2019-09-01",
},
{
resType: "Microsoft.Compute/disks",
name: parameters.DiskName.Value,
apiVersion: "2019-07-01",
},
{
resType: "Microsoft.Compute/images",
name: parameters.ImageName.Value,
apiVersion: "2019-07-01",
},
}
// Delete all the resources
for _, resourceToDelete := range resourcesToDelete {
resourceID := fmt.Sprintf(
"subscriptions/%s/resourceGroups/%s/providers/%s/%s",
creds.SubscriptionID,
creds.ResourceGroup,
resourceToDelete.resType,
resourceToDelete.name,
)
err := deleteResource(resourcesClient, resourceID, resourceToDelete.apiVersion)
if err != nil {
log.Printf("deleting the resource %s errored: %v", resourceToDelete.name, err)
retErr = wrapErrorf(retErr, "cannot delete the resource %s: %v", resourceToDelete.name, err)
// do not return here, try deleting as much as possible
}
}
// Delete the deployment
// This actually does not delete any resources created by the
// deployment as one might think. Therefore the code above
// is needed.
result, err := deploymentsClient.Delete(context.Background(), creds.ResourceGroup, deploymentName)
if err != nil {
retErr = wrapErrorf(retErr, "cannot create the request for the deployment deletion: %v", err)
return
}
err = result.WaitForCompletionRef(context.Background(), deploymentsClient.Client)
if err != nil {
retErr = wrapErrorf(retErr, "waiting for the deployment deletion failed: %v", err)
return
}
_, err = result.Result(deploymentsClient)
if err != nil {
retErr = wrapErrorf(retErr, "cannot retrieve the deployment deletion result: %v", err)
return
}
}()
if err != nil {
return fmt.Errorf("creating a deployment failed: %v", err)
}
err = deploymentFuture.WaitForCompletionRef(context.Background(), deploymentsClient.Client)
if err != nil {
return fmt.Errorf("waiting for deployment completion failed: %v", err)
}
_, err = deploymentFuture.Result(deploymentsClient)
if err != nil {
return fmt.Errorf("retrieving the deployment result failed: %v", err)
}
// get the IP address
publicIPAddressClient := network.NewPublicIPAddressesClient(creds.SubscriptionID)
publicIPAddressClient.Authorizer = authorizer
publicIPAddress, err := publicIPAddressClient.Get(context.Background(), creds.ResourceGroup, parameters.PublicIPAddressName.Value, "")
if err != nil {
return fmt.Errorf("cannot get the ip address details: %v", err)
}
return f(*publicIPAddress.IPAddress)
}

View file

@ -0,0 +1,56 @@
// +build integration
package azuretest
import (
"encoding/json"
"fmt"
"os"
"github.com/osbuild/osbuild-composer/cmd/osbuild-image-tests/constants"
)
// loadDeploymentTemplate loads the deployment template from the specified
// path and return it as a "dynamically" typed object
func loadDeploymentTemplate() (interface{}, error) {
f, err := os.Open(constants.TestPaths.AzureDeploymentTemplate)
if err != nil {
return nil, fmt.Errorf("cannot open the deployment file: %v", err)
}
defer f.Close()
var result interface{}
err = json.NewDecoder(f).Decode(&result)
if err != nil {
return nil, fmt.Errorf("cannot decode the deployment file: %v", err)
}
return result, nil
}
// struct for encoding a deployment parameter
type deploymentParameter struct {
Value string `json:"value"`
}
func newDeploymentParameter(value string) deploymentParameter {
return deploymentParameter{Value: value}
}
// struct for encoding deployment parameters
type deploymentParameters struct {
NetworkInterfaceName deploymentParameter `json:"networkInterfaceName"`
NetworkSecurityGroupName deploymentParameter `json:"networkSecurityGroupName"`
VirtualNetworkName deploymentParameter `json:"virtualNetworkName"`
PublicIPAddressName deploymentParameter `json:"publicIPAddressName"`
VirtualMachineName deploymentParameter `json:"virtualMachineName"`
DiskName deploymentParameter `json:"diskName"`
ImageName deploymentParameter `json:"imageName"`
Location deploymentParameter `json:"location"`
ImagePath deploymentParameter `json:"imagePath"`
AdminUsername deploymentParameter `json:"adminUsername"`
AdminPublicKey deploymentParameter `json:"adminPublicKey"`
}

View file

@ -0,0 +1,271 @@
// +build integration
package boot
import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
"runtime"
"strconv"
"strings"
"time"
"github.com/osbuild/osbuild-composer/cmd/osbuild-image-tests/constants"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/distro"
)
// withNetworkNamespace provides the function f with a new network namespace
// which is deleted immediately after f returns
func withNetworkNamespace(f func(ns netNS) error) error {
ns, err := newNetworkNamespace()
if err != nil {
return err
}
defer func() {
err := ns.Delete()
if err != nil {
log.Printf("cannot delete network namespace: %#v", err)
}
}()
return f(ns)
}
// withTempFile provides the function f with a new temporary file
// dir and pattern parameters have the same semantics as in ioutil.TempFile
func withTempFile(dir, pattern string, f func(file *os.File) error) error {
tempFile, err := ioutil.TempFile(dir, pattern)
if err != nil {
return fmt.Errorf("cannot create the temporary file: %#v", err)
}
defer func() {
err := os.Remove(tempFile.Name())
if err != nil {
log.Printf("cannot remove the temporary file: %#v", err)
}
}()
return f(tempFile)
}
func withTempDir(dir, pattern string, f func(dir string) error) error {
tempDir, err := ioutil.TempDir(dir, pattern)
if err != nil {
return fmt.Errorf("cannot create the temporary directory %#v", err)
}
defer func() {
err := os.RemoveAll(tempDir)
if err != nil {
log.Printf("cannot remove the temporary directory: %#v", err)
}
}()
return f(tempDir)
}
// writeCloudInitSO creates cloud-init iso from specified userData and
// metaData and writes it to the writer
func writeCloudInitISO(writer io.Writer, userData, metaData string) error {
isoCmd := exec.Command(
"genisoimage",
"-quiet",
"-input-charset", "utf-8",
"-volid", "cidata",
"-joliet",
"-rock",
userData,
metaData,
)
isoCmd.Stdout = writer
isoCmd.Stderr = os.Stderr
err := isoCmd.Run()
if err != nil {
return fmt.Errorf("cannot create cloud-init iso: %#v", err)
}
return nil
}
// withBootedQemuImage boots the specified image in the specified namespace
// using qemu. The VM is killed immediately after function returns.
func withBootedQemuImage(image string, ns netNS, f func() error) error {
return withTempFile("", "osbuild-image-tests-cloudinit", func(cloudInitFile *os.File) error {
err := writeCloudInitISO(
cloudInitFile,
constants.TestPaths.UserData,
constants.TestPaths.MetaData,
)
if err != nil {
return err
}
err = cloudInitFile.Close()
if err != nil {
return fmt.Errorf("cannot close temporary cloudinit file: %#v", err)
}
var qemuCmd *exec.Cmd
if common.CurrentArch() == "x86_64" {
hostDistroName, _, err := distro.GetHostDistroName()
if err != nil {
return fmt.Errorf("cannot determing the current distro: %v", err)
}
var qemuPath string
if strings.HasPrefix(hostDistroName, "rhel") {
qemuPath = "/usr/libexec/qemu-kvm"
} else {
qemuPath = "qemu-system-x86_64"
}
qemuCmd = ns.NamespacedCommand(
qemuPath,
"-cpu", "host",
"-smp", strconv.Itoa(runtime.NumCPU()),
"-m", "1024",
"-snapshot",
"-M", "accel=kvm",
"-cdrom", cloudInitFile.Name(),
"-net", "nic,model=rtl8139", "-net", "user,hostfwd=tcp::22-:22",
"-nographic",
image,
)
} else if common.CurrentArch() == "aarch64" {
// This command does not use KVM as I was unable to make it work in Beaker,
// once we have machines that can use KVM, enable it to make it faster
qemuCmd = ns.NamespacedCommand(
"qemu-system-aarch64",
"-cpu", "host",
"-M", "virt",
"-m", "2048",
// As opposed to x86_64, aarch64 uses UEFI, this one comes from edk2-aarch64 package on Fedora
"-bios", "/usr/share/edk2/aarch64/QEMU_EFI.fd",
"-boot", "efi",
"-M", "accel=kvm",
"-snapshot",
"-cdrom", cloudInitFile.Name(),
"-net", "nic,model=rtl8139", "-net", "user,hostfwd=tcp::22-:22",
"-nographic",
image,
)
} else {
panic("Running on unknown architecture.")
}
err = qemuCmd.Start()
if err != nil {
return fmt.Errorf("cannot start the qemu process: %#v", err)
}
defer func() {
err := killProcessCleanly(qemuCmd.Process, time.Second)
if err != nil {
log.Printf("cannot kill the qemu process: %#v", err)
}
}()
return f()
})
}
// withBootedNspawnImage boots the specified image in the specified namespace
// using nspawn. The VM is killed immediately after function returns.
func withBootedNspawnImage(image string, ns netNS, f func() error) error {
cmd := exec.Command(
"systemd-nspawn",
"--boot", "--register=no",
"--image", image,
"--network-namespace-path", ns.Path(),
)
err := cmd.Start()
if err != nil {
return fmt.Errorf("cannot start the systemd-nspawn process: %#v", err)
}
defer func() {
err := killProcessCleanly(cmd.Process, time.Second)
if err != nil {
log.Printf("cannot kill the systemd-nspawn process: %#v", err)
}
}()
return f()
}
// withBootedNspawnImage boots the specified directory in the specified namespace
// using nspawn. The VM is killed immediately after function returns.
func withBootedNspawnDirectory(dir string, ns netNS, f func() error) error {
cmd := exec.Command(
"systemd-nspawn",
"--boot", "--register=no",
"--directory", dir,
"--network-namespace-path", ns.Path(),
)
err := cmd.Start()
if err != nil {
return fmt.Errorf("cannot start the systemd-nspawn process: %#v", err)
}
defer func() {
err := killProcessCleanly(cmd.Process, time.Second)
if err != nil {
log.Printf("cannot kill the systemd-nspawn process: %#v", err)
}
}()
return f()
}
// withExtractedTarArchive extracts the provided archive and passes
// a path to the result to the function f. The result is deleted
// immediately after the function returns.
func withExtractedTarArchive(archive string, f func(dir string) error) error {
return withTempDir("", "tar-archive", func(dir string) error {
cmd := exec.Command(
"tar",
"xf", archive,
"-C", dir,
)
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
err := cmd.Run()
if err != nil {
return fmt.Errorf("cannot untar the archive: %#v", err)
}
return f(dir)
})
}
// withSSHKeyPair runs the function f with a newly generated
// ssh key-pair, they key-pair is deleted immediately after
// the function f returns
func withSSHKeyPair(f func(privateKey, publicKey string) error) error {
return withTempDir("", "keys", func(dir string) error {
privateKey := dir + "/id_rsa"
publicKey := dir + "/id_rsa.pub"
cmd := exec.Command("ssh-keygen",
"-N", "",
"-f", privateKey,
)
err := cmd.Run()
if err != nil {
return fmt.Errorf("ssh-keygen failed: %#v", err)
}
return f(privateKey, publicKey)
})
}

65
internal/boot/helpers.go Normal file
View file

@ -0,0 +1,65 @@
// +build integration
package boot
import (
"log"
"os"
"syscall"
"time"
"github.com/google/uuid"
)
// durationMin returns the smaller of two given durations
func durationMin(a, b time.Duration) time.Duration {
if a < b {
return a
}
return b
}
// killProcessCleanly firstly sends SIGTERM to the process. If it still exists
// after the specified timeout, it sends SIGKILL
func killProcessCleanly(process *os.Process, timeout time.Duration) error {
err := process.Signal(syscall.SIGTERM)
if err != nil {
log.Printf("cannot send SIGTERM to process, sending SIGKILL instead: %#v", err)
return process.Kill()
}
const pollInterval = 10 * time.Millisecond
for {
p, err := os.FindProcess(process.Pid)
if err != nil {
return nil
}
err = p.Signal(syscall.Signal(0))
if err != nil {
return nil
}
sleep := durationMin(pollInterval, timeout)
if sleep == 0 {
break
}
timeout -= sleep
time.Sleep(sleep)
}
return process.Kill()
}
// generateRandomString generates a new random string with specified prefix.
// The random part is based on UUID.
func generateRandomString(prefix string) (string, error) {
id, err := uuid.NewRandom()
if err != nil {
return "", err
}
return prefix + id.String(), nil
}

148
internal/boot/netns.go Normal file
View file

@ -0,0 +1,148 @@
// +build integration
package boot
import (
"context"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"runtime"
"syscall"
"golang.org/x/sys/unix"
)
const netnsDir = "/var/run/netns"
// Network namespace abstraction
type netNS string
// newNetworkNamespace returns a new network namespace with a random
// name. The calling goroutine remains in the same namespace
// as before the call.
func newNetworkNamespace() (netNS, error) {
// This method needs to unshare the current thread. Go runtime can switch
// the goroutine to run on a different thread at any point, so we need
// to ensure that this method runs in the same thread for its whole
// lifetime.
runtime.LockOSThread()
defer runtime.UnlockOSThread()
_, err := os.Stat(netnsDir)
if err != nil {
if os.IsNotExist(err) {
err := os.Mkdir(netnsDir, 0755)
if err != nil {
return "", fmt.Errorf("cannot create %s: %#v", netnsDir, err)
}
} else {
return "", fmt.Errorf("cannot stat %s: %#v", netnsDir, err)
}
}
f, err := ioutil.TempFile(netnsDir, "osbuild-composer-namespace")
if err != nil {
return "", fmt.Errorf("cannot create a tempfile: %#v", err)
}
// We want to remove the temporary file if the namespace initialization fails.
// The best method I could thought of is to have the following variable
// denoting if the initialization was successful. It is set to true right
// before the end of this function.
initOK := false
defer func() {
if !initOK {
err := os.Remove(f.Name())
if err != nil {
log.Printf("cannot remove the temporary namespace: %#v", err)
}
}
}()
oldNS, err := os.Open("/proc/self/ns/net")
if err != nil {
return "", fmt.Errorf("cannot open the current namespace: %#v", err)
}
err = syscall.Unshare(syscall.CLONE_NEWNET)
if err != nil {
return "", fmt.Errorf("cannot unshare the network namespace")
}
defer func() {
err = unix.Setns(int(oldNS.Fd()), syscall.CLONE_NEWNET)
if err != nil {
// We cannot return to the original namespace.
// As we don't know nothing about affected threads, let's just
// quit immediately.
log.Fatalf("returning to the original namespace failed, quitting: %#v", err)
}
}()
cmd := exec.Command("ip", "link", "set", "up", "dev", "lo")
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stderr
err = cmd.Run()
if err != nil {
return "", fmt.Errorf("cannot set up a loopback device in the new namespace: %#v", err)
}
cmd = exec.Command("mount", "-o", "bind", "/proc/self/ns/net", f.Name())
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stderr
err = cmd.Run()
if err != nil {
return "", fmt.Errorf("cannot bind mount the new namespace: %#v", err)
}
ns := netNS(path.Base(f.Name()))
// Initialization OK, do not delete the namespace file.
initOK = true
return ns, nil
}
// NamespaceCommand returns an *exec.Cmd struct with the difference
// that it's prepended by "ip netns exec NAMESPACE_NAME" command, which
// runs the command in a namespaced environment.
func (n netNS) NamespacedCommand(name string, arg ...string) *exec.Cmd {
args := []string{"netns", "exec", string(n), name}
args = append(args, arg...)
return exec.Command("ip", args...)
}
// NamespaceCommand returns an *exec.Cmd struct with the difference
// that it's prepended by "ip netns exec NAMESPACE_NAME" command, which
// runs the command in a namespaced environment.
func (n netNS) NamespacedCommandContext(ctx context.Context, name string, arg ...string) *exec.Cmd {
args := []string{"netns", "exec", string(n), name}
args = append(args, arg...)
return exec.CommandContext(ctx, "ip", args...)
}
// Path returns the path to the namespace file
func (n netNS) Path() string {
return path.Join(netnsDir, string(n))
}
// Delete deletes the namespaces
func (n netNS) Delete() error {
cmd := exec.Command("umount", n.Path())
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
err := cmd.Run()
if err != nil {
return fmt.Errorf("cannot unmount the network namespace: %#v", err)
}
err = os.Remove(n.Path())
if err != nil {
return fmt.Errorf("cannot delete the network namespace file: %#v", err)
}
return nil
}

View file

@ -0,0 +1,133 @@
// +build integration
package openstacktest
import (
"fmt"
"os"
"github.com/gophercloud/gophercloud"
"github.com/gophercloud/gophercloud/openstack"
"github.com/gophercloud/gophercloud/openstack/imageservice/v2/images"
"github.com/gophercloud/gophercloud/openstack/imageservice/v2/imagedata"
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
)
const WaitTimeout = 600 // in seconds
func UploadImageToOpenStack(p *gophercloud.ProviderClient, imagePath string, imageName string) (*images.Image, error) {
client, err := openstack.NewImageServiceV2(p, gophercloud.EndpointOpts{
Region: os.Getenv("OS_REGION_NAME"),
})
if err != nil {
return nil, fmt.Errorf("Error creating ImageService client: %v", err)
}
// create a new image which gives us the ID
image, err := images.Create(client, images.CreateOpts{
Name: imageName,
DiskFormat: "qcow2",
ContainerFormat: "bare",
}).Extract()
if err != nil {
return image, fmt.Errorf("Creating image failed: %v", err)
}
// then upload the actual binary data
imageData, err := os.Open(imagePath)
if err != nil {
return image, fmt.Errorf("Error opening %s: %v", imagePath, err)
}
defer imageData.Close()
err = imagedata.Upload(client, image.ID, imageData).ExtractErr()
if err != nil {
return image, fmt.Errorf("Upload to OpenStack failed: %v", err)
}
// wait for the status to change from Queued to Active
err = gophercloud.WaitFor(WaitTimeout, func() (bool, error) {
actual, err := images.Get(client, image.ID).Extract()
return actual.Status == images.ImageStatusActive, err
})
if err != nil {
return image, fmt.Errorf("Waiting for image to become Active failed: %v", err)
}
return image, nil
}
func DeleteImageFromOpenStack(p *gophercloud.ProviderClient, imageUUID string) error {
client, err := openstack.NewImageServiceV2(p, gophercloud.EndpointOpts{
Region: os.Getenv("OS_REGION_NAME"),
})
if err != nil {
return fmt.Errorf("Error creating ImageService client: %v", err)
}
err = images.Delete(client, imageUUID).ExtractErr()
if err != nil {
return fmt.Errorf("cannot delete the image: %v", err)
}
return nil
}
func WithBootedImageInOpenStack(p *gophercloud.ProviderClient, imageID, userData string, f func(address string) error) (retErr error) {
client, err := openstack.NewComputeV2(p, gophercloud.EndpointOpts{
Region: os.Getenv("OS_REGION_NAME"),
})
if err != nil {
return fmt.Errorf("Error creating Compute client: %v", err)
}
server, err := servers.Create(client, servers.CreateOpts{
Name: "osbuild-composer-vm-for-" + imageID,
FlavorRef: "77b8cf27-be16-40d9-95b1-81db4522be1e", // ci.m1.medium.ephemeral
Networks: []servers.Network{ // provider_net_cci_2
servers.Network{UUID: "74e8faa7-87ba-41b2-a000-438013194814"},
},
ImageRef: imageID,
UserData: []byte(userData),
}).Extract()
if err != nil {
return fmt.Errorf("Cannot create instance: %v", err)
}
// cleanup
defer func(){
err := servers.ForceDelete(client, server.ID).ExtractErr()
if err != nil {
fmt.Printf("Force deleting instance %s failed: %v", server.ID, err)
return
}
}()
// wait for the status to become Active
err = servers.WaitForStatus(client, server.ID, "ACTIVE", WaitTimeout)
if err != nil {
return fmt.Errorf("Waiting for instance %s to become Active failed: %v", server.ID, err)
}
// get server details again to refresh the IP addresses
server, err = servers.Get(client, server.ID).Extract()
if err != nil {
return fmt.Errorf("Cannot get instance details: %v\n", err)
}
// server.AccessIPv4 is empty so list all addresses and
// get the first fixed one. ssh should be equally happy with v4 or v6
var fixedIP string
for _, networkAddresses := range server.Addresses["provider_net_cci_2"].([]interface{}) {
address := networkAddresses.(map[string]interface{})
if address["OS-EXT-IPS:type"] == "fixed" {
fixedIP = address["addr"].(string)
break
}
}
if fixedIP == "" {
return fmt.Errorf("Cannot find IP address for instance %s", server.ID)
}
return f(fixedIP)
}

View file

@ -0,0 +1,228 @@
// +build integration
package vmwaretest
import (
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
// importing the packages registers these cli commands
"github.com/vmware/govmomi/govc/cli"
_ "github.com/vmware/govmomi/govc/datastore"
_ "github.com/vmware/govmomi/govc/importx"
_ "github.com/vmware/govmomi/govc/vm"
_ "github.com/vmware/govmomi/govc/vm/guest"
)
const WaitTimeout = 6000 // in seconds
type AuthOptions struct {
Host string
Username string
Password string
Datacenter string
Cluster string
Network string
Datastore string
Folder string
}
func AuthOptionsFromEnv() (*AuthOptions, error) {
host, hostExists := os.LookupEnv("GOVMOMI_URL")
username, userExists := os.LookupEnv("GOVMOMI_USERNAME")
password, pwdExists := os.LookupEnv("GOVMOMI_PASSWORD")
datacenter, dcExists := os.LookupEnv("GOVMOMI_DATACENTER")
cluster, clusterExists := os.LookupEnv("GOVMOMI_CLUSTER")
network, netExists := os.LookupEnv("GOVMOMI_NETWORK")
datastore, dsExists := os.LookupEnv("GOVMOMI_DATASTORE")
folder, folderExists := os.LookupEnv("GOVMOMI_FOLDER")
// If only one/two of them are not set, then fail
if !hostExists {
return nil, errors.New("GOVMOMI_URL not set")
}
if !userExists {
return nil, errors.New("GOVMOMI_USERNAME not set")
}
if !pwdExists {
return nil, errors.New("GOVMOMI_PASSWORD not set")
}
if !dcExists {
return nil, errors.New("GOVMOMI_DATACENTER not set")
}
if !clusterExists {
return nil, errors.New("GOVMOMI_CLUSTER not set")
}
if !netExists {
return nil, errors.New("GOVMOMI_NETWORK not set")
}
if !dsExists {
return nil, errors.New("GOVMOMI_DATASTORE not set")
}
if !folderExists {
return nil, errors.New("GOVMOMI_FOLDER not set")
}
return &AuthOptions{
Host: host,
Username: username,
Password: password,
Datacenter: datacenter,
Cluster: cluster,
Network: network,
Datastore: datastore,
Folder: folder,
}, nil
}
func ImportImage(creds *AuthOptions, imagePath, imageName string) error {
args := []string{
"import.vmdk",
fmt.Sprintf("-u=%s:%s@%s", creds.Username, creds.Password, creds.Host),
"-k=true",
fmt.Sprintf("-pool=%s/Resources", creds.Cluster),
fmt.Sprintf("-dc=%s", creds.Datacenter),
fmt.Sprintf("-ds=%s", creds.Datastore),
imagePath,
imageName,
}
retcode := cli.Run(args)
if retcode != 0 {
return errors.New("importing vmdk failed")
}
return nil
}
func DeleteImage(creds *AuthOptions, directoryName string) error {
retcode := cli.Run([]string{
"datastore.rm",
"-f=true",
fmt.Sprintf("-u=%s:%s@%s", creds.Username, creds.Password, creds.Host),
"-k=true",
fmt.Sprintf("-dc=%s", creds.Datacenter),
fmt.Sprintf("-ds=%s", creds.Datastore),
directoryName + "*", // because vm.create creates another directory with _1 prefix
})
if retcode != 0 {
return errors.New("deleting directory failed")
}
return nil
}
func runWithStdout(args []string) (string, int) {
oldStdout := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
retcode := cli.Run(args)
w.Close()
out, _ := ioutil.ReadAll(r)
os.Stdout = oldStdout
return strings.TrimSpace(string(out)), retcode
}
func WithBootedImage(creds *AuthOptions, imagePath, imageName, publicKey string, f func(address string) error) (retErr error) {
vmdkBaseName := filepath.Base(imagePath)
args := []string{
"vm.create",
fmt.Sprintf("-u=%s:%s@%s", creds.Username, creds.Password, creds.Host),
"-k=true",
fmt.Sprintf("-pool=%s/Resources", creds.Cluster),
fmt.Sprintf("-dc=%s", creds.Datacenter),
fmt.Sprintf("-ds=%s", creds.Datastore),
fmt.Sprintf("-folder=%s", creds.Folder),
fmt.Sprintf("-net=%s", creds.Network),
"-m=2048", "-g=rhel8_64Guest", "-on=true", "-firmware=bios",
fmt.Sprintf("-disk=%s/%s", imageName, vmdkBaseName),
"--disk.controller=ide",
imageName,
}
retcode := cli.Run(args)
if retcode != 0 {
return errors.New("Creating VM from vmdk failed")
}
defer func() {
args = []string{
"vm.destroy",
fmt.Sprintf("-u=%s:%s@%s", creds.Username, creds.Password, creds.Host),
"-k=true",
imageName,
}
retcode := cli.Run(args)
if retcode != 0 {
fmt.Printf("Deleting VM %s failed", imageName)
return
}
}()
// note: by default this will wait/block until an IP address is returned
// note: using exec() instead of running the command b/c .Run() returns an int
args = []string{
"vm.ip",
fmt.Sprintf("-u=%s:%s@%s", creds.Username, creds.Password, creds.Host),
"-k=true",
imageName,
}
ipAddress, retcode := runWithStdout(args)
if retcode != 0 {
return errors.New("Getting IP address for VM failed")
}
// Disabled b/c of https://github.com/vmware/govmomi/issues/2054
// upload public key on the VM
//args = []string{
// "guest.mkdir",
// fmt.Sprintf("-u=%s:%s@%s", creds.Username, creds.Password, creds.Host),
// "-k=true",
// fmt.Sprintf("-vm=%s", imageName),
// "-p", "/root/.ssh",
//}
//retcode = cli.Run(args)
//if retcode != 0 {
// return errors.New("mkdir /root/.ssh on VM failed")
//}
//args = []string{
// "guest.upload",
// fmt.Sprintf("-u=%s:%s@%s", creds.Username, creds.Password, creds.Host),
// "-k=true",
// fmt.Sprintf("-vm=%s", imageName),
// "-f=true",
// publicKey, // this is a file path
// "/root/.ssh/authorized_keys",
//}
//retcode = cli.Run(args)
//if retcode != 0 {
// return errors.New("Uploading public key to VM failed")
//}
return f(ipAddress)
}
// hard-coded SSH keys b/c we're having troubles uploading publicKey
// to the VM, see https://github.com/vmware/govmomi/issues/2054
func WithSSHKeyPair(f func(privateKey, publicKey string) error) error {
public := "/usr/share/tests/osbuild-composer/keyring/id_rsa.pub"
private := "/usr/share/tests/osbuild-composer/keyring/id_rsa"
return f(private, public)
}