tests/image: add booting tests

This commit makes the osbuild-image-tests binary doing the same set of tests
like the old test/run script.

Changes from test/run:
- qemu/nspawn are now killed gracefully. Firstly, SIGTERM is sent.
  If the process doesn't exit till the timeout, SIGKILL is sent.
  I changed this because nspawn leaves some artifacts behind when killed
  by SIGKILL.
- the unsharing of network namespace now works differently because of
  systemd issue #15079

cmd/osbuild-image-tests/context-managers.go

@@ -0,0 +1,200 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"time"
+)
+
+// withNetworkNamespace provides the function f with a new network namespace
+// which is deleted immediately after f returns
+func withNetworkNamespace(f func(ns netNS) error) error {
+	ns, err := newNetworkNamespace()
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		err := ns.Delete()
+		if err != nil {
+			log.Printf("cannot delete network namespace: %v", err)
+		}
+	}()
+
+	return f(ns)
+}
+
+// withTempFile provides the function f with a new temporary file
+// dir and pattern parameters have the same semantics as in ioutil.TempFile
+func withTempFile(dir, pattern string, f func(file *os.File) error) error {
+	tempFile, err := ioutil.TempFile(dir, pattern)
+	if err != nil {
+		return fmt.Errorf("cannot create the temporary file: %v", err)
+	}
+
+	defer func() {
+		err := os.Remove(tempFile.Name())
+		if err != nil {
+			log.Printf("cannot remove the temporary file: %v", err)
+		}
+	}()
+
+	return f(tempFile)
+}
+
+// writeCloudInitSO creates cloud-init iso from specified userData and
+// metaData and writes it to the writer
+func writeCloudInitISO(writer io.Writer, userData, metaData string) error {
+	isoCmd := exec.Command(
+		"genisoimage",
+		"-quiet",
+		"-input-charset", "utf-8",
+		"-volid", "cidata",
+		"-joliet",
+		"-rock",
+		userData,
+		metaData,
+	)
+	isoCmd.Stdout = writer
+	isoCmd.Stderr = os.Stderr
+
+	err := isoCmd.Run()
+	if err != nil {
+		return fmt.Errorf("cannot create cloud-init iso: %v", err)
+	}
+
+	return nil
+}
+
+// withBootedQemuImage boots the specified image in the specified namespace
+// using qemu. The VM is killed immediately after function returns.
+func withBootedQemuImage(image string, ns netNS, f func() error) error {
+	return withTempFile("", "osbuild-image-tests-cloudinit", func(cloudInitFile *os.File) error {
+		err := writeCloudInitISO(
+			cloudInitFile,
+			"/usr/share/tests/osbuild-composer/cloud-init/user-data",
+			"/usr/share/tests/osbuild-composer/cloud-init/meta-data",
+		)
+		if err != nil {
+			return err
+		}
+
+		err = cloudInitFile.Close()
+		if err != nil {
+			return fmt.Errorf("cannot close temporary cloudinit file: %v", err)
+		}
+
+		qemuCmd := ns.NamespacedCommand(
+			"qemu-system-x86_64",
+			"-m", "2048",
+			"-snapshot",
+			"-accel", "accel=kvm:hvf:tcg",
+			"-cdrom", cloudInitFile.Name(),
+			"-net", "nic,model=rtl8139", "-net", "user,hostfwd=tcp::22-:22",
+			"-nographic",
+			image,
+		)
+
+		err = qemuCmd.Start()
+		if err != nil {
+			return fmt.Errorf("cannot start the qemu process: %v", err)
+		}
+
+		defer func() {
+			err := killProcessCleanly(qemuCmd.Process, time.Second)
+			if err != nil {
+				log.Printf("cannot kill the qemu process: %v", err)
+			}
+		}()
+
+		return f()
+	})
+}
+
+// withBootedNspawnImage boots the specified image in the specified namespace
+// using nspawn. The VM is killed immediately after function returns.
+func withBootedNspawnImage(image, name string, ns netNS, f func() error) error {
+	cmd := exec.Command(
+		"systemd-nspawn",
+		"--boot", "--register=no",
+		"-M", name,
+		"--image", image,
+		"--network-namespace-path", ns.Path(),
+	)
+
+	err := cmd.Start()
+	if err != nil {
+		return fmt.Errorf("cannot start the systemd-nspawn process: %v", err)
+	}
+
+	defer func() {
+		err := killProcessCleanly(cmd.Process, time.Second)
+		if err != nil {
+			log.Printf("cannot kill the systemd-nspawn process: %v", err)
+		}
+	}()
+
+	return f()
+}
+
+// withBootedNspawnImage boots the specified directory in the specified namespace
+// using nspawn. The VM is killed immediately after function returns.
+func withBootedNspawnDirectory(dir, name string, ns netNS, f func() error) error {
+	cmd := exec.Command(
+		"systemd-nspawn",
+		"--boot", "--register=no",
+		"-M", name,
+		"--directory", dir,
+		"--network-namespace-path", ns.Path(),
+	)
+
+	err := cmd.Start()
+	if err != nil {
+		return fmt.Errorf("cannot start the systemd-nspawn process: %v", err)
+	}
+
+	defer func() {
+		err := killProcessCleanly(cmd.Process, time.Second)
+		if err != nil {
+			log.Printf("cannot kill the systemd-nspawn process: %v", err)
+		}
+	}()
+
+	return f()
+}
+
+// withExtractedTarArchive extracts the provided archive and passes
+// a path to the result to the function f. The result is deleted
+// immediately after the function returns.
+func withExtractedTarArchive(archive string, f func(dir string) error) error {
+	dir, err := ioutil.TempDir("", "tar-archive")
+	if err != nil {
+		return fmt.Errorf("cannot create a temporary dir: %v", err)
+	}
+
+	defer func() {
+		err := os.RemoveAll(dir)
+		if err != nil {
+			log.Printf("cannot remove the temporary dir: %v", err)
+		}
+	}()
+
+	cmd := exec.Command(
+		"tar",
+		"xf", archive,
+		"-C", dir,
+	)
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("cannot untar the archive: %v", err)
+	}
+
+	return f(dir)
+}

cmd/osbuild-image-tests/helpers.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"log"
+	"os"
+	"syscall"
+	"time"
+)
+
+// durationMin returns the smaller of two given durations
+func durationMin(a, b time.Duration) time.Duration {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+// killProcessCleanly firstly sends SIGTERM to the process. If it still exists
+// after the specified timeout, it sends SIGKILL
+func killProcessCleanly(process *os.Process, timeout time.Duration) error {
+	err := process.Signal(syscall.SIGTERM)
+	if err != nil {
+		log.Printf("cannot send SIGTERM to process, sending SIGKILL instead: %v", err)
+		return process.Kill()
+	}
+
+	const pollInterval = 10 * time.Millisecond
+
+	for {
+		p, err := os.FindProcess(process.Pid)
+		if err != nil {
+			return nil
+		}
+
+		err = p.Signal(syscall.Signal(0))
+		if err != nil {
+			return nil
+		}
+
+		sleep := durationMin(pollInterval, timeout)
+		if sleep == 0 {
+			break
+		}
+
+		timeout -= sleep
+		time.Sleep(sleep)
+	}
+
+	return process.Kill()
+}

cmd/osbuild-image-tests/netns.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"runtime"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+const netnsDir = "/var/run/netns"
+
+// Network namespace abstraction
+type netNS string
+
+// newNetworkNamespace returns a new network namespace with a random
+// name. The calling goroutine remains in the same namespace
+// as before the call.
+func newNetworkNamespace() (netNS, error) {
+	// This method needs to unshare the current thread. Go runtime can switch
+	// the goroutine to run on a different thread at any point, so we need
+	// to ensure that this method runs in the same thread for its whole
+	// lifetime.
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	_, err := os.Stat(netnsDir)
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			err := os.Mkdir(netnsDir, 0755)
+			if err != nil {
+				return "", fmt.Errorf("cannot create %s: %v", netnsDir, err)
+			}
+		} else {
+			return "", fmt.Errorf("cannot stat %s: %v", netnsDir, err)
+		}
+	}
+
+	f, err := ioutil.TempFile(netnsDir, "osbuild-composer-namespace")
+	if err != nil {
+		return "", fmt.Errorf("cannot create a tempfile: %v", err)
+	}
+
+	// We want to remove the temporary file if the namespace initialization fails.
+	// The best method I could thought of is to have the following variable
+	// denoting if the initialization was successful. It is set to true right
+	// before the end of this function.
+	initOK := false
+	defer func() {
+		if !initOK {
+			err := os.Remove(f.Name())
+			if err != nil {
+				log.Printf("cannot remove the temporary namespace: %v", err)
+			}
+		}
+	}()
+
+	oldNS, err := os.Open("/proc/self/ns/net")
+	if err != nil {
+		return "", fmt.Errorf("cannot open the current namespace: %v", err)
+	}
+
+	err = syscall.Unshare(syscall.CLONE_NEWNET)
+	if err != nil {
+		return "", fmt.Errorf("cannot unshare the network namespace")
+	}
+	defer func() {
+		err = unix.Setns(int(oldNS.Fd()), syscall.CLONE_NEWNET)
+		if err != nil {
+			// We cannot return to the original namespace.
+			// As we don't know nothing about affected threads, let's just
+			// quit immediately.
+			log.Fatalf("returning to the original namespace failed, quitting: %v", err)
+		}
+	}()
+
+	cmd := exec.Command("ip", "link", "set", "up", "dev", "lo")
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stderr
+	err = cmd.Run()
+	if err != nil {
+		return "", fmt.Errorf("cannot set up a loopback device in the new namespace: %v", err)
+	}
+
+	cmd = exec.Command("mount", "-o", "bind", "/proc/self/ns/net", f.Name())
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stderr
+	err = cmd.Run()
+	if err != nil {
+		return "", fmt.Errorf("cannot bind mount the new namespace: %v", err)
+	}
+
+	ns := netNS(path.Base(f.Name()))
+
+	// Initialization OK, do not delete the namespace file.
+	initOK = true
+	return ns, nil
+}
+
+// NamespaceCommand returns an *exec.Cmd struct with the difference
+// that it's prepended by "ip netns exec NAMESPACE_NAME" command, which
+// runs the command in a namespaced environment.
+func (n netNS) NamespacedCommand(name string, arg ...string) *exec.Cmd {
+	args := []string{"netns", "exec", string(n), name}
+	args = append(args, arg...)
+	return exec.Command("ip", args...)
+}
+
+// NamespaceCommand returns an *exec.Cmd struct with the difference
+// that it's prepended by "ip netns exec NAMESPACE_NAME" command, which
+// runs the command in a namespaced environment.
+func (n netNS) NamespacedCommandContext(ctx context.Context, name string, arg ...string) *exec.Cmd {
+	args := []string{"netns", "exec", string(n), name}
+	args = append(args, arg...)
+	return exec.CommandContext(ctx, "ip", args...)
+}
+
+// Path returns the path to the namespace file
+func (n netNS) Path() string {
+	return path.Join(netnsDir, string(n))
+}
+
+// Delete deletes the namespaces
+func (n netNS) Delete() error {
+	cmd := exec.Command("umount", n.Path())
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("cannot unmount the network namespace: %v", err)
+	}
+
+	err = os.Remove(n.Path())
+	if err != nil {
+		return fmt.Errorf("cannot delete the network namespace file: %v", err)
+	}
+
+	return nil
+}

cmd/osbuild-image-tests/osbuild-image-tests.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
@@ -12,8 +13,10 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
+
 	"github.com/osbuild/osbuild-composer/internal/common"
 	"github.com/osbuild/osbuild-composer/internal/distro"
 )
@@ -26,6 +29,9 @@ type testcaseStruct struct {
 	} `json:"compose-request"`
 	Manifest  json.RawMessage
 	ImageInfo json.RawMessage `json:"image-info"`
+	Boot      *struct {
+		Type string
+	}
 }
 
 // runOsbuild runs osbuild with the specified manifest and store.
@@ -70,6 +76,8 @@ func runOsbuild(manifest []byte, store string) (string, error) {
 // extractXZ extracts an xz archive, it's just a simple wrapper around unxz(1).
 func extractXZ(archivePath string) error {
 	cmd := exec.Command("unxz", archivePath)
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stderr
 	if err := cmd.Run(); err != nil {
 		return fmt.Errorf("cannot extract xz archive: %v", err)
 	}
@@ -124,9 +132,111 @@ func testImageInfo(imagePath string, rawImageInfoExpected []byte) error {
 	return nil
 }
 
+type timeoutError struct{}
+
+func (*timeoutError) Error() string { return "" }
+
+// trySSHOnce tries to test the running image using ssh once
+// It returns timeoutError if ssh command returns 255, if it runs for more
+// that 10 seconds or if systemd-is-running returns starting.
+// It returns nil if systemd-is-running returns running or degraded.
+// It can also return other errors in other error cases.
+func trySSHOnce(ns netNS) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	cmd := ns.NamespacedCommandContext(
+		ctx,
+		"ssh",
+		"-p", "22",
+		"-i", "/usr/share/tests/osbuild-composer/keyring/id_rsa",
+		"-o", "StrictHostKeyChecking=no",
+		"-o", "UserKnownHostsFile=/dev/null",
+		"redhat@localhost",
+		"systemctl --wait is-system-running",
+	)
+	output, err := cmd.Output()
+
+	if ctx.Err() == context.DeadlineExceeded {
+		return &timeoutError{}
+	}
+
+	if err != nil {
+		if exitError, ok := err.(*exec.ExitError); ok {
+			if exitError.ExitCode() == 255 {
+				return &timeoutError{}
+			}
+		} else {
+			return fmt.Errorf("ssh command failed from unknown reason: %v", err)
+		}
+	}
+
+	outputString := strings.TrimSpace(string(output))
+	switch outputString {
+	case "running":
+		return nil
+	case "degraded":
+		log.Print("ssh test passed, but the system is degraded")
+		return nil
+	case "starting":
+		return &timeoutError{}
+	default:
+		return fmt.Errorf("ssh test failed, system status is: %s", outputString)
+	}
+}
+
+// testSSH tests the running image using ssh.
+// It tries 20 attempts before giving up. If a major error occurs, it might
+// return earlier.
+func testSSH(ns netNS) error {
+	const attempts = 20
+	for i := 0; i < attempts; i++ {
+		err := trySSHOnce(ns)
+		if err == nil {
+			return nil
+		}
+		if _, ok := err.(*timeoutError); !ok {
+			return err
+		}
+
+		time.Sleep(10 * time.Second)
+	}
+
+	return fmt.Errorf("ssh test failure, %d attempts were made", attempts)
+}
+
+// testBoot tests if the image is able to successfully boot
+// Before the test it boots the image respecting the specified bootType.
+// The test passes if the function is able to connect to the image via ssh
+// in defined number of attempts and systemd-is-running returns running
+// or degraded status.
+func testBoot(imagePath string, bootType string, outputID string) error {
+	return withNetworkNamespace(func(ns netNS) error {
+		switch bootType {
+		case "qemu":
+			fallthrough
+		case "qemu-extract":
+			return withBootedQemuImage(imagePath, ns, func() error {
+				return testSSH(ns)
+			})
+		case "nspawn":
+			return withBootedNspawnImage(imagePath, outputID, ns, func() error {
+				return testSSH(ns)
+			})
+		case "nspawn-extract":
+			return withExtractedTarArchive(imagePath, func(dir string) error {
+				return withBootedNspawnDirectory(dir, outputID, ns, func() error {
+					return testSSH(ns)
+				})
+			})
+		default:
+			panic("unknown boot type!")
+		}
+	})
+}
+
 // testImage performs a series of tests specified in the testcase
 // on an image
-func testImage(testcase testcaseStruct, imagePath string) error {
+func testImage(testcase testcaseStruct, imagePath, outputID string) error {
 	if testcase.ImageInfo != nil {
 		log.Print("[image info sub-test] running")
 		err := testImageInfo(imagePath, testcase.ImageInfo)
@@ -139,6 +249,18 @@ func testImage(testcase testcaseStruct, imagePath string) error {
 		log.Print("[image info sub-test] not defined, skipping")
 	}
 
+	if testcase.Boot != nil {
+		log.Print("[boot sub-test] running")
+		err := testBoot(imagePath, testcase.Boot.Type, outputID)
+		if err != nil {
+			log.Print("[boot sub-test] failed")
+			return err
+		}
+		log.Print("[boot sub-test] succeeded")
+	} else {
+		log.Print("[boot sub-test] not defined, skipping")
+	}
+
 	return nil
 }
 
@@ -163,16 +285,19 @@ func runTestcase(testcase testcaseStruct) error {
 
 	imagePath := fmt.Sprintf("%s/refs/%s/%s", store, outputID, testcase.ComposeRequest.Filename)
 
-	// if the result is xz archive, extract it
+	// if the result is xz archive but not tar.xz archive, extract it
 	base, ex := splitExtension(imagePath)
 	if ex == ".xz" {
-		if err := extractXZ(imagePath); err != nil {
+		_, ex = splitExtension(base)
-			return err
+		if ex != ".tar" {
+			if err := extractXZ(imagePath); err != nil {
+				return err
+			}
+			imagePath = base
 		}
-		imagePath = base
 	}
 
-	return testImage(testcase, imagePath)
+	return testImage(testcase, imagePath, outputID)
 }
 
 // getAllCases returns paths to all testcases in the testcase directory

go.mod

@@ -16,5 +16,5 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/stretchr/testify v1.4.0
 	golang.org/x/net v0.0.0-20200202094626-16171245cfb2 // indirect
-	golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4 // indirect
+	golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4
 )

golang-github-osbuild-composer.spec

@@ -96,6 +96,11 @@ install -m 0644 -vp repositories/*                          %{buildroot}%{_datad
 
 install -m 0755 -vd                                         %{buildroot}%{_datadir}/tests/osbuild-composer/cases
 install -m 0644 -vp test/cases/*                            %{buildroot}%{_datadir}/tests/osbuild-composer/cases/
+install -m 0755 -vd                                         %{buildroot}%{_datadir}/tests/osbuild-composer/keyring
+install -m 0600 -vp test/keyring/*                          %{buildroot}%{_datadir}/tests/osbuild-composer/keyring/
+
+install -m 0755 -vd                                         %{buildroot}%{_datadir}/tests/osbuild-composer/cloud-init
+install -m 0644 -vp test/cloud-init/*                       %{buildroot}%{_datadir}/tests/osbuild-composer/cloud-init/
 
 install -m 0755 -vd                                         %{buildroot}%{_unitdir}
 install -m 0644 -vp distribution/*.{service,socket}         %{buildroot}%{_unitdir}/
@@ -149,13 +154,15 @@ Summary:	Integration tests
 Requires: 	osbuild-composer
 Requires: 	composer-cli
 Requires:	createrepo_c
+Requires:   genisoimage
+Requires:   qemu-kvm-core
 
 %description tests
 Integration tests to be run on a pristine-dedicated system to test the osbuild-composer package.
 
 %files tests
-%{_libexecdir}/tests/osbuild-composer/*
+%{_libexecdir}/tests/osbuild-composer/
-%{_datadir}/tests/osbuild-composer/*
+%{_datadir}/tests/osbuild-composer/
 %{_libexecdir}/osbuild-composer/image-info
 
 %package worker

vendor/modules.txt

@@ -84,6 +84,7 @@ github.com/stretchr/testify/assert
 golang.org/x/net/http/httpproxy
 golang.org/x/net/idna
 # golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4
+golang.org/x/sys/unix
 golang.org/x/sys/windows
 golang.org/x/sys/windows/registry
 # golang.org/x/text v0.3.0