From b4a7bc6467627099c548ef4635154757b8a76f31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Budai?= Date: Fri, 13 Mar 2020 14:08:23 +0100 Subject: [PATCH] tests/image: add booting tests This commit makes the osbuild-image-tests binary doing the same set of tests like the old test/run script. Changes from test/run: - qemu/nspawn are now killed gracefully. Firstly, SIGTERM is sent. If the process doesn't exit till the timeout, SIGKILL is sent. I changed this because nspawn leaves some artifacts behind when killed by SIGKILL. - the unsharing of network namespace now works differently because of systemd issue #15079 --- cmd/osbuild-image-tests/context-managers.go | 200 ++++++++++++++++++ cmd/osbuild-image-tests/helpers.go | 50 +++++ cmd/osbuild-image-tests/netns.go | 146 +++++++++++++ .../osbuild-image-tests.go | 137 +++++++++++- go.mod | 2 +- golang-github-osbuild-composer.spec | 11 +- vendor/modules.txt | 1 + 7 files changed, 538 insertions(+), 9 deletions(-) create mode 100644 cmd/osbuild-image-tests/context-managers.go create mode 100644 cmd/osbuild-image-tests/helpers.go create mode 100644 cmd/osbuild-image-tests/netns.go diff --git a/cmd/osbuild-image-tests/context-managers.go b/cmd/osbuild-image-tests/context-managers.go new file mode 100644 index 000000000..b766e2b83 --- /dev/null +++ b/cmd/osbuild-image-tests/context-managers.go @@ -0,0 +1,200 @@ +package main + +import ( + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "time" +) + +// withNetworkNamespace provides the function f with a new network namespace +// which is deleted immediately after f returns +func withNetworkNamespace(f func(ns netNS) error) error { + ns, err := newNetworkNamespace() + if err != nil { + return err + } + + defer func() { + err := ns.Delete() + if err != nil { + log.Printf("cannot delete network namespace: %v", err) + } + }() + + return f(ns) +} + +// withTempFile provides the function f with a new temporary file +// dir and pattern parameters have the same semantics as in ioutil.TempFile +func withTempFile(dir, pattern string, f func(file *os.File) error) error { + tempFile, err := ioutil.TempFile(dir, pattern) + if err != nil { + return fmt.Errorf("cannot create the temporary file: %v", err) + } + + defer func() { + err := os.Remove(tempFile.Name()) + if err != nil { + log.Printf("cannot remove the temporary file: %v", err) + } + }() + + return f(tempFile) +} + +// writeCloudInitSO creates cloud-init iso from specified userData and +// metaData and writes it to the writer +func writeCloudInitISO(writer io.Writer, userData, metaData string) error { + isoCmd := exec.Command( + "genisoimage", + "-quiet", + "-input-charset", "utf-8", + "-volid", "cidata", + "-joliet", + "-rock", + userData, + metaData, + ) + isoCmd.Stdout = writer + isoCmd.Stderr = os.Stderr + + err := isoCmd.Run() + if err != nil { + return fmt.Errorf("cannot create cloud-init iso: %v", err) + } + + return nil +} + +// withBootedQemuImage boots the specified image in the specified namespace +// using qemu. The VM is killed immediately after function returns. +func withBootedQemuImage(image string, ns netNS, f func() error) error { + return withTempFile("", "osbuild-image-tests-cloudinit", func(cloudInitFile *os.File) error { + err := writeCloudInitISO( + cloudInitFile, + "/usr/share/tests/osbuild-composer/cloud-init/user-data", + "/usr/share/tests/osbuild-composer/cloud-init/meta-data", + ) + if err != nil { + return err + } + + err = cloudInitFile.Close() + if err != nil { + return fmt.Errorf("cannot close temporary cloudinit file: %v", err) + } + + qemuCmd := ns.NamespacedCommand( + "qemu-system-x86_64", + "-m", "2048", + "-snapshot", + "-accel", "accel=kvm:hvf:tcg", + "-cdrom", cloudInitFile.Name(), + "-net", "nic,model=rtl8139", "-net", "user,hostfwd=tcp::22-:22", + "-nographic", + image, + ) + + err = qemuCmd.Start() + if err != nil { + return fmt.Errorf("cannot start the qemu process: %v", err) + } + + defer func() { + err := killProcessCleanly(qemuCmd.Process, time.Second) + if err != nil { + log.Printf("cannot kill the qemu process: %v", err) + } + }() + + return f() + }) +} + +// withBootedNspawnImage boots the specified image in the specified namespace +// using nspawn. The VM is killed immediately after function returns. +func withBootedNspawnImage(image, name string, ns netNS, f func() error) error { + cmd := exec.Command( + "systemd-nspawn", + "--boot", "--register=no", + "-M", name, + "--image", image, + "--network-namespace-path", ns.Path(), + ) + + err := cmd.Start() + if err != nil { + return fmt.Errorf("cannot start the systemd-nspawn process: %v", err) + } + + defer func() { + err := killProcessCleanly(cmd.Process, time.Second) + if err != nil { + log.Printf("cannot kill the systemd-nspawn process: %v", err) + } + }() + + return f() +} + +// withBootedNspawnImage boots the specified directory in the specified namespace +// using nspawn. The VM is killed immediately after function returns. +func withBootedNspawnDirectory(dir, name string, ns netNS, f func() error) error { + cmd := exec.Command( + "systemd-nspawn", + "--boot", "--register=no", + "-M", name, + "--directory", dir, + "--network-namespace-path", ns.Path(), + ) + + err := cmd.Start() + if err != nil { + return fmt.Errorf("cannot start the systemd-nspawn process: %v", err) + } + + defer func() { + err := killProcessCleanly(cmd.Process, time.Second) + if err != nil { + log.Printf("cannot kill the systemd-nspawn process: %v", err) + } + }() + + return f() +} + +// withExtractedTarArchive extracts the provided archive and passes +// a path to the result to the function f. The result is deleted +// immediately after the function returns. +func withExtractedTarArchive(archive string, f func(dir string) error) error { + dir, err := ioutil.TempDir("", "tar-archive") + if err != nil { + return fmt.Errorf("cannot create a temporary dir: %v", err) + } + + defer func() { + err := os.RemoveAll(dir) + if err != nil { + log.Printf("cannot remove the temporary dir: %v", err) + } + }() + + cmd := exec.Command( + "tar", + "xf", archive, + "-C", dir, + ) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + + err = cmd.Run() + if err != nil { + return fmt.Errorf("cannot untar the archive: %v", err) + } + + return f(dir) +} diff --git a/cmd/osbuild-image-tests/helpers.go b/cmd/osbuild-image-tests/helpers.go new file mode 100644 index 000000000..71e4f60f5 --- /dev/null +++ b/cmd/osbuild-image-tests/helpers.go @@ -0,0 +1,50 @@ +package main + +import ( + "log" + "os" + "syscall" + "time" +) + +// durationMin returns the smaller of two given durations +func durationMin(a, b time.Duration) time.Duration { + if a < b { + return a + } + return b +} + +// killProcessCleanly firstly sends SIGTERM to the process. If it still exists +// after the specified timeout, it sends SIGKILL +func killProcessCleanly(process *os.Process, timeout time.Duration) error { + err := process.Signal(syscall.SIGTERM) + if err != nil { + log.Printf("cannot send SIGTERM to process, sending SIGKILL instead: %v", err) + return process.Kill() + } + + const pollInterval = 10 * time.Millisecond + + for { + p, err := os.FindProcess(process.Pid) + if err != nil { + return nil + } + + err = p.Signal(syscall.Signal(0)) + if err != nil { + return nil + } + + sleep := durationMin(pollInterval, timeout) + if sleep == 0 { + break + } + + timeout -= sleep + time.Sleep(sleep) + } + + return process.Kill() +} diff --git a/cmd/osbuild-image-tests/netns.go b/cmd/osbuild-image-tests/netns.go new file mode 100644 index 000000000..f05145428 --- /dev/null +++ b/cmd/osbuild-image-tests/netns.go @@ -0,0 +1,146 @@ +package main + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "runtime" + "syscall" + + "golang.org/x/sys/unix" +) + +const netnsDir = "/var/run/netns" + +// Network namespace abstraction +type netNS string + +// newNetworkNamespace returns a new network namespace with a random +// name. The calling goroutine remains in the same namespace +// as before the call. +func newNetworkNamespace() (netNS, error) { + // This method needs to unshare the current thread. Go runtime can switch + // the goroutine to run on a different thread at any point, so we need + // to ensure that this method runs in the same thread for its whole + // lifetime. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + _, err := os.Stat(netnsDir) + + if err != nil { + if os.IsNotExist(err) { + err := os.Mkdir(netnsDir, 0755) + if err != nil { + return "", fmt.Errorf("cannot create %s: %v", netnsDir, err) + } + } else { + return "", fmt.Errorf("cannot stat %s: %v", netnsDir, err) + } + } + + f, err := ioutil.TempFile(netnsDir, "osbuild-composer-namespace") + if err != nil { + return "", fmt.Errorf("cannot create a tempfile: %v", err) + } + + // We want to remove the temporary file if the namespace initialization fails. + // The best method I could thought of is to have the following variable + // denoting if the initialization was successful. It is set to true right + // before the end of this function. + initOK := false + defer func() { + if !initOK { + err := os.Remove(f.Name()) + if err != nil { + log.Printf("cannot remove the temporary namespace: %v", err) + } + } + }() + + oldNS, err := os.Open("/proc/self/ns/net") + if err != nil { + return "", fmt.Errorf("cannot open the current namespace: %v", err) + } + + err = syscall.Unshare(syscall.CLONE_NEWNET) + if err != nil { + return "", fmt.Errorf("cannot unshare the network namespace") + } + defer func() { + err = unix.Setns(int(oldNS.Fd()), syscall.CLONE_NEWNET) + if err != nil { + // We cannot return to the original namespace. + // As we don't know nothing about affected threads, let's just + // quit immediately. + log.Fatalf("returning to the original namespace failed, quitting: %v", err) + } + }() + + cmd := exec.Command("ip", "link", "set", "up", "dev", "lo") + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stderr + err = cmd.Run() + if err != nil { + return "", fmt.Errorf("cannot set up a loopback device in the new namespace: %v", err) + } + + cmd = exec.Command("mount", "-o", "bind", "/proc/self/ns/net", f.Name()) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stderr + err = cmd.Run() + if err != nil { + return "", fmt.Errorf("cannot bind mount the new namespace: %v", err) + } + + ns := netNS(path.Base(f.Name())) + + // Initialization OK, do not delete the namespace file. + initOK = true + return ns, nil +} + +// NamespaceCommand returns an *exec.Cmd struct with the difference +// that it's prepended by "ip netns exec NAMESPACE_NAME" command, which +// runs the command in a namespaced environment. +func (n netNS) NamespacedCommand(name string, arg ...string) *exec.Cmd { + args := []string{"netns", "exec", string(n), name} + args = append(args, arg...) + return exec.Command("ip", args...) +} + +// NamespaceCommand returns an *exec.Cmd struct with the difference +// that it's prepended by "ip netns exec NAMESPACE_NAME" command, which +// runs the command in a namespaced environment. +func (n netNS) NamespacedCommandContext(ctx context.Context, name string, arg ...string) *exec.Cmd { + args := []string{"netns", "exec", string(n), name} + args = append(args, arg...) + return exec.CommandContext(ctx, "ip", args...) +} + +// Path returns the path to the namespace file +func (n netNS) Path() string { + return path.Join(netnsDir, string(n)) +} + +// Delete deletes the namespaces +func (n netNS) Delete() error { + cmd := exec.Command("umount", n.Path()) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + err := cmd.Run() + if err != nil { + return fmt.Errorf("cannot unmount the network namespace: %v", err) + } + + err = os.Remove(n.Path()) + if err != nil { + return fmt.Errorf("cannot delete the network namespace file: %v", err) + } + + return nil +} diff --git a/cmd/osbuild-image-tests/osbuild-image-tests.go b/cmd/osbuild-image-tests/osbuild-image-tests.go index d95b4c30d..818b10f22 100644 --- a/cmd/osbuild-image-tests/osbuild-image-tests.go +++ b/cmd/osbuild-image-tests/osbuild-image-tests.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "encoding/json" "flag" "fmt" @@ -12,8 +13,10 @@ import ( "os/exec" "path/filepath" "strings" + "time" "github.com/google/go-cmp/cmp" + "github.com/osbuild/osbuild-composer/internal/common" "github.com/osbuild/osbuild-composer/internal/distro" ) @@ -26,6 +29,9 @@ type testcaseStruct struct { } `json:"compose-request"` Manifest json.RawMessage ImageInfo json.RawMessage `json:"image-info"` + Boot *struct { + Type string + } } // runOsbuild runs osbuild with the specified manifest and store. @@ -70,6 +76,8 @@ func runOsbuild(manifest []byte, store string) (string, error) { // extractXZ extracts an xz archive, it's just a simple wrapper around unxz(1). func extractXZ(archivePath string) error { cmd := exec.Command("unxz", archivePath) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stderr if err := cmd.Run(); err != nil { return fmt.Errorf("cannot extract xz archive: %v", err) } @@ -124,9 +132,111 @@ func testImageInfo(imagePath string, rawImageInfoExpected []byte) error { return nil } +type timeoutError struct{} + +func (*timeoutError) Error() string { return "" } + +// trySSHOnce tries to test the running image using ssh once +// It returns timeoutError if ssh command returns 255, if it runs for more +// that 10 seconds or if systemd-is-running returns starting. +// It returns nil if systemd-is-running returns running or degraded. +// It can also return other errors in other error cases. +func trySSHOnce(ns netNS) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cmd := ns.NamespacedCommandContext( + ctx, + "ssh", + "-p", "22", + "-i", "/usr/share/tests/osbuild-composer/keyring/id_rsa", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "redhat@localhost", + "systemctl --wait is-system-running", + ) + output, err := cmd.Output() + + if ctx.Err() == context.DeadlineExceeded { + return &timeoutError{} + } + + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + if exitError.ExitCode() == 255 { + return &timeoutError{} + } + } else { + return fmt.Errorf("ssh command failed from unknown reason: %v", err) + } + } + + outputString := strings.TrimSpace(string(output)) + switch outputString { + case "running": + return nil + case "degraded": + log.Print("ssh test passed, but the system is degraded") + return nil + case "starting": + return &timeoutError{} + default: + return fmt.Errorf("ssh test failed, system status is: %s", outputString) + } +} + +// testSSH tests the running image using ssh. +// It tries 20 attempts before giving up. If a major error occurs, it might +// return earlier. +func testSSH(ns netNS) error { + const attempts = 20 + for i := 0; i < attempts; i++ { + err := trySSHOnce(ns) + if err == nil { + return nil + } + if _, ok := err.(*timeoutError); !ok { + return err + } + + time.Sleep(10 * time.Second) + } + + return fmt.Errorf("ssh test failure, %d attempts were made", attempts) +} + +// testBoot tests if the image is able to successfully boot +// Before the test it boots the image respecting the specified bootType. +// The test passes if the function is able to connect to the image via ssh +// in defined number of attempts and systemd-is-running returns running +// or degraded status. +func testBoot(imagePath string, bootType string, outputID string) error { + return withNetworkNamespace(func(ns netNS) error { + switch bootType { + case "qemu": + fallthrough + case "qemu-extract": + return withBootedQemuImage(imagePath, ns, func() error { + return testSSH(ns) + }) + case "nspawn": + return withBootedNspawnImage(imagePath, outputID, ns, func() error { + return testSSH(ns) + }) + case "nspawn-extract": + return withExtractedTarArchive(imagePath, func(dir string) error { + return withBootedNspawnDirectory(dir, outputID, ns, func() error { + return testSSH(ns) + }) + }) + default: + panic("unknown boot type!") + } + }) +} + // testImage performs a series of tests specified in the testcase // on an image -func testImage(testcase testcaseStruct, imagePath string) error { +func testImage(testcase testcaseStruct, imagePath, outputID string) error { if testcase.ImageInfo != nil { log.Print("[image info sub-test] running") err := testImageInfo(imagePath, testcase.ImageInfo) @@ -139,6 +249,18 @@ func testImage(testcase testcaseStruct, imagePath string) error { log.Print("[image info sub-test] not defined, skipping") } + if testcase.Boot != nil { + log.Print("[boot sub-test] running") + err := testBoot(imagePath, testcase.Boot.Type, outputID) + if err != nil { + log.Print("[boot sub-test] failed") + return err + } + log.Print("[boot sub-test] succeeded") + } else { + log.Print("[boot sub-test] not defined, skipping") + } + return nil } @@ -163,16 +285,19 @@ func runTestcase(testcase testcaseStruct) error { imagePath := fmt.Sprintf("%s/refs/%s/%s", store, outputID, testcase.ComposeRequest.Filename) - // if the result is xz archive, extract it + // if the result is xz archive but not tar.xz archive, extract it base, ex := splitExtension(imagePath) if ex == ".xz" { - if err := extractXZ(imagePath); err != nil { - return err + _, ex = splitExtension(base) + if ex != ".tar" { + if err := extractXZ(imagePath); err != nil { + return err + } + imagePath = base } - imagePath = base } - return testImage(testcase, imagePath) + return testImage(testcase, imagePath, outputID) } // getAllCases returns paths to all testcases in the testcase directory diff --git a/go.mod b/go.mod index a00a8ba58..4dd391d09 100644 --- a/go.mod +++ b/go.mod @@ -16,5 +16,5 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/stretchr/testify v1.4.0 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 // indirect - golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4 // indirect + golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4 ) diff --git a/golang-github-osbuild-composer.spec b/golang-github-osbuild-composer.spec index 5b87fe09e..1fab7c37e 100644 --- a/golang-github-osbuild-composer.spec +++ b/golang-github-osbuild-composer.spec @@ -96,6 +96,11 @@ install -m 0644 -vp repositories/* %{buildroot}%{_datad install -m 0755 -vd %{buildroot}%{_datadir}/tests/osbuild-composer/cases install -m 0644 -vp test/cases/* %{buildroot}%{_datadir}/tests/osbuild-composer/cases/ +install -m 0755 -vd %{buildroot}%{_datadir}/tests/osbuild-composer/keyring +install -m 0600 -vp test/keyring/* %{buildroot}%{_datadir}/tests/osbuild-composer/keyring/ + +install -m 0755 -vd %{buildroot}%{_datadir}/tests/osbuild-composer/cloud-init +install -m 0644 -vp test/cloud-init/* %{buildroot}%{_datadir}/tests/osbuild-composer/cloud-init/ install -m 0755 -vd %{buildroot}%{_unitdir} install -m 0644 -vp distribution/*.{service,socket} %{buildroot}%{_unitdir}/ @@ -149,13 +154,15 @@ Summary: Integration tests Requires: osbuild-composer Requires: composer-cli Requires: createrepo_c +Requires: genisoimage +Requires: qemu-kvm-core %description tests Integration tests to be run on a pristine-dedicated system to test the osbuild-composer package. %files tests -%{_libexecdir}/tests/osbuild-composer/* -%{_datadir}/tests/osbuild-composer/* +%{_libexecdir}/tests/osbuild-composer/ +%{_datadir}/tests/osbuild-composer/ %{_libexecdir}/osbuild-composer/image-info %package worker diff --git a/vendor/modules.txt b/vendor/modules.txt index 7fc5b0635..9c526fbc6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -84,6 +84,7 @@ github.com/stretchr/testify/assert golang.org/x/net/http/httpproxy golang.org/x/net/idna # golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4 +golang.org/x/sys/unix golang.org/x/sys/windows golang.org/x/sys/windows/registry # golang.org/x/text v0.3.0