store: verify that old versions of the store can still be unmarshalled

The new tool osbuild-store-dump saves store.json to the current working
directory, with more or less arbitrary data in it.

This has been executed on osubild-composer-{12,13} (mutatis mutandis),
and the results are saved in `internal/store/test`. A new test is added
which loads these stores and does very basic verification on them having
been loaded correctly.

This is mostly meant to catch regressions that means old stores are able
to make composer crash, or lose all its data. It would not catch minor
errors that leave the stores syntactically correct.

Signed-off-by: Tom Gundersen <teg@jklm.no>

cmd/osbuild-store-dump/main.go

@@ -0,0 +1,184 @@
+// This fills and saves a store with more or less arbitrary data. It is meant to generate test stores as
+// test data for testing upgrades to composer.
+package main
+
+import (
+	"os"
+	"path"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/osbuild/osbuild-composer/internal/blueprint"
+	"github.com/osbuild/osbuild-composer/internal/distro"
+	"github.com/osbuild/osbuild-composer/internal/distro/fedora32"
+	"github.com/osbuild/osbuild-composer/internal/rpmmd"
+	"github.com/osbuild/osbuild-composer/internal/store"
+	"github.com/osbuild/osbuild-composer/internal/target"
+)
+
+func getManifest(bp blueprint.Blueprint, t distro.ImageType, a distro.Arch, d distro.Distro, rpmmd rpmmd.RPMMD, repos []rpmmd.RepoConfig) distro.Manifest {
+	packages, excludePackages := t.BasePackages()
+	pkgs, _, err := rpmmd.Depsolve(packages, excludePackages, repos, d.ModulePlatformID(), a.Name())
+	if err != nil {
+		panic(err)
+	}
+	buildPkgs, _, err := rpmmd.Depsolve(t.BuildPackages(), nil, repos, d.ModulePlatformID(), a.Name())
+	if err != nil {
+		panic(err)
+	}
+	manifest, err := t.Manifest(bp.Customizations, distro.ImageOptions{}, repos, pkgs, buildPkgs)
+	if err != nil {
+		panic(err)
+	}
+
+	return manifest
+}
+
+func main() {
+	cwd, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+	id1, err := uuid.NewRandom()
+	if err != nil {
+		panic(err)
+	}
+	id2, err := uuid.NewRandom()
+	if err != nil {
+		panic(err)
+	}
+	hostname := "my-host"
+	description := "Mostly harmless."
+	password := "password"
+	sshKey := "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC61wMCjOSHwbVb4VfVyl5sn497qW4PsdQ7Ty7aD6wDNZ/QjjULkDV/yW5WjDlDQ7UqFH0Sr7vywjqDizUAqK7zM5FsUKsUXWHWwg/ehKg8j9xKcMv11AkFoUoujtfAujnKODkk58XSA9whPr7qcw3vPrmog680pnMSzf9LC7J6kXfs6lkoKfBh9VnlxusCrw2yg0qI1fHAZBLPx7mW6+me71QZsS6sVz8v8KXyrXsKTdnF50FjzHcK9HXDBtSJS5wA3fkcRYymJe0o6WMWNdgSRVpoSiWaHHmFgdMUJaYoCfhXzyl7LtNb3Q+Sveg+tJK7JaRXBLMUllOlJ6ll5Hod root@localhost"
+	home := "/home/my-home"
+	shell := "/bin/true"
+	uid := 42
+	gid := 42
+	bp1 := blueprint.Blueprint{
+		Name:        "my-blueprint-1",
+		Description: "My first blueprint",
+		Packages: []blueprint.Package{
+			{
+				Name: "tmux",
+			},
+		},
+		Groups: []blueprint.Group{
+			{
+				Name: "core",
+			},
+		},
+	}
+	bp2 := blueprint.Blueprint{
+		Name:        "my-blueprint-2",
+		Description: "My second blueprint",
+		Version:     "0.0.2",
+		Customizations: &blueprint.Customizations{
+			Hostname: &hostname,
+			Kernel: &blueprint.KernelCustomization{
+				Append: "debug",
+			},
+			SSHKey: []blueprint.SSHKeyCustomization{
+				{
+					User: "me",
+					Key:  sshKey,
+				},
+			},
+			User: []blueprint.UserCustomization{
+				{
+					Name:        "myself",
+					Description: &description,
+					Password:    &password,
+					Key:         &sshKey,
+					Home:        &home,
+					Shell:       &shell,
+					Groups: []string{
+						"wheel",
+					},
+					UID: &uid,
+					GID: &gid,
+				},
+			},
+		},
+	}
+	awsTarget := target.NewAWSTarget(
+		&target.AWSTargetOptions{
+			Filename:        "image.ami",
+			Region:          "far-away-1",
+			AccessKeyID:     "MyKey",
+			SecretAccessKey: "MySecret",
+			Bucket:          "list",
+			Key:             "image",
+		},
+	)
+	awsTarget.Uuid = id1
+	awsTarget.ImageName = "My Image"
+	awsTarget.Created = time.Now()
+
+	d := fedora32.New()
+	a, err := d.GetArch("x86_64")
+	if err != nil {
+		panic(err)
+	}
+	t1, err := a.GetImageType("qcow2")
+	if err != nil {
+		panic(err)
+	}
+	t2, err := a.GetImageType("fedora-iot-commit")
+	if err != nil {
+		panic(err)
+	}
+	allRepos, err := rpmmd.LoadRepositories([]string{cwd}, "fedora-32")
+	if err != nil {
+		panic(err)
+	}
+	repos := allRepos["x86_64"]
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		panic("os.UserHomeDir(): " + err.Error())
+	}
+	rpmmd := rpmmd.NewRPMMD(path.Join(homeDir, ".cache/osbuild-composer/rpmmd"))
+
+	s := store.New(&cwd, a, nil)
+	if s == nil {
+		panic("could not create store")
+	}
+	err = s.PushBlueprint(bp1, "message 1")
+	if err != nil {
+		panic(err)
+	}
+	err = s.PushBlueprint(bp1, "message 2")
+	if err != nil {
+		panic(err)
+	}
+	err = s.PushBlueprintToWorkspace(bp2)
+	if err != nil {
+		panic(err)
+	}
+	err = s.PushCompose(id1,
+		getManifest(bp2, t1, a, d, rpmmd, repos),
+		t1,
+		&bp2,
+		0,
+		[]*target.Target{
+			awsTarget,
+		},
+		id1,
+	)
+	if err != nil {
+		panic(err)
+	}
+	err = s.PushCompose(id2,
+		getManifest(bp2, t2, a, d, rpmmd, repos),
+		t2,
+		&bp2,
+		0,
+		[]*target.Target{
+			awsTarget,
+		},
+		id2,
+	)
+	if err != nil {
+		panic(err)
+	}
+}

internal/store/json_test.go

@@ -1,14 +1,20 @@
 package store
 
 import (
+	"encoding/json"
+	"io/ioutil"
+	"path/filepath"
 	"reflect"
 	"testing"
 
 	"github.com/google/uuid"
 	"github.com/osbuild/osbuild-composer/internal/blueprint"
 	"github.com/osbuild/osbuild-composer/internal/distro"
+	"github.com/osbuild/osbuild-composer/internal/distro/fedora32"
 	"github.com/osbuild/osbuild-composer/internal/distro/fedoratest"
 	"github.com/osbuild/osbuild-composer/internal/distro/test_distro"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func Test_imageTypeToCompatString(t *testing.T) {
@@ -202,3 +208,27 @@ func Test_newCommitsV0(t *testing.T) {
 		})
 	}
 }
+
+func Test_upgrade(t *testing.T) {
+	assert := assert.New(t)
+	testPath, err := filepath.Abs("./test/*.json")
+	require.NoError(t, err)
+	fileNames, err := filepath.Glob(testPath)
+	assert.NoErrorf(err, "Could not read test store directory '%s': %v", testPath, err)
+	require.Greaterf(t, len(fileNames), 0, "No test stores found in %s", testPath)
+	for _, fileName := range fileNames {
+		var storeStruct storeV0
+		file, err := ioutil.ReadFile(fileName)
+		assert.NoErrorf(err, "Could not read test-store '%s': %v", fileName, err)
+		err = json.Unmarshal([]byte(file), &storeStruct)
+		assert.NoErrorf(err, "Could not parse test-store '%s': %v", fileName, err)
+		arch, err := fedora32.New().GetArch("x86_64")
+		assert.NoError(err)
+		store := newStoreFromV0(storeStruct, arch, nil)
+		assert.Equal(1, len(store.blueprints))
+		assert.Equal(1, len(store.blueprintsChanges))
+		assert.Equal(1, len(store.blueprintsCommits))
+		assert.LessOrEqual(1, len(store.composes))
+		assert.Equal(1, len(store.workspace))
+	}
+}