From f4991cb1ca7f92e844465192c185f9509adfb2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Budai?= Date: Thu, 28 Nov 2019 15:15:27 +0100 Subject: [PATCH] api: Add support for upload API This commit introduces basic support for upload API. Currently, all the routes required by cockpit-composer are supported (except for /compose/log). Also, ComposeEntry struct is moved outside of the store package. I decided to do it because it isn't connected in any way to store, it's more connected to API. Due to this move there's currently a known bug that image size is not returned. This should be solved by moving Image struct inside Compose struct by follow-up PR. --- internal/jobqueue/api_test.go | 6 +- internal/mocks/rpmmd/fixtures.go | 122 ++++++++++++++---- internal/store/store.go | 112 ++++------------- internal/weldr/api.go | 94 ++++++++++---- internal/weldr/api_test.go | 209 ++++++++++++++++++++++++++----- internal/weldr/compose.go | 88 +++++++++++++ internal/weldr/upload.go | 110 ++++++++++++++++ 7 files changed, 570 insertions(+), 171 deletions(-) create mode 100644 internal/weldr/compose.go create mode 100644 internal/weldr/upload.go diff --git a/internal/jobqueue/api_test.go b/internal/jobqueue/api_test.go index b74f767e0..34bdf1daf 100644 --- a/internal/jobqueue/api_test.go +++ b/internal/jobqueue/api_test.go @@ -43,13 +43,13 @@ func TestCreate(t *testing.T) { store := store.New(nil, distro.New("fedora-30")) api := jobqueue.New(nil, store) - err := store.PushCompose(id, &blueprint.Blueprint{}, "tar") + err := store.PushCompose(id, &blueprint.Blueprint{}, "tar", nil) if err != nil { t.Fatalf("error pushing compose: %v", err) } test.TestRoute(t, api, false, "POST", "/job-queue/v1/jobs", `{}`, http.StatusCreated, - `{"id":"ffffffff-ffff-ffff-ffff-ffffffffffff","pipeline":{"build":{"pipeline":{"stages":[{"name":"org.osbuild.dnf","options":{"repos":[{"metalink":"https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever\u0026arch=$basearch","gpgkey":"-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFturGcBEACv0xBo91V2n0uEC2vh69ywCiSyvUgN/AQH8EZpCVtM7NyjKgKm\nbbY4G3R0M3ir1xXmvUDvK0493/qOiFrjkplvzXFTGpPTi0ypqGgxc5d0ohRA1M75\nL+0AIlXoOgHQ358/c4uO8X0JAA1NYxCkAW1KSJgFJ3RjukrfqSHWthS1d4o8fhHy\nKJKEnirE5hHqB50dafXrBfgZdaOs3C6ppRIePFe2o4vUEapMTCHFw0woQR8Ah4/R\nn7Z9G9Ln+0Cinmy0nbIDiZJ+pgLAXCOWBfDUzcOjDGKvcpoZharA07c0q1/5ojzO\n4F0Fh4g/BUmtrASwHfcIbjHyCSr1j/3Iz883iy07gJY5Yhiuaqmp0o0f9fgHkG53\n2xCU1owmACqaIBNQMukvXRDtB2GJMuKa/asTZDP6R5re+iXs7+s9ohcRRAKGyAyc\nYKIQKcaA+6M8T7/G+TPHZX6HJWqJJiYB+EC2ERblpvq9TPlLguEWcmvjbVc31nyq\nSDoO3ncFWKFmVsbQPTbP+pKUmlLfJwtb5XqxNR5GEXSwVv4I7IqBmJz1MmRafnBZ\ng0FJUtH668GnldO20XbnSVBr820F5SISMXVwCXDXEvGwwiB8Lt8PvqzXnGIFDAu3\nDlQI5sxSqpPVWSyw08ppKT2Tpmy8adiBotLfaCFl2VTHwOae48X2dMPBvQARAQAB\ntDFGZWRvcmEgKDMwKSA8ZmVkb3JhLTMwLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJbbqxnAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRDvPBEfz8ZZudTnD/9170LL3nyTVUCFmBjT9wZ4gYnpwtKVPa/pKnxbbS+Bmmac\ng9TrT9pZbqOHrNJLiZ3Zx1Hp+8uxr3Lo6kbYwImLhkOEDrf4aP17HfQ6VYFbQZI8\nf79OFxWJ7si9+3gfzeh9UYFEqOQfzIjLWFyfnas0OnV/P+RMQ1Zr+vPRqO7AR2va\nN9wg+Xl7157dhXPCGYnGMNSoxCbpRs0JNlzvJMuAea5nTTznRaJZtK/xKsqLn51D\nK07k9MHVFXakOH8QtMCUglbwfTfIpO5YRq5imxlWbqsYWVQy1WGJFyW6hWC0+RcJ\nOx5zGtOfi4/dN+xJ+ibnbyvy/il7Qm+vyFhCYqIPyS5m2UVJUuao3eApE38k78/o\n8aQOTnFQZ+U1Sw+6woFTxjqRQBXlQm2+7Bt3bqGATg4sXXWPbmwdL87Ic+mxn/ml\nSMfQux/5k6iAu1kQhwkO2YJn9eII6HIPkW+2m5N1JsUyJQe4cbtZE5Yh3TRA0dm7\n+zoBRfCXkOW4krchbgww/ptVmzMMP7GINJdROrJnsGl5FVeid9qHzV7aZycWSma7\nCxBYB1J8HCbty5NjtD6XMYRrMLxXugvX6Q4NPPH+2NKjzX4SIDejS6JjgrP3KA3O\npMuo7ZHMfveBngv8yP+ZD/1sS6l+dfExvdaJdOdgFCnp4p3gPbw5+Lv70HrMjA==\n=BfZ/\n-----END PGP PUBLIC KEY BLOCK-----\n","checksum":"sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"}],"packages":["dnf","e2fsprogs","policycoreutils","qemu-img","systemd","grub2-pc","tar"],"releasever":"30","basearch":"x86_64"}}]},"runner":"org.osbuild.fedora30"},"stages":[{"name":"org.osbuild.dnf","options":{"repos":[{"metalink":"https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever\u0026arch=$basearch","gpgkey":"-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFturGcBEACv0xBo91V2n0uEC2vh69ywCiSyvUgN/AQH8EZpCVtM7NyjKgKm\nbbY4G3R0M3ir1xXmvUDvK0493/qOiFrjkplvzXFTGpPTi0ypqGgxc5d0ohRA1M75\nL+0AIlXoOgHQ358/c4uO8X0JAA1NYxCkAW1KSJgFJ3RjukrfqSHWthS1d4o8fhHy\nKJKEnirE5hHqB50dafXrBfgZdaOs3C6ppRIePFe2o4vUEapMTCHFw0woQR8Ah4/R\nn7Z9G9Ln+0Cinmy0nbIDiZJ+pgLAXCOWBfDUzcOjDGKvcpoZharA07c0q1/5ojzO\n4F0Fh4g/BUmtrASwHfcIbjHyCSr1j/3Iz883iy07gJY5Yhiuaqmp0o0f9fgHkG53\n2xCU1owmACqaIBNQMukvXRDtB2GJMuKa/asTZDP6R5re+iXs7+s9ohcRRAKGyAyc\nYKIQKcaA+6M8T7/G+TPHZX6HJWqJJiYB+EC2ERblpvq9TPlLguEWcmvjbVc31nyq\nSDoO3ncFWKFmVsbQPTbP+pKUmlLfJwtb5XqxNR5GEXSwVv4I7IqBmJz1MmRafnBZ\ng0FJUtH668GnldO20XbnSVBr820F5SISMXVwCXDXEvGwwiB8Lt8PvqzXnGIFDAu3\nDlQI5sxSqpPVWSyw08ppKT2Tpmy8adiBotLfaCFl2VTHwOae48X2dMPBvQARAQAB\ntDFGZWRvcmEgKDMwKSA8ZmVkb3JhLTMwLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJbbqxnAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRDvPBEfz8ZZudTnD/9170LL3nyTVUCFmBjT9wZ4gYnpwtKVPa/pKnxbbS+Bmmac\ng9TrT9pZbqOHrNJLiZ3Zx1Hp+8uxr3Lo6kbYwImLhkOEDrf4aP17HfQ6VYFbQZI8\nf79OFxWJ7si9+3gfzeh9UYFEqOQfzIjLWFyfnas0OnV/P+RMQ1Zr+vPRqO7AR2va\nN9wg+Xl7157dhXPCGYnGMNSoxCbpRs0JNlzvJMuAea5nTTznRaJZtK/xKsqLn51D\nK07k9MHVFXakOH8QtMCUglbwfTfIpO5YRq5imxlWbqsYWVQy1WGJFyW6hWC0+RcJ\nOx5zGtOfi4/dN+xJ+ibnbyvy/il7Qm+vyFhCYqIPyS5m2UVJUuao3eApE38k78/o\n8aQOTnFQZ+U1Sw+6woFTxjqRQBXlQm2+7Bt3bqGATg4sXXWPbmwdL87Ic+mxn/ml\nSMfQux/5k6iAu1kQhwkO2YJn9eII6HIPkW+2m5N1JsUyJQe4cbtZE5Yh3TRA0dm7\n+zoBRfCXkOW4krchbgww/ptVmzMMP7GINJdROrJnsGl5FVeid9qHzV7aZycWSma7\nCxBYB1J8HCbty5NjtD6XMYRrMLxXugvX6Q4NPPH+2NKjzX4SIDejS6JjgrP3KA3O\npMuo7ZHMfveBngv8yP+ZD/1sS6l+dfExvdaJdOdgFCnp4p3gPbw5+Lv70HrMjA==\n=BfZ/\n-----END PGP PUBLIC KEY BLOCK-----\n","checksum":"sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"}],"packages":["policycoreutils","selinux-policy-targeted","kernel","firewalld","chrony","langpacks-en"],"exclude_packages":["dracut-config-rescue"],"releasever":"30","basearch":"x86_64"}},{"name":"org.osbuild.fix-bls","options":{}},{"name":"org.osbuild.locale","options":{"language":"en_US"}},{"name":"org.osbuild.grub2","options":{"root_fs_uuid":"76a22bf4-f153-4541-b6c7-0332c0dfaeac","boot_fs_uuid":"00000000-0000-0000-0000-000000000000","kernel_opts":"ro biosdevname=0 net.ifnames=0"}},{"name":"org.osbuild.selinux","options":{"file_contexts":"etc/selinux/targeted/contexts/files/file_contexts"}}],"assembler":{"name":"org.osbuild.tar","options":{"filename":"root.tar.xz"}}},"targets":[{"image_name":"","name":"org.osbuild.local","options":{"location":"/var/lib/osbuild-composer/outputs/ffffffff-ffff-ffff-ffff-ffffffffffff"}}]}`) + `{"id":"ffffffff-ffff-ffff-ffff-ffffffffffff","pipeline":{"build":{"pipeline":{"stages":[{"name":"org.osbuild.dnf","options":{"repos":[{"metalink":"https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever\u0026arch=$basearch","gpgkey":"-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFturGcBEACv0xBo91V2n0uEC2vh69ywCiSyvUgN/AQH8EZpCVtM7NyjKgKm\nbbY4G3R0M3ir1xXmvUDvK0493/qOiFrjkplvzXFTGpPTi0ypqGgxc5d0ohRA1M75\nL+0AIlXoOgHQ358/c4uO8X0JAA1NYxCkAW1KSJgFJ3RjukrfqSHWthS1d4o8fhHy\nKJKEnirE5hHqB50dafXrBfgZdaOs3C6ppRIePFe2o4vUEapMTCHFw0woQR8Ah4/R\nn7Z9G9Ln+0Cinmy0nbIDiZJ+pgLAXCOWBfDUzcOjDGKvcpoZharA07c0q1/5ojzO\n4F0Fh4g/BUmtrASwHfcIbjHyCSr1j/3Iz883iy07gJY5Yhiuaqmp0o0f9fgHkG53\n2xCU1owmACqaIBNQMukvXRDtB2GJMuKa/asTZDP6R5re+iXs7+s9ohcRRAKGyAyc\nYKIQKcaA+6M8T7/G+TPHZX6HJWqJJiYB+EC2ERblpvq9TPlLguEWcmvjbVc31nyq\nSDoO3ncFWKFmVsbQPTbP+pKUmlLfJwtb5XqxNR5GEXSwVv4I7IqBmJz1MmRafnBZ\ng0FJUtH668GnldO20XbnSVBr820F5SISMXVwCXDXEvGwwiB8Lt8PvqzXnGIFDAu3\nDlQI5sxSqpPVWSyw08ppKT2Tpmy8adiBotLfaCFl2VTHwOae48X2dMPBvQARAQAB\ntDFGZWRvcmEgKDMwKSA8ZmVkb3JhLTMwLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJbbqxnAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRDvPBEfz8ZZudTnD/9170LL3nyTVUCFmBjT9wZ4gYnpwtKVPa/pKnxbbS+Bmmac\ng9TrT9pZbqOHrNJLiZ3Zx1Hp+8uxr3Lo6kbYwImLhkOEDrf4aP17HfQ6VYFbQZI8\nf79OFxWJ7si9+3gfzeh9UYFEqOQfzIjLWFyfnas0OnV/P+RMQ1Zr+vPRqO7AR2va\nN9wg+Xl7157dhXPCGYnGMNSoxCbpRs0JNlzvJMuAea5nTTznRaJZtK/xKsqLn51D\nK07k9MHVFXakOH8QtMCUglbwfTfIpO5YRq5imxlWbqsYWVQy1WGJFyW6hWC0+RcJ\nOx5zGtOfi4/dN+xJ+ibnbyvy/il7Qm+vyFhCYqIPyS5m2UVJUuao3eApE38k78/o\n8aQOTnFQZ+U1Sw+6woFTxjqRQBXlQm2+7Bt3bqGATg4sXXWPbmwdL87Ic+mxn/ml\nSMfQux/5k6iAu1kQhwkO2YJn9eII6HIPkW+2m5N1JsUyJQe4cbtZE5Yh3TRA0dm7\n+zoBRfCXkOW4krchbgww/ptVmzMMP7GINJdROrJnsGl5FVeid9qHzV7aZycWSma7\nCxBYB1J8HCbty5NjtD6XMYRrMLxXugvX6Q4NPPH+2NKjzX4SIDejS6JjgrP3KA3O\npMuo7ZHMfveBngv8yP+ZD/1sS6l+dfExvdaJdOdgFCnp4p3gPbw5+Lv70HrMjA==\n=BfZ/\n-----END PGP PUBLIC KEY BLOCK-----\n","checksum":"sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"}],"packages":["dnf","e2fsprogs","policycoreutils","qemu-img","systemd","grub2-pc","tar"],"releasever":"30","basearch":"x86_64"}}]},"runner":"org.osbuild.fedora30"},"stages":[{"name":"org.osbuild.dnf","options":{"repos":[{"metalink":"https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever\u0026arch=$basearch","gpgkey":"-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFturGcBEACv0xBo91V2n0uEC2vh69ywCiSyvUgN/AQH8EZpCVtM7NyjKgKm\nbbY4G3R0M3ir1xXmvUDvK0493/qOiFrjkplvzXFTGpPTi0ypqGgxc5d0ohRA1M75\nL+0AIlXoOgHQ358/c4uO8X0JAA1NYxCkAW1KSJgFJ3RjukrfqSHWthS1d4o8fhHy\nKJKEnirE5hHqB50dafXrBfgZdaOs3C6ppRIePFe2o4vUEapMTCHFw0woQR8Ah4/R\nn7Z9G9Ln+0Cinmy0nbIDiZJ+pgLAXCOWBfDUzcOjDGKvcpoZharA07c0q1/5ojzO\n4F0Fh4g/BUmtrASwHfcIbjHyCSr1j/3Iz883iy07gJY5Yhiuaqmp0o0f9fgHkG53\n2xCU1owmACqaIBNQMukvXRDtB2GJMuKa/asTZDP6R5re+iXs7+s9ohcRRAKGyAyc\nYKIQKcaA+6M8T7/G+TPHZX6HJWqJJiYB+EC2ERblpvq9TPlLguEWcmvjbVc31nyq\nSDoO3ncFWKFmVsbQPTbP+pKUmlLfJwtb5XqxNR5GEXSwVv4I7IqBmJz1MmRafnBZ\ng0FJUtH668GnldO20XbnSVBr820F5SISMXVwCXDXEvGwwiB8Lt8PvqzXnGIFDAu3\nDlQI5sxSqpPVWSyw08ppKT2Tpmy8adiBotLfaCFl2VTHwOae48X2dMPBvQARAQAB\ntDFGZWRvcmEgKDMwKSA8ZmVkb3JhLTMwLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJbbqxnAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRDvPBEfz8ZZudTnD/9170LL3nyTVUCFmBjT9wZ4gYnpwtKVPa/pKnxbbS+Bmmac\ng9TrT9pZbqOHrNJLiZ3Zx1Hp+8uxr3Lo6kbYwImLhkOEDrf4aP17HfQ6VYFbQZI8\nf79OFxWJ7si9+3gfzeh9UYFEqOQfzIjLWFyfnas0OnV/P+RMQ1Zr+vPRqO7AR2va\nN9wg+Xl7157dhXPCGYnGMNSoxCbpRs0JNlzvJMuAea5nTTznRaJZtK/xKsqLn51D\nK07k9MHVFXakOH8QtMCUglbwfTfIpO5YRq5imxlWbqsYWVQy1WGJFyW6hWC0+RcJ\nOx5zGtOfi4/dN+xJ+ibnbyvy/il7Qm+vyFhCYqIPyS5m2UVJUuao3eApE38k78/o\n8aQOTnFQZ+U1Sw+6woFTxjqRQBXlQm2+7Bt3bqGATg4sXXWPbmwdL87Ic+mxn/ml\nSMfQux/5k6iAu1kQhwkO2YJn9eII6HIPkW+2m5N1JsUyJQe4cbtZE5Yh3TRA0dm7\n+zoBRfCXkOW4krchbgww/ptVmzMMP7GINJdROrJnsGl5FVeid9qHzV7aZycWSma7\nCxBYB1J8HCbty5NjtD6XMYRrMLxXugvX6Q4NPPH+2NKjzX4SIDejS6JjgrP3KA3O\npMuo7ZHMfveBngv8yP+ZD/1sS6l+dfExvdaJdOdgFCnp4p3gPbw5+Lv70HrMjA==\n=BfZ/\n-----END PGP PUBLIC KEY BLOCK-----\n","checksum":"sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"}],"packages":["policycoreutils","selinux-policy-targeted","kernel","firewalld","chrony","langpacks-en"],"exclude_packages":["dracut-config-rescue"],"releasever":"30","basearch":"x86_64"}},{"name":"org.osbuild.fix-bls","options":{}},{"name":"org.osbuild.locale","options":{"language":"en_US"}},{"name":"org.osbuild.grub2","options":{"root_fs_uuid":"76a22bf4-f153-4541-b6c7-0332c0dfaeac","boot_fs_uuid":"00000000-0000-0000-0000-000000000000","kernel_opts":"ro biosdevname=0 net.ifnames=0"}},{"name":"org.osbuild.selinux","options":{"file_contexts":"etc/selinux/targeted/contexts/files/file_contexts"}}],"assembler":{"name":"org.osbuild.tar","options":{"filename":"root.tar.xz"}}},"targets":[{"image_name":"","name":"org.osbuild.local","options":{"location":"/var/lib/osbuild-composer/outputs/ffffffff-ffff-ffff-ffff-ffffffffffff"},"status":"RUNNING"}]}`, "created", "uuid") } func testUpdateTransition(t *testing.T, from, to string, expectedStatus int) { @@ -58,7 +58,7 @@ func testUpdateTransition(t *testing.T, from, to string, expectedStatus int) { api := jobqueue.New(nil, store) if from != "VOID" { - err := store.PushCompose(id, &blueprint.Blueprint{}, "tar") + err := store.PushCompose(id, &blueprint.Blueprint{}, "tar", nil) if err != nil { t.Fatalf("error pushing compose: %v", err) } diff --git a/internal/mocks/rpmmd/fixtures.go b/internal/mocks/rpmmd/fixtures.go index 65c4662ff..2bfbb9504 100644 --- a/internal/mocks/rpmmd/fixtures.go +++ b/internal/mocks/rpmmd/fixtures.go @@ -10,6 +10,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/distro" "github.com/osbuild/osbuild-composer/internal/rpmmd" "github.com/osbuild/osbuild-composer/internal/store" + "github.com/osbuild/osbuild-composer/internal/target" ) type FixtureGenerator func() Fixture @@ -54,44 +55,78 @@ func generatePackageList() rpmmd.PackageList { func createBaseStoreFixture() *store.Store { var bName = "test" - var b = blueprint.Blueprint{Name: bName, Version: "0.0.0"} + var b = blueprint.Blueprint{ + Name: bName, + Version: "0.0.0", + Packages: []blueprint.Package{}, + Modules: []blueprint.Package{}, + Groups: []blueprint.Group{}, + Customizations: nil, + } var date = time.Date(2019, 11, 27, 13, 19, 0, 0, time.FixedZone("UTC+1", 60*60)) + + var localTarget = &target.Target{ + Uuid: uuid.MustParse("20000000-0000-0000-0000-000000000000"), + Name: "org.osbuild.local", + ImageName: "localimage", + Created: date, + Status: "WAITING", + Options: &target.LocalTargetOptions{ + Location: "/tmp/localimage", + }, + } + + var awsTarget = &target.Target{ + Uuid: uuid.MustParse("10000000-0000-0000-0000-000000000000"), + Name: "org.osbuild.aws", + ImageName: "awsimage", + Created: date, + Status: "WAITING", + Options: &target.AWSTargetOptions{ + Region: "frankfurt", + AccessKeyID: "accesskey", + SecretAccessKey: "secretkey", + Bucket: "clay", + Key: "imagekey", + }, + } + d := distro.New("fedora-30") s := store.New(nil, d) s.Blueprints[bName] = b s.Composes = map[uuid.UUID]store.Compose{ - uuid.MustParse("e65f76f8-b0d9-4974-9dd7-745ae80b4721"): store.Compose{ + uuid.MustParse("30000000-0000-0000-0000-000000000000"): store.Compose{ QueueStatus: "WAITING", Blueprint: &b, OutputType: "tar", - Targets: nil, + Targets: []*target.Target{localTarget, awsTarget}, JobCreated: date, }, - uuid.MustParse("e65f76f8-b0d9-4974-9dd7-745ae80b4722"): store.Compose{ + uuid.MustParse("30000000-0000-0000-0000-000000000001"): store.Compose{ QueueStatus: "RUNNING", Blueprint: &b, OutputType: "tar", - Targets: nil, + Targets: []*target.Target{localTarget}, JobCreated: date, JobStarted: date, }, - uuid.MustParse("e65f76f8-b0d9-4974-9dd7-745ae80b4723"): store.Compose{ + uuid.MustParse("30000000-0000-0000-0000-000000000002"): store.Compose{ QueueStatus: "FINISHED", Blueprint: &b, OutputType: "tar", - Targets: nil, + Targets: []*target.Target{localTarget, awsTarget}, JobCreated: date, JobStarted: date, JobFinished: date, }, - uuid.MustParse("e65f76f8-b0d9-4974-9dd7-745ae80b4724"): store.Compose{ + uuid.MustParse("30000000-0000-0000-0000-000000000003"): store.Compose{ QueueStatus: "FAILED", Blueprint: &b, OutputType: "tar", - Targets: nil, + Targets: []*target.Target{localTarget, awsTarget}, JobCreated: date, JobStarted: date, JobFinished: date, @@ -101,6 +136,44 @@ func createBaseStoreFixture() *store.Store { return s } +func createBaseDepsolveFixture() []rpmmd.PackageSpec { + return []rpmmd.PackageSpec{ + { + Name: "dep-package1", + Epoch: 0, + Version: "1.33", + Release: "2.fc30", + Arch: "x86_64", + }, + { + Name: "dep-package2", + Epoch: 0, + Version: "2.9", + Release: "1.fc30", + Arch: "x86_64", + }, + } +} + +func createStoreWithoutComposesFixture() *store.Store { + var bName = "test" + var b = blueprint.Blueprint{ + Name: bName, + Version: "0.0.0", + Packages: []blueprint.Package{}, + Modules: []blueprint.Package{}, + Groups: []blueprint.Group{}, + Customizations: nil, + } + + d := distro.New("fedora-30") + s := store.New(nil, d) + + s.Blueprints[bName] = b + + return s +} + func BaseFixture() Fixture { return Fixture{ fetchPackageList{ @@ -108,28 +181,27 @@ func BaseFixture() Fixture { nil, }, depsolve{ - []rpmmd.PackageSpec{ - { - Name: "dep-package1", - Epoch: 0, - Version: "1.33", - Release: "2.fc30", - Arch: "x86_64", - }, - { - Name: "dep-package2", - Epoch: 0, - Version: "2.9", - Release: "1.fc30", - Arch: "x86_64", - }, - }, + createBaseDepsolveFixture(), nil, }, createBaseStoreFixture(), } } +func NoComposesFixture() Fixture { + return Fixture{ + fetchPackageList{ + generatePackageList(), + nil, + }, + depsolve{ + createBaseDepsolveFixture(), + nil, + }, + createStoreWithoutComposesFixture(), + } +} + func NonExistingPackage() Fixture { return Fixture{ fetchPackageList{ diff --git a/internal/store/store.go b/internal/store/store.go index 73d04e005..8fc57dc71 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -220,95 +220,26 @@ func (s *Store) ListBlueprints() []string { return names } -type ComposeEntry struct { - ID uuid.UUID `json:"id"` - Blueprint string `json:"blueprint"` - Version string `json:"version"` - ComposeType string `json:"compose_type"` - ImageSize int64 `json:"image_size"` - QueueStatus string `json:"queue_status"` - JobCreated float64 `json:"job_created"` - JobStarted float64 `json:"job_started,omitempty"` - JobFinished float64 `json:"job_finished,omitempty"` -} - -func (s *Store) ListQueue(uuids []uuid.UUID) []*ComposeEntry { +func (s *Store) GetAllComposes() map[uuid.UUID]Compose { s.mu.RLock() defer s.mu.RUnlock() - newCompose := func(id uuid.UUID, compose Compose) *ComposeEntry { - switch compose.QueueStatus { - case "WAITING": - return &ComposeEntry{ - ID: id, - Blueprint: compose.Blueprint.Name, - Version: compose.Blueprint.Version, - ComposeType: compose.OutputType, - QueueStatus: compose.QueueStatus, - JobCreated: float64(compose.JobCreated.UnixNano()) / 1000000000, - } - case "RUNNING": - return &ComposeEntry{ - ID: id, - Blueprint: compose.Blueprint.Name, - Version: compose.Blueprint.Version, - ComposeType: compose.OutputType, - QueueStatus: compose.QueueStatus, - JobCreated: float64(compose.JobCreated.UnixNano()) / 1000000000, - JobStarted: float64(compose.JobStarted.UnixNano()) / 1000000000, - } - case "FINISHED": - image, err := s.GetImage(id) - imageSize := int64(0) - if err == nil { - imageSize = image.Size - } - return &ComposeEntry{ - ID: id, - Blueprint: compose.Blueprint.Name, - Version: compose.Blueprint.Version, - ComposeType: compose.OutputType, - ImageSize: imageSize, - QueueStatus: compose.QueueStatus, - JobCreated: float64(compose.JobCreated.UnixNano()) / 1000000000, - JobStarted: float64(compose.JobStarted.UnixNano()) / 1000000000, - JobFinished: float64(compose.JobFinished.UnixNano()) / 1000000000, - } - case "FAILED": - return &ComposeEntry{ - ID: id, - Blueprint: compose.Blueprint.Name, - Version: compose.Blueprint.Version, - ComposeType: compose.OutputType, - QueueStatus: compose.QueueStatus, - JobCreated: float64(compose.JobCreated.UnixNano()) / 1000000000, - JobStarted: float64(compose.JobStarted.UnixNano()) / 1000000000, - JobFinished: float64(compose.JobFinished.UnixNano()) / 1000000000, - } - default: - panic("invalid compose state") - } - } + composes := make(map[uuid.UUID]Compose) - var composes []*ComposeEntry - if uuids == nil { - composes = make([]*ComposeEntry, 0, len(s.Composes)) - for id, compose := range s.Composes { - composes = append(composes, newCompose(id, compose)) - } - } else { - composes = make([]*ComposeEntry, 0, len(uuids)) - for _, id := range uuids { - if compose, exists := s.Composes[id]; exists { - composes = append(composes, newCompose(id, compose)) - } - } - } + for id, compose := range s.Composes { + newCompose := compose + newCompose.Targets = []*target.Target{} - // make this function output more predictable - sort.Slice(composes, func(i, j int) bool { - return composes[i].ID.String() < composes[j].ID.String() - }) + for _, t := range compose.Targets { + newTarget := *t + newCompose.Targets = append(newCompose.Targets, &newTarget) + } + + newBlueprint := *compose.Blueprint + newCompose.Blueprint = &newBlueprint + + composes[id] = newCompose + } return composes } @@ -448,7 +379,7 @@ func (s *Store) DeleteBlueprintFromWorkspace(name string) { }) } -func (s *Store) PushCompose(composeID uuid.UUID, bp *blueprint.Blueprint, composeType string) error { +func (s *Store) PushCompose(composeID uuid.UUID, bp *blueprint.Blueprint, composeType string, uploadTarget *target.Target) error { targets := []*target.Target{ target.NewLocalTarget( &target.LocalTargetOptions{ @@ -456,6 +387,11 @@ func (s *Store) PushCompose(composeID uuid.UUID, bp *blueprint.Blueprint, compos }, ), } + + if uploadTarget != nil { + targets = append(targets, uploadTarget) + } + pipeline, err := s.distro.Pipeline(bp, composeType) if err != nil { return err @@ -488,6 +424,9 @@ func (s *Store) PopCompose() Job { } compose.JobStarted = time.Now() compose.QueueStatus = "RUNNING" + for _, t := range compose.Targets { + t.Status = "RUNNING" + } s.Composes[job.ComposeID] = compose return nil }) @@ -518,6 +457,9 @@ func (s *Store) UpdateCompose(composeID uuid.UUID, status string) error { return &NotRunningError{"compose was not running"} } compose.QueueStatus = status + for _, t := range compose.Targets { + t.Status = status + } s.Composes[composeID] = compose default: return &InvalidRequestError{"invalid state transition"} diff --git a/internal/weldr/api.go b/internal/weldr/api.go index 81894d62b..b31a631f7 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -15,10 +15,10 @@ import ( "github.com/julienschmidt/httprouter" "github.com/osbuild/osbuild-composer/internal/blueprint" + "github.com/osbuild/osbuild-composer/internal/distro" "github.com/osbuild/osbuild-composer/internal/rpmmd" "github.com/osbuild/osbuild-composer/internal/store" - - "github.com/osbuild/osbuild-composer/internal/distro" + "github.com/osbuild/osbuild-composer/internal/target" ) type API struct { @@ -137,6 +137,18 @@ func verifyRequestVersion(writer http.ResponseWriter, params httprouter.Params, return true } +func isRequestVersionAtLeast(params httprouter.Params, minVersion uint) bool { + versionString := params.ByName("version") + + version, err := strconv.ParseUint(versionString, 10, 0) + + if err != nil { + panic(err) + } + + return uint(version) >= minVersion +} + func methodNotAllowedHandler(writer http.ResponseWriter, request *http.Request) { writer.WriteHeader(http.StatusMethodNotAllowed) } @@ -1110,15 +1122,16 @@ func (api *API) blueprintDeleteWorkspaceHandler(writer http.ResponseWriter, requ // Schedule new compose by first translating the appropriate blueprint into a pipeline and then // pushing it into the channel for waiting builds. func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { - if !verifyRequestVersion(writer, params, 0) { // TODO: version 1 API + if !verifyRequestVersion(writer, params, 0) { return } // https://weldr.io/lorax/pylorax.api.html#pylorax.api.v0.v0_compose_start type ComposeRequest struct { - BlueprintName string `json:"blueprint_name"` - ComposeType string `json:"compose_type"` - Branch string `json:"branch"` + BlueprintName string `json:"blueprint_name"` + ComposeType string `json:"compose_type"` + Branch string `json:"branch"` + Upload *UploadRequest `json:"upload"` } type ComposeReply struct { BuildID uuid.UUID `json:"build_id"` @@ -1152,12 +1165,27 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request Status: true, } + var uploadTarget *target.Target + if isRequestVersionAtLeast(params, 1) && cr.Upload != nil { + uploadTarget, err = UploadRequestToTarget(*cr.Upload) + + if err != nil { + errors := responseError{ + ID: "UploadError", + Msg: fmt.Sprintf("bad input format: %s", err.Error()), + } + + statusResponseError(writer, http.StatusBadRequest, errors) + return + } + } + bp := blueprint.Blueprint{} changed := false found := api.store.GetBlueprint(cr.BlueprintName, &bp, &changed) // TODO: what to do with changed? if found { - err := api.store.PushCompose(reply.BuildID, &bp, cr.ComposeType) + err := api.store.PushCompose(reply.BuildID, &bp, cr.ComposeType, uploadTarget) // TODO: we should probably do some kind of blueprint validation in future // for now, let's just 500 and bail out @@ -1183,7 +1211,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request } func (api *API) composeTypesHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { - if !verifyRequestVersion(writer, params, 0) { // TODO: version 1 API + if !verifyRequestVersion(writer, params, 0) { return } type composeType struct { @@ -1203,24 +1231,22 @@ func (api *API) composeTypesHandler(writer http.ResponseWriter, request *http.Re } func (api *API) composeQueueHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { - if !verifyRequestVersion(writer, params, 0) { // TODO: version 1 API + if !verifyRequestVersion(writer, params, 0) { return } var reply struct { - New []*store.ComposeEntry `json:"new"` - Run []*store.ComposeEntry `json:"run"` + New []*ComposeEntry `json:"new"` + Run []*ComposeEntry `json:"run"` } - reply.New = make([]*store.ComposeEntry, 0) - reply.Run = make([]*store.ComposeEntry, 0) - - for _, entry := range api.store.ListQueue(nil) { - switch entry.QueueStatus { + composes := api.store.GetAllComposes() + for id, compose := range composes { + switch compose.QueueStatus { case "WAITING": - reply.New = append(reply.New, entry) + reply.New = append(reply.New, composeToComposeEntry(id, compose, isRequestVersionAtLeast(params, 1))) case "RUNNING": - reply.Run = append(reply.Run, entry) + reply.Run = append(reply.Run, composeToComposeEntry(id, compose, isRequestVersionAtLeast(params, 1))) } } @@ -1229,12 +1255,12 @@ func (api *API) composeQueueHandler(writer http.ResponseWriter, request *http.Re func (api *API) composeStatusHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { // TODO: lorax has some params: /api/v0/compose/status/[?blueprint=&status=&type=] - if !verifyRequestVersion(writer, params, 0) { // TODO: version 1 API + if !verifyRequestVersion(writer, params, 0) { return } var reply struct { - UUIDs []*store.ComposeEntry `json:"uuids"` + UUIDs []*ComposeEntry `json:"uuids"` } uuidsParam := params.ByName("uuids") @@ -1257,7 +1283,9 @@ func (api *API) composeStatusHandler(writer http.ResponseWriter, request *http.R uuids = append(uuids, id) } } - reply.UUIDs = api.store.ListQueue(uuids) + composes := api.store.GetAllComposes() + + reply.UUIDs = composesToComposeEntries(composes, uuids, isRequestVersionAtLeast(params, 1)) json.NewEncoder(writer).Encode(reply) } @@ -1305,29 +1333,41 @@ func (api *API) composeImageHandler(writer http.ResponseWriter, request *http.Re } func (api *API) composeFinishedHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { - if !verifyRequestVersion(writer, params, 0) { // TODO: version 1 API + if !verifyRequestVersion(writer, params, 0) { return } var reply struct { - Finished []interface{} `json:"finished"` + Finished []*ComposeEntry `json:"finished"` } - reply.Finished = make([]interface{}, 0) + composes := api.store.GetAllComposes() + for _, entry := range composesToComposeEntries(composes, nil, isRequestVersionAtLeast(params, 1)) { + switch entry.QueueStatus { + case "FINISHED": + reply.Finished = append(reply.Finished, entry) + } + } json.NewEncoder(writer).Encode(reply) } func (api *API) composeFailedHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { - if !verifyRequestVersion(writer, params, 0) { // TODO: version 1 API + if !verifyRequestVersion(writer, params, 0) { return } var reply struct { - Failed []interface{} `json:"failed"` + Failed []*ComposeEntry `json:"failed"` } - reply.Failed = make([]interface{}, 0) + composes := api.store.GetAllComposes() + for _, entry := range composesToComposeEntries(composes, nil, isRequestVersionAtLeast(params, 1)) { + switch entry.QueueStatus { + case "FAILED": + reply.Failed = append(reply.Failed, entry) + } + } json.NewEncoder(writer).Encode(reply) } diff --git a/internal/weldr/api_test.go b/internal/weldr/api_test.go index d6179b874..a3b1f75b2 100644 --- a/internal/weldr/api_test.go +++ b/internal/weldr/api_test.go @@ -8,20 +8,24 @@ import ( "testing" "time" + "github.com/osbuild/osbuild-composer/internal/blueprint" + "github.com/osbuild/osbuild-composer/internal/distro" + _ "github.com/osbuild/osbuild-composer/internal/distro/test" rpmmd_mock "github.com/osbuild/osbuild-composer/internal/mocks/rpmmd" + "github.com/osbuild/osbuild-composer/internal/store" + "github.com/osbuild/osbuild-composer/internal/target" "github.com/osbuild/osbuild-composer/internal/test" "github.com/osbuild/osbuild-composer/internal/weldr" - "github.com/osbuild/osbuild-composer/internal/distro" - _ "github.com/osbuild/osbuild-composer/internal/distro/test" + "github.com/google/go-cmp/cmp" ) -func createWeldrAPI(fixtureGenerator rpmmd_mock.FixtureGenerator) *weldr.API { +func createWeldrAPI(fixtureGenerator rpmmd_mock.FixtureGenerator) (*weldr.API, *store.Store) { fixture := fixtureGenerator() rpm := rpmmd_mock.NewRPMMDMock(fixture) d := distro.New("test") - return weldr.New(rpm, d, nil, fixture.Store) + return weldr.New(rpm, d, nil, fixture.Store), fixture.Store } func TestBasic(t *testing.T) { @@ -46,7 +50,7 @@ func TestBasic(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(rpmmd_mock.BaseFixture) + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) test.TestRoute(t, api, true, "GET", c.Path, ``, c.ExpectedStatus, c.ExpectedJSON) } } @@ -63,7 +67,7 @@ func TestBlueprintsNew(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(rpmmd_mock.BaseFixture) + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) test.TestRoute(t, api, true, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON) } } @@ -80,7 +84,7 @@ func TestBlueprintsWorkspace(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(rpmmd_mock.BaseFixture) + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) test.SendHTTP(api, true, "POST", "/api/v0/blueprints/new", `{"name":"test","description":"Test","packages":[{"name":"httpd","version":"2.4.*"}],"version":"0.0.0"}`) test.TestRoute(t, api, true, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON) } @@ -101,7 +105,7 @@ func TestBlueprintsInfo(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(rpmmd_mock.BaseFixture) + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) test.SendHTTP(api, true, "POST", "/api/v0/blueprints/new", `{"name":"test1","description":"Test","packages":[{"name":"httpd","version":"2.4.*"}],"version":"0.0.0"}`) test.SendHTTP(api, true, "POST", "/api/v0/blueprints/new", `{"name":"test2","description":"Test","packages":[{"name":"httpd","version":"2.4.*"}],"version":"0.0.0"}`) test.SendHTTP(api, true, "POST", "/api/v0/blueprints/workspace", `{"name":"test2","description":"Test","packages":[{"name":"systemd","version":"123"}],"version":"0.0.0"}`) @@ -122,7 +126,7 @@ func TestBlueprintsFreeze(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(c.Fixture) + api, _ := createWeldrAPI(c.Fixture) test.SendHTTP(api, false, "POST", "/api/v0/blueprints/new", `{"name":"test","description":"Test","packages":[{"name":"dep-package1","version":"*"}],"version":"0.0.0"}`) test.TestRoute(t, api, false, "GET", c.Path, ``, c.ExpectedStatus, c.ExpectedJSON) test.SendHTTP(api, false, "DELETE", "/api/v0/blueprints/delete/test", ``) @@ -141,7 +145,7 @@ func TestBlueprintsDiff(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(rpmmd_mock.BaseFixture) + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) test.SendHTTP(api, true, "POST", "/api/v0/blueprints/new", `{"name":"test","description":"Test","packages":[{"name":"httpd","version":"2.4.*"}],"version":"0.0.0"}`) test.SendHTTP(api, true, "POST", "/api/v0/blueprints/workspace", `{"name":"test","description":"Test","packages":[{"name":"systemd","version":"123"}],"version":"0.0.0"}`) test.TestRoute(t, api, true, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON) @@ -161,7 +165,7 @@ func TestBlueprintsDelete(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(rpmmd_mock.BaseFixture) + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) test.SendHTTP(api, true, "POST", "/api/v0/blueprints/new", `{"name":"test","description":"Test","packages":[{"name":"httpd","version":"2.4.*"}],"version":"0.0.0"}`) test.TestRoute(t, api, true, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON) test.SendHTTP(api, true, "DELETE", "/api/v0/blueprints/delete/test", ``) @@ -169,7 +173,7 @@ func TestBlueprintsDelete(t *testing.T) { } func TestBlueprintsChanges(t *testing.T) { - api := createWeldrAPI(rpmmd_mock.BaseFixture) + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) rand.Seed(time.Now().UnixNano()) id := strconv.Itoa(rand.Int()) ignoreFields := []string{"commit", "timestamp"} @@ -184,24 +188,121 @@ func TestBlueprintsChanges(t *testing.T) { } func TestCompose(t *testing.T) { + expectedComposeLocal := &store.Compose{ + QueueStatus: "WAITING", + Blueprint: &blueprint.Blueprint{ + Name: "test", + Version: "0.0.0", + Packages: []blueprint.Package{}, + Modules: []blueprint.Package{}, + Groups: []blueprint.Group{}, + Customizations: nil, + }, + OutputType: "tar", + Targets: []*target.Target{ + { + Name: "org.osbuild.local", + Created: time.Time{}, + Status: "WAITING", + Options: &target.LocalTargetOptions{}, + }, + }, + } + + expectedComposeLocalAndAws := &store.Compose{ + QueueStatus: "WAITING", + Blueprint: &blueprint.Blueprint{ + Name: "test", + Version: "0.0.0", + Packages: []blueprint.Package{}, + Modules: []blueprint.Package{}, + Groups: []blueprint.Group{}, + Customizations: nil, + }, + OutputType: "tar", + Targets: []*target.Target{ + { + Name: "org.osbuild.local", + Status: "WAITING", + Options: &target.LocalTargetOptions{}, + }, + { + Name: "org.osbuild.aws", + Status: "WAITING", + ImageName: "test_upload", + Options: &target.AWSTargetOptions{ + Region: "frankfurt", + AccessKeyID: "accesskey", + SecretAccessKey: "secretkey", + Bucket: "clay", + Key: "imagekey", + }, + }, + }, + } + var cases = []struct { - External bool + External bool + Method string + Path string + Body string + ExpectedStatus int + ExpectedJSON string + ExpectedCompose *store.Compose + IgnoreFields []string + }{ + {true, "POST", "/api/v0/compose", `{"blueprint_name": "http-server","compose_type": "tar","branch": "master"}`, http.StatusBadRequest, `{"status":false,"errors":[{"id":"UnknownBlueprint","msg":"Unknown blueprint name: http-server"}]}`, nil, []string{"build_id"}}, + {false, "POST", "/api/v0/compose", `{"blueprint_name": "test","compose_type": "tar","branch": "master"}`, http.StatusOK, `{"status": true}`, expectedComposeLocal, []string{"build_id"}}, + {false, "POST", "/api/v1/compose", `{"blueprint_name": "test","compose_type":"tar","branch":"master","upload":{"image_name":"test_upload","provider":"aws","settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}}`, http.StatusOK, `{"status": true}`, expectedComposeLocalAndAws, []string{"build_id"}}, + } + + for _, c := range cases { + api, s := createWeldrAPI(rpmmd_mock.NoComposesFixture) + test.TestRoute(t, api, c.External, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON, c.IgnoreFields...) + + if c.ExpectedStatus != http.StatusOK { + continue + } + + if len(s.Composes) != 1 { + t.Fatalf("%s: bad compose count in store: %d", c.Path, len(s.Composes)) + } + + // I have no idea how to get the compose in better way + var compose store.Compose + for _, c := range s.Composes { + compose = c + break + } + + if diff := cmp.Diff(compose, *c.ExpectedCompose, test.IgnoreDates(), test.IgnoreUuids(), test.Ignore("Targets.Options.Location")); diff != "" { + t.Errorf("%s: compose in store isn't the same as expected, diff:\n%s", c.Path, diff) + } + + } +} + +func TestComposeStatus(t *testing.T) { + var cases = []struct { + Fixture rpmmd_mock.FixtureGenerator Method string Path string Body string ExpectedStatus int ExpectedJSON string - IgnoreFields []string }{ - {true, "POST", "/api/v0/compose", `{"blueprint_name": "http-server","compose_type": "tar","branch": "master"}`, http.StatusBadRequest, `{"status":false,"errors":[{"id":"UnknownBlueprint","msg":"Unknown blueprint name: http-server"}]}`, []string{"build_id"}}, - {false, "POST", "/api/v0/compose", `{"blueprint_name": "test","compose_type": "tar","branch": "master"}`, http.StatusOK, `{"status": true}`, []string{"build_id"}}, + {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/status/30000000-0000-0000-0000-000000000000,30000000-0000-0000-0000-000000000002", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"WAITING","job_created":1574857140},{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/status/*", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"WAITING","job_created":1574857140},{"id":"30000000-0000-0000-0000-000000000001","blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"RUNNING","job_created":1574857140,"job_started":1574857140},{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140},{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/status/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"WAITING","job_created":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`}, + } + + if len(os.Getenv("OSBUILD_COMPOSER_TEST_EXTERNAL")) > 0 { + t.Skip("This test is for internal testing only") } for _, c := range cases { - api := createWeldrAPI(rpmmd_mock.BaseFixture) - test.SendHTTP(api, c.External, "POST", "/api/v0/blueprints/new", `{"name":"test","description":"Test","packages":[{"name":"httpd","version":"2.4.*"}],"version":"0.0.0"}`) - test.TestRoute(t, api, c.External, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON, c.IgnoreFields...) - test.SendHTTP(api, c.External, "DELETE", "/api/v0/blueprints/delete/test", ``) + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) + test.TestRoute(t, api, false, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON, "id", "job_created", "job_started") } } @@ -213,9 +314,9 @@ func TestComposeQueue(t *testing.T) { Body string ExpectedStatus int ExpectedJSON string - IgnoreFields []string }{ - {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/queue", ``, http.StatusOK, `{"new":[{"blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"WAITING"}],"run":[{"blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"RUNNING"}]}`, []string{"id", "job_created", "job_started"}}, + {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/queue", ``, http.StatusOK, `{"new":[{"blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"WAITING"}],"run":[{"blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"RUNNING"}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/queue", ``, http.StatusOK, `{"new":[{"blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"WAITING","uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}],"run":[{"blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"RUNNING"}]}`}, } if len(os.Getenv("OSBUILD_COMPOSER_TEST_EXTERNAL")) > 0 { @@ -223,8 +324,54 @@ func TestComposeQueue(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(rpmmd_mock.BaseFixture) - test.TestRoute(t, api, false, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON, c.IgnoreFields...) + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) + test.TestRoute(t, api, false, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON, "id", "job_created", "job_started") + } +} + +func TestComposeFinished(t *testing.T) { + var cases = []struct { + Fixture rpmmd_mock.FixtureGenerator + Method string + Path string + Body string + ExpectedStatus int + ExpectedJSON string + }{ + {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/finished", ``, http.StatusOK, `{"finished":[{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/finished", ``, http.StatusOK, `{"finished":[{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`}, + } + + if len(os.Getenv("OSBUILD_COMPOSER_TEST_EXTERNAL")) > 0 { + t.Skip("This test is for internal testing only") + } + + for _, c := range cases { + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) + test.TestRoute(t, api, false, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON, "id", "job_created", "job_started") + } +} + +func TestComposeFailed(t *testing.T) { + var cases = []struct { + Fixture rpmmd_mock.FixtureGenerator + Method string + Path string + Body string + ExpectedStatus int + ExpectedJSON string + }{ + {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/failed", ``, http.StatusOK, `{"failed":[{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/failed", ``, http.StatusOK, `{"failed":[{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"tar","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`}, + } + + if len(os.Getenv("OSBUILD_COMPOSER_TEST_EXTERNAL")) > 0 { + t.Skip("This test is for internal testing only") + } + + for _, c := range cases { + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) + test.TestRoute(t, api, false, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON, "id", "job_created", "job_started") } } @@ -241,7 +388,7 @@ func TestSourcesNew(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(rpmmd_mock.BaseFixture) + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) test.TestRoute(t, api, true, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON) test.SendHTTP(api, true, "DELETE", "/api/v0/projects/source/delete/fish", ``) } @@ -260,7 +407,7 @@ func TestSourcesDelete(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(rpmmd_mock.BaseFixture) + api, _ := createWeldrAPI(rpmmd_mock.BaseFixture) test.SendHTTP(api, true, "POST", "/api/v0/projects/source/new", `{"name": "fish","url": "https://download.opensuse.org/repositories/shells:/fish:/release:/3/Fedora_29/","type": "yum-baseurl","check_ssl": false,"check_gpg": false}`) test.TestRoute(t, api, true, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON) test.SendHTTP(api, true, "DELETE", "/api/v0/projects/source/delete/fish", ``) @@ -280,7 +427,7 @@ func TestProjectsDepsolve(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(c.Fixture) + api, _ := createWeldrAPI(c.Fixture) test.TestRoute(t, api, true, "GET", c.Path, ``, c.ExpectedStatus, c.ExpectedJSON) } } @@ -301,7 +448,7 @@ func TestProjectsInfo(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(c.Fixture) + api, _ := createWeldrAPI(c.Fixture) test.TestRoute(t, api, true, "GET", c.Path, ``, c.ExpectedStatus, c.ExpectedJSON) } } @@ -323,7 +470,7 @@ func TestModulesInfo(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(c.Fixture) + api, _ := createWeldrAPI(c.Fixture) test.TestRoute(t, api, true, "GET", c.Path, ``, c.ExpectedStatus, c.ExpectedJSON) } } @@ -342,7 +489,7 @@ func TestProjectsList(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(c.Fixture) + api, _ := createWeldrAPI(c.Fixture) test.TestRoute(t, api, true, "GET", c.Path, ``, c.ExpectedStatus, c.ExpectedJSON) } } @@ -364,7 +511,7 @@ func TestModulesList(t *testing.T) { } for _, c := range cases { - api := createWeldrAPI(c.Fixture) + api, _ := createWeldrAPI(c.Fixture) test.TestRoute(t, api, true, "GET", c.Path, ``, c.ExpectedStatus, c.ExpectedJSON) } } diff --git a/internal/weldr/compose.go b/internal/weldr/compose.go new file mode 100644 index 000000000..80c2ccdd8 --- /dev/null +++ b/internal/weldr/compose.go @@ -0,0 +1,88 @@ +package weldr + +import ( + "github.com/google/uuid" + "github.com/osbuild/osbuild-composer/internal/store" + "sort" +) + +type ComposeEntry struct { + ID uuid.UUID `json:"id"` + Blueprint string `json:"blueprint"` + Version string `json:"version"` + ComposeType string `json:"compose_type"` + ImageSize int64 `json:"image_size"` + QueueStatus string `json:"queue_status"` + JobCreated float64 `json:"job_created"` + JobStarted float64 `json:"job_started,omitempty"` + JobFinished float64 `json:"job_finished,omitempty"` + Uploads []UploadResponse `json:"uploads,omitempty"` +} + +func composeToComposeEntry(id uuid.UUID, compose store.Compose, includeUploads bool) *ComposeEntry { + var composeEntry ComposeEntry + + composeEntry.ID = id + composeEntry.Blueprint = compose.Blueprint.Name + composeEntry.Version = compose.Blueprint.Version + composeEntry.ComposeType = compose.OutputType + composeEntry.QueueStatus = compose.QueueStatus + + if includeUploads { + composeEntry.Uploads = TargetsToUploadResponses(compose.Targets) + } + + switch compose.QueueStatus { + case "WAITING": + composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000 + + case "RUNNING": + composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000 + composeEntry.JobStarted = float64(compose.JobStarted.UnixNano()) / 1000000000 + + case "FINISHED": + //image, err := s.GetImage(id) + //imageSize := int64(0) + //if err == nil { + // imageSize = image.Size + //} + // TODO: this is currently broken! + composeEntry.ImageSize = int64(0) + composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000 + composeEntry.JobStarted = float64(compose.JobStarted.UnixNano()) / 1000000000 + composeEntry.JobFinished = float64(compose.JobFinished.UnixNano()) / 1000000000 + + case "FAILED": + composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000 + composeEntry.JobStarted = float64(compose.JobStarted.UnixNano()) / 1000000000 + composeEntry.JobFinished = float64(compose.JobFinished.UnixNano()) / 1000000000 + default: + panic("invalid compose state") + } + + return &composeEntry +} + +func composesToComposeEntries(composes map[uuid.UUID]store.Compose, uuids []uuid.UUID, includeUploads bool) []*ComposeEntry { + var composeEntries []*ComposeEntry + if uuids == nil { + composeEntries = make([]*ComposeEntry, 0, len(composes)) + for id, compose := range composes { + composeEntries = append(composeEntries, composeToComposeEntry(id, compose, includeUploads)) + } + } else { + composeEntries = make([]*ComposeEntry, 0, len(uuids)) + for _, id := range uuids { + if compose, exists := composes[id]; exists { + composeEntries = append(composeEntries, composeToComposeEntry(id, compose, includeUploads)) + } + } + } + + // make this function output more predictable + sort.Slice(composeEntries, func(i, j int) bool { + return composeEntries[i].ID.String() < composeEntries[j].ID.String() + }) + + return composeEntries +} diff --git a/internal/weldr/upload.go b/internal/weldr/upload.go new file mode 100644 index 000000000..44ef52457 --- /dev/null +++ b/internal/weldr/upload.go @@ -0,0 +1,110 @@ +package weldr + +import ( + "encoding/json" + "errors" + "time" + + "github.com/google/uuid" + "github.com/osbuild/osbuild-composer/internal/target" +) + +type UploadResponse struct { + Uuid uuid.UUID `json:"uuid"` + Status string `json:"status"` + ProviderName string `json:"provider_name"` + ImageName string `json:"image_name"` + CreationTime float64 `json:"creation_time"` + Settings target.TargetOptions `json:"settings"` +} + +type UploadRequest struct { + Provider string `json:"provider"` + ImageName string `json:"image_name"` + Settings target.TargetOptions `json:"settings"` +} + +type rawUploadRequest struct { + Provider string `json:"provider"` + ImageName string `json:"image_name"` + Settings json.RawMessage `json:"settings"` +} + +func (u *UploadRequest) UnmarshalJSON(data []byte) error { + var rawUpload rawUploadRequest + err := json.Unmarshal(data, &rawUpload) + if err != nil { + return err + } + + // we need to convert provider name to target name to use the unmarshaller + targetName := providerNameToTargetNameMap[rawUpload.Provider] + options, err := target.UnmarshalTargetOptions(targetName, rawUpload.Settings) + + u.Provider = rawUpload.Provider + u.ImageName = rawUpload.ImageName + u.Settings = options + + return err +} + +var targetNameToProviderNameMap = map[string]string{ + "org.osbuild.aws": "aws", + "org.osbuild.azure": "azure", +} + +var providerNameToTargetNameMap = map[string]string{ + "aws": "org.osbuild.aws", + "azure": "org.osbuild.azure", +} + +func targetToUploadResponse(t *target.Target) UploadResponse { + var u UploadResponse + + providerName, providerExist := targetNameToProviderNameMap[t.Name] + if !providerExist { + panic("target name " + t.Name + " is not defined in conversion map!") + } + + u.CreationTime = float64(t.Created.UnixNano()) / 1000000000 + u.ImageName = t.ImageName + u.ProviderName = providerName + u.Status = t.Status + u.Uuid = t.Uuid + u.Settings = t.Options + + return u +} + +func TargetsToUploadResponses(targets []*target.Target) []UploadResponse { + var uploads []UploadResponse + for _, t := range targets { + if t.Name == "org.osbuild.local" { + continue + } + + upload := targetToUploadResponse(t) + + uploads = append(uploads, upload) + } + + return uploads +} + +func UploadRequestToTarget(u UploadRequest) (*target.Target, error) { + var t target.Target + targetName, targetExist := providerNameToTargetNameMap[u.Provider] + + if !targetExist { + return nil, errors.New("Unknown provider name " + u.Provider) + } + + t.Uuid = uuid.New() + t.ImageName = u.ImageName + t.Options = u.Settings + t.Name = targetName + t.Status = "WAITING" + t.Created = time.Now() + + return &t, nil +}