From f89a9671be577fc7ee083b7c651b1b870602959a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Budai?= Date: Tue, 3 Dec 2019 15:05:46 +0100 Subject: [PATCH] store: add image struct into compose struct As a part of f4991cb1 ComposeEntry struct was removed from store package. This change made sense because this struct is connected more with API than with store - store uses its own Compose struct. In addition, converters between Compose and ComposeEntry were added. Unfortunately, ComposeEntry contains ImageSize which was not stored in Compose but retrieved from store using GetImage method. This made those converters dependent on the store, which was messy. To solve this issue this commit adds image struct into Compose struct. The content of image struct is generated on the worker side - when the worker sets the compose status to FINISHED, it also sends Image struct with detailed information about the result. --- cmd/osbuild-worker/main.go | 15 ++++---- internal/jobqueue/api.go | 8 ++-- internal/jobqueue/api_test.go | 2 +- internal/jobqueue/job.go | 58 +++++++++++++++++++++------- internal/store/store.go | 71 ++++++++++++----------------------- internal/weldr/api.go | 46 +++++++++++++++++++---- internal/weldr/compose.go | 15 ++++---- 7 files changed, 127 insertions(+), 88 deletions(-) diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go index 0824cf9ec..459fcd6c8 100644 --- a/cmd/osbuild-worker/main.go +++ b/cmd/osbuild-worker/main.go @@ -11,6 +11,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/distro" "github.com/osbuild/osbuild-composer/internal/jobqueue" + "github.com/osbuild/osbuild-composer/internal/store" ) type ComposerClient struct { @@ -53,9 +54,9 @@ func (c *ComposerClient) AddJob() (*jobqueue.Job, error) { return job, nil } -func (c *ComposerClient) UpdateJob(job *jobqueue.Job, status string) error { +func (c *ComposerClient) UpdateJob(job *jobqueue.Job, status string, image *store.Image) error { var b bytes.Buffer - json.NewEncoder(&b).Encode(&jobqueue.JobStatus{status}) + json.NewEncoder(&b).Encode(&jobqueue.JobStatus{status, image}) req, err := http.NewRequest("PATCH", "http://localhost/job-queue/v1/jobs/"+job.ID.String(), &b) if err != nil { return err @@ -82,23 +83,23 @@ func handleJob(client *ComposerClient, distro distro.Distro) { panic(err) } - client.UpdateJob(job, "RUNNING") + client.UpdateJob(job, "RUNNING", nil) fmt.Printf("Running job %s\n", job.ID.String()) - err, errs := job.Run(distro) + image, err, errs := job.Run(distro) if err != nil { - client.UpdateJob(job, "FAILED") + client.UpdateJob(job, "FAILED", nil) return } for _, err := range errs { if err != nil { - client.UpdateJob(job, "FAILED") + client.UpdateJob(job, "FAILED", nil) return } } - client.UpdateJob(job, "FINISHED") + client.UpdateJob(job, "FINISHED", image) } func main() { diff --git a/internal/jobqueue/api.go b/internal/jobqueue/api.go index 60c3d1aea..f00256fb6 100644 --- a/internal/jobqueue/api.go +++ b/internal/jobqueue/api.go @@ -93,12 +93,10 @@ func (api *API) addJobHandler(writer http.ResponseWriter, request *http.Request, nextJob := api.store.PopCompose() writer.WriteHeader(http.StatusCreated) - json.NewEncoder(writer).Encode(replyBody{nextJob.ComposeID, nextJob.Pipeline, nextJob.Targets}) + json.NewEncoder(writer).Encode(replyBody{nextJob.ComposeID, nextJob.Pipeline, nextJob.Targets, nextJob.OutputType}) } func (api *API) updateJobHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { - type requestBody JobStatus - contentType := request.Header["Content-Type"] if len(contentType) != 1 || contentType[0] != "application/json" { statusResponseError(writer, http.StatusUnsupportedMediaType) @@ -111,14 +109,14 @@ func (api *API) updateJobHandler(writer http.ResponseWriter, request *http.Reque return } - var body requestBody + var body JobStatus err = json.NewDecoder(request.Body).Decode(&body) if err != nil { statusResponseError(writer, http.StatusBadRequest, "invalid status: "+err.Error()) return } - err = api.store.UpdateCompose(id, body.Status) + err = api.store.UpdateCompose(id, body.Status, body.Image) if err != nil { switch err.(type) { case *store.NotFoundError: diff --git a/internal/jobqueue/api_test.go b/internal/jobqueue/api_test.go index 34bdf1daf..86217ff92 100644 --- a/internal/jobqueue/api_test.go +++ b/internal/jobqueue/api_test.go @@ -49,7 +49,7 @@ func TestCreate(t *testing.T) { } test.TestRoute(t, api, false, "POST", "/job-queue/v1/jobs", `{}`, http.StatusCreated, - `{"id":"ffffffff-ffff-ffff-ffff-ffffffffffff","pipeline":{"build":{"pipeline":{"stages":[{"name":"org.osbuild.dnf","options":{"repos":[{"metalink":"https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever\u0026arch=$basearch","gpgkey":"-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFturGcBEACv0xBo91V2n0uEC2vh69ywCiSyvUgN/AQH8EZpCVtM7NyjKgKm\nbbY4G3R0M3ir1xXmvUDvK0493/qOiFrjkplvzXFTGpPTi0ypqGgxc5d0ohRA1M75\nL+0AIlXoOgHQ358/c4uO8X0JAA1NYxCkAW1KSJgFJ3RjukrfqSHWthS1d4o8fhHy\nKJKEnirE5hHqB50dafXrBfgZdaOs3C6ppRIePFe2o4vUEapMTCHFw0woQR8Ah4/R\nn7Z9G9Ln+0Cinmy0nbIDiZJ+pgLAXCOWBfDUzcOjDGKvcpoZharA07c0q1/5ojzO\n4F0Fh4g/BUmtrASwHfcIbjHyCSr1j/3Iz883iy07gJY5Yhiuaqmp0o0f9fgHkG53\n2xCU1owmACqaIBNQMukvXRDtB2GJMuKa/asTZDP6R5re+iXs7+s9ohcRRAKGyAyc\nYKIQKcaA+6M8T7/G+TPHZX6HJWqJJiYB+EC2ERblpvq9TPlLguEWcmvjbVc31nyq\nSDoO3ncFWKFmVsbQPTbP+pKUmlLfJwtb5XqxNR5GEXSwVv4I7IqBmJz1MmRafnBZ\ng0FJUtH668GnldO20XbnSVBr820F5SISMXVwCXDXEvGwwiB8Lt8PvqzXnGIFDAu3\nDlQI5sxSqpPVWSyw08ppKT2Tpmy8adiBotLfaCFl2VTHwOae48X2dMPBvQARAQAB\ntDFGZWRvcmEgKDMwKSA8ZmVkb3JhLTMwLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJbbqxnAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRDvPBEfz8ZZudTnD/9170LL3nyTVUCFmBjT9wZ4gYnpwtKVPa/pKnxbbS+Bmmac\ng9TrT9pZbqOHrNJLiZ3Zx1Hp+8uxr3Lo6kbYwImLhkOEDrf4aP17HfQ6VYFbQZI8\nf79OFxWJ7si9+3gfzeh9UYFEqOQfzIjLWFyfnas0OnV/P+RMQ1Zr+vPRqO7AR2va\nN9wg+Xl7157dhXPCGYnGMNSoxCbpRs0JNlzvJMuAea5nTTznRaJZtK/xKsqLn51D\nK07k9MHVFXakOH8QtMCUglbwfTfIpO5YRq5imxlWbqsYWVQy1WGJFyW6hWC0+RcJ\nOx5zGtOfi4/dN+xJ+ibnbyvy/il7Qm+vyFhCYqIPyS5m2UVJUuao3eApE38k78/o\n8aQOTnFQZ+U1Sw+6woFTxjqRQBXlQm2+7Bt3bqGATg4sXXWPbmwdL87Ic+mxn/ml\nSMfQux/5k6iAu1kQhwkO2YJn9eII6HIPkW+2m5N1JsUyJQe4cbtZE5Yh3TRA0dm7\n+zoBRfCXkOW4krchbgww/ptVmzMMP7GINJdROrJnsGl5FVeid9qHzV7aZycWSma7\nCxBYB1J8HCbty5NjtD6XMYRrMLxXugvX6Q4NPPH+2NKjzX4SIDejS6JjgrP3KA3O\npMuo7ZHMfveBngv8yP+ZD/1sS6l+dfExvdaJdOdgFCnp4p3gPbw5+Lv70HrMjA==\n=BfZ/\n-----END PGP PUBLIC KEY BLOCK-----\n","checksum":"sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"}],"packages":["dnf","e2fsprogs","policycoreutils","qemu-img","systemd","grub2-pc","tar"],"releasever":"30","basearch":"x86_64"}}]},"runner":"org.osbuild.fedora30"},"stages":[{"name":"org.osbuild.dnf","options":{"repos":[{"metalink":"https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever\u0026arch=$basearch","gpgkey":"-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFturGcBEACv0xBo91V2n0uEC2vh69ywCiSyvUgN/AQH8EZpCVtM7NyjKgKm\nbbY4G3R0M3ir1xXmvUDvK0493/qOiFrjkplvzXFTGpPTi0ypqGgxc5d0ohRA1M75\nL+0AIlXoOgHQ358/c4uO8X0JAA1NYxCkAW1KSJgFJ3RjukrfqSHWthS1d4o8fhHy\nKJKEnirE5hHqB50dafXrBfgZdaOs3C6ppRIePFe2o4vUEapMTCHFw0woQR8Ah4/R\nn7Z9G9Ln+0Cinmy0nbIDiZJ+pgLAXCOWBfDUzcOjDGKvcpoZharA07c0q1/5ojzO\n4F0Fh4g/BUmtrASwHfcIbjHyCSr1j/3Iz883iy07gJY5Yhiuaqmp0o0f9fgHkG53\n2xCU1owmACqaIBNQMukvXRDtB2GJMuKa/asTZDP6R5re+iXs7+s9ohcRRAKGyAyc\nYKIQKcaA+6M8T7/G+TPHZX6HJWqJJiYB+EC2ERblpvq9TPlLguEWcmvjbVc31nyq\nSDoO3ncFWKFmVsbQPTbP+pKUmlLfJwtb5XqxNR5GEXSwVv4I7IqBmJz1MmRafnBZ\ng0FJUtH668GnldO20XbnSVBr820F5SISMXVwCXDXEvGwwiB8Lt8PvqzXnGIFDAu3\nDlQI5sxSqpPVWSyw08ppKT2Tpmy8adiBotLfaCFl2VTHwOae48X2dMPBvQARAQAB\ntDFGZWRvcmEgKDMwKSA8ZmVkb3JhLTMwLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJbbqxnAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRDvPBEfz8ZZudTnD/9170LL3nyTVUCFmBjT9wZ4gYnpwtKVPa/pKnxbbS+Bmmac\ng9TrT9pZbqOHrNJLiZ3Zx1Hp+8uxr3Lo6kbYwImLhkOEDrf4aP17HfQ6VYFbQZI8\nf79OFxWJ7si9+3gfzeh9UYFEqOQfzIjLWFyfnas0OnV/P+RMQ1Zr+vPRqO7AR2va\nN9wg+Xl7157dhXPCGYnGMNSoxCbpRs0JNlzvJMuAea5nTTznRaJZtK/xKsqLn51D\nK07k9MHVFXakOH8QtMCUglbwfTfIpO5YRq5imxlWbqsYWVQy1WGJFyW6hWC0+RcJ\nOx5zGtOfi4/dN+xJ+ibnbyvy/il7Qm+vyFhCYqIPyS5m2UVJUuao3eApE38k78/o\n8aQOTnFQZ+U1Sw+6woFTxjqRQBXlQm2+7Bt3bqGATg4sXXWPbmwdL87Ic+mxn/ml\nSMfQux/5k6iAu1kQhwkO2YJn9eII6HIPkW+2m5N1JsUyJQe4cbtZE5Yh3TRA0dm7\n+zoBRfCXkOW4krchbgww/ptVmzMMP7GINJdROrJnsGl5FVeid9qHzV7aZycWSma7\nCxBYB1J8HCbty5NjtD6XMYRrMLxXugvX6Q4NPPH+2NKjzX4SIDejS6JjgrP3KA3O\npMuo7ZHMfveBngv8yP+ZD/1sS6l+dfExvdaJdOdgFCnp4p3gPbw5+Lv70HrMjA==\n=BfZ/\n-----END PGP PUBLIC KEY BLOCK-----\n","checksum":"sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"}],"packages":["policycoreutils","selinux-policy-targeted","kernel","firewalld","chrony","langpacks-en"],"exclude_packages":["dracut-config-rescue"],"releasever":"30","basearch":"x86_64"}},{"name":"org.osbuild.fix-bls","options":{}},{"name":"org.osbuild.locale","options":{"language":"en_US"}},{"name":"org.osbuild.grub2","options":{"root_fs_uuid":"76a22bf4-f153-4541-b6c7-0332c0dfaeac","boot_fs_uuid":"00000000-0000-0000-0000-000000000000","kernel_opts":"ro biosdevname=0 net.ifnames=0"}},{"name":"org.osbuild.selinux","options":{"file_contexts":"etc/selinux/targeted/contexts/files/file_contexts"}}],"assembler":{"name":"org.osbuild.tar","options":{"filename":"root.tar.xz"}}},"targets":[{"image_name":"","name":"org.osbuild.local","options":{"location":"/var/lib/osbuild-composer/outputs/ffffffff-ffff-ffff-ffff-ffffffffffff"},"status":"RUNNING"}]}`, "created", "uuid") + `{"id":"ffffffff-ffff-ffff-ffff-ffffffffffff","output_type":"tar","pipeline":{"build":{"pipeline":{"stages":[{"name":"org.osbuild.dnf","options":{"repos":[{"metalink":"https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever\u0026arch=$basearch","gpgkey":"-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFturGcBEACv0xBo91V2n0uEC2vh69ywCiSyvUgN/AQH8EZpCVtM7NyjKgKm\nbbY4G3R0M3ir1xXmvUDvK0493/qOiFrjkplvzXFTGpPTi0ypqGgxc5d0ohRA1M75\nL+0AIlXoOgHQ358/c4uO8X0JAA1NYxCkAW1KSJgFJ3RjukrfqSHWthS1d4o8fhHy\nKJKEnirE5hHqB50dafXrBfgZdaOs3C6ppRIePFe2o4vUEapMTCHFw0woQR8Ah4/R\nn7Z9G9Ln+0Cinmy0nbIDiZJ+pgLAXCOWBfDUzcOjDGKvcpoZharA07c0q1/5ojzO\n4F0Fh4g/BUmtrASwHfcIbjHyCSr1j/3Iz883iy07gJY5Yhiuaqmp0o0f9fgHkG53\n2xCU1owmACqaIBNQMukvXRDtB2GJMuKa/asTZDP6R5re+iXs7+s9ohcRRAKGyAyc\nYKIQKcaA+6M8T7/G+TPHZX6HJWqJJiYB+EC2ERblpvq9TPlLguEWcmvjbVc31nyq\nSDoO3ncFWKFmVsbQPTbP+pKUmlLfJwtb5XqxNR5GEXSwVv4I7IqBmJz1MmRafnBZ\ng0FJUtH668GnldO20XbnSVBr820F5SISMXVwCXDXEvGwwiB8Lt8PvqzXnGIFDAu3\nDlQI5sxSqpPVWSyw08ppKT2Tpmy8adiBotLfaCFl2VTHwOae48X2dMPBvQARAQAB\ntDFGZWRvcmEgKDMwKSA8ZmVkb3JhLTMwLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJbbqxnAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRDvPBEfz8ZZudTnD/9170LL3nyTVUCFmBjT9wZ4gYnpwtKVPa/pKnxbbS+Bmmac\ng9TrT9pZbqOHrNJLiZ3Zx1Hp+8uxr3Lo6kbYwImLhkOEDrf4aP17HfQ6VYFbQZI8\nf79OFxWJ7si9+3gfzeh9UYFEqOQfzIjLWFyfnas0OnV/P+RMQ1Zr+vPRqO7AR2va\nN9wg+Xl7157dhXPCGYnGMNSoxCbpRs0JNlzvJMuAea5nTTznRaJZtK/xKsqLn51D\nK07k9MHVFXakOH8QtMCUglbwfTfIpO5YRq5imxlWbqsYWVQy1WGJFyW6hWC0+RcJ\nOx5zGtOfi4/dN+xJ+ibnbyvy/il7Qm+vyFhCYqIPyS5m2UVJUuao3eApE38k78/o\n8aQOTnFQZ+U1Sw+6woFTxjqRQBXlQm2+7Bt3bqGATg4sXXWPbmwdL87Ic+mxn/ml\nSMfQux/5k6iAu1kQhwkO2YJn9eII6HIPkW+2m5N1JsUyJQe4cbtZE5Yh3TRA0dm7\n+zoBRfCXkOW4krchbgww/ptVmzMMP7GINJdROrJnsGl5FVeid9qHzV7aZycWSma7\nCxBYB1J8HCbty5NjtD6XMYRrMLxXugvX6Q4NPPH+2NKjzX4SIDejS6JjgrP3KA3O\npMuo7ZHMfveBngv8yP+ZD/1sS6l+dfExvdaJdOdgFCnp4p3gPbw5+Lv70HrMjA==\n=BfZ/\n-----END PGP PUBLIC KEY BLOCK-----\n","checksum":"sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"}],"packages":["dnf","e2fsprogs","policycoreutils","qemu-img","systemd","grub2-pc","tar"],"releasever":"30","basearch":"x86_64"}}]},"runner":"org.osbuild.fedora30"},"stages":[{"name":"org.osbuild.dnf","options":{"repos":[{"metalink":"https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever\u0026arch=$basearch","gpgkey":"-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFturGcBEACv0xBo91V2n0uEC2vh69ywCiSyvUgN/AQH8EZpCVtM7NyjKgKm\nbbY4G3R0M3ir1xXmvUDvK0493/qOiFrjkplvzXFTGpPTi0ypqGgxc5d0ohRA1M75\nL+0AIlXoOgHQ358/c4uO8X0JAA1NYxCkAW1KSJgFJ3RjukrfqSHWthS1d4o8fhHy\nKJKEnirE5hHqB50dafXrBfgZdaOs3C6ppRIePFe2o4vUEapMTCHFw0woQR8Ah4/R\nn7Z9G9Ln+0Cinmy0nbIDiZJ+pgLAXCOWBfDUzcOjDGKvcpoZharA07c0q1/5ojzO\n4F0Fh4g/BUmtrASwHfcIbjHyCSr1j/3Iz883iy07gJY5Yhiuaqmp0o0f9fgHkG53\n2xCU1owmACqaIBNQMukvXRDtB2GJMuKa/asTZDP6R5re+iXs7+s9ohcRRAKGyAyc\nYKIQKcaA+6M8T7/G+TPHZX6HJWqJJiYB+EC2ERblpvq9TPlLguEWcmvjbVc31nyq\nSDoO3ncFWKFmVsbQPTbP+pKUmlLfJwtb5XqxNR5GEXSwVv4I7IqBmJz1MmRafnBZ\ng0FJUtH668GnldO20XbnSVBr820F5SISMXVwCXDXEvGwwiB8Lt8PvqzXnGIFDAu3\nDlQI5sxSqpPVWSyw08ppKT2Tpmy8adiBotLfaCFl2VTHwOae48X2dMPBvQARAQAB\ntDFGZWRvcmEgKDMwKSA8ZmVkb3JhLTMwLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJbbqxnAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRDvPBEfz8ZZudTnD/9170LL3nyTVUCFmBjT9wZ4gYnpwtKVPa/pKnxbbS+Bmmac\ng9TrT9pZbqOHrNJLiZ3Zx1Hp+8uxr3Lo6kbYwImLhkOEDrf4aP17HfQ6VYFbQZI8\nf79OFxWJ7si9+3gfzeh9UYFEqOQfzIjLWFyfnas0OnV/P+RMQ1Zr+vPRqO7AR2va\nN9wg+Xl7157dhXPCGYnGMNSoxCbpRs0JNlzvJMuAea5nTTznRaJZtK/xKsqLn51D\nK07k9MHVFXakOH8QtMCUglbwfTfIpO5YRq5imxlWbqsYWVQy1WGJFyW6hWC0+RcJ\nOx5zGtOfi4/dN+xJ+ibnbyvy/il7Qm+vyFhCYqIPyS5m2UVJUuao3eApE38k78/o\n8aQOTnFQZ+U1Sw+6woFTxjqRQBXlQm2+7Bt3bqGATg4sXXWPbmwdL87Ic+mxn/ml\nSMfQux/5k6iAu1kQhwkO2YJn9eII6HIPkW+2m5N1JsUyJQe4cbtZE5Yh3TRA0dm7\n+zoBRfCXkOW4krchbgww/ptVmzMMP7GINJdROrJnsGl5FVeid9qHzV7aZycWSma7\nCxBYB1J8HCbty5NjtD6XMYRrMLxXugvX6Q4NPPH+2NKjzX4SIDejS6JjgrP3KA3O\npMuo7ZHMfveBngv8yP+ZD/1sS6l+dfExvdaJdOdgFCnp4p3gPbw5+Lv70HrMjA==\n=BfZ/\n-----END PGP PUBLIC KEY BLOCK-----\n","checksum":"sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"}],"packages":["policycoreutils","selinux-policy-targeted","kernel","firewalld","chrony","langpacks-en"],"exclude_packages":["dracut-config-rescue"],"releasever":"30","basearch":"x86_64"}},{"name":"org.osbuild.fix-bls","options":{}},{"name":"org.osbuild.locale","options":{"language":"en_US"}},{"name":"org.osbuild.grub2","options":{"root_fs_uuid":"76a22bf4-f153-4541-b6c7-0332c0dfaeac","boot_fs_uuid":"00000000-0000-0000-0000-000000000000","kernel_opts":"ro biosdevname=0 net.ifnames=0"}},{"name":"org.osbuild.selinux","options":{"file_contexts":"etc/selinux/targeted/contexts/files/file_contexts"}}],"assembler":{"name":"org.osbuild.tar","options":{"filename":"root.tar.xz"}}},"targets":[{"image_name":"","name":"org.osbuild.local","options":{"location":"/var/lib/osbuild-composer/outputs/ffffffff-ffff-ffff-ffff-ffffffffffff"},"status":"RUNNING"}]}`, "created", "uuid") } func testUpdateTransition(t *testing.T, from, to string, expectedStatus int) { diff --git a/internal/jobqueue/job.go b/internal/jobqueue/job.go index 7d6c47729..fd948adb4 100644 --- a/internal/jobqueue/job.go +++ b/internal/jobqueue/job.go @@ -8,36 +8,40 @@ import ( "os/exec" "github.com/google/uuid" + "github.com/osbuild/osbuild-composer/internal/distro" "github.com/osbuild/osbuild-composer/internal/pipeline" + "github.com/osbuild/osbuild-composer/internal/store" "github.com/osbuild/osbuild-composer/internal/target" "github.com/osbuild/osbuild-composer/internal/upload/awsupload" ) type Job struct { - ID uuid.UUID `json:"id"` - Pipeline *pipeline.Pipeline `json:"pipeline"` - Targets []*target.Target `json:"targets"` + ID uuid.UUID `json:"id"` + Pipeline *pipeline.Pipeline `json:"pipeline"` + Targets []*target.Target `json:"targets"` + OutputType string `json:"output_type"` } type JobStatus struct { - Status string `json:"status"` + Status string `json:"status"` + Image *store.Image `json:"image"` } -func (job *Job) Run(d distro.Distro) (error, []error) { +func (job *Job) Run(d distro.Distro) (*store.Image, error, []error) { build := pipeline.Build{ Runner: d.Runner(), } buildFile, err := ioutil.TempFile("", "osbuild-worker-build-env-*") if err != nil { - return err, nil + return nil, err, nil } defer os.Remove(buildFile.Name()) err = json.NewEncoder(buildFile).Encode(build) if err != nil { - return err, nil + return nil, err, nil } cmd := exec.Command( @@ -50,22 +54,22 @@ func (job *Job) Run(d distro.Distro) (error, []error) { stdin, err := cmd.StdinPipe() if err != nil { - return err, nil + return nil, err, nil } stdout, err := cmd.StdoutPipe() if err != nil { - return err, nil + return nil, err, nil } err = cmd.Start() if err != nil { - return err, nil + return nil, err, nil } err = json.NewEncoder(stdin).Encode(job.Pipeline) if err != nil { - return err, nil + return nil, err, nil } stdin.Close() @@ -75,14 +79,21 @@ func (job *Job) Run(d distro.Distro) (error, []error) { } err = json.NewDecoder(stdout).Decode(&result) if err != nil { - return err, nil + return nil, err, nil } err = cmd.Wait() if err != nil { - return err, nil + return nil, err, nil } + filename, mimeType, err := d.FilenameFromType(job.OutputType) + if err != nil { + return nil, err, nil + } + + var image store.Image + var r []error for _, t := range job.Targets { @@ -102,6 +113,25 @@ func (job *Job) Run(d distro.Distro) (error, []error) { r = append(r, err) continue } + + imagePath := options.Location + "/" + filename + file, err := os.Open(imagePath) + if err != nil { + r = append(r, err) + continue + } + + fileStat, err := file.Stat() + if err != nil { + return nil, err, nil + } + + image = store.Image{ + Path: imagePath, + Mime: mimeType, + Size: fileStat.Size(), + } + case *target.AWSTargetOptions: a, err := awsupload.New(options.Region, options.AccessKeyID, options.SecretAccessKey) if err != nil { @@ -132,5 +162,5 @@ func (job *Job) Run(d distro.Distro) (error, []error) { r = append(r, nil) } - return nil, r + return &image, nil, r } diff --git a/internal/store/store.go b/internal/store/store.go index 285638699..459e1d24e 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -51,19 +51,20 @@ type Compose struct { JobCreated time.Time `json:"job_created"` JobStarted time.Time `json:"job_started"` JobFinished time.Time `json:"job_finished"` + Image *Image `json:"image"` } // A Job contains the information about a compose a worker needs to process it. type Job struct { - ComposeID uuid.UUID - Pipeline *pipeline.Pipeline - Targets []*target.Target + ComposeID uuid.UUID + Pipeline *pipeline.Pipeline + Targets []*target.Target + OutputType string } // An Image represents the image resulting from a compose. type Image struct { - File *os.File - Name string + Path string Mime string Size int64 } @@ -223,6 +224,14 @@ func (s *Store) ListBlueprints() []string { return names } +func (s *Store) GetCompose(id uuid.UUID) (Compose, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + compose, exists := s.Composes[id] + return compose, exists +} + func (s *Store) GetAllComposes() map[uuid.UUID]Compose { s.mu.RLock() defer s.mu.RUnlock() @@ -431,9 +440,10 @@ func (s *Store) PushCompose(composeID uuid.UUID, bp *blueprint.Blueprint, compos return nil }) s.pendingJobs <- Job{ - ComposeID: composeID, - Pipeline: pipeline, - Targets: targets, + ComposeID: composeID, + Pipeline: pipeline, + Targets: targets, + OutputType: composeType, } return nil @@ -457,7 +467,7 @@ func (s *Store) PopCompose() Job { return job } -func (s *Store) UpdateCompose(composeID uuid.UUID, status string) error { +func (s *Store) UpdateCompose(composeID uuid.UUID, status string, image *Image) error { return s.change(func() error { compose, exists := s.Composes[composeID] if !exists { @@ -484,6 +494,11 @@ func (s *Store) UpdateCompose(composeID uuid.UUID, status string) error { for _, t := range compose.Targets { t.Status = status } + + if status == "FINISHED" { + compose.Image = image + } + s.Composes[composeID] = compose default: return &InvalidRequestError{"invalid state transition"} @@ -492,44 +507,6 @@ func (s *Store) UpdateCompose(composeID uuid.UUID, status string) error { }) } -func (s *Store) GetImage(composeID uuid.UUID) (*Image, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - if compose, exists := s.Composes[composeID]; exists { - if compose.QueueStatus != "FINISHED" { - return nil, &InvalidRequestError{"compose was not finished"} - } - name, mime, err := s.distro.FilenameFromType(compose.OutputType) - if err != nil { - panic("invalid output type") - } - for _, t := range compose.Targets { - switch options := t.Options.(type) { - case *target.LocalTargetOptions: - file, err := os.Open(options.Location + "/" + name) - if err == nil { - fileStat, err := file.Stat() - if err != nil { - return nil, &NotFoundError{"image info could not be found"} - } - size := fileStat.Size() - - return &Image{ - File: file, - Name: name, - Mime: mime, - Size: size, - }, nil - } - } - } - return nil, &NotFoundError{"image could not be found"} - } - - return nil, &NotFoundError{"compose could not be found"} -} - func (s *Store) PushSource(source SourceConfig) { s.change(func() error { s.Sources[source.Name] = source diff --git a/internal/weldr/api.go b/internal/weldr/api.go index 6d774c33a..6486a1b86 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -9,6 +9,8 @@ import ( "net" "net/http" "net/url" + "os" + "path/filepath" "sort" "strconv" "strings" @@ -1370,21 +1372,51 @@ func (api *API) composeImageHandler(writer http.ResponseWriter, request *http.Re return } - image, err := api.store.GetImage(uuid) - if err != nil { + compose, exists := api.store.GetCompose(uuid) + if !exists { errors := responseError{ ID: "BuildMissingFile", - Msg: fmt.Sprintf("Build %s is missing image file %s", uuidString, image.Name), + Msg: fmt.Sprintf("Compose %s doesn't exist", uuidString), } statusResponseError(writer, http.StatusBadRequest, errors) return } - writer.Header().Set("Content-Disposition", "attachment; filename="+uuid.String()+"-"+image.Name) - writer.Header().Set("Content-Type", image.Mime) - writer.Header().Set("Content-Length", fmt.Sprintf("%d", image.Size)) + if compose.QueueStatus != "FINISHED" { + errors := responseError{ + ID: "BuildInWrongState", + Msg: fmt.Sprintf("Build %s is in wrong state: %s", uuidString, compose.QueueStatus), + } + statusResponseError(writer, http.StatusBadRequest, errors) + return + } - io.Copy(writer, image.File) + if compose.Image == nil { + errors := responseError{ + ID: "BuildMissingFile", + Msg: fmt.Sprintf("Compose %s doesn't have an image assigned", uuidString), + } + statusResponseError(writer, http.StatusBadRequest, errors) + return + } + + imageName := filepath.Base(compose.Image.Path) + + file, err := os.Open(compose.Image.Path) + if err != nil { + errors := responseError{ + ID: "BuildMissingFile", + Msg: fmt.Sprintf("Build %s is missing file %s!", uuidString, imageName), + } + statusResponseError(writer, http.StatusBadRequest, errors) + return + } + + writer.Header().Set("Content-Disposition", "attachment; filename="+uuid.String()+"-"+imageName) + writer.Header().Set("Content-Type", compose.Image.Mime) + writer.Header().Set("Content-Length", fmt.Sprintf("%d", compose.Image.Size)) + + io.Copy(writer, file) } func (api *API) composeFinishedHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { diff --git a/internal/weldr/compose.go b/internal/weldr/compose.go index 80c2ccdd8..705188f34 100644 --- a/internal/weldr/compose.go +++ b/internal/weldr/compose.go @@ -3,6 +3,7 @@ package weldr import ( "github.com/google/uuid" "github.com/osbuild/osbuild-composer/internal/store" + "log" "sort" ) @@ -41,13 +42,13 @@ func composeToComposeEntry(id uuid.UUID, compose store.Compose, includeUploads b composeEntry.JobStarted = float64(compose.JobStarted.UnixNano()) / 1000000000 case "FINISHED": - //image, err := s.GetImage(id) - //imageSize := int64(0) - //if err == nil { - // imageSize = image.Size - //} - // TODO: this is currently broken! - composeEntry.ImageSize = int64(0) + if compose.Image != nil { + composeEntry.ImageSize = compose.Image.Size + } else { + log.Printf("finished compose with id %s has nil image\n", id.String()) + composeEntry.ImageSize = 0 + } + composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000 composeEntry.JobStarted = float64(compose.JobStarted.UnixNano()) / 1000000000 composeEntry.JobFinished = float64(compose.JobFinished.UnixNano()) / 1000000000