From b7e7bafb2ef6029701a8c7f6da980dac781f93d7 Mon Sep 17 00:00:00 2001 From: Gianluca Zuccarelli Date: Tue, 31 Jan 2023 12:04:56 +0000 Subject: [PATCH] internal/remotefile: create a remote file resolver Create a resolver to fetch the contents of a remote file which can be used at build time. The initial usecase for this resolver is to resolve remote gpgkeys but the resolver has been made more generic for general files. --- internal/remotefile/client.go | 60 ++++++++++++++++ internal/remotefile/client_test.go | 73 +++++++++++++++++++ internal/remotefile/resolver.go | 66 +++++++++++++++++ internal/remotefile/resolver_test.go | 102 +++++++++++++++++++++++++++ internal/remotefile/spec.go | 9 +++ 5 files changed, 310 insertions(+) create mode 100644 internal/remotefile/client.go create mode 100644 internal/remotefile/client_test.go create mode 100644 internal/remotefile/resolver.go create mode 100644 internal/remotefile/resolver_test.go create mode 100644 internal/remotefile/spec.go diff --git a/internal/remotefile/client.go b/internal/remotefile/client.go new file mode 100644 index 000000000..c51f9c34f --- /dev/null +++ b/internal/remotefile/client.go @@ -0,0 +1,60 @@ +package remotefile + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/url" +) + +type Client struct { + client *http.Client +} + +func NewClient() *Client { + return &Client{ + client: &http.Client{}, + } +} + +func (c *Client) makeRequest(u *url.URL) ([]byte, error) { + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + output, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return output, nil +} + +func (c *Client) validateURL(u string) (*url.URL, error) { + if u == "" { + return nil, fmt.Errorf("File resolver: url is required") + } + parsedURL, err := url.ParseRequestURI(u) + if err != nil { + return nil, fmt.Errorf("File resolver: invalid url %s", u) + } + return parsedURL, nil +} + +// resolve and return the contents of a remote file +// which can be used later, in the pipeline +func (c *Client) Resolve(u string) ([]byte, error) { + parsedURL, err := c.validateURL(u) + if err != nil { + return nil, err + } + + return c.makeRequest(parsedURL) +} diff --git a/internal/remotefile/client_test.go b/internal/remotefile/client_test.go new file mode 100644 index 000000000..6d40e97ae --- /dev/null +++ b/internal/remotefile/client_test.go @@ -0,0 +1,73 @@ +package remotefile + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func makeTestServer() *httptest.Server { + // use a simple mock server to test the client + // and file content resolver + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/key1" { + fmt.Fprintln(w, "key1") + } + if r.URL.Path == "/key2" { + fmt.Fprintln(w, "key2") + } + })) +} + +func TestClientResolve(t *testing.T) { + server := makeTestServer() + + url := server.URL + "/key1" + + client := NewClient() + + output, err := client.Resolve(url) + assert.NoError(t, err) + + expectedOutput := "key1\n" + + assert.Equal(t, expectedOutput, string(output)) +} + +func TestInputSpecValidation(t *testing.T) { + server := makeTestServer() + + test := []struct { + name string + url string + want error + }{ + { + name: "valid input spec", + url: server.URL + "/key1", + want: nil, + }, + { + name: "missing url spec", + url: "", + want: fmt.Errorf("File resolver: url is required"), + }, + } + + client := NewClient() + + for _, tt := range test { + url, err := client.validateURL(tt.url) + if tt.want == nil { + assert.NoError(t, err) + assert.Equal(t, tt.url, url.String()) + } else { + assert.EqualError(t, err, tt.want.Error()) + assert.Nil(t, url) + } + } + +} diff --git a/internal/remotefile/resolver.go b/internal/remotefile/resolver.go new file mode 100644 index 000000000..90745cc1c --- /dev/null +++ b/internal/remotefile/resolver.go @@ -0,0 +1,66 @@ +package remotefile + +import ( + "context" + + "github.com/osbuild/osbuild-composer/internal/worker/clienterrors" +) + +type resolveResult struct { + url string + content []byte + err error +} + +// TODO: could make this more generic +// since this is shared with the container +// resolver +type Resolver struct { + jobs int + queue chan resolveResult + + ctx context.Context +} + +func NewResolver() *Resolver { + return &Resolver{ + ctx: context.Background(), + queue: make(chan resolveResult, 2), + } +} + +func (r *Resolver) Add(url string) { + client := NewClient() + r.jobs += 1 + + go func() { + content, err := client.Resolve(url) + r.queue <- resolveResult{url: url, content: content, err: err} + }() +} + +func (r *Resolver) Finish() []Spec { + + resultItems := make([]Spec, 0, r.jobs) + for r.jobs > 0 { + result := <-r.queue + r.jobs -= 1 + + var resultError *clienterrors.Error + if result.err != nil { + resultError = clienterrors.WorkerClientError( + clienterrors.ErrorRemoteFileResolution, + result.err.Error(), + result.url, + ) + } + + resultItems = append(resultItems, Spec{ + URL: result.url, + Content: result.content, + ResolutionError: resultError, + }) + } + + return resultItems +} diff --git a/internal/remotefile/resolver_test.go b/internal/remotefile/resolver_test.go new file mode 100644 index 000000000..4c8371ef2 --- /dev/null +++ b/internal/remotefile/resolver_test.go @@ -0,0 +1,102 @@ +package remotefile + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSingleInputResolver(t *testing.T) { + server := makeTestServer() + url := server.URL + "/key1" + + resolver := NewResolver() + + expectedOutput := Spec{ + URL: url, + Content: []byte("key1\n"), + ResolutionError: nil, + } + + resolver.Add(url) + + resultItems := resolver.Finish() + assert.Contains(t, resultItems, expectedOutput) + + for _, item := range resultItems { + assert.Nil(t, item.ResolutionError) + } +} + +func TestMultiInputResolver(t *testing.T) { + server := makeTestServer() + + urlOne := server.URL + "/key1" + urlTwo := server.URL + "/key2" + + expectedOutputOne := Spec{ + URL: urlOne, + Content: []byte("key1\n"), + ResolutionError: nil, + } + + expectedOutputTwo := Spec{ + URL: urlTwo, + Content: []byte("key2\n"), + ResolutionError: nil, + } + + resolver := NewResolver() + + resolver.Add(urlOne) + resolver.Add(urlTwo) + + resultItems := resolver.Finish() + + assert.Contains(t, resultItems, expectedOutputOne) + assert.Contains(t, resultItems, expectedOutputTwo) + + for _, item := range resultItems { + assert.Nil(t, item.ResolutionError) + } +} + +func TestInvalidInputResolver(t *testing.T) { + url := "" + + resolver := NewResolver() + + resolver.Add(url) + + expectedErr := fmt.Errorf("File resolver: url is required") + + resultItems := resolver.Finish() + + for _, item := range resultItems { + assert.Equal(t, item.ResolutionError.Reason, expectedErr.Error()) + } +} + +func TestMultiInvalidInputResolver(t *testing.T) { + urlOne := "" + urlTwo := "hello" + + resolver := NewResolver() + + resolver.Add(urlOne) + resolver.Add(urlTwo) + + expectedErrMessageOne := "File resolver: url is required" + expectedErrMessageTwo := fmt.Sprintf("File resolver: invalid url %s", urlTwo) + + resultItems := resolver.Finish() + + errs := []string{} + for _, item := range resultItems { + errs = append(errs, item.ResolutionError.Reason) + } + + assert.Contains(t, errs, expectedErrMessageOne) + assert.Contains(t, errs, expectedErrMessageTwo) +} diff --git a/internal/remotefile/spec.go b/internal/remotefile/spec.go new file mode 100644 index 000000000..9f2875c70 --- /dev/null +++ b/internal/remotefile/spec.go @@ -0,0 +1,9 @@ +package remotefile + +import "github.com/osbuild/osbuild-composer/internal/worker/clienterrors" + +type Spec struct { + URL string + Content []byte + ResolutionError *clienterrors.Error +}