diff --git a/internal/remotefile/client.go b/internal/remotefile/client.go new file mode 100644 index 000000000..c51f9c34f --- /dev/null +++ b/internal/remotefile/client.go @@ -0,0 +1,60 @@ +package remotefile + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/url" +) + +type Client struct { + client *http.Client +} + +func NewClient() *Client { + return &Client{ + client: &http.Client{}, + } +} + +func (c *Client) makeRequest(u *url.URL) ([]byte, error) { + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + output, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return output, nil +} + +func (c *Client) validateURL(u string) (*url.URL, error) { + if u == "" { + return nil, fmt.Errorf("File resolver: url is required") + } + parsedURL, err := url.ParseRequestURI(u) + if err != nil { + return nil, fmt.Errorf("File resolver: invalid url %s", u) + } + return parsedURL, nil +} + +// resolve and return the contents of a remote file +// which can be used later, in the pipeline +func (c *Client) Resolve(u string) ([]byte, error) { + parsedURL, err := c.validateURL(u) + if err != nil { + return nil, err + } + + return c.makeRequest(parsedURL) +} diff --git a/internal/remotefile/client_test.go b/internal/remotefile/client_test.go new file mode 100644 index 000000000..6d40e97ae --- /dev/null +++ b/internal/remotefile/client_test.go @@ -0,0 +1,73 @@ +package remotefile + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func makeTestServer() *httptest.Server { + // use a simple mock server to test the client + // and file content resolver + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/key1" { + fmt.Fprintln(w, "key1") + } + if r.URL.Path == "/key2" { + fmt.Fprintln(w, "key2") + } + })) +} + +func TestClientResolve(t *testing.T) { + server := makeTestServer() + + url := server.URL + "/key1" + + client := NewClient() + + output, err := client.Resolve(url) + assert.NoError(t, err) + + expectedOutput := "key1\n" + + assert.Equal(t, expectedOutput, string(output)) +} + +func TestInputSpecValidation(t *testing.T) { + server := makeTestServer() + + test := []struct { + name string + url string + want error + }{ + { + name: "valid input spec", + url: server.URL + "/key1", + want: nil, + }, + { + name: "missing url spec", + url: "", + want: fmt.Errorf("File resolver: url is required"), + }, + } + + client := NewClient() + + for _, tt := range test { + url, err := client.validateURL(tt.url) + if tt.want == nil { + assert.NoError(t, err) + assert.Equal(t, tt.url, url.String()) + } else { + assert.EqualError(t, err, tt.want.Error()) + assert.Nil(t, url) + } + } + +} diff --git a/internal/remotefile/resolver.go b/internal/remotefile/resolver.go new file mode 100644 index 000000000..90745cc1c --- /dev/null +++ b/internal/remotefile/resolver.go @@ -0,0 +1,66 @@ +package remotefile + +import ( + "context" + + "github.com/osbuild/osbuild-composer/internal/worker/clienterrors" +) + +type resolveResult struct { + url string + content []byte + err error +} + +// TODO: could make this more generic +// since this is shared with the container +// resolver +type Resolver struct { + jobs int + queue chan resolveResult + + ctx context.Context +} + +func NewResolver() *Resolver { + return &Resolver{ + ctx: context.Background(), + queue: make(chan resolveResult, 2), + } +} + +func (r *Resolver) Add(url string) { + client := NewClient() + r.jobs += 1 + + go func() { + content, err := client.Resolve(url) + r.queue <- resolveResult{url: url, content: content, err: err} + }() +} + +func (r *Resolver) Finish() []Spec { + + resultItems := make([]Spec, 0, r.jobs) + for r.jobs > 0 { + result := <-r.queue + r.jobs -= 1 + + var resultError *clienterrors.Error + if result.err != nil { + resultError = clienterrors.WorkerClientError( + clienterrors.ErrorRemoteFileResolution, + result.err.Error(), + result.url, + ) + } + + resultItems = append(resultItems, Spec{ + URL: result.url, + Content: result.content, + ResolutionError: resultError, + }) + } + + return resultItems +} diff --git a/internal/remotefile/resolver_test.go b/internal/remotefile/resolver_test.go new file mode 100644 index 000000000..4c8371ef2 --- /dev/null +++ b/internal/remotefile/resolver_test.go @@ -0,0 +1,102 @@ +package remotefile + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSingleInputResolver(t *testing.T) { + server := makeTestServer() + url := server.URL + "/key1" + + resolver := NewResolver() + + expectedOutput := Spec{ + URL: url, + Content: []byte("key1\n"), + ResolutionError: nil, + } + + resolver.Add(url) + + resultItems := resolver.Finish() + assert.Contains(t, resultItems, expectedOutput) + + for _, item := range resultItems { + assert.Nil(t, item.ResolutionError) + } +} + +func TestMultiInputResolver(t *testing.T) { + server := makeTestServer() + + urlOne := server.URL + "/key1" + urlTwo := server.URL + "/key2" + + expectedOutputOne := Spec{ + URL: urlOne, + Content: []byte("key1\n"), + ResolutionError: nil, + } + + expectedOutputTwo := Spec{ + URL: urlTwo, + Content: []byte("key2\n"), + ResolutionError: nil, + } + + resolver := NewResolver() + + resolver.Add(urlOne) + resolver.Add(urlTwo) + + resultItems := resolver.Finish() + + assert.Contains(t, resultItems, expectedOutputOne) + assert.Contains(t, resultItems, expectedOutputTwo) + + for _, item := range resultItems { + assert.Nil(t, item.ResolutionError) + } +} + +func TestInvalidInputResolver(t *testing.T) { + url := "" + + resolver := NewResolver() + + resolver.Add(url) + + expectedErr := fmt.Errorf("File resolver: url is required") + + resultItems := resolver.Finish() + + for _, item := range resultItems { + assert.Equal(t, item.ResolutionError.Reason, expectedErr.Error()) + } +} + +func TestMultiInvalidInputResolver(t *testing.T) { + urlOne := "" + urlTwo := "hello" + + resolver := NewResolver() + + resolver.Add(urlOne) + resolver.Add(urlTwo) + + expectedErrMessageOne := "File resolver: url is required" + expectedErrMessageTwo := fmt.Sprintf("File resolver: invalid url %s", urlTwo) + + resultItems := resolver.Finish() + + errs := []string{} + for _, item := range resultItems { + errs = append(errs, item.ResolutionError.Reason) + } + + assert.Contains(t, errs, expectedErrMessageOne) + assert.Contains(t, errs, expectedErrMessageTwo) +} diff --git a/internal/remotefile/spec.go b/internal/remotefile/spec.go new file mode 100644 index 000000000..9f2875c70 --- /dev/null +++ b/internal/remotefile/spec.go @@ -0,0 +1,9 @@ +package remotefile + +import "github.com/osbuild/osbuild-composer/internal/worker/clienterrors" + +type Spec struct { + URL string + Content []byte + ResolutionError *clienterrors.Error +}