internal/remotefile: create a remote file resolver

Create a resolver to fetch the contents of a
remote file which can be used at build time.
The initial usecase for this resolver is to
resolve remote gpgkeys but the resolver has
been made more generic for general files.

internal/remotefile/client.go

@@ -0,0 +1,60 @@
+package remotefile
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+)
+
+type Client struct {
+	client *http.Client
+}
+
+func NewClient() *Client {
+	return &Client{
+		client: &http.Client{},
+	}
+}
+
+func (c *Client) makeRequest(u *url.URL) ([]byte, error) {
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+	output, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return output, nil
+}
+
+func (c *Client) validateURL(u string) (*url.URL, error) {
+	if u == "" {
+		return nil, fmt.Errorf("File resolver: url is required")
+	}
+	parsedURL, err := url.ParseRequestURI(u)
+	if err != nil {
+		return nil, fmt.Errorf("File resolver: invalid url %s", u)
+	}
+	return parsedURL, nil
+}
+
+// resolve and return the contents of a remote file
+// which can be used later, in the pipeline
+func (c *Client) Resolve(u string) ([]byte, error) {
+	parsedURL, err := c.validateURL(u)
+	if err != nil {
+		return nil, err
+	}
+
+	return c.makeRequest(parsedURL)
+}

internal/remotefile/client_test.go

@@ -0,0 +1,73 @@
+package remotefile
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func makeTestServer() *httptest.Server {
+	// use a simple mock server to test the client
+	// and file content resolver
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/key1" {
+			fmt.Fprintln(w, "key1")
+		}
+		if r.URL.Path == "/key2" {
+			fmt.Fprintln(w, "key2")
+		}
+	}))
+}
+
+func TestClientResolve(t *testing.T) {
+	server := makeTestServer()
+
+	url := server.URL + "/key1"
+
+	client := NewClient()
+
+	output, err := client.Resolve(url)
+	assert.NoError(t, err)
+
+	expectedOutput := "key1\n"
+
+	assert.Equal(t, expectedOutput, string(output))
+}
+
+func TestInputSpecValidation(t *testing.T) {
+	server := makeTestServer()
+
+	test := []struct {
+		name string
+		url  string
+		want error
+	}{
+		{
+			name: "valid input spec",
+			url:  server.URL + "/key1",
+			want: nil,
+		},
+		{
+			name: "missing url spec",
+			url:  "",
+			want: fmt.Errorf("File resolver: url is required"),
+		},
+	}
+
+	client := NewClient()
+
+	for _, tt := range test {
+		url, err := client.validateURL(tt.url)
+		if tt.want == nil {
+			assert.NoError(t, err)
+			assert.Equal(t, tt.url, url.String())
+		} else {
+			assert.EqualError(t, err, tt.want.Error())
+			assert.Nil(t, url)
+		}
+	}
+
+}

internal/remotefile/resolver.go

@@ -0,0 +1,66 @@
+package remotefile
+
+import (
+	"context"
+
+	"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
+)
+
+type resolveResult struct {
+	url     string
+	content []byte
+	err     error
+}
+
+// TODO: could make this more generic
+// since this is shared with the container
+// resolver
+type Resolver struct {
+	jobs  int
+	queue chan resolveResult
+
+	ctx context.Context
+}
+
+func NewResolver() *Resolver {
+	return &Resolver{
+		ctx:   context.Background(),
+		queue: make(chan resolveResult, 2),
+	}
+}
+
+func (r *Resolver) Add(url string) {
+	client := NewClient()
+	r.jobs += 1
+
+	go func() {
+		content, err := client.Resolve(url)
+		r.queue <- resolveResult{url: url, content: content, err: err}
+	}()
+}
+
+func (r *Resolver) Finish() []Spec {
+
+	resultItems := make([]Spec, 0, r.jobs)
+	for r.jobs > 0 {
+		result := <-r.queue
+		r.jobs -= 1
+
+		var resultError *clienterrors.Error
+		if result.err != nil {
+			resultError = clienterrors.WorkerClientError(
+				clienterrors.ErrorRemoteFileResolution,
+				result.err.Error(),
+				result.url,
+			)
+		}
+
+		resultItems = append(resultItems, Spec{
+			URL:             result.url,
+			Content:         result.content,
+			ResolutionError: resultError,
+		})
+	}
+
+	return resultItems
+}

internal/remotefile/resolver_test.go

@@ -0,0 +1,102 @@
+package remotefile
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSingleInputResolver(t *testing.T) {
+	server := makeTestServer()
+	url := server.URL + "/key1"
+
+	resolver := NewResolver()
+
+	expectedOutput := Spec{
+		URL:             url,
+		Content:         []byte("key1\n"),
+		ResolutionError: nil,
+	}
+
+	resolver.Add(url)
+
+	resultItems := resolver.Finish()
+	assert.Contains(t, resultItems, expectedOutput)
+
+	for _, item := range resultItems {
+		assert.Nil(t, item.ResolutionError)
+	}
+}
+
+func TestMultiInputResolver(t *testing.T) {
+	server := makeTestServer()
+
+	urlOne := server.URL + "/key1"
+	urlTwo := server.URL + "/key2"
+
+	expectedOutputOne := Spec{
+		URL:             urlOne,
+		Content:         []byte("key1\n"),
+		ResolutionError: nil,
+	}
+
+	expectedOutputTwo := Spec{
+		URL:             urlTwo,
+		Content:         []byte("key2\n"),
+		ResolutionError: nil,
+	}
+
+	resolver := NewResolver()
+
+	resolver.Add(urlOne)
+	resolver.Add(urlTwo)
+
+	resultItems := resolver.Finish()
+
+	assert.Contains(t, resultItems, expectedOutputOne)
+	assert.Contains(t, resultItems, expectedOutputTwo)
+
+	for _, item := range resultItems {
+		assert.Nil(t, item.ResolutionError)
+	}
+}
+
+func TestInvalidInputResolver(t *testing.T) {
+	url := ""
+
+	resolver := NewResolver()
+
+	resolver.Add(url)
+
+	expectedErr := fmt.Errorf("File resolver: url is required")
+
+	resultItems := resolver.Finish()
+
+	for _, item := range resultItems {
+		assert.Equal(t, item.ResolutionError.Reason, expectedErr.Error())
+	}
+}
+
+func TestMultiInvalidInputResolver(t *testing.T) {
+	urlOne := ""
+	urlTwo := "hello"
+
+	resolver := NewResolver()
+
+	resolver.Add(urlOne)
+	resolver.Add(urlTwo)
+
+	expectedErrMessageOne := "File resolver: url is required"
+	expectedErrMessageTwo := fmt.Sprintf("File resolver: invalid url %s", urlTwo)
+
+	resultItems := resolver.Finish()
+
+	errs := []string{}
+	for _, item := range resultItems {
+		errs = append(errs, item.ResolutionError.Reason)
+	}
+
+	assert.Contains(t, errs, expectedErrMessageOne)
+	assert.Contains(t, errs, expectedErrMessageTwo)
+}

internal/remotefile/spec.go

@@ -0,0 +1,9 @@
+package remotefile
+
+import "github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
+
+type Spec struct {
+	URL             string
+	Content         []byte
+	ResolutionError *clienterrors.Error
+}