internal/upload: Add support for upload to GCP and CLI tool using it

Add new internal upload target for Google Cloud Platform and
osbuild-upload-gcp CLI tool which uses the API.

Supported features are:
- Authenticate with GCP using explicitly provided JSON credentials
  file or let the authentication be handled automatically by the
  Google cloud client library. The later is useful e.g. when the worker
  is running in GCP VM instance, which has associated permissions with
  it.
- Upload an existing image file into existing Storage bucket.
- Verify MD5 checksum of the uploaded image file against the local
  file's checksum.
- Import the uploaded image file into Compute Node as an Image.
- Delete the uploaded image file after a successful image import.
- Delete all cache files from storage created as part of the image
  import build job.
- Share the imported image with a list of specified accounts.

GCP-specific image type is not yet added, since GCP supports importing
VMDK and VHD images, which the osbuild-composer already supports.

Update go.mod, vendor/ content and SPEC file with new dependencies.

Signed-off-by: Tomas Hozza <thozza@redhat.com>

vendor/github.com/jstemmer/go-junit-report/.gitignore

@@ -0,0 +1 @@
+go-junit-report

vendor/github.com/jstemmer/go-junit-report/.travis.yml

@@ -0,0 +1,16 @@
+language: go
+
+go:
+  - tip
+  - "1.13.x"
+  - "1.12.x"
+  - "1.11.x"
+  - "1.10.x"
+  - "1.9.x"
+  - "1.8.x"
+  - "1.7.x"
+  - "1.6.x"
+  - "1.5.x"
+  - "1.4.x"
+  - "1.3.x"
+  - "1.2.x"

vendor/github.com/jstemmer/go-junit-report/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Joel Stemmer
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/jstemmer/go-junit-report/README.md

@@ -0,0 +1,49 @@
+# go-junit-report
+
+Converts `go test` output to an xml report, suitable for applications that
+expect junit xml reports (e.g. [Jenkins](http://jenkins-ci.org)).
+
+[![Build Status][travis-badge]][travis-link]
+[![Report Card][report-badge]][report-link]
+
+## Installation
+
+Go version 1.2 or higher is required. Install or update using the `go get`
+command:
+
+```bash
+go get -u github.com/jstemmer/go-junit-report
+```
+
+## Usage
+
+go-junit-report reads the `go test` verbose output from standard in and writes
+junit compatible XML to standard out.
+
+```bash
+go test -v 2>&1 | go-junit-report > report.xml
+```
+
+Note that it also can parse benchmark output with `-bench` flag:
+```bash
+go test -v -bench . -count 5 2>&1 | go-junit-report > report.xml
+```
+
+## Contribution
+
+Create an Issue and discuss the fix or feature, then fork the package.
+Clone to github.com/jstemmer/go-junit-report.  This is necessary because go import uses this path.
+Fix or implement feature. Test and then commit change.
+Specify #Issue and describe change in the commit message.
+Create Pull Request. It can be merged by owner or administrator then.
+
+### Run Tests
+
+```bash
+go test
+```
+
+[travis-badge]: https://travis-ci.org/jstemmer/go-junit-report.svg
+[travis-link]: https://travis-ci.org/jstemmer/go-junit-report
+[report-badge]: https://goreportcard.com/badge/github.com/jstemmer/go-junit-report
+[report-link]: https://goreportcard.com/report/github.com/jstemmer/go-junit-report

vendor/github.com/jstemmer/go-junit-report/formatter/formatter.go

@@ -0,0 +1,182 @@
+package formatter
+
+import (
+	"bufio"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/jstemmer/go-junit-report/parser"
+)
+
+// JUnitTestSuites is a collection of JUnit test suites.
+type JUnitTestSuites struct {
+	XMLName xml.Name         `xml:"testsuites"`
+	Suites  []JUnitTestSuite `xml:"testsuite"`
+}
+
+// JUnitTestSuite is a single JUnit test suite which may contain many
+// testcases.
+type JUnitTestSuite struct {
+	XMLName    xml.Name        `xml:"testsuite"`
+	Tests      int             `xml:"tests,attr"`
+	Failures   int             `xml:"failures,attr"`
+	Time       string          `xml:"time,attr"`
+	Name       string          `xml:"name,attr"`
+	Properties []JUnitProperty `xml:"properties>property,omitempty"`
+	TestCases  []JUnitTestCase `xml:"testcase"`
+}
+
+// JUnitTestCase is a single test case with its result.
+type JUnitTestCase struct {
+	XMLName     xml.Name          `xml:"testcase"`
+	Classname   string            `xml:"classname,attr"`
+	Name        string            `xml:"name,attr"`
+	Time        string            `xml:"time,attr"`
+	SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"`
+	Failure     *JUnitFailure     `xml:"failure,omitempty"`
+}
+
+// JUnitSkipMessage contains the reason why a testcase was skipped.
+type JUnitSkipMessage struct {
+	Message string `xml:"message,attr"`
+}
+
+// JUnitProperty represents a key/value pair used to define properties.
+type JUnitProperty struct {
+	Name  string `xml:"name,attr"`
+	Value string `xml:"value,attr"`
+}
+
+// JUnitFailure contains data related to a failed test.
+type JUnitFailure struct {
+	Message  string `xml:"message,attr"`
+	Type     string `xml:"type,attr"`
+	Contents string `xml:",chardata"`
+}
+
+// JUnitReportXML writes a JUnit xml representation of the given report to w
+// in the format described at http://windyroad.org/dl/Open%20Source/JUnit.xsd
+func JUnitReportXML(report *parser.Report, noXMLHeader bool, goVersion string, w io.Writer) error {
+	suites := JUnitTestSuites{}
+
+	// convert Report to JUnit test suites
+	for _, pkg := range report.Packages {
+		pkg.Benchmarks = mergeBenchmarks(pkg.Benchmarks)
+		ts := JUnitTestSuite{
+			Tests:      len(pkg.Tests) + len(pkg.Benchmarks),
+			Failures:   0,
+			Time:       formatTime(pkg.Duration),
+			Name:       pkg.Name,
+			Properties: []JUnitProperty{},
+			TestCases:  []JUnitTestCase{},
+		}
+
+		classname := pkg.Name
+		if idx := strings.LastIndex(classname, "/"); idx > -1 && idx < len(pkg.Name) {
+			classname = pkg.Name[idx+1:]
+		}
+
+		// properties
+		if goVersion == "" {
+			// if goVersion was not specified as a flag, fall back to version reported by runtime
+			goVersion = runtime.Version()
+		}
+		ts.Properties = append(ts.Properties, JUnitProperty{"go.version", goVersion})
+		if pkg.CoveragePct != "" {
+			ts.Properties = append(ts.Properties, JUnitProperty{"coverage.statements.pct", pkg.CoveragePct})
+		}
+
+		// individual test cases
+		for _, test := range pkg.Tests {
+			testCase := JUnitTestCase{
+				Classname: classname,
+				Name:      test.Name,
+				Time:      formatTime(test.Duration),
+				Failure:   nil,
+			}
+
+			if test.Result == parser.FAIL {
+				ts.Failures++
+				testCase.Failure = &JUnitFailure{
+					Message:  "Failed",
+					Type:     "",
+					Contents: strings.Join(test.Output, "\n"),
+				}
+			}
+
+			if test.Result == parser.SKIP {
+				testCase.SkipMessage = &JUnitSkipMessage{strings.Join(test.Output, "\n")}
+			}
+
+			ts.TestCases = append(ts.TestCases, testCase)
+		}
+
+		// individual benchmarks
+		for _, benchmark := range pkg.Benchmarks {
+			benchmarkCase := JUnitTestCase{
+				Classname: classname,
+				Name:      benchmark.Name,
+				Time:      formatBenchmarkTime(benchmark.Duration),
+			}
+
+			ts.TestCases = append(ts.TestCases, benchmarkCase)
+		}
+
+		suites.Suites = append(suites.Suites, ts)
+	}
+
+	// to xml
+	bytes, err := xml.MarshalIndent(suites, "", "\t")
+	if err != nil {
+		return err
+	}
+
+	writer := bufio.NewWriter(w)
+
+	if !noXMLHeader {
+		writer.WriteString(xml.Header)
+	}
+
+	writer.Write(bytes)
+	writer.WriteByte('\n')
+	writer.Flush()
+
+	return nil
+}
+
+func mergeBenchmarks(benchmarks []*parser.Benchmark) []*parser.Benchmark {
+	var merged []*parser.Benchmark
+	benchmap := make(map[string][]*parser.Benchmark)
+	for _, bm := range benchmarks {
+		if _, ok := benchmap[bm.Name]; !ok {
+			merged = append(merged, &parser.Benchmark{Name: bm.Name})
+		}
+		benchmap[bm.Name] = append(benchmap[bm.Name], bm)
+	}
+
+	for _, bm := range merged {
+		for _, b := range benchmap[bm.Name] {
+			bm.Allocs += b.Allocs
+			bm.Bytes += b.Bytes
+			bm.Duration += b.Duration
+		}
+		n := len(benchmap[bm.Name])
+		bm.Allocs /= n
+		bm.Bytes /= n
+		bm.Duration /= time.Duration(n)
+	}
+
+	return merged
+}
+
+func formatTime(d time.Duration) string {
+	return fmt.Sprintf("%.3f", d.Seconds())
+}
+
+func formatBenchmarkTime(d time.Duration) string {
+	return fmt.Sprintf("%.9f", d.Seconds())
+}

vendor/github.com/jstemmer/go-junit-report/go-junit-report.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/jstemmer/go-junit-report/formatter"
+	"github.com/jstemmer/go-junit-report/parser"
+)
+
+var (
+	noXMLHeader   = flag.Bool("no-xml-header", false, "do not print xml header")
+	packageName   = flag.String("package-name", "", "specify a package name (compiled test have no package name in output)")
+	goVersionFlag = flag.String("go-version", "", "specify the value to use for the go.version property in the generated XML")
+	setExitCode   = flag.Bool("set-exit-code", false, "set exit code to 1 if tests failed")
+)
+
+func main() {
+	flag.Parse()
+
+	if flag.NArg() != 0 {
+		fmt.Fprintf(os.Stderr, "%s does not accept positional arguments\n", os.Args[0])
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	// Read input
+	report, err := parser.Parse(os.Stdin, *packageName)
+	if err != nil {
+		fmt.Printf("Error reading input: %s\n", err)
+		os.Exit(1)
+	}
+
+	// Write xml
+	err = formatter.JUnitReportXML(report, *noXMLHeader, *goVersionFlag, os.Stdout)
+	if err != nil {
+		fmt.Printf("Error writing XML: %s\n", err)
+		os.Exit(1)
+	}
+
+	if *setExitCode && report.Failures() > 0 {
+		os.Exit(1)
+	}
+}

vendor/github.com/jstemmer/go-junit-report/go.mod

@@ -0,0 +1,3 @@
+module github.com/jstemmer/go-junit-report
+
+go 1.2

vendor/github.com/jstemmer/go-junit-report/parser/parser.go

@@ -0,0 +1,319 @@
+package parser
+
+import (
+	"bufio"
+	"io"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Result represents a test result.
+type Result int
+
+// Test result constants
+const (
+	PASS Result = iota
+	FAIL
+	SKIP
+)
+
+// Report is a collection of package tests.
+type Report struct {
+	Packages []Package
+}
+
+// Package contains the test results of a single package.
+type Package struct {
+	Name        string
+	Duration    time.Duration
+	Tests       []*Test
+	Benchmarks  []*Benchmark
+	CoveragePct string
+
+	// Time is deprecated, use Duration instead.
+	Time int // in milliseconds
+}
+
+// Test contains the results of a single test.
+type Test struct {
+	Name     string
+	Duration time.Duration
+	Result   Result
+	Output   []string
+
+	SubtestIndent string
+
+	// Time is deprecated, use Duration instead.
+	Time int // in milliseconds
+}
+
+// Benchmark contains the results of a single benchmark.
+type Benchmark struct {
+	Name     string
+	Duration time.Duration
+	// number of B/op
+	Bytes int
+	// number of allocs/op
+	Allocs int
+}
+
+var (
+	regexStatus   = regexp.MustCompile(`--- (PASS|FAIL|SKIP): (.+) \((\d+\.\d+)(?: seconds|s)\)`)
+	regexIndent   = regexp.MustCompile(`^([ \t]+)---`)
+	regexCoverage = regexp.MustCompile(`^coverage:\s+(\d+\.\d+)%\s+of\s+statements(?:\sin\s.+)?$`)
+	regexResult   = regexp.MustCompile(`^(ok|FAIL)\s+([^ ]+)\s+(?:(\d+\.\d+)s|\(cached\)|(\[\w+ failed]))(?:\s+coverage:\s+(\d+\.\d+)%\sof\sstatements(?:\sin\s.+)?)?$`)
+	// regexBenchmark captures 3-5 groups: benchmark name, number of times ran, ns/op (with or without decimal), B/op (optional), and allocs/op (optional).
+	regexBenchmark       = regexp.MustCompile(`^(Benchmark[^ -]+)(?:-\d+\s+|\s+)(\d+)\s+(\d+|\d+\.\d+)\sns/op(?:\s+(\d+)\sB/op)?(?:\s+(\d+)\sallocs/op)?`)
+	regexOutput          = regexp.MustCompile(`(    )*\t(.*)`)
+	regexSummary         = regexp.MustCompile(`^(PASS|FAIL|SKIP)$`)
+	regexPackageWithTest = regexp.MustCompile(`^# ([^\[\]]+) \[[^\]]+\]$`)
+)
+
+// Parse parses go test output from reader r and returns a report with the
+// results. An optional pkgName can be given, which is used in case a package
+// result line is missing.
+func Parse(r io.Reader, pkgName string) (*Report, error) {
+	reader := bufio.NewReader(r)
+
+	report := &Report{make([]Package, 0)}
+
+	// keep track of tests we find
+	var tests []*Test
+
+	// keep track of benchmarks we find
+	var benchmarks []*Benchmark
+
+	// sum of tests' time, use this if current test has no result line (when it is compiled test)
+	var testsTime time.Duration
+
+	// current test
+	var cur string
+
+	// coverage percentage report for current package
+	var coveragePct string
+
+	// stores mapping between package name and output of build failures
+	var packageCaptures = map[string][]string{}
+
+	// the name of the package which it's build failure output is being captured
+	var capturedPackage string
+
+	// capture any non-test output
+	var buffers = map[string][]string{}
+
+	// parse lines
+	for {
+		l, _, err := reader.ReadLine()
+		if err != nil && err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+
+		line := string(l)
+
+		if strings.HasPrefix(line, "=== RUN ") {
+			// new test
+			cur = strings.TrimSpace(line[8:])
+			tests = append(tests, &Test{
+				Name:   cur,
+				Result: FAIL,
+				Output: make([]string, 0),
+			})
+
+			// clear the current build package, so output lines won't be added to that build
+			capturedPackage = ""
+		} else if matches := regexBenchmark.FindStringSubmatch(line); len(matches) == 6 {
+			bytes, _ := strconv.Atoi(matches[4])
+			allocs, _ := strconv.Atoi(matches[5])
+
+			benchmarks = append(benchmarks, &Benchmark{
+				Name:     matches[1],
+				Duration: parseNanoseconds(matches[3]),
+				Bytes:    bytes,
+				Allocs:   allocs,
+			})
+		} else if strings.HasPrefix(line, "=== PAUSE ") {
+			continue
+		} else if strings.HasPrefix(line, "=== CONT ") {
+			cur = strings.TrimSpace(line[8:])
+			continue
+		} else if matches := regexResult.FindStringSubmatch(line); len(matches) == 6 {
+			if matches[5] != "" {
+				coveragePct = matches[5]
+			}
+			if strings.HasSuffix(matches[4], "failed]") {
+				// the build of the package failed, inject a dummy test into the package
+				// which indicate about the failure and contain the failure description.
+				tests = append(tests, &Test{
+					Name:   matches[4],
+					Result: FAIL,
+					Output: packageCaptures[matches[2]],
+				})
+			} else if matches[1] == "FAIL" && !containsFailures(tests) && len(buffers[cur]) > 0 {
+				// This package didn't have any failing tests, but still it
+				// failed with some output. Create a dummy test with the
+				// output.
+				tests = append(tests, &Test{
+					Name:   "Failure",
+					Result: FAIL,
+					Output: buffers[cur],
+				})
+				buffers[cur] = buffers[cur][0:0]
+			}
+
+			// all tests in this package are finished
+			report.Packages = append(report.Packages, Package{
+				Name:        matches[2],
+				Duration:    parseSeconds(matches[3]),
+				Tests:       tests,
+				Benchmarks:  benchmarks,
+				CoveragePct: coveragePct,
+
+				Time: int(parseSeconds(matches[3]) / time.Millisecond), // deprecated
+			})
+
+			buffers[cur] = buffers[cur][0:0]
+			tests = make([]*Test, 0)
+			benchmarks = make([]*Benchmark, 0)
+			coveragePct = ""
+			cur = ""
+			testsTime = 0
+		} else if matches := regexStatus.FindStringSubmatch(line); len(matches) == 4 {
+			cur = matches[2]
+			test := findTest(tests, cur)
+			if test == nil {
+				continue
+			}
+
+			// test status
+			if matches[1] == "PASS" {
+				test.Result = PASS
+			} else if matches[1] == "SKIP" {
+				test.Result = SKIP
+			} else {
+				test.Result = FAIL
+			}
+
+			if matches := regexIndent.FindStringSubmatch(line); len(matches) == 2 {
+				test.SubtestIndent = matches[1]
+			}
+
+			test.Output = buffers[cur]
+
+			test.Name = matches[2]
+			test.Duration = parseSeconds(matches[3])
+			testsTime += test.Duration
+
+			test.Time = int(test.Duration / time.Millisecond) // deprecated
+		} else if matches := regexCoverage.FindStringSubmatch(line); len(matches) == 2 {
+			coveragePct = matches[1]
+		} else if matches := regexOutput.FindStringSubmatch(line); capturedPackage == "" && len(matches) == 3 {
+			// Sub-tests start with one or more series of 4-space indents, followed by a hard tab,
+			// followed by the test output
+			// Top-level tests start with a hard tab.
+			test := findTest(tests, cur)
+			if test == nil {
+				continue
+			}
+			test.Output = append(test.Output, matches[2])
+		} else if strings.HasPrefix(line, "# ") {
+			// indicates a capture of build output of a package. set the current build package.
+			packageWithTestBinary := regexPackageWithTest.FindStringSubmatch(line)
+			if packageWithTestBinary != nil {
+				// Sometimes, the text after "# " shows the name of the test binary
+				// ("<package>.test") in addition to the package
+				// e.g.: "# package/name [package/name.test]"
+				capturedPackage = packageWithTestBinary[1]
+			} else {
+				capturedPackage = line[2:]
+			}
+		} else if capturedPackage != "" {
+			// current line is build failure capture for the current built package
+			packageCaptures[capturedPackage] = append(packageCaptures[capturedPackage], line)
+		} else if regexSummary.MatchString(line) {
+			// unset current test name so any additional output after the
+			// summary is captured separately.
+			cur = ""
+		} else {
+			// buffer anything else that we didn't recognize
+			buffers[cur] = append(buffers[cur], line)
+
+			// if we have a current test, also append to its output
+			test := findTest(tests, cur)
+			if test != nil {
+				if strings.HasPrefix(line, test.SubtestIndent+"    ") {
+					test.Output = append(test.Output, strings.TrimPrefix(line, test.SubtestIndent+"    "))
+				}
+			}
+		}
+	}
+
+	if len(tests) > 0 {
+		// no result line found
+		report.Packages = append(report.Packages, Package{
+			Name:        pkgName,
+			Duration:    testsTime,
+			Time:        int(testsTime / time.Millisecond),
+			Tests:       tests,
+			Benchmarks:  benchmarks,
+			CoveragePct: coveragePct,
+		})
+	}
+
+	return report, nil
+}
+
+func parseSeconds(t string) time.Duration {
+	if t == "" {
+		return time.Duration(0)
+	}
+	// ignore error
+	d, _ := time.ParseDuration(t + "s")
+	return d
+}
+
+func parseNanoseconds(t string) time.Duration {
+	// note: if input < 1 ns precision, result will be 0s.
+	if t == "" {
+		return time.Duration(0)
+	}
+	// ignore error
+	d, _ := time.ParseDuration(t + "ns")
+	return d
+}
+
+func findTest(tests []*Test, name string) *Test {
+	for i := len(tests) - 1; i >= 0; i-- {
+		if tests[i].Name == name {
+			return tests[i]
+		}
+	}
+	return nil
+}
+
+func containsFailures(tests []*Test) bool {
+	for _, test := range tests {
+		if test.Result == FAIL {
+			return true
+		}
+	}
+	return false
+}
+
+// Failures counts the number of failed tests in this report
+func (r *Report) Failures() int {
+	count := 0
+
+	for _, p := range r.Packages {
+		for _, t := range p.Tests {
+			if t.Result == FAIL {
+				count++
+			}
+		}
+	}
+
+	return count
+}