Introduce Azure upload test

The test is very simple, it first upload a file with random content and
then downloads it back and compare hashes.

cmd/osbuild-upload-azure/main.go

@@ -35,7 +35,7 @@ func main() {
 
 	fmt.Println("Image to upload is:", fileName)
 
-	azure.UploadImage(azure.Credentials{
+	err := azure.UploadImage(azure.Credentials{
 		StorageAccount:   storageAccount,
 		StorageAccessKey: storageAccessKey,
 	}, azure.ImageMetadata{
@@ -43,4 +43,7 @@ func main() {
 		ContainerName: containerName,
 	}, fileName, threads)
 
+	if err != nil {
+		fmt.Println("Error: ", err)
+	}
 }

go.sum

@@ -1,9 +1,9 @@
-github.com/aws/aws-sdk-go v1.25.37 h1:gBtB/F3dophWpsUQKN/Kni+JzYEH2mGHF4hWNtfED1w=
-github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo=
 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
 github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o=
 github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=
+github.com/aws/aws-sdk-go v1.25.37 h1:gBtB/F3dophWpsUQKN/Kni+JzYEH2mGHF4hWNtfED1w=
+github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c=
 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=

internal/upload/azure/azure.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"crypto/md5"
 	"fmt"
 	"io"
 	"net/url"
@@ -13,6 +14,14 @@ import (
 	"github.com/Azure/azure-storage-blob-go/azblob"
 )
 
+type errorString struct {
+	s string
+}
+
+func (e *errorString) Error() string {
+	return e.s
+}
+
 // Credentials contains credentials to connect to your account
 // It uses so called "Client credentials", see the official documentation for more information:
 // https://docs.microsoft.com/en-us/azure/go/azure-sdk-go-authorization#available-authentication-types-and-methods
@@ -55,6 +64,7 @@ func UploadImage(credentials Credentials, metadata ImageMetadata, fileName strin
 	if err != nil {
 		return err
 	}
+	defer imageFile.Close()
 
 	// Stat image to get the file size
 	stat, err := imageFile.Stat()
@@ -62,12 +72,27 @@ func UploadImage(credentials Credentials, metadata ImageMetadata, fileName strin
 		return err
 	}
 
+	// Hash the imageFile
+	imageFileHash := md5.New()
+	if _, err := io.Copy(imageFileHash, imageFile); err != nil {
+		return err
+	}
+	// Move the cursor back to the start of the imageFile
+	if _, err := imageFile.Seek(0, 0); err != nil {
+		return err
+	}
+
 	// Create page blob URL. Page blob is required for VM images
 	blobURL := containerURL.NewPageBlobURL(metadata.ImageName)
 	_, err = blobURL.Create(ctx, stat.Size(), 0, azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{})
 	if err != nil {
 		return err
 	}
+	// Wrong MD5 does not seem to have any impact on the upload
+	_, err = blobURL.SetHTTPHeaders(ctx, azblob.BlobHTTPHeaders{ContentMD5: imageFileHash.Sum(nil)}, azblob.BlobAccessConditions{})
+	if err != nil {
+		return err
+	}
 
 	// Create control variables
 	// This channel simulates behavior of a semaphore and bounds the number of parallel threads
@@ -110,11 +135,25 @@ func UploadImage(credentials Credentials, metadata ImageMetadata, fileName strin
 		}(counter, buffer, n)
 		counter++
 	}
+	// Wait for all gorutines to finish
 	wg.Wait()
+	// Check any errors during the transmission using a nonblocking read from the channel
 	select {
 	case err := <-errorInGorutine:
 		return err
 	default:
-		return nil
 	}
+	// Check properties, specifically MD5 sum of the blob
+	props, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{})
+	if err != nil {
+		return err
+	}
+	var blobChecksum []byte = props.ContentMD5()
+	var fileChecksum []byte = imageFileHash.Sum(nil)
+
+	if bytes.Compare(blobChecksum, fileChecksum) != 0 {
+		return &errorString{"error during image upload. the image seems to be corrupted"}
+	}
+
+	return nil
 }

internal/upload/azure/azure_test.go

@@ -0,0 +1,117 @@
+package azure
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"os/exec"
+	"path"
+	"testing"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+)
+
+func handleErrors(t *testing.T, err error) {
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func loadEnvVar(t *testing.T, envVarName string) (string, bool) {
+	variable, exists := os.LookupEnv(envVarName)
+	if !exists {
+		t.Logf("Environment variable does not exist: %s", envVarName)
+		return "", false
+	}
+	return variable, true
+}
+
+func TestAzure_FileUpload(t *testing.T) {
+	// Load configuration
+	storageAccount, saExists := loadEnvVar(t, "AZURE_STORAGE_ACCOUNT")
+	storageAccessKey, sakExists := loadEnvVar(t, "AZURE_STORAGE_ACCESS_KEY")
+	containerName, cnExists := loadEnvVar(t, "AZURE_STORAGE_CONTAINER")
+	fileName := "/tmp/testing-image.vhd"
+	var threads int = 4
+
+	// Workaround Travis security feature. If non of the variables is set, just ignore the test
+	if saExists == false && sakExists == false && cnExists == false {
+		t.Log("No AZURE configuration provided, assuming that this is running in CI. Skipping the test.")
+		return
+	}
+	// If only one/two of them are not set, then fail
+	if saExists == false || sakExists == false || cnExists == false {
+		t.Fatal("You need to define all variables for AZURE connection.")
+	}
+
+	// Prepare the file
+	cmd := exec.Command("dd", "bs=512", "count=512", "if=/dev/urandom", fmt.Sprintf("of=%s", fileName))
+	err := cmd.Run()
+	handleErrors(t, err)
+	t.Log("Image to upload is:", fileName)
+
+	// Calculate MD5 sum of the file
+	f, err := os.Open(fileName)
+	handleErrors(t, err)
+
+	h := md5.New()
+	io.Copy(h, f)
+	handleErrors(t, err)
+	imageChecksum := h.Sum(nil)
+	f.Close()
+
+	credentials := Credentials{
+		StorageAccount:   storageAccount,
+		StorageAccessKey: storageAccessKey,
+	}
+	metadata := ImageMetadata{
+		ImageName:     path.Base(fileName),
+		ContainerName: containerName,
+	}
+	// Upload the image
+	err = UploadImage(credentials, metadata, fileName, threads)
+	handleErrors(t, err)
+
+	// Download the image
+	// Create a default request pipeline using your storage account name and account key.
+	credential, err := azblob.NewSharedKeyCredential(credentials.StorageAccount, credentials.StorageAccessKey)
+	handleErrors(t, err)
+
+	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+
+	// get storage account blob service URL endpoint.
+	URL, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s", credentials.StorageAccount, metadata.ContainerName))
+
+	// Create a ContainerURL object that wraps the container URL and a request
+	// pipeline to make requests.
+	containerURL := azblob.NewContainerURL(*URL, p)
+
+	// Create the container, use a never-expiring context
+	ctx := context.Background()
+
+	blobURL := containerURL.NewPageBlobURL(metadata.ImageName)
+
+	get, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false)
+	handleErrors(t, err)
+	blobData := &bytes.Buffer{}
+	reader := get.Body(azblob.RetryReaderOptions{})
+	blobData.ReadFrom(reader)
+	reader.Close() // The client must close the response body when finished with it
+	blobBytes := blobData.Bytes()
+	blobChecksum := md5.Sum(blobBytes)
+	t.Logf("Local image checksum:      %x\n", imageChecksum)
+	t.Logf("Downloaded blob checksum : %x\n", blobChecksum)
+
+	// Delete the blob and throw away any errors
+	_, _ = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{})
+	_ = os.Remove(fileName)
+
+	if bytes.Compare(imageChecksum, blobChecksum[:]) != 0 {
+		t.Fatalf("Checksums do not match! Local file: %x, cloud blob: %x", imageChecksum, blobChecksum[:])
+	}
+
+}