diff --git a/cmd/osbuild-upload-azure/main.go b/cmd/osbuild-upload-azure/main.go index 779a2e13b..74e03f581 100644 --- a/cmd/osbuild-upload-azure/main.go +++ b/cmd/osbuild-upload-azure/main.go @@ -35,7 +35,7 @@ func main() { fmt.Println("Image to upload is:", fileName) - azure.UploadImage(azure.Credentials{ + err := azure.UploadImage(azure.Credentials{ StorageAccount: storageAccount, StorageAccessKey: storageAccessKey, }, azure.ImageMetadata{ @@ -43,4 +43,7 @@ func main() { ContainerName: containerName, }, fileName, threads) + if err != nil { + fmt.Println("Error: ", err) + } } diff --git a/go.sum b/go.sum index 163bc1388..538763b38 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,9 @@ -github.com/aws/aws-sdk-go v1.25.37 h1:gBtB/F3dophWpsUQKN/Kni+JzYEH2mGHF4hWNtfED1w= -github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o= github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= +github.com/aws/aws-sdk-go v1.25.37 h1:gBtB/F3dophWpsUQKN/Kni+JzYEH2mGHF4hWNtfED1w= +github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= diff --git a/internal/upload/azure/azure.go b/internal/upload/azure/azure.go index eec8717ca..88f07ea9a 100644 --- a/internal/upload/azure/azure.go +++ b/internal/upload/azure/azure.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "crypto/md5" "fmt" "io" "net/url" @@ -13,6 +14,14 @@ import ( "github.com/Azure/azure-storage-blob-go/azblob" ) +type errorString struct { + s string +} + +func (e *errorString) Error() string { + return e.s +} + // Credentials contains credentials to connect to your account // It uses so called "Client credentials", see the official documentation for more information: // https://docs.microsoft.com/en-us/azure/go/azure-sdk-go-authorization#available-authentication-types-and-methods @@ -55,6 +64,7 @@ func UploadImage(credentials Credentials, metadata ImageMetadata, fileName strin if err != nil { return err } + defer imageFile.Close() // Stat image to get the file size stat, err := imageFile.Stat() @@ -62,12 +72,27 @@ func UploadImage(credentials Credentials, metadata ImageMetadata, fileName strin return err } + // Hash the imageFile + imageFileHash := md5.New() + if _, err := io.Copy(imageFileHash, imageFile); err != nil { + return err + } + // Move the cursor back to the start of the imageFile + if _, err := imageFile.Seek(0, 0); err != nil { + return err + } + // Create page blob URL. Page blob is required for VM images blobURL := containerURL.NewPageBlobURL(metadata.ImageName) _, err = blobURL.Create(ctx, stat.Size(), 0, azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{}) if err != nil { return err } + // Wrong MD5 does not seem to have any impact on the upload + _, err = blobURL.SetHTTPHeaders(ctx, azblob.BlobHTTPHeaders{ContentMD5: imageFileHash.Sum(nil)}, azblob.BlobAccessConditions{}) + if err != nil { + return err + } // Create control variables // This channel simulates behavior of a semaphore and bounds the number of parallel threads @@ -110,11 +135,25 @@ func UploadImage(credentials Credentials, metadata ImageMetadata, fileName strin }(counter, buffer, n) counter++ } + // Wait for all gorutines to finish wg.Wait() + // Check any errors during the transmission using a nonblocking read from the channel select { case err := <-errorInGorutine: return err default: - return nil } + // Check properties, specifically MD5 sum of the blob + props, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{}) + if err != nil { + return err + } + var blobChecksum []byte = props.ContentMD5() + var fileChecksum []byte = imageFileHash.Sum(nil) + + if bytes.Compare(blobChecksum, fileChecksum) != 0 { + return &errorString{"error during image upload. the image seems to be corrupted"} + } + + return nil } diff --git a/internal/upload/azure/azure_test.go b/internal/upload/azure/azure_test.go new file mode 100644 index 000000000..79b4bf2d6 --- /dev/null +++ b/internal/upload/azure/azure_test.go @@ -0,0 +1,117 @@ +package azure + +import ( + "bytes" + "context" + "crypto/md5" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path" + "testing" + + "github.com/Azure/azure-storage-blob-go/azblob" +) + +func handleErrors(t *testing.T, err error) { + if err != nil { + t.Fatal(err) + } +} + +func loadEnvVar(t *testing.T, envVarName string) (string, bool) { + variable, exists := os.LookupEnv(envVarName) + if !exists { + t.Logf("Environment variable does not exist: %s", envVarName) + return "", false + } + return variable, true +} + +func TestAzure_FileUpload(t *testing.T) { + // Load configuration + storageAccount, saExists := loadEnvVar(t, "AZURE_STORAGE_ACCOUNT") + storageAccessKey, sakExists := loadEnvVar(t, "AZURE_STORAGE_ACCESS_KEY") + containerName, cnExists := loadEnvVar(t, "AZURE_STORAGE_CONTAINER") + fileName := "/tmp/testing-image.vhd" + var threads int = 4 + + // Workaround Travis security feature. If non of the variables is set, just ignore the test + if saExists == false && sakExists == false && cnExists == false { + t.Log("No AZURE configuration provided, assuming that this is running in CI. Skipping the test.") + return + } + // If only one/two of them are not set, then fail + if saExists == false || sakExists == false || cnExists == false { + t.Fatal("You need to define all variables for AZURE connection.") + } + + // Prepare the file + cmd := exec.Command("dd", "bs=512", "count=512", "if=/dev/urandom", fmt.Sprintf("of=%s", fileName)) + err := cmd.Run() + handleErrors(t, err) + t.Log("Image to upload is:", fileName) + + // Calculate MD5 sum of the file + f, err := os.Open(fileName) + handleErrors(t, err) + + h := md5.New() + io.Copy(h, f) + handleErrors(t, err) + imageChecksum := h.Sum(nil) + f.Close() + + credentials := Credentials{ + StorageAccount: storageAccount, + StorageAccessKey: storageAccessKey, + } + metadata := ImageMetadata{ + ImageName: path.Base(fileName), + ContainerName: containerName, + } + // Upload the image + err = UploadImage(credentials, metadata, fileName, threads) + handleErrors(t, err) + + // Download the image + // Create a default request pipeline using your storage account name and account key. + credential, err := azblob.NewSharedKeyCredential(credentials.StorageAccount, credentials.StorageAccessKey) + handleErrors(t, err) + + p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + + // get storage account blob service URL endpoint. + URL, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s", credentials.StorageAccount, metadata.ContainerName)) + + // Create a ContainerURL object that wraps the container URL and a request + // pipeline to make requests. + containerURL := azblob.NewContainerURL(*URL, p) + + // Create the container, use a never-expiring context + ctx := context.Background() + + blobURL := containerURL.NewPageBlobURL(metadata.ImageName) + + get, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false) + handleErrors(t, err) + blobData := &bytes.Buffer{} + reader := get.Body(azblob.RetryReaderOptions{}) + blobData.ReadFrom(reader) + reader.Close() // The client must close the response body when finished with it + blobBytes := blobData.Bytes() + blobChecksum := md5.Sum(blobBytes) + t.Logf("Local image checksum: %x\n", imageChecksum) + t.Logf("Downloaded blob checksum : %x\n", blobChecksum) + + // Delete the blob and throw away any errors + _, _ = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}) + _ = os.Remove(fileName) + + if bytes.Compare(imageChecksum, blobChecksum[:]) != 0 { + t.Fatalf("Checksums do not match! Local file: %x, cloud blob: %x", imageChecksum, blobChecksum[:]) + } + +}