upload/azure: skip uploading empty pages
The size of the page blob is defined on creation and the blob is zero-initialized. Therefore, we can just skip all the pages that contain only zeros. This should save a lot of bandwidth if used on sparse files as e.g. operating system images. (:
This commit is contained in:
parent
abe6ccfb50
commit
943ead790e
1 changed files with 20 additions and 0 deletions
|
|
@ -62,6 +62,17 @@ const DefaultUploadThreads = 16
|
||||||
// See https://learn.microsoft.com/en-us/rest/api/storageservices/put-page
|
// See https://learn.microsoft.com/en-us/rest/api/storageservices/put-page
|
||||||
const PageBlobMaxUploadPagesBytes = 4 * 1024 * 1024
|
const PageBlobMaxUploadPagesBytes = 4 * 1024 * 1024
|
||||||
|
|
||||||
|
// allZerosSlice returns true if all values in the slice are equal to 0
|
||||||
|
func allZerosSlice(slice []byte) bool {
|
||||||
|
for i := 0; i < len(slice); i++ {
|
||||||
|
if slice[i] != 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// UploadPageBlob takes the metadata and credentials required to upload the image specified by `fileName`
|
// UploadPageBlob takes the metadata and credentials required to upload the image specified by `fileName`
|
||||||
// It can speed up the upload by using goroutines. The number of parallel goroutines is bounded by
|
// It can speed up the upload by using goroutines. The number of parallel goroutines is bounded by
|
||||||
// the `threads` argument.
|
// the `threads` argument.
|
||||||
|
|
@ -143,6 +154,15 @@ func (c StorageClient) UploadPageBlob(metadata BlobMetadata, fileName string, th
|
||||||
if n == 0 {
|
if n == 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Skip the uploading part if there are only zeros in the buffer.
|
||||||
|
// We already defined the size of the blob in the initial call and the blob is zero-initialized,
|
||||||
|
// so this pushing zeros would actually be a no-op.
|
||||||
|
if allZerosSlice(buffer) {
|
||||||
|
counter++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
semaphore <- 1
|
semaphore <- 1
|
||||||
go func(counter int64, buffer []byte, n int) {
|
go func(counter int64, buffer []byte, n int) {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue