feb357e538
Use case -------- If Endpoint is not set and Region is - upload to AWS S3 If both the Endpoint and Region are set - upload the Generic S3 via Weldr API If neither the Endpoint and Region are set - upload the Generic S3 via Composer API (use configuration) jobimpl-osbuild --------------- Add configuration fields for Generic S3 upload Support S3 upload requests coming from Weldr or Composer API to either AWS or Generic S3 Weldr API for Generic S3 requires that all connection parameters but the credentials be passed in the API call Composer API for Generic S3 requires that all conneciton parameters are taken from the configuration Adjust to the consolidation in Target and UploadOptions Target and UploadOptions ------------------------ Add the fields that were specific to the Generic S3 structures to the AWS S3 one Remove the structures for Generic S3 and always use the AWS S3 ones Worker Main ----------- Add Endpoint, Region, Bucket, CABundle and SkipSSLVerification to the configuration structure Pass the values to the Server Weldr API --------- Keep the generic.s3 provider name to maintain the API, but unmarshel into awsS3UploadSettings tests - api.sh -------------- Allow the caller to specifiy either AWS or Generic S3 upload targets for specific image types Implement the pieces required for testing upload to a Generic S3 service In some cases generalize the AWS S3 functions for reuse GitLab CI --------- Add test case for api.sh tests with edge-commit and generic S3
328 lines
9.6 KiB
Go
328 lines
9.6 KiB
Go
package weldr
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/osbuild/osbuild-composer/internal/common"
|
|
"github.com/osbuild/osbuild-composer/internal/distro"
|
|
"github.com/sirupsen/logrus"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/osbuild/osbuild-composer/internal/target"
|
|
)
|
|
|
|
type uploadResponse struct {
|
|
UUID uuid.UUID `json:"uuid"`
|
|
Status common.ImageBuildState `json:"status"`
|
|
ProviderName string `json:"provider_name"`
|
|
ImageName string `json:"image_name"`
|
|
CreationTime float64 `json:"creation_time"`
|
|
Settings uploadSettings `json:"settings"`
|
|
}
|
|
|
|
type uploadSettings interface {
|
|
isUploadSettings()
|
|
}
|
|
|
|
type awsUploadSettings struct {
|
|
Region string `json:"region"`
|
|
AccessKeyID string `json:"accessKeyID,omitempty"`
|
|
SecretAccessKey string `json:"secretAccessKey,omitempty"`
|
|
SessionToken string `json:"sessionToken,omitempty"`
|
|
Bucket string `json:"bucket"`
|
|
Key string `json:"key"`
|
|
}
|
|
|
|
func (awsUploadSettings) isUploadSettings() {}
|
|
|
|
type awsS3UploadSettings struct {
|
|
Region string `json:"region"`
|
|
AccessKeyID string `json:"accessKeyID,omitempty"`
|
|
SecretAccessKey string `json:"secretAccessKey,omitempty"`
|
|
SessionToken string `json:"sessionToken,omitempty"`
|
|
Bucket string `json:"bucket"`
|
|
Key string `json:"key"`
|
|
Endpoint string `json:"endpoint"`
|
|
CABundle string `json:"ca_bundle"`
|
|
SkipSSLVerification bool `json:"skip_ssl_verification"`
|
|
}
|
|
|
|
func (awsS3UploadSettings) isUploadSettings() {}
|
|
|
|
type azureUploadSettings struct {
|
|
StorageAccount string `json:"storageAccount,omitempty"`
|
|
StorageAccessKey string `json:"storageAccessKey,omitempty"`
|
|
Container string `json:"container"`
|
|
}
|
|
|
|
func (azureUploadSettings) isUploadSettings() {}
|
|
|
|
type gcpUploadSettings struct {
|
|
Filename string `json:"filename"`
|
|
Region string `json:"region"`
|
|
Bucket string `json:"bucket"`
|
|
Object string `json:"object"`
|
|
|
|
// base64 encoded GCP credentials JSON file
|
|
Credentials string `json:"credentials,omitempty"`
|
|
}
|
|
|
|
func (gcpUploadSettings) isUploadSettings() {}
|
|
|
|
type vmwareUploadSettings struct {
|
|
Host string `json:"host"`
|
|
Username string `json:"username"`
|
|
Password string `json:"password"`
|
|
Datacenter string `json:"datacenter"`
|
|
Cluster string `json:"cluster"`
|
|
Datastore string `json:"datastore"`
|
|
}
|
|
|
|
func (vmwareUploadSettings) isUploadSettings() {}
|
|
|
|
type ociUploadSettings struct {
|
|
Tenancy string `json:"tenancy"`
|
|
Region string `json:"region"`
|
|
User string `json:"user"`
|
|
Bucket string `json:"bucket"`
|
|
Namespace string `json:"namespace"`
|
|
PrivateKey string `json:"private_key"`
|
|
Fingerprint string `json:"fingerprint"`
|
|
Compartment string `json:"compartment"`
|
|
}
|
|
|
|
func (ociUploadSettings) isUploadSettings() {}
|
|
|
|
type uploadRequest struct {
|
|
Provider string `json:"provider"`
|
|
ImageName string `json:"image_name"`
|
|
Settings uploadSettings `json:"settings"`
|
|
}
|
|
|
|
type rawUploadRequest struct {
|
|
Provider string `json:"provider"`
|
|
ImageName string `json:"image_name"`
|
|
Settings json.RawMessage `json:"settings"`
|
|
}
|
|
|
|
func (u *uploadRequest) UnmarshalJSON(data []byte) error {
|
|
var rawUploadRequest rawUploadRequest
|
|
err := json.Unmarshal(data, &rawUploadRequest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var settings uploadSettings
|
|
switch rawUploadRequest.Provider {
|
|
case "azure":
|
|
settings = new(azureUploadSettings)
|
|
case "aws":
|
|
settings = new(awsUploadSettings)
|
|
case "aws.s3":
|
|
settings = new(awsS3UploadSettings)
|
|
case "gcp":
|
|
settings = new(gcpUploadSettings)
|
|
case "vmware":
|
|
settings = new(vmwareUploadSettings)
|
|
case "oci":
|
|
settings = new(ociUploadSettings)
|
|
case "generic.s3":
|
|
// While the API still accepts provider type "generic.s3", the request is handled
|
|
// in the same way as for a request with provider type "aws.s3"
|
|
settings = new(awsS3UploadSettings)
|
|
default:
|
|
return errors.New("unexpected provider name")
|
|
}
|
|
err = json.Unmarshal(rawUploadRequest.Settings, settings)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
u.Provider = rawUploadRequest.Provider
|
|
u.ImageName = rawUploadRequest.ImageName
|
|
u.Settings = settings
|
|
|
|
return err
|
|
}
|
|
|
|
// Converts a `Target` to a serializable `uploadResponse`.
|
|
//
|
|
// This ignore the status in `targets`, because that's never set correctly.
|
|
// Instead, it sets each target's status to the ImageBuildState equivalent of
|
|
// `state`.
|
|
//
|
|
// This also ignores any sensitive data passed into targets. Access keys may
|
|
// be passed as input to composer, but should not be possible to be queried.
|
|
func targetsToUploadResponses(targets []*target.Target, state ComposeState) []uploadResponse {
|
|
var uploads []uploadResponse
|
|
for _, t := range targets {
|
|
upload := uploadResponse{
|
|
UUID: t.Uuid,
|
|
ImageName: t.ImageName,
|
|
CreationTime: float64(t.Created.UnixNano()) / 1000000000,
|
|
}
|
|
|
|
switch state {
|
|
case ComposeWaiting:
|
|
upload.Status = common.IBWaiting
|
|
case ComposeRunning:
|
|
upload.Status = common.IBRunning
|
|
case ComposeFinished:
|
|
upload.Status = common.IBFinished
|
|
case ComposeFailed:
|
|
upload.Status = common.IBFailed
|
|
}
|
|
|
|
switch options := t.Options.(type) {
|
|
case *target.AWSTargetOptions:
|
|
upload.ProviderName = "aws"
|
|
upload.Settings = &awsUploadSettings{
|
|
Region: options.Region,
|
|
Bucket: options.Bucket,
|
|
Key: options.Key,
|
|
// AccessKeyID and SecretAccessKey are intentionally not included.
|
|
}
|
|
uploads = append(uploads, upload)
|
|
case *target.AzureTargetOptions:
|
|
upload.ProviderName = "azure"
|
|
upload.Settings = &azureUploadSettings{
|
|
Container: options.Container,
|
|
// StorageAccount and StorageAccessKey are intentionally not included.
|
|
}
|
|
uploads = append(uploads, upload)
|
|
case *target.GCPTargetOptions:
|
|
upload.ProviderName = "gcp"
|
|
upload.Settings = &gcpUploadSettings{
|
|
Filename: options.Filename,
|
|
Region: options.Region,
|
|
Bucket: options.Bucket,
|
|
Object: options.Object,
|
|
// Credentials are intentionally not included.
|
|
}
|
|
uploads = append(uploads, upload)
|
|
case *target.VMWareTargetOptions:
|
|
upload.ProviderName = "vmware"
|
|
upload.Settings = &vmwareUploadSettings{
|
|
Host: options.Host,
|
|
Cluster: options.Cluster,
|
|
Datacenter: options.Datacenter,
|
|
Datastore: options.Datastore,
|
|
// Username and Password are intentionally not included.
|
|
}
|
|
uploads = append(uploads, upload)
|
|
case *target.AWSS3TargetOptions:
|
|
upload.ProviderName = "aws.s3"
|
|
upload.Settings = &awsS3UploadSettings{
|
|
Region: options.Region,
|
|
Bucket: options.Bucket,
|
|
Key: options.Key,
|
|
// AccessKeyID and SecretAccessKey are intentionally not included.
|
|
}
|
|
uploads = append(uploads, upload)
|
|
}
|
|
}
|
|
|
|
return uploads
|
|
}
|
|
|
|
func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target.Target {
|
|
var t target.Target
|
|
|
|
t.Uuid = uuid.New()
|
|
t.ImageName = u.ImageName
|
|
t.Status = common.IBWaiting
|
|
t.Created = time.Now()
|
|
|
|
switch options := u.Settings.(type) {
|
|
case *awsUploadSettings:
|
|
t.Name = "org.osbuild.aws"
|
|
t.Options = &target.AWSTargetOptions{
|
|
Filename: imageType.Filename(),
|
|
Region: options.Region,
|
|
AccessKeyID: options.AccessKeyID,
|
|
SecretAccessKey: options.SecretAccessKey,
|
|
SessionToken: options.SessionToken,
|
|
Bucket: options.Bucket,
|
|
Key: options.Key,
|
|
}
|
|
case *awsS3UploadSettings:
|
|
t.Name = "org.osbuild.aws.s3"
|
|
t.Options = &target.AWSS3TargetOptions{
|
|
Filename: imageType.Filename(),
|
|
Region: options.Region,
|
|
AccessKeyID: options.AccessKeyID,
|
|
SecretAccessKey: options.SecretAccessKey,
|
|
SessionToken: options.SessionToken,
|
|
Bucket: options.Bucket,
|
|
Key: options.Key,
|
|
Endpoint: options.Endpoint,
|
|
CABundle: options.CABundle,
|
|
SkipSSLVerification: options.SkipSSLVerification,
|
|
}
|
|
case *azureUploadSettings:
|
|
t.Name = "org.osbuild.azure"
|
|
t.Options = &target.AzureTargetOptions{
|
|
Filename: imageType.Filename(),
|
|
StorageAccount: options.StorageAccount,
|
|
StorageAccessKey: options.StorageAccessKey,
|
|
Container: options.Container,
|
|
}
|
|
case *gcpUploadSettings:
|
|
t.Name = "org.osbuild.gcp"
|
|
|
|
var gcpCredentials []byte
|
|
var err error
|
|
if options.Credentials != "" {
|
|
gcpCredentials, err = base64.StdEncoding.DecodeString(options.Credentials)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// The uploaded image object name must have 'tar.gz' suffix to be imported
|
|
objectName := options.Object
|
|
if !strings.HasSuffix(objectName, ".tar.gz") {
|
|
objectName = objectName + ".tar.gz"
|
|
logrus.Infof("[GCP] object name must end with '.tar.gz', using %q as the object name", objectName)
|
|
}
|
|
|
|
t.Options = &target.GCPTargetOptions{
|
|
Filename: imageType.Filename(),
|
|
Region: options.Region,
|
|
Os: imageType.Arch().Distro().Name(),
|
|
Bucket: options.Bucket,
|
|
Object: objectName,
|
|
Credentials: gcpCredentials,
|
|
}
|
|
case *vmwareUploadSettings:
|
|
t.Name = "org.osbuild.vmware"
|
|
t.Options = &target.VMWareTargetOptions{
|
|
Filename: imageType.Filename(),
|
|
Username: options.Username,
|
|
Password: options.Password,
|
|
Host: options.Host,
|
|
Cluster: options.Cluster,
|
|
Datacenter: options.Datacenter,
|
|
Datastore: options.Datastore,
|
|
}
|
|
case *ociUploadSettings:
|
|
t.Name = "org.osbuild.oci"
|
|
t.Options = &target.OCITargetOptions{
|
|
User: options.User,
|
|
Tenancy: options.Tenancy,
|
|
Region: options.Region,
|
|
FileName: imageType.Filename(),
|
|
PrivateKey: options.PrivateKey,
|
|
Fingerprint: options.Fingerprint,
|
|
Bucket: options.Bucket,
|
|
Namespace: options.Namespace,
|
|
Compartment: options.Compartment,
|
|
}
|
|
}
|
|
|
|
return &t
|
|
}
|