Upload to HTTPS S3 - Support self signed certificate

API
---
Allow the user to pass the CA public certification or skip the verification

AWSCloud
--------
Restore the old version of newAwsFromCreds for access to AWS
Create a new method newAwsFromCredsWithEndpoint for Generic S3 which sets the endpoint and optionally overrides the CA Bundle or skips the SSL certificate verification

jobimpl-osbuild
---------------
Update with the new parameters

osbuild-upload-generic-s3
-------------------------
Add ca-bunlde and skip-ssl-verification flags

tests
-----
Split the tests into http, https with certificate and https skip certificate check
Create a new base test for S3 over HTTPS for secure and insecure
Move the generic S3 test to tools to reuse for secure and insecure connections
All S3 tests now use the aws cli tool
Update the libvirt test to be able to download over HTTPS
Update the RPM spec

Kill container with sudo

internal/cloud/awscloud/awscloud.go

@@ -1,7 +1,9 @@
 package awscloud
 
 import (
+	"crypto/tls"
 	"fmt"
+	"net/http"
 	"os"
 	"time"
 
@@ -22,14 +24,11 @@ type AWS struct {
 }
 
 // Create a new session from the credentials and the region and returns an *AWS object initialized with it.
-func newAwsFromCreds(creds *credentials.Credentials, region string, endpoint *string) (*AWS, error) {
+func newAwsFromCreds(creds *credentials.Credentials, region string) (*AWS, error) {
 	// Create a Session with a custom region
-	s3ForcePathStyle := endpoint != nil
 	sess, err := session.NewSession(&aws.Config{
-		Credentials:      creds,
-		Region:           aws.String(region),
-		Endpoint:         endpoint,
-		S3ForcePathStyle: &s3ForcePathStyle,
+		Credentials: creds,
+		Region:      aws.String(region),
 	})
 	if err != nil {
 		return nil, err
@@ -44,7 +43,7 @@ func newAwsFromCreds(creds *credentials.Credentials, region string, endpoint *st
 
 // Initialize a new AWS object from individual bits. SessionToken is optional
 func New(region string, accessKeyID string, accessKey string, sessionToken string) (*AWS, error) {
-	return newAwsFromCreds(credentials.NewStaticCredentials(accessKeyID, accessKey, sessionToken), region, nil)
+	return newAwsFromCreds(credentials.NewStaticCredentials(accessKeyID, accessKey, sessionToken), region)
 }
 
 // Initializes a new AWS object with the credentials info found at filename's location.
@@ -57,18 +56,60 @@ func New(region string, accessKeyID string, accessKey string, sessionToken strin
 // "AWS_SHARED_CREDENTIALS_FILE" env variable or will default to
 // $HOME/.aws/credentials.
 func NewFromFile(filename string, region string) (*AWS, error) {
-	return newAwsFromCreds(credentials.NewSharedCredentials(filename, "default"), region, nil)
+	return newAwsFromCreds(credentials.NewSharedCredentials(filename, "default"), region)
 }
 
 // Initialize a new AWS object from defaults.
 // Looks for env variables, shared credential file, and EC2 Instance Roles.
 func NewDefault(region string) (*AWS, error) {
-	return newAwsFromCreds(nil, region, nil)
+	return newAwsFromCreds(nil, region)
+}
+
+// Create a new session from the credentials and the region and returns an *AWS object initialized with it.
+func newAwsFromCredsWithEndpoint(creds *credentials.Credentials, region, endpoint, caBundle string, skipSSLVerification bool) (*AWS, error) {
+	// Create a Session with a custom region
+	s3ForcePathStyle := true
+	sessionOptions := session.Options{
+		Config: aws.Config{
+			Credentials:      creds,
+			Region:           aws.String(region),
+			Endpoint:         &endpoint,
+			S3ForcePathStyle: &s3ForcePathStyle,
+		},
+	}
+
+	if caBundle != "" {
+		caBundleReader, err := os.Open(caBundle)
+		if err != nil {
+			return nil, err
+		}
+		defer caBundleReader.Close()
+		sessionOptions.CustomCABundle = caBundleReader
+	}
+
+	if skipSSLVerification {
+		transport := http.DefaultTransport.(*http.Transport).Clone()
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec G402
+		sessionOptions.Config.HTTPClient = &http.Client{
+			Transport: transport,
+		}
+	}
+
+	sess, err := session.NewSessionWithOptions(sessionOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	return &AWS{
+		uploader: s3manager.NewUploader(sess),
+		ec2:      ec2.New(sess),
+		s3:       s3.New(sess),
+	}, nil
 }
 
 // Initialize a new AWS object targeting a specific endpoint from individual bits. SessionToken is optional
-func NewForEndpoint(endpoint, region string, accessKeyID string, accessKey string, sessionToken string) (*AWS, error) {
-	return newAwsFromCreds(credentials.NewStaticCredentials(accessKeyID, accessKey, sessionToken), region, &endpoint)
+func NewForEndpoint(endpoint, region, accessKeyID, accessKey, sessionToken, caBundle string, skipSSLVerification bool) (*AWS, error) {
+	return newAwsFromCredsWithEndpoint(credentials.NewStaticCredentials(accessKeyID, accessKey, sessionToken), region, endpoint, caBundle, skipSSLVerification)
 }
 
 // Initializes a new AWS object targeting a specific endpoint with the credentials info found at filename's location.
@@ -80,8 +121,8 @@ func NewForEndpoint(endpoint, region string, accessKeyID string, accessKey strin
 // If filename is empty the underlying function will look for the
 // "AWS_SHARED_CREDENTIALS_FILE" env variable or will default to
 // $HOME/.aws/credentials.
-func NewForEndpointFromFile(filename string, endpoint, region string) (*AWS, error) {
-	return newAwsFromCreds(credentials.NewSharedCredentials(filename, "default"), region, &endpoint)
+func NewForEndpointFromFile(filename, endpoint, region, caBundle string, skipSSLVerification bool) (*AWS, error) {
+	return newAwsFromCredsWithEndpoint(credentials.NewSharedCredentials(filename, "default"), region, endpoint, caBundle, skipSSLVerification)
 }
 
 func (a *AWS) Upload(filename, bucket, key string) (*s3manager.UploadOutput, error) {