worker: implement koji job types
The three new job types osbuild-koji, koji-init, and koji-finalize allows the different tasks to be split appart and in particular for there to be several builds on different architectures as part of a given compose.
This commit is contained in:
Tom Gundersen
Lars Karlitski
parent
a2895376ae
commit
0e382e9cf4
6 changed files with 429 additions and 0 deletions
181
cmd/osbuild-worker/jobimpl-koji-finalize.go
Normal file
181
cmd/osbuild-worker/jobimpl-koji-finalize.go
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/osbuild/osbuild-composer/internal/upload/koji"
|
||||
"github.com/osbuild/osbuild-composer/internal/worker"
|
||||
)
|
||||
|
||||
type KojiFinalizeJobImpl struct {
|
||||
KojiServers map[string]koji.GSSAPICredentials
|
||||
}
|
||||
|
||||
func (impl *KojiFinalizeJobImpl) kojiImport(
|
||||
server string,
|
||||
build koji.ImageBuild,
|
||||
buildRoots []koji.BuildRoot,
|
||||
images []koji.Image,
|
||||
directory, token string) error {
|
||||
// Koji for some reason needs TLS renegotiation enabled.
|
||||
// Clone the default http transport and enable renegotiation.
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
Renegotiation: tls.RenegotiateOnceAsClient,
|
||||
}
|
||||
|
||||
serverURL, err := url.Parse(server)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
creds, exists := impl.KojiServers[serverURL.Hostname()]
|
||||
if !exists {
|
||||
return fmt.Errorf("Koji server has not been configured: %s", serverURL.Hostname())
|
||||
}
|
||||
|
||||
k, err := koji.NewFromGSSAPI(server, &creds, transport)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
err := k.Logout()
|
||||
if err != nil {
|
||||
log.Printf("koji logout failed: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
_, err = k.CGImport(build, buildRoots, images, directory, token)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not import build into koji: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (impl *KojiFinalizeJobImpl) kojiFail(server string, buildID int, token string) error {
|
||||
// Koji for some reason needs TLS renegotiation enabled.
|
||||
// Clone the default http transport and enable renegotiation.
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
Renegotiation: tls.RenegotiateOnceAsClient,
|
||||
}
|
||||
|
||||
serverURL, err := url.Parse(server)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
creds, exists := impl.KojiServers[serverURL.Hostname()]
|
||||
if !exists {
|
||||
return fmt.Errorf("Koji server has not been configured: %s", serverURL.Hostname())
|
||||
}
|
||||
|
||||
k, err := koji.NewFromGSSAPI(server, &creds, transport)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
err := k.Logout()
|
||||
if err != nil {
|
||||
log.Printf("koji logout failed: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return k.CGFailBuild(buildID, token)
|
||||
}
|
||||
|
||||
func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error {
|
||||
var args worker.KojiFinalizeJob
|
||||
err := job.Args(&args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var failure bool
|
||||
|
||||
var initArgs worker.KojiInitJobResult
|
||||
err = job.DynamicArgs(0, &initArgs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if initArgs.KojiError != nil {
|
||||
failure = true
|
||||
}
|
||||
|
||||
build := koji.ImageBuild{
|
||||
BuildID: initArgs.BuildID,
|
||||
TaskID: args.TaskID,
|
||||
Name: args.Name,
|
||||
Version: args.Version,
|
||||
Release: args.Release,
|
||||
StartTime: int64(args.StartTime),
|
||||
EndTime: time.Now().Unix(),
|
||||
}
|
||||
|
||||
var buildRoots []koji.BuildRoot
|
||||
var images []koji.Image
|
||||
for i := 1; i < job.NDynamicArgs(); i++ {
|
||||
var buildArgs worker.OSBuildKojiJobResult
|
||||
err = job.DynamicArgs(i, &buildArgs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !buildArgs.OSBuildOutput.Success || buildArgs.KojiError != nil {
|
||||
failure = true
|
||||
break
|
||||
}
|
||||
buildRoots = append(buildRoots, koji.BuildRoot{
|
||||
ID: uint64(i),
|
||||
Host: koji.Host{
|
||||
Os: buildArgs.HostOS,
|
||||
Arch: buildArgs.Arch,
|
||||
},
|
||||
ContentGenerator: koji.ContentGenerator{
|
||||
Name: "osbuild",
|
||||
Version: "0", // TODO: put the correct version here
|
||||
},
|
||||
Container: koji.Container{
|
||||
Type: "none",
|
||||
Arch: buildArgs.Arch,
|
||||
},
|
||||
Tools: []koji.Tool{},
|
||||
RPMs: osbuildStagesToRPMs(buildArgs.OSBuildOutput.Build.Stages),
|
||||
})
|
||||
images = append(images, koji.Image{
|
||||
BuildRootID: uint64(i),
|
||||
Filename: args.KojiFilenames[i-1],
|
||||
FileSize: buildArgs.ImageSize,
|
||||
Arch: buildArgs.Arch,
|
||||
ChecksumType: "md5",
|
||||
MD5: buildArgs.ImageHash,
|
||||
Type: "image",
|
||||
RPMs: osbuildStagesToRPMs(buildArgs.OSBuildOutput.Stages),
|
||||
Extra: koji.ImageExtra{
|
||||
Info: koji.ImageExtraInfo{
|
||||
Arch: buildArgs.Arch,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
var result worker.KojiFinalizeJobResult
|
||||
if failure {
|
||||
result.KojiError = impl.kojiFail(args.Server, int(initArgs.BuildID), initArgs.Token)
|
||||
} else {
|
||||
result.KojiError = impl.kojiImport(args.Server, build, buildRoots, images, args.KojiDirectory, initArgs.Token)
|
||||
}
|
||||
|
||||
err = job.Update(&result)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error reporting job result: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
71
cmd/osbuild-worker/jobimpl-koji-init.go
Normal file
71
cmd/osbuild-worker/jobimpl-koji-init.go
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"github.com/osbuild/osbuild-composer/internal/upload/koji"
|
||||
"github.com/osbuild/osbuild-composer/internal/worker"
|
||||
)
|
||||
|
||||
type KojiInitJobImpl struct {
|
||||
KojiServers map[string]koji.GSSAPICredentials
|
||||
}
|
||||
|
||||
func (impl *KojiInitJobImpl) kojiInit(server, name, version, release string) (string, uint64, error) {
|
||||
// Koji for some reason needs TLS renegotiation enabled.
|
||||
// Clone the default http transport and enable renegotiation.
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
Renegotiation: tls.RenegotiateOnceAsClient,
|
||||
}
|
||||
|
||||
serverURL, err := url.Parse(server)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
creds, exists := impl.KojiServers[serverURL.Hostname()]
|
||||
if !exists {
|
||||
return "", 0, fmt.Errorf("Koji server has not been configured: %s", serverURL.Hostname())
|
||||
}
|
||||
|
||||
k, err := koji.NewFromGSSAPI(server, &creds, transport)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer func() {
|
||||
err := k.Logout()
|
||||
if err != nil {
|
||||
log.Printf("koji logout failed: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
buildInfo, err := k.CGInitBuild(name, version, release)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
return buildInfo.Token, uint64(buildInfo.BuildID), nil
|
||||
}
|
||||
|
||||
func (impl *KojiInitJobImpl) Run(job worker.Job) error {
|
||||
var args worker.KojiInitJob
|
||||
err := job.Args(&args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var result worker.KojiInitJobResult
|
||||
result.Token, result.BuildID, result.KojiError = impl.kojiInit(args.Server, args.Name, args.Version, args.Release)
|
||||
|
||||
err = job.Update(&result)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error reporting job result: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
108
cmd/osbuild-worker/jobimpl-osbuild-koji.go
Normal file
108
cmd/osbuild-worker/jobimpl-osbuild-koji.go
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/osbuild/osbuild-composer/internal/common"
|
||||
"github.com/osbuild/osbuild-composer/internal/distro"
|
||||
"github.com/osbuild/osbuild-composer/internal/upload/koji"
|
||||
"github.com/osbuild/osbuild-composer/internal/worker"
|
||||
)
|
||||
|
||||
type OSBuildKojiJobImpl struct {
|
||||
Store string
|
||||
KojiServers map[string]koji.GSSAPICredentials
|
||||
}
|
||||
|
||||
func (impl *OSBuildKojiJobImpl) kojiUpload(file *os.File, server, directory, filename string) (string, uint64, error) {
|
||||
// Koji for some reason needs TLS renegotiation enabled.
|
||||
// Clone the default http transport and enable renegotiation.
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
Renegotiation: tls.RenegotiateOnceAsClient,
|
||||
}
|
||||
|
||||
serverURL, err := url.Parse(server)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
creds, exists := impl.KojiServers[serverURL.Hostname()]
|
||||
if !exists {
|
||||
return "", 0, fmt.Errorf("Koji server has not been configured: %s", serverURL.Hostname())
|
||||
}
|
||||
|
||||
k, err := koji.NewFromGSSAPI(server, &creds, transport)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer func() {
|
||||
err := k.Logout()
|
||||
if err != nil {
|
||||
log.Printf("koji logout failed: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return k.Upload(file, directory, filename)
|
||||
}
|
||||
|
||||
func (impl *OSBuildKojiJobImpl) Run(job worker.Job) error {
|
||||
outputDirectory, err := ioutil.TempDir("/var/tmp", "osbuild-worker-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating temporary output directory: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
err := os.RemoveAll(outputDirectory)
|
||||
if err != nil {
|
||||
log.Printf("Error removing temporary output directory (%s): %v", outputDirectory, err)
|
||||
}
|
||||
}()
|
||||
|
||||
var args worker.OSBuildKojiJob
|
||||
err = job.Args(&args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var initArgs worker.KojiInitJobResult
|
||||
err = job.DynamicArgs(0, &initArgs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var result worker.OSBuildKojiJobResult
|
||||
result.Arch = common.CurrentArch()
|
||||
result.HostOS, err = distro.GetRedHatRelease()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if initArgs.KojiError == nil {
|
||||
result.OSBuildOutput, err = RunOSBuild(args.Manifest, impl.Store, outputDirectory, os.Stderr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if result.OSBuildOutput.Success {
|
||||
f, err := os.Open(path.Join(outputDirectory, args.ImageName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.ImageHash, result.ImageSize, result.KojiError = impl.kojiUpload(f, args.KojiServer, args.KojiDirectory, args.KojiFilename)
|
||||
}
|
||||
}
|
||||
|
||||
err = job.Update(&result)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error reporting job result: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -154,6 +154,16 @@ func main() {
|
|||
Store: store,
|
||||
KojiServers: kojiServers,
|
||||
},
|
||||
"osbuild-koji": &OSBuildKojiJobImpl{
|
||||
Store: store,
|
||||
KojiServers: kojiServers,
|
||||
},
|
||||
"koji-init": &KojiInitJobImpl{
|
||||
KojiServers: kojiServers,
|
||||
},
|
||||
"koji-finalize": &KojiFinalizeJobImpl{
|
||||
KojiServers: kojiServers,
|
||||
},
|
||||
}
|
||||
|
||||
acceptedJobTypes := []string{}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ type Job interface {
|
|||
Id() uuid.UUID
|
||||
Type() string
|
||||
Args(args interface{}) error
|
||||
DynamicArgs(i int, args interface{}) error
|
||||
NDynamicArgs() int
|
||||
Update(result interface{}) error
|
||||
Canceled() (bool, error)
|
||||
UploadArtifact(name string, reader io.Reader) error
|
||||
|
|
@ -151,6 +153,18 @@ func (j *job) Args(args interface{}) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (j *job) NDynamicArgs() int {
|
||||
return len(j.dynamicArgs)
|
||||
}
|
||||
|
||||
func (j *job) DynamicArgs(i int, args interface{}) error {
|
||||
err := json.Unmarshal(j.dynamicArgs[i], args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error parsing job arguments: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j *job) Update(result interface{}) error {
|
||||
var buf bytes.Buffer
|
||||
err := json.NewEncoder(&buf).Encode(api.UpdateJobJSONRequestBody{
|
||||
|
|
|
|||
|
|
@ -26,6 +26,51 @@ type OSBuildJobResult struct {
|
|||
TargetErrors []string `json:"target_errors,omitempty"`
|
||||
}
|
||||
|
||||
type KojiInitJob struct {
|
||||
Server string `json:"server"`
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Release string `json:"release"`
|
||||
}
|
||||
|
||||
type KojiInitJobResult struct {
|
||||
BuildID uint64 `json:"build_id"`
|
||||
Token string `json:"token"`
|
||||
KojiError error `json:"koji_error"`
|
||||
}
|
||||
|
||||
type OSBuildKojiJob struct {
|
||||
Manifest distro.Manifest `json:"manifest"`
|
||||
ImageName string `json:"image_name"`
|
||||
KojiServer string `json:"koji_server"`
|
||||
KojiDirectory string `json:"koji_directory"`
|
||||
KojiFilename string `json:"koji_filename"`
|
||||
}
|
||||
|
||||
type OSBuildKojiJobResult struct {
|
||||
HostOS string `json:"host_os"`
|
||||
Arch string `json:"arch"`
|
||||
OSBuildOutput *osbuild.Result `json:"osbuild_output"`
|
||||
ImageHash string `json:"image_hash"`
|
||||
ImageSize uint64 `json:"image_size"`
|
||||
KojiError error `json:"koji_error"`
|
||||
}
|
||||
|
||||
type KojiFinalizeJob struct {
|
||||
Server string `json:"server"`
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Release string `json:"release"`
|
||||
KojiFilenames []string `json:"koji_filenames"`
|
||||
KojiDirectory string `json:"koji_directory"`
|
||||
TaskID uint64 `json:"task_id"` /* https://pagure.io/koji/issue/215 */
|
||||
StartTime uint64 `json:"start_time"`
|
||||
}
|
||||
|
||||
type KojiFinalizeJobResult struct {
|
||||
KojiError error `json:"koji_error"`
|
||||
}
|
||||
|
||||
//
|
||||
// JSON-serializable types for the HTTP API
|
||||
//
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue