debian-forge-composer/internal/worker/json.go

package worker

import (
	"encoding/json"
	"fmt"

	"github.com/osbuild/osbuild-composer/internal/distro"
	"github.com/osbuild/osbuild-composer/internal/osbuild"
	"github.com/osbuild/osbuild-composer/internal/rpmmd"
	"github.com/osbuild/osbuild-composer/internal/target"
	"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
)

//
// JSON-serializable types for the jobqueue
//

type OSBuildJob struct {
	Manifest distro.Manifest `json:"manifest,omitempty"`
	// Index of the ManifestJobByIDResult instance in the job's dynamic arguments slice
	ManifestDynArgsIdx *int             `json:"manifest_dyn_args_idx,omitempty"`
	Targets            []*target.Target `json:"targets,omitempty"`
	PipelineNames      *PipelineNames   `json:"pipeline_names,omitempty"`
}

// OsbuildExports returns a slice of osbuild pipeline names, which should be
// exported as part of running osbuild image build for the job. The pipeline
// names are gathered from the targets specified in the job.
func (j OSBuildJob) OsbuildExports() []string {
	exports := []string{}
	seenExports := map[string]bool{}
	for _, target := range j.Targets {
		exists := seenExports[target.OsbuildArtifact.ExportName]
		if !exists {
			seenExports[target.OsbuildArtifact.ExportName] = true
			exports = append(exports, target.OsbuildArtifact.ExportName)
		}
	}
	return exports
}

type JobResult struct {
	JobError *clienterrors.Error `json:"job_error,omitempty"`
}

type OSBuildJobResult struct {
	Success       bool                   `json:"success"`
	OSBuildOutput *osbuild.Result        `json:"osbuild_output,omitempty"`
	TargetResults []*target.TargetResult `json:"target_results,omitempty"`
	UploadStatus  string                 `json:"upload_status"`
	PipelineNames *PipelineNames         `json:"pipeline_names,omitempty"`
	// Host OS of the worker which handled the job
	HostOS string `json:"host_os"`
	// Architecture of the worker which handled the job
	Arch string `json:"arch"`
	JobResult
}

// TargetErrors returns a slice of *clienterrors.Error gathered
// from the job result's target results. If there were no target errors
// then the returned slice will be empty.
func (j *OSBuildJobResult) TargetErrors() []*clienterrors.Error {
	targetErrors := []*clienterrors.Error{}

	for _, targetResult := range j.TargetResults {
		if targetResult.TargetError != nil {
			targetError := targetResult.TargetError
			// Add the target name to the error details, because the error reason
			// may not contain any information to determine the type of the target
			// which failed.
			targetErrors = append(targetErrors, clienterrors.WorkerClientError(targetError.ID, targetError.Reason, targetResult.Name))
		}
	}

	return targetErrors
}

// TargetResultsByName iterates over TargetResults attached to the Job result and
// returns a slice of Target results of the provided name (type). If there were no
// TargetResults of the desired type attached to the Job results, the returned
// slice will be empty.
func (j *OSBuildJobResult) TargetResultsByName(name target.TargetName) []*target.TargetResult {
	targetResults := []*target.TargetResult{}
	for _, targetResult := range j.TargetResults {
		if targetResult.Name == name {
			targetResults = append(targetResults, targetResult)
		}
	}
	return targetResults
}

type KojiInitJob struct {
	Server  string `json:"server"`
	Name    string `json:"name"`
	Version string `json:"version"`
	Release string `json:"release"`
}

type KojiInitJobResult struct {
	BuildID   uint64 `json:"build_id"`
	Token     string `json:"token"`
	KojiError string `json:"koji_error"`
	JobResult
}

type KojiFinalizeJob struct {
	Server        string   `json:"server"`
	Name          string   `json:"name"`
	Version       string   `json:"version"`
	Release       string   `json:"release"`
	KojiFilenames []string `json:"koji_filenames"`
	KojiDirectory string   `json:"koji_directory"`
	TaskID        uint64   `json:"task_id"` /* https://pagure.io/koji/issue/215 */
	StartTime     uint64   `json:"start_time"`
}

type KojiFinalizeJobResult struct {
	KojiError string `json:"koji_error"`
	JobResult
}

// PipelineNames is used to provide two pieces of information related to a job:
// 1. A categorization of each pipeline into one of two groups
// // 2. A pipeline ordering when the lists are concatenated: build -> os
// 2. A pipeline ordering when the lists are concatenated: build -> os
type PipelineNames struct {
	Build   []string `json:"build"`
	Payload []string `json:"payload"`
}

// Returns a concatenated list of the pipeline names
func (pn *PipelineNames) All() []string {
	return append(pn.Build, pn.Payload...)
}

// DepsolveJob defines the parameters of one or more depsolve jobs.  Each named
// list of package sets defines a separate job.  Lists with multiple package
// sets are depsolved in a chain, combining the results of sequential depsolves
// into a single PackageSpec list in the result.  Each PackageSet defines the
// repositories it will be depsolved against.
type DepsolveJob struct {
	PackageSets      map[string][]rpmmd.PackageSet `json:"grouped_package_sets"`
	ModulePlatformID string                        `json:"module_platform_id"`
	Arch             string                        `json:"arch"`
	Releasever       string                        `json:"releasever"`
}

// Custom marshaller for keeping compatibility with older workers.  The
// serialised format encompasses both the old and new DepsolveJob formats.  It
// is meant to be temporarily used to transition from the old to the new
// format.
func (ds DepsolveJob) MarshalJSON() ([]byte, error) {
	// NOTE: Common, top level repositories aren't used because in the new
	// format they don't exist; putting all required repositories on all
	// package sets as PackageSetsRepos should produce the same behaviour so
	// there's no need to try and figure out the "common" ones.
	// This also makes it possible to use old workers for new image types that
	// are incompatible with having common repos for all package sets (RHEL 7.9).
	compatJob := struct {
		// new format
		GroupedPackageSets map[string][]rpmmd.PackageSet `json:"grouped_package_sets"`
		ModulePlatformID   string                        `json:"module_platform_id"`
		Arch               string                        `json:"arch"`
		Releasever         string                        `json:"releasever"`

		// old format elements
		PackageSetsChains map[string][]string           `json:"package_sets_chains"`
		PackageSets       map[string]rpmmd.PackageSet   `json:"package_sets"`
		PackageSetsRepos  map[string][]rpmmd.RepoConfig `json:"package_sets_repositories,omitempty"`
	}{
		// new format substruct
		GroupedPackageSets: ds.PackageSets,
		ModulePlatformID:   ds.ModulePlatformID,
		Arch:               ds.Arch,
		Releasever:         ds.Releasever,
	}

	// build equivalent old format substruct
	pkgSetRepos := make(map[string][]rpmmd.RepoConfig)
	pkgSets := make(map[string]rpmmd.PackageSet)
	chains := make(map[string][]string)
	for chainName, pkgSetChain := range ds.PackageSets {
		if len(pkgSetChain) == 1 {
			// single element "chain" (i.e., not a chain)
			pkgSets[chainName] = rpmmd.PackageSet{
				Include: pkgSetChain[0].Include,
				Exclude: pkgSetChain[0].Exclude,
			}
			pkgSetRepos[chainName] = pkgSetChain[0].Repositories
			continue
		}
		chain := make([]string, len(pkgSetChain))
		for idx, set := range pkgSetChain {
			// the names of the individual sets in the chain don't matter, as long
			// as they match the keys for the repo configs
			setName := fmt.Sprintf("%s-%d", chainName, idx)
			// the package set (without repos)
			pkgSets[setName] = rpmmd.PackageSet{
				Include: set.Include,
				Exclude: set.Exclude,
			}
			// set repositories
			pkgSetRepos[setName] = set.Repositories
			// add name to the chain
			chain[idx] = setName
		}
		chains[chainName] = chain
	}

	compatJob.PackageSets = pkgSets
	compatJob.PackageSetsChains = chains
	compatJob.PackageSetsRepos = pkgSetRepos

	return json.Marshal(compatJob)
}

type ErrorType string

const (
	DepsolveErrorType ErrorType = "depsolve"
	OtherErrorType    ErrorType = "other"
)

type DepsolveJobResult struct {
	PackageSpecs map[string][]rpmmd.PackageSpec `json:"package_specs"`
	Error        string                         `json:"error"`
	ErrorType    ErrorType                      `json:"error_type"`
	JobResult
}

type ManifestJobByID struct{}

type ManifestJobByIDResult struct {
	Manifest distro.Manifest `json:"data,omitempty"`
	Error    string          `json:"error"`
	JobResult
}

type ContainerSpec struct {
	Source    string `json:"source"`
	Name      string `json:"name"`
	TLSVerify *bool  `json:"tls-verify,omitempty"`

	ImageID string `json:"image_id"`
	Digest  string `json:"digest"`
}

type ContainerResolveJob struct {
	Arch  string          `json:"arch"`
	Specs []ContainerSpec `json:"specs"`
}

type ContainerResolveJobResult struct {
	Specs []ContainerSpec `json:"specs"`

	JobResult
}

type OSTreeResolveSpec struct {
	URL    string `json:"url"`
	Ref    string `json:"ref"`
	Parent string `json:"parent"`
	RHSM   bool   `json:"rhsm"`
}

type OSTreeResolveJob struct {
	Specs []OSTreeResolveSpec `json:"ostree_resolve_specs"`
}

type OSTreeResolveResultSpec struct {
	URL      string `json:"url"`
	Ref      string `json:"ref"`
	Checksum string `json:"checksum"`
	RHSM     bool   `json:"bool"`
}

type OSTreeResolveJobResult struct {
	Specs []OSTreeResolveResultSpec `json:"ostree_resolve_result_specs"`

	JobResult
}

type AWSEC2ShareJob struct {
	Ami               string   `json:"ami"`
	Region            string   `json:"region"`
	ShareWithAccounts []string `json:"shareWithAccounts"`
}

type AWSEC2ShareJobResult struct {
	JobResult

	Ami    string `json:"ami"`
	Region string `json:"region"`
}

type AWSEC2CopyJob struct {
	Ami          string `json:"ami"`
	SourceRegion string `json:"source_region"`
	TargetRegion string `json:"target_region"`
	TargetName   string `json:"target_name"`
}

type AWSEC2CopyJobResult struct {
	JobResult

	Ami    string `json:"ami"`
	Region string `json:"region"`
}

//
// JSON-serializable types for the client
//

type updateJobRequest struct {
	Result interface{} `json:"result"`
}

func (j *OSBuildJob) UnmarshalJSON(data []byte) error {
	// handles unmarshalling old jobs in the queue that don't contain newer fields
	// adds default/fallback values to missing data
	type aliasType OSBuildJob
	type compatType struct {
		aliasType
		// Deprecated: Exports should not be used. The export is set in the `Target.OsbuildExport`
		Exports []string `json:"export_stages,omitempty"`
	}
	var compat compatType
	if err := json.Unmarshal(data, &compat); err != nil {
		return err
	}
	if compat.PipelineNames == nil {
		compat.PipelineNames = &PipelineNames{
			Build:   distro.BuildPipelinesFallback(),
			Payload: distro.PayloadPipelinesFallback(),
		}
	}

	// Exports used to be specified in the job, but there could be always only a single export specified.
	if len(compat.Exports) != 0 {
		if len(compat.Exports) > 1 {
			return fmt.Errorf("osbuild job has more than one exports specified")
		}
		export := compat.Exports[0]
		// add the single export to each target
		for idx := range compat.Targets {
			target := compat.Targets[idx]
			if target.OsbuildArtifact.ExportName == "" {
				target.OsbuildArtifact.ExportName = export
			} else if target.OsbuildArtifact.ExportName != export {
				return fmt.Errorf("osbuild job has different global exports and export in the target specified at the same time")
			}
			compat.Targets[idx] = target
		}
	}

	*j = OSBuildJob(compat.aliasType)
	return nil
}

func (j OSBuildJob) MarshalJSON() ([]byte, error) {
	type aliasType OSBuildJob
	type compatType struct {
		aliasType
		// Depredated: Exports should not be used. The export is set in the `Target.OsbuildExport`
		Exports []string `json:"export_stages,omitempty"`
	}
	compat := compatType{
		aliasType: aliasType(j),
	}
	compat.Exports = j.OsbuildExports()

	data, err := json.Marshal(&compat)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func (j *OSBuildJobResult) UnmarshalJSON(data []byte) error {
	// handles unmarshalling old jobs in the queue that don't contain newer fields
	// adds default/fallback values to missing data
	type aliastype OSBuildJobResult
	var alias aliastype
	if err := json.Unmarshal(data, &alias); err != nil {
		return err
	}
	if alias.PipelineNames == nil {
		alias.PipelineNames = &PipelineNames{
			Build:   distro.BuildPipelinesFallback(),
			Payload: distro.PayloadPipelinesFallback(),
		}
	}
	*j = OSBuildJobResult(alias)
	return nil
}