debian-forge-composer/internal/worker/json.go

package worker

import (
	"encoding/json"
	"fmt"
	"runtime/debug"

	"github.com/osbuild/images/pkg/distro"
	"github.com/osbuild/images/pkg/manifest"
	"github.com/osbuild/images/pkg/osbuild"
	"github.com/osbuild/images/pkg/rpmmd"
	"github.com/osbuild/osbuild-composer/internal/target"
	"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
	"golang.org/x/exp/slices"
)

//
// JSON-serializable types for the jobqueue
//

type OSBuildJob struct {
	Manifest manifest.OSBuildManifest `json:"manifest,omitempty"`
	// Index of the ManifestJobByIDResult instance in the job's dynamic arguments slice
	ManifestDynArgsIdx *int             `json:"manifest_dyn_args_idx,omitempty"`
	Targets            []*target.Target `json:"targets,omitempty"`
	PipelineNames      *PipelineNames   `json:"pipeline_names,omitempty"`
	// The ImageBootMode is just copied to the result by the worker, so that
	// the value can be accessed job which depend on it.
	// (string representation of distro.BootMode values)
	ImageBootMode string `json:"image_boot_mode,omitempty"`
}

// OsbuildExports returns a slice of osbuild pipeline names, which should be
// exported as part of running osbuild image build for the job. The pipeline
// names are gathered from the targets specified in the job.
func (j OSBuildJob) OsbuildExports() []string {
	exports := []string{}
	seenExports := map[string]bool{}
	for _, target := range j.Targets {
		exists := seenExports[target.OsbuildArtifact.ExportName]
		if !exists {
			seenExports[target.OsbuildArtifact.ExportName] = true
			exports = append(exports, target.OsbuildArtifact.ExportName)
		}
	}
	return exports
}

type JobResult struct {
	JobError *clienterrors.Error `json:"job_error,omitempty"`
}

type OSBuildJobResult struct {
	Success       bool                   `json:"success"`
	OSBuildOutput *osbuild.Result        `json:"osbuild_output,omitempty"`
	TargetResults []*target.TargetResult `json:"target_results,omitempty"`
	UploadStatus  string                 `json:"upload_status"`
	PipelineNames *PipelineNames         `json:"pipeline_names,omitempty"`
	// Host OS of the worker which handled the job
	HostOS string `json:"host_os"`
	// Architecture of the worker which handled the job
	Arch string `json:"arch"`
	// Boot mode supported by the image
	// (string representation of distro.BootMode values)
	ImageBootMode string `json:"image_boot_mode,omitempty"`
	JobResult
}

// TargetErrors returns a slice of *clienterrors.Error gathered
// from the job result's target results. If there were no target errors
// then the returned slice will be empty.
func (j *OSBuildJobResult) TargetErrors() []*clienterrors.Error {
	targetErrors := []*clienterrors.Error{}

	for _, targetResult := range j.TargetResults {
		if targetResult.TargetError != nil {
			targetError := targetResult.TargetError
			// Add the target name to the error details, because the error reason
			// may not contain any information to determine the type of the target
			// which failed.
			targetErrors = append(targetErrors, clienterrors.WorkerClientError(targetError.ID, targetError.Reason, targetResult.Name))
		}
	}

	return targetErrors
}

// TargetResultsByName iterates over TargetResults attached to the Job result and
// returns a slice of Target results of the provided name (type). If there were no
// TargetResults of the desired type attached to the Job results, the returned
// slice will be empty.
func (j *OSBuildJobResult) TargetResultsByName(name target.TargetName) []*target.TargetResult {
	targetResults := []*target.TargetResult{}
	for _, targetResult := range j.TargetResults {
		if targetResult.Name == name {
			targetResults = append(targetResults, targetResult)
		}
	}
	return targetResults
}

// TargetResultsFilterByName iterates over TargetResults attached to the Job result and
// returns a slice of Target results excluding the provided names (types). If there were
// no TargetResults left after filtering, the returned slice will be empty.
func (j *OSBuildJobResult) TargetResultsFilterByName(excludeNames []target.TargetName) []*target.TargetResult {
	targetResults := []*target.TargetResult{}
	for _, targetResult := range j.TargetResults {
		if !slices.Contains(excludeNames, targetResult.Name) {
			targetResults = append(targetResults, targetResult)
		}
	}
	return targetResults
}

func (j *FileResolveJobResult) ResolutionErrors() []*clienterrors.Error {
	resolutionErrors := []*clienterrors.Error{}

	for _, result := range j.Results {
		if result.ResolutionError != nil {
			resolutionErrors = append(resolutionErrors, result.ResolutionError)
		}
	}

	return resolutionErrors
}

type KojiInitJob struct {
	Server  string `json:"server"`
	Name    string `json:"name"`
	Version string `json:"version"`
	Release string `json:"release"`
}

type KojiInitJobResult struct {
	BuildID   uint64 `json:"build_id"`
	Token     string `json:"token"`
	KojiError string `json:"koji_error"`
	JobResult
}

type KojiFinalizeJob struct {
	Server  string `json:"server"`
	Name    string `json:"name"`
	Version string `json:"version"`
	Release string `json:"release"`
	// TODO: eventually deprecate and remove KojiFilenames, since the image filenames are now set in the KojiTargetResultOptions.
	KojiFilenames []string `json:"koji_filenames"`
	KojiDirectory string   `json:"koji_directory"`
	TaskID        uint64   `json:"task_id"` /* https://pagure.io/koji/issue/215 */
	StartTime     uint64   `json:"start_time"`
}

type KojiFinalizeJobResult struct {
	KojiError string `json:"koji_error"`
	JobResult
}

// PipelineNames is used to provide two pieces of information related to a job:
// 1. A categorization of each pipeline into one of two groups
// // 2. A pipeline ordering when the lists are concatenated: build -> os
// 2. A pipeline ordering when the lists are concatenated: build -> os
type PipelineNames struct {
	Build   []string `json:"build"`
	Payload []string `json:"payload"`
}

// Returns a concatenated list of the pipeline names
func (pn *PipelineNames) All() []string {
	return append(pn.Build, pn.Payload...)
}

// DepsolveJob defines the parameters of one or more depsolve jobs.  Each named
// list of package sets defines a separate job.  Lists with multiple package
// sets are depsolved in a chain, combining the results of sequential depsolves
// into a single PackageSpec list in the result.  Each PackageSet defines the
// repositories it will be depsolved against.
type DepsolveJob struct {
	PackageSets      map[string][]rpmmd.PackageSet `json:"grouped_package_sets"`
	ModulePlatformID string                        `json:"module_platform_id"`
	Arch             string                        `json:"arch"`
	Releasever       string                        `json:"releasever"`
}

// Custom marshaller for keeping compatibility with older workers.  The
// serialised format encompasses both the old and new DepsolveJob formats.  It
// is meant to be temporarily used to transition from the old to the new
// format.
func (ds DepsolveJob) MarshalJSON() ([]byte, error) {
	// NOTE: Common, top level repositories aren't used because in the new
	// format they don't exist; putting all required repositories on all
	// package sets as PackageSetsRepos should produce the same behaviour so
	// there's no need to try and figure out the "common" ones.
	// This also makes it possible to use old workers for new image types that
	// are incompatible with having common repos for all package sets (RHEL 7.9).
	compatJob := struct {
		// new format
		GroupedPackageSets map[string][]rpmmd.PackageSet `json:"grouped_package_sets"`
		ModulePlatformID   string                        `json:"module_platform_id"`
		Arch               string                        `json:"arch"`
		Releasever         string                        `json:"releasever"`

		// old format elements
		PackageSetsChains map[string][]string           `json:"package_sets_chains"`
		PackageSets       map[string]rpmmd.PackageSet   `json:"package_sets"`
		PackageSetsRepos  map[string][]rpmmd.RepoConfig `json:"package_sets_repositories,omitempty"`
	}{
		// new format substruct
		GroupedPackageSets: ds.PackageSets,
		ModulePlatformID:   ds.ModulePlatformID,
		Arch:               ds.Arch,
		Releasever:         ds.Releasever,
	}

	// build equivalent old format substruct
	pkgSetRepos := make(map[string][]rpmmd.RepoConfig)
	pkgSets := make(map[string]rpmmd.PackageSet)
	chains := make(map[string][]string)
	for chainName, pkgSetChain := range ds.PackageSets {
		if len(pkgSetChain) == 1 {
			// single element "chain" (i.e., not a chain)
			pkgSets[chainName] = rpmmd.PackageSet{
				Include: pkgSetChain[0].Include,
				Exclude: pkgSetChain[0].Exclude,
			}
			pkgSetRepos[chainName] = pkgSetChain[0].Repositories
			continue
		}
		chain := make([]string, len(pkgSetChain))
		for idx, set := range pkgSetChain {
			// the names of the individual sets in the chain don't matter, as long
			// as they match the keys for the repo configs
			setName := fmt.Sprintf("%s-%d", chainName, idx)
			// the package set (without repos)
			pkgSets[setName] = rpmmd.PackageSet{
				Include: set.Include,
				Exclude: set.Exclude,
			}
			// set repositories
			pkgSetRepos[setName] = set.Repositories
			// add name to the chain
			chain[idx] = setName
		}
		chains[chainName] = chain
	}

	compatJob.PackageSets = pkgSets
	compatJob.PackageSetsChains = chains
	compatJob.PackageSetsRepos = pkgSetRepos

	return json.Marshal(compatJob)
}

type ErrorType string

const (
	DepsolveErrorType ErrorType = "depsolve"
	OtherErrorType    ErrorType = "other"
)

type DepsolveJobResult struct {
	PackageSpecs map[string][]rpmmd.PackageSpec `json:"package_specs"`
	Error        string                         `json:"error"`
	ErrorType    ErrorType                      `json:"error_type"`
	JobResult
}

type ManifestJobByID struct{}

// OSBuildComposerDepModule contains information about a module used by
// osbuild-composer which could affect the manifest content.
type OSBuildComposerDepModule struct {
	Path    string                    `json:"path"`
	Version string                    `json:"version"`
	Replace *OSBuildComposerDepModule `json:"replace,omitempty"`
}

// ComposerDepModuleFromDebugModule converts a debug.Module instance
// to an OSBuildComposerDepModule instance.
func ComposerDepModuleFromDebugModule(module *debug.Module) *OSBuildComposerDepModule {
	if module == nil {
		return nil
	}
	depModule := &OSBuildComposerDepModule{
		Path:    module.Path,
		Version: module.Version,
	}
	if module.Replace != nil {
		depModule.Replace = &OSBuildComposerDepModule{
			Path:    module.Replace.Path,
			Version: module.Replace.Version,
		}
	}
	return depModule
}

// ManifestInfo contains information about the environment in which
// the manifest was produced and which could affect its content.
type ManifestInfo struct {
	OSBuildComposerVersion string `json:"osbuild_composer_version"`
	// List of relevant modules used by osbuild-composer which
	// could affect the manifest content.
	OSBuildComposerDeps []*OSBuildComposerDepModule `json:"osbuild_composer_deps,omitempty"`
}

type ManifestJobByIDResult struct {
	Manifest     manifest.OSBuildManifest `json:"data,omitempty"`
	ManifestInfo ManifestInfo             `json:"info,omitempty"`
	Error        string                   `json:"error"`
	JobResult
}

type ContainerSpec struct {
	Source    string `json:"source"`
	Name      string `json:"name"`
	TLSVerify *bool  `json:"tls-verify,omitempty"`

	ImageID    string `json:"image_id"`
	Digest     string `json:"digest"`
	ListDigest string `json:"list-digest,omitempty"`
}

type ContainerResolveJob struct {
	Arch  string          `json:"arch"`
	Specs []ContainerSpec `json:"specs"`
}

type ContainerResolveJobResult struct {
	Specs []ContainerSpec `json:"specs"`

	JobResult
}

type FileResolveJob struct {
	URLs []string `json:"urls"`
}

type FileResolveJobResultItem struct {
	URL             string              `json:"url"`
	Content         []byte              `json:"content"`
	ResolutionError *clienterrors.Error `json:"target_error,omitempty"`
}

type FileResolveJobResult struct {
	Success bool                       `json:"success"`
	Results []FileResolveJobResultItem `json:"results"`
	JobResult
}

type OSTreeResolveSpec struct {
	URL  string `json:"url"`
	Ref  string `json:"ref"`
	RHSM bool   `json:"rhsm"`
}

type OSTreeResolveJob struct {
	Specs []OSTreeResolveSpec `json:"ostree_resolve_specs"`
}

type OSTreeResolveResultSpec struct {
	URL      string `json:"url"`
	Ref      string `json:"ref"`
	Checksum string `json:"checksum"`
	RHSM     bool   `json:"bool"` // NOTE: kept for backwards compatibility; remove after a few releases
	Secrets  string `json:"secrets"`
}

type OSTreeResolveJobResult struct {
	Specs []OSTreeResolveResultSpec `json:"ostree_resolve_result_specs"`

	JobResult
}

type AWSEC2ShareJob struct {
	Ami               string   `json:"ami"`
	Region            string   `json:"region"`
	ShareWithAccounts []string `json:"shareWithAccounts"`
}

type AWSEC2ShareJobResult struct {
	JobResult

	Ami    string `json:"ami"`
	Region string `json:"region"`
}

type AWSEC2CopyJob struct {
	Ami          string `json:"ami"`
	SourceRegion string `json:"source_region"`
	TargetRegion string `json:"target_region"`
	TargetName   string `json:"target_name"`
}

type AWSEC2CopyJobResult struct {
	JobResult

	Ami    string `json:"ami"`
	Region string `json:"region"`
}

//
// JSON-serializable types for the client
//

type updateJobRequest struct {
	Result interface{} `json:"result"`
}

func (j *OSBuildJob) UnmarshalJSON(data []byte) error {
	// handles unmarshalling old jobs in the queue that don't contain newer fields
	// adds default/fallback values to missing data
	type aliasType OSBuildJob
	type compatType struct {
		aliasType
		// Deprecated: Exports should not be used. The export is set in the `Target.OsbuildExport`
		Exports []string `json:"export_stages,omitempty"`
	}
	var compat compatType
	if err := json.Unmarshal(data, &compat); err != nil {
		return err
	}
	if compat.PipelineNames == nil {
		compat.PipelineNames = &PipelineNames{
			Build:   distro.BuildPipelinesFallback(),
			Payload: distro.PayloadPipelinesFallback(),
		}
	}

	// Exports used to be specified in the job, but there could be always only a single export specified.
	if len(compat.Exports) != 0 {
		if len(compat.Exports) > 1 {
			return fmt.Errorf("osbuild job has more than one exports specified")
		}
		export := compat.Exports[0]
		// add the single export to each target
		for idx := range compat.Targets {
			target := compat.Targets[idx]
			if target.OsbuildArtifact.ExportName == "" {
				target.OsbuildArtifact.ExportName = export
			} else if target.OsbuildArtifact.ExportName != export {
				return fmt.Errorf("osbuild job has different global exports and export in the target specified at the same time")
			}
			compat.Targets[idx] = target
		}
	}

	*j = OSBuildJob(compat.aliasType)
	return nil
}

func (j OSBuildJob) MarshalJSON() ([]byte, error) {
	type aliasType OSBuildJob
	type compatType struct {
		aliasType
		// Depredated: Exports should not be used. The export is set in the `Target.OsbuildExport`
		Exports []string `json:"export_stages,omitempty"`
	}
	compat := compatType{
		aliasType: aliasType(j),
	}
	compat.Exports = j.OsbuildExports()

	data, err := json.Marshal(&compat)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func (j *OSBuildJobResult) UnmarshalJSON(data []byte) error {
	// handles unmarshalling old jobs in the queue that don't contain newer fields
	// adds default/fallback values to missing data
	type aliastype OSBuildJobResult
	var alias aliastype
	if err := json.Unmarshal(data, &alias); err != nil {
		return err
	}
	if alias.PipelineNames == nil {
		alias.PipelineNames = &PipelineNames{
			Build:   distro.BuildPipelinesFallback(),
			Payload: distro.PayloadPipelinesFallback(),
		}
	}
	*j = OSBuildJobResult(alias)
	return nil
}