osbuild-worker: attach pipeline names to jobs

Pipeline names are added to each job before adding to the queue. When a
job is finished, the names are copied to the Result object as well. This
is done for both OSBuild and Koji jobs.

The pipeline names in the result are primarily used to separate package
lists into build and payload/image packages in two cases:
1. Koji builds: for reporting the build root and image package lists to
   Koji (in Koji finalize).
2. Cloud API (v1 and v2): for reporting the payload packages in the
   metadata request.

The pipeline names are also used to print the system log output in the
order in which pipelines are executed. This still isn't used when
printing the OSBuild Result (osbuild2.Result.Write()) and we still rely
on sorting by pipeline name
(see https://github.com/osbuild/osbuild-composer/pull/1330).

cmd/osbuild-worker/jobimpl-koji-finalize.go

@@ -8,7 +8,6 @@ import (
 	"net/url"
 	"time"
 
-	osbuild "github.com/osbuild/osbuild-composer/internal/osbuild2"
 	"github.com/osbuild/osbuild-composer/internal/rpmmd"
 	"github.com/osbuild/osbuild-composer/internal/upload/koji"
 	"github.com/osbuild/osbuild-composer/internal/worker"
@@ -133,10 +132,14 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error {
 	var buildRoots []koji.BuildRoot
 	var images []koji.Image
 	for i, buildArgs := range osbuildKojiResults {
-		buildPipelineMd := buildArgs.OSBuildOutput.Metadata["build"]
-		buildRPMs := rpmmd.OSBuildMetadataToRPMs(buildPipelineMd)
+		buildRPMs := make([]rpmmd.RPM, 0)
+		// collect packages from stages in build pipelines
+		for _, plName := range buildArgs.PipelineNames.Build {
+			buildPipelineMd := buildArgs.OSBuildOutput.Metadata[plName]
+			buildRPMs = append(buildRPMs, rpmmd.OSBuildMetadataToRPMs(buildPipelineMd)...)
+		}
 		// this dedupe is usually not necessary since we generally only have
-		// one rpm stage in the build pipeline, but it's not invalid to have
+		// one rpm stage in one build pipeline, but it's not invalid to have
 		// multiple
 		buildRPMs = rpmmd.DeduplicateRPMs(buildRPMs)
 		buildRoots = append(buildRoots, koji.BuildRoot{
@@ -157,21 +160,13 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error {
 			RPMs:  buildRPMs,
 		})
 
-		// collect metadata from all other pipelines
-		// use pipeline name + stage name as key while collecting since all RPM
-		// stage metadata will have the same key within a single pipeline
-		imagePipelinesMd := make(map[string]osbuild.StageMetadata)
-		for pipelineName, pipelineMetadata := range buildArgs.OSBuildOutput.Metadata {
-			if pipelineName == "build" {
-				continue
-			}
-			for stageName, stageMetadata := range pipelineMetadata {
-				imagePipelinesMd[pipelineName+":"+stageName] = stageMetadata
-			}
+		// collect packages from stages in payload pipelines
+		imageRPMs := make([]rpmmd.RPM, 0)
+		for _, plName := range buildArgs.PipelineNames.Payload {
+			payloadPipelineMd := buildArgs.OSBuildOutput.Metadata[plName]
+			imageRPMs = append(imageRPMs, rpmmd.OSBuildMetadataToRPMs(payloadPipelineMd)...)
 		}
 
-		// collect packages from all stages
-		imageRPMs := rpmmd.OSBuildMetadataToRPMs(imagePipelinesMd)
 		// deduplicate
 		imageRPMs = rpmmd.DeduplicateRPMs(imageRPMs)
 

cmd/osbuild-worker/jobimpl-osbuild-koji.go

@@ -116,6 +116,9 @@ func (impl *OSBuildKojiJobImpl) Run(job worker.Job) error {
 		}
 	}
 
+	// copy pipeline info to the result
+	result.PipelineNames = args.PipelineNames
+
 	err = job.Update(&result)
 	if err != nil {
 		return fmt.Errorf("Error reporting job result: %v", err)

cmd/osbuild-worker/jobimpl-osbuild.go

@@ -94,6 +94,8 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error {
 		}
 		args.Manifest = manifestJR.Manifest
 	}
+	// copy pipeline info to the result
+	osbuildJobResult.PipelineNames = args.PipelineNames
 
 	// The specification allows multiple upload targets because it is an array, but we don't support it.
 	// Return an error to osbuild-composer.
@@ -121,7 +123,13 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error {
 	}
 
 	// Include pipeline stages output inside the worker's logs.
-	for pipelineName, pipelineLog := range osbuildJobResult.OSBuildOutput.Log {
+	// Order pipelines based on PipelineNames from job
+	for _, pipelineName := range osbuildJobResult.PipelineNames.All() {
+		pipelineLog, hasLog := osbuildJobResult.OSBuildOutput.Log[pipelineName]
+		if !hasLog {
+			// no pipeline output
+			continue
+		}
 		log.Printf("%s pipeline results:\n", pipelineName)
 		for _, stageResult := range pipelineLog {
 			if stageResult.Success {