worker/server: add JobDependencyChainErrors() method

Add new `JobDependencyChainErrors()` method for gathering a stack trace
of job errors from the job's dependencies which caused it to fail.

The `JobDependencyChainErrors()` implementation uses job-type specific
`...Status()` methods intentionally, because job-type specific status
methods check the job's result in a slightly different way and set
the result.JobError to a specific value. Due to this reason, it would
not be practical to introduce a generic `JobStatus()` method and get rid
of the `switch` block, because in reality, the new method would have
to implement an equivalent `switch` block as well.

Add unit test covering the method functionality.
This commit is contained in:
Tomas Hozza 2022-05-24 13:59:16 +02:00 committed by Tom Gundersen
parent 5bd02f2f27
commit fa37005a32
2 changed files with 790 additions and 0 deletions

View file

@ -174,6 +174,94 @@ func (s *Server) CheckBuildDependencies(dep uuid.UUID, jobErr *clienterrors.Erro
return nil
}
// DependencyChainErrors recursively gathers all errors from job's dependencies,
// which caused it to fail. If the job didn't fail, `nil` is returned.
func (s *Server) JobDependencyChainErrors(id uuid.UUID) (*clienterrors.Error, error) {
jobType, err := s.JobType(id)
if err != nil {
return nil, err
}
var jobResult *JobResult
var jobDeps []uuid.UUID
switch jobType {
case JobTypeOSBuild:
var osbuildJR OSBuildJobResult
_, jobDeps, err = s.OSBuildJobStatus(id, &osbuildJR)
if err != nil {
return nil, err
}
jobResult = &osbuildJR.JobResult
case JobTypeDepsolve:
var depsolveJR DepsolveJobResult
_, jobDeps, err = s.DepsolveJobStatus(id, &depsolveJR)
if err != nil {
return nil, err
}
jobResult = &depsolveJR.JobResult
case JobTypeManifestIDOnly:
var manifestJR ManifestJobByIDResult
_, jobDeps, err = s.ManifestJobStatus(id, &manifestJR)
if err != nil {
return nil, err
}
jobResult = &manifestJR.JobResult
case JobTypeKojiInit:
var kojiInitJR KojiInitJobResult
_, jobDeps, err = s.KojiInitJobStatus(id, &kojiInitJR)
if err != nil {
return nil, err
}
jobResult = &kojiInitJR.JobResult
case JobTypeOSBuildKoji:
var osbuildKojiJR OSBuildKojiJobResult
_, jobDeps, err = s.OSBuildKojiJobStatus(id, &osbuildKojiJR)
if err != nil {
return nil, err
}
jobResult = &osbuildKojiJR.JobResult
case JobTypeKojiFinalize:
var kojiFinalizeJR KojiFinalizeJobResult
_, jobDeps, err = s.KojiFinalizeJobStatus(id, &kojiFinalizeJR)
if err != nil {
return nil, err
}
jobResult = &kojiFinalizeJR.JobResult
default:
return nil, fmt.Errorf("unexpected job type: %s", jobType)
}
if jobError := jobResult.JobError; jobError != nil {
depErrors := []*clienterrors.Error{}
if jobError.IsDependencyError() {
// check job's dependencies
for _, dep := range jobDeps {
depError, err := s.JobDependencyChainErrors(dep)
if err != nil {
return nil, err
}
if depError != nil {
depErrors = append(depErrors, depError)
}
}
}
if len(depErrors) > 0 {
jobError.Details = depErrors
}
return jobError, nil
}
return nil, nil
}
func (s *Server) OSBuildJobStatus(id uuid.UUID, result *OSBuildJobResult) (*JobStatus, []uuid.UUID, error) {
jobType, _, status, deps, err := s.jobStatus(id, result)
if err != nil {

View file

@ -1014,3 +1014,705 @@ func TestDepsolveJobArgsCompat(t *testing.T) {
assert.Equal(newJob, newJobW)
}
}
type testJob struct {
main interface{}
deps []testJob
result interface{}
}
func enqueueAndFinishTestJobDependencies(s *worker.Server, deps []testJob) ([]uuid.UUID, error) {
ids := []uuid.UUID{}
for _, dep := range deps {
var depUUIDs []uuid.UUID
var err error
if len(dep.deps) > 0 {
depUUIDs, err = enqueueAndFinishTestJobDependencies(s, dep.deps)
if err != nil {
return nil, err
}
}
var id uuid.UUID
switch dep.main.(type) {
case *worker.OSBuildJob:
job := dep.main.(*worker.OSBuildJob)
id, err = s.EnqueueOSBuildAsDependency(distro.X86_64ArchName, job, depUUIDs, "")
if err != nil {
return nil, err
}
case *worker.ManifestJobByID:
job := dep.main.(*worker.ManifestJobByID)
if len(depUUIDs) != 1 {
return nil, fmt.Errorf("exactly one dependency is expected for ManifestJobByID, got: %d", len(depUUIDs))
}
id, err = s.EnqueueManifestJobByID(job, depUUIDs[0], "")
if err != nil {
return nil, err
}
case *worker.DepsolveJob:
job := dep.main.(*worker.DepsolveJob)
if len(depUUIDs) != 0 {
return nil, fmt.Errorf("dependencies are not supported for DepsolveJob, got: %d", len(depUUIDs))
}
id, err = s.EnqueueDepsolve(job, "")
if err != nil {
return nil, err
}
case *worker.KojiInitJob:
job := dep.main.(*worker.KojiInitJob)
if len(depUUIDs) != 0 {
return nil, fmt.Errorf("dependencies are not supported for KojiInitJob, got: %d", len(depUUIDs))
}
id, err = s.EnqueueKojiInit(job, "")
if err != nil {
return nil, err
}
case *worker.OSBuildKojiJob:
job := dep.main.(*worker.OSBuildKojiJob)
if len(depUUIDs) != 2 {
return nil, fmt.Errorf("exactly two dependency is expected for OSBuildKojiJob, got: %d", len(depUUIDs))
}
id, err = s.EnqueueOSBuildKojiAsDependency(distro.X86_64ArchName, job, depUUIDs[1], depUUIDs[0], "")
if err != nil {
return nil, err
}
case *worker.KojiFinalizeJob:
job := dep.main.(*worker.KojiFinalizeJob)
if len(depUUIDs) < 2 {
return nil, fmt.Errorf("at least two dependencies are expected for KojiFinalizeJob, got: %d", len(depUUIDs))
}
id, err = s.EnqueueKojiFinalize(job, depUUIDs[0], depUUIDs[1:], "")
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unexpected job type")
}
// request the previously added Job
_, token, _, _, _, err := s.RequestJobById(context.Background(), distro.X86_64ArchName, id)
if err != nil {
return nil, err
}
result, err := json.Marshal(dep.result)
if err != nil {
return nil, err
}
// mark the job as finished using the defined job result
err = s.FinishJob(token, result)
if err != nil {
return nil, err
}
ids = append(ids, id)
}
return ids, nil
}
func TestJobDependencyChainErrors(t *testing.T) {
var cases = []struct {
job testJob
expectedError *clienterrors.Error
}{
// osbuild + manifest + depsolve
// failed depsolve
{
job: testJob{
main: &worker.OSBuildJob{},
deps: []testJob{
{
main: &worker.ManifestJobByID{},
deps: []testJob{
{
main: &worker.DepsolveJob{},
result: &worker.DepsolveJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorDNFDepsolveError,
Reason: "package X not found",
},
},
},
},
},
result: &worker.ManifestJobByIDResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorDepsolveDependency,
Reason: "depsolve dependency job failed",
},
},
},
},
},
result: &worker.OSBuildJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorManifestDependency,
Reason: "manifest dependency job failed",
},
},
},
},
expectedError: &clienterrors.Error{
ID: clienterrors.ErrorManifestDependency,
Reason: "manifest dependency job failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorDepsolveDependency,
Reason: "depsolve dependency job failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorDNFDepsolveError,
Reason: "package X not found",
},
},
},
},
},
},
// osbuild + manifest + depsolve
// failed manifest
{
job: testJob{
main: &worker.OSBuildJob{},
deps: []testJob{
{
main: &worker.ManifestJobByID{},
deps: []testJob{
{
main: &worker.DepsolveJob{},
result: &worker.DepsolveJobResult{},
},
},
result: &worker.ManifestJobByIDResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorManifestGeneration,
Reason: "failed to generate manifest",
},
},
},
},
},
result: &worker.OSBuildJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorManifestDependency,
Reason: "manifest dependency job failed",
},
},
},
},
expectedError: &clienterrors.Error{
ID: clienterrors.ErrorManifestDependency,
Reason: "manifest dependency job failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorManifestGeneration,
Reason: "failed to generate manifest",
},
},
},
},
// osbuild + manifest + depsolve
// failed osbuild
{
job: testJob{
main: &worker.OSBuildJob{},
deps: []testJob{
{
main: &worker.ManifestJobByID{},
deps: []testJob{
{
main: &worker.DepsolveJob{},
result: &worker.DepsolveJobResult{},
},
},
result: &worker.ManifestJobByIDResult{
JobResult: worker.JobResult{},
},
},
},
result: &worker.OSBuildJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorEmptyManifest,
Reason: "empty manifest received",
},
},
},
},
expectedError: &clienterrors.Error{
ID: clienterrors.ErrorEmptyManifest,
Reason: "empty manifest received",
},
},
// koji-init + osbuild-koji + manifest + depsolve
// failed depsolve
{
job: testJob{
main: &worker.OSBuildKojiJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
{
main: &worker.ManifestJobByID{},
deps: []testJob{
{
main: &worker.DepsolveJob{},
result: &worker.DepsolveJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorDNFDepsolveError,
Reason: "package X not found",
},
},
},
},
},
result: &worker.ManifestJobByIDResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorDepsolveDependency,
Reason: "depsolve dependency job failed",
},
},
},
},
},
result: &worker.OSBuildKojiJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorManifestDependency,
Reason: "manifest dependency job failed",
},
},
},
},
expectedError: &clienterrors.Error{
ID: clienterrors.ErrorManifestDependency,
Reason: "manifest dependency job failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorDepsolveDependency,
Reason: "depsolve dependency job failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorDNFDepsolveError,
Reason: "package X not found",
},
},
},
},
},
},
// koji-init + (osbuild-koji + manifest + depsolve) + (osbuild-koji + manifest + depsolve) + koji-finalize
// failed one depsolve
{
job: testJob{
main: &worker.KojiFinalizeJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
// failed build
{
main: &worker.OSBuildKojiJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
{
main: &worker.ManifestJobByID{},
deps: []testJob{
{
main: &worker.DepsolveJob{},
result: &worker.DepsolveJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorDNFDepsolveError,
Reason: "package X not found",
},
},
},
},
},
result: &worker.ManifestJobByIDResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorDepsolveDependency,
Reason: "depsolve dependency job failed",
},
},
},
},
},
result: &worker.OSBuildKojiJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorManifestDependency,
Reason: "manifest dependency job failed",
},
},
},
},
// passed build
{
main: &worker.OSBuildKojiJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
{
main: &worker.ManifestJobByID{},
deps: []testJob{
{
main: &worker.DepsolveJob{},
result: &worker.DepsolveJobResult{},
},
},
result: &worker.ManifestJobByIDResult{},
},
},
result: &worker.OSBuildKojiJobResult{
OSBuildOutput: &osbuild2.Result{},
},
},
},
result: &worker.KojiFinalizeJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorKojiFailedDependency,
Reason: "one build failed",
},
},
},
},
expectedError: &clienterrors.Error{
ID: clienterrors.ErrorKojiFailedDependency,
Reason: "one build failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorManifestDependency,
Reason: "manifest dependency job failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorDepsolveDependency,
Reason: "depsolve dependency job failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorDNFDepsolveError,
Reason: "package X not found",
},
},
},
},
},
},
},
},
// koji-init + (osbuild-koji + manifest + depsolve) + (osbuild-koji + manifest + depsolve) + koji-finalize
// failed both depsolve
{
job: testJob{
main: &worker.KojiFinalizeJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
// failed build
{
main: &worker.OSBuildKojiJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
{
main: &worker.ManifestJobByID{},
deps: []testJob{
{
main: &worker.DepsolveJob{},
result: &worker.DepsolveJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorDNFDepsolveError,
Reason: "package X not found",
},
},
},
},
},
result: &worker.ManifestJobByIDResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorDepsolveDependency,
Reason: "depsolve dependency job failed",
},
},
},
},
},
result: &worker.OSBuildKojiJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorManifestDependency,
Reason: "manifest dependency job failed",
},
},
},
},
// failed build
{
main: &worker.OSBuildKojiJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
{
main: &worker.ManifestJobByID{},
deps: []testJob{
{
main: &worker.DepsolveJob{},
result: &worker.DepsolveJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorDNFDepsolveError,
Reason: "package Y not found",
},
},
},
},
},
result: &worker.ManifestJobByIDResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorDepsolveDependency,
Reason: "depsolve dependency job failed",
},
},
},
},
},
result: &worker.OSBuildKojiJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorManifestDependency,
Reason: "manifest dependency job failed",
},
},
},
},
},
result: &worker.KojiFinalizeJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorKojiFailedDependency,
Reason: "two builds failed",
},
},
},
},
expectedError: &clienterrors.Error{
ID: clienterrors.ErrorKojiFailedDependency,
Reason: "two builds failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorManifestDependency,
Reason: "manifest dependency job failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorDepsolveDependency,
Reason: "depsolve dependency job failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorDNFDepsolveError,
Reason: "package X not found",
},
},
},
},
},
{
ID: clienterrors.ErrorManifestDependency,
Reason: "manifest dependency job failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorDepsolveDependency,
Reason: "depsolve dependency job failed",
Details: []*clienterrors.Error{
{
ID: clienterrors.ErrorDNFDepsolveError,
Reason: "package Y not found",
},
},
},
},
},
},
},
},
// koji-init + (osbuild-koji + manifest + depsolve) + (osbuild-koji + manifest + depsolve) + koji-finalize
// failed koji-finalize
{
job: testJob{
main: &worker.KojiFinalizeJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
// passed build
{
main: &worker.OSBuildKojiJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
{
main: &worker.ManifestJobByID{},
deps: []testJob{
{
main: &worker.DepsolveJob{},
result: &worker.DepsolveJobResult{},
},
},
result: &worker.ManifestJobByIDResult{},
},
},
result: &worker.OSBuildKojiJobResult{
OSBuildOutput: &osbuild2.Result{},
},
},
// passed build
{
main: &worker.OSBuildKojiJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
{
main: &worker.ManifestJobByID{},
deps: []testJob{
{
main: &worker.DepsolveJob{},
result: &worker.DepsolveJobResult{},
},
},
result: &worker.ManifestJobByIDResult{},
},
},
result: &worker.OSBuildKojiJobResult{
OSBuildOutput: &osbuild2.Result{},
},
},
},
result: &worker.KojiFinalizeJobResult{
JobResult: worker.JobResult{
JobError: &clienterrors.Error{
ID: clienterrors.ErrorKojiFinalize,
Reason: "koji-finalize failed",
},
},
},
},
expectedError: &clienterrors.Error{
ID: clienterrors.ErrorKojiFinalize,
Reason: "koji-finalize failed",
},
},
// koji-init + (osbuild-koji + manifest + depsolve) + (osbuild-koji + manifest + depsolve) + koji-finalize
// all passed
{
job: testJob{
main: &worker.KojiFinalizeJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
// passed build
{
main: &worker.OSBuildKojiJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
{
main: &worker.ManifestJobByID{},
deps: []testJob{
{
main: &worker.DepsolveJob{},
result: &worker.DepsolveJobResult{},
},
},
result: &worker.ManifestJobByIDResult{},
},
},
result: &worker.OSBuildKojiJobResult{
OSBuildOutput: &osbuild2.Result{},
},
},
// passed build
{
main: &worker.OSBuildKojiJob{},
deps: []testJob{
{
main: &worker.KojiInitJob{},
result: &worker.KojiInitJobResult{},
},
{
main: &worker.ManifestJobByID{},
deps: []testJob{
{
main: &worker.DepsolveJob{},
result: &worker.DepsolveJobResult{},
},
},
result: &worker.ManifestJobByIDResult{},
},
},
result: &worker.OSBuildKojiJobResult{
OSBuildOutput: &osbuild2.Result{},
},
},
},
result: &worker.KojiFinalizeJobResult{
JobResult: worker.JobResult{},
},
},
expectedError: nil,
},
}
for idx, c := range cases {
t.Logf("Test case #%d", idx)
server := newTestServer(t, t.TempDir(), time.Duration(0), "/api/worker/v1")
ids, err := enqueueAndFinishTestJobDependencies(server, []testJob{c.job})
require.Nil(t, err)
require.Len(t, ids, 1)
mainJobID := ids[0]
errors, err := server.JobDependencyChainErrors(mainJobID)
require.Nil(t, err)
assert.EqualValues(t, c.expectedError, errors)
}
}