internal/worker/server: return an error on depsolve timeout HMS-2989

Fixes the special case that if no worker is available and we
generate an internal timeout and cancel the depsolve including all
followup jobs, no error was propagated.

internal/jobqueue/fsjobqueue/fsjobqueue.go

@@ -403,6 +403,40 @@ func (q *fsJobQueue) CancelJob(id uuid.UUID) error {
 	return nil
 }
 
+func (q *fsJobQueue) FailJob(id uuid.UUID, result interface{}) error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	j, err := q.readJob(id)
+	if err != nil {
+		return err
+	}
+
+	if !j.FinishedAt.IsZero() {
+		return jobqueue.ErrFinished
+	}
+
+	if !j.StartedAt.IsZero() {
+		return jobqueue.ErrRunning
+	}
+
+	j.Result, err = json.Marshal(result)
+	if err != nil {
+		return err
+	}
+
+	j.StartedAt = time.Now()
+	j.FinishedAt = time.Now()
+	j.Token = uuid.New()
+
+	err = q.db.Write(id.String(), j)
+	if err != nil {
+		return fmt.Errorf("error writing job %s: %v", id, err)
+	}
+
+	return nil
+}
+
 func (q *fsJobQueue) JobStatus(id uuid.UUID) (jobType string, channel string, result json.RawMessage, queued, started, finished time.Time, canceled bool, deps []uuid.UUID, dependents []uuid.UUID, err error) {
 	j, err := q.readJob(id)
 	if err != nil {

internal/jobqueue/jobqueuetest/jobqueuetest.go

@@ -7,6 +7,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/osbuild/osbuild-composer/internal/worker"
+	"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
 	"os"
 	"sync"
 	"testing"
@@ -55,6 +57,7 @@ func TestJobQueue(t *testing.T, makeJobQueue MakeJobQueue) {
 	t.Run("multiple-channels", wrap(testMultipleChannels))
 	t.Run("100-dequeuers", wrap(test100dequeuers))
 	t.Run("workers", wrap(testWorkers))
+	t.Run("fail", wrap(testFail))
 }
 
 func pushTestJob(t *testing.T, q jobqueue.JobQueue, jobType string, args interface{}, dependencies []uuid.UUID, channel string) uuid.UUID {
@@ -762,3 +765,56 @@ func testWorkers(t *testing.T, q jobqueue.JobQueue) {
 	err = q.DeleteWorker(w2)
 	require.NoError(t, err)
 }
+
+func testFail(t *testing.T, q jobqueue.JobQueue) {
+	startTime := time.Now()
+
+	FailedJobErrorResult := worker.JobResult{
+		JobError: clienterrors.New(clienterrors.ErrorDepsolveTimeout,
+			"Test timeout reason",
+			"Test timeout details",
+		),
+	}
+
+	testReason, err := json.Marshal(FailedJobErrorResult)
+	require.NoError(t, err)
+
+	// set a non-existing job to failed
+	err = q.FailJob(uuid.New(), testReason)
+	require.Error(t, err)
+
+	// Cancel a pending job
+	id := pushTestJob(t, q, "coralreef", nil, nil, "testchannel")
+	require.NotEmpty(t, id)
+
+	err = q.FailJob(id, testReason)
+	require.NoError(t, err)
+
+	//nolint:golint,ineffassign
+	jobType, channel, result, queued, started, finished, canceled, _, _, err := q.JobStatus(id)
+	require.NoError(t, err)
+
+	endTime := time.Now()
+	type JobResult struct {
+		JobError *clienterrors.Error `json:"job_error"`
+	}
+	var r1 JobResult
+	err = json.Unmarshal(result, &r1)
+	require.NoError(t, err)
+
+	require.NotNil(t, r1)
+	require.Equal(t, "Test timeout reason", r1.JobError.Reason)
+	require.Equal(t, "Test timeout details", r1.JobError.Details)
+	require.Equal(t, clienterrors.ErrorDepsolveTimeout, r1.JobError.ID)
+	require.Equal(t, "testchannel", channel)
+	require.Equal(t, "coralreef", jobType)
+	require.Equal(t, false, canceled)
+
+	allTimings := []time.Time{queued, started, finished}
+
+	for _, tmr := range allTimings {
+		require.Less(t, startTime, tmr)
+		require.Greater(t, endTime, tmr)
+	}
+
+}