fix: bound blocking calls and stop failing silently (#1174)

Jobs occasionally go silent ([example](https://gitea.com/gitea/runner/actions/runs/805045/jobs/1055123)) mid-run and Gitea reaped them after `ZOMBIE_TASK_TIMEOUT`, with no error in the log. This contains a number of related fixes, all with full test coverage:

1. Bound every RPC to Gitea with a timeout, a stalled report otherwise parked logs and heartbeats for the whole job.
2. Cap `runner.fetch_timeout` at that ceiling.
3. Let only the daemon loop close its own channel, the race panicked the process.
4. Stop the job on any terminal server result, not just `RESULT_CANCELLED`.
5. Report that result instead of relabelling it as cancelled.
6. Log reporting failures once at each end of an outage instead of discarding them.
7. Clamp the acknowledged log index, a too-large ack panicked on a slice bound.
8. Stop reading server health from a `FetchTask` deadline, it marked the runner healthy and reset the error backoff on a timeout.
9. Return an error from the Docker version probe instead of a `logrus` panic.
10. Pass the context to go-git's fetch and pull.
11. Fail the clone when a refresh dies on a cancelled context.
12. Set `terminationGracePeriodSeconds` in the Kubernetes examples.

Also contains a deprecation fix for goreleaser.

Reviewed-on: https://gitea.com/gitea/runner/pulls/1174
Reviewed-by: bircni <bircni@icloud.com>
Co-authored-by: silverwind <me@silverwind.io>

internal/app/cmd/daemon.go

@@ -158,6 +158,7 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
 			cfg.Runner.Insecure,
 			reg.UUID,
 			reg.Token,
+			config.RequestTimeout,
 		)
 
 		runner := run.NewRunner(cfg, reg, cli)

internal/app/cmd/register.go

@@ -366,6 +366,7 @@ func doRegister(ctx context.Context, cfg *config.Config, inputs *registerInputs)
 		cfg.Runner.Insecure,
 		"",
 		"",
+		config.RequestTimeout,
 	)
 
 	for {

internal/app/poll/poller.go

@@ -68,6 +68,8 @@ type Poller struct {
 type workerState struct {
 	consecutiveEmpty  int64
 	consecutiveErrors int64
+	// fetchTimedOut suppresses repeats of the fetch timeout warning.
+	fetchTimedOut bool
 	// lastBackoff is the last interval reported to the PollBackoffSeconds gauge;
 	// used to suppress redundant no-op Set calls when the backoff plateaus
 	// (e.g. at FetchIntervalMax).
@@ -346,16 +348,20 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
 		TasksVersion: v,
 	}))
 
-	// DeadlineExceeded is the designed idle path for a long-poll: the server
-	// found no work within FetchTimeout. Treat it as an empty response and do
-	// not record the duration — the timeout value would swamp the histogram.
+	// Our own deadline proves nothing either way: today Gitea answers immediately,
+	// so it means a slow server, and once it holds the request open it is an idle
+	// poll. Back off without claiming the server is healthy, warn once a streak so
+	// neither case floods, and keep it out of the latency histogram.
 	if errors.Is(err, context.DeadlineExceeded) {
-		p.markHealthyPoll()
+		if !s.fetchTimedOut {
+			s.fetchTimedOut = true
+			log.Warnf("fetching a task timed out after %s, raise runner.fetch_timeout if this persists", p.cfg.Runner.FetchTimeout)
+		}
 		s.consecutiveEmpty++
-		s.consecutiveErrors = 0 // timeout is a healthy idle response
 		metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultEmpty).Inc()
 		return nil, false
 	}
+	s.fetchTimedOut = false
 	metrics.PollFetchDuration.Observe(time.Since(start).Seconds())
 
 	if err != nil {
@@ -368,7 +374,13 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
 			p.shutdownPolling()
 			return nil, false
 		}
-		log.WithError(err).Error("failed to fetch task")
+		// Not a long poll, so a deadline can mean the server assigned a task
+		// this runner never received.
+		if errors.Is(err, context.DeadlineExceeded) {
+			log.WithError(err).Errorf("fetching a task timed out after %s", p.cfg.Runner.FetchTimeout)
+		} else {
+			log.WithError(err).Error("failed to fetch task")
+		}
 		p.lastPollFailed.Store(true)
 		s.consecutiveErrors++
 		metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultError).Inc()

internal/app/poll/poller_test.go

@@ -16,6 +16,7 @@ import (
 
 	connect_go "connectrpc.com/connect"
 	runnerv1 "gitea.dev/actionslib/runner/v1"
+	"github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
@@ -61,11 +62,7 @@ func TestPoller_WorkerStateCounters(t *testing.T) {
 // increments only the per-worker error counter, not the empty counter.
 func TestPoller_FetchErrorIncrementsErrorsOnly(t *testing.T) {
 	client := mocks.NewClient(t)
-	client.On("FetchTask", mock.Anything, mock.Anything).Return(
-		func(_ context.Context, _ *connect_go.Request[runnerv1.FetchTaskRequest]) (*connect_go.Response[runnerv1.FetchTaskResponse], error) {
-			return nil, errors.New("network unreachable")
-		},
-	)
+	client.On("FetchTask", mock.Anything, mock.Anything).Return(nil, errors.New("network unreachable"))
 
 	cfg, err := config.LoadDefault("")
 	require.NoError(t, err)
@@ -78,6 +75,33 @@ func TestPoller_FetchErrorIncrementsErrorsOnly(t *testing.T) {
 	assert.Equal(t, int64(0), s.consecutiveEmpty)
 }
 
+// A deadline is the idle path once the server holds the request open, and a
+// symptom before then, so it must neither reset the error count nor assert that
+// the server is reachable.
+func TestPoller_FetchTimeoutIsNoSignal(t *testing.T) {
+	client := mocks.NewClient(t)
+	client.On("FetchTask", mock.Anything, mock.Anything).Return(nil, context.DeadlineExceeded)
+
+	cfg, err := config.LoadDefault("")
+	require.NoError(t, err)
+	p := &Poller{client: client, cfg: cfg}
+	p.lastPollFailed.Store(true)
+
+	hook := test.NewGlobal()
+	defer hook.Reset()
+
+	s := &workerState{consecutiveErrors: 2}
+	_, ok := p.fetchTask(context.Background(), s)
+	require.False(t, ok)
+	assert.Equal(t, int64(2), s.consecutiveErrors)
+	assert.Equal(t, int64(1), s.consecutiveEmpty)
+	assert.True(t, p.lastPollFailed.Load(), "a timeout must not clear a known failure")
+
+	// An idle runner against a server holding the request open must not flood.
+	_, _ = p.fetchTask(context.Background(), s)
+	assert.Len(t, hook.AllEntries(), 1)
+}
+
 // TestPoller_FetchUnauthenticatedStopsPolling verifies that an Unauthenticated
 // response marks the runner as unregistered and cancels the polling context so
 // the daemon can exit instead of retrying forever.