fix: bound blocking calls and stop failing silently (#1174)

Jobs occasionally go silent ([example](https://gitea.com/gitea/runner/actions/runs/805045/jobs/1055123)) mid-run and Gitea reaped them after `ZOMBIE_TASK_TIMEOUT`, with no error in the log. This contains a number of related fixes, all with full test coverage:

1. Bound every RPC to Gitea with a timeout, a stalled report otherwise parked logs and heartbeats for the whole job.
2. Cap `runner.fetch_timeout` at that ceiling.
3. Let only the daemon loop close its own channel, the race panicked the process.
4. Stop the job on any terminal server result, not just `RESULT_CANCELLED`.
5. Report that result instead of relabelling it as cancelled.
6. Log reporting failures once at each end of an outage instead of discarding them.
7. Clamp the acknowledged log index, a too-large ack panicked on a slice bound.
8. Stop reading server health from a `FetchTask` deadline, it marked the runner healthy and reset the error backoff on a timeout.
9. Return an error from the Docker version probe instead of a `logrus` panic.
10. Pass the context to go-git's fetch and pull.
11. Fail the clone when a refresh dies on a cancelled context.
12. Set `terminationGracePeriodSeconds` in the Kubernetes examples.

Also contains a deprecation fix for goreleaser.

Reviewed-on: https://gitea.com/gitea/runner/pulls/1174
Reviewed-by: bircni <bircni@icloud.com>
Co-authored-by: silverwind <me@silverwind.io>

internal/pkg/config/config.example.yaml

@@ -34,6 +34,7 @@ runner:
   # Whether skip verifying the TLS certificate of the Gitea instance.
   #insecure: false
   # The timeout for fetching the job from the Gitea instance.
+  # Values above the 60s RPC timeout are capped to it.
   #fetch_timeout: 5s
   # The interval for fetching the job from the Gitea instance.
   #fetch_interval: 2s

internal/pkg/config/config.go

@@ -20,6 +20,9 @@ import (
 	"go.yaml.in/yaml/v4"
 )
 
+// RequestTimeout bounds every RPC to Gitea, and with it runner.fetch_timeout.
+const RequestTimeout = 60 * time.Second
+
 // DefaultPostTaskScriptTimeout is the fallback cap on how long the post-task
 // script may run when post_task_script is set without an explicit timeout. It is
 // applied both at config load (for a configured script) and at the point of use
@@ -334,6 +337,10 @@ func LoadDefault(file string) (*Config, error) {
 		log.Warnf("runner.tool_cache_mode %q with capacity %d: two jobs writing the same tool version at once corrupt it",
 			ToolCacheModeShared, cfg.Runner.Capacity)
 	}
+	if cfg.Runner.FetchTimeout > RequestTimeout {
+		log.Warnf("fetch_timeout (%v) exceeds the RPC timeout (%v), capping it", cfg.Runner.FetchTimeout, RequestTimeout)
+		cfg.Runner.FetchTimeout = RequestTimeout
+	}
 	if cfg.Runner.FetchIntervalMax < cfg.Runner.FetchInterval {
 		log.Warnf("fetch_interval_max (%v) is less than fetch_interval (%v), setting fetch_interval_max to fetch_interval",
 			cfg.Runner.FetchIntervalMax, cfg.Runner.FetchInterval)

internal/pkg/config/config_test.go

@@ -416,3 +416,15 @@ func TestLoadDefault_ShippedConfigsChangeNothing(t *testing.T) {
 	}
 	assert.Empty(t, hook.AllEntries())
 }
+
+func TestLoadDefault_ClampsFetchTimeout(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte(`
+runner:
+  fetch_timeout: 120s
+`), 0o600))
+
+	cfg, err := LoadDefault(path)
+	require.NoError(t, err)
+	assert.Equal(t, RequestTimeout, cfg.Runner.FetchTimeout)
+}