fix: bound blocking calls and stop failing silently (#1174)

Jobs occasionally go silent ([example](https://gitea.com/gitea/runner/actions/runs/805045/jobs/1055123)) mid-run and Gitea reaped them after `ZOMBIE_TASK_TIMEOUT`, with no error in the log. This contains a number of related fixes, all with full test coverage:

1. Bound every RPC to Gitea with a timeout, a stalled report otherwise parked logs and heartbeats for the whole job.
2. Cap `runner.fetch_timeout` at that ceiling.
3. Let only the daemon loop close its own channel, the race panicked the process.
4. Stop the job on any terminal server result, not just `RESULT_CANCELLED`.
5. Report that result instead of relabelling it as cancelled.
6. Log reporting failures once at each end of an outage instead of discarding them.
7. Clamp the acknowledged log index, a too-large ack panicked on a slice bound.
8. Stop reading server health from a `FetchTask` deadline, it marked the runner healthy and reset the error backoff on a timeout.
9. Return an error from the Docker version probe instead of a `logrus` panic.
10. Pass the context to go-git's fetch and pull.
11. Fail the clone when a refresh dies on a cancelled context.
12. Set `terminationGracePeriodSeconds` in the Kubernetes examples.

Also contains a deprecation fix for goreleaser.

Reviewed-on: https://gitea.com/gitea/runner/pulls/1174
Reviewed-by: bircni <bircni@icloud.com>
Co-authored-by: silverwind <me@silverwind.io>

internal/pkg/client/http.go

@@ -17,7 +17,7 @@ import (
 	"gitea.dev/actionslib/runner/v1/runnerv1connect"
 )
 
-func getHTTPClient(endpoint string, insecure bool) *http.Client {
+func getHTTPClient(endpoint string, insecure bool, timeout time.Duration) *http.Client {
 	transport := &http.Transport{
 		Proxy:               http.ProxyFromEnvironment,
 		MaxIdleConns:        10,
@@ -29,11 +29,13 @@ func getHTTPClient(endpoint string, insecure bool) *http.Client {
 			InsecureSkipVerify: true,
 		}
 	}
-	return &http.Client{Transport: transport}
+	return &http.Client{Transport: transport, Timeout: timeout}
 }
 
-// New returns a new runner client.
-func New(endpoint string, insecure bool, uuid, token string, opts ...connect.ClientOption) *HTTPClient {
+// New returns a new runner client. timeout bounds every RPC: without it a
+// stalled connection parks the reporter for the whole job context, so logs and
+// heartbeats stop together and the task is reaped as a zombie.
+func New(endpoint string, insecure bool, uuid, token string, timeout time.Duration, opts ...connect.ClientOption) *HTTPClient {
 	baseURL := strings.TrimRight(endpoint, "/") + "/api/actions"
 
 	opts = append(opts, connect.WithInterceptors(connect.UnaryInterceptorFunc(func(next connect.UnaryFunc) connect.UnaryFunc {
@@ -49,7 +51,7 @@ func New(endpoint string, insecure bool, uuid, token string, opts ...connect.Cli
 		}
 	})))
 
-	httpClient := getHTTPClient(endpoint, insecure)
+	httpClient := getHTTPClient(endpoint, insecure, timeout)
 	return &HTTPClient{
 		PingServiceClient: pingv1connect.NewPingServiceClient(
 			httpClient,

internal/pkg/client/http_test.go

@@ -8,6 +8,7 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 
 	"connectrpc.com/connect"
 	pingv1 "gitea.dev/actionslib/ping/v1"
@@ -17,7 +18,8 @@ import (
 func TestGetHTTPClientUsesProxyFromEnvironment(t *testing.T) {
 	t.Setenv("HTTP_PROXY", "http://proxy.example.com:8080")
 
-	client := getHTTPClient("http://gitea.example.com", false)
+	client := getHTTPClient("http://gitea.example.com", false, time.Minute)
+	require.Equal(t, time.Minute, client.Timeout)
 	transport, ok := client.Transport.(*http.Transport)
 	require.True(t, ok)
 
@@ -32,7 +34,7 @@ func TestGetHTTPClientUsesProxyFromEnvironment(t *testing.T) {
 
 func TestGetHTTPClientInsecureTLS(t *testing.T) {
 	// insecure only takes effect for https endpoints
-	httpsInsecure := getHTTPClient("https://gitea.example.com", true)
+	httpsInsecure := getHTTPClient("https://gitea.example.com", true, time.Minute)
 	transport, ok := httpsInsecure.Transport.(*http.Transport)
 	require.True(t, ok)
 	require.NotNil(t, transport.TLSClientConfig)
@@ -47,7 +49,7 @@ func TestGetHTTPClientInsecureTLS(t *testing.T) {
 		{"http insecure ignored", "http://gitea.example.com", true},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			c := getHTTPClient(tc.endpoint, tc.insecure)
+			c := getHTTPClient(tc.endpoint, tc.insecure, time.Minute)
 			tr, ok := c.Transport.(*http.Transport)
 			require.True(t, ok)
 			require.Nil(t, tr.TLSClientConfig)
@@ -66,7 +68,7 @@ func TestNewSetsBaseURLAndHeaders(t *testing.T) {
 	defer server.Close()
 
 	// trailing slash must be trimmed before "/api/actions" is appended
-	c := New(server.URL+"/", false, "the-uuid", "the-token")
+	c := New(server.URL+"/", false, "the-uuid", "the-token", time.Minute)
 	// Address returns the endpoint as supplied (untrimmed)
 	require.Equal(t, server.URL+"/", c.Address())
 	require.False(t, c.Insecure())
@@ -87,7 +89,7 @@ func TestNewOmitsEmptyHeaders(t *testing.T) {
 	}))
 	defer server.Close()
 
-	c := New(server.URL, false, "", "")
+	c := New(server.URL, false, "", "", time.Minute)
 	_, _ = c.Ping(t.Context(), connect.NewRequest(&pingv1.PingRequest{Data: "hi"}))
 
 	require.Empty(t, gotHeaders.Get(UUIDHeader))