Compare commits

..

3 Commits

Author SHA1 Message Date
Lunny Xiao c0a07cfb72 enhance: download each action repository once per job (#1178)
A job downloads each action repository once, keyed on the clone URL and ref, so repeated `uses:` and different paths of one repository share a checkout. The download is reported once as `{org}/{repo}@{ref}`, the way actions/runner reports it.

The action itself is still read per step, because a repository without an action file gets a synthetic action built from that step's `with.args`.

Fixes https://gitea.com/gitea/runner/issues/1159

---------

Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1178
Reviewed-by: bircni <bircni@icloud.com>
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2026-08-22 17:01:39 +00:00
Lunny Xiao 2fa5fe7121 fix: follow the act move in the paths outside Go code
`make test-dind` runs a hardcoded package path, and the `.gitignore`
negation for the test secrets fixture stopped matching.

Assisted-by: Codet:GPT-5.1-Codex
2026-08-07 21:49:42 -07:00
Lunny Xiao 825c6af07c refactor: move act under internal
The runner is an application, not a library. `act/model` and
`act/exprparser` were the last packages anything outside this repository
consumed and they now live in actionslib, so nothing needs the rest of
`act` to be importable, and keeping it importable invites the coupling
that was just removed.

Import paths only, the files are unchanged.

Assisted-by: Codet:GPT-5.1-Codex
2026-08-07 21:40:48 -07:00
336 changed files with 5886 additions and 5933 deletions
+10 -4
View File
@@ -20,10 +20,11 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with: with:
fetch-depth: 0 fetch-depth: 0
# Custom publishers (the R2 upload below) run as the very last # Custom publishers (the R2 mirror below) run as the very last
# step of goreleaser's publish pipeline, after the Gitea release # step of goreleaser's publish pipeline, after the Gitea release
# has already been created. Fail here instead, before anything # has already been created and every artifact already uploaded
# is built or published, if the R2 secrets are missing. # to S3. Fail here instead, before anything is built or
# published, if the R2 secrets are missing.
- name: check R2 configuration - name: check R2 configuration
run: sh scripts/upload-r2.sh --check-config run: sh scripts/upload-r2.sh --check-config
env: env:
@@ -42,6 +43,11 @@ jobs:
args: release --nightly args: release --nightly
env: env:
GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
S3_REGION: ${{ secrets.AWS_REGION }}
S3_BUCKET: ${{ secrets.AWS_BUCKET }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }} R2_BUCKET: ${{ secrets.R2_BUCKET }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
@@ -77,7 +83,7 @@ jobs:
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
- name: Set up Docker BuildX - name: Set up Docker BuildX
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
- name: Login to DockerHub - name: Login to DockerHub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
+10 -4
View File
@@ -12,10 +12,11 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with: with:
fetch-depth: 0 # all history for all branches and tags fetch-depth: 0 # all history for all branches and tags
# Custom publishers (the R2 upload below) run as the very last # Custom publishers (the R2 mirror below) run as the very last
# step of goreleaser's publish pipeline, after the Gitea release # step of goreleaser's publish pipeline, after the Gitea release
# has already been created. Fail here instead, before anything # has already been created and every artifact already uploaded
# is built or published, if the R2 secrets are missing. # to S3. Fail here instead, before anything is built or
# published, if the R2 secrets are missing.
- name: check R2 configuration - name: check R2 configuration
run: sh scripts/upload-r2.sh --check-config run: sh scripts/upload-r2.sh --check-config
env: env:
@@ -41,6 +42,11 @@ jobs:
args: release args: release
env: env:
GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
S3_REGION: ${{ secrets.AWS_REGION }}
S3_BUCKET: ${{ secrets.AWS_BUCKET }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }} R2_BUCKET: ${{ secrets.R2_BUCKET }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
@@ -80,7 +86,7 @@ jobs:
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
- name: Set up Docker BuildX - name: Set up Docker BuildX
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
- name: Login to DockerHub - name: Login to DockerHub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
+10 -37
View File
@@ -5,18 +5,17 @@ on:
- main - main
pull_request: pull_request:
env:
# The runner image ships a stale docker.io login; point docker at an empty config so
# image pulls go straight to anonymous instead of attempting (and failing) that auth
# first. The path must be a literal: the `runner` context is unavailable in workflow-level
# env, so `${{ runner.temp }}` would resolve to empty and config.Dir() would fall back
# to ~/.docker with the stale credentials.
DOCKER_CONFIG: /tmp/docker-noauth
jobs: jobs:
lint: lint:
name: check and test name: check and test
runs-on: ubuntu-latest runs-on: ubuntu-latest
env:
# The runner image ships a stale docker.io login; point docker at an empty config so
# image pulls go straight to anonymous instead of attempting (and failing) that auth
# first. The path must be a literal: the `runner` context is unavailable in job-level
# env, so `${{ runner.temp }}` would resolve to empty and config.Dir() would fall back
# to ~/.docker with the stale credentials.
DOCKER_CONFIG: /tmp/docker-noauth
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
@@ -25,20 +24,13 @@ jobs:
check-latest: true check-latest: true
- name: prepare anonymous docker config - name: prepare anonymous docker config
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json" run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`; # Pre-pull internal/act/runner's two largest base images so a slow pull can't dominate `make test`;
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host # the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
# daemon retains them between runs, so this is usually a fast manifest re-check. # daemon retains them between runs, so this is usually a fast manifest re-check.
- name: pre-pull test images - name: pre-pull test images
env:
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:3638d9a6fe4030bd716be989438248074489337ba3275657f93595428be4fc03 # renovate: datasource=docker
TEST_SERVICE_IMAGE: nginx:alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
run: | run: |
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do for img in node:24-bookworm-slim nginx:alpine; do
for attempt in 1 2 3; do for try in 1 2 3; do docker pull "$img" && break || sleep 5; done
docker pull "$image" && break
[ "$attempt" = 3 ] || sleep 5
done
docker tag "$image" "${image%@*}"
done done
- name: lint - name: lint
run: make lint run: make lint
@@ -57,22 +49,3 @@ jobs:
run: | run: |
make coverage-report make coverage-report
cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY" cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY"
e2e:
name: gitea compatibility (${{ matrix.gitea_image }})
runs-on: ubuntu-latest
strategy:
matrix:
gitea_image:
- gitea/gitea:latest
- gitea/gitea:main-nightly
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: 'go.mod'
check-latest: true
- name: prepare anonymous docker config
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
- name: e2e compatibility
run: make test-e2e E2E_GITEA_IMAGE=${{ matrix.gitea_image }}
+1 -1
View File
@@ -1,6 +1,6 @@
/gitea-runner /gitea-runner
.env .env
!/act/runner/testdata/secrets/.env !/internal/act/runner/testdata/secrets/.env
.runner .runner
.runner.lock .runner.lock
coverage.txt coverage.txt
+6 -8
View File
@@ -46,7 +46,8 @@ linters:
gocritic: gocritic:
enabled-checks: enabled-checks:
- equalFold - equalFold
disabled-checks: [] disabled-checks:
- ifElseChain
revive: revive:
severity: error severity: error
rules: rules:
@@ -70,14 +71,10 @@ linters:
- name: unexported-return - name: unexported-return
- name: var-declaration - name: var-declaration
- name: var-naming - name: var-naming
arguments:
- [] # AllowList - do not remove as args for the rule are positional and won't work without lists first
- [] # DenyList
- - skip-initialism-name-checks: true
staticcheck: staticcheck:
checks: checks:
- all - all
testifylint: {} - -ST1005
usetesting: usetesting:
os-temp-dir: true os-temp-dir: true
perfsprint: perfsprint:
@@ -95,6 +92,8 @@ linters:
generated: lax generated: lax
presets: presets:
- comments - comments
- common-false-positives
- legacy
- std-error-handling - std-error-handling
rules: rules:
- linters: - linters:
@@ -119,8 +118,7 @@ formatters:
- blank - blank
- default - default
gofumpt: gofumpt:
extra: extra-rules: true
group-params: true
exclusions: exclusions:
generated: lax generated: lax
run: run:
+19 -7
View File
@@ -83,12 +83,24 @@ builds:
- cmd: sh .goreleaser.checksum.sh {{ .Path }} - cmd: sh .goreleaser.checksum.sh {{ .Path }}
- cmd: sh .goreleaser.checksum.sh {{ .Path }}.xz - cmd: sh .goreleaser.checksum.sh {{ .Path }}.xz
# Uploads every release artifact to Cloudflare R2. The `blobs:` pipe blobs:
# isn't usable here since it authenticates from the global AWS_* env -
# with no per-entry credentials; `publishers:` supports per-entry provider: s3
# `env:` instead, so it's used to invoke scripts/upload-r2.sh once per bucket: "{{ .Env.S3_BUCKET }}"
# artifact. Custom publishers inherit almost nothing from the region: "{{ .Env.S3_REGION }}"
# environment, hence the explicit R2_* forwarding below. directory: "gitea-runner/{{.Version}}"
extra_files:
- glob: ./**.xz
- glob: ./**.sha256
# Mirrors the S3 `blobs:` upload above into Cloudflare R2 during the
# parallel S3+R2 period (S3 will be removed once migration completes).
# A second `blobs:` entry is impossible here since the blob pipe
# authenticates from the global AWS_* env with no per-entry
# credentials; `publishers:` supports per-entry `env:` instead, so
# it's used to invoke scripts/upload-r2.sh once per artifact. Custom
# publishers inherit almost nothing from the environment, hence the
# explicit R2_* forwarding below.
# #
# This publisher fires 109 times for 73 distinct keys because # This publisher fires 109 times for 73 distinct keys because
# goreleaser's release pipe already registers `release.extra_files` # goreleaser's release pipe already registers `release.extra_files`
@@ -113,7 +125,7 @@ publishers:
- R2_SECRET_ACCESS_KEY={{ index .Env "R2_SECRET_ACCESS_KEY" }} - R2_SECRET_ACCESS_KEY={{ index .Env "R2_SECRET_ACCESS_KEY" }}
archives: archives:
- formats: [binary] - format: binary
name_template: "{{ .Binary }}" name_template: "{{ .Binary }}"
allow_different_binary_count: true allow_different_binary_count: true
-9
View File
@@ -30,12 +30,3 @@ depending on the prefix:
encoded forms too. encoded forms too.
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the reporter - Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the reporter
decodes exactly those two when folding a location into an annotation. decodes exactly those two when folding a location into an annotation.
## End-to-end compatibility tests
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
parallel.
The suite shares one Gitea and regular runner. Cache and ephemeral scenarios use isolated
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
+3 -3
View File
@@ -1,7 +1,7 @@
### BUILDER STAGE ### BUILDER STAGE
# #
# #
FROM golang:1.27-alpine3.23 AS builder FROM golang:1.26-alpine3.23 AS builder
# Do not remove `git` here, it is required for getting runner version when executing `make build` # Do not remove `git` here, it is required for getting runner version when executing `make build`
RUN apk add --no-cache make git RUN apk add --no-cache make git
@@ -17,7 +17,7 @@ RUN make clean && make build
### DIND VARIANT ### DIND VARIANT
# #
# #
FROM docker:29.7.2-dind AS dind FROM docker:29.6.2-dind AS dind
ARG VERSION=dev ARG VERSION=dev
@@ -37,7 +37,7 @@ ENTRYPOINT ["s6-svscan","/etc/s6"]
### DIND-ROOTLESS VARIANT ### DIND-ROOTLESS VARIANT
# #
# #
FROM docker:29.7.2-dind-rootless AS dind-rootless FROM docker:29.6.2-dind-rootless AS dind-rootless
ARG VERSION=dev ARG VERSION=dev
+4 -13
View File
@@ -5,7 +5,7 @@ GO ?= go
SHASUM ?= shasum -a 256 SHASUM ?= shasum -a 256
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" ) HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
XGO_PACKAGE ?= src.techknowlogick.com/xgo@v1.9.0 # renovate: datasource=go XGO_PACKAGE ?= src.techknowlogick.com/xgo@v1.9.0 # renovate: datasource=go
XGO_VERSION := go-1.27.x XGO_VERSION := go-1.26.x
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/gxz@v0.5.16 # renovate: datasource=go GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/gxz@v0.5.16 # renovate: datasource=go
LINUX_ARCHS ?= linux/amd64,linux/arm64 LINUX_ARCHS ?= linux/amd64,linux/arm64
@@ -18,8 +18,8 @@ DOCKER_TAG ?= nightly
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG) DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.13.1 # renovate: datasource=go GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2 # renovate: datasource=go
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.7.0 # renovate: datasource=go GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.6.0 # renovate: datasource=go
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8 GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
@@ -137,7 +137,7 @@ lint-pr-title: ## lint PR title against Conventional Commits (set PR_TITLE=...)
.PHONY: security-check .PHONY: security-check
security-check: security-check:
$(GO) run $(GOVULNCHECK_PACKAGE) -show color ./... || true GOEXPERIMENT= $(GO) run $(GOVULNCHECK_PACKAGE) -show color ./... || true
.PHONY: tidy .PHONY: tidy
tidy: ## run go mod tidy tidy: ## run go mod tidy
@@ -171,15 +171,6 @@ coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md
test-dind: ## run the daemon-facing tests against the built dind image (TARGET=dind|dind-rootless) test-dind: ## run the daemon-facing tests against the built dind image (TARGET=dind|dind-rootless)
@./scripts/test-dind.sh $(TARGET) @./scripts/test-dind.sh $(TARGET)
E2E_JOB_IMAGE ?= node:24-bookworm@sha256:934240a162082fd8b8a2f90cd5114446443f1eba1c5378f6687167ca405e6584 # renovate: datasource=docker
SERVICE_IMAGE ?= nginx:1.31.4-alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
E2E_CONCURRENCY ?= 8
.PHONY: test-e2e
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
.PHONY: install .PHONY: install
install: $(GOFILES) ## install the runner binary via `go install` install: $(GOFILES) ## install the runner binary via `go install`
$(GO) install -v -tags '$(TAGS)' -ldflags '-s -w $(EXTLDFLAGS) $(LDFLAGS)' $(GO) install -v -tags '$(TAGS)' -ldflags '-s -w $(EXTLDFLAGS) $(LDFLAGS)'
+2 -46
View File
@@ -158,32 +158,6 @@ An edit keeps the comments and the key order of the file. Indentation becomes tw
`config get`, `set`, `add` and `remove` use `config.yaml` (or `config.yml`) from the working directory, then from the directory of the binary, and print their choice to stderr. `config init` writes `config.yaml` in the working directory, and refuses to overwrite an existing config without `--force`. Pass `-c` for another path. `config get`, `set`, `add` and `remove` use `config.yaml` (or `config.yml`) from the working directory, then from the directory of the binary, and print their choice to stderr. `config init` writes `config.yaml` in the working directory, and refuses to overwrite an existing config without `--force`. Pass `-c` for another path.
#### Tool cache
Setup actions like `setup-go` install tools into `RUNNER_TOOL_CACHE`, which is `/opt/hostedtoolcache` inside a job. `runner.tool_cache_mode` selects what backs it:
| Mode | Tool cache | Trade-off |
| --- | --- | --- |
| `none` (default) | Per job, provided by the job image | A version the image lacks is downloaded in every job |
| `shared` | One volume reused by every job | Two jobs writing the same tool version at once corrupt it, so use it only with `runner.capacity: 1` |
With `none`, tools must come from the job image. Install them into `/opt/hostedtoolcache/<tool>/<version>/<arch>`, with an empty `<arch>.complete` file next to the directory:
```dockerfile
RUN GO=$(curl -fsSL 'https://go.dev/dl/?mode=json' | grep -oP '"version": "\Kgo1\.26\.[0-9]*' | head -1); \
DIR="/opt/hostedtoolcache/go/${GO#go}/x64" && \
mkdir -p "$(dirname "$DIR")" && \
curl -fsSL "https://dl.google.com/go/${GO}.linux-amd64.tar.gz" | tar -xz -C /tmp && \
mv /tmp/go "$DIR" && \
touch "${DIR}.complete"
```
A workflow requesting a minor version, `go-version: "1.26"`, resolves to the newest matching version in the cache, so a patch update in the image still hits it.
Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavour is the one that ships tools in this layout.
`gitea-runner exec` reads no config file and takes `--tool-cache-mode` instead, defaulting to `none`.
#### Environment variables #### Environment variables
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below. Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
@@ -228,7 +202,7 @@ a workflow with `runs-on: ubuntu-latest` is executed in the `runner-images:ubunt
Names may themselves contain a colon (for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`); only `host` and `docker` are treated as schemas. Names may themselves contain a colon (for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`); only `host` and `docker` are treated as schemas.
If a job's `runs-on` matches none of the runner's labels, or sets no `runs-on` at all, it still runs: in `runner.default_image` where docker is available, on the host where it is not. Images maintained for this purpose are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images). If a job's `runs-on` matches none of the runner's labels, the job still runs, in the default `docker.gitea.com/runner-images:ubuntu-latest` image. Images maintained for this purpose are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
Labels are chosen at registration time (`--labels`, or the interactive prompt) and can be changed afterwards by editing `runner.labels` in the config file, or in the Gitea UI under the runner's settings. Labels are chosen at registration time (`--labels`, or the interactive prompt) and can be changed afterwards by editing `runner.labels` in the config file, or in the Gitea UI under the runner's settings.
@@ -299,12 +273,6 @@ A password in a proxy URL is hidden in job logs. Any step can still read it, bec
Each runner starts its own cache server automatically. Cache entries are local to that runner — runners do not share a cache by default. Each runner starts its own cache server automatically. Cache entries are local to that runner — runners do not share a cache by default.
**Eviction**
An entry nothing has read or written for `retention` is removed, and a repository past `repo_size_limit` loses its least recently accessed entries until it fits; `size_limit` caps the whole cache the same way. Age alone never retires an entry still in use, and whatever these allow, the cache keeps free space above `health_check.min_free_disk_space_mb` when health checks are enabled.
These apply where the cache server runs, so on a shared server they belong in *its* config, not the runners'. See `retention`, `repo_size_limit`, `size_limit` and `sweep_interval` in [config.example.yaml](internal/pkg/config/config.example.yaml) for units and defaults.
**Cache service v2** **Cache service v2**
`actions/cache@v4.2` and later can use the *cache service v2* API. The runner serves it from the same store as v1, on by default, and it works with `external_server`. Turn it off with: `actions/cache@v4.2` and later can use the *cache service v2* API. The runner serves it from the same store as v1, on by default, and it works with `external_server`. Turn it off with:
@@ -314,7 +282,7 @@ cache:
v2: false v2: false
``` ```
Those actions refuse any host they do not take for GitHub. Rather than misreport the server URL, the runner edits that check out of the action's own bundle on its way into the job, undone whenever the action is downloaded again. A bundle it does not recognise is left alone and keeps to v1. The same edit lets the stock `actions/upload-artifact` and `actions/download-artifact` work from `v4.4.0` on, without the `gitea-upload-artifact` fork, so it is made whatever `v2` says: that setting only governs the API the runner advertises. Set `runner.patch_actions: false` to leave every bundle exactly as shipped, an escape hatch for an action the edit breaks. The artifact actions then refuse again and the cache client keeps to v1. Those actions refuse any host they do not take for GitHub. Rather than misreport the server URL, the runner edits that check out of the action's own bundle and keeps the untouched copy beside it; a bundle it does not recognise is left alone and keeps to v1. The same edit lets the stock `actions/upload-artifact` and `actions/download-artifact` work from `v4.4.0` on, without the `gitea-upload-artifact` fork.
**Shared cache across multiple runners** **Shared cache across multiple runners**
@@ -345,8 +313,6 @@ Run one dedicated `gitea-runner cache-server` that all runners point at.
# external_secret_file: /path/to/secret # secret can also be passed via a file # external_secret_file: /path/to/secret # secret can also be passed via a file
``` ```
Jobs reach the cache server at `external_server`, so when a reverse proxy fronts the server, point `external_server` at the proxy. The cache server itself needs no extra configuration.
Alternatively, mount the same NFS/CIFS share on every runner and point `cache.dir` at it — simpler, but with weaker isolation between repositories. Alternatively, mount the same NFS/CIFS share on every runner and point `cache.dir` at it — simpler, but with weaker isolation between repositories.
**S3 / MinIO** — mount object storage as a FUSE filesystem (e.g. [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) or [goofys](https://github.com/kahing/goofys)) and set `cache.dir` to the mount point. **S3 / MinIO** — mount object storage as a FUSE filesystem (e.g. [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) or [goofys](https://github.com/kahing/goofys)) and set `cache.dir` to the mount point.
@@ -386,16 +352,6 @@ Both hooks are synchronous and block the job while they run. Either one exiting
See **[docs/job-hooks.md](docs/job-hooks.md)** for the execution order, environment, and platform notes. See **[docs/job-hooks.md](docs/job-hooks.md)** for the execution order, environment, and platform notes.
#### Local job logs (`log.job.dir`)
Set `log.job.dir` to a path and the runner writes a copy of every task's log there as `<start time>-task-<id>.log`: the rows exactly as Gitea received them, with the same secrets masked and the job's result on the last line. Off by default, and what Gitea shows does not change.
#### Secret masking
A job's secrets and its `::add-mask::` values are hidden from what the runner writes and uploads: the job log, the local copy above, job summaries, and the names of the containers it creates. A job output carrying one is skipped with a warning rather than sent masked, as GitHub does, so a downstream `needs.<job>.outputs.<name>` reading it is empty.
`log.job.retention` (default `168h`) is how long a log is kept, expired ones being deleted as new tasks start, and `log.job.max_size` (default `1GB`) caps one log. Keep `retention` above `runner.timeout` so a long job cannot outlive its own log, and prefer local disk, the file is written while the job runs. Only the runner's own user can read it.
### Example Deployments ### Example Deployments
Check out the [examples](examples) directory for sample deployment types. Check out the [examples](examples) directory for sample deployment types.
-189
View File
@@ -1,189 +0,0 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// Copyright 2021 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package artifacts
import (
"bytes"
"compress/gzip"
"encoding/json/v2"
"io"
"maps"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/julienschmidt/httprouter"
"github.com/stretchr/testify/require"
)
func TestArtifactFlow(t *testing.T) {
artifactPath := t.TempDir()
router := httprouter.New()
uploads(router, artifactPath)
downloads(router, artifactPath)
server := httptest.NewServer(router)
defer server.Close()
baseURL := server.URL
client := server.Client()
client.Timeout = 5 * time.Second
request := func(t *testing.T, method, rawURL string, body io.Reader, header http.Header) (int, []byte) {
t.Helper()
req, err := http.NewRequest(method, rawURL, body)
require.NoError(t, err)
maps.Copy(req.Header, header)
resp, err := client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
data, err := io.ReadAll(resp.Body)
require.NoError(t, err)
return resp.StatusCode, data
}
t.Run("upload-and-download", func(t *testing.T) {
const runID, item, content = "1", "my-artifact/data.txt", "hello artifact\n"
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
var prep FileContainerResourceURL
require.NoError(t, json.Unmarshal(data, &prep))
require.Equal(t, baseURL+"/upload/"+runID, prep.FileContainerResourceURL)
status, data = request(t, http.MethodPut, prep.FileContainerResourceURL+"?itemPath="+url.QueryEscape(item), strings.NewReader(content), nil)
require.Equal(t, http.StatusOK, status, string(data))
var msg ResponseMessage
require.NoError(t, json.Unmarshal(data, &msg))
require.Equal(t, "success", msg.Message)
status, data = request(t, http.MethodPatch, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
require.NoError(t, json.Unmarshal(data, &msg))
require.Equal(t, "success", msg.Message)
status, data = request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
var list NamedFileContainerResourceURLResponse
require.NoError(t, json.Unmarshal(data, &list))
require.Equal(t, 1, list.Count)
require.Equal(t, "my-artifact", list.Value[0].Name)
status, data = request(t, http.MethodGet, list.Value[0].FileContainerResourceURL+"?itemPath=my-artifact", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
var items ContainerItemResponse
require.NoError(t, json.Unmarshal(data, &items))
require.Len(t, items.Value, 1)
require.Equal(t, "file", items.Value[0].ItemType)
require.Equal(t, "my-artifact/data.txt", items.Value[0].Path)
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
require.Equal(t, http.StatusOK, status)
require.Equal(t, content, string(data))
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "my-artifact", "data.txt"))
require.NoError(t, err)
require.Equal(t, content, string(stored))
})
t.Run("content-range", func(t *testing.T) {
const rawURL = "/upload/4?itemPath=chunks.txt"
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
http.Header{"Content-Range": []string{"bytes 0-4/11"}})
require.Equal(t, http.StatusOK, status, string(data))
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
http.Header{"Content-Range": []string{"bytes 5-11/11"}})
require.Equal(t, http.StatusOK, status, string(data))
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
require.NoError(t, err)
require.Equal(t, "first-second", string(stored))
})
t.Run("gzip-roundtrip", func(t *testing.T) {
const runID, item, content = "2", "logs/app.log", "compressed payload\n"
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
_, err := gz.Write([]byte(content))
require.NoError(t, err)
require.NoError(t, gz.Close())
status, data := request(t, http.MethodPut, baseURL+"/upload/"+runID+"?itemPath="+url.QueryEscape(item),
&buf, http.Header{"Content-Encoding": []string{"gzip"}})
require.Equal(t, http.StatusOK, status, string(data))
// stored compressed, with the server's gzip marker suffix
_, err = os.Stat(filepath.Join(artifactPath, runID, "logs", "app.log.gz__"))
require.NoError(t, err)
status, data = request(t, http.MethodGet, baseURL+"/download/"+runID+"?itemPath=logs", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
var items ContainerItemResponse
require.NoError(t, json.Unmarshal(data, &items))
require.Len(t, items.Value, 1)
require.Equal(t, "logs/app.log", items.Value[0].Path)
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
require.Equal(t, http.StatusOK, status)
require.Equal(t, content, string(data))
})
// GHSL-2023-004: an itemPath that climbs out of the run directory must be neutralised so the
// blob cannot be written outside the artifact root.
t.Run("GHSL-2023-004", func(t *testing.T) {
const runID, content = "3", "contained\n"
status, data := request(t, http.MethodPut, baseURL+"/upload/"+runID+"?itemPath="+url.QueryEscape("../../escape.txt"),
strings.NewReader(content), nil)
require.Equal(t, http.StatusOK, status, string(data))
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "escape.txt"))
require.NoError(t, err)
require.Equal(t, content, string(stored))
_, err = os.Stat(filepath.Join(filepath.Dir(artifactPath), "escape.txt"))
require.True(t, os.IsNotExist(err), "upload escaped the artifact root")
status, data = request(t, http.MethodGet, baseURL+"/artifact/"+runID+"/escape.txt", nil, nil)
require.Equal(t, http.StatusOK, status)
require.Equal(t, content, string(data))
})
}
func TestSafeResolve(t *testing.T) {
baseDir := "/foo/bar"
tests := map[string]struct {
input string
want string
}{
"simple": {input: "baz", want: "/foo/bar/baz"},
"nested": {input: "baz/blue", want: "/foo/bar/baz/blue"},
"dots in middle": {input: "baz/../../blue", want: "/foo/bar/blue"},
"leading dots": {input: "../../parent", want: "/foo/bar/parent"},
"root path": {input: "/root", want: "/foo/bar/root"},
"root": {input: "/", want: "/foo/bar"},
"empty": {input: "", want: "/foo/bar"},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
require.Equal(t, tc.want, safeResolve(baseDir, tc.input))
})
}
}
func TestServeEmptyArtifactPathReturnsCancelableNoop(t *testing.T) {
cancel := Serve(t.Context(), "", "127.0.0.1", "0")
require.NotNil(t, cancel)
cancel()
}
-153
View File
@@ -1,153 +0,0 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// Copyright 2024 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package filecollector
import (
"archive/tar"
"bytes"
"context"
"io"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
git "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestIgnoredTrackedfile(t *testing.T) {
repoDir := filepath.Join(t.TempDir(), "mygitrepo")
repo, err := git.PlainInit(repoDir, false)
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".gitignore"), []byte(".*\n"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".env"), []byte("test=val1\n"), 0o644))
worktree, err := repo.Worktree()
require.NoError(t, err)
_, err = worktree.Add(".gitignore")
require.NoError(t, err)
var archive bytes.Buffer
tw := tar.NewWriter(&archive)
patterns, err := gitignore.ReadPatterns(worktree.Filesystem, nil)
require.NoError(t, err)
ignorer := gitignore.NewMatcher(patterns)
fc := &FileCollector{
Ignorer: ignorer,
SrcPath: repoDir,
SrcPrefix: repoDir + string(filepath.Separator),
Handler: &TarCollector{
TarWriter: tw,
},
}
err = filepath.Walk(repoDir, fc.CollectFiles(context.Background(), nil))
assert.NoError(t, err, "successfully collect files")
require.NoError(t, tw.Close())
tr := tar.NewReader(&archive)
h, err := tr.Next()
assert.NoError(t, err, "tar must not be empty") //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, ".gitignore", h.Name)
_, err = tr.Next()
assert.ErrorIs(t, err, io.EOF, "tar must only contain one element")
}
func TestSymlinks(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("creating symlinks requires elevated privileges on Windows")
}
repoDir := filepath.Join(t.TempDir(), "mygitrepo")
repo, err := git.PlainInit(repoDir, false)
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".env"), []byte("test=val1\n"), 0o644))
require.NoError(t, os.Symlink(".env", filepath.Join(repoDir, "test.env")))
worktree, err := repo.Worktree()
require.NoError(t, err)
_, err = worktree.Add("test.env")
require.NoError(t, err)
var archive bytes.Buffer
tw := tar.NewWriter(&archive)
fc := &FileCollector{
SrcPath: repoDir,
SrcPrefix: repoDir + string(filepath.Separator),
Handler: &TarCollector{
TarWriter: tw,
},
}
err = filepath.Walk(repoDir, fc.CollectFiles(context.Background(), nil))
assert.NoError(t, err, "successfully collect files")
require.NoError(t, tw.Close())
tr := tar.NewReader(&archive)
h, err := tr.Next()
files := map[string]tar.Header{}
for err == nil {
files[h.Name] = *h
h, err = tr.Next()
}
assert.Equal(t, ".env", files[".env"].Name)
assert.Equal(t, "test.env", files["test.env"].Name)
assert.Equal(t, ".env", files["test.env"].Linkname)
assert.ErrorIs(t, err, io.EOF, "tar must be read cleanly to EOF")
}
// Regression for https://gitea.com/gitea/runner/issues/876 and /941:
// re-copying an action directory must overwrite a pre-existing read-only
// file (e.g. a git pack .idx at mode 0444) instead of failing with EACCES
// on macOS or "Access is denied" on Windows.
func TestCopyCollectorWriteFileOverwritesReadOnlyFile(t *testing.T) {
dst := t.TempDir()
target := filepath.Join(dst, "sub", "pack.idx")
require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755))
require.NoError(t, os.WriteFile(target, []byte("old"), 0o444))
src := filepath.Join(t.TempDir(), "pack.idx")
require.NoError(t, os.WriteFile(src, []byte("new"), 0o444))
fi, err := os.Stat(src)
require.NoError(t, err)
cc := &CopyCollector{DstDir: dst}
require.NoError(t, cc.WriteFile("sub/pack.idx", fi, "", strings.NewReader("new")))
got, err := os.ReadFile(target)
require.NoError(t, err)
assert.Equal(t, "new", string(got))
}
// Without the destination removal, os.Symlink fails with EEXIST when the
// path already holds a regular file from an earlier copy of the action.
func TestCopyCollectorWriteFileOverwritesFileWithSymlink(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("creating symlinks requires elevated privileges on Windows")
}
dst := t.TempDir()
target := filepath.Join(dst, "link")
require.NoError(t, os.WriteFile(target, []byte("stale"), 0o644))
fi, err := os.Lstat(target)
require.NoError(t, err)
cc := &CopyCollector{DstDir: dst}
require.NoError(t, cc.WriteFile("link", fi, "target", nil))
resolved, err := os.Readlink(target)
require.NoError(t, err)
assert.Equal(t, "target", resolved)
}
func TestFileCollectorCancellationAndWalkError(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
cancel()
walk := (&FileCollector{}).CollectFiles(ctx, nil)
err := walk("file", nil, nil)
require.EqualError(t, err, "copy cancelled")
err = walk("file", nil, os.ErrPermission)
require.ErrorIs(t, err, os.ErrPermission)
}
-21
View File
@@ -1,21 +0,0 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Copyright 2022 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package lookpath
import (
"runtime"
"strings"
)
func getenv(env map[string]string, name string) string {
if runtime.GOOS == "windows" {
for key, value := range env {
if strings.EqualFold(name, key) {
return value
}
}
}
return env[name]
}
-165
View File
@@ -1,165 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package runner
import (
"bytes"
"context"
"os"
"path/filepath"
"regexp"
"strings"
"gitea.com/gitea/runner/act/common"
"gitea.dev/actionslib/pkg/model"
)
// Actions bundle the @actions toolkit into their own JavaScript, and two of its lines keep it
// from working against Gitea. Both are edited out of the bundle the runner downloaded.
//
// isGhes() takes any host that is not github.com, *.ghe.com or *.localhost for GitHub
// Enterprise. @actions/cache then forces the v1 API, and @actions/artifact refuses outright,
// which is why the stock upload-artifact aborts here. The edit empties the last of the three
// hostname tests, so `endsWith('.LOCALHOST')` becomes `endsWith(”)`, which every hostname
// satisfies: one string literal, no call sites to resolve, and the same answer the toolkit's own
// proposed ACTIONS_VENDOR switch would give. Gitea already makes this edit by hand in its fork
// of upload-artifact.
//
// getCacheServiceURL() then resolves the cache service from ACTIONS_RESULTS_URL alone, where v1
// reads ACTIONS_CACHE_URL first. Both reads there are given the same preference, which is what
// keeps the runner out of the artifact path: the results URL still points at Gitea.
//
// Either of these landing upstream makes this file deletable:
//
// https://github.com/actions/toolkit/pull/2123 — an ACTIONS_VENDOR switch, naming Gitea
// https://github.com/actions/toolkit/issues/2439 — treat ACTIONS_RESULTS_URL as the signal
const (
CacheServiceV2Env = "ACTIONS_CACHE_SERVICE_V2"
cacheURLEnv = "ACTIONS_CACHE_URL"
resultsURLEnv = "ACTIONS_RESULTS_URL"
// localhostHost is the suffix isGhes accepts.
localhostHost = ".LOCALHOST"
// artifactRefusal is the only thing the gate guards in @actions/artifact, which is what makes
// such a bundle safe to open. A bundle carrying neither toolkit uses isGhes for something this
// runner has not looked at, and is left alone.
artifactRefusal = "GHESNotSupportedError"
maxBundleSize = 64 << 20
)
var (
// localhostTest matches the third hostname test of isGhes, in any quoting. The match is case
// sensitive on purpose, and that is load-bearing: isGhes uppercases the hostname before
// testing it, while undici, bundled into all of these actions, tests a lowercase ".localhost"
// in isURLPotentiallyTrustworthy. Opening that one would tell its HTTP client that every URL
// is trustworthy. Uppercase, the literal occurs nowhere but this test, across 118 bundles
// covering every major version of sixteen actions.
localhostTest = regexp.MustCompile(`endsWith\s*\(\s*` + quoted(regexp.QuoteMeta(localhostHost)) + `\s*\)`)
// serviceURLBranches matches both branches of getCacheServiceURL at once: the v1 branch reads
// the cache URL and falls back to the results URL, the v2 branch just below reads the results
// URL alone. That `||` pairing is the only place the two variables are read together, so
// matching them as one expression is what keeps the edit inside this function rather than
// anywhere they happen to sit near each other. The branches are 21 bytes apart minified and
// 63 not, across every bundle measured.
serviceURLBranches = regexp.MustCompile(`(` + envRead(cacheURLEnv) + `\s*\|\|\s*)(` +
envRead(resultsURLEnv) + `)((?s).{0,256}?)(` + envRead(resultsURLEnv) + `)`)
// cacheURLFirst gives both reads the preference the v1 branch already had.
cacheURLFirst = []byte(`${1}(process.env.` + cacheURLEnv + `||${2})${3}(process.env.` + cacheURLEnv + `||${4})`)
)
func envRead(name string) string {
return `process\s*\.\s*env\s*(?:\.\s*` + name + `\b|\[\s*` + quoted(name) + `\s*\])`
}
// quoted matches a string literal in any of the three quote characters. RE2 has no
// backreferences, so the pairs are spelled out.
func quoted(pattern string) string {
return "(?:'" + pattern + "'|\"" + pattern + "\"|`" + pattern + "`)"
}
// actionScriptPaths returns the entrypoints of a node action, the only kind with a bundle. Only
// remote actions get here: a local one lives in the user's checkout, which the runner does not
// rewrite.
func actionScriptPaths(dir string, action *model.Action) []string {
if action == nil || !action.Runs.Using.IsNode() {
return nil
}
var paths []string
for _, script := range []string{action.Runs.Pre, action.Runs.Main, action.Runs.Post} {
if script == "" {
continue
}
path := filepath.Join(dir, script)
// `runs` is the action's own yaml, and a key pointing outside its directory is not ours.
if rel, err := filepath.Rel(dir, path); err != nil || strings.HasPrefix(rel, "..") {
continue
}
paths = append(paths, path)
}
return paths
}
// patchActions edits the toolkit in an action's bundles. The caller holds the action directory's
// clone lock, which is what keeps another job's checkout from resetting them before the copy.
func patchActions(ctx context.Context, scripts []string) {
for _, script := range scripts {
switch patched, err := patchBundle(script); {
case err != nil:
common.Logger(ctx).Warnf("actions toolkit: %s left unpatched: %v", script, err)
case patched:
common.Logger(ctx).Debugf("actions toolkit: patched %s", script)
}
}
}
func patchBundle(script string) (bool, error) {
info, err := os.Stat(script)
if err != nil {
return false, err
}
if info.Size() > maxBundleSize {
return false, nil
}
data, err := os.ReadFile(script)
if err != nil {
return false, err
}
patched, ok := patchedBundle(data)
if !ok {
return false, nil
}
// No atomic write needed: every prepare checks the action out and hard resets it.
return true, os.WriteFile(script, patched, info.Mode().Perm())
}
// patchedBundle opens the GHES gate, and where the cache toolkit is present, points the cache
// service at the cache server. A bundle this runner cannot account for comes back untouched.
func patchedBundle(data []byte) ([]byte, bool) {
// Literals before regex: most bundles carry neither toolkit and stop here. The artifact gate
// guards a refusal with no URL to move, so it opens alone; the cache gate opens only with its
// service URL, since a bundle whose getter this cannot find is better left on v1.
artifact := bytes.Contains(data, []byte(artifactRefusal))
cache := bytes.Contains(data, []byte(CacheServiceV2Env)) && serviceURLBranches.Match(data)
if !artifact && !cache {
return data, false
}
if !localhostTest.Match(data) {
return data, false
}
opened := localhostTest.ReplaceAllFunc(data, func(test []byte) []byte {
// Drop the hostname from the test rather than rewriting the call, so the bundle's own
// quoting survives and the result stays valid even inside a string literal.
return bytes.Replace(test, []byte(localhostHost), nil, 1)
})
if cache {
opened = serviceURLBranches.ReplaceAll(opened, cacheURLFirst)
}
return opened, true
}
-82
View File
@@ -1,82 +0,0 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Copyright 2022 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package runner
import (
"context"
"strings"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/container"
"gitea.dev/actionslib/pkg/model"
"github.com/kballard/go-shellquote"
)
type stepDocker struct {
Step *model.Step
RunContext *RunContext
env map[string]string
}
func (sd *stepDocker) pre() common.Executor { return common.NewPipelineExecutor() }
func (sd *stepDocker) main() common.Executor {
sd.env = map[string]string{}
return runStepExecutor(sd, stepStageMain, sd.runUsesContainer())
}
func (sd *stepDocker) post() common.Executor { return common.NewPipelineExecutor() }
func (sd *stepDocker) getRunContext() *RunContext {
return sd.RunContext
}
func (sd *stepDocker) getGithubContext(ctx context.Context) *model.GithubContext {
return sd.getRunContext().getGithubContext(ctx)
}
func (sd *stepDocker) getStepModel() *model.Step {
return sd.Step
}
func (sd *stepDocker) getEnv() *map[string]string {
return &sd.env
}
func (sd *stepDocker) getIfExpression(_ context.Context, _ stepStage) string {
return sd.Step.If.Value
}
func (sd *stepDocker) runUsesContainer() common.Executor {
rc := sd.RunContext
step := sd.Step
return func(ctx context.Context) error {
image := strings.TrimPrefix(step.Uses, "docker://")
eval := rc.NewExpressionEvaluator(ctx)
cmd, err := shellquote.Split(eval.Interpolate(ctx, step.With["args"]))
if err != nil {
return err
}
var entrypoint []string
if entry := eval.Interpolate(ctx, step.With["entrypoint"]); entry != "" {
entrypoint = []string{entry}
}
stepContainer := newStepContainer(ctx, sd, image, cmd, entrypoint, "")
return common.NewPipelineExecutor(
stepContainer.Pull(rc.Config.ForcePull),
stepContainer.Remove(),
stepContainer.Create(rc.Config.ContainerCapAdd, rc.Config.ContainerCapDrop),
stepContainer.Start(true),
).Finally(stepContainer.Close())(ctx)
}
}
var ContainerNewContainer = container.NewContainer
-8
View File
@@ -1,8 +0,0 @@
name: test network setup
on: push
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Resolve the container hostname
run: getent hosts "$(hostname -f)"
-28
View File
@@ -1,28 +0,0 @@
name: output
on: push
jobs:
build_output:
runs-on: ubuntu-latest
steps:
- id: set_1
run: |
echo "::set-output name=var_1::$(echo var1)"
echo "::set-output name=var_2::$(echo var2)"
- id: set_2
run: |
echo "::set-output name=var_3::$(echo var3)"
outputs:
variable_1: ${{ steps.set_1.outputs.var_1 }}
variable_2: ${{ steps.set_1.outputs.var_2 }}
variable_3: ${{ steps.set_2.outputs.var_3 }}
build:
needs: build_output
runs-on: ubuntu-latest
steps:
- name: Check outputs
run: |
test "${{ needs.build_output.outputs.variable_1 }}" = var1
test "${{ needs.build_output.outputs.variable_2 }}" = var2
test "${{ needs.build_output.outputs.variable_3 }}" = var3
-17
View File
@@ -1,17 +0,0 @@
name: steps context
on: push
jobs:
check:
runs-on: ubuntu-latest
steps:
- id: first
run: exit 0
- id: second
continue-on-error: true
run: exit 1
- run: |
test '${{ steps.first.conclusion }}' = success
test '${{ steps.second.conclusion }}' = success
test '${{ steps.first.outcome }}' = success
test '${{ steps.second.outcome }}' = failure
-30
View File
@@ -1,30 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build e2e
package e2e
import (
"testing"
)
func testActionsCacheRoundTrip(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
v2 bool
}{
{name: "cache_v2", v2: true},
{name: "cache_v1", v2: false},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
api, repo, _ := startIsolatedScenario(t, "cache.yml", "e2e-cache", runnerOptions{cacheV2: &tc.v2})
wfRun := waitForRun(t, api, repo)
requireSuccess(t, api, repo, wfRun.ID)
})
}
}
-74
View File
@@ -1,74 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build e2e
package e2e
import (
"context"
"errors"
"net/http"
"testing"
"time"
)
func testRunCancellation(t *testing.T) {
t.Parallel()
ctx := t.Context()
api, repo := newScenario(t)
err := api.CancelRun(ctx, repo, 0)
cancelSupported := !errors.Is(err, ErrCancelUnsupported)
var response statusError
if cancelSupported && (!errors.As(err, &response) || response.code != http.StatusNotFound) {
t.Fatalf("probe run cancellation: %v", err)
}
pushWorkflow(t, api, repo, "cancel.yml")
wfRun := waitForRun(t, api, repo)
waitForRunningJobLog(t, api, repo, wfRun.ID, "e2e-live-log-marker")
if !cancelSupported {
requireSuccess(t, api, repo, wfRun.ID)
return
}
if err := api.CancelRun(ctx, repo, wfRun.ID); err != nil {
dumpRunLogs(t, api, repo, wfRun.ID)
t.Fatalf("cancel run: %v", err)
}
completed, err := api.WaitForRunConclusion(ctx, repo, wfRun.ID, time.Minute)
if err != nil {
dumpRunLogs(t, api, repo, wfRun.ID)
t.Fatalf("cancelled run did not finish promptly: %v", err)
}
if completed.Conclusion != "cancelled" {
dumpRunLogs(t, api, repo, wfRun.ID)
t.Fatalf("cancelled run concluded %q, want cancelled", completed.Conclusion)
}
}
func waitForRunningJobLog(t *testing.T, api *GiteaAPI, repo string, runID int64, substr string) {
t.Helper()
ctx, cancel := context.WithTimeout(t.Context(), time.Minute)
defer cancel()
for {
jobs, err := api.Jobs(ctx, repo, runID)
if err != nil {
t.Fatalf("list jobs: %v", err)
}
if len(jobs) > 0 && jobs[0].Status == "in_progress" {
logs, err := api.JobLogs(ctx, repo, jobs[0].ID)
if err == nil && commandRow(logs, substr) == substr {
return
}
}
select {
case <-ctx.Done():
t.Fatalf("running job for run %d never logged %q", runID, substr)
case <-time.After(pollInterval):
}
}
}
-99
View File
@@ -1,99 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build e2e
package e2e
import (
"os"
"strings"
"testing"
)
func TestCompatibility(t *testing.T) {
if skipReason != "" {
t.Skip(skipReason)
}
sharedPoller := startRunner(t, "", "ubuntu-latest", runnerOptions{capacity: 16})
t.Cleanup(func() {
select {
case <-sharedPoller.Done():
t.Error("shared runner stopped")
default:
}
})
t.Run("payloads", testPayloads)
t.Run("cache", testActionsCacheRoundTrip)
t.Run("cancellation_and_log_streaming", testRunCancellation)
t.Run("dispatch", testWorkflowDispatch)
t.Run("ephemeral", testEphemeralRunner)
}
func testPayloads(t *testing.T) {
t.Parallel()
ctx := t.Context()
const secretValue = "s3cr3t-value-xyz"
api, repo := newScenario(t)
if err := api.CreateSecret(ctx, repo, "FOO", secretValue); err != nil {
t.Fatalf("create secret: %v", err)
}
if err := api.CreateVariable(ctx, repo, "GREETING", "hello-from-a-variable"); err != nil {
t.Fatalf("create variable: %v", err)
}
if err := api.CreateVariable(ctx, repo, "E2E_SERVICE_IMAGE", os.Getenv("SERVICE_IMAGE")); err != nil {
t.Fatalf("create service image variable: %v", err)
}
pushWorkflow(t, api, repo, "payloads.yml")
wfRun := waitForRun(t, api, repo)
requireSuccess(t, api, repo, wfRun.ID)
logs := runLogs(t, api, repo, wfRun.ID)
for _, want := range []string{
"hello-from-a-variable",
"plain-100%-done;-[bracket]",
"multiline-first",
"multiline-second",
"notice-payload-here",
"warning-payload-here",
"error-payload-here",
"group-payload-here",
"inside-the-group",
"received=produced-value-42",
"cell-a-1",
"cell-a-2",
"cell-b-1",
"cell-b-2",
} {
if !strings.Contains(logs, want) {
t.Errorf("stored logs are missing %q", want)
}
}
if forwarded := commandRow(logs, "::notice::encoded-first"); forwarded != "::notice::encoded-first%0Aencoded-second" {
t.Errorf("forwarded command row is %q, want its payload passed through unchanged", forwarded)
}
if !strings.Contains(logs, "plain-100%25-done") {
t.Error("the emitted command did not escape %")
}
if strings.Contains(logs, secretValue) || !strings.Contains(logs, "***") {
t.Error("job logs did not mask the secret")
}
if t.Failed() {
t.Logf("full stored logs:\n%s", logs)
}
}
func commandRow(logs, prefix string) string {
for line := range strings.SplitSeq(logs, "\n") {
_, payload, found := strings.Cut(line, "Z ")
if found && strings.HasPrefix(strings.TrimRight(payload, "\r"), prefix) {
return strings.TrimRight(payload, "\r")
}
}
return ""
}
-54
View File
@@ -1,54 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build e2e
package e2e
import (
"context"
"testing"
"time"
)
func testWorkflowDispatch(t *testing.T) {
t.Parallel()
ctx := t.Context()
api, repo := newScenario(t)
pushWorkflow(t, api, repo, "dispatch.yml")
branch, err := api.DefaultBranch(ctx, repo)
if err != nil {
t.Fatalf("get default branch: %v", err)
}
inputs := map[string]string{"subject": "dispatch-input-value"}
if err := waitForDispatch(ctx, api, repo, branch, inputs); err != nil {
t.Fatalf("dispatch workflow: %v", err)
}
dispatched := waitForRun(t, api, repo)
if dispatched.Event != "workflow_dispatch" {
t.Fatalf("run event is %q, want workflow_dispatch", dispatched.Event)
}
requireSuccess(t, api, repo, dispatched.ID)
}
// Gitea indexes workflows asynchronously after the Contents API commit.
func waitForDispatch(ctx context.Context, api *GiteaAPI, repo, branch string, inputs map[string]string) error {
ctx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
var lastErr error
for {
lastErr = api.DispatchWorkflow(ctx, repo, "dispatch.yml", branch, inputs)
if lastErr == nil {
return nil
}
select {
case <-ctx.Done():
return lastErr
case <-time.After(pollInterval):
}
}
}
-8
View File
@@ -1,8 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build e2e
// Package e2e tests the runner against a real gitea/gitea container (build tag e2e).
// Run with `make test-e2e`; see DEVELOPMENT.md.
package e2e
-28
View File
@@ -1,28 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build e2e
package e2e
import (
"testing"
"time"
)
func testEphemeralRunner(t *testing.T) {
t.Parallel()
api, repo, poller := startIsolatedScenario(t, "ephemeral.yml", "e2e-ephemeral", runnerOptions{ephemeral: true})
wfRun := waitForRun(t, api, repo)
requireSuccess(t, api, repo, wfRun.ID)
select {
case <-poller.Done():
case <-time.After(5 * time.Second):
t.Fatal("ephemeral runner was not deleted")
}
if !poller.Unregistered() {
t.Fatal("ephemeral runner stopped without being deleted")
}
}
-217
View File
@@ -1,217 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build e2e
package e2e
import (
"bytes"
"context"
"encoding/base64"
"encoding/json/v2"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
)
const pollInterval = 250 * time.Millisecond
type GiteaAPI struct {
baseURL string
token string
}
type ActionRun struct {
ID int64 `json:"id"`
Status string `json:"status"`
Conclusion string `json:"conclusion"`
Event string `json:"event"`
}
type ActionJob struct {
ID int64 `json:"id"`
Name string `json:"name"`
Status string `json:"status"`
Conclusion string `json:"conclusion"`
}
func (a *GiteaAPI) CreateRepo(ctx context.Context, name string) error {
body := map[string]any{"name": name, "auto_init": true}
return a.doJSON(ctx, http.MethodPost, "/api/v1/user/repos", body, nil)
}
func (a *GiteaAPI) CreateFile(ctx context.Context, repo, path, content, message string) error {
body := map[string]any{
"content": base64.StdEncoding.EncodeToString([]byte(content)),
"message": message,
}
url := fmt.Sprintf("/api/v1/repos/%s/%s/contents/%s", giteaAdminUser, repo, path)
return a.doJSON(ctx, http.MethodPost, url, body, nil)
}
func (a *GiteaAPI) CreateSecret(ctx context.Context, repo, name, value string) error {
body := map[string]any{"data": value}
url := fmt.Sprintf("/api/v1/repos/%s/%s/actions/secrets/%s", giteaAdminUser, repo, name)
return a.doJSON(ctx, http.MethodPut, url, body, nil)
}
func (a *GiteaAPI) DefaultBranch(ctx context.Context, repo string) (string, error) {
var resp struct {
DefaultBranch string `json:"default_branch"`
}
url := fmt.Sprintf("/api/v1/repos/%s/%s", giteaAdminUser, repo)
if err := a.doJSON(ctx, http.MethodGet, url, nil, &resp); err != nil {
return "", err
}
return resp.DefaultBranch, nil
}
func (a *GiteaAPI) CreateVariable(ctx context.Context, repo, name, value string) error {
body := map[string]any{"value": value}
url := fmt.Sprintf("/api/v1/repos/%s/%s/actions/variables/%s", giteaAdminUser, repo, name)
return a.doJSON(ctx, http.MethodPost, url, body, nil)
}
func (a *GiteaAPI) DispatchWorkflow(ctx context.Context, repo, workflowID, ref string, inputs map[string]string) error {
body := map[string]any{"ref": ref}
if len(inputs) > 0 {
body["inputs"] = inputs
}
url := fmt.Sprintf("/api/v1/repos/%s/%s/actions/workflows/%s/dispatches", giteaAdminUser, repo, workflowID)
return a.doJSON(ctx, http.MethodPost, url, body, nil)
}
var ErrCancelUnsupported = errors.New("run cancellation is unsupported by this gitea version")
func (a *GiteaAPI) CancelRun(ctx context.Context, repo string, runID int64) error {
url := fmt.Sprintf("/api/v1/repos/%s/%s/actions/runs/%d/cancel", giteaAdminUser, repo, runID)
err := a.doJSON(ctx, http.MethodPost, url, nil, nil)
var status statusError
if errors.As(err, &status) && status.routeAbsent() {
return ErrCancelUnsupported
}
return err
}
func (a *GiteaAPI) Runs(ctx context.Context, repo string) ([]ActionRun, error) {
var resp struct {
WorkflowRuns []ActionRun `json:"workflow_runs"`
}
url := fmt.Sprintf("/api/v1/repos/%s/%s/actions/runs?limit=1", giteaAdminUser, repo)
if err := a.doJSON(ctx, http.MethodGet, url, nil, &resp); err != nil {
return nil, err
}
return resp.WorkflowRuns, nil
}
func (a *GiteaAPI) WaitForRunConclusion(ctx context.Context, repo string, runID int64, timeout time.Duration) (*ActionRun, error) {
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
url := fmt.Sprintf("/api/v1/repos/%s/%s/actions/runs/%d", giteaAdminUser, repo, runID)
for {
var run ActionRun
if err := a.doJSON(ctx, http.MethodGet, url, nil, &run); err != nil {
return nil, err
}
if run.Status == "completed" {
return &run, nil
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("run %d did not complete within %s (last status %q): %w", runID, timeout, run.Status, ctx.Err())
case <-time.After(pollInterval):
}
}
}
func (a *GiteaAPI) Jobs(ctx context.Context, repo string, runID int64) ([]ActionJob, error) {
var resp struct {
Jobs []ActionJob `json:"jobs"`
}
url := fmt.Sprintf("/api/v1/repos/%s/%s/actions/runs/%d/jobs", giteaAdminUser, repo, runID)
if err := a.doJSON(ctx, http.MethodGet, url, nil, &resp); err != nil {
return nil, err
}
return resp.Jobs, nil
}
func (a *GiteaAPI) JobLogs(ctx context.Context, repo string, jobID int64) (string, error) {
url := fmt.Sprintf("/api/v1/repos/%s/%s/actions/jobs/%d/logs", giteaAdminUser, repo, jobID)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, a.baseURL+url, nil)
if err != nil {
return "", err
}
req.Header.Set("Authorization", "token "+a.token)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode >= 300 {
return "", fmt.Errorf("GET %s: %d: %s", url, resp.StatusCode, body)
}
return string(body), nil
}
func (a *GiteaAPI) doJSON(ctx context.Context, method, path string, reqBody, respBody any) error {
var bodyReader io.Reader
if reqBody != nil {
encoded, err := json.Marshal(reqBody)
if err != nil {
return err
}
bodyReader = bytes.NewReader(encoded)
}
req, err := http.NewRequestWithContext(ctx, method, a.baseURL+path, bodyReader)
if err != nil {
return err
}
req.Header.Set("Authorization", "token "+a.token)
if reqBody != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 300 {
return statusError{method: method, path: path, code: resp.StatusCode, body: string(body)}
}
if respBody != nil && len(body) > 0 {
return json.Unmarshal(body, respBody)
}
return nil
}
type statusError struct {
method, path string
code int
body string
}
func (e statusError) Error() string {
return fmt.Sprintf("%s %s: %d: %s", e.method, e.path, e.code, e.body)
}
func (e statusError) routeAbsent() bool {
return e.code == http.StatusNotFound && strings.TrimSpace(e.body) == "404 page not found"
}
-365
View File
@@ -1,365 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build e2e
package e2e
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/netip"
"os"
"strconv"
"strings"
"time"
"gitea.com/gitea/runner/act/container"
apicontainer "github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/network"
mobyclient "github.com/moby/moby/client"
)
const (
giteaAdminUser = "e2e-admin"
giteaAdminMail = "e2e-admin@example.com"
)
type GiteaFixture struct {
cli mobyclient.APIClient
id string
image string
version string
baseURL string
network string // set in container mode; jobs must join it
adminToken string
}
func dockerClient(ctx context.Context) (mobyclient.APIClient, error) {
cli, err := container.GetDockerClient(ctx)
if err != nil {
return nil, err
}
if _, err := cli.Ping(ctx, mobyclient.PingOptions{}); err != nil {
return nil, fmt.Errorf("docker daemon unreachable: %w", err)
}
return cli, nil
}
// Prefer docker bridge gateway, then LAN IP; never 127.0.0.1 (ACTIONS_RUNTIME_URL must reach the host from job containers).
func hostAddress(ctx context.Context, cli mobyclient.APIClient) netip.Addr {
if gateway, ok := bridgeGateway(ctx, cli); ok && bindable(gateway) {
return gateway
}
loopback := netip.AddrFrom4([4]byte{127, 0, 0, 1})
conn, err := net.Dial("udp", "192.0.2.1:80") // TEST-NET-1: routing table only
if err != nil {
return loopback
}
defer conn.Close()
addr, ok := conn.LocalAddr().(*net.UDPAddr)
if !ok {
return loopback
}
return addr.AddrPort().Addr().Unmap()
}
func bridgeGateway(ctx context.Context, cli mobyclient.APIClient) (netip.Addr, bool) {
inspected, err := cli.NetworkInspect(ctx, "bridge", mobyclient.NetworkInspectOptions{})
if err != nil {
return netip.Addr{}, false
}
for _, cfg := range inspected.Network.IPAM.Config {
if cfg.Gateway.Is4() {
return cfg.Gateway, true
}
}
return netip.Addr{}, false
}
// When tests run inside a container (sibling docker daemon), join that network and address by name.
func selfNetwork(ctx context.Context, cli mobyclient.APIClient) (string, bool) {
if _, err := os.Stat("/.dockerenv"); err != nil {
return "", false
}
id, err := os.ReadFile("/proc/sys/kernel/hostname")
if err != nil {
id, err = os.ReadFile("/etc/hostname")
}
if err != nil {
return "", false
}
inspected, err := cli.ContainerInspect(ctx, strings.TrimSpace(string(id)), mobyclient.ContainerInspectOptions{})
if err != nil || inspected.Container.NetworkSettings == nil {
return "", false
}
for name := range inspected.Container.NetworkSettings.Networks {
if name != "host" && name != "none" && name != "bridge" { // only user-defined nets have DNS
return name, true
}
}
return "", false
}
func bindable(addr netip.Addr) bool {
l, err := net.Listen("tcp", net.JoinHostPort(addr.String(), "0"))
if err != nil {
return false
}
_ = l.Close()
return true
}
func freeHostPort(host string) (int, error) {
l, err := net.Listen("tcp", net.JoinHostPort(host, "0"))
if err != nil {
return 0, err
}
defer l.Close()
addr, ok := l.Addr().(*net.TCPAddr)
if !ok {
return 0, fmt.Errorf("unexpected listener address type %T", l.Addr())
}
return addr.Port, nil
}
func StartGitea(ctx context.Context, cli mobyclient.APIClient) (*GiteaFixture, error) {
closeClient := true
defer func() {
if closeClient {
_ = cli.Close()
}
}()
image := os.Getenv("E2E_GITEA_IMAGE")
if image == "" {
return nil, errors.New("E2E_GITEA_IMAGE is not set")
}
name := fmt.Sprintf("gitea-runner-e2e-%d", time.Now().UnixNano())
containerPort := network.MustParsePort("3000/tcp")
var (
baseURL string
hostConfig = &apicontainer.HostConfig{}
netConfig *network.NetworkingConfig
sharedNet, _ = selfNetwork(ctx, cli)
)
if sharedNet != "" {
baseURL = fmt.Sprintf("http://%s:3000", name)
netConfig = &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{sharedNet: {}},
}
} else {
host := hostAddress(ctx, cli)
port, err := freeHostPort(host.String())
if err != nil {
return nil, fmt.Errorf("find a free host port: %w", err)
}
baseURL = fmt.Sprintf("http://%s:%d", host, port)
hostConfig.PortBindings = network.PortMap{
containerPort: []network.PortBinding{{HostIP: host, HostPort: strconv.Itoa(port)}},
}
}
if _, err := cli.ImageInspect(ctx, image); err != nil {
return nil, fmt.Errorf("inspect gitea image %s: %w", image, err)
}
resp, err := cli.ContainerCreate(ctx, mobyclient.ContainerCreateOptions{
Config: &apicontainer.Config{
Image: image,
Env: []string{
"GITEA__security__INSTALL_LOCK=true",
"GITEA__database__DB_TYPE=sqlite3",
"GITEA__actions__ENABLED=true",
"GITEA__server__ROOT_URL=" + baseURL + "/",
},
ExposedPorts: network.PortSet{containerPort: struct{}{}},
},
HostConfig: hostConfig,
NetworkingConfig: netConfig,
Name: name,
})
if err != nil {
return nil, fmt.Errorf("create gitea container: %w", err)
}
f := &GiteaFixture{cli: cli, id: resp.ID, image: image, baseURL: baseURL, network: sharedNet}
closeClient = false
if _, err := cli.ContainerStart(ctx, f.id, mobyclient.ContainerStartOptions{}); err != nil {
_ = f.Close(ctx)
return nil, fmt.Errorf("start gitea container: %w", err)
}
if err := f.waitHealthy(ctx); err != nil {
_ = f.Close(ctx)
return nil, fmt.Errorf("gitea did not become healthy: %w", err)
}
if err := f.bootstrapAdmin(ctx); err != nil {
_ = f.Close(ctx)
return nil, fmt.Errorf("bootstrap gitea admin: %w", err)
}
if err := f.readVersion(ctx); err != nil {
_ = f.Close(ctx)
return nil, fmt.Errorf("read gitea version: %w", err)
}
return f, nil
}
func (f *GiteaFixture) readVersion(ctx context.Context) error {
var body struct {
Version string `json:"version"`
}
if err := f.doJSON(ctx, http.MethodGet, "/api/v1/version", nil, &body); err != nil {
return err
}
f.version = body.Version
return nil
}
func (f *GiteaFixture) waitHealthy(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, 90*time.Second)
defer cancel()
url := f.baseURL + "/api/healthz"
for {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err == nil {
resp, err := http.DefaultClient.Do(req)
if err == nil {
_ = resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return nil
}
}
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(pollInterval):
}
}
}
func (f *GiteaFixture) bootstrapAdmin(ctx context.Context) error {
password := randomToken(16)
if _, err := f.exec(ctx, []string{
"gitea", "admin", "user", "create",
"--username", giteaAdminUser,
"--password", password,
"--email", giteaAdminMail,
"--admin",
"--must-change-password=false",
}); err != nil {
return fmt.Errorf("create admin user: %w", err)
}
out, err := f.exec(ctx, []string{
"gitea", "admin", "user", "generate-access-token",
"--username", giteaAdminUser,
"--scopes", "all",
"-t", "e2e-admin-token",
})
if err != nil {
return fmt.Errorf("generate admin token: %w", err)
}
token := extractToken(out)
if token == "" {
return fmt.Errorf("could not parse access token from CLI output: %q", out)
}
f.adminToken = token
return nil
}
// Last field of `gitea admin user generate-access-token` output (format varies by release).
func extractToken(cliOutput string) string {
fields := strings.Fields(cliOutput)
if len(fields) == 0 {
return ""
}
return fields[len(fields)-1]
}
func randomToken(n int) string {
b := make([]byte, n)
_, _ = rand.Read(b)
return hex.EncodeToString(b)
}
func (f *GiteaFixture) exec(ctx context.Context, cmd []string) (string, error) {
created, err := f.cli.ExecCreate(ctx, f.id, mobyclient.ExecCreateOptions{
Cmd: cmd,
User: "git", // gitea refuses root
AttachStdout: true,
AttachStderr: true,
})
if err != nil {
return "", err
}
attached, err := f.cli.ExecAttach(ctx, created.ID, mobyclient.ExecAttachOptions{})
if err != nil {
return "", err
}
defer attached.Close()
var out bytes.Buffer
if _, err := io.Copy(&out, attached.Reader); err != nil {
return "", err
}
inspected, err := f.cli.ExecInspect(ctx, created.ID, mobyclient.ExecInspectOptions{})
if err != nil {
return "", err
}
if inspected.ExitCode != 0 {
return "", fmt.Errorf("exec %v exited %d: %s", cmd, inspected.ExitCode, out.String())
}
return out.String(), nil
}
func (f *GiteaFixture) RegistrationToken(ctx context.Context, repo string) (string, error) {
var body struct {
Token string `json:"token"`
}
path := "/api/v1/admin/actions/runners/registration-token"
if repo != "" {
path = fmt.Sprintf("/api/v1/repos/%s/%s/actions/runners/registration-token", giteaAdminUser, repo)
}
if err := f.doJSON(ctx, http.MethodPost, path, nil, &body); err != nil {
return "", err
}
return body.Token, nil
}
func (f *GiteaFixture) doJSON(ctx context.Context, method, path string, reqBody, respBody any) error {
api := &GiteaAPI{baseURL: f.baseURL, token: f.adminToken}
return api.doJSON(ctx, method, path, reqBody, respBody)
}
func (f *GiteaFixture) Close(ctx context.Context) error {
if f.id == "" {
return f.cli.Close()
}
_, removeErr := f.cli.ContainerRemove(ctx, f.id, mobyclient.ContainerRemoveOptions{Force: true})
return errors.Join(removeErr, f.cli.Close())
}
-162
View File
@@ -1,162 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build e2e
package e2e
import (
"context"
"fmt"
"os"
"regexp"
"strings"
"testing"
"time"
"gitea.com/gitea/runner/internal/app/poll"
)
const runTimeout = 3 * time.Minute
var nonRepoChars = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)
func repoName(t *testing.T) string {
return strings.ToLower(nonRepoChars.ReplaceAllString(t.Name(), "-"))
}
var fixture *GiteaFixture
var skipReason string
func TestMain(m *testing.M) {
os.Exit(runSuite(m))
}
func runSuite(m *testing.M) int {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
cli, err := dockerClient(ctx)
if err != nil {
skipReason = fmt.Sprintf("docker unavailable: %v", err)
return m.Run()
}
f, err := StartGitea(ctx, cli)
if err != nil {
fmt.Fprintf(os.Stderr, "start gitea fixture: %v\n", err)
return 1
}
fixture = f
defer func() { _ = fixture.Close(context.Background()) }()
fmt.Fprintf(os.Stderr, "gitea fixture: image=%s version=%s\n", fixture.image, fixture.version)
return m.Run()
}
func newScenario(t *testing.T) (*GiteaAPI, string) {
t.Helper()
repo := repoName(t)
api := &GiteaAPI{baseURL: fixture.baseURL, token: fixture.adminToken}
if err := api.CreateRepo(t.Context(), repo); err != nil {
t.Fatalf("create repo: %v", err)
}
return api, repo
}
func startIsolatedScenario(t *testing.T, workflow, label string, options runnerOptions) (*GiteaAPI, string, *poll.Poller) {
t.Helper()
api, repo := newScenario(t)
poller := startRunner(t, repo, label, options)
pushWorkflow(t, api, repo, workflow)
return api, repo, poller
}
func pushWorkflow(t *testing.T, api *GiteaAPI, repo, workflow string) {
t.Helper()
content, err := os.ReadFile("testdata/workflows/" + workflow)
if err != nil {
t.Fatalf("read workflow fixture %s: %v", workflow, err)
}
if err := api.CreateFile(t.Context(), repo, ".gitea/workflows/"+workflow, string(content), "add "+workflow); err != nil {
t.Fatalf("push workflow %s: %v", workflow, err)
}
}
func requireSuccess(t *testing.T, api *GiteaAPI, repo string, runID int64) {
t.Helper()
completed, err := api.WaitForRunConclusion(t.Context(), repo, runID, runTimeout)
if err != nil {
dumpRunLogs(t, api, repo, runID)
t.Fatalf("wait for run: %v", err)
}
if completed.Conclusion != "success" {
dumpRunLogs(t, api, repo, runID)
t.Fatalf("run concluded %q, want success", completed.Conclusion)
}
}
func runLogs(t *testing.T, api *GiteaAPI, repo string, runID int64) string {
t.Helper()
ctx := t.Context()
jobs, err := api.Jobs(ctx, repo, runID)
if err != nil {
t.Fatalf("list jobs: %v", err)
}
var all strings.Builder
for _, job := range jobs {
logs, err := api.JobLogs(ctx, repo, job.ID)
if err != nil {
t.Fatalf("job logs for %s: %v", job.Name, err)
}
all.WriteString(logs)
}
return all.String()
}
func dumpRunLogs(t *testing.T, api *GiteaAPI, repo string, runID int64) {
t.Helper()
ctx := t.Context()
jobs, err := api.Jobs(ctx, repo, runID)
if err != nil {
t.Logf("dump run %d: list jobs: %v", runID, err)
return
}
for _, job := range jobs {
logs, err := api.JobLogs(ctx, repo, job.ID)
if err != nil {
t.Logf("job %q (%s): logs unavailable: %v", job.Name, job.Conclusion, err)
continue
}
t.Logf("job %q concluded %q:\n%s", job.Name, job.Conclusion, logs)
}
}
func waitForRun(t *testing.T, api *GiteaAPI, repo string) *ActionRun {
t.Helper()
ctx, cancel := context.WithTimeout(t.Context(), time.Minute)
defer cancel()
for {
runs, err := api.Runs(ctx, repo)
if err != nil {
t.Fatalf("get latest run: %v", err)
}
if len(runs) > 0 {
return &runs[0]
}
select {
case <-ctx.Done():
t.Fatalf("no run appeared for %s within timeout", repo)
case <-time.After(pollInterval):
}
}
}
-120
View File
@@ -1,120 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build e2e
package e2e
import (
"context"
"errors"
"os"
"testing"
"time"
"gitea.com/gitea/runner/internal/app/poll"
"gitea.com/gitea/runner/internal/app/run"
"gitea.com/gitea/runner/internal/pkg/client"
"gitea.com/gitea/runner/internal/pkg/config"
"gitea.com/gitea/runner/internal/pkg/labels"
"connectrpc.com/connect"
pingv1 "gitea.dev/actionslib/ping/v1"
runnerv1 "gitea.dev/actionslib/runner/v1"
)
type runnerOptions struct {
capacity int
ephemeral bool
cacheV2 *bool
}
func startRunner(t *testing.T, repo, labelName string, options runnerOptions) *poll.Poller {
t.Helper()
ctx := t.Context()
token, err := fixture.RegistrationToken(ctx, repo)
if err != nil {
t.Fatalf("get registration token: %v", err)
}
cfg, err := config.LoadDefault("")
if err != nil {
t.Fatalf("load default config: %v", err)
}
if options.cacheV2 != nil {
cfg.Cache.V2 = options.cacheV2
}
cfg.Container.DockerHost = os.Getenv("DOCKER_HOST")
if cfg.Container.DockerHost == "" {
cfg.Container.DockerHost = "unix:///var/run/docker.sock"
}
cfg.Cache.Dir = t.TempDir() + "/cache"
cfg.Runner.Insecure = true
cfg.Runner.FetchInterval = 250 * time.Millisecond // faster than prod defaults for local fixture
cfg.Runner.FetchIntervalMax = 250 * time.Millisecond
cfg.Runner.StateReportInterval = 500 * time.Millisecond // so cancel reaches the job quickly
cfg.Runner.LogReportInterval = 500 * time.Millisecond
cfg.Runner.Capacity = max(options.capacity, 1)
if fixture.network != "" {
cfg.Container.Network = fixture.network
}
rawLabel := labelName + ":docker://" + os.Getenv("E2E_JOB_IMAGE")
label, err := labels.Parse(rawLabel)
if err != nil {
t.Fatalf("parse label %q: %v", rawLabel, err)
}
labelNames := []string{label.Name}
pingCli := client.New(fixture.baseURL, cfg.Runner.Insecure, "", "", config.RequestTimeout)
if _, err := pingCli.Ping(ctx, connect.NewRequest(&pingv1.PingRequest{Data: t.Name()})); err != nil {
t.Fatalf("ping %s: %v", fixture.baseURL, err)
}
regResp, err := pingCli.Register(ctx, connect.NewRequest(&runnerv1.RegisterRequest{
Name: t.Name(),
Token: token,
Version: "e2e",
Labels: labelNames,
Ephemeral: options.ephemeral,
Capabilities: run.RunnerCapabilities(),
}))
if err != nil {
t.Fatalf("register runner: %v", err)
}
if options.ephemeral && !regResp.Msg.Runner.Ephemeral {
t.Fatal("gitea did not grant ephemeral registration")
}
reg := &config.Registration{
ID: regResp.Msg.Runner.Id,
UUID: regResp.Msg.Runner.Uuid,
Name: regResp.Msg.Runner.Name,
Token: regResp.Msg.Runner.Token,
Address: fixture.baseURL,
Labels: []string{rawLabel},
Ephemeral: regResp.Msg.Runner.Ephemeral,
}
cli := client.New(fixture.baseURL, cfg.Runner.Insecure, reg.UUID, reg.Token, config.RequestTimeout)
runner := run.NewRunner(cfg, reg, cli)
declResp, err := runner.Declare(ctx, labelNames)
if err != nil {
_ = runner.Close()
t.Fatalf("declare runner: %v", err)
}
runner.SetCapabilitiesFromDeclare(declResp)
poller := poll.New(cfg, cli, runner)
go poller.Poll()
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := errors.Join(poller.Shutdown(ctx), runner.Close()); err != nil {
t.Logf("runner shutdown: %v", err)
}
})
return poller
}
-26
View File
@@ -1,26 +0,0 @@
name: cache
on: push
jobs:
save:
runs-on: e2e-cache
steps:
- run: |
mkdir -p cached
echo "cached-payload" > cached/data.txt
- uses: actions/cache@v4
with:
path: cached
key: e2e-cache-${{ github.run_id }}
restore:
needs: save
runs-on: e2e-cache
steps:
- uses: actions/cache@v4
id: restore
with:
path: cached
key: e2e-cache-${{ github.run_id }}
- run: |
echo "cache-hit=${{ steps.restore.outputs.cache-hit }}"
test "${{ steps.restore.outputs.cache-hit }}" = "true"
grep -q cached-payload cached/data.txt
-9
View File
@@ -1,9 +0,0 @@
name: cancel
on: push
jobs:
slow:
runs-on: ubuntu-latest
steps:
- run: |
echo e2e-live-log-marker
timeout 2s tail -f /dev/null || true
-11
View File
@@ -1,11 +0,0 @@
name: dispatch
on:
workflow_dispatch:
inputs:
subject:
required: true
jobs:
greet:
runs-on: ubuntu-latest
steps:
- run: test "${{ inputs.subject }}" = dispatch-input-value
-7
View File
@@ -1,7 +0,0 @@
name: ephemeral
on: push
jobs:
hello:
runs-on: e2e-ephemeral
steps:
- run: echo hello
-53
View File
@@ -1,53 +0,0 @@
name: payloads
on: push
jobs:
verify:
runs-on: ubuntu-latest
services:
web:
image: ${{ vars.E2E_SERVICE_IMAGE }}
steps:
- run: |
echo 'plain-100%-done;-[bracket]'
printf 'multiline-first\nmultiline-second\n'
echo '::notice::notice-payload-here'
echo '::warning::warning-payload-here'
echo '::error::error-payload-here'
echo '::group::group-payload-here'
echo 'inside-the-group'
echo '::endgroup::'
echo '::notice::encoded-first%0Aencoded-second'
echo "the variable is ${{ vars.GREETING }}"
echo "the secret is ${{ secrets.FOO }}"
curl --connect-timeout 1 --max-time 2 --retry 30 --retry-delay 1 --retry-max-time 30 --retry-all-errors -fsS -o /dev/null http://web/
produce:
runs-on: ubuntu-latest
outputs:
token: ${{ steps.emit.outputs.token }}
steps:
- id: emit
run: echo "token=produced-value-42" >> "$GITHUB_OUTPUT"
- run: echo "artifact content" > payload.txt
- uses: actions/upload-artifact@v4
with:
name: payload
path: payload.txt
consume:
needs: produce
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
name: payload
- run: grep -q 'artifact content' payload.txt
- run: |
echo "received=${{ needs.produce.outputs.token }}"
test "${{ needs.produce.outputs.token }}" = "produced-value-42"
cell:
runs-on: ubuntu-latest
strategy:
matrix:
letter: [a, b]
number: [1, 2]
steps:
- run: echo "cell-${{ matrix.letter }}-${{ matrix.number }}"
-2
View File
@@ -11,8 +11,6 @@ Each example persists **two** things, and it is worth knowing which is which:
- `/data` is the runner's working directory. It holds the `.runner` registration file and, optionally, the config file — so the runner re-attaches to the server instead of registering again. - `/data` is the runner's working directory. It holds the `.runner` registration file and, optionally, the config file — so the runner re-attaches to the server instead of registering again.
- The Docker daemon's data root holds the images pulled for jobs (`/var/lib/docker` for the dind sidecar, `/home/rootless/.local/share/docker` for `dind-rootless`). It is *not* under `/data`. If you drop this volume, the examples still work, but the image cache is discarded whenever the pod is recreated and every job re-pulls its images. - The Docker daemon's data root holds the images pulled for jobs (`/var/lib/docker` for the dind sidecar, `/home/rootless/.local/share/docker` for `dind-rootless`). It is *not* under `/data`. If you drop this volume, the examples still work, but the image cache is discarded whenever the pod is recreated and every job re-pulls its images.
- Kubernetes SIGKILLs a pod 30s after SIGTERM by default, long before a job finishes and reports its result, which leaves tasks the server can only reap as zombies. The manifests raise `terminationGracePeriodSeconds` to three hours, matching the systemd example and the `runner.timeout` job ceiling; set `runner.shutdown_timeout` below that so the runner drains jobs within the window rather than being killed mid-cleanup.
Files in this directory: Files in this directory:
- [`dind-docker.yaml`](dind-docker.yaml) - [`dind-docker.yaml`](dind-docker.yaml)
-1
View File
@@ -56,7 +56,6 @@ spec:
app: runner app: runner
spec: spec:
restartPolicy: Always restartPolicy: Always
terminationGracePeriodSeconds: 10800 # keep above runner.shutdown_timeout, see README
volumes: volumes:
- name: docker-socket - name: docker-socket
emptyDir: {} emptyDir: {}
-38
View File
@@ -1,38 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package kubernetes_test
import (
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var gracePeriod = regexp.MustCompile(`terminationGracePeriodSeconds: (\d+)`)
// Without it Kubernetes SIGKILLs the pod 30s after SIGTERM, mid-job.
func TestManifestsSetTerminationGracePeriod(t *testing.T) {
files, err := filepath.Glob("*.yaml")
require.NoError(t, err)
require.NotEmpty(t, files)
for _, file := range files {
content, err := os.ReadFile(file)
require.NoError(t, err)
if !strings.Contains(string(content), "containers:") {
continue
}
match := gracePeriod.FindStringSubmatch(string(content))
require.NotNil(t, match, file)
seconds, err := strconv.Atoi(match[1])
require.NoError(t, err)
assert.GreaterOrEqual(t, seconds, 3600, file)
}
}
-1
View File
@@ -56,7 +56,6 @@ spec:
app: runner app: runner
spec: spec:
restartPolicy: Always restartPolicy: Always
terminationGracePeriodSeconds: 10800 # keep above runner.shutdown_timeout, see README
volumes: volumes:
- name: runner-data - name: runner-data
persistentVolumeClaim: persistentVolumeClaim:
@@ -33,7 +33,6 @@ spec:
app: runner app: runner
spec: spec:
restartPolicy: Always restartPolicy: Always
terminationGracePeriodSeconds: 10800 # keep above runner.shutdown_timeout, see README
volumes: volumes:
- name: docker-socket - name: docker-socket
emptyDir: {} emptyDir: {}
+34 -29
View File
@@ -1,8 +1,8 @@
module gitea.com/gitea/runner module gitea.com/gitea/runner
go 1.27 go 1.26.0
toolchain go1.27.0 toolchain go1.26.5
require ( require (
connectrpc.com/connect v1.20.0 connectrpc.com/connect v1.20.0
@@ -12,9 +12,8 @@ require (
github.com/containerd/errdefs v1.0.0 github.com/containerd/errdefs v1.0.0
github.com/creack/pty v1.1.24 github.com/creack/pty v1.1.24
github.com/distribution/reference v0.6.0 github.com/distribution/reference v0.6.0
github.com/docker/cli v29.7.2+incompatible github.com/docker/cli v29.6.2+incompatible
github.com/docker/go-connections v0.8.1 github.com/docker/go-connections v0.8.1
github.com/docker/go-units v0.5.0
github.com/go-git/go-billy/v5 v5.9.1 github.com/go-git/go-billy/v5 v5.9.1
github.com/go-git/go-git/v5 v5.19.2 github.com/go-git/go-git/v5 v5.19.2
github.com/gobwas/glob v0.2.3 github.com/gobwas/glob v0.2.3
@@ -23,7 +22,7 @@ require (
github.com/julienschmidt/httprouter v1.3.0 github.com/julienschmidt/httprouter v1.3.0
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/mattn/go-isatty v0.0.24 github.com/mattn/go-isatty v0.0.24
github.com/moby/go-archive v0.3.3 github.com/moby/go-archive v0.2.1
github.com/moby/moby/api v1.55.0 github.com/moby/moby/api v1.55.0
github.com/moby/moby/client v0.5.1 github.com/moby/moby/client v0.5.1
github.com/moby/patternmatcher v0.6.1 github.com/moby/patternmatcher v0.6.1
@@ -31,41 +30,43 @@ require (
github.com/opencontainers/selinux v1.15.1 github.com/opencontainers/selinux v1.15.1
github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_golang v1.24.1
github.com/prometheus/client_model v0.6.2 github.com/prometheus/client_model v0.6.2
github.com/sirupsen/logrus v1.10.1 github.com/sirupsen/logrus v1.9.4
github.com/spf13/cobra v1.10.2 github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10 github.com/spf13/pflag v1.0.10
github.com/stretchr/testify v1.12.1 github.com/stretchr/testify v1.11.1
github.com/timshannon/bolthold v0.0.0-20240314194003-30aac6950928 github.com/timshannon/bolthold v0.0.0-20240314194003-30aac6950928
go.etcd.io/bbolt v1.5.0 go.etcd.io/bbolt v1.5.0
go.yaml.in/yaml/v4 v4.0.0-rc.3 go.yaml.in/yaml/v4 v4.0.0-rc.3
golang.org/x/net v0.58.0 golang.org/x/net v0.57.0
golang.org/x/sync v0.22.0 golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0 golang.org/x/sys v0.47.0
golang.org/x/term v0.45.0 golang.org/x/term v0.45.0
golang.org/x/text v0.41.0 golang.org/x/text v0.40.0
google.golang.org/protobuf v1.36.12 google.golang.org/protobuf v1.36.11
gotest.tools/v3 v3.5.2 gotest.tools/v3 v3.5.2
tags.cncf.io/container-device-interface v1.1.0 tags.cncf.io/container-device-interface v1.1.0
) )
require ( require (
cyphar.com/go-pathrs v0.2.5 // indirect cyphar.com/go-pathrs v0.2.3 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/ProtonMail/go-crypto v1.4.1 // indirect github.com/ProtonMail/go-crypto v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
github.com/cloudflare/circl v1.6.5 // indirect github.com/cloudflare/circl v1.6.3 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect github.com/containerd/log v0.1.0 // indirect
github.com/cyphar/filepath-securejoin v0.7.0 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect
github.com/docker/docker-credential-helpers v0.9.8 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/docker-credential-helpers v0.9.6 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect
github.com/fatih/color v1.19.0 // indirect github.com/fatih/color v1.19.0 // indirect
github.com/felixge/httpsnoop v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
@@ -73,18 +74,19 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.6.0 // indirect github.com/kevinburke/ssh_config v1.6.0 // indirect
github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/compress v1.19.1 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-runewidth v0.0.28 // indirect github.com/mattn/go-runewidth v0.0.21 // indirect
github.com/mattn/go-shellwords v1.0.14 // indirect github.com/mattn/go-shellwords v1.0.12 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/sys/sequential v0.7.0 // indirect github.com/moby/sys/sequential v0.7.0 // indirect
github.com/moby/sys/user v0.4.1 // indirect github.com/moby/sys/user v0.4.1 // indirect
github.com/moby/sys/userns v0.2.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect
github.com/rhysd/actionlint v1.7.12 // indirect github.com/rhysd/actionlint v1.7.12 // indirect
@@ -97,11 +99,14 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
go.opentelemetry.io/otel v1.45.0 // indirect go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.45.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.45.0 // indirect go.opentelemetry.io/otel/sdk v1.43.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
golang.org/x/crypto v0.55.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.54.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
) )
+57 -60
View File
@@ -1,7 +1,7 @@
connectrpc.com/connect v1.20.0 h1:6TNDAB+WeNd2uolWNlYczB5E0KNNaVMNUEx8JEUsPmQ= connectrpc.com/connect v1.20.0 h1:6TNDAB+WeNd2uolWNlYczB5E0KNNaVMNUEx8JEUsPmQ=
connectrpc.com/connect v1.20.0/go.mod h1:A2ygJrukXwWy32vkCAAHNVguZrqZ+jeZ9rGRnGR4dN4= connectrpc.com/connect v1.20.0/go.mod h1:A2ygJrukXwWy32vkCAAHNVguZrqZ+jeZ9rGRnGR4dN4=
cyphar.com/go-pathrs v0.2.5 h1:SnX9FBvnoyn3lUs1dkMgZ52bAETpirNu3FTRh5HlRik= cyphar.com/go-pathrs v0.2.3 h1:0pH8gep37wB0BgaXrEaN1OtZhUMeS7VvaejSr6i822o=
cyphar.com/go-pathrs v0.2.5/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= cyphar.com/go-pathrs v0.2.3/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc=
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
gitea.dev/actionslib v0.7.0 h1:JCV8eeIGwjlXcuSr7ojEdQC22VoE2466+K+D9vuQWKQ= gitea.dev/actionslib v0.7.0 h1:JCV8eeIGwjlXcuSr7ojEdQC22VoE2466+K+D9vuQWKQ=
@@ -11,8 +11,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw=
github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
@@ -27,8 +27,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA= github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c= github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
@@ -38,16 +38,17 @@ github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
github.com/cyphar/filepath-securejoin v0.7.0 h1:s0Y3ITPy6sQn5xt54DuYvTF8hu134ooYLUb58DX/HjE= github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE=
github.com/cyphar/filepath-securejoin v0.7.0/go.mod h1:ymLGms/u3BYaviIiuKFnUx8EkQEZeK6cInNoAPJA3o4= github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/cli v29.7.2+incompatible h1:dlkwallR8XqfeVnA2ELEhdwvb4lsSwuB4IgsG8Q9cLY= github.com/docker/cli v29.6.2+incompatible h1:/bjePvcbbFTnRrMfWJBY7AjfICdsiLVgHn6LwTVOcqw=
github.com/docker/cli v29.7.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v29.6.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/docker-credential-helpers v0.9.8 h1:bIREROb7So6PRlq6KTtdS9MPEjC29OQRkFNlvK2OX8Q= github.com/docker/docker-credential-helpers v0.9.6 h1:cT2PbRPSlnMmNTfT2TDMXRyQ1KMWHG7xoTLBcn1ZNv0=
github.com/docker/docker-credential-helpers v0.9.8/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c= github.com/docker/docker-credential-helpers v0.9.6/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c=
github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M= github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M=
github.com/docker/go-connections v0.8.1/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-connections v0.8.1/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -58,8 +59,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@@ -71,8 +72,8 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod
github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY=
github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
@@ -99,10 +100,10 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY=
github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -112,34 +113,30 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/mattn/go-runewidth v0.0.28 h1:rPyg2ybwEKPebvpzVWe1gKBkH8EQFkxO4Y0hjBeLaBU= github.com/mattn/go-runewidth v0.0.21 h1:jJKAZiQH+2mIinzCJIaIG9Be1+0NR+5sz/lYEEjdM8w=
github.com/mattn/go-runewidth v0.0.28/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8= github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/mattn/go-shellwords v1.0.14 h1:yUKzIgsCnosndOASY6/enly1EAuaXeFSQ7cdyA3OuYg= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
github.com/mattn/go-shellwords v1.0.14/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/go-archive v0.3.3 h1:OxxR9paxsluYi+zDUEXTTaIxtkK3viymW+Ka7vRhhME= github.com/moby/go-archive v0.2.1 h1:fAa0wUS/ikZKyx7o/1fhUYmhZ7RgpthdeoDhJvunTLc=
github.com/moby/go-archive v0.3.3/go.mod h1:Npdv43fFqlhZW7Xo8fbm3ZMYFvAGNviUPqX21VERbcE= github.com/moby/go-archive v0.2.1/go.mod h1:Npdv43fFqlhZW7Xo8fbm3ZMYFvAGNviUPqX21VERbcE=
github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=
github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw= github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw=
github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM= github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM=
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
github.com/moby/sys/mount v0.3.5 h1:eS3fsZTjHaBihwjp4/+5Z3jxqLXYsbwxqpVSfFv3M00=
github.com/moby/sys/mount v0.3.5/go.mod h1:WUQDO+/uCiCIkIztx8SrwIDVn2dtMFRBebRhpDFT71M=
github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg=
github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4=
github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8=
github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o=
github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0= github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0=
github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y= github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y=
github.com/moby/sys/userns v0.2.0 h1:nEtDtp7NCV/6dutSklNe8FrENPwFdc4mXnZqC/JWgXM= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
github.com/moby/sys/userns v0.2.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
@@ -154,6 +151,7 @@ github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU=
github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
@@ -173,8 +171,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.10.1 h1:xi4336Zh11WpU14fXR6I67V3yaTPQYwRx2WEtHbRg4Q= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.10.1/go.mod h1:vsQHnG7xzNsxk3NrwboUiWPnIC3dmbjcGPykD7+tiHk= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/skeema/knownhosts v1.3.2 h1:EDL9mgf4NzwMXCTfaxSD/o/a5fxDw/xL9nkU28JjdBg= github.com/skeema/knownhosts v1.3.2 h1:EDL9mgf4NzwMXCTfaxSD/o/a5fxDw/xL9nkU28JjdBg=
github.com/skeema/knownhosts v1.3.2/go.mod h1:bEg3iQAuw+jyiw+484wwFJoKSLwcfd7fqRy+N0QTiow= github.com/skeema/knownhosts v1.3.2/go.mod h1:bEg3iQAuw+jyiw+484wwFJoKSLwcfd7fqRy+N0QTiow=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
@@ -193,8 +191,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/timshannon/bolthold v0.0.0-20240314194003-30aac6950928 h1:zjNCuOOhh1TKRU0Ru3PPPJt80z7eReswCao91gBLk00= github.com/timshannon/bolthold v0.0.0-20240314194003-30aac6950928 h1:zjNCuOOhh1TKRU0Ru3PPPJt80z7eReswCao91gBLk00=
github.com/timshannon/bolthold v0.0.0-20240314194003-30aac6950928/go.mod h1:PCFYfAEfKT+Nd6zWvUpsXduMR1bXFLf0uGSlEF05MCI= github.com/timshannon/bolthold v0.0.0-20240314194003-30aac6950928/go.mod h1:PCFYfAEfKT+Nd6zWvUpsXduMR1bXFLf0uGSlEF05MCI=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
@@ -212,35 +210,34 @@ go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M= go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0=
go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go= go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -257,11 +254,11 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
@@ -5,13 +5,12 @@
package artifactcache package artifactcache
import ( import (
"cmp"
"context" "context"
"crypto/hmac" "crypto/hmac"
"crypto/rand" "crypto/rand"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"encoding/json/v2" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@@ -21,15 +20,13 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"regexp" "regexp"
"slices"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time" "time"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"gitea.com/gitea/runner/internal/pkg/disk"
"github.com/julienschmidt/httprouter" "github.com/julienschmidt/httprouter"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
@@ -62,9 +59,6 @@ type JobCredential struct {
// remote runner registers with. // remote runner registers with.
Results string `json:"results"` Results string `json:"results"`
InsecureTLS bool `json:"insecure_tls"` InsecureTLS bool `json:"insecure_tls"`
// PublicURL is this server as a reverse proxy makes the job reach it, not the listen address.
PublicURL string `json:"public_url"`
} }
// credEntry holds a registered job's credential along with an active // credEntry holds a registered job's credential along with an active
@@ -106,38 +100,19 @@ type Handler struct {
credMu sync.RWMutex credMu sync.RWMutex
creds map[string]*credEntry creds map[string]*credEntry
policy Policy
// freeDisk is a field so tests can drive evictForFreeSpace without a full volume.
freeDisk func(string) (uint64, error)
}
// Options configures a cache server started by StartHandler; the zero value is usable.
type Options struct {
Dir string
OutboundIP string
Port uint16
// InternalSecret, when non-empty, enables a control-plane API at
// /_internal/{register,revoke} that lets a remote runner pre-register the
// per-job ACTIONS_RUNTIME_TOKENs it expects this server to honor. The
// embedded in-process handler leaves it empty and registers tokens via the
// in-process RegisterJob method directly.
InternalSecret string
Policy Policy
Logger logrus.FieldLogger
} }
// StartHandler opens the on-disk cache store and starts the HTTP server. // StartHandler opens the on-disk cache store and starts the HTTP server.
func StartHandler(opts Options) (*Handler, error) { //
dir, logger := opts.Dir, opts.Logger // internalSecret, when non-empty, enables a control-plane API at
// /_internal/{register,revoke} that lets a remote runner pre-register the
// per-job ACTIONS_RUNTIME_TOKENs it expects this server to honor. The
// embedded in-process handler leaves it empty and registers tokens via the
// in-process RegisterJob method directly.
func StartHandler(dir, outboundIP string, port uint16, internalSecret string, logger logrus.FieldLogger) (*Handler, error) {
h := &Handler{ h := &Handler{
creds: make(map[string]*credEntry), creds: make(map[string]*credEntry),
internalSecret: opts.InternalSecret, internalSecret: internalSecret,
policy: opts.Policy.withDefaults(),
freeDisk: disk.FreeBytes,
} }
if logger == nil { if logger == nil {
@@ -167,8 +142,8 @@ func StartHandler(opts Options) (*Handler, error) {
} }
h.storage = storage h.storage = storage
if opts.OutboundIP != "" { if outboundIP != "" {
h.outboundIP = opts.OutboundIP h.outboundIP = outboundIP
} else if ip := common.GetOutboundIP(); ip == nil { } else if ip := common.GetOutboundIP(); ip == nil {
return nil, errors.New("unable to determine outbound IP address") return nil, errors.New("unable to determine outbound IP address")
} else { } else {
@@ -207,7 +182,7 @@ func StartHandler(opts Options) (*Handler, error) {
// can break Docker Desktop variants where the host's outbound IP is not // can break Docker Desktop variants where the host's outbound IP is not
// routable from inside the container network. Authentication is enforced // routable from inside the container network. Authentication is enforced
// by the bearer middleware and per-repo scoping, not by reachability. // by the bearer middleware and per-repo scoping, not by reachability.
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", opts.Port)) listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -237,13 +212,6 @@ func (h *Handler) ExternalURL() string {
return fmt.Sprintf("http://%s:%d", h.outboundIP, h.port) return fmt.Sprintf("http://%s:%d", h.outboundIP, h.port)
} }
func (h *Handler) baseURL(cred JobCredential) string {
if base := strings.TrimRight(cred.PublicURL, "/"); base != "" {
return base
}
return h.ExternalURL()
}
// RegisterJob makes token a valid bearer credential for cache requests from // RegisterJob makes token a valid bearer credential for cache requests from
// the given repository and returns a function that removes it. The runner // the given repository and returns a function that removes it. The runner
// calls this at job start and defers the returned func so that the credential // calls this at job start and defers the returned func so that the credential
@@ -357,8 +325,8 @@ func (h *Handler) Close() error {
func (h *Handler) openDB() (*bolthold.Store, error) { func (h *Handler) openDB() (*bolthold.Store, error) {
return bolthold.Open(filepath.Join(h.dir, "bolt.db"), 0o644, &bolthold.Options{ return bolthold.Open(filepath.Join(h.dir, "bolt.db"), 0o644, &bolthold.Options{
Encoder: func(value any) ([]byte, error) { return json.Marshal(value) }, Encoder: json.Marshal,
Decoder: func(data []byte, value any) error { return json.Unmarshal(data, value) }, Decoder: json.Unmarshal,
Options: &bbolt.Options{ Options: &bbolt.Options{
Timeout: 5 * time.Second, Timeout: 5 * time.Second,
NoGrowSync: bbolt.DefaultOptions.NoGrowSync, NoGrowSync: bbolt.DefaultOptions.NoGrowSync,
@@ -391,7 +359,7 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request, _ httprouter.Para
} }
h.responseJSON(w, r, 200, map[string]any{ h.responseJSON(w, r, 200, map[string]any{
"result": "hit", "result": "hit",
"archiveLocation": h.signedArtifactURL(cred, cache.ID, time.Now().Add(artifactURLTTL)), "archiveLocation": h.signedArtifactURL(cache.ID, time.Now().Add(artifactURLTTL)),
"cacheKey": cache.Key, "cacheKey": cache.Key,
}) })
} }
@@ -412,9 +380,6 @@ func (h *Handler) lookupCache(db *bolthold.Store, repo string, keys []string, ve
_ = db.Delete(cache.ID, cache) _ = db.Delete(cache.ID, cache)
return nil, nil //nolint:nilnil // absence is not an error here return nil, nil //nolint:nilnil // absence is not an error here
} }
// Handing out a download URL counts as access, or eviction could drop the entry between
// this call and the GET that follows it.
h.touch(db, cache)
return cache, nil return cache, nil
} }
@@ -422,15 +387,13 @@ func (h *Handler) lookupCache(db *bolthold.Store, repo string, keys []string, ve
func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
cred := credFromContext(r.Context()) cred := credFromContext(r.Context())
api := &Request{} api := &Request{}
if err := json.UnmarshalRead(r.Body, api); err != nil { if err := json.NewDecoder(r.Body).Decode(api); err != nil {
h.responseJSON(w, r, 400, err) h.responseJSON(w, r, 400, err)
return return
} }
cache := &Cache{Repo: cred.Repo, Key: api.Key, Version: api.Version, Size: api.Size} cache := api.ToCache()
if cache.Size == 0 { cache.Repo = cred.Repo
cache.Size = -1
}
db, err := h.openDB() db, err := h.openDB()
if err != nil { if err != nil {
h.responseJSON(w, r, 500, err) h.responseJSON(w, r, 500, err)
@@ -554,22 +517,13 @@ func (h *Handler) commitCache(cache *Cache) error {
// write real size back to cache, it may be different from the current value when the request doesn't specify it. // write real size back to cache, it may be different from the current value when the request doesn't specify it.
cache.Size = written cache.Size = written
cache.Complete = true cache.Complete = true
cache.UsedAt = time.Now().Unix() // a just-written entry counts as accessed, so it cannot be its own eviction victim
db, err := h.openDB() db, err := h.openDB()
if err != nil { if err != nil {
return err return err
} }
defer db.Close() defer db.Close()
if err := db.Update(cache.ID, cache); err != nil { return db.Update(cache.ID, cache)
return err
}
// A commit is the only thing that grows the store, so the only thing that can push the
// volume under the floor.
h.evictRepo(db, cache.Repo)
h.evictTotal(db)
h.evictForFreeSpace(db)
return nil
} }
// GET /_apis/artifactcache/artifacts/:id // GET /_apis/artifactcache/artifacts/:id
@@ -687,12 +641,12 @@ func (h *Handler) ResultsURL(cred JobCredential) string {
if h == nil || cred.Results == "" { if h == nil || cred.Results == "" {
return "" return ""
} }
return h.baseURL(cred) return h.ExternalURL()
} }
func (h *Handler) internalRegister(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { func (h *Handler) internalRegister(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
var body internalRegisterBody var body internalRegisterBody
if err := json.UnmarshalRead(r.Body, &body); err != nil { if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
h.responseJSON(w, r, http.StatusBadRequest, err) h.responseJSON(w, r, http.StatusBadRequest, err)
return return
} }
@@ -708,7 +662,7 @@ func (h *Handler) internalRegister(w http.ResponseWriter, r *http.Request, _ htt
// POST /_internal/revoke // POST /_internal/revoke
func (h *Handler) internalRevoke(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { func (h *Handler) internalRevoke(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
var body internalRevokeBody var body internalRevokeBody
if err := json.UnmarshalRead(r.Body, &body); err != nil { if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
h.responseJSON(w, r, http.StatusBadRequest, err) h.responseJSON(w, r, http.StatusBadRequest, err)
return return
} }
@@ -746,16 +700,16 @@ func (h *Handler) computeSignature(purpose string, cacheID, exp int64) string {
} }
// signedURL builds a URL under path that signedAuth accepts for the same purpose. // signedURL builds a URL under path that signedAuth accepts for the same purpose.
func (h *Handler) signedURL(cred JobCredential, path, purpose string, cacheID uint64, exp time.Time) string { func (h *Handler) signedURL(path, purpose string, cacheID uint64, exp time.Time) string {
expUnix := exp.Unix() expUnix := exp.Unix()
q := url.Values{} q := url.Values{}
q.Set("exp", strconv.FormatInt(expUnix, 10)) q.Set("exp", strconv.FormatInt(expUnix, 10))
q.Set("sig", h.computeSignature(purpose, int64(cacheID), expUnix)) q.Set("sig", h.computeSignature(purpose, int64(cacheID), expUnix))
return fmt.Sprintf("%s%s/%d?%s", h.baseURL(cred), path, cacheID, q.Encode()) return fmt.Sprintf("%s%s/%d?%s", h.ExternalURL(), path, cacheID, q.Encode())
} }
func (h *Handler) signedArtifactURL(cred JobCredential, cacheID uint64, exp time.Time) string { func (h *Handler) signedArtifactURL(cacheID uint64, exp time.Time) string {
return h.signedURL(cred, apiPath+"/artifacts", "", cacheID, exp) return h.signedURL(apiPath+"/artifacts", "", cacheID, exp)
} }
// if not found, return (nil, nil) instead of an error. // if not found, return (nil, nil) instead of an error.
@@ -857,43 +811,12 @@ func (h *Handler) touchCache(id uint64, requireIncomplete bool) error {
} }
const ( const (
miB = 1024 * 1024 keepUsed = 30 * 24 * time.Hour
keepUnused = 7 * 24 * time.Hour
defaultSweepInterval = time.Hour keepTemp = 5 * time.Minute
keepOld = 5 * time.Minute
// inUseGrace matches artifactURLTTL so an entry outlives every signed URL still usable
// for it, and no sweep cuts off a download in progress.
inUseGrace = artifactURLTTL
// uploadStallTimeout is how long a reservation may sit without a chunk before it counts
// as abandoned. Widening it also widens the window for findExactCache to hand a finalize
// a stale reservation.
uploadStallTimeout = 5 * time.Minute
defaultMinFreeDisk = 1024 * miB
) )
// Policy bounds what the cache server keeps: a retention window counted from last access,
// and size limits that evict least recently accessed first. A zero limit is no limit.
type Policy struct {
Retention time.Duration // Retention removes entries nothing has read or written within this window. Zero keeps them regardless of age.
RepoSizeLimit int64 // RepoSizeLimit caps one repository's completed entries in bytes, evicting least recently accessed first.
SizeLimit int64 // SizeLimit caps every repository's completed entries together, in bytes.
SweepInterval time.Duration // SweepInterval is the minimum time between two eviction sweeps.
MinFreeDisk int64 // MinFreeDisk is volume headroom the cache will not eat into. Tracks the runner's health-check floor rather than taking a key of its own.
}
func (p Policy) withDefaults() Policy {
// The limits default in config.LoadDefault, so a written 0 means off.
if p.MinFreeDisk <= 0 {
p.MinFreeDisk = defaultMinFreeDisk
}
if p.SweepInterval <= 0 {
p.SweepInterval = defaultSweepInterval
}
return p
}
func (h *Handler) gcCache() { func (h *Handler) gcCache() {
if h.gcing.Load() { if h.gcing.Load() {
return return
@@ -903,7 +826,7 @@ func (h *Handler) gcCache() {
} }
defer h.gcing.Store(false) defer h.gcing.Store(false)
if time.Since(h.gcAt) < h.policy.SweepInterval { if time.Since(h.gcAt) < time.Hour {
h.logger.Debugf("skip gc: %v", h.gcAt.String()) h.logger.Debugf("skip gc: %v", h.gcAt.String())
return return
} }
@@ -916,60 +839,71 @@ func (h *Handler) gcCache() {
} }
defer db.Close() defer db.Close()
h.evictIncomplete(db) // Remove the caches which are not completed for a while, they are most likely to be broken.
h.evictExpired(db)
h.evictSuperseded(db)
h.evictOversized(db)
h.evictForFreeSpace(db)
}
// evictForFreeSpace bounds the volume itself, so it also covers bytes the cache never
// accounted for.
func (h *Handler) evictForFreeSpace(db *bolthold.Store) {
free, err := h.freeDisk(h.dir)
if err != nil {
h.logger.Debugf("free disk check: %v", err) // unsupported platform, treat as unavailable rather than full
return
}
if free >= uint64(h.policy.MinFreeDisk) {
return
}
caches := h.completedByUse(db)
total, shortfall := totalSize(caches), h.policy.MinFreeDisk-int64(free)
if total <= shortfall {
// Say so, or shedding everything and still being short reads as the backstop working.
h.logger.Warnf("cache volume is %d MiB short of the free space floor with only %d MiB of cache on it; something else is filling it", shortfall/miB, total/miB)
}
h.evictTo(db, caches, total-shortfall, "the cache volume")
}
// evictIncomplete removes uploads that stopped part way, which are most likely broken.
func (h *Handler) evictIncomplete(db *bolthold.Store) {
h.sweep(db, bolthold.
Where("UsedAt").Lt(time.Now().Add(-uploadStallTimeout).Unix()).
And("Complete").Eq(false).
Index("UsedAt"))
}
func (h *Handler) evictExpired(db *bolthold.Store) {
if h.policy.Retention <= 0 {
return
}
// Never below inUseGrace, or a short retention would outrun a signed URL already issued.
window := max(h.policy.Retention, inUseGrace)
h.sweep(db, bolthold.Where("UsedAt").Lt(time.Now().Add(-window).Unix()).Index("UsedAt"))
}
// evictSuperseded removes entries a newer one with the same key and version replaced. The
// aggregation includes Repo so two repos sharing a (key, version) do not evict each other.
func (h *Handler) evictSuperseded(db *bolthold.Store) {
results, err := db.FindAggregate(&Cache{}, bolthold.Where("Complete").Eq(true).Index("Complete"), "Repo", "Key", "Version")
if err != nil {
h.logger.Warnf("find aggregate caches: %v", err)
return
}
var caches []*Cache var caches []*Cache
if err := db.Find(&caches, bolthold.
Where("UsedAt").Lt(time.Now().Add(-keepTemp).Unix()).
And("Complete").Eq(false),
); err != nil {
h.logger.Warnf("find caches: %v", err)
} else {
for _, cache := range caches {
h.storage.Remove(cache.ID)
if err := db.Delete(cache.ID, cache); err != nil {
h.logger.Warnf("delete cache: %v", err)
continue
}
h.logger.Infof("deleted cache: %+v", cache)
}
}
// Remove the old caches which have not been used recently.
caches = caches[:0]
if err := db.Find(&caches, bolthold.
Where("UsedAt").Lt(time.Now().Add(-keepUnused).Unix()),
); err != nil {
h.logger.Warnf("find caches: %v", err)
} else {
for _, cache := range caches {
h.storage.Remove(cache.ID)
if err := db.Delete(cache.ID, cache); err != nil {
h.logger.Warnf("delete cache: %v", err)
continue
}
h.logger.Infof("deleted cache: %+v", cache)
}
}
// Remove the old caches which are too old.
caches = caches[:0]
if err := db.Find(&caches, bolthold.
Where("CreatedAt").Lt(time.Now().Add(-keepUsed).Unix()),
); err != nil {
h.logger.Warnf("find caches: %v", err)
} else {
for _, cache := range caches {
h.storage.Remove(cache.ID)
if err := db.Delete(cache.ID, cache); err != nil {
h.logger.Warnf("delete cache: %v", err)
continue
}
h.logger.Infof("deleted cache: %+v", cache)
}
}
// Remove the old caches with the same key and version within the same
// repository, keep the latest one. Aggregation must include Repo so two
// repos that happen to share a (key, version) do not evict each other —
// otherwise per-repo scoping holds for reads but one repo can age
// another out after keepOld.
// Also keep the olds which have been used recently for a while in case of the cache is still in use.
if results, err := db.FindAggregate(
&Cache{},
bolthold.Where("Complete").Eq(true),
"Repo", "Key", "Version",
); err != nil {
h.logger.Warnf("find aggregate caches: %v", err)
} else {
for _, result := range results { for _, result := range results {
if result.Count() <= 1 { if result.Count() <= 1 {
continue continue
@@ -978,144 +912,20 @@ func (h *Handler) evictSuperseded(db *bolthold.Store) {
caches = caches[:0] caches = caches[:0]
result.Reduction(&caches) result.Reduction(&caches)
for _, cache := range caches[:len(caches)-1] { for _, cache := range caches[:len(caches)-1] {
if inUse(cache) { if time.Since(time.Unix(cache.UsedAt, 0)) < keepOld {
// Keep it since it has been used recently, even if it's old.
// Or it could break downloading in process.
continue continue
} }
h.deleteCache(db, cache) h.storage.Remove(cache.ID)
}
}
}
// evictOversized applies the per-repository limit, then the whole-store one. Only completed
// entries count, since only those carry a size measured at commit rather than claimed.
func (h *Handler) evictOversized(db *bolthold.Store) {
if h.policy.RepoSizeLimit > 0 {
byRepo := make(map[string][]*Cache)
for _, cache := range h.completedByUse(db) {
byRepo[cache.Repo] = append(byRepo[cache.Repo], cache)
}
for repo, caches := range byRepo {
h.evictTo(db, caches, h.policy.RepoSizeLimit, "repository "+repo)
}
}
h.evictTotal(db)
}
// evictTotal caps the store as a whole. It re-queries because the per-repo pass may have
// deleted rows an earlier result still holds.
func (h *Handler) evictTotal(db *bolthold.Store) {
if h.policy.SizeLimit <= 0 {
return
}
h.evictTo(db, h.completedByUse(db), h.policy.SizeLimit, "the cache")
}
// evictRepo reclaims space when a commit pushes a repo over, rather than at the next sweep.
func (h *Handler) evictRepo(db *bolthold.Store, repo string) {
if h.policy.RepoSizeLimit <= 0 {
return
}
h.evictTo(db, h.cachesByUse(db, bolthold.Where("Repo").Eq(repo).And("Complete").Eq(true).Index("Repo")), h.policy.RepoSizeLimit, "repository "+repo)
}
// evictTo deletes until caches fit limit. caches must be ordered by UsedAt ascending.
func (h *Handler) evictTo(db *bolthold.Store, caches []*Cache, limit int64, scope string) {
// An entry bigger than the limit never fits, so it goes on its own account instead of
// dragging every neighbour out first and then following them next sweep.
fits := caches[:0]
for _, cache := range caches {
if cache.Size <= limit {
fits = append(fits, cache)
continue
}
if !inUse(cache) {
h.logger.Warnf("cache %q is %d MiB on its own, over the limit for %s; dropping it", cache.Key, cache.Size/miB, scope)
h.deleteCache(db, cache)
}
}
caches = fits
total := totalSize(caches)
var freed int64
for _, cache := range caches {
if total <= limit {
break
}
if inUse(cache) || !h.deleteCache(db, cache) {
continue
}
total -= cache.Size
freed += cache.Size
}
if freed > 0 {
h.logger.Warnf("evicted %d MiB from %s, least recently used first", freed/miB, scope)
}
}
// inUse reports whether an entry was read or written recently enough that removing it
// could break a download in progress.
func inUse(cache *Cache) bool {
return time.Since(time.Unix(cache.UsedAt, 0)) < inUseGrace
}
// touch stamps UsedAt through the caller's store, a bolt write on the read path. It cannot
// go through touchCache, which opens its own store and would block on the exclusive lock
// for as long as the caller holds one.
func (h *Handler) touch(db *bolthold.Store, cache *Cache) {
cache.UsedAt = time.Now().Unix()
if err := db.Update(cache.ID, cache); err != nil {
h.logger.Warnf("touch cache: %v", err)
}
}
func (h *Handler) sweep(db *bolthold.Store, query *bolthold.Query) {
for _, cache := range h.caches(db, query) {
h.deleteCache(db, cache)
}
}
func (h *Handler) caches(db *bolthold.Store, query *bolthold.Query) []*Cache {
var caches []*Cache
if err := db.Find(&caches, query); err != nil {
h.logger.Warnf("find caches: %v", err)
}
return caches
}
// cachesByUse returns matches least recently accessed first, sorting here rather than with
// bolthold's SortBy, which reflects over every field it compares.
func (h *Handler) cachesByUse(db *bolthold.Store, query *bolthold.Query) []*Cache {
caches := h.caches(db, query)
slices.SortFunc(caches, func(a, b *Cache) int { return cmp.Compare(a.UsedAt, b.UsedAt) })
return caches
}
// completedByUse returns every entry the size limits count, least recently accessed first.
func (h *Handler) completedByUse(db *bolthold.Store) []*Cache {
return h.cachesByUse(db, bolthold.Where("Complete").Eq(true).Index("Complete"))
}
func totalSize(caches []*Cache) int64 {
var total int64
for _, cache := range caches {
total += cache.Size
}
return total
}
// deleteCache drops an entry and its bytes, reporting whether it went fully. The blob goes
// first, so a failed unlink leaves the row for the next sweep instead of orphaning bytes.
func (h *Handler) deleteCache(db *bolthold.Store, cache *Cache) bool {
if err := h.storage.Remove(cache.ID); err != nil {
h.logger.Warnf("remove cache blob: %v", err)
return false
}
if err := db.Delete(cache.ID, cache); err != nil { if err := db.Delete(cache.ID, cache); err != nil {
h.logger.Warnf("delete cache: %v", err) h.logger.Warnf("delete cache: %v", err)
return false continue
} }
h.logger.Infof("deleted cache: %+v", cache) h.logger.Infof("deleted cache: %+v", cache)
return true }
}
}
} }
func (h *Handler) responseJSON(w http.ResponseWriter, r *http.Request, code int, v ...any) { func (h *Handler) responseJSON(w http.ResponseWriter, r *http.Request, code int, v ...any) {
@@ -7,8 +7,7 @@ package artifactcache
import ( import (
"bytes" "bytes"
"crypto/rand" "crypto/rand"
"encoding/json/v2" "encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@@ -42,19 +41,16 @@ func (b *bearerTransport) RoundTrip(r *http.Request) (*http.Response, error) {
var testClient = &http.Client{Transport: &bearerTransport{token: testToken}} var testClient = &http.Client{Transport: &bearerTransport{token: testToken}}
// testRetention mirrors config.DefaultCacheRetention; Policy has no defaults of its own.
const testRetention = 7 * 24 * time.Hour
// signArtifactURL builds a signed download URL the same way the server does; // signArtifactURL builds a signed download URL the same way the server does;
// tests use it to reach the get handler directly without going through a // tests use it to reach the get handler directly without going through a
// find/cache-hit round trip. // find/cache-hit round trip.
func signArtifactURL(h *Handler, id int64) string { func signArtifactURL(h *Handler, id int64) string {
return h.signedArtifactURL(JobCredential{}, uint64(id), time.Now().Add(artifactURLTTL)) return h.signedArtifactURL(uint64(id), time.Now().Add(artifactURLTTL))
} }
func TestHandler(t *testing.T) { func TestHandler(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{Dir: dir}) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
handler.RegisterJob(testToken, JobCredential{Repo: testRepo}) handler.RegisterJob(testToken, JobCredential{Repo: testRepo})
@@ -136,7 +132,7 @@ func TestHandler(t *testing.T) {
defer resp.Body.Close() defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode) assert.Equal(t, 200, resp.StatusCode)
require.NoError(t, json.UnmarshalRead(resp.Body, &first)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&first))
assert.NotZero(t, first.CacheID) assert.NotZero(t, first.CacheID)
} }
{ {
@@ -151,7 +147,7 @@ func TestHandler(t *testing.T) {
defer resp.Body.Close() defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode) assert.Equal(t, 200, resp.StatusCode)
require.NoError(t, json.UnmarshalRead(resp.Body, &second)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&second))
assert.NotZero(t, second.CacheID) assert.NotZero(t, second.CacheID)
} }
@@ -204,7 +200,7 @@ func TestHandler(t *testing.T) {
got := struct { got := struct {
CacheID uint64 `json:"cacheId"` CacheID uint64 `json:"cacheId"`
}{} }{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
id = got.CacheID id = got.CacheID
} }
{ {
@@ -259,7 +255,7 @@ func TestHandler(t *testing.T) {
got := struct { got := struct {
CacheID uint64 `json:"cacheId"` CacheID uint64 `json:"cacheId"`
}{} }{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
id = got.CacheID id = got.CacheID
} }
{ {
@@ -315,7 +311,7 @@ func TestHandler(t *testing.T) {
got := struct { got := struct {
CacheID uint64 `json:"cacheId"` CacheID uint64 `json:"cacheId"`
}{} }{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
id = got.CacheID id = got.CacheID
} }
{ {
@@ -362,7 +358,7 @@ func TestHandler(t *testing.T) {
got := struct { got := struct {
CacheID uint64 `json:"cacheId"` CacheID uint64 `json:"cacheId"`
}{} }{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
id = got.CacheID id = got.CacheID
} }
@@ -413,7 +409,7 @@ func TestHandler(t *testing.T) {
got := struct { got := struct {
CacheID uint64 `json:"cacheId"` CacheID uint64 `json:"cacheId"`
}{} }{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
id = got.CacheID id = got.CacheID
} }
{ {
@@ -493,7 +489,7 @@ func TestHandler(t *testing.T) {
ArchiveLocation string `json:"archiveLocation"` ArchiveLocation string `json:"archiveLocation"`
CacheKey string `json:"cacheKey"` CacheKey string `json:"cacheKey"`
}{} }{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
assert.Equal(t, "hit", got.Result) assert.Equal(t, "hit", got.Result)
assert.Equal(t, keys[except], got.CacheKey) assert.Equal(t, keys[except], got.CacheKey)
@@ -528,7 +524,7 @@ func TestHandler(t *testing.T) {
ArchiveLocation string `json:"archiveLocation"` ArchiveLocation string `json:"archiveLocation"`
CacheKey string `json:"cacheKey"` CacheKey string `json:"cacheKey"`
}{} }{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
assert.Equal(t, "hit", got.Result) assert.Equal(t, "hit", got.Result)
assert.Equal(t, key, got.CacheKey) assert.Equal(t, key, got.CacheKey)
assert.NotEqual(t, strings.ToLower(key), got.CacheKey) assert.NotEqual(t, strings.ToLower(key), got.CacheKey)
@@ -577,7 +573,7 @@ func TestHandler(t *testing.T) {
ArchiveLocation string `json:"archiveLocation"` ArchiveLocation string `json:"archiveLocation"`
CacheKey string `json:"cacheKey"` CacheKey string `json:"cacheKey"`
}{} }{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
assert.Equal(t, keys[expect], got.CacheKey) assert.Equal(t, keys[expect], got.CacheKey)
contentResp, err := testClient.Get(got.ArchiveLocation) contentResp, err := testClient.Get(got.ArchiveLocation)
@@ -633,7 +629,7 @@ func TestHandler(t *testing.T) {
ArchiveLocation string `json:"archiveLocation"` ArchiveLocation string `json:"archiveLocation"`
CacheKey string `json:"cacheKey"` CacheKey string `json:"cacheKey"`
}{} }{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
assert.Equal(t, keys[expect], got.CacheKey) assert.Equal(t, keys[expect], got.CacheKey)
contentResp, err := testClient.Get(got.ArchiveLocation) contentResp, err := testClient.Get(got.ArchiveLocation)
@@ -660,7 +656,7 @@ func backdateCache(t *testing.T, handler *Handler, key string, age time.Duration
require.NoError(t, db.Update(caches[0].ID, caches[0])) require.NoError(t, db.Update(caches[0].ID, caches[0]))
} }
func uploadCacheNormally(t *testing.T, base, key, version string, content []byte) { func uploadCacheNormally(t *testing.T, base, key, version string, content []byte) { //nolint:unparam // pre-existing issue from nektos/act
var id uint64 var id uint64
{ {
body, err := json.Marshal(&Request{ body, err := json.Marshal(&Request{
@@ -677,7 +673,7 @@ func uploadCacheNormally(t *testing.T, base, key, version string, content []byte
got := struct { got := struct {
CacheID uint64 `json:"cacheId"` CacheID uint64 `json:"cacheId"`
}{} }{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
id = got.CacheID id = got.CacheID
} }
{ {
@@ -708,7 +704,7 @@ func uploadCacheNormally(t *testing.T, base, key, version string, content []byte
ArchiveLocation string `json:"archiveLocation"` ArchiveLocation string `json:"archiveLocation"`
CacheKey string `json:"cacheKey"` CacheKey string `json:"cacheKey"`
}{} }{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
assert.Equal(t, "hit", got.Result) assert.Equal(t, "hit", got.Result)
assert.Equal(t, key, got.CacheKey) assert.Equal(t, key, got.CacheKey)
archiveLocation = got.ArchiveLocation archiveLocation = got.ArchiveLocation
@@ -726,7 +722,7 @@ func uploadCacheNormally(t *testing.T, base, key, version string, content []byte
func TestHandler_gcCache(t *testing.T) { func TestHandler_gcCache(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{Dir: dir, Policy: Policy{Retention: testRetention}}) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer func() { defer func() {
@@ -756,8 +752,8 @@ func TestHandler_gcCache(t *testing.T) {
Key: "test_key_2", Key: "test_key_2",
Version: "test_version", Version: "test_version",
Complete: false, Complete: false,
UsedAt: now.Add(-(inUseGrace + time.Second)).Unix(), UsedAt: now.Add(-(keepTemp + time.Second)).Unix(),
CreatedAt: now.Add(-(inUseGrace + time.Hour)).Unix(), CreatedAt: now.Add(-(keepTemp + time.Hour)).Unix(),
}, },
Kept: false, Kept: false,
}, },
@@ -767,21 +763,21 @@ func TestHandler_gcCache(t *testing.T) {
Key: "test_key_3", Key: "test_key_3",
Version: "test_version", Version: "test_version",
Complete: true, Complete: true,
UsedAt: now.Add(-(testRetention + time.Second)).Unix(), UsedAt: now.Add(-(keepUnused + time.Second)).Unix(),
CreatedAt: now.Add(-(testRetention + time.Hour)).Unix(), CreatedAt: now.Add(-(keepUnused + time.Hour)).Unix(),
}, },
Kept: false, Kept: false,
}, },
{ {
// should be kept, since age alone does not retire an entry that is still used. // should be removed, since it's used but too old.
Cache: &Cache{ Cache: &Cache{
Key: "test_key_3", Key: "test_key_3",
Version: "test_version", Version: "test_version",
Complete: true, Complete: true,
UsedAt: now.Unix(), UsedAt: now.Unix(),
CreatedAt: now.Add(-365 * 24 * time.Hour).Unix(), CreatedAt: now.Add(-(keepUsed + time.Second)).Unix(),
}, },
Kept: true, Kept: false,
}, },
{ {
// should be kept, since it has a newer edition but be used recently. // should be kept, since it has a newer edition but be used recently.
@@ -789,7 +785,7 @@ func TestHandler_gcCache(t *testing.T) {
Key: "test_key_1", Key: "test_key_1",
Version: "test_version", Version: "test_version",
Complete: true, Complete: true,
UsedAt: now.Add(-(inUseGrace - time.Minute)).Unix(), UsedAt: now.Add(-(keepOld - time.Minute)).Unix(),
CreatedAt: now.Add(-(time.Hour + time.Second)).Unix(), CreatedAt: now.Add(-(time.Hour + time.Second)).Unix(),
}, },
Kept: true, Kept: true,
@@ -800,7 +796,7 @@ func TestHandler_gcCache(t *testing.T) {
Key: "test_key_1", Key: "test_key_1",
Version: "test_version", Version: "test_version",
Complete: true, Complete: true,
UsedAt: now.Add(-(inUseGrace + time.Second)).Unix(), UsedAt: now.Add(-(keepOld + time.Second)).Unix(),
CreatedAt: now.Add(-(time.Hour + time.Second)).Unix(), CreatedAt: now.Add(-(time.Hour + time.Second)).Unix(),
}, },
Kept: false, Kept: false,
@@ -833,265 +829,11 @@ func TestHandler_gcCache(t *testing.T) {
require.NoError(t, db.Close()) require.NoError(t, db.Close())
} }
// TestHandler_evictPolicy covers the non-default policies; TestHandler_gcCache covers the
// defaults across every pass.
func TestHandler_evictPolicy(t *testing.T) {
now := time.Now()
stale := func(d time.Duration) int64 { return now.Add(-d).Unix() }
mib := func(n int64) int64 { return n * miB }
for _, tc := range []struct {
name string
policy Policy
entries []*Cache
kept []string
}{
{
name: "a zero retention keeps an entry nothing has touched",
policy: Policy{Retention: 0},
entries: []*Cache{
{Key: "idle", UsedAt: stale(testRetention + time.Hour)},
},
kept: []string{"idle"},
},
{
name: "evicts least recently accessed until the repository fits",
policy: Policy{RepoSizeLimit: mib(10)},
entries: []*Cache{
{Repo: "o/a", Key: "oldest", Size: mib(4), UsedAt: stale(3 * time.Hour)},
{Repo: "o/a", Key: "middle", Size: mib(4), UsedAt: stale(2 * time.Hour)},
{Repo: "o/a", Key: "newest", Size: mib(4), UsedAt: stale(time.Hour)},
},
kept: []string{"middle", "newest"},
},
{
name: "spares entries that may still be downloading",
policy: Policy{RepoSizeLimit: mib(10)},
entries: []*Cache{
{Repo: "o/a", Key: "fresh_1", Size: mib(6), UsedAt: stale(time.Minute)},
{Repo: "o/a", Key: "fresh_2", Size: mib(6), UsedAt: stale(time.Minute)},
},
kept: []string{"fresh_1", "fresh_2"},
},
{
name: "one repository over its limit leaves another alone",
policy: Policy{RepoSizeLimit: mib(10)},
entries: []*Cache{
{Repo: "o/a", Key: "a_old", Size: mib(6), UsedAt: stale(3 * time.Hour)},
{Repo: "o/a", Key: "a_new", Size: mib(6), UsedAt: stale(time.Hour)},
{Repo: "o/b", Key: "b_old", Size: mib(6), UsedAt: stale(4 * time.Hour)},
},
kept: []string{"a_new", "b_old"},
},
{
name: "the total limit evicts across repositories once each fits its own",
policy: Policy{RepoSizeLimit: mib(10), SizeLimit: mib(12)},
entries: []*Cache{
{Repo: "o/a", Key: "a_old", Size: mib(8), UsedAt: stale(3 * time.Hour)},
{Repo: "o/b", Key: "b_new", Size: mib(8), UsedAt: stale(time.Hour)},
},
kept: []string{"b_new"},
},
{
// Retention below inUseGrace would otherwise drop an entry whose signed URL a job
// is still holding.
name: "a retention shorter than the grace still spares a just-served entry",
policy: Policy{Retention: time.Minute},
entries: []*Cache{
{Key: "just_served", UsedAt: stale(2 * time.Minute)},
{Key: "idle", UsedAt: stale(time.Hour)},
},
kept: []string{"just_served"},
},
{
name: "an entry over the limit goes without emptying the repository",
policy: Policy{RepoSizeLimit: mib(10)},
entries: []*Cache{
{Repo: "o/a", Key: "keeps", Size: mib(4), UsedAt: stale(3 * time.Hour)},
{Repo: "o/a", Key: "huge", Size: mib(20), UsedAt: stale(2 * time.Hour)},
},
kept: []string{"keeps"},
},
{
name: "a zero limit keeps everything",
policy: Policy{RepoSizeLimit: 0},
entries: []*Cache{
{Repo: "o/a", Key: "huge_1", Size: mib(100), UsedAt: stale(3 * time.Hour)},
{Repo: "o/a", Key: "huge_2", Size: mib(100), UsedAt: stale(2 * time.Hour)},
},
kept: []string{"huge_1", "huge_2"},
},
} {
t.Run(tc.name, func(t *testing.T) {
for _, e := range tc.entries {
e.Complete = true // only completed entries carry a measured size, so only they count
}
handler := newTestHandler(t, tc.policy, tc.entries...)
handler.gcAt = time.Time{} // ensure gcCache will not skip
handler.gcCache()
assert.ElementsMatch(t, tc.kept, keptKeys(t, handler, tc.entries))
})
}
}
// TestHandler_evictForFreeSpace proves the volume backstop sheds only what it must, and only
// when the disk is actually short.
func TestHandler_evictForFreeSpace(t *testing.T) {
free := func(n int64) func(string) (uint64, error) {
return func(string) (uint64, error) { return uint64(n), nil }
}
for _, tc := range []struct {
name string
freeDisk func(string) (uint64, error)
kept []string
}{
{"ample free space evicts nothing", free(defaultMinFreeDisk), []string{"oldest", "middle", "newest"}},
{"a small shortfall sheds one entry", free(defaultMinFreeDisk - 4*miB), []string{"middle", "newest"}},
{"a shortfall the cache cannot cover sheds all of it", free(0), nil},
{
"an unreadable volume is treated as unavailable, not as full",
func(string) (uint64, error) { return 0, errors.New("unsupported") },
[]string{"oldest", "middle", "newest"},
},
} {
t.Run(tc.name, func(t *testing.T) {
now := time.Now()
entries := []*Cache{
{Key: "oldest", Complete: true, Size: 4 * miB, UsedAt: now.Add(-3 * time.Hour).Unix()},
{Key: "middle", Complete: true, Size: 4 * miB, UsedAt: now.Add(-2 * time.Hour).Unix()},
{Key: "newest", Complete: true, Size: 4 * miB, UsedAt: now.Add(-time.Hour).Unix()},
}
handler := newTestHandler(t, Policy{}, entries...)
handler.freeDisk = tc.freeDisk
db, err := handler.openDB()
require.NoError(t, err)
handler.evictForFreeSpace(db)
require.NoError(t, db.Close())
assert.ElementsMatch(t, tc.kept, keptKeys(t, handler, entries))
})
}
}
// TestHandler_SweepKeepsEntryWhenBlobSurvives proves a failed unlink leaves the row in place,
// so the next sweep retries rather than orphaning bytes no row points at and no limit counts.
func TestHandler_SweepKeepsEntryWhenBlobSurvives(t *testing.T) {
cache := &Cache{Key: "stuck", Complete: true, UsedAt: time.Now().Add(-(testRetention + time.Hour)).Unix()}
handler := newTestHandler(t, Policy{Retention: testRetention}, cache)
// A non-empty directory where the blob belongs makes os.Remove fail on every platform.
blob := handler.storage.filename(cache.ID)
require.NoError(t, os.MkdirAll(blob, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(blob, "held"), []byte("x"), 0o600))
handler.gcAt = time.Time{}
handler.gcCache()
assert.Equal(t, []string{"stuck"}, keptKeys(t, handler, []*Cache{cache}), "the entry must outlive a blob that could not be removed")
}
// TestHandler_FindProtectsFromEviction covers the window between a find handing out a signed
// download URL and the GET that redeems it: the entry promised to a job must not be the next
// eviction victim just because its last access predates the find.
func TestHandler_FindProtectsFromEviction(t *testing.T) {
// 12 MiB against a 10 MiB limit, so exactly one entry has to go.
wanted := &Cache{Repo: testRepo, Key: "wanted", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-3 * time.Hour).Unix()}
other := &Cache{Repo: testRepo, Key: "other", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-2 * time.Hour).Unix()}
newest := &Cache{Repo: testRepo, Key: "newest", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-time.Hour).Unix()}
handler := newTestHandler(t, Policy{RepoSizeLimit: 10 * miB}, wanted, other, newest)
writeBlob(t, handler, wanted.ID) // find only reports a hit when the blob is on disk
resp, err := testClient.Get(fmt.Sprintf("%s%s/cache?keys=wanted&version=v", handler.ExternalURL(), apiPath))
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, 200, resp.StatusCode)
// Evict directly: the request above kicked off an async gcCache, and writing gcAt here
// to drive gcCache would race its read.
db, err := handler.openDB()
require.NoError(t, err)
defer func() { require.NoError(t, db.Close()) }()
handler.evictOversized(db)
require.NoError(t, db.Get(wanted.ID, &Cache{}), "the entry just promised to a job must survive")
assert.ErrorIs(t, db.Get(other.ID, &Cache{}), bolthold.ErrNotFound, "the next least recently used goes instead")
}
// TestHandler_evictOnCommit proves a repository that goes over its limit gets space back at
// once, rather than waiting out the collection interval.
func TestHandler_evictOnCommit(t *testing.T) {
full := &Cache{Repo: testRepo, Key: "full", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-time.Hour).Unix()}
handler := newTestHandler(t, Policy{RepoSizeLimit: 4 * miB}, full)
// StartHandler already stamped gcAt, so the periodic sweep stays rate-limited out and
// only the commit path can evict.
uploadCacheNormally(t, handler.ExternalURL()+apiPath, "new", "v", []byte("some content"))
assert.Empty(t, keptKeys(t, handler, []*Cache{full}))
}
func TestHandler_gcCacheInterval(t *testing.T) {
cache := &Cache{Key: "temp", UsedAt: time.Now().Add(-time.Hour).Unix()}
// Half the default, so a sweep 45m ago is still inside the default but past this one.
handler := newTestHandler(t, Policy{SweepInterval: 30 * time.Minute}, cache)
handler.gcAt = time.Now().Add(-45 * time.Minute) // past the configured interval, still inside the default
handler.gcCache()
assert.Empty(t, keptKeys(t, handler, []*Cache{cache}))
}
// newTestHandler starts a handler with testToken registered, seeded with entries.
func newTestHandler(t *testing.T, policy Policy, entries ...*Cache) *Handler {
t.Helper()
handler, err := StartHandler(Options{
Dir: filepath.Join(t.TempDir(), "artifactcache"),
OutboundIP: "127.0.0.1",
Policy: policy,
})
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, handler.Close()) })
handler.RegisterJob(testToken, JobCredential{Repo: testRepo})
db, err := handler.openDB()
require.NoError(t, err)
for _, e := range entries {
require.NoError(t, insertCache(db, e))
}
require.NoError(t, db.Close())
return handler
}
// keptKeys reports which of entries are still in the store.
func keptKeys(t *testing.T, handler *Handler, entries []*Cache) []string {
t.Helper()
db, err := handler.openDB()
require.NoError(t, err)
defer func() { require.NoError(t, db.Close()) }()
var kept []string
for _, e := range entries {
if err := db.Get(e.ID, &Cache{}); err == nil {
kept = append(kept, e.Key)
}
}
return kept
}
// writeBlob gives an entry the on-disk bytes that find and get require.
func writeBlob(t *testing.T, handler *Handler, id uint64) {
t.Helper()
require.NoError(t, handler.storage.Write(id, 0, strings.NewReader("a")))
_, err := handler.storage.Commit(id, 1)
require.NoError(t, err)
}
// TestHandler_RejectsMissingBearer covers the advisory's root cause: // TestHandler_RejectsMissingBearer covers the advisory's root cause:
// unauthenticated access to management endpoints is now refused with 401. // unauthenticated access to management endpoints is now refused with 401.
func TestHandler_RejectsMissingBearer(t *testing.T) { func TestHandler_RejectsMissingBearer(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{Dir: dir}) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
@@ -1124,7 +866,7 @@ func TestHandler_RejectsMissingBearer(t *testing.T) {
// accepted after RegisterJob; stale/forged tokens cannot be replayed. // accepted after RegisterJob; stale/forged tokens cannot be replayed.
func TestHandler_RejectsUnknownBearer(t *testing.T) { func TestHandler_RejectsUnknownBearer(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{Dir: dir}) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
@@ -1144,7 +886,7 @@ func TestHandler_RejectsUnknownBearer(t *testing.T) {
// working the moment the job ends instead of living for the runner's lifetime. // working the moment the job ends instead of living for the runner's lifetime.
func TestHandler_UnregisterRevokes(t *testing.T) { func TestHandler_UnregisterRevokes(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{Dir: dir}) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
@@ -1175,7 +917,7 @@ func TestHandler_UnregisterRevokes(t *testing.T) {
// invisible to queries scoped to repoB. // invisible to queries scoped to repoB.
func TestHandler_CrossRepoIsolation(t *testing.T) { func TestHandler_CrossRepoIsolation(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{Dir: dir}) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
handler.RegisterJob("token-a", JobCredential{Repo: "owner/repoA"}) handler.RegisterJob("token-a", JobCredential{Repo: "owner/repoA"})
@@ -1197,7 +939,7 @@ func TestHandler_CrossRepoIsolation(t *testing.T) {
var reserved struct { var reserved struct {
CacheID uint64 `json:"cacheId"` CacheID uint64 `json:"cacheId"`
} }
require.NoError(t, json.UnmarshalRead(resp.Body, &reserved)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&reserved))
resp.Body.Close() resp.Body.Close()
require.NotZero(t, reserved.CacheID) require.NotZero(t, reserved.CacheID)
@@ -1241,7 +983,7 @@ func TestHandler_CrossRepoIsolation(t *testing.T) {
// working after artifactURLTTL even if the bearer token is still registered. // working after artifactURLTTL even if the bearer token is still registered.
func TestHandler_ArtifactSignature(t *testing.T) { func TestHandler_ArtifactSignature(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{Dir: dir}) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
handler.RegisterJob(testToken, JobCredential{Repo: testRepo}) handler.RegisterJob(testToken, JobCredential{Repo: testRepo})
@@ -1256,7 +998,7 @@ func TestHandler_ArtifactSignature(t *testing.T) {
}) })
t.Run("tampered signature", func(t *testing.T) { t.Run("tampered signature", func(t *testing.T) {
good := signArtifactURL(handler, 1) good := handler.signedArtifactURL(1, time.Now().Add(artifactURLTTL))
bad := good[:len(good)-4] + "dead" bad := good[:len(good)-4] + "dead"
resp, err := testClient.Get(bad) resp, err := testClient.Get(bad)
require.NoError(t, err) require.NoError(t, err)
@@ -1265,7 +1007,7 @@ func TestHandler_ArtifactSignature(t *testing.T) {
}) })
t.Run("expired signature", func(t *testing.T) { t.Run("expired signature", func(t *testing.T) {
expired := handler.signedArtifactURL(JobCredential{}, 1, time.Now().Add(-time.Second)) expired := handler.signedArtifactURL(1, time.Now().Add(-time.Second))
resp, err := testClient.Get(expired) resp, err := testClient.Get(expired)
require.NoError(t, err) require.NoError(t, err)
resp.Body.Close() resp.Body.Close()
@@ -1274,10 +1016,10 @@ func TestHandler_ArtifactSignature(t *testing.T) {
t.Run("signature from a different server", func(t *testing.T) { t.Run("signature from a different server", func(t *testing.T) {
dir2 := filepath.Join(t.TempDir(), "artifactcache2") dir2 := filepath.Join(t.TempDir(), "artifactcache2")
other, err := StartHandler(Options{Dir: dir2}) other, err := StartHandler(dir2, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer other.Close() defer other.Close()
otherURL := signArtifactURL(other, 1) otherURL := other.signedArtifactURL(1, time.Now().Add(artifactURLTTL))
// Rewrite the host so the request still lands on our handler, but // Rewrite the host so the request still lands on our handler, but
// the signature was computed with a different secret. // the signature was computed with a different secret.
parts := strings.SplitN(otherURL, apiPath, 2) parts := strings.SplitN(otherURL, apiPath, 2)
@@ -1296,13 +1038,13 @@ func TestHandler_ArtifactSignature(t *testing.T) {
func TestHandler_SecretPersistsAcrossRestarts(t *testing.T) { func TestHandler_SecretPersistsAcrossRestarts(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
first, err := StartHandler(Options{Dir: dir, OutboundIP: "127.0.0.1"}) first, err := StartHandler(dir, "127.0.0.1", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
exp := time.Now().Add(artifactURLTTL).Unix() exp := time.Now().Add(artifactURLTTL).Unix()
sig := first.computeSignature("", 42, exp) sig := first.computeSignature("", 42, exp)
require.NoError(t, first.Close()) require.NoError(t, first.Close())
second, err := StartHandler(Options{Dir: dir, OutboundIP: "127.0.0.1"}) second, err := StartHandler(dir, "127.0.0.1", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer second.Close() defer second.Close()
@@ -1314,7 +1056,7 @@ func TestHandler_SecretPersistsAcrossRestarts(t *testing.T) {
// the auth refactor. // the auth refactor.
func TestHandler_ArtifactSignatureDownload(t *testing.T) { func TestHandler_ArtifactSignatureDownload(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{Dir: dir}) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
handler.RegisterJob(testToken, JobCredential{Repo: testRepo}) handler.RegisterJob(testToken, JobCredential{Repo: testRepo})
@@ -1331,7 +1073,7 @@ func TestHandler_ArtifactSignatureDownload(t *testing.T) {
var hit struct { var hit struct {
ArchiveLocation string `json:"archiveLocation"` ArchiveLocation string `json:"archiveLocation"`
} }
require.NoError(t, json.UnmarshalRead(resp.Body, &hit)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&hit))
resp.Body.Close() resp.Body.Close()
require.Contains(t, hit.ArchiveLocation, "sig=") require.Contains(t, hit.ArchiveLocation, "sig=")
@@ -1354,7 +1096,7 @@ func TestHandler_ArtifactSignatureDownload(t *testing.T) {
// (restart mid-task, retry), which must not kill the live job's auth. // (restart mid-task, retry), which must not kill the live job's auth.
func TestHandler_RegisterJob_RefCounted(t *testing.T) { func TestHandler_RegisterJob_RefCounted(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{Dir: dir}) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
@@ -1383,10 +1125,10 @@ func TestHandler_RegisterJob_RefCounted(t *testing.T) {
// TestHandler_GC_PerRepoDedup ensures duplicate-pruning does not evict // TestHandler_GC_PerRepoDedup ensures duplicate-pruning does not evict
// another repo's entry. Two repos reserve the same (key, version); after the // another repo's entry. Two repos reserve the same (key, version); after the
// inUseGrace window, GC must keep the one from each repo. // keepOld window, GC must keep the one from each repo.
func TestHandler_GC_PerRepoDedup(t *testing.T) { func TestHandler_GC_PerRepoDedup(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{Dir: dir}) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
handler.RegisterJob("tok-a", JobCredential{Repo: "owner/repoA"}) handler.RegisterJob("tok-a", JobCredential{Repo: "owner/repoA"})
@@ -1400,7 +1142,7 @@ func TestHandler_GC_PerRepoDedup(t *testing.T) {
db, err := handler.openDB() db, err := handler.openDB()
require.NoError(t, err) require.NoError(t, err)
now := time.Now().Unix() now := time.Now().Unix()
stale := time.Now().Add(-inUseGrace - time.Minute).Unix() stale := time.Now().Add(-keepOld - time.Minute).Unix()
a := &Cache{Repo: "owner/repoA", Key: key, Version: version, Complete: true, CreatedAt: stale, UsedAt: stale, Size: 1} a := &Cache{Repo: "owner/repoA", Key: key, Version: version, Complete: true, CreatedAt: stale, UsedAt: stale, Size: 1}
b := &Cache{Repo: "owner/repoB", Key: key, Version: version, Complete: true, CreatedAt: now, UsedAt: now, Size: 1} b := &Cache{Repo: "owner/repoB", Key: key, Version: version, Complete: true, CreatedAt: now, UsedAt: now, Size: 1}
require.NoError(t, insertCache(db, a)) require.NoError(t, insertCache(db, a))
@@ -1437,7 +1179,7 @@ func TestHandler_GC_PerRepoDedup(t *testing.T) {
// register/revoke when the feature is off. // register/revoke when the feature is off.
func TestHandler_InternalAPI_Disabled(t *testing.T) { func TestHandler_InternalAPI_Disabled(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{Dir: dir}) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
@@ -1455,7 +1197,7 @@ func TestHandler_InternalAPI_Disabled(t *testing.T) {
func TestHandler_InternalAPI_AuthAndUsage(t *testing.T) { func TestHandler_InternalAPI_AuthAndUsage(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
const secret = "internal-secret" const secret = "internal-secret"
handler, err := StartHandler(Options{Dir: dir, InternalSecret: secret}) handler, err := StartHandler(dir, "", 0, secret, nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
@@ -5,8 +5,7 @@ package artifactcache
import ( import (
"cmp" "cmp"
"encoding/json/jsontext" "encoding/json"
"encoding/json/v2"
"encoding/xml" "encoding/xml"
"errors" "errors"
"fmt" "fmt"
@@ -78,7 +77,6 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
h.twirpError(w, r, twirpInternal, err) h.twirpError(w, r, twirpInternal, err)
return return
} else if existing != nil { } else if existing != nil {
h.touch(db, existing) // the client skips the upload, so this is the only sign the entry is still in use
h.twirpNotOK(w, r) h.twirpNotOK(w, r)
return return
} }
@@ -99,7 +97,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
h.responseJSON(w, r, http.StatusOK, map[string]any{ h.responseJSON(w, r, http.StatusOK, map[string]any{
"ok": true, "ok": true,
"signed_upload_url": h.signedURL(cred, blobPath, blobUploadPurpose, cache.ID, time.Now().Add(blobUploadURLTTL)), "signed_upload_url": h.signedURL(blobPath, blobUploadPurpose, cache.ID, time.Now().Add(blobUploadURLTTL)),
}) })
} }
@@ -129,7 +127,7 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
} }
db.Close() // commitCache needs the store closed db.Close() // commitCache needs the store closed
cache.Size = int64(cmp.Or(req.SizeBytes, req.SizeBytesCamel)) cache.Size, _ = cmp.Or(req.SizeBytes, req.SizeBytesCamel).Int64()
if err := h.commitCache(cache); err != nil { if err := h.commitCache(cache); err != nil {
h.logger.Errorf("finalize cache %d (%s): %v", cache.ID, cache.Key, err) h.logger.Errorf("finalize cache %d (%s): %v", cache.ID, cache.Key, err)
h.twirpNotOK(w, r) h.twirpNotOK(w, r)
@@ -170,7 +168,7 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
h.responseJSON(w, r, http.StatusOK, map[string]any{ h.responseJSON(w, r, http.StatusOK, map[string]any{
"ok": true, "ok": true,
"signed_download_url": h.signedArtifactURL(cred, cache.ID, time.Now().Add(artifactURLTTL)), "signed_download_url": h.signedArtifactURL(cache.ID, time.Now().Add(artifactURLTTL)),
"matched_key": cache.Key, "matched_key": cache.Key,
}) })
} }
@@ -248,8 +246,8 @@ type (
v2FinalizeRequest struct { v2FinalizeRequest struct {
Key string `json:"key"` Key string `json:"key"`
Version string `json:"version"` Version string `json:"version"`
SizeBytes twirpInt64 `json:"size_bytes"` SizeBytes json.Number `json:"size_bytes"`
SizeBytesCamel twirpInt64 `json:"sizeBytes"` SizeBytesCamel json.Number `json:"sizeBytes"`
} }
v2DownloadRequest struct { v2DownloadRequest struct {
@@ -260,31 +258,6 @@ type (
} }
) )
// twirpInt64 accepts its value as the JSON string the mapping prescribes or as a bare number.
type twirpInt64 int64
func (n *twirpInt64) UnmarshalJSONFrom(dec *jsontext.Decoder) error {
val, err := dec.ReadValue()
if err != nil {
return err
}
digits := []byte(val)
switch val.Kind() {
case 'n': // absent, keep the zero value
return nil
case '"':
if digits, err = jsontext.AppendUnquote(nil, val); err != nil {
return err
}
}
parsed, err := strconv.ParseInt(string(digits), 10, 64)
if err != nil {
return err
}
*n = twirpInt64(parsed)
return nil
}
func (d v2DownloadRequest) keys() []string { func (d v2DownloadRequest) keys() []string {
restoreKeys := d.RestoreKeys restoreKeys := d.RestoreKeys
if len(restoreKeys) == 0 { if len(restoreKeys) == 0 {
@@ -295,6 +268,6 @@ func (d v2DownloadRequest) keys() []string {
func decodeTwirpRequest[T any](r *http.Request) (T, error) { func decodeTwirpRequest[T any](r *http.Request) (T, error) {
var req T var req T
err := json.UnmarshalRead(io.LimitReader(r.Body, 1<<20), &req) err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&req)
return req, err return req, err
} }
@@ -6,12 +6,12 @@ package artifactcache
import ( import (
"bytes" "bytes"
"encoding/base64" "encoding/base64"
"encoding/json/v2" "encoding/json"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"path/filepath"
"strconv" "strconv"
"strings"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -32,7 +32,7 @@ func v2Call(t *testing.T, handler *Handler, client *http.Client, method string,
require.Equal(t, http.StatusOK, resp.StatusCode) require.Equal(t, http.StatusOK, resp.StatusCode)
got := map[string]any{} got := map[string]any{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
return got return got
} }
@@ -66,6 +66,16 @@ func getURL(t *testing.T, url string) []byte {
return body return body
} }
func startTestHandler(t *testing.T) *Handler {
t.Helper()
handler, err := StartHandler(filepath.Join(t.TempDir(), "artifactcache"), "127.0.0.1", 0, "", nil)
require.NoError(t, err)
t.Cleanup(func() { _ = handler.Close() })
handler.RegisterJob(testToken, JobCredential{Repo: testRepo})
return handler
}
// saveV2 runs the reserve/upload/finalize sequence and returns the finalize response along // saveV2 runs the reserve/upload/finalize sequence and returns the finalize response along
// with the upload URL it used. // with the upload URL it used.
func saveV2(t *testing.T, handler *Handler, key, version string, content []byte) (finalized map[string]any, uploadURL string) { func saveV2(t *testing.T, handler *Handler, key, version string, content []byte) (finalized map[string]any, uploadURL string) {
@@ -87,7 +97,7 @@ func saveV2(t *testing.T, handler *Handler, key, version string, content []byte)
// URLs it is handed: unsigned requests are refused, an upload URL cannot be replayed to read // URLs it is handed: unsigned requests are refused, an upload URL cannot be replayed to read
// or to replace a finalized entry. // or to replace a finalized entry.
func TestCacheServiceV2RoundTrip(t *testing.T) { func TestCacheServiceV2RoundTrip(t *testing.T) {
handler := newTestHandler(t, Policy{}) handler := startTestHandler(t)
content := []byte("the cached archive") content := []byte("the cached archive")
unsigned := fmt.Sprintf("%s%s/1", handler.ExternalURL(), blobPath) unsigned := fmt.Sprintf("%s%s/1", handler.ExternalURL(), blobPath)
@@ -116,7 +126,7 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
// A large archive is staged as blocks and only put in order by the final block list, so // A large archive is staged as blocks and only put in order by the final block list, so
// blocks that arrive out of order must still be assembled the way the client asked. // blocks that arrive out of order must still be assembled the way the client asked.
func TestCacheServiceV2BlockUpload(t *testing.T) { func TestCacheServiceV2BlockUpload(t *testing.T) {
handler := newTestHandler(t, Policy{}) handler := startTestHandler(t)
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"}) created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
uploadURL, _ := created["signed_upload_url"].(string) uploadURL, _ := created["signed_upload_url"].(string)
@@ -153,7 +163,7 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
} }
func TestCacheServiceV2Lookups(t *testing.T) { func TestCacheServiceV2Lookups(t *testing.T) {
handler := newTestHandler(t, Policy{}) handler := startTestHandler(t)
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x")) saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
require.Equal(t, true, saved["ok"]) require.Equal(t, true, saved["ok"])
@@ -188,18 +198,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.NotEmpty(t, reserved["signed_upload_url"]) assert.NotEmpty(t, reserved["signed_upload_url"])
}) })
t.Run("a proxied job is handed the address its runner registered", func(t *testing.T) {
const proxy = "https://cache.example.invalid"
handler.RegisterJob("proxied", JobCredential{Repo: testRepo, PublicURL: proxy + "/"})
client := &http.Client{Transport: &bearerTransport{token: "proxied"}}
created := v2Call(t, handler, client, "CreateCacheEntry", map[string]any{"key": "proxied-key", "version": "v1"})
assert.True(t, strings.HasPrefix(created["signed_upload_url"].(string), proxy+blobPath+"/"))
got := v2Call(t, handler, client, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
assert.True(t, strings.HasPrefix(got["signed_download_url"].(string), proxy+apiPath+"/artifacts/"))
})
t.Run("finalizing without a reservation is not ok", func(t *testing.T) { t.Run("finalizing without a reservation is not ok", func(t *testing.T) {
got := v2Call(t, handler, testClient, "FinalizeCacheEntryUpload", map[string]any{ got := v2Call(t, handler, testClient, "FinalizeCacheEntryUpload", map[string]any{
"key": "never-reserved", "version": "v1", "size_bytes": 1, "key": "never-reserved", "version": "v1", "size_bytes": 1,
@@ -227,7 +225,7 @@ func TestCacheServiceV2Lookups(t *testing.T) {
require.Equal(t, http.StatusOK, resp.StatusCode) require.Equal(t, http.StatusOK, resp.StatusCode)
got := map[string]any{} got := map[string]any{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got)) require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
assert.Equal(t, "deps-abc", got["cacheKey"]) assert.Equal(t, "deps-abc", got["cacheKey"])
assert.NotEmpty(t, got["archiveLocation"]) assert.NotEmpty(t, got["archiveLocation"])
}) })
@@ -10,6 +10,23 @@ type Request struct {
Size int64 `json:"cacheSize"` Size int64 `json:"cacheSize"`
} }
func (c *Request) ToCache() *Cache {
if c == nil {
return nil
}
ret := &Cache{
Key: c.Key,
Version: c.Version,
Size: c.Size,
}
if c.Size == 0 {
// So the request comes from old versions of actions, like `actions/cache@v2`.
// It doesn't send cache size. Set it to -1 to indicate that.
ret.Size = -1
}
return ret
}
type Cache struct { type Cache struct {
ID uint64 `json:"id" boltholdKey:"ID"` ID uint64 `json:"id" boltholdKey:"ID"`
Repo string `json:"repo" boltholdIndex:"Repo"` Repo string `json:"repo" boltholdIndex:"Repo"`
@@ -24,7 +24,7 @@ func TestFrontResultsService(t *testing.T) {
})) }))
defer gitea.Close() defer gitea.Close()
handler, err := StartHandler(Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1"}) handler, err := StartHandler(t.TempDir(), "127.0.0.1", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
const token = "forward-token" const token = "forward-token"
@@ -143,13 +143,9 @@ func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
http.ServeFile(w, r, name) http.ServeFile(w, r, name)
} }
// Remove deletes an entry's blob and any staged parts. It reports failure so the caller can func (s *Storage) Remove(id uint64) {
// keep the entry and retry, rather than dropping the only reference to bytes on disk. _ = os.Remove(s.filename(id))
func (s *Storage) Remove(id uint64) error { _ = os.RemoveAll(s.tempDir(id))
if err := os.Remove(s.filename(id)); err != nil && !os.IsNotExist(err) {
return err
}
return os.RemoveAll(s.tempDir(id))
} }
func (s *Storage) filename(id uint64) string { func (s *Storage) filename(id uint64) string {
@@ -6,7 +6,7 @@ package artifacts
import ( import (
"context" "context"
"encoding/json/v2" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@@ -17,7 +17,7 @@ import (
"strings" "strings"
"time" "time"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"github.com/julienschmidt/httprouter" "github.com/julienschmidt/httprouter"
) )
@@ -50,29 +50,65 @@ type ResponseMessage struct {
Message string `json:"message"` Message string `json:"message"`
} }
type WritableFile interface {
io.WriteCloser
}
type WriteFS interface {
OpenWritable(name string) (WritableFile, error)
OpenAppendable(name string) (WritableFile, error)
}
type readWriteFSImpl struct{}
func (fwfs readWriteFSImpl) Open(name string) (fs.File, error) {
return os.Open(name)
}
func (fwfs readWriteFSImpl) OpenWritable(name string) (WritableFile, error) {
if err := os.MkdirAll(filepath.Dir(name), os.ModePerm); err != nil {
return nil, err
}
return os.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o644)
}
func (fwfs readWriteFSImpl) OpenAppendable(name string) (WritableFile, error) {
if err := os.MkdirAll(filepath.Dir(name), os.ModePerm); err != nil {
return nil, err
}
file, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0o644)
if err != nil {
return nil, err
}
_, err = file.Seek(0, io.SeekEnd)
if err != nil {
return nil, err
}
return file, nil
}
var gzipExtension = ".gz__" var gzipExtension = ".gz__"
func safeResolve(baseDir, relPath string) string { func safeResolve(baseDir, relPath string) string {
return filepath.Join(baseDir, filepath.Clean(filepath.Join(string(os.PathSeparator), relPath))) return filepath.Join(baseDir, filepath.Clean(filepath.Join(string(os.PathSeparator), relPath)))
} }
func writeJSON(w http.ResponseWriter, value any) { func uploads(router *httprouter.Router, baseDir string, fsys WriteFS) {
data, err := json.Marshal(value)
if err != nil {
panic(err)
}
if _, err := w.Write(data); err != nil {
panic(err)
}
}
func uploads(router *httprouter.Router, baseDir string) {
router.POST("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { router.POST("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
runID := params.ByName("runId") runID := params.ByName("runId")
writeJSON(w, FileContainerResourceURL{ json, err := json.Marshal(FileContainerResourceURL{
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, runID), FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, runID),
}) })
if err != nil {
panic(err)
}
_, err = w.Write(json)
if err != nil {
panic(err)
}
}) })
router.PUT("/upload/:runId", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { router.PUT("/upload/:runId", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
@@ -86,47 +122,67 @@ func uploads(router *httprouter.Router, baseDir string) {
safeRunPath := safeResolve(baseDir, runID) safeRunPath := safeResolve(baseDir, runID)
safePath := safeResolve(safeRunPath, itemPath) safePath := safeResolve(safeRunPath, itemPath)
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil { file, err := func() (WritableFile, error) {
panic(err) contentRange := req.Header.Get("Content-Range")
if contentRange != "" && !strings.HasPrefix(contentRange, "bytes 0-") {
return fsys.OpenAppendable(safePath)
} }
flags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC return fsys.OpenWritable(safePath)
appendUpload := req.Header.Get("Content-Range") }()
if appendUpload != "" && !strings.HasPrefix(appendUpload, "bytes 0-") {
flags = os.O_CREATE | os.O_WRONLY | os.O_APPEND
}
file, err := os.OpenFile(safePath, flags, 0o644)
if err != nil { if err != nil {
panic(err) panic(err)
} }
defer file.Close() defer file.Close()
if req.Body == nil {
panic(errors.New("no body given")) writer, ok := file.(io.Writer)
if !ok {
panic(errors.New("File is not writable"))
} }
_, err = io.Copy(file, req.Body) if req.Body == nil {
panic(errors.New("No body given"))
}
_, err = io.Copy(writer, req.Body)
if err != nil { if err != nil {
panic(err) panic(err)
} }
writeJSON(w, ResponseMessage{ json, err := json.Marshal(ResponseMessage{
Message: "success", Message: "success",
}) })
if err != nil {
panic(err)
}
_, err = w.Write(json)
if err != nil {
panic(err)
}
}) })
router.PATCH("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { router.PATCH("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
writeJSON(w, ResponseMessage{ json, err := json.Marshal(ResponseMessage{
Message: "success", Message: "success",
}) })
if err != nil {
panic(err)
}
_, err = w.Write(json)
if err != nil {
panic(err)
}
}) })
} }
func downloads(router *httprouter.Router, baseDir string) { func downloads(router *httprouter.Router, baseDir string, fsys fs.FS) {
router.GET("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { router.GET("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
runID := params.ByName("runId") runID := params.ByName("runId")
safePath := safeResolve(baseDir, runID) safePath := safeResolve(baseDir, runID)
entries, err := os.ReadDir(safePath) entries, err := fs.ReadDir(fsys, safePath)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@@ -139,10 +195,18 @@ func downloads(router *httprouter.Router, baseDir string) {
}) })
} }
writeJSON(w, NamedFileContainerResourceURLResponse{ json, err := json.Marshal(NamedFileContainerResourceURLResponse{
Count: len(list), Count: len(list),
Value: list, Value: list,
}) })
if err != nil {
panic(err)
}
_, err = w.Write(json)
if err != nil {
panic(err)
}
}) })
router.GET("/download/:container", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { router.GET("/download/:container", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
@@ -151,7 +215,7 @@ func downloads(router *httprouter.Router, baseDir string) {
safePath := safeResolve(baseDir, filepath.Join(container, itemPath)) safePath := safeResolve(baseDir, filepath.Join(container, itemPath))
var files []ContainerItem var files []ContainerItem
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error { err := fs.WalkDir(fsys, safePath, func(path string, entry fs.DirEntry, err error) error {
if !entry.IsDir() { if !entry.IsDir() {
rel, err := filepath.Rel(safePath, path) rel, err := filepath.Rel(safePath, path)
if err != nil { if err != nil {
@@ -177,9 +241,17 @@ func downloads(router *httprouter.Router, baseDir string) {
panic(err) panic(err)
} }
writeJSON(w, ContainerItemResponse{ json, err := json.Marshal(ContainerItemResponse{
Value: files, Value: files,
}) })
if err != nil {
panic(err)
}
_, err = w.Write(json)
if err != nil {
panic(err)
}
}) })
router.GET("/artifact/*path", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { router.GET("/artifact/*path", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
@@ -187,16 +259,15 @@ func downloads(router *httprouter.Router, baseDir string) {
safePath := safeResolve(baseDir, path) safePath := safeResolve(baseDir, path)
file, err := os.Open(safePath) file, err := fsys.Open(safePath)
if err != nil { if err != nil {
// try gzip file // try gzip file
file, err = os.Open(safePath + gzipExtension) file, err = fsys.Open(safePath + gzipExtension)
if err != nil { if err != nil {
panic(err) panic(err)
} }
w.Header().Add("Content-Encoding", "gzip") w.Header().Add("Content-Encoding", "gzip")
} }
defer file.Close()
_, err = io.Copy(w, file) _, err = io.Copy(w, file)
if err != nil { if err != nil {
@@ -216,8 +287,9 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
router := httprouter.New() router := httprouter.New()
logger.Debugf("Artifacts base path '%s'", artifactPath) logger.Debugf("Artifacts base path '%s'", artifactPath)
uploads(router, artifactPath) fsys := readWriteFSImpl{}
downloads(router, artifactPath) uploads(router, artifactPath, fsys)
downloads(router, artifactPath, fsys)
server := &http.Server{ server := &http.Server{
Addr: fmt.Sprintf("%s:%s", addr, port), Addr: fmt.Sprintf("%s:%s", addr, port),
+481
View File
@@ -0,0 +1,481 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// Copyright 2021 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package artifacts
import (
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
"maps"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"testing/fstest"
"time"
"github.com/julienschmidt/httprouter"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type writableMapFile struct {
fstest.MapFile
}
func (f *writableMapFile) Write(data []byte) (int, error) {
f.Data = data
return len(data), nil
}
func (f *writableMapFile) Close() error {
return nil
}
type writeMapFS struct {
fstest.MapFS
}
func (fsys writeMapFS) OpenWritable(name string) (WritableFile, error) {
file := &writableMapFile{
MapFile: fstest.MapFile{
Data: []byte("content2"),
},
}
fsys.MapFS[name] = &file.MapFile
return file, nil
}
func (fsys writeMapFS) OpenAppendable(name string) (WritableFile, error) {
file := &writableMapFile{
MapFile: fstest.MapFile{
Data: []byte("content2"),
},
}
fsys.MapFS[name] = &file.MapFile
return file, nil
}
func TestNewArtifactUploadPrepare(t *testing.T) {
assert := assert.New(t)
memfs := fstest.MapFS(map[string]*fstest.MapFile{})
router := httprouter.New()
uploads(router, "artifact/server/path", writeMapFS{memfs})
req, _ := http.NewRequest(http.MethodPost, "http://localhost/_apis/pipelines/workflows/1/artifacts", nil)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if status := rr.Code; status != http.StatusOK {
assert.Fail("Wrong status")
}
response := FileContainerResourceURL{}
err := json.Unmarshal(rr.Body.Bytes(), &response)
if err != nil {
panic(err)
}
assert.Equal("http://localhost/upload/1", response.FileContainerResourceURL)
}
func TestArtifactUploadBlob(t *testing.T) {
assert := assert.New(t)
memfs := fstest.MapFS(map[string]*fstest.MapFile{})
router := httprouter.New()
uploads(router, "artifact/server/path", writeMapFS{memfs})
req, _ := http.NewRequest(http.MethodPut, "http://localhost/upload/1?itemPath=some/file", strings.NewReader("content"))
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if status := rr.Code; status != http.StatusOK {
assert.Fail("Wrong status")
}
response := ResponseMessage{}
err := json.Unmarshal(rr.Body.Bytes(), &response)
if err != nil {
panic(err)
}
assert.Equal("success", response.Message)
assert.Equal("content", string(memfs["artifact/server/path/1/some/file"].Data))
}
func TestFinalizeArtifactUpload(t *testing.T) {
assert := assert.New(t)
memfs := fstest.MapFS(map[string]*fstest.MapFile{})
router := httprouter.New()
uploads(router, "artifact/server/path", writeMapFS{memfs})
req, _ := http.NewRequest(http.MethodPatch, "http://localhost/_apis/pipelines/workflows/1/artifacts", nil)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if status := rr.Code; status != http.StatusOK {
assert.Fail("Wrong status")
}
response := ResponseMessage{}
err := json.Unmarshal(rr.Body.Bytes(), &response)
if err != nil {
panic(err)
}
assert.Equal("success", response.Message)
}
func TestListArtifacts(t *testing.T) {
assert := assert.New(t)
memfs := fstest.MapFS(map[string]*fstest.MapFile{
"artifact/server/path/1/file.txt": {
Data: []byte(""),
},
})
router := httprouter.New()
downloads(router, "artifact/server/path", memfs)
req, _ := http.NewRequest(http.MethodGet, "http://localhost/_apis/pipelines/workflows/1/artifacts", nil)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if status := rr.Code; status != http.StatusOK {
assert.FailNow(fmt.Sprintf("Wrong status: %d", status))
}
response := NamedFileContainerResourceURLResponse{}
err := json.Unmarshal(rr.Body.Bytes(), &response)
if err != nil {
panic(err)
}
assert.Equal(1, response.Count)
assert.Equal("file.txt", response.Value[0].Name)
assert.Equal("http://localhost/download/1", response.Value[0].FileContainerResourceURL)
}
func TestListArtifactContainer(t *testing.T) {
assert := assert.New(t)
memfs := fstest.MapFS(map[string]*fstest.MapFile{
"artifact/server/path/1/some/file": {
Data: []byte(""),
},
})
router := httprouter.New()
downloads(router, "artifact/server/path", memfs)
req, _ := http.NewRequest(http.MethodGet, "http://localhost/download/1?itemPath=some/file", nil)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if status := rr.Code; status != http.StatusOK {
assert.FailNow(fmt.Sprintf("Wrong status: %d", status))
}
response := ContainerItemResponse{}
err := json.Unmarshal(rr.Body.Bytes(), &response)
if err != nil {
panic(err)
}
assert.Len(response.Value, 1)
assert.Equal("some/file", response.Value[0].Path)
assert.Equal("file", response.Value[0].ItemType)
assert.Equal("http://localhost/artifact/1/some/file/.", response.Value[0].ContentLocation)
}
func TestDownloadArtifactFile(t *testing.T) {
assert := assert.New(t)
memfs := fstest.MapFS(map[string]*fstest.MapFile{
"artifact/server/path/1/some/file": {
Data: []byte("content"),
},
})
router := httprouter.New()
downloads(router, "artifact/server/path", memfs)
req, _ := http.NewRequest(http.MethodGet, "http://localhost/artifact/1/some/file", nil)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if status := rr.Code; status != http.StatusOK {
assert.FailNow(fmt.Sprintf("Wrong status: %d", status))
}
data := rr.Body.Bytes()
assert.Equal("content", string(data))
}
// TestArtifactFlow drives the real Serve() artifact server over a loopback socket, exercising
// the same upload -> finalize -> list -> download protocol the upload-artifact/download-artifact
// actions speak. Running it in-process (rather than from a job container) keeps it network-free
// and reachable everywhere, including when the CI job is itself a container.
func TestArtifactFlow(t *testing.T) {
artifactPath := t.TempDir()
// Serve the exact routes Serve() wires up, on a real loopback socket via httptest. httptest
// picks a free port and Close() tears the server down synchronously — avoiding both the
// port-rebind race and Serve()'s detached ListenAndServe goroutine, which logger.Fatal()s
// (process exit) on a bind error and can outlive the test's temp-dir cleanup.
router := httprouter.New()
fsys := readWriteFSImpl{}
uploads(router, artifactPath, fsys)
downloads(router, artifactPath, fsys)
server := httptest.NewServer(router)
defer server.Close()
baseURL := server.URL
client := server.Client()
client.Timeout = 5 * time.Second
// request performs one HTTP call and returns the status and body. The default transport adds
// Accept-Encoding: gzip and transparently decompresses, so gzipped downloads come back plain.
request := func(t *testing.T, method, rawURL string, body io.Reader, header http.Header) (int, []byte) {
t.Helper()
req, err := http.NewRequest(method, rawURL, body)
require.NoError(t, err)
maps.Copy(req.Header, header)
resp, err := client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
data, err := io.ReadAll(resp.Body)
require.NoError(t, err)
return resp.StatusCode, data
}
t.Run("upload-and-download", func(t *testing.T) {
const runID, item, content = "1", "my-artifact/data.txt", "hello artifact\n"
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
var prep FileContainerResourceURL
require.NoError(t, json.Unmarshal(data, &prep))
require.Equal(t, baseURL+"/upload/"+runID, prep.FileContainerResourceURL)
status, data = request(t, http.MethodPut, prep.FileContainerResourceURL+"?itemPath="+url.QueryEscape(item), strings.NewReader(content), nil)
require.Equal(t, http.StatusOK, status, string(data))
var msg ResponseMessage
require.NoError(t, json.Unmarshal(data, &msg))
require.Equal(t, "success", msg.Message)
status, data = request(t, http.MethodPatch, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
status, data = request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
var list NamedFileContainerResourceURLResponse
require.NoError(t, json.Unmarshal(data, &list))
require.Equal(t, 1, list.Count)
require.Equal(t, "my-artifact", list.Value[0].Name)
status, data = request(t, http.MethodGet, list.Value[0].FileContainerResourceURL+"?itemPath=my-artifact", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
var items ContainerItemResponse
require.NoError(t, json.Unmarshal(data, &items))
require.Len(t, items.Value, 1)
require.Equal(t, "file", items.Value[0].ItemType)
require.Equal(t, "my-artifact/data.txt", items.Value[0].Path)
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
require.Equal(t, http.StatusOK, status)
require.Equal(t, content, string(data))
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "my-artifact", "data.txt"))
require.NoError(t, err)
require.Equal(t, content, string(stored))
})
t.Run("gzip-roundtrip", func(t *testing.T) {
const runID, item, content = "2", "logs/app.log", "compressed payload\n"
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
_, err := gz.Write([]byte(content))
require.NoError(t, err)
require.NoError(t, gz.Close())
status, data := request(t, http.MethodPut, baseURL+"/upload/"+runID+"?itemPath="+url.QueryEscape(item),
&buf, http.Header{"Content-Encoding": []string{"gzip"}})
require.Equal(t, http.StatusOK, status, string(data))
// stored compressed, with the server's gzip marker suffix
_, err = os.Stat(filepath.Join(artifactPath, runID, "logs", "app.log.gz__"))
require.NoError(t, err)
status, data = request(t, http.MethodGet, baseURL+"/download/"+runID+"?itemPath=logs", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
var items ContainerItemResponse
require.NoError(t, json.Unmarshal(data, &items))
require.Len(t, items.Value, 1)
require.Equal(t, "logs/app.log", items.Value[0].Path)
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
require.Equal(t, http.StatusOK, status)
require.Equal(t, content, string(data))
})
// GHSL-2023-004: an itemPath that climbs out of the run directory must be neutralised so the
// blob cannot be written outside the artifact root.
t.Run("GHSL-2023-004", func(t *testing.T) {
const runID, content = "3", "contained\n"
status, data := request(t, http.MethodPut, baseURL+"/upload/"+runID+"?itemPath="+url.QueryEscape("../../escape.txt"),
strings.NewReader(content), nil)
require.Equal(t, http.StatusOK, status, string(data))
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "escape.txt"))
require.NoError(t, err)
require.Equal(t, content, string(stored))
_, err = os.Stat(filepath.Join(filepath.Dir(artifactPath), "escape.txt"))
require.True(t, os.IsNotExist(err), "upload escaped the artifact root")
status, data = request(t, http.MethodGet, baseURL+"/artifact/"+runID+"/escape.txt", nil, nil)
require.Equal(t, http.StatusOK, status)
require.Equal(t, content, string(data))
})
}
func TestMkdirFsImplSafeResolve(t *testing.T) {
assert := assert.New(t)
baseDir := "/foo/bar"
tests := map[string]struct {
input string
want string
}{
"simple": {input: "baz", want: "/foo/bar/baz"},
"nested": {input: "baz/blue", want: "/foo/bar/baz/blue"},
"dots in middle": {input: "baz/../../blue", want: "/foo/bar/blue"},
"leading dots": {input: "../../parent", want: "/foo/bar/parent"},
"root path": {input: "/root", want: "/foo/bar/root"},
"root": {input: "/", want: "/foo/bar"},
"empty": {input: "", want: "/foo/bar"},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
assert.Equal(tc.want, safeResolve(baseDir, tc.input))
})
}
}
func TestReadWriteFSWritableAndAppendable(t *testing.T) {
fsys := readWriteFSImpl{}
name := filepath.Join(t.TempDir(), "nested", "artifact.txt")
w, err := fsys.OpenWritable(name)
require.NoError(t, err)
_, err = w.Write([]byte("first"))
require.NoError(t, err)
require.NoError(t, w.Close())
w, err = fsys.OpenAppendable(name)
require.NoError(t, err)
_, err = w.Write([]byte("-second"))
require.NoError(t, err)
require.NoError(t, w.Close())
got, err := os.ReadFile(name)
require.NoError(t, err)
require.Equal(t, "first-second", string(got))
w, err = fsys.OpenWritable(name)
require.NoError(t, err)
_, err = w.Write([]byte("replaced"))
require.NoError(t, err)
require.NoError(t, w.Close())
got, err = os.ReadFile(name)
require.NoError(t, err)
require.Equal(t, "replaced", string(got))
}
func TestServeEmptyArtifactPathReturnsCancelableNoop(t *testing.T) {
cancel := Serve(t.Context(), "", "127.0.0.1", "0")
require.NotNil(t, cancel)
cancel()
}
func TestDownloadArtifactFileUnsafePath(t *testing.T) {
assert := assert.New(t)
memfs := fstest.MapFS(map[string]*fstest.MapFile{
"artifact/server/path/some/file": {
Data: []byte("content"),
},
})
router := httprouter.New()
downloads(router, "artifact/server/path", memfs)
req, _ := http.NewRequest(http.MethodGet, "http://localhost/artifact/2/../../some/file", nil)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if status := rr.Code; status != http.StatusOK {
assert.FailNow(fmt.Sprintf("Wrong status: %d", status))
}
data := rr.Body.Bytes()
assert.Equal("content", string(data))
}
func TestArtifactUploadBlobUnsafePath(t *testing.T) {
assert := assert.New(t)
memfs := fstest.MapFS(map[string]*fstest.MapFile{})
router := httprouter.New()
uploads(router, "artifact/server/path", writeMapFS{memfs})
req, _ := http.NewRequest(http.MethodPut, "http://localhost/upload/1?itemPath=../../some/file", strings.NewReader("content"))
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if status := rr.Code; status != http.StatusOK {
assert.Fail("Wrong status")
}
response := ResponseMessage{}
err := json.Unmarshal(rr.Body.Bytes(), &response)
if err != nil {
panic(err)
}
assert.Equal("success", response.Message)
assert.Equal("content", string(memfs["artifact/server/path/1/some/file"].Data))
}
@@ -54,6 +54,22 @@ func NewPipelineExecutor(executors ...Executor) Executor {
return rtn return rtn
} }
// NewConditionalExecutor creates a new executor based on conditions
func NewConditionalExecutor(conditional Conditional, trueExecutor, falseExecutor Executor) Executor {
return func(ctx context.Context) error {
if conditional(ctx) {
if trueExecutor != nil {
return trueExecutor(ctx)
}
} else {
if falseExecutor != nil {
return falseExecutor(ctx)
}
}
return nil
}
}
// NewErrorExecutor creates a new executor that always errors out // NewErrorExecutor creates a new executor that always errors out
func NewErrorExecutor(err error) Executor { func NewErrorExecutor(err error) Executor {
return func(ctx context.Context) error { return func(ctx context.Context) error {
@@ -171,8 +187,15 @@ func (e Executor) Finally(finally Executor) Executor {
err := e(ctx) err := e(ctx)
err2 := finally(ctx) err2 := finally(ctx)
if err2 != nil { if err2 != nil {
return fmt.Errorf("error occurred running finally: %v (original error: %v)", err2, err) return fmt.Errorf("Error occurred running finally: %v (original error: %v)", err2, err)
} }
return err return err
} }
} }
// Not return an inverted conditional
func (c Conditional) Not() Conditional {
return func(ctx context.Context) bool {
return !c(ctx)
}
}
@@ -45,6 +45,43 @@ func TestNewWorkflow(t *testing.T) {
assert.Equal(2, runcount) assert.Equal(2, runcount)
} }
func TestNewConditionalExecutor(t *testing.T) {
assert := assert.New(t)
ctx := context.Background()
trueCount := 0
falseCount := 0
err := NewConditionalExecutor(func(ctx context.Context) bool {
return false
}, func(ctx context.Context) error {
trueCount++
return nil
}, func(ctx context.Context) error {
falseCount++
return nil
})(ctx)
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(0, trueCount)
assert.Equal(1, falseCount)
err = NewConditionalExecutor(func(ctx context.Context) bool {
return true
}, func(ctx context.Context) error {
trueCount++
return nil
}, func(ctx context.Context) error {
falseCount++
return nil
})(ctx)
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(1, trueCount)
assert.Equal(1, falseCount)
}
// concurrencyProbe returns an executor recording the peak number of concurrent copies. Copies // concurrencyProbe returns an executor recording the peak number of concurrent copies. Copies
// block until wantActive are in flight so the peak is exact without sleeping, and later copies // block until wantActive are in flight so the peak is exact without sleeping, and later copies
// find the gate already open so the last one still finishes with no partner left. // find the gate already open so the last one still finishes with no partner left.
@@ -186,3 +223,10 @@ func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
t.Fatalf("finally error = %q, want both cleanup and original error", err) t.Fatalf("finally error = %q, want both cleanup and original error", err)
} }
} }
func TestConditionalNot(t *testing.T) {
cond := Conditional(func(context.Context) bool { return false })
if !cond.Not()(context.Background()) {
t.Fatal("inverted conditional should be true")
}
}
@@ -15,7 +15,7 @@ import (
"strings" "strings"
"sync" "sync"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"gitea.com/gitea/runner/internal/pkg/lock" "gitea.com/gitea/runner/internal/pkg/lock"
"github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5"
@@ -36,6 +36,7 @@ var (
cloneLocks lock.Keyed[string] // key: clone target directory cloneLocks lock.Keyed[string] // key: clone target directory
ErrShortRef = errors.New("short SHA references are not supported") ErrShortRef = errors.New("short SHA references are not supported")
ErrNoRepo = errors.New("unable to find git repo")
) )
// AcquireCloneLock returns an unlock function after locking the per-directory mutex for dir. // AcquireCloneLock returns an unlock function after locking the per-directory mutex for dir.
@@ -186,16 +187,19 @@ func FindGitRef(ctx context.Context, file string) (string, error) {
} }
// FindGithubRepo get the repo // FindGithubRepo get the repo
func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, error) { func FindGithubRepo(ctx context.Context, file, githubInstance, remoteName string) (string, error) {
goGitMu.Lock() goGitMu.Lock()
defer goGitMu.Unlock() defer goGitMu.Unlock()
if remoteName == "" {
remoteName = "origin"
}
url, err := findGitRemoteURL(ctx, file, "origin") url, err := findGitRemoteURL(ctx, file, remoteName)
if err != nil { if err != nil {
return "", err return "", err
} }
_, slug := findGitSlug(url, githubInstance) _, slug, err := findGitSlug(url, githubInstance)
return slug, nil return slug, err
} }
func findGitRemoteURL(_ context.Context, file, remoteName string) (string, error) { func findGitRemoteURL(_ context.Context, file, remoteName string) (string, error) {
@@ -222,25 +226,25 @@ func findGitRemoteURL(_ context.Context, file, remoteName string) (string, error
return remote.Config().URLs[0], nil return remote.Config().URLs[0], nil
} }
func findGitSlug(url, githubInstance string) (string, string) { func findGitSlug(url, githubInstance string) (string, string, error) { //nolint:unparam // pre-existing issue from nektos/act
if matches := codeCommitHTTPRegex.FindStringSubmatch(url); matches != nil { if matches := codeCommitHTTPRegex.FindStringSubmatch(url); matches != nil {
return "CodeCommit", matches[2] return "CodeCommit", matches[2], nil
} else if matches := codeCommitSSHRegex.FindStringSubmatch(url); matches != nil { } else if matches := codeCommitSSHRegex.FindStringSubmatch(url); matches != nil {
return "CodeCommit", matches[2] return "CodeCommit", matches[2], nil
} else if matches := githubHTTPRegex.FindStringSubmatch(url); matches != nil { } else if matches := githubHTTPRegex.FindStringSubmatch(url); matches != nil {
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2]) return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2]), nil
} else if matches := githubSSHRegex.FindStringSubmatch(url); matches != nil { } else if matches := githubSSHRegex.FindStringSubmatch(url); matches != nil {
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2]) return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2]), nil
} else if githubInstance != "github.com" { } else if githubInstance != "github.com" {
gheHTTPRegex := regexp.MustCompile(fmt.Sprintf(`^https?://%s/(.+)/(.+?)(?:.git)?$`, githubInstance)) gheHTTPRegex := regexp.MustCompile(fmt.Sprintf(`^https?://%s/(.+)/(.+?)(?:.git)?$`, githubInstance))
gheSSHRegex := regexp.MustCompile(githubInstance + "[:/](.+)/(.+?)(?:.git)?$") gheSSHRegex := regexp.MustCompile(githubInstance + "[:/](.+)/(.+?)(?:.git)?$")
if matches := gheHTTPRegex.FindStringSubmatch(url); matches != nil { if matches := gheHTTPRegex.FindStringSubmatch(url); matches != nil {
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2]) return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2]), nil
} else if matches := gheSSHRegex.FindStringSubmatch(url); matches != nil { } else if matches := gheSSHRegex.FindStringSubmatch(url); matches != nil {
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2]) return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2]), nil
} }
} }
return "", url return "", url, nil
} }
// NewGitCloneExecutorInput the input for the NewGitCloneExecutor // NewGitCloneExecutorInput the input for the NewGitCloneExecutor
@@ -274,12 +278,11 @@ func CloneIfRequired(ctx context.Context, refName plumbing.ReferenceName, input
return r, true, nil return r, true, nil
} }
switch { if err != nil {
case err != nil:
logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err) logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err)
case len(remote.Config().URLs) == 0: } else if len(remote.Config().URLs) == 0 {
logger.Debugf("Removing cached clone at %s because origin has no URL", input.Dir) logger.Debugf("Removing cached clone at %s because origin has no URL", input.Dir)
default: } else {
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, remote.Config().URLs[0], input.URL) logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, remote.Config().URLs[0], input.URL)
} }
if err := os.RemoveAll(input.Dir); err != nil { if err := os.RemoveAll(input.Dir); err != nil {
@@ -342,16 +345,6 @@ func gitOptions(token string) (fetchOptions git.FetchOptions, pullOptions git.Pu
return fetchOptions, pullOptions return fetchOptions, pullOptions
} }
// staleRefreshErr reports why a failed refresh must abort: the resolve and
// checkout that follow are local and succeed on a cancelled context, which
// would hand back the cached revision as if it were fresh.
func staleRefreshErr(ctx context.Context, err error) error {
if err == nil || errors.Is(err, git.NoErrAlreadyUpToDate) {
return nil
}
return ctx.Err()
}
// NewGitCloneExecutor creates an executor to clone git repos // NewGitCloneExecutor creates an executor to clone git repos
func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor { func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
return func(ctx context.Context) error { return func(ctx context.Context) error {
@@ -392,7 +385,7 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
} }
if !isOfflineMode { if !isOfflineMode {
err = r.FetchContext(ctx, &fetchOptions) err = r.Fetch(&fetchOptions)
if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
return err return err
} }
@@ -461,17 +454,18 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
switch { switch {
case !isOfflineMode && !shallow: case !isOfflineMode && !shallow:
// In shallow mode the depth-limited fetch above already advanced the ref. // In shallow mode the depth-limited fetch above already advanced the ref.
if err = w.PullContext(ctx, &pullOptions); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { if err = w.Pull(&pullOptions); err != nil && err != git.NoErrAlreadyUpToDate {
logger.Debugf("Unable to pull %s: %v", refName, err) logger.Debugf("Unable to pull %s: %v", refName, err)
} }
if err := staleRefreshErr(ctx, err); err != nil {
return err
}
case isOfflineMode && reused: case isOfflineMode && reused:
reusedMsg = " (reused in offline mode)" reusedMsg = " (offline mode)"
} }
logger.Debugf("Cloned %s to %s%s", input.URL, input.Dir, reusedMsg) if reused {
logger.Debugf("Reused %s at %s%s", input.URL, input.Dir, reusedMsg)
} else {
logger.Debugf("Cloned %s to %s", input.URL, input.Dir)
}
if hash.String() != input.Ref && refType == "branch" { if hash.String() != input.Ref && refType == "branch" {
logger.Debugf("Provided ref is not a sha. Updating branch ref after pull") logger.Debugf("Provided ref is not a sha. Updating branch ref after pull")
@@ -6,24 +6,18 @@ package git
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"net/http"
"net/http/httptest"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
"sync"
"syscall" "syscall"
"testing" "testing"
"time" "time"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
gogit "github.com/go-git/go-git/v5"
gogitconfig "github.com/go-git/go-git/v5/config"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
logrustest "github.com/sirupsen/logrus/hooks/test" logrustest "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -51,7 +45,9 @@ func TestFindGitSlug(t *testing.T) {
} }
for _, tt := range slugTests { for _, tt := range slugTests {
provider, slug := findGitSlug(tt.url, "github.com") provider, slug, err := findGitSlug(tt.url, "github.com")
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(tt.provider, provider) assert.Equal(tt.provider, provider)
assert.Equal(tt.slug, slug) assert.Equal(tt.slug, slug)
} }
@@ -85,20 +81,45 @@ func cleanGitHooks(dir string) error {
return nil return nil
} }
func TestFindGithubRepoUsesOrigin(t *testing.T) { func TestFindGitRemoteURL(t *testing.T) {
assert := assert.New(t)
basedir := t.TempDir()
err := gitCmd("init", basedir)
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
err = cleanGitHooks(basedir)
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
remoteURL := "https://git-codecommit.us-east-1.amazonaws.com/v1/repos/my-repo-name"
err = gitCmd("-C", basedir, "remote", "add", "origin", remoteURL)
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
u, err := findGitRemoteURL(context.Background(), basedir, "origin")
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(remoteURL, u)
remoteURL = "git@github.com/AwesomeOwner/MyAwesomeRepo.git"
err = gitCmd("-C", basedir, "remote", "add", "upstream", remoteURL)
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
u, err = findGitRemoteURL(context.Background(), basedir, "upstream")
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(remoteURL, u)
}
func TestFindGithubRepoUsesOriginAndCustomRemote(t *testing.T) {
basedir := t.TempDir() basedir := t.TempDir()
const remoteURL = "https://github.com/owner/repo.git"
require.NoError(t, gitCmd("init", basedir)) require.NoError(t, gitCmd("init", basedir))
require.NoError(t, cleanGitHooks(basedir)) require.NoError(t, cleanGitHooks(basedir))
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL)) require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", "https://github.com/owner/repo.git"))
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "ghe", "git@git.example.com:team/project.git"))
url, err := findGitRemoteURL(context.Background(), basedir, "origin") slug, err := FindGithubRepo(context.Background(), basedir, "github.com", "")
require.NoError(t, err)
require.Equal(t, remoteURL, url)
slug, err := FindGithubRepo(context.Background(), basedir, "github.com")
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "owner/repo", slug) require.Equal(t, "owner/repo", slug)
slug, err = FindGithubRepo(context.Background(), basedir, "git.example.com", "ghe")
require.NoError(t, err)
require.Equal(t, "team/project", slug)
} }
func TestGitFindRef(t *testing.T) { func TestGitFindRef(t *testing.T) {
@@ -352,22 +373,28 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) {
// Prime the cache with an online clone of main. // Prime the cache with an online clone of main.
cacheDir := t.TempDir() cacheDir := t.TempDir()
logger, hook := logrustest.NewNullLogger()
logger.SetLevel(log.DebugLevel)
ctx := common.WithLogger(context.Background(), logger.WithField("job", "j1"))
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{ require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, URL: remoteDir,
Ref: "main", Ref: "main",
Dir: cacheDir, Dir: cacheDir,
})(context.Background())) })(ctx))
assert.Contains(t, logMessages(hook), "Cloned "+remoteDir+" to "+cacheDir)
t.Run("cached branch resolves without fetching", func(t *testing.T) { t.Run("cached branch resolves without fetching", func(t *testing.T) {
// Offline reuse of a cached branch must succeed even though ResolveRevision(input.Ref) // Offline reuse of a cached branch must succeed even though ResolveRevision(input.Ref)
// finds no local refs/heads/<ref>. // finds no local refs/heads/<ref>.
hook.Reset()
err := NewGitCloneExecutor(NewGitCloneExecutorInput{ err := NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, URL: remoteDir,
Ref: "main", Ref: "main",
Dir: cacheDir, Dir: cacheDir,
OfflineMode: true, OfflineMode: true,
})(context.Background()) })(ctx)
require.NoError(t, err) require.NoError(t, err)
assert.Contains(t, logMessages(hook), "Reused "+remoteDir+" at "+cacheDir+" (offline mode)")
out, err := exec.Command("git", "-C", cacheDir, "log", "--oneline", "-1", "--format=%s").Output() out, err := exec.Command("git", "-C", cacheDir, "log", "--oneline", "-1", "--format=%s").Output()
require.NoError(t, err) require.NoError(t, err)
@@ -424,6 +451,14 @@ func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) {
} }
} }
func logMessages(hook *logrustest.Hook) []string {
messages := []string{}
for _, entry := range hook.AllEntries() {
messages = append(messages, entry.Message)
}
return messages
}
func TestGitCloneExecutorShallow(t *testing.T) { func TestGitCloneExecutorShallow(t *testing.T) {
// Build a local "remote" with several commits on main plus a tag, so a full clone would pull noticeably more history than a shallow one. // Build a local "remote" with several commits on main plus a tag, so a full clone would pull noticeably more history than a shallow one.
remoteDir := t.TempDir() remoteDir := t.TempDir()
@@ -589,55 +624,3 @@ func TestAcquireCloneLock(t *testing.T) {
} }
}) })
} }
// An unresponsive remote must not pin a job: the refresh has to be interruptible.
func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
block := make(chan struct{})
reached := make(chan struct{})
var once sync.Once
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
once.Do(func() { close(reached) })
<-block
}))
t.Cleanup(func() {
close(block)
server.Close()
})
dir := filepath.Join(t.TempDir(), "cached-action")
repo, err := gogit.PlainInit(dir, false)
require.NoError(t, err)
_, err = repo.CreateRemote(&gogitconfig.RemoteConfig{Name: "origin", URLs: []string{server.URL}})
require.NoError(t, err)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
done := make(chan error, 1)
go func() {
done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir})(ctx)
}()
select {
case <-reached:
case <-time.After(10 * time.Second):
t.Fatal("the executor never reached the remote")
}
cancel()
select {
case err := <-done:
require.Error(t, err)
case <-time.After(10 * time.Second):
t.Fatal("fetch ignored context cancellation")
}
}
func TestStaleRefreshErr(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
require.NoError(t, staleRefreshErr(ctx, errors.New("remote hung up")))
cancel()
require.ErrorIs(t, staleRefreshErr(ctx, errors.New("remote hung up")), context.Canceled)
require.NoError(t, staleRefreshErr(ctx, gogit.NoErrAlreadyUpToDate))
}
@@ -46,14 +46,14 @@ func (lw *lineWriter) Write(p []byte) (n int, err error) {
line, err := pBuf.ReadString('\n') line, err := pBuf.ReadString('\n')
w, _ := lw.buffer.WriteString(line) w, _ := lw.buffer.WriteString(line)
written += w written += w
if err != nil { if err == nil {
if err == io.EOF {
break
}
return written, err
}
lw.handleLine(lw.buffer.String()) lw.handleLine(lw.buffer.String())
lw.buffer.Reset() lw.buffer.Reset()
} else if err == io.EOF {
break
} else {
return written, err
}
} }
return written, nil return written, nil
@@ -10,7 +10,7 @@ import (
"fmt" "fmt"
"io" "io"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"github.com/docker/go-connections/nat" "github.com/docker/go-connections/nat"
"github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/container"
@@ -41,8 +41,7 @@ type NewContainerInput struct {
Privileged bool Privileged bool
UsernsMode string UsernsMode string
Platform string Platform string
RunnerOptions string // container options the runner was configured with, trusted Options string
WorkflowOptions string // container options the workflow asked for, untrusted
NetworkAliases []string NetworkAliases []string
ExposedPorts nat.PortSet ExposedPorts nat.PortSet
PortBindings nat.PortMap PortBindings nat.PortMap
@@ -89,7 +88,9 @@ type Info struct {
// Container for managing docker run containers // Container for managing docker run containers
type Container interface { type Container interface {
Create(capAdd, capDrop []string) common.Executor Create(capAdd, capDrop []string) common.Executor
ConnectToNetwork(name string) common.Executor
Copy(destPath string, files ...*FileEntry) common.Executor Copy(destPath string, files ...*FileEntry) common.Executor
CopyTarStream(ctx context.Context, destPath string, tarStream io.Reader) error
CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor
GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error)
Inspect(ctx context.Context) (*Info, error) Inspect(ctx context.Context) (*Info, error)
@@ -9,7 +9,7 @@ package container
import ( import (
"context" "context"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"github.com/distribution/reference" "github.com/distribution/reference"
"github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config"
@@ -12,7 +12,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"github.com/moby/go-archive" "github.com/moby/go-archive"
"github.com/moby/go-archive/compression" "github.com/moby/go-archive/compression"
@@ -17,7 +17,8 @@
package container package container
import ( import (
"encoding/json/jsontext" "bytes"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"net" "net"
@@ -350,7 +351,7 @@ type containerConfig struct {
// parse parses the args for the specified command and generates a Config, // parse parses the args for the specified command and generates a Config,
// a HostConfig and returns them with the specified command. // a HostConfig and returns them with the specified command.
// If the specified args are not valid, it will return an error. // If the specified args are not valid, it will return an error.
func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*containerConfig, error) { //nolint:gocyclo,unparam // verbatim copy from docker/cli func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*containerConfig, error) { //nolint:gocyclo // verbatim copy from docker/cli
var ( var (
attachStdin = copts.attach.Get("stdin") attachStdin = copts.attach.Get("stdin")
attachStdout = copts.attach.Get("stdout") attachStdout = copts.attach.Get("stdout")
@@ -958,11 +959,11 @@ func parseSecurityOpts(securityOpts []string) ([]string, error) {
if err != nil { if err != nil {
return securityOpts, fmt.Errorf("opening seccomp profile (%s) failed: %w", v, err) return securityOpts, fmt.Errorf("opening seccomp profile (%s) failed: %w", v, err)
} }
profile := jsontext.Value(f) var b bytes.Buffer
if err := profile.Compact(); err != nil { if err := json.Compact(&b, f); err != nil {
return securityOpts, fmt.Errorf("compacting json for seccomp profile (%s) failed: %w", v, err) return securityOpts, fmt.Errorf("compacting json for seccomp profile (%s) failed: %w", v, err)
} }
securityOpts[key] = "seccomp=" + string(profile) securityOpts[key] = "seccomp=" + b.String()
} }
} }
} }
@@ -10,9 +10,7 @@ import (
"fmt" "fmt"
"io" "io"
"slices" "slices"
"strings"
"github.com/docker/cli/opts"
"github.com/kballard/go-shellquote" "github.com/kballard/go-shellquote"
"github.com/spf13/pflag" "github.com/spf13/pflag"
) )
@@ -53,16 +51,15 @@ func parseContainerOptions(options string) (*pflag.FlagSet, *containerOptions, *
flags := pflag.NewFlagSet("container_flags", pflag.ContinueOnError) flags := pflag.NewFlagSet("container_flags", pflag.ContinueOnError)
flags.SetOutput(io.Discard) flags.SetOutput(io.Discard)
copts := addFlags(flags) copts := addFlags(flags)
copts.env = opts.NewListOpts(validateEnv) // addFlags registered this field's address, so the swap takes effect
cf := registerCreateFlags(flags) cf := registerCreateFlags(flags)
args, err := shellquote.Split(options) args, err := shellquote.Split(options)
if err != nil { if err != nil {
return flags, copts, cf, fmt.Errorf("cannot split container options: '%s': '%w'", options, err) return flags, copts, cf, fmt.Errorf("Cannot split container options: '%s': '%w'", options, err)
} }
if err := flags.Parse(args); err != nil { if err := flags.Parse(args); err != nil {
return flags, copts, cf, fmt.Errorf("cannot parse container options: '%s': '%w'", options, err) return flags, copts, cf, fmt.Errorf("Cannot parse container options: '%s': '%w'", options, err)
} }
return flags, copts, cf, nil return flags, copts, cf, nil
@@ -76,30 +73,6 @@ func createFlagsFromOptions(options string) *createFlags {
return cf return cf
} }
// validateEnv is opts.ValidateEnv without its lookup of a bare name in the runner's environment.
func validateEnv(val string) (string, error) {
if name, _, _ := strings.Cut(val, "="); name == "" {
return "", errors.New("invalid environment variable: " + val)
}
return val, nil
}
// rejectHostReadingOptions refuses the flags naming files that are read here, on the
// runner, rather than in the container.
func rejectHostReadingOptions(options string) error {
flags, _, _, err := parseContainerOptions(options)
if err != nil {
return err
}
for _, name := range []string{"env-file", "label-file"} {
if flags.Changed(name) {
return fmt.Errorf("container option --%s reads files from the runner and is not allowed in a workflow", name)
}
}
return nil
}
func (cf *createFlags) validate() error { func (cf *createFlags) validate() error {
if !slices.Contains(pullPolicies, cf.pull) { if !slices.Contains(pullPolicies, cf.pull) {
return fmt.Errorf("invalid --pull option %q: must be one of %q", cf.pull, pullPolicies) return fmt.Errorf("invalid --pull option %q: must be one of %q", cf.pull, pullPolicies)
@@ -50,13 +50,13 @@ func TestCreateFlagsValidate(t *testing.T) {
} }
func TestNewContainerAppliesCreateFlags(t *testing.T) { func TestNewContainerAppliesCreateFlags(t *testing.T) {
input := &NewContainerInput{Platform: "linux/amd64", RunnerOptions: "--pull never", WorkflowOptions: "--platform linux/arm64"} input := &NewContainerInput{Platform: "linux/amd64", Options: "--platform linux/arm64 --pull never"}
cr, ok := NewContainer(input).(*containerReference) cr, ok := NewContainer(input).(*containerReference)
require.True(t, ok) require.True(t, ok)
assert.Equal(t, "linux/arm64", input.Platform) assert.Equal(t, "linux/arm64", input.Platform)
assert.Equal(t, pullPolicyNever, cr.pullPolicy) assert.Equal(t, pullPolicyNever, cr.pullPolicy)
kept := &NewContainerInput{Platform: "linux/amd64", RunnerOptions: "--privileged"} kept := &NewContainerInput{Platform: "linux/amd64", Options: "--privileged"}
NewContainer(kept) NewContainer(kept)
assert.Equal(t, "linux/amd64", kept.Platform) assert.Equal(t, "linux/amd64", kept.Platform)
} }
@@ -8,7 +8,7 @@ package container
import ( import (
"bufio" "bufio"
"encoding/json/v2" "encoding/json"
"errors" "errors"
"io" "io"
@@ -20,8 +20,8 @@ type dockerMessage struct {
Stream string `json:"stream"` Stream string `json:"stream"`
Error string `json:"error"` Error string `json:"error"`
ErrorDetail struct { ErrorDetail struct {
Message string `json:"message"` Message string
} `json:"errorDetail"` }
Status string `json:"status"` Status string `json:"status"`
Progress string `json:"progress"` Progress string `json:"progress"`
} }
@@ -60,16 +60,15 @@ func logDockerResponse(logger logrus.FieldLogger, dockerResponse io.ReadCloser,
return errors.New(msg.ErrorDetail.Message) return errors.New(msg.ErrorDetail.Message)
} }
switch { if msg.Status != "" {
case msg.Status != "":
if msg.Progress != "" { if msg.Progress != "" {
writeLog(logger, isError, "%s :: %s :: %s\n", msg.Status, msg.ID, msg.Progress) writeLog(logger, isError, "%s :: %s :: %s\n", msg.Status, msg.ID, msg.Progress)
} else { } else {
writeLog(logger, isError, "%s :: %s\n", msg.Status, msg.ID) writeLog(logger, isError, "%s :: %s\n", msg.Status, msg.ID)
} }
case msg.Stream != "": } else if msg.Stream != "" {
writeLog(logger, isError, "%s", msg.Stream) writeLog(logger, isError, "%s", msg.Stream)
default: } else {
writeLog(logger, false, "Unable to handle line: %s", string(line)) writeLog(logger, false, "Unable to handle line: %s", string(line))
} }
} }
@@ -13,7 +13,7 @@ import (
"strings" "strings"
"time" "time"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"github.com/moby/moby/client" "github.com/moby/moby/client"
) )
@@ -32,9 +32,9 @@ func TestRemoveOrphanNetworks(t *testing.T) {
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{ client.On("NetworkList", ctx, mobyclient.NetworkListOptions{
Filters: make(mobyclient.Filters).Add("label", runnerUUIDLabel+"=runner-1"), Filters: make(mobyclient.Filters).Add("label", runnerUUIDLabel+"=runner-1"),
}).Return(mobyclient.NetworkListResult{Items: []network.Summary{ }).Return(mobyclient.NetworkListResult{Items: []network.Summary{
{ID: "orphan"}, {Network: network.Network{ID: "orphan"}},
{ID: "busy"}, {Network: network.Network{ID: "busy"}},
{ID: "starting"}, {Network: network.Network{ID: "starting"}},
}}, nil) }}, nil)
client.On("NetworkInspect", ctx, "orphan", mobyclient.NetworkInspectOptions{}). client.On("NetworkInspect", ctx, "orphan", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{}, nil) Return(mobyclient.NetworkInspectResult{}, nil)
@@ -11,7 +11,7 @@ import (
"fmt" "fmt"
"strings" "strings"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"github.com/distribution/reference" "github.com/distribution/reference"
"github.com/moby/moby/api/pkg/authconfig" "github.com/moby/moby/api/pkg/authconfig"
@@ -15,7 +15,6 @@ import (
"io" "io"
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"regexp" "regexp"
"runtime" "runtime"
"slices" "slices"
@@ -23,8 +22,8 @@ import (
"strings" "strings"
"time" "time"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"gitea.com/gitea/runner/act/filecollector" "gitea.com/gitea/runner/internal/act/filecollector"
"dario.cat/mergo" "dario.cat/mergo"
cerrdefs "github.com/containerd/errdefs" cerrdefs "github.com/containerd/errdefs"
@@ -58,7 +57,7 @@ func NewContainer(input *NewContainerInput) ExecutionsEnvironment {
cr := new(containerReference) cr := new(containerReference)
cr.input = input cr.input = input
// Resolved up front because the image pull runs before the container is created. // Resolved up front because the image pull runs before the container is created.
cf := createFlagsFromOptions(input.allOptions()) cf := createFlagsFromOptions(input.Options)
if cf.platform != "" { if cf.platform != "" {
cr.input.Platform = cf.platform cr.input.Platform = cf.platform
} }
@@ -66,14 +65,37 @@ func NewContainer(input *NewContainerInput) ExecutionsEnvironment {
return cr return cr
} }
// supportsContainerImagePlatform reports whether the Docker server API version func (cr *containerReference) ConnectToNetwork(name string) common.Executor {
// is 1.41 and beyond return common.
func supportsContainerImagePlatform(ctx context.Context, cli client.APIClient) (bool, error) { NewDebugExecutor("docker network connect %s %s", name, cr.input.Name).
Then(
common.NewPipelineExecutor(
cr.connect(),
cr.connectToNetwork(name, cr.input.NetworkAliases),
).IfNot(common.Dryrun),
)
}
func (cr *containerReference) connectToNetwork(name string, aliases []string) common.Executor {
return func(ctx context.Context) error {
_, err := cr.cli.NetworkConnect(ctx, name, client.NetworkConnectOptions{
Container: cr.input.Name,
EndpointConfig: &network.EndpointSettings{
Aliases: aliases,
},
})
return err
}
}
// supportsContainerImagePlatform returns true if the underlying Docker server
// API version is 1.41 and beyond
func supportsContainerImagePlatform(ctx context.Context, cli client.APIClient) bool {
ver, err := cli.ServerVersion(ctx, client.ServerVersionOptions{}) ver, err := cli.ServerVersion(ctx, client.ServerVersionOptions{})
if err != nil { if err != nil {
return false, fmt.Errorf("get docker API version: %w", err) common.Logger(ctx).Panicf("Failed to get Docker API Version: %s", err)
} }
return versions.GreaterThanOrEqualTo(ver.APIVersion, "1.41"), nil return versions.GreaterThanOrEqualTo(ver.APIVersion, "1.41")
} }
func (cr *containerReference) Create(capAdd, capDrop []string) common.Executor { func (cr *containerReference) Create(capAdd, capDrop []string) common.Executor {
@@ -525,39 +547,29 @@ func (cr *containerReference) waitForRemoval(ctx context.Context, idOrName strin
} }
} }
// allOptions puts the runner's options first, so a flag both sources set ends up the workflow's.
func (input *NewContainerInput) allOptions() string {
return strings.TrimSpace(input.RunnerOptions + " " + input.WorkflowOptions)
}
func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig, error) { func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig, error) {
logger := common.Logger(ctx) logger := common.Logger(ctx)
options := cr.input.allOptions() input := cr.input
if options == "" { if input.Options == "" {
return config, hostConfig, nil return config, hostConfig, nil
} }
// For Gitea, checked here because the parse below is what would read those files
if err := rejectHostReadingOptions(cr.input.WorkflowOptions); err != nil {
return nil, nil, err
}
// parse configuration from CLI container.options // parse configuration from CLI container.options
flags, copts, cf, err := parseContainerOptions(options) flags, copts, cf, err := parseContainerOptions(input.Options)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
if err := cf.validate(); err != nil { if err := cf.validate(); err != nil {
return nil, nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err) return nil, nil, fmt.Errorf("Cannot process container options: '%s': '%w'", input.Options, err)
} }
// FIXME: If everything is fine after gitea/act v0.260.0, remove the following comment. // FIXME: If everything is fine after gitea/act v0.260.0, remove the following comment.
// In the old fork version, the code is // In the old fork version, the code is
// if len(copts.netMode.Value()) == 0 { // if len(copts.netMode.Value()) == 0 {
// if err = copts.netMode.Set("host"); err != nil { // if err = copts.netMode.Set("host"); err != nil {
// return nil, nil, fmt.Errorf("cannot parse networkmode=host. This is an internal error and should not happen: '%w'", err) // return nil, nil, fmt.Errorf("Cannot parse networkmode=host. This is an internal error and should not happen: '%w'", err)
// } // }
// } // }
// And it has been commented with: // And it has been commented with:
@@ -569,7 +581,7 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
if len(copts.netMode.Value()) == 0 { if len(copts.netMode.Value()) == 0 {
if err = copts.netMode.Set(cr.input.NetworkMode); err != nil { if err = copts.netMode.Set(cr.input.NetworkMode); err != nil {
return nil, nil, fmt.Errorf("cannot parse networkmode=%s. This is an internal error and should not happen: '%w'", cr.input.NetworkMode, err) return nil, nil, fmt.Errorf("Cannot parse networkmode=%s. This is an internal error and should not happen: '%w'", cr.input.NetworkMode, err)
} }
} }
@@ -581,23 +593,24 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
containerConfig, err := parse(flags, copts, runtime.GOOS) containerConfig, err := parse(flags, copts, runtime.GOOS)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err) return nil, nil, fmt.Errorf("Cannot process container options: '%s': '%w'", input.Options, err)
} }
// For Gitea, forcing --privileged off is not enough, other options reach the host too // For Gitea
// When privileged mode is disabled, container.options is workflow-controlled
// untrusted input. Strip the HostConfig fields that would let a workflow break
// out of the container (host namespaces, capability expansion, security profile
// overrides, device and runtime access). Otherwise these survive into the final
// HostConfig even though --privileged is forced off.
if !hostConfig.Privileged { if !hostConfig.Privileged {
trusted, err := parseOptionsHostConfig(cr.input.RunnerOptions) sanitizeOptionsHostConfig(logger, containerConfig.HostConfig)
if err != nil {
return nil, nil, err
}
sanitizeOptionsHostConfig(logger, containerConfig.HostConfig, trusted)
} }
logger.Debugf("Custom container.Config from options ==> %+v", containerConfig.Config) logger.Debugf("Custom container.Config from options ==> %+v", containerConfig.Config)
err = mergo.Merge(config, containerConfig.Config, mergo.WithOverride, mergo.WithAppendSlice) err = mergo.Merge(config, containerConfig.Config, mergo.WithOverride, mergo.WithAppendSlice)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("cannot merge container.Config options: '%s': '%w'", options, err) return nil, nil, fmt.Errorf("Cannot merge container.Config options: '%s': '%w'", input.Options, err)
} }
logger.Debugf("Merged container.Config ==> %+v", config) logger.Debugf("Merged container.Config ==> %+v", config)
@@ -609,15 +622,14 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
networkMode := hostConfig.NetworkMode networkMode := hostConfig.NetworkMode
err = mergo.Merge(hostConfig, containerConfig.HostConfig, mergo.WithOverride) err = mergo.Merge(hostConfig, containerConfig.HostConfig, mergo.WithOverride)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("cannot merge container.HostConfig options: '%s': '%w'", options, err) return nil, nil, fmt.Errorf("Cannot merge container.HostConfig options: '%s': '%w'", input.Options, err)
} }
hostConfig.Binds = binds hostConfig.Binds = binds
hostConfig.Mounts = mounts hostConfig.Mounts = mounts
if cf.name != "" { if cf.name != "" {
logger.Warn("--name in the options will be ignored.") logger.Warn("--name in the options will be ignored.")
} }
// the runner's own network mode was put into copts above, so ask the flags instead if len(copts.netMode.Value()) > 0 {
if flags.Changed("network") || flags.Changed("net") {
logger.Warn("--network and --net in the options will be ignored.") logger.Warn("--network and --net in the options will be ignored.")
} }
hostConfig.NetworkMode = networkMode hostConfig.NetworkMode = networkMode
@@ -670,17 +682,11 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
} }
var platSpecs *specs.Platform var platSpecs *specs.Platform
if cr.input.Platform != "" { if cr.input.Platform != "" && supportsContainerImagePlatform(ctx, cr.cli) {
// Dropping the platform silently would build for the host arch. platSpecs, err = parsePlatform(cr.input.Platform)
supported, err := supportsContainerImagePlatform(ctx, cr.cli)
if err != nil { if err != nil {
return err return err
} }
if supported {
if platSpecs, err = parsePlatform(cr.input.Platform); err != nil {
return err
}
}
} }
hostConfig := &container.HostConfig{ hostConfig := &container.HostConfig{
@@ -932,6 +938,42 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
} }
} }
func (cr *containerReference) CopyTarStream(ctx context.Context, destPath string, tarStream io.Reader) error {
if cr.id == "" {
return cr.missingContainerError("copy to %s", destPath)
}
// Mkdir, with a path relative to the DestinationPath ("/") below. Docker 29.5+
// rejects absolute tar entry names with "path escapes from parent".
buf := &bytes.Buffer{}
tw := tar.NewWriter(buf)
_ = tw.WriteHeader(&tar.Header{
Name: strings.TrimPrefix(destPath, "/"),
Mode: 0o777,
Typeflag: tar.TypeDir,
})
tw.Close()
_, err := cr.cli.CopyToContainer(ctx, cr.id, client.CopyToContainerOptions{
DestinationPath: "/",
Content: buf,
})
if err != nil {
return fmt.Errorf("failed to mkdir to copy content to container: %w", err)
}
// Copy Content
_, err = cr.cli.CopyToContainer(ctx, cr.id, client.CopyToContainerOptions{
DestinationPath: destPath,
Content: tarStream,
})
if err != nil {
return fmt.Errorf("failed to copy content to container: %w", err)
}
// If this fails, then folders have wrong permissions on non root container
if cr.UID != 0 || cr.GID != 0 {
_ = cr.Exec([]string{"chown", "-R", fmt.Sprintf("%d:%d", cr.UID, cr.GID), destPath}, nil, "0", "")(ctx)
}
return nil
}
func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore bool) common.Executor { func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore bool) common.Executor {
return func(ctx context.Context) error { return func(ctx context.Context) error {
if cr.id == "" { if cr.id == "" {
@@ -973,6 +1015,7 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore bool
} }
fc := &filecollector.FileCollector{ fc := &filecollector.FileCollector{
Fs: &filecollector.DefaultFs{},
Ignorer: ignorer, Ignorer: ignorer,
SrcPath: srcPath, SrcPath: srcPath,
SrcPrefix: srcPrefix, SrcPrefix: srcPrefix,
@@ -1123,64 +1166,74 @@ func (cr *containerReference) wait() common.Executor {
} }
// For Gitea // For Gitea
// sanitizeOptionsHostConfig takes back everything a workflow could escape the container with, // sanitizeOptionsHostConfig clears the HostConfig fields parsed from a
// setting each field to trusted, which is what the runner's own options parse to on their own. // workflow-controlled container.options string that could be used to escape the
// Only for unprivileged mode, since privileged mode grants host access anyway. // container when privileged mode is disabled. It must only be called when the
func sanitizeOptionsHostConfig(logger logrus.FieldLogger, hostConfig, trusted *container.HostConfig) { // runner has privileged mode turned off; with privileged mode enabled the
resetOption(logger, "--pid", &hostConfig.PidMode, trusted.PidMode) // administrator has already opted into host access.
resetOption(logger, "--ipc", &hostConfig.IpcMode, trusted.IpcMode) func sanitizeOptionsHostConfig(logger logrus.FieldLogger, hostConfig *container.HostConfig) {
resetOption(logger, "--uts", &hostConfig.UTSMode, trusted.UTSMode) warn := func(option string) {
resetOption(logger, "--cgroupns", &hostConfig.CgroupnsMode, trusted.CgroupnsMode) logger.Warnf("container option %q is not allowed when privileged mode is disabled and will be ignored", option)
resetOption(logger, "--userns", &hostConfig.UsernsMode, trusted.UsernsMode) // --userns=host would undo the remapping the runner asked for }
resetOption(logger, "--cap-add", &hostConfig.CapAdd, trusted.CapAdd)
resetOption(logger, "--security-opt", &hostConfig.SecurityOpt, trusted.SecurityOpt)
resetOption(logger, "--device", &hostConfig.Devices, trusted.Devices)
resetOption(logger, "--device-cgroup-rule", &hostConfig.DeviceCgroupRules, trusted.DeviceCgroupRules)
resetOption(logger, "--gpus", &hostConfig.DeviceRequests, trusted.DeviceRequests)
resetOption(logger, "--volumes-from", &hostConfig.VolumesFrom, trusted.VolumesFrom)
resetOption(logger, "--runtime", &hostConfig.Runtime, trusted.Runtime)
resetOption(logger, "--cgroup-parent", &hostConfig.CgroupParent, trusted.CgroupParent)
resetOption(logger, "--sysctl", &hostConfig.Sysctls, trusted.Sysctls)
resetOption(logger, "--isolation", &hostConfig.Isolation, trusted.Isolation) // windows: process isolation drops the hyper-v boundary
resetOption(logger, "--volume-driver", &hostConfig.VolumeDriver, trusted.VolumeDriver)
// systempaths=unconfined lands in these two rather than in SecurityOpt
resetOption(logger, "--security-opt", &hostConfig.MaskedPaths, trusted.MaskedPaths)
resetOption(logger, "--security-opt", &hostConfig.ReadonlyPaths, trusted.ReadonlyPaths)
// a driver mounts what it likes, e.g. local with device= binds any host path, which if hostConfig.PidMode != "" {
// valid_volumes never gets to see warn("--pid")
hostConfig.Mounts = slices.DeleteFunc(hostConfig.Mounts, func(mt mount.Mount) bool { hostConfig.PidMode = ""
if mt.VolumeOptions == nil || mt.VolumeOptions.DriverConfig == nil ||
slices.ContainsFunc(trusted.Mounts, func(t mount.Mount) bool { return reflect.DeepEqual(t, mt) }) {
return false
} }
logger.Warnf("volume driver of %q in the workflow is not allowed when privileged mode is disabled and will be ignored", mt.Source) if hostConfig.IpcMode != "" {
return true warn("--ipc")
}) hostConfig.IpcMode = ""
}
// resetOption puts a field back to the runner's own value. It compares the values rather than
// the flags, so a field that more than one option feeds cannot slip through.
func resetOption[T any](logger logrus.FieldLogger, option string, field *T, trusted T) {
if reflect.DeepEqual(*field, trusted) {
return
} }
logger.Warnf("container option %q in the workflow is not allowed when privileged mode is disabled and will be ignored", option) if hostConfig.UTSMode != "" {
*field = trusted warn("--uts")
} hostConfig.UTSMode = ""
// parseOptionsHostConfig parses one options string on its own, to see what it alone asks for.
// Even "" goes through the parser, or its empty slices and maps would differ from a real parse.
func parseOptionsHostConfig(options string) (*container.HostConfig, error) {
flags, copts, _, err := parseContainerOptions(options)
if err != nil {
return nil, err
} }
containerConfig, err := parse(flags, copts, runtime.GOOS) if hostConfig.CgroupnsMode != "" {
if err != nil { warn("--cgroupns")
return nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err) hostConfig.CgroupnsMode = ""
}
// UsernsMode is set from the runner-controlled input; never let options
// override it (e.g. --userns=host disables user namespace remapping).
if hostConfig.UsernsMode != "" {
warn("--userns")
hostConfig.UsernsMode = ""
}
if len(hostConfig.CapAdd) > 0 {
warn("--cap-add")
hostConfig.CapAdd = nil
}
if len(hostConfig.SecurityOpt) > 0 {
warn("--security-opt")
hostConfig.SecurityOpt = nil
}
if len(hostConfig.Devices) > 0 {
warn("--device")
hostConfig.Devices = nil
}
if len(hostConfig.DeviceCgroupRules) > 0 {
warn("--device-cgroup-rule")
hostConfig.DeviceCgroupRules = nil
}
if len(hostConfig.DeviceRequests) > 0 {
warn("--gpus")
hostConfig.DeviceRequests = nil
}
if len(hostConfig.VolumesFrom) > 0 {
warn("--volumes-from")
hostConfig.VolumesFrom = nil
}
if hostConfig.Runtime != "" {
warn("--runtime")
hostConfig.Runtime = ""
}
if hostConfig.CgroupParent != "" {
warn("--cgroup-parent")
hostConfig.CgroupParent = ""
}
if len(hostConfig.Sysctls) > 0 {
warn("--sysctl")
hostConfig.Sysctls = nil
} }
return containerConfig.HostConfig, nil
} }
// For Gitea // For Gitea
@@ -5,6 +5,7 @@
package container package container
import ( import (
"archive/tar"
"bufio" "bufio"
"bytes" "bytes"
"context" "context"
@@ -16,8 +17,9 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"time"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
cerrdefs "github.com/containerd/errdefs" cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/api/pkg/stdcopy" "github.com/moby/moby/api/pkg/stdcopy"
@@ -77,11 +79,6 @@ type mockDockerClient struct {
mock.Mock mock.Mock
} }
func (m *mockDockerClient) ServerVersion(ctx context.Context, opts mobyclient.ServerVersionOptions) (mobyclient.ServerVersionResult, error) {
args := m.Called(ctx, opts)
return args.Get(0).(mobyclient.ServerVersionResult), args.Error(1)
}
func (m *mockDockerClient) ExecCreate(ctx context.Context, id string, opts mobyclient.ExecCreateOptions) (mobyclient.ExecCreateResult, error) { func (m *mockDockerClient) ExecCreate(ctx context.Context, id string, opts mobyclient.ExecCreateOptions) (mobyclient.ExecCreateResult, error) {
args := m.Called(ctx, id, opts) args := m.Called(ctx, id, opts)
return args.Get(0).(mobyclient.ExecCreateResult), args.Error(1) return args.Get(0).(mobyclient.ExecCreateResult), args.Error(1)
@@ -147,17 +144,12 @@ func (m *mockDockerClient) NetworkRemove(ctx context.Context, id string, opts mo
return args.Get(0).(mobyclient.NetworkRemoveResult), args.Error(1) return args.Get(0).(mobyclient.NetworkRemoveResult), args.Error(1)
} }
type interruptReader struct { type endlessReader struct {
started chan struct{} io.Reader
interrupted chan struct{}
stopped chan struct{}
} }
func (r *interruptReader) Read(_ []byte) (int, error) { func (r endlessReader) Read(_ []byte) (n int, err error) {
close(r.started) return 1, nil
<-r.interrupted
close(r.stopped)
return 0, io.EOF
} }
type mockConn struct { type mockConn struct {
@@ -177,17 +169,16 @@ func (m *mockConn) Close() (err error) {
func TestDockerExecAbort(t *testing.T) { func TestDockerExecAbort(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
reader := &interruptReader{started: make(chan struct{}), interrupted: make(chan struct{}), stopped: make(chan struct{})}
conn := &mockConn{} conn := &mockConn{}
conn.On("Write", []byte{3}). conn.On("Write", mock.AnythingOfType("[]uint8")).Return(1, nil)
Run(func(mock.Arguments) { close(reader.interrupted) }).
Return(1, nil)
client := &mockDockerClient{} client := &mockDockerClient{}
client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).Return(mobyclient.ExecCreateResult{ID: "id"}, nil) client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).Return(mobyclient.ExecAttachResult{ client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).Return(mobyclient.ExecAttachResult{
HijackedResponse: mobyclient.HijackedResponse{
Conn: conn, Conn: conn,
Reader: bufio.NewReader(reader), Reader: bufio.NewReader(endlessReader{}),
},
}, nil) }, nil)
cr := &containerReference{ cr := &containerReference{
@@ -204,11 +195,11 @@ func TestDockerExecAbort(t *testing.T) {
channel <- cr.exec([]string{""}, map[string]string{}, "user", "workdir")(ctx) channel <- cr.exec([]string{""}, map[string]string{}, "user", "workdir")(ctx)
}() }()
<-reader.started time.Sleep(500 * time.Millisecond)
cancel() cancel()
err := <-channel err := <-channel
<-reader.stopped
assert.ErrorIs(t, err, context.Canceled) //nolint:testifylint // pre-existing issue from nektos/act assert.ErrorIs(t, err, context.Canceled) //nolint:testifylint // pre-existing issue from nektos/act
conn.AssertExpectations(t) conn.AssertExpectations(t)
@@ -223,8 +214,10 @@ func TestDockerExecFailure(t *testing.T) {
client := &mockDockerClient{} client := &mockDockerClient{}
client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).Return(mobyclient.ExecCreateResult{ID: "id"}, nil) client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).Return(mobyclient.ExecAttachResult{ client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).Return(mobyclient.ExecAttachResult{
HijackedResponse: mobyclient.HijackedResponse{
Conn: conn, Conn: conn,
Reader: bufio.NewReader(strings.NewReader("output")), Reader: bufio.NewReader(strings.NewReader("output")),
},
}, nil) }, nil)
client.On("ExecInspect", ctx, "id", mobyclient.ExecInspectOptions{}).Return(mobyclient.ExecInspectResult{ client.On("ExecInspect", ctx, "id", mobyclient.ExecInspectOptions{}).Return(mobyclient.ExecInspectResult{
ExitCode: 1, ExitCode: 1,
@@ -276,8 +269,10 @@ func TestDockerAttachFlushesTrailingLine(t *testing.T) {
client := &mockDockerClient{} client := &mockDockerClient{}
client.On("ContainerAttach", ctx, "123", mock.AnythingOfType("client.ContainerAttachOptions")). client.On("ContainerAttach", ctx, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
Return(mobyclient.ContainerAttachResult{ Return(mobyclient.ContainerAttachResult{
HijackedResponse: mobyclient.HijackedResponse{
Conn: &mockConn{}, Conn: &mockConn{},
Reader: bufio.NewReader(framed), Reader: bufio.NewReader(framed),
},
}, nil) }, nil)
statusCh := make(chan container.WaitResponse, 1) statusCh := make(chan container.WaitResponse, 1)
@@ -342,6 +337,116 @@ func TestDockerWaitFailure(t *testing.T) {
client.AssertExpectations(t) client.AssertExpectations(t)
} }
func TestDockerCopyTarStream(t *testing.T) {
ctx := context.Background()
client := &mockDockerClient{}
client.On("CopyToContainer", ctx, "123", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
return opts.DestinationPath == "/" && opts.Content != nil
})).Return(mobyclient.CopyToContainerResult{}, nil)
client.On("CopyToContainer", ctx, "123", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
return opts.DestinationPath == "/var/run/act" && opts.Content != nil
})).Return(mobyclient.CopyToContainerResult{}, nil)
cr := &containerReference{
id: "123",
cli: client,
input: &NewContainerInput{
Image: "image",
},
}
_ = cr.CopyTarStream(ctx, "/var/run/act", &bytes.Buffer{})
client.AssertExpectations(t)
}
// Docker 29.5+ rejects absolute names in the mkdir tarball with
// "path escapes from parent", since it is extracted relative to "/".
func TestDockerCopyTarStreamMkdirEntryIsRelative(t *testing.T) {
ctx := context.Background()
var mkdirNames []string
client := &mockDockerClient{}
client.On("CopyToContainer", ctx, "123", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
if opts.DestinationPath != "/" || opts.Content == nil {
return false
}
tr := tar.NewReader(opts.Content)
for {
hdr, err := tr.Next()
if err != nil {
break
}
mkdirNames = append(mkdirNames, hdr.Name)
}
return true
})).Return(mobyclient.CopyToContainerResult{}, nil)
client.On("CopyToContainer", ctx, "123", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
return opts.DestinationPath == "/var/run/act" && opts.Content != nil
})).Return(mobyclient.CopyToContainerResult{}, nil)
cr := &containerReference{
id: "123",
cli: client,
input: &NewContainerInput{
Image: "image",
},
}
require.NoError(t, cr.CopyTarStream(ctx, "/var/run/act", &bytes.Buffer{}))
assert.Equal(t, []string{"var/run/act"}, mkdirNames)
client.AssertExpectations(t)
}
func TestDockerCopyTarStreamErrorInCopyFiles(t *testing.T) {
ctx := context.Background()
merr := errors.New("Failure")
client := &mockDockerClient{}
client.On("CopyToContainer", ctx, "123", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
return opts.DestinationPath == "/" && opts.Content != nil
})).Return(mobyclient.CopyToContainerResult{}, merr)
cr := &containerReference{
id: "123",
cli: client,
input: &NewContainerInput{
Image: "image",
},
}
err := cr.CopyTarStream(ctx, "/var/run/act", &bytes.Buffer{})
assert.ErrorIs(t, err, merr) //nolint:testifylint // pre-existing issue from nektos/act
client.AssertExpectations(t)
}
func TestDockerCopyTarStreamErrorInMkdir(t *testing.T) {
ctx := context.Background()
merr := errors.New("Failure")
client := &mockDockerClient{}
client.On("CopyToContainer", ctx, "123", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
return opts.DestinationPath == "/" && opts.Content != nil
})).Return(mobyclient.CopyToContainerResult{}, nil)
client.On("CopyToContainer", ctx, "123", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
return opts.DestinationPath == "/var/run/act" && opts.Content != nil
})).Return(mobyclient.CopyToContainerResult{}, merr)
cr := &containerReference{
id: "123",
cli: client,
input: &NewContainerInput{
Image: "image",
},
}
err := cr.CopyTarStream(ctx, "/var/run/act", &bytes.Buffer{})
assert.ErrorIs(t, err, merr) //nolint:testifylint // pre-existing issue from nektos/act
client.AssertExpectations(t)
}
// A remove that raced the daemon's AutoRemove teardown is not a failure and must not // A remove that raced the daemon's AutoRemove teardown is not a failure and must not
// be logged as one. // be logged as one.
func TestRemoveIgnoresAutoRemoveRace(t *testing.T) { func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
@@ -472,6 +577,7 @@ func TestRejectsMissingContainer(t *testing.T) {
} }
check("copyContent", cr.copyContent("/var/run/act", &FileEntry{Name: "x", Mode: 0o644})(ctx)) check("copyContent", cr.copyContent("/var/run/act", &FileEntry{Name: "x", Mode: 0o644})(ctx))
check("copyDir", cr.copyDir("/var/run/act", "/src", false)(ctx)) check("copyDir", cr.copyDir("/var/run/act", "/src", false)(ctx))
check("CopyTarStream", cr.CopyTarStream(ctx, "/var/run/act", &bytes.Buffer{}))
check("exec", cr.exec([]string{"echo"}, nil, "", "")(ctx)) check("exec", cr.exec([]string{"echo"}, nil, "", "")(ctx))
_, err := cr.GetContainerArchive(ctx, "/var/run/act/x") _, err := cr.GetContainerArchive(ctx, "/var/run/act/x")
check("GetContainerArchive", err) check("GetContainerArchive", err)
@@ -507,6 +613,35 @@ func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
client.AssertExpectations(t) client.AssertExpectations(t)
} }
// TestDockerCopyToSymlinkPath is a regression test for gitea/runner#981. Most base images
// symlink /var/run to /run, so copying into /var/run/act traverses that symlink. The broken
// docker 29.5.1 daemon fails the extraction with "mkdirat var/run: file exists" (fixed in
// 29.5.2). Running against the daemon shipped in the dind image, this catches a bad bump.
func TestDockerCopyToSymlinkPath(t *testing.T) {
requireDocker(t)
ctx := context.Background()
rc := NewContainer(&NewContainerInput{
Image: "alpine:latest",
Entrypoint: []string{"sleep", "30"},
Name: "act-test-symlink-" + time.Now().Format("20060102150405.000000"),
AutoRemove: true,
})
require.NoError(t, rc.Pull(false)(ctx))
require.NoError(t, rc.Create(nil, nil)(ctx))
require.NoError(t, rc.Start(false)(ctx))
t.Cleanup(func() {
_ = rc.Remove()(ctx)
_ = rc.Close()(ctx)
})
// CopyTarStream first creates the destination directory by extracting a tar at "/",
// which makes the daemon mkdir var, then var/run (the symlink), then act — the exact
// step that fails on the broken daemon.
err := rc.CopyTarStream(ctx, "/var/run/act/actions/", &bytes.Buffer{})
require.NoError(t, err)
}
// Type assert containerReference implements ExecutionsEnvironment // Type assert containerReference implements ExecutionsEnvironment
var _ ExecutionsEnvironment = &containerReference{} var _ ExecutionsEnvironment = &containerReference{}
@@ -583,59 +718,11 @@ func TestCheckVolumes(t *testing.T) {
} }
} }
// A volume driver decides for itself what it mounts, e.g. the local driver with device= binds
// any host path, which valid_volumes never gets to see.
func TestMergeContainerConfigsDropsVolumeDriversFromWorkflows(t *testing.T) {
const escape = "--mount type=volume,src=job-escape,dst=/host,volume-driver=local,volume-opt=type=none,volume-opt=o=bind,volume-opt=device=/"
hostConfig, _ := mergeOptions(t, "", escape+" --mount type=volume,src=job-plain,dst=/cache", false)
require.Len(t, hostConfig.Mounts, 1)
assert.Equal(t, "job-plain", hostConfig.Mounts[0].Source)
// the same mount from the runner's own options is the administrator's to make
hostConfig, _ = mergeOptions(t, escape, "", false)
require.Len(t, hostConfig.Mounts, 1)
assert.Equal(t, "job-escape", hostConfig.Mounts[0].Source)
}
// Both of these are read here, on the runner, so a workflow could read the runner's files
// and environment with them.
func TestMergeContainerConfigsKeepsTheRunnersFilesAndEnvToItself(t *testing.T) {
hostFile := filepath.Join(t.TempDir(), "host.env")
require.NoError(t, os.WriteFile(hostFile, []byte("STOLEN=from-the-host\n"), 0o600))
t.Setenv("RUNNER_SECRET", "s3cr3t")
for _, option := range []string{"--env-file " + hostFile, "--label-file " + hostFile} {
logger, _ := test.NewNullLogger()
cr := &containerReference{input: &NewContainerInput{NetworkMode: "bridge", WorkflowOptions: option}}
_, _, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
require.ErrorContains(t, err, "not allowed in a workflow")
// the runner reading its own files is what those options are for
cr = &containerReference{input: &NewContainerInput{NetworkMode: "bridge", RunnerOptions: option}}
_, _, err = cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
require.NoError(t, err)
}
// a bare name is no longer resolved from the runner's environment, for either source
logger, _ := test.NewNullLogger()
cr := &containerReference{input: &NewContainerInput{
NetworkMode: "bridge",
RunnerOptions: "--env RUNNER_SECRET",
WorkflowOptions: "--env RUNNER_SECRET --env GIVEN=value",
}}
config, _, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
require.NoError(t, err)
assert.Equal(t, []string{"RUNNER_SECRET", "RUNNER_SECRET", "GIVEN=value"}, config.Env)
}
func TestSanitizeOptionsHostConfig(t *testing.T) { func TestSanitizeOptionsHostConfig(t *testing.T) {
logger, _ := test.NewNullLogger() logger, _ := test.NewNullLogger()
// every field the sanitizer resets, so a reset dropped in a refactor fails here dangerous := func() *container.HostConfig {
hostConfig := &container.HostConfig{ return &container.HostConfig{
PidMode: "host", PidMode: "host",
IpcMode: "host", IpcMode: "host",
UTSMode: "host", UTSMode: "host",
@@ -645,71 +732,94 @@ func TestSanitizeOptionsHostConfig(t *testing.T) {
SecurityOpt: []string{"seccomp=unconfined", "apparmor=unconfined"}, SecurityOpt: []string{"seccomp=unconfined", "apparmor=unconfined"},
VolumesFrom: []string{"other"}, VolumesFrom: []string{"other"},
Runtime: "runc", Runtime: "runc",
Isolation: "process", Resources: container.Resources{
VolumeDriver: "rogue",
MaskedPaths: []string{},
ReadonlyPaths: []string{},
CgroupParent: "/custom", CgroupParent: "/custom",
Devices: []container.DeviceMapping{{PathOnHost: "/dev/sda", PathInContainer: "/dev/sda", CgroupPermissions: "rwm"}}, Devices: []container.DeviceMapping{{PathOnHost: "/dev/sda", PathInContainer: "/dev/sda", CgroupPermissions: "rwm"}},
DeviceCgroupRules: []string{"a *:* rwm"}, DeviceCgroupRules: []string{"a *:* rwm"},
DeviceRequests: []container.DeviceRequest{{Count: -1, Capabilities: [][]string{{"gpu"}}}}, },
Sysctls: map[string]string{"net.ipv4.ip_forward": "1"}, Sysctls: map[string]string{"net.ipv4.ip_forward": "1"},
} }
}
sanitizeOptionsHostConfig(logger, hostConfig, &container.HostConfig{}) hostConfig := dangerous()
sanitizeOptionsHostConfig(logger, hostConfig)
assert.Equal(t, &container.HostConfig{}, hostConfig) assert.Empty(t, string(hostConfig.PidMode))
assert.Empty(t, string(hostConfig.IpcMode))
assert.Empty(t, string(hostConfig.UTSMode))
assert.Empty(t, string(hostConfig.CgroupnsMode))
assert.Empty(t, string(hostConfig.UsernsMode))
assert.Empty(t, hostConfig.CapAdd)
assert.Empty(t, hostConfig.SecurityOpt)
assert.Empty(t, hostConfig.Devices)
assert.Empty(t, hostConfig.DeviceCgroupRules)
assert.Empty(t, hostConfig.VolumesFrom)
assert.Empty(t, hostConfig.Runtime)
assert.Empty(t, hostConfig.CgroupParent)
assert.Empty(t, hostConfig.Sysctls)
} }
// mergeOptions merges both option sources into a bare container, returning the result and its log. func TestMergeContainerConfigsStripsDangerousOptionsWhenUnprivileged(t *testing.T) {
func mergeOptions(t *testing.T, runnerOptions, workflowOptions string, privileged bool) (*container.HostConfig, *test.Hook) { // OS-independent options only: --device parsing requires a linux/windows
t.Helper() // server OS, which is not guaranteed for the test host.
logger, hook := test.NewNullLogger() const dangerousOptions = "--pid=host --ipc=host --uts=host --cgroupns=host " +
cr := &containerReference{input: &NewContainerInput{ "--userns=host --cap-add=ALL --security-opt seccomp=unconfined " +
RunnerOptions: runnerOptions, "--security-opt apparmor=unconfined --volumes-from other " +
WorkflowOptions: workflowOptions, "--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1"
t.Run("unprivileged strips host-escape options", func(t *testing.T) {
logger, _ := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
cr := &containerReference{
input: &NewContainerInput{
Options: dangerousOptions,
NetworkMode: "bridge", NetworkMode: "bridge",
UsernsMode: "private", UsernsMode: "private",
}} },
}
_, hostConfig, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{ _, hostConfig, err := cr.mergeContainerConfigs(ctx, &container.Config{}, &container.HostConfig{
Privileged: privileged, Privileged: false,
UsernsMode: container.UsernsMode("private"), UsernsMode: container.UsernsMode("private"),
NetworkMode: container.NetworkMode("bridge"), NetworkMode: container.NetworkMode("bridge"),
}) })
require.NoError(t, err) require.NoError(t, err)
return hostConfig, hook
}
func TestMergeContainerConfigsStripsDangerousOptionsWhenUnprivileged(t *testing.T) { assert.False(t, hostConfig.Privileged)
// OS-independent options only, --device and --gpus need a linux/windows server OS assert.Empty(t, string(hostConfig.PidMode))
const dangerousOptions = "--pid=host --ipc=host --uts=host --cgroupns=host " + assert.Empty(t, string(hostConfig.IpcMode))
"--userns=host --cap-add=ALL --security-opt seccomp=unconfined " + assert.Empty(t, string(hostConfig.UTSMode))
"--security-opt apparmor=unconfined --volumes-from other --isolation process " + assert.Empty(t, string(hostConfig.CgroupnsMode))
"--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1" // UsernsMode must keep the runner-controlled value, not the one from options.
assert.Equal(t, "private", string(hostConfig.UsernsMode))
assert.Empty(t, hostConfig.CapAdd)
assert.Empty(t, hostConfig.SecurityOpt)
assert.Empty(t, hostConfig.VolumesFrom)
assert.Empty(t, hostConfig.Runtime)
assert.Empty(t, hostConfig.CgroupParent)
assert.Empty(t, hostConfig.Sysctls)
})
// whatever the workflow adds, an unprivileged container comes out exactly as the runner's t.Run("privileged preserves options", func(t *testing.T) {
// own options alone describe it, field for field logger, _ := test.NewNullLogger()
for _, runnerOptions := range []string{"--shm-size 1g", dangerousOptions, "--cap-add SYS_ADMIN --security-opt seccomp=unconfined"} { ctx := common.WithLogger(context.Background(), logger)
runnerOnly, _ := mergeOptions(t, runnerOptions, "", false) cr := &containerReference{
withWorkflow, _ := mergeOptions(t, runnerOptions, dangerousOptions, false) input: &NewContainerInput{
Options: "--pid=host --cap-add=ALL --security-opt seccomp=unconfined",
assert.Equal(t, runnerOnly, withWorkflow, "runner options: %q", runnerOptions) NetworkMode: "bridge",
},
} }
// the same options from the runner reach the daemon, even --userns, which no workflow may set _, hostConfig, err := cr.mergeContainerConfigs(ctx, &container.Config{}, &container.HostConfig{
kept, _ := mergeOptions(t, dangerousOptions, "", false) Privileged: true,
assert.Equal(t, "host", string(kept.PidMode)) NetworkMode: container.NetworkMode("bridge"),
assert.Equal(t, []string{"ALL"}, kept.CapAdd) })
assert.Equal(t, "runc", kept.Runtime) require.NoError(t, err)
assert.Equal(t, "host", string(kept.UsernsMode))
assert.False(t, kept.Privileged)
// privileged is the administrator opting in, so the workflow's options are honored assert.Equal(t, "host", string(hostConfig.PidMode))
privileged, _ := mergeOptions(t, "", dangerousOptions, true) assert.Equal(t, []string{"ALL"}, hostConfig.CapAdd)
assert.Equal(t, "host", string(privileged.PidMode)) assert.Equal(t, []string{"seccomp=unconfined"}, hostConfig.SecurityOpt)
assert.Equal(t, []string{"ALL"}, privileged.CapAdd) })
assert.Equal(t, []string{"seccomp=unconfined", "apparmor=unconfined"}, privileged.SecurityOpt)
} }
func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) { func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
@@ -808,7 +918,7 @@ func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
cr := &containerReference{ cr := &containerReference{
input: &NewContainerInput{ input: &NewContainerInput{
NetworkMode: "bridge", NetworkMode: "bridge",
RunnerOptions: "--volume /host/tools:/opt/hostedtoolcache", Options: "--volume /host/tools:/opt/hostedtoolcache",
}, },
} }
@@ -820,26 +930,3 @@ func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
assert.Equal(t, []string{"/var/run/docker.sock:/var/run/docker.sock", "/host/tools:/opt/hostedtoolcache"}, hostConf.Binds) assert.Equal(t, []string{"/var/run/docker.sock:/var/run/docker.sock", "/host/tools:/opt/hostedtoolcache"}, hostConf.Binds)
assert.Empty(t, hostConf.Mounts) assert.Empty(t, hostConf.Mounts)
} }
func TestMergeContainerConfigsWarnsOnlyAboutOptionsThatWereGiven(t *testing.T) {
warnings := func(runnerOptions, workflowOptions string) int {
_, hook := mergeOptions(t, runnerOptions, workflowOptions, false)
return len(hook.AllEntries())
}
assert.Zero(t, warnings("--volume /host/tools:/opt/hostedtoolcache", ""))
assert.Zero(t, warnings("", "--shm-size 1g"))
assert.Equal(t, 1, warnings("--network host", ""))
}
// A dead daemon must fail the job, not panic through logrus and not silently
// drop the requested platform.
func TestSupportsContainerImagePlatformDaemonError(t *testing.T) {
cli := &mockDockerClient{}
cli.On("ServerVersion", mock.Anything, mock.Anything).
Return(mobyclient.ServerVersionResult{}, errors.New("cannot connect to the Docker daemon"))
supported, err := supportsContainerImagePlatform(t.Context(), cli)
require.ErrorContains(t, err, "cannot connect to the Docker daemon")
assert.False(t, supported)
}
+138
View File
@@ -0,0 +1,138 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// Copyright 2024 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"fmt"
"os"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
)
var CommonSocketLocations = []string{
"/var/run/docker.sock",
"/run/podman/podman.sock",
"$HOME/.colima/docker.sock",
"$XDG_RUNTIME_DIR/docker.sock",
"$XDG_RUNTIME_DIR/podman/podman.sock",
`\\.\pipe\docker_engine`,
"$HOME/.docker/run/docker.sock",
}
// returns socket URI or false if not found any
func socketLocation() (string, bool) {
if dockerHost, exists := os.LookupEnv("DOCKER_HOST"); exists {
return dockerHost, true
}
for _, p := range CommonSocketLocations {
if _, err := os.Lstat(os.ExpandEnv(p)); err == nil {
if strings.HasPrefix(p, `\\.\`) {
return "npipe://" + filepath.ToSlash(os.ExpandEnv(p)), true
}
return "unix://" + filepath.ToSlash(os.ExpandEnv(p)), true
}
}
return "", false
}
// This function, `isDockerHostURI`, takes a string argument `daemonPath`. It checks if the
// `daemonPath` is a valid Docker host URI. It does this by checking if the scheme of the URI (the
// part before "://") contains only alphabetic characters. If it does, the function returns true,
// indicating that the `daemonPath` is a Docker host URI. If it doesn't, or if the "://" delimiter
// is not found in the `daemonPath`, the function returns false.
func isDockerHostURI(daemonPath string) bool {
if before, _, ok := strings.Cut(daemonPath, "://"); ok {
scheme := before
if strings.IndexFunc(scheme, func(r rune) bool {
return (r < 'a' || r > 'z') && (r < 'A' || r > 'Z')
}) == -1 {
return true
}
}
return false
}
type SocketAndHost struct {
Socket string
Host string
}
func GetSocketAndHost(containerSocket string) (SocketAndHost, error) {
log.Debugf("Handling container host and socket")
// Prefer DOCKER_HOST, don't override it
dockerHost, hasDockerHost := socketLocation()
socketHost := SocketAndHost{Socket: containerSocket, Host: dockerHost}
// ** socketHost.Socket cases **
// Case 1: User does _not_ want to mount a daemon socket (passes a dash)
// Case 2: User passes a filepath to the socket; is that even valid?
// Case 3: User passes a valid socket; do nothing
// Case 4: User omitted the flag; set a sane default
// ** DOCKER_HOST cases **
// Case A: DOCKER_HOST is set; use it, i.e. do nothing
// Case B: DOCKER_HOST is empty; use sane defaults
// Set host for sanity's sake, when the socket isn't useful
if !hasDockerHost && (socketHost.Socket == "-" || !isDockerHostURI(socketHost.Socket) || socketHost.Socket == "") {
// Cases: 1B, 2B, 4B
socket, found := socketLocation()
socketHost.Host = socket
hasDockerHost = found
}
// A - (dash) in socketHost.Socket means don't mount, preserve this value
// otherwise if socketHost.Socket is a filepath don't use it as socket
// Exit early if we're in an invalid state (e.g. when no DOCKER_HOST and user supplied "-", a dash or omitted)
if !hasDockerHost && socketHost.Socket != "" && !isDockerHostURI(socketHost.Socket) {
// Cases: 1B, 2B
// Should we early-exit here, since there is no host nor socket to talk to?
return SocketAndHost{}, fmt.Errorf("DOCKER_HOST was not set, couldn't be found in the usual locations, and the container daemon socket ('%s') is invalid", socketHost.Socket)
}
// Default to DOCKER_HOST if set
if socketHost.Socket == "" && hasDockerHost {
// Cases: 4A
log.Debugf("Defaulting container socket to DOCKER_HOST")
socketHost.Socket = socketHost.Host
}
// Set sane default socket location if user omitted it
if socketHost.Socket == "" {
// Cases: 4B
socket, _ := socketLocation()
// socket is empty if it isn't found, so assignment here is at worst a no-op
log.Debugf("Defaulting container socket to default '%s'", socket)
socketHost.Socket = socket
}
// Exit if both the DOCKER_HOST and socket are fulfilled
if hasDockerHost {
// Cases: 1A, 2A, 3A, 4A
if !isDockerHostURI(socketHost.Socket) {
// Cases: 1A, 2A
log.Debugf("DOCKER_HOST is set, but socket is invalid '%s'", socketHost.Socket)
}
return socketHost, nil
}
// Set a sane DOCKER_HOST default if we can
if isDockerHostURI(socketHost.Socket) {
// Cases: 3B
log.Debugf("Setting DOCKER_HOST to container socket '%s'", socketHost.Socket)
socketHost.Host = socketHost.Socket
// Both DOCKER_HOST and container socket are valid; short-circuit exit
return socketHost, nil
}
// Here there is no DOCKER_HOST _and_ the supplied container socket is not a valid URI (either invalid or a file path)
// Cases: 2B <- but is already handled at the top
// I.e. this path should never be taken
return SocketAndHost{}, fmt.Errorf("no DOCKER_HOST and an invalid container socket '%s'", socketHost.Socket)
}
@@ -0,0 +1,167 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// Copyright 2024 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"os"
"testing"
log "github.com/sirupsen/logrus"
assert "github.com/stretchr/testify/assert"
)
func init() {
log.SetLevel(log.DebugLevel)
}
var originalCommonSocketLocations = CommonSocketLocations
func isolateSocketEnv(t *testing.T) {
t.Helper()
t.Cleanup(func() { CommonSocketLocations = originalCommonSocketLocations })
if host, ok := os.LookupEnv("DOCKER_HOST"); ok {
t.Setenv("DOCKER_HOST", host)
} else {
t.Cleanup(func() { os.Unsetenv("DOCKER_HOST") })
}
}
func TestGetSocketAndHostWithSocket(t *testing.T) {
// Arrange
isolateSocketEnv(t)
dockerHost := "unix:///my/docker/host.sock"
socketURI := "/path/to/my.socket"
t.Setenv("DOCKER_HOST", dockerHost)
// Act
ret, err := GetSocketAndHost(socketURI)
// Assert
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, SocketAndHost{socketURI, dockerHost}, ret)
}
func TestGetSocketAndHostNoSocket(t *testing.T) {
// Arrange
dockerHost := "unix:///my/docker/host.sock"
t.Setenv("DOCKER_HOST", dockerHost)
// Act
ret, err := GetSocketAndHost("")
// Assert
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, SocketAndHost{dockerHost, dockerHost}, ret)
}
func TestGetSocketAndHostOnlySocket(t *testing.T) {
// Arrange
isolateSocketEnv(t)
socketURI := "/path/to/my.socket"
os.Unsetenv("DOCKER_HOST")
defaultSocket, defaultSocketFound := socketLocation()
// Act
ret, err := GetSocketAndHost(socketURI)
// Assert
assert.NoError(t, err, "Expected no error from GetSocketAndHost") //nolint:testifylint // pre-existing issue from nektos/act
assert.True(t, defaultSocketFound, "Expected to find default socket")
assert.Equal(t, socketURI, ret.Socket, "Expected socket to match common location")
assert.Equal(t, defaultSocket, ret.Host, "Expected ret.Host to match default socket location")
}
func TestGetSocketAndHostDontMount(t *testing.T) {
// Arrange
isolateSocketEnv(t)
dockerHost := "unix:///my/docker/host.sock"
t.Setenv("DOCKER_HOST", dockerHost)
// Act
ret, err := GetSocketAndHost("-")
// Assert
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, SocketAndHost{"-", dockerHost}, ret)
}
func TestGetSocketAndHostNoHostNoSocket(t *testing.T) {
// Arrange
isolateSocketEnv(t)
os.Unsetenv("DOCKER_HOST")
defaultSocket, found := socketLocation()
// Act
ret, err := GetSocketAndHost("")
// Assert
assert.True(t, found, "Expected a default socket to be found")
assert.NoError(t, err, "Expected no error from GetSocketAndHost") //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, SocketAndHost{defaultSocket, defaultSocket}, ret, "Expected to match default socket location")
}
// Catch
// > Your code breaks setting DOCKER_HOST if shouldMount is false.
// > This happens if neither DOCKER_HOST nor --container-daemon-socket has a value, but socketLocation() returns a URI
func TestGetSocketAndHostNoHostNoSocketDefaultLocation(t *testing.T) {
// Arrange
isolateSocketEnv(t)
mySocketFile, tmpErr := os.CreateTemp(t.TempDir(), "act-*.sock")
mySocket := mySocketFile.Name()
unixSocket := "unix://" + mySocket
defer os.RemoveAll(mySocket)
assert.NoError(t, tmpErr) //nolint:testifylint // pre-existing issue from nektos/act
os.Unsetenv("DOCKER_HOST")
CommonSocketLocations = []string{mySocket}
defaultSocket, found := socketLocation()
// Act
ret, err := GetSocketAndHost("")
// Assert
assert.Equal(t, unixSocket, defaultSocket, "Expected default socket to match common socket location")
assert.True(t, found, "Expected default socket to be found")
assert.NoError(t, err, "Expected no error from GetSocketAndHost") //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, SocketAndHost{unixSocket, unixSocket}, ret, "Expected to match default socket location")
}
func TestGetSocketAndHostNoHostInvalidSocket(t *testing.T) {
// Arrange
isolateSocketEnv(t)
os.Unsetenv("DOCKER_HOST")
mySocket := "/my/socket/path.sock"
CommonSocketLocations = []string{"/unusual", "/socket", "/location"}
defaultSocket, found := socketLocation()
// Act
ret, err := GetSocketAndHost(mySocket)
// Assert
assert.False(t, found, "Expected no default socket to be found")
assert.Equal(t, "", defaultSocket, "Expected no default socket to be found") //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, SocketAndHost{}, ret, "Expected to match default socket location")
assert.Error(t, err, "Expected an error in invalid state")
}
func TestGetSocketAndHostOnlySocketValidButUnusualLocation(t *testing.T) {
// Arrange
isolateSocketEnv(t)
socketURI := "unix:///path/to/my.socket"
CommonSocketLocations = []string{"/unusual", "/location"}
os.Unsetenv("DOCKER_HOST")
defaultSocket, found := socketLocation()
// Act
ret, err := GetSocketAndHost(socketURI)
// Assert
// Default socket locations
assert.Equal(t, "", defaultSocket, "Expect default socket location to be empty") //nolint:testifylint // pre-existing issue from nektos/act
assert.False(t, found, "Expected no default socket to be found")
// Sane default
assert.NoError(t, err, "Expect no error from GetSocketAndHost") //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, socketURI, ret.Host, "Expect host to default to unusual socket")
}
@@ -12,7 +12,7 @@ import (
"runtime" "runtime"
"time" "time"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"github.com/moby/moby/api/types/system" "github.com/moby/moby/api/types/system"
) )
@@ -9,7 +9,7 @@ package container
import ( import (
"context" "context"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"github.com/moby/moby/client" "github.com/moby/moby/client"
) )
@@ -21,12 +21,11 @@ import (
"sync/atomic" "sync/atomic"
"time" "time"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"gitea.com/gitea/runner/act/filecollector" "gitea.com/gitea/runner/internal/act/filecollector"
"gitea.com/gitea/runner/act/lookpath" "gitea.com/gitea/runner/internal/act/lookpath"
"gitea.com/gitea/runner/internal/pkg/process" "gitea.com/gitea/runner/internal/pkg/process"
"github.com/creack/pty"
"github.com/go-git/go-billy/v5/helper/polyfill" "github.com/go-git/go-billy/v5/helper/polyfill"
"github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5/plumbing/format/gitignore" "github.com/go-git/go-git/v5/plumbing/format/gitignore"
@@ -72,6 +71,12 @@ func (e *HostEnvironment) Create(_, _ []string) common.Executor {
} }
} }
func (e *HostEnvironment) ConnectToNetwork(name string) common.Executor {
return func(ctx context.Context) error {
return nil
}
}
func (e *HostEnvironment) Close() common.Executor { func (e *HostEnvironment) Close() common.Executor {
return func(ctx context.Context) error { return func(ctx context.Context) error {
return nil return nil
@@ -92,6 +97,33 @@ func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Exec
} }
} }
func (e *HostEnvironment) CopyTarStream(ctx context.Context, destPath string, tarStream io.Reader) error {
if err := os.RemoveAll(destPath); err != nil {
return err
}
tr := tar.NewReader(tarStream)
cp := &filecollector.CopyCollector{
DstDir: destPath,
}
for {
ti, err := tr.Next()
if errors.Is(err, io.EOF) {
return nil
} else if err != nil {
return err
}
if ti.FileInfo().IsDir() {
continue
}
if ctx.Err() != nil {
return errors.New("CopyTarStream has been cancelled")
}
if err := cp.WriteFile(ti.Name, ti.FileInfo(), ti.Linkname, tr); err != nil {
return err
}
}
}
func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor { func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor {
return func(ctx context.Context) error { return func(ctx context.Context) error {
logger := common.Logger(ctx) logger := common.Logger(ctx)
@@ -110,6 +142,7 @@ func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore bool) c
ignorer = gitignore.NewMatcher(ps) ignorer = gitignore.NewMatcher(ps)
} }
fc := &filecollector.FileCollector{ fc := &filecollector.FileCollector{
Fs: &filecollector.DefaultFs{},
Ignorer: ignorer, Ignorer: ignorer,
SrcPath: srcPath, SrcPath: srcPath,
SrcPrefix: srcPrefix, SrcPrefix: srcPrefix,
@@ -147,6 +180,7 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin
srcPrefix += string(filepath.Separator) srcPrefix += string(filepath.Separator)
} }
fc := &filecollector.FileCollector{ fc := &filecollector.FileCollector{
Fs: &filecollector.DefaultFs{},
SrcPath: srcPath, SrcPath: srcPath,
SrcPrefix: srcPrefix, SrcPrefix: srcPrefix,
Handler: tc, Handler: tc,
@@ -212,8 +246,24 @@ func (w *ptyWriter) Write(buf []byte) (int, error) {
return w.Out.Write(buf) return w.Out.Write(buf)
} }
type localEnv struct {
env map[string]string
}
func (l *localEnv) Getenv(name string) string {
if runtime.GOOS == "windows" {
for k, v := range l.env {
if strings.EqualFold(name, k) {
return v
}
}
return ""
}
return l.env[name]
}
func lookupPathHost(cmd string, env map[string]string, writer io.Writer) (string, error) { func lookupPathHost(cmd string, env map[string]string, writer io.Writer) (string, error) {
f, err := lookpath.LookPath2(cmd, env) f, err := lookpath.LookPath2(cmd, &localEnv{env: env})
if err != nil { if err != nil {
err := "Cannot find: " + cmd + " in PATH" err := "Cannot find: " + cmd + " in PATH"
if _, _err := writer.Write([]byte(err + "\n")); _err != nil { if _, _err := writer.Write([]byte(err + "\n")); _err != nil {
@@ -225,7 +275,7 @@ func lookupPathHost(cmd string, env map[string]string, writer io.Writer) (string
} }
func setupPty(cmd *exec.Cmd, cmdline string) (*os.File, *os.File, error) { func setupPty(cmd *exec.Cmd, cmdline string) (*os.File, *os.File, error) {
ppty, tty, err := pty.Open() ppty, tty, err := openPty()
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@@ -351,7 +401,8 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
} }
err = cmd.Wait() err = cmd.Wait()
if err != nil { if err != nil {
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return ExitCodeError(exitErr.ExitCode()) return ExitCodeError(exitErr.ExitCode())
} }
return err return err
@@ -17,7 +17,7 @@ import (
"testing" "testing"
"time" "time"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -12,7 +12,7 @@ import (
"io" "io"
"strings" "strings"
"gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/act/common"
"golang.org/x/text/encoding/unicode" "golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform" "golang.org/x/text/transform"
@@ -46,10 +46,9 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
} }
singleLineEnv := strings.Index(line, "=") singleLineEnv := strings.Index(line, "=")
multiLineEnv := strings.Index(line, "<<") multiLineEnv := strings.Index(line, "<<")
switch { if singleLineEnv != -1 && (multiLineEnv == -1 || singleLineEnv < multiLineEnv) {
case singleLineEnv != -1 && (multiLineEnv == -1 || singleLineEnv < multiLineEnv):
localEnv[line[:singleLineEnv]] = line[singleLineEnv+1:] localEnv[line[:singleLineEnv]] = line[singleLineEnv+1:]
case multiLineEnv != -1: } else if multiLineEnv != -1 {
multiLineEnvContent := "" multiLineEnvContent := ""
multiLineEnvDelimiter := line[multiLineEnv+2:] multiLineEnvDelimiter := line[multiLineEnv+2:]
delimiterFound := false delimiterFound := false
@@ -71,7 +70,7 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter) return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter)
} }
localEnv[line[:multiLineEnv]] = multiLineEnvContent localEnv[line[:multiLineEnv]] = multiLineEnvContent
default: } else {
return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line) return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line)
} }
} }

Some files were not shown because too many files have changed in this diff Show More