[CI] Woodpecker based releases process

(cherry picked from commit c2a7aaeee8)
(cherry picked from commit 6b6007fbce)
(cherry picked from commit 63608a221e)
(cherry picked from commit 5cfe60baa7)
(cherry picked from commit 2af4c73d12)
(cherry picked from commit 1985959bfe)
(cherry picked from commit 880424c77e)
(cherry picked from commit c78a861d1b)
(cherry picked from commit 25c1227011)
(cherry picked from commit 7195e894ee)
(cherry picked from commit cf15153873)
(cherry picked from commit 9bee773c95)
(cherry picked from commit 581c3060da)
(cherry picked from commit bf550f9b2c)
(cherry picked from commit b570eca0b9)

[CI] implementation: Woodpecker based CI (squash)

Upgrade xgo to Go v1.20 for building binaries

(cherry picked from commit 6308c776b6)

[CI] v1.20: switch PR check from Woodpecker CI to Forgejo Actions

The PR checks for v1.19 still rely on Woodpecker CI. Keeping
.woodpecker in v1.20 while both Woodpecker CI & Forgejo Actions are
enabled would dupicate the checks.

The release process in releases remains Woodpecker CI.

(cherry picked from commit 93e42f3f53)
(cherry picked from commit 599c5162ad)
(cherry picked from commit 6f8b723a55)
(cherry picked from commit e238d7d72f)
(cherry picked from commit 93e3ebff86)
(cherry picked from commit 26bc89ed7b)

.gitignore

@@ -1,3 +1,6 @@
+# Emacs
+*~
+
 # Compiled Object files, Static and Dynamic libs (Shared Objects)
 *.o
 *.a

Makefile

@@ -415,8 +415,7 @@ lint-go-windows:
 .PHONY: lint-go-vet
 lint-go-vet:
 	@echo "Running go vet..."
-	@GOOS= GOARCH= $(GO) build code.gitea.io/gitea-vet
+	@$(GO) vet $(GO_PACKAGES)
-	@$(GO) vet -vettool=gitea-vet $(GO_PACKAGES)
 
 .PHONY: lint-editorconfig
 lint-editorconfig:
@@ -829,7 +828,7 @@ $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ)
 	CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@
 
 .PHONY: release
-release: frontend generate release-windows release-linux release-darwin release-freebsd release-copy release-compress vendor release-sources release-docs release-check
+release: frontend generate release-linux release-copy release-compress vendor release-sources release-check
 
 $(DIST_DIRS):
 	mkdir -p $(DIST_DIRS)

releases/Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine:3.17
+
+RUN echo root > state

releases/Dockerfile-rootless

@@ -0,0 +1,3 @@
+FROM alpine:3.17
+
+RUN echo rootless > state

releases/binaries-pull-push-test.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+
+set -ex
+
+test_teardown() {
+    setup_api
+    api DELETE repos/$PUSH_USER/forgejo/releases/tags/$TAG || true
+    api DELETE repos/$PUSH_USER/forgejo/tags/$TAG || true
+    rm -fr dist/release
+    setup_tea
+    $BIN_DIR/tea login delete $RELEASETEAMUSER || true
+}
+
+test_setup() {
+    mkdir -p $RELEASE_DIR
+    touch $RELEASE_DIR/file-one.txt
+    touch $RELEASE_DIR/file-two.txt
+}
+
+test_ensure_tag() {
+    api DELETE repos/$PUSH_USER/forgejo/tags/$TAG || true
+    #
+    # idempotent
+    #
+    ensure_tag
+    api GET repos/$PUSH_USER/forgejo/tags/$TAG > /tmp/tag1.json
+    ensure_tag
+    api GET repos/$PUSH_USER/forgejo/tags/$TAG > /tmp/tag2.json
+    diff -u /tmp/tag[12].json
+    #
+    # sanity check on the SHA of an existing tag
+    #
+    (
+	CI_COMMIT_SHA=12345
+	! ensure_tag
+    )
+    api DELETE repos/$PUSH_USER/forgejo/tags/$TAG
+}
+
+#
+# Running the test locally instead of within Woodpecker
+#
+# 1. Setup: obtain a token at https://codeberg.org/user/settings/applications
+# 2. Run: RELEASETEAMUSER=<username> RELEASETEAMTOKEn=<apptoken> binaries-pull-push-test.sh test_run
+# 3. Verify: (optional) manual verification at https://codeberg.org/<username>/forgejo/releases
+# 4. Cleanup: RELEASETEAMUSER=<username> RELEASETEAMTOKEn=<apptoken> binaries-pull-push-test.sh test_teardown
+#
+test_run() {
+    test_teardown
+    to_push=/tmp/binaries-releases-to-push
+    pulled=/tmp/binaries-releases-pulled
+    RELEASE_DIR=$to_push
+    test_setup
+    test_ensure_tag
+    echo "================================ TEST BEGIN"
+    push
+    RELEASE_DIR=$pulled
+    pull
+    diff -r $to_push $pulled
+    echo "================================ TEST END"
+}
+
+: ${CI_REPO_OWNER:=dachary}
+: ${PULL_USER=$CI_REPO_OWNER}
+: ${PUSH_USER=$CI_REPO_OWNER}
+: ${CI_COMMIT_TAG:=W17.8.20-1}
+: ${CI_COMMIT_SHA:=$(git rev-parse HEAD)}
+
+. $(dirname $0)/binaries-pull-push.sh

releases/binaries-pull-push.sh

@@ -0,0 +1,88 @@
+#!/bin/sh
+
+set -ex
+
+: ${PULL_USER:=forgejo-integration}
+if test "$CI_REPO" = "forgejo/release" ; then
+    : ${PUSH_USER:=forgejo}
+else
+    : ${PUSH_USER:=forgejo-experimental}
+fi
+: ${TAG:=${CI_COMMIT_TAG}}
+: ${DOMAIN:=codeberg.org}
+: ${RELEASE_DIR:=dist/release}
+: ${BIN_DIR:=/tmp}
+: ${TEA_VERSION:=0.9.0}
+
+
+setup_tea() {
+    if ! test -f $BIN_DIR/tea ; then
+	curl -sL https://dl.gitea.io/tea/$TEA_VERSION/tea-$TEA_VERSION-linux-amd64 > $BIN_DIR/tea
+	chmod +x $BIN_DIR/tea
+    fi
+}
+
+ensure_tag() {
+    if api GET repos/$PUSH_USER/forgejo/tags/$TAG > /tmp/tag.json ; then
+	local sha=$(jq --raw-output .commit.sha < /tmp/tag.json)
+	if test "$sha" != "$CI_COMMIT_SHA" ; then
+	    cat /tmp/tag.json
+	    echo "the tag SHA in the $PUSH_USER repository does not match the tag SHA that triggered the build: $CI_COMMIT_SHA"
+	    false
+	fi
+    else
+	api POST repos/$PUSH_USER/forgejo/tags --data-raw '{"tag_name": "'$CI_COMMIT_TAG'", "target": "'$CI_COMMIT_SHA'"}'
+    fi
+}
+
+upload() {
+    ASSETS=$(ls $RELEASE_DIR/* | sed -e 's/^/-a /')
+    echo "${CI_COMMIT_TAG}" | grep -qi '\-rc' && export RELEASETYPE="--prerelease" && echo "Uploading as Pre-Release"
+    echo "${CI_COMMIT_TAG}" | grep -qi '\-test' && export RELEASETYPE="--draft" && echo "Uploading as Draft"
+    test ${RELEASETYPE+false} || echo "Uploading as Stable"
+    ensure_tag
+    anchor=$(echo $CI_COMMIT_TAG | sed -e 's/^v//' -e 's/[^a-zA-Z0-9]/-/g')
+    $BIN_DIR/tea release create $ASSETS --repo $PUSH_USER/forgejo --note "See https://codeberg.org/forgejo/forgejo/src/branch/forgejo/RELEASE-NOTES.md#${anchor}" --tag $CI_COMMIT_TAG --title $CI_COMMIT_TAG ${RELEASETYPE}
+}
+
+push() {
+    setup_api
+    setup_tea
+    GITEA_SERVER_TOKEN=$RELEASETEAMTOKEN $BIN_DIR/tea login add --name $RELEASETEAMUSER --url $DOMAIN
+    upload
+}
+
+setup_api() {
+    if ! which jq || ! which curl ; then
+	apk --update --no-cache add jq curl
+    fi
+}
+
+api() {
+    method=$1
+    shift
+    path=$1
+    shift
+
+    curl --fail -X $method -sS -H "Content-Type: application/json" -H "Authorization: token $RELEASETEAMTOKEN" "$@" https://$DOMAIN/api/v1/$path
+}
+
+pull() {
+    setup_api
+    (
+	mkdir -p $RELEASE_DIR
+	cd $RELEASE_DIR
+	api GET repos/$PULL_USER/forgejo/releases/tags/$TAG > /tmp/assets.json
+	jq --raw-output '.assets[] | "\(.name) \(.browser_download_url)"' < /tmp/assets.json | while read name url ; do
+	    wget --quiet -O $name $url
+	done
+    )
+}
+
+
+missing() {
+    echo need pull or push argument got nothing
+    exit 1
+}
+
+${@:-missing}

releases/container-images-pull-verify-push-test.sh

@@ -0,0 +1,70 @@
+#!/bin/sh
+
+set -ex
+
+image_delete() {
+    curl -sS -H @$TOKEN_HEADER -X DELETE https://$DOMAIN/v2/$1/forgejo/manifests/$2
+}
+
+#
+# Create the same set of images that buildx would
+#
+test_setup() {
+    dir=$(dirname $0)
+
+    for suffix in '' '-rootless' ; do
+	(
+	    cd $dir
+	    manifests=""
+	    for arch in $ARCHS ; do
+		image=$(arch_image_name $PULL_USER $arch $suffix)
+		docker build -f Dockerfile$suffix --platform linux/$arch -t $image .
+		docker push $image
+		images="$images $image"
+	    done
+	    manifest=$(image_name $PULL_USER $suffix)
+	    docker manifest rm $manifest || true
+	    docker manifest create $manifest $images
+	    image_put $PULL_USER $(image_tag $suffix) $manifest
+	)
+    done
+}
+
+test_teardown() {
+    authenticate
+    for suffix in '' '-rootless' ; do
+	image_delete $PULL_USER $(image_tag $suffix)
+	image_delete $PUSH_USER $(image_tag $suffix)
+	image_delete $PUSH_USER $(short_image_tag $suffix)
+	for arch in $ARCHS ; do
+	    image_delete $PULL_USER $(arch_image_tag $arch $suffix)
+	    image_delete $PUSH_USER $(arch_image_tag $arch $suffix)
+	done
+    done
+}
+
+#
+# Running the test locally instead of within Woodpecker
+#
+# 1. Setup: obtain a token at https://codeberg.org/user/settings/applications
+# 2. Run: RELEASETEAMUSER=<username> RELEASETEAMTOKEn=<apptoken> container-images-pull-verify-push-test.sh test_run
+# 3. Verify: (optional) manual verification at https://codeberg.org/<username>/-/packages/container/forgejo/versions
+# 4. Cleanup: RELEASETEAMUSER=<username> RELEASETEAMTOKEn=<apptoken> container-images-pull-verify-push-test.sh test_teardown
+#
+test_run() {
+    boot
+    test_teardown
+    test_setup
+    VERIFY_STRING=something
+    VERIFY_COMMAND="echo $VERIFY_STRING"
+    echo "================================ TEST BEGIN"
+    main
+    echo "================================ TEST END"
+}
+
+: ${CI_REPO_OWNER:=dachary}
+: ${PULL_USER:=$CI_REPO_OWNER}
+: ${PUSH_USER:=$CI_REPO_OWNER}
+: ${CI_COMMIT_TAG:=v17.1.42-2}
+
+. $(dirname $0)/container-images-pull-verify-push.sh

releases/container-images-pull-verify-push.sh

@@ -0,0 +1,122 @@
+#!/bin/sh
+
+set -ex
+
+: ${DOCKER_HOST:=unix:///var/run/docker.sock}
+: ${ARCHS:=amd64 arm64}
+: ${PULL_USER:=forgejo-integration}
+if test "$CI_REPO" = "forgejo/release" ; then
+    : ${PUSH_USER:=forgejo}
+else
+    : ${PUSH_USER:=forgejo-experimental}
+fi
+: ${INTEGRATION_IMAGE:=codeberg.org/$PULL_USER/forgejo}
+: ${TAG:=${CI_COMMIT_TAG##v}}
+: ${SHORT_TAG=${TAG%.*-*}}
+: ${DOMAIN:=codeberg.org}
+: ${TOKEN_HEADER:=/tmp/token$$}
+trap "rm -f ${TOKEN_HEADER}" EXIT
+
+: ${VERIFY:=true}
+VERIFY_COMMAND='gitea --version'
+VERIFY_STRING='built with'
+
+publish() {
+    for suffix in '' '-rootless' ; do
+	images=""
+	for arch in $ARCHS ; do
+	    #
+	    # Get the image from the integration user
+	    #
+	    image=$(image_name $PULL_USER $suffix)
+	    docker pull --platform linux/$arch $image
+	    #
+	    # Verify it is usable
+	    #
+	    if $VERIFY ; then
+		docker run --platform linux/$arch --rm $image $VERIFY_COMMAND | grep "$VERIFY_STRING"
+	    fi
+	    #
+	    # Push the image with a tag reflecting the architecture to the repo owner
+	    #
+	    arch_image=$(arch_image_name $PUSH_USER $arch $suffix)
+	    docker tag $image $arch_image
+	    docker push $arch_image
+	    images="$images $arch_image"
+	done
+
+	#
+	# Push a manifest with all the architectures to the repo owner
+	#
+	manifest=$(image_name $PUSH_USER $suffix)
+	docker manifest rm $manifest || true
+	docker manifest create $manifest $images
+	image_put $PUSH_USER $(image_tag $suffix) $manifest
+	image_put $PUSH_USER $(short_image_tag $suffix) $manifest
+	#
+	# Sanity check to ensure the manifest that are published can actualy
+	# be used.
+	#
+	for arch in $ARCHS ; do
+	    docker pull --platform linux/$arch $(image_name $PUSH_USER $suffix)
+	    docker pull --platform linux/$arch $(short_image_name $PUSH_USER $suffix)
+	done
+    done
+}
+
+boot() {
+    if docker version ; then
+	return
+    fi
+    apk --update --no-cache add coredns jq curl
+    ( echo ".:53 {" ; echo "  forward . /etc/resolv.conf"; echo "}" ) > /etc/coredns/Corefile
+    coredns -conf /etc/coredns/Corefile &
+    /usr/local/bin/dockerd --data-root /var/lib/docker --host=$DOCKER_HOST --dns 172.17.0.3 &
+    for i in $(seq 60) ; do
+	docker version && break
+	sleep 1
+    done
+    docker version || exit 1
+}
+
+authenticate() {
+    echo "$RELEASETEAMTOKEN" | docker login --password-stdin --username "$RELEASETEAMUSER" $DOMAIN
+    curl -u$RELEASETEAMUSER:$RELEASETEAMTOKEN -sS https://$DOMAIN/v2/token | jq --raw-output '"Authorization: token \(.token)"' > $TOKEN_HEADER
+}
+
+image_put() {
+    docker manifest inspect $3 > /tmp/manifest.json
+    curl -sS -H @$TOKEN_HEADER -X PUT --data-binary @/tmp/manifest.json https://$DOMAIN/v2/$1/forgejo/manifests/$2
+}
+
+main() {
+    boot
+    authenticate
+    publish
+}
+
+image_name() {
+    echo $DOMAIN/$1/forgejo:$(image_tag $2)
+}
+
+image_tag() {
+    echo $TAG$1
+}
+
+short_image_name() {
+    echo $DOMAIN/$1/forgejo:$(short_image_tag $2)
+}
+
+short_image_tag() {
+    echo $SHORT_TAG$1
+}
+
+arch_image_name() {
+    echo $DOMAIN/$1/forgejo:$(arch_image_tag $2 $3)
+}
+
+arch_image_tag() {
+    echo $TAG-$1$2
+}
+
+${@:-main}

releases/woodpecker-build/binaries.yml

@@ -0,0 +1,107 @@
+platform: linux/amd64
+
+when:
+  event: tag
+  tag: v*
+
+variables:
+ - &node_image 'node:18'
+ - &golang_image 'golang:1.20'
+ - &alpine_image 'alpine:3.18'
+ - &gpg_sign_image 'plugins/gpgsign:1'
+ - &xgo_image 'techknowlogick/xgo:go-1.20.x'
+ - &gpg_sign_image 'plugins/gpgsign:1'
+ - &goproxy_override ''
+ - &goproxy_setup |-
+      if [ -n "$${GOPROXY_OVERRIDE:-}" ]; then
+        export GOPROXY="$${GOPROXY_OVERRIDE}";
+        echo "Using goproxy from goproxy_override \"$${GOPROXY}\"";
+      elif [ -n "$${GOPROXY_DEFAULT:-}" ]; then
+        export GOPROXY="$${GOPROXY_DEFAULT}";
+        echo "Using goproxy from goproxy_default (secret) not displaying";
+      else
+        export GOPROXY="https://proxy.golang.org,direct";
+        echo "No goproxy overrides or defaults given, using \"$${GOPROXY}\"";
+      fi
+
+workspace:
+  base: /source
+  path: /
+
+pipeline:
+  fetch-tags:
+    image: *golang_image
+    pull: true
+    group: deps
+    commands:
+     - git config --add safe.directory '*'
+     - git fetch --tags --force
+
+  deps-frontend:
+    image: *node_image
+    pull: true
+    group: deps
+    commands:
+      - make deps-frontend
+
+  deps-backend:
+    image: *golang_image
+    pull: true
+    group: deps
+    environment:
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+      - goproxy_default
+    commands:
+    - *goproxy_setup
+    - make deps-backend
+
+  static:
+    image: *xgo_image
+    pull: true
+    commands:
+      - *goproxy_setup
+      - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get -qqy install nodejs
+      - export PATH=$PATH:$GOPATH/bin
+      - make CI=true LINUX_ARCHS=linux/amd64,linux/arm64,linux/arm-6 release
+    environment:
+      TAGS: 'bindata sqlite sqlite_unlock_notify'
+      DEBIAN_FRONTEND: 'noninteractive'
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+      - goproxy_default
+
+  #
+  # See https://codeberg.org/forgejo/forgejo/issues/230 for a discussion on this
+  # compilation stage. The goal is just to verify the build does not break, not that
+  # the binary produced actually works.
+  #
+  freebsd:
+    image: *xgo_image
+    group: build
+    commands:
+      - *goproxy_setup
+      - export PATH=$PATH:$GOPATH/bin
+      - make CI=false release-freebsd
+    environment:
+      TAGS: 'bindata sqlite sqlite_unlock_notify'
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+      - goproxy_default
+
+  verifyruns:
+    image: *golang_image
+    commands:
+      - ./dist/release/forgejo-*-amd64 --version | grep 'built with'
+      - apt-get update
+      - apt-get install -y qemu-user-static
+      - /usr/bin/qemu-aarch64-static ./dist/release/forgejo-*-arm64 --version | grep 'built with'
+      - /usr/bin/qemu-arm-static ./dist/release/forgejo-*-arm-6 --version | grep 'built with'
+
+  push-integration:
+    image: *alpine_image
+    commands:
+      - PUSH_USER=$CI_REPO_OWNER releases/binaries-pull-push.sh push
+    secrets:
+      - releaseteamtoken
+      - releaseteamuser

releases/woodpecker-build/container-images.yml

@@ -0,0 +1,65 @@
+platform: linux/amd64
+
+when:
+  event: tag
+  tag: v*
+
+variables:
+ - &golang_image 'golang:1.20'
+ - &dind_image 'docker:20.10-dind'
+ - &buildx_image 'woodpeckerci/plugin-docker-buildx:2.0.0'
+ - &integration_image 'codeberg.org/forgejo-integration/forgejo'
+ - &dockerfile_root 'Dockerfile'
+# for testing purposes
+# - &dockerfile_root 'releases/Dockerfile'
+ - &dockerfile_rootless 'Dockerfile.rootless'
+# for testing purposes
+# - &dockerfile_rootless 'releases/Dockerfile-rootless'
+ - &verify 'true'
+# for testing purposes
+# - &verify 'false'
+ - &archs 'amd64 arm64'
+
+pipeline:
+  fetch-tags:
+    image: *golang_image
+    pull: true
+    commands:
+     - git config --add safe.directory '*'
+     - git fetch --tags --force
+
+  build-root:
+    image: *buildx_image
+    group: integration
+    pull: true
+    settings:
+      platforms: linux/amd64,linux/arm64
+      dockerfile: *dockerfile_root
+      registry:
+        from_secret: domain
+      tag: ${CI_COMMIT_TAG##v}
+      repo: *integration_image
+      build_args:
+        - GOPROXY=https://proxy.golang.org
+      password:
+        from_secret: releaseteamtoken
+      username:
+        from_secret: releaseteamuser
+
+  build-rootless:
+    image: *buildx_image
+    group: integration
+    pull: true
+    settings:
+      platforms: linux/amd64,linux/arm64
+      dockerfile: *dockerfile_rootless
+      registry:
+        from_secret: domain
+      tag: ${CI_COMMIT_TAG##v}-rootless
+      repo: *integration_image
+      build_args:
+        - GOPROXY=https://proxy.golang.org
+      password:
+        from_secret: releaseteamtoken
+      username:
+        from_secret: releaseteamuser

releases/woodpecker-build/releases-helper.yml

@@ -0,0 +1,34 @@
+platform: linux/amd64
+
+when:
+  event: push
+
+variables:
+ - &dind_image 'docker:20.10-dind'
+ - &alpine_image 'alpine:3.17'
+
+pipeline:
+  container-images-pull-verify-push:
+    image: *dind_image
+    group: integration
+    commands:
+# arm64 would require qemu-user-static which is not available on alpline
+# the test coverage does not change much and running the tests test locally
+# is possible if there is a doubt
+      - ARCHS=amd64 ./releases/container-images-pull-verify-push-test.sh test_run
+      - ./releases/container-images-pull-verify-push-test.sh test_teardown
+    secrets:
+      - releaseteamuser
+      - releaseteamtoken
+      - domain
+
+  binaries-pull-push:
+    image: *alpine_image
+    group: integration
+    commands:
+      - ./releases/binaries-pull-push-test.sh test_run
+      - ./releases/binaries-pull-push-test.sh test_teardown
+    secrets:
+      - releaseteamuser
+      - releaseteamtoken
+      - domain

releases/woodpecker-publish/binaries.yml

@@ -0,0 +1,36 @@
+platform: linux/amd64
+
+when:
+  event: tag
+
+variables:
+ - &dind_image 'docker:20.10-dind'
+ - &gpg_sign_image 'plugins/gpgsign:1'
+
+pipeline:
+
+  pull:
+    image: *dind_image
+    commands:
+      - ./releases/binaries-pull-push.sh pull
+
+  gpg-sign:
+    image: *gpg_sign_image
+    pull: true
+    settings:
+      detach_sign: true
+      excludes:
+        - "dist/release/*.sha256"
+      files:
+        - "dist/release/*"
+      key:
+        from_secret: releaseteamgpg
+
+  push:
+    image: *dind_image
+    commands:
+      - ./releases/binaries-pull-push.sh push
+    secrets:
+      - releaseteamtoken
+      - releaseteamuser
+      - domain

releases/woodpecker-publish/container-images.yml

@@ -0,0 +1,27 @@
+platform: linux/amd64
+
+when:
+  event: tag
+
+variables:
+ - &dind_image 'docker:20.10-dind'
+ - &integration_image 'codeberg.org/forgejo-integration/forgejo'
+ - &verify 'true'
+# for testing purposes
+# - &verify 'false'
+ - &archs 'amd64 arm64'
+
+pipeline:
+
+  publish:
+    image: *dind_image
+    environment:
+      INTEGRATION_IMAGE: *integration_image
+      VERIFY: *verify
+      ARCHS: *archs
+    commands:
+      - ./releases/container-images-pull-verify-push.sh
+    secrets:
+      - releaseteamtoken
+      - releaseteamuser
+      - domain