[CI] Woodpecker based releases process

(cherry picked from commit c2a7aaeee82293793b2740251dc5fd27dfb32ddb)
(cherry picked from commit 6b6007fbced1cecb15aeb09e709b23855c43177e)
(cherry picked from commit 63608a221e551a0e1822901be62f37a557622edd)
(cherry picked from commit 5cfe60baa7a3d250458a8b562976f605357c806b)
(cherry picked from commit 2af4c73d12a13ac5e0cd9bb02ff4195e3d88c47c)
(cherry picked from commit 1985959bfea6df1cb965dc0a963d8fb49eddc2f3)
(cherry picked from commit 880424c77ee46f7cfabccbf8eb546a66e05c0e1c)
(cherry picked from commit c78a861d1bccc92df94ef01157ee3828b960063e)
(cherry picked from commit 25c122701130c3cfce1e11e88ba0d0de8bc32dad)
(cherry picked from commit 7195e894ee0dd348b02b034a0821c52f055c0eb9)
(cherry picked from commit cf15153873f6007d233811faf171d0828461152b)
(cherry picked from commit 9bee773c95298a65ad0df2e17eb11270b61191b6)
(cherry picked from commit 581c3060da3561569c9b03c6931064b55438944c)
(cherry picked from commit bf550f9b2c6fb09fba02c5884cf1206024780149)
(cherry picked from commit b570eca0b950e94dc78a91b57391ed031dccefd4)

[CI] implementation: Woodpecker based CI (squash)

Upgrade xgo to Go v1.20 for building binaries

(cherry picked from commit 6308c776b6e8fb8a904d08b178f61a7dab1ef971)

[CI] v1.20: switch PR check from Woodpecker CI to Forgejo Actions

The PR checks for v1.19 still rely on Woodpecker CI. Keeping
.woodpecker in v1.20 while both Woodpecker CI & Forgejo Actions are
enabled would dupicate the checks.

The release process in releases remains Woodpecker CI.

(cherry picked from commit 93e42f3f53da1084db21b354994449a32e4abca4)
(cherry picked from commit 599c5162ad069fa5cb239a91486152cd47c77bf7)
(cherry picked from commit 6f8b723a55277c1e15827dcdfaa9a0ffe3ea42b4)
(cherry picked from commit e238d7d72fd5aa9741e9c5e76f37f5aff8db2892)
(cherry picked from commit 93e3ebff86a19f16619946a1aacff1642f0593c6)
(cherry picked from commit 26bc89ed7b3c241f9718ebb6a907ac4f8060ae0b)
(cherry picked from commit 8a9ff9b0a473ebd345876b8c6ce603fbf2e324af)
This commit is contained in:
Loïc Dachary 2022-11-05 00:00:07 +01:00 committed by Earl Warren
parent a06b0723a9
commit de0dbb8579
No known key found for this signature in database
GPG key ID: 0579CB2928A78A00
13 changed files with 629 additions and 3 deletions

3
.gitignore vendored
View file

@ -1,3 +1,6 @@
# Emacs
*~
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a

View file

@ -415,8 +415,7 @@ lint-go-windows:
.PHONY: lint-go-vet
lint-go-vet:
@echo "Running go vet..."
@GOOS= GOARCH= $(GO) build code.gitea.io/gitea-vet
@$(GO) vet -vettool=gitea-vet $(GO_PACKAGES)
@$(GO) vet $(GO_PACKAGES)
.PHONY: lint-editorconfig
lint-editorconfig:
@ -829,7 +828,7 @@ $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ)
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@
.PHONY: release
release: frontend generate release-windows release-linux release-darwin release-freebsd release-copy release-compress vendor release-sources release-docs release-check
release: frontend generate release-linux release-copy release-compress vendor release-sources release-check
$(DIST_DIRS):
mkdir -p $(DIST_DIRS)

3
releases/Dockerfile Normal file
View file

@ -0,0 +1,3 @@
FROM alpine:3.17
RUN echo root > state

View file

@ -0,0 +1,3 @@
FROM alpine:3.17
RUN echo rootless > state

View file

@ -0,0 +1,69 @@
#!/bin/sh
set -ex
test_teardown() {
setup_api
api DELETE repos/$PUSH_USER/forgejo/releases/tags/$TAG || true
api DELETE repos/$PUSH_USER/forgejo/tags/$TAG || true
rm -fr dist/release
setup_tea
$BIN_DIR/tea login delete $RELEASETEAMUSER || true
}
test_setup() {
mkdir -p $RELEASE_DIR
touch $RELEASE_DIR/file-one.txt
touch $RELEASE_DIR/file-two.txt
}
test_ensure_tag() {
api DELETE repos/$PUSH_USER/forgejo/tags/$TAG || true
#
# idempotent
#
ensure_tag
api GET repos/$PUSH_USER/forgejo/tags/$TAG > /tmp/tag1.json
ensure_tag
api GET repos/$PUSH_USER/forgejo/tags/$TAG > /tmp/tag2.json
diff -u /tmp/tag[12].json
#
# sanity check on the SHA of an existing tag
#
(
CI_COMMIT_SHA=12345
! ensure_tag
)
api DELETE repos/$PUSH_USER/forgejo/tags/$TAG
}
#
# Running the test locally instead of within Woodpecker
#
# 1. Setup: obtain a token at https://codeberg.org/user/settings/applications
# 2. Run: RELEASETEAMUSER=<username> RELEASETEAMTOKEn=<apptoken> binaries-pull-push-test.sh test_run
# 3. Verify: (optional) manual verification at https://codeberg.org/<username>/forgejo/releases
# 4. Cleanup: RELEASETEAMUSER=<username> RELEASETEAMTOKEn=<apptoken> binaries-pull-push-test.sh test_teardown
#
test_run() {
test_teardown
to_push=/tmp/binaries-releases-to-push
pulled=/tmp/binaries-releases-pulled
RELEASE_DIR=$to_push
test_setup
test_ensure_tag
echo "================================ TEST BEGIN"
push
RELEASE_DIR=$pulled
pull
diff -r $to_push $pulled
echo "================================ TEST END"
}
: ${CI_REPO_OWNER:=dachary}
: ${PULL_USER=$CI_REPO_OWNER}
: ${PUSH_USER=$CI_REPO_OWNER}
: ${CI_COMMIT_TAG:=W17.8.20-1}
: ${CI_COMMIT_SHA:=$(git rev-parse HEAD)}
. $(dirname $0)/binaries-pull-push.sh

88
releases/binaries-pull-push.sh Executable file
View file

@ -0,0 +1,88 @@
#!/bin/sh
set -ex
: ${PULL_USER:=forgejo-integration}
if test "$CI_REPO" = "forgejo/release" ; then
: ${PUSH_USER:=forgejo}
else
: ${PUSH_USER:=forgejo-experimental}
fi
: ${TAG:=${CI_COMMIT_TAG}}
: ${DOMAIN:=codeberg.org}
: ${RELEASE_DIR:=dist/release}
: ${BIN_DIR:=/tmp}
: ${TEA_VERSION:=0.9.0}
setup_tea() {
if ! test -f $BIN_DIR/tea ; then
curl -sL https://dl.gitea.io/tea/$TEA_VERSION/tea-$TEA_VERSION-linux-amd64 > $BIN_DIR/tea
chmod +x $BIN_DIR/tea
fi
}
ensure_tag() {
if api GET repos/$PUSH_USER/forgejo/tags/$TAG > /tmp/tag.json ; then
local sha=$(jq --raw-output .commit.sha < /tmp/tag.json)
if test "$sha" != "$CI_COMMIT_SHA" ; then
cat /tmp/tag.json
echo "the tag SHA in the $PUSH_USER repository does not match the tag SHA that triggered the build: $CI_COMMIT_SHA"
false
fi
else
api POST repos/$PUSH_USER/forgejo/tags --data-raw '{"tag_name": "'$CI_COMMIT_TAG'", "target": "'$CI_COMMIT_SHA'"}'
fi
}
upload() {
ASSETS=$(ls $RELEASE_DIR/* | sed -e 's/^/-a /')
echo "${CI_COMMIT_TAG}" | grep -qi '\-rc' && export RELEASETYPE="--prerelease" && echo "Uploading as Pre-Release"
echo "${CI_COMMIT_TAG}" | grep -qi '\-test' && export RELEASETYPE="--draft" && echo "Uploading as Draft"
test ${RELEASETYPE+false} || echo "Uploading as Stable"
ensure_tag
anchor=$(echo $CI_COMMIT_TAG | sed -e 's/^v//' -e 's/[^a-zA-Z0-9]/-/g')
$BIN_DIR/tea release create $ASSETS --repo $PUSH_USER/forgejo --note "See https://codeberg.org/forgejo/forgejo/src/branch/forgejo/RELEASE-NOTES.md#${anchor}" --tag $CI_COMMIT_TAG --title $CI_COMMIT_TAG ${RELEASETYPE}
}
push() {
setup_api
setup_tea
GITEA_SERVER_TOKEN=$RELEASETEAMTOKEN $BIN_DIR/tea login add --name $RELEASETEAMUSER --url $DOMAIN
upload
}
setup_api() {
if ! which jq || ! which curl ; then
apk --update --no-cache add jq curl
fi
}
api() {
method=$1
shift
path=$1
shift
curl --fail -X $method -sS -H "Content-Type: application/json" -H "Authorization: token $RELEASETEAMTOKEN" "$@" https://$DOMAIN/api/v1/$path
}
pull() {
setup_api
(
mkdir -p $RELEASE_DIR
cd $RELEASE_DIR
api GET repos/$PULL_USER/forgejo/releases/tags/$TAG > /tmp/assets.json
jq --raw-output '.assets[] | "\(.name) \(.browser_download_url)"' < /tmp/assets.json | while read name url ; do
wget --quiet -O $name $url
done
)
}
missing() {
echo need pull or push argument got nothing
exit 1
}
${@:-missing}

View file

@ -0,0 +1,70 @@
#!/bin/sh
set -ex
image_delete() {
curl -sS -H @$TOKEN_HEADER -X DELETE https://$DOMAIN/v2/$1/forgejo/manifests/$2
}
#
# Create the same set of images that buildx would
#
test_setup() {
dir=$(dirname $0)
for suffix in '' '-rootless' ; do
(
cd $dir
manifests=""
for arch in $ARCHS ; do
image=$(arch_image_name $PULL_USER $arch $suffix)
docker build -f Dockerfile$suffix --platform linux/$arch -t $image .
docker push $image
images="$images $image"
done
manifest=$(image_name $PULL_USER $suffix)
docker manifest rm $manifest || true
docker manifest create $manifest $images
image_put $PULL_USER $(image_tag $suffix) $manifest
)
done
}
test_teardown() {
authenticate
for suffix in '' '-rootless' ; do
image_delete $PULL_USER $(image_tag $suffix)
image_delete $PUSH_USER $(image_tag $suffix)
image_delete $PUSH_USER $(short_image_tag $suffix)
for arch in $ARCHS ; do
image_delete $PULL_USER $(arch_image_tag $arch $suffix)
image_delete $PUSH_USER $(arch_image_tag $arch $suffix)
done
done
}
#
# Running the test locally instead of within Woodpecker
#
# 1. Setup: obtain a token at https://codeberg.org/user/settings/applications
# 2. Run: RELEASETEAMUSER=<username> RELEASETEAMTOKEn=<apptoken> container-images-pull-verify-push-test.sh test_run
# 3. Verify: (optional) manual verification at https://codeberg.org/<username>/-/packages/container/forgejo/versions
# 4. Cleanup: RELEASETEAMUSER=<username> RELEASETEAMTOKEn=<apptoken> container-images-pull-verify-push-test.sh test_teardown
#
test_run() {
boot
test_teardown
test_setup
VERIFY_STRING=something
VERIFY_COMMAND="echo $VERIFY_STRING"
echo "================================ TEST BEGIN"
main
echo "================================ TEST END"
}
: ${CI_REPO_OWNER:=dachary}
: ${PULL_USER:=$CI_REPO_OWNER}
: ${PUSH_USER:=$CI_REPO_OWNER}
: ${CI_COMMIT_TAG:=v17.1.42-2}
. $(dirname $0)/container-images-pull-verify-push.sh

View file

@ -0,0 +1,122 @@
#!/bin/sh
set -ex
: ${DOCKER_HOST:=unix:///var/run/docker.sock}
: ${ARCHS:=amd64 arm64}
: ${PULL_USER:=forgejo-integration}
if test "$CI_REPO" = "forgejo/release" ; then
: ${PUSH_USER:=forgejo}
else
: ${PUSH_USER:=forgejo-experimental}
fi
: ${INTEGRATION_IMAGE:=codeberg.org/$PULL_USER/forgejo}
: ${TAG:=${CI_COMMIT_TAG##v}}
: ${SHORT_TAG=${TAG%.*-*}}
: ${DOMAIN:=codeberg.org}
: ${TOKEN_HEADER:=/tmp/token$$}
trap "rm -f ${TOKEN_HEADER}" EXIT
: ${VERIFY:=true}
VERIFY_COMMAND='gitea --version'
VERIFY_STRING='built with'
publish() {
for suffix in '' '-rootless' ; do
images=""
for arch in $ARCHS ; do
#
# Get the image from the integration user
#
image=$(image_name $PULL_USER $suffix)
docker pull --platform linux/$arch $image
#
# Verify it is usable
#
if $VERIFY ; then
docker run --platform linux/$arch --rm $image $VERIFY_COMMAND | grep "$VERIFY_STRING"
fi
#
# Push the image with a tag reflecting the architecture to the repo owner
#
arch_image=$(arch_image_name $PUSH_USER $arch $suffix)
docker tag $image $arch_image
docker push $arch_image
images="$images $arch_image"
done
#
# Push a manifest with all the architectures to the repo owner
#
manifest=$(image_name $PUSH_USER $suffix)
docker manifest rm $manifest || true
docker manifest create $manifest $images
image_put $PUSH_USER $(image_tag $suffix) $manifest
image_put $PUSH_USER $(short_image_tag $suffix) $manifest
#
# Sanity check to ensure the manifest that are published can actualy
# be used.
#
for arch in $ARCHS ; do
docker pull --platform linux/$arch $(image_name $PUSH_USER $suffix)
docker pull --platform linux/$arch $(short_image_name $PUSH_USER $suffix)
done
done
}
boot() {
if docker version ; then
return
fi
apk --update --no-cache add coredns jq curl
( echo ".:53 {" ; echo " forward . /etc/resolv.conf"; echo "}" ) > /etc/coredns/Corefile
coredns -conf /etc/coredns/Corefile &
/usr/local/bin/dockerd --data-root /var/lib/docker --host=$DOCKER_HOST --dns 172.17.0.3 &
for i in $(seq 60) ; do
docker version && break
sleep 1
done
docker version || exit 1
}
authenticate() {
echo "$RELEASETEAMTOKEN" | docker login --password-stdin --username "$RELEASETEAMUSER" $DOMAIN
curl -u$RELEASETEAMUSER:$RELEASETEAMTOKEN -sS https://$DOMAIN/v2/token | jq --raw-output '"Authorization: token \(.token)"' > $TOKEN_HEADER
}
image_put() {
docker manifest inspect $3 > /tmp/manifest.json
curl -sS -H @$TOKEN_HEADER -X PUT --data-binary @/tmp/manifest.json https://$DOMAIN/v2/$1/forgejo/manifests/$2
}
main() {
boot
authenticate
publish
}
image_name() {
echo $DOMAIN/$1/forgejo:$(image_tag $2)
}
image_tag() {
echo $TAG$1
}
short_image_name() {
echo $DOMAIN/$1/forgejo:$(short_image_tag $2)
}
short_image_tag() {
echo $SHORT_TAG$1
}
arch_image_name() {
echo $DOMAIN/$1/forgejo:$(arch_image_tag $2 $3)
}
arch_image_tag() {
echo $TAG-$1$2
}
${@:-main}

View file

@ -0,0 +1,107 @@
platform: linux/amd64
when:
event: tag
tag: v*
variables:
- &node_image 'node:18'
- &golang_image 'golang:1.20'
- &alpine_image 'alpine:3.18'
- &gpg_sign_image 'plugins/gpgsign:1'
- &xgo_image 'techknowlogick/xgo:go-1.20.x'
- &gpg_sign_image 'plugins/gpgsign:1'
- &goproxy_override ''
- &goproxy_setup |-
if [ -n "$${GOPROXY_OVERRIDE:-}" ]; then
export GOPROXY="$${GOPROXY_OVERRIDE}";
echo "Using goproxy from goproxy_override \"$${GOPROXY}\"";
elif [ -n "$${GOPROXY_DEFAULT:-}" ]; then
export GOPROXY="$${GOPROXY_DEFAULT}";
echo "Using goproxy from goproxy_default (secret) not displaying";
else
export GOPROXY="https://proxy.golang.org,direct";
echo "No goproxy overrides or defaults given, using \"$${GOPROXY}\"";
fi
workspace:
base: /source
path: /
pipeline:
fetch-tags:
image: *golang_image
pull: true
group: deps
commands:
- git config --add safe.directory '*'
- git fetch --tags --force
deps-frontend:
image: *node_image
pull: true
group: deps
commands:
- make deps-frontend
deps-backend:
image: *golang_image
pull: true
group: deps
environment:
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- goproxy_default
commands:
- *goproxy_setup
- make deps-backend
static:
image: *xgo_image
pull: true
commands:
- *goproxy_setup
- curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get -qqy install nodejs
- export PATH=$PATH:$GOPATH/bin
- make CI=true LINUX_ARCHS=linux/amd64,linux/arm64,linux/arm-6 release
environment:
TAGS: 'bindata sqlite sqlite_unlock_notify'
DEBIAN_FRONTEND: 'noninteractive'
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- goproxy_default
#
# See https://codeberg.org/forgejo/forgejo/issues/230 for a discussion on this
# compilation stage. The goal is just to verify the build does not break, not that
# the binary produced actually works.
#
freebsd:
image: *xgo_image
group: build
commands:
- *goproxy_setup
- export PATH=$PATH:$GOPATH/bin
- make CI=false release-freebsd
environment:
TAGS: 'bindata sqlite sqlite_unlock_notify'
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- goproxy_default
verifyruns:
image: *golang_image
commands:
- ./dist/release/forgejo-*-amd64 --version | grep 'built with'
- apt-get update
- apt-get install -y qemu-user-static
- /usr/bin/qemu-aarch64-static ./dist/release/forgejo-*-arm64 --version | grep 'built with'
- /usr/bin/qemu-arm-static ./dist/release/forgejo-*-arm-6 --version | grep 'built with'
push-integration:
image: *alpine_image
commands:
- PUSH_USER=$CI_REPO_OWNER releases/binaries-pull-push.sh push
secrets:
- releaseteamtoken
- releaseteamuser

View file

@ -0,0 +1,65 @@
platform: linux/amd64
when:
event: tag
tag: v*
variables:
- &golang_image 'golang:1.20'
- &dind_image 'docker:20.10-dind'
- &buildx_image 'woodpeckerci/plugin-docker-buildx:2.0.0'
- &integration_image 'codeberg.org/forgejo-integration/forgejo'
- &dockerfile_root 'Dockerfile'
# for testing purposes
# - &dockerfile_root 'releases/Dockerfile'
- &dockerfile_rootless 'Dockerfile.rootless'
# for testing purposes
# - &dockerfile_rootless 'releases/Dockerfile-rootless'
- &verify 'true'
# for testing purposes
# - &verify 'false'
- &archs 'amd64 arm64'
pipeline:
fetch-tags:
image: *golang_image
pull: true
commands:
- git config --add safe.directory '*'
- git fetch --tags --force
build-root:
image: *buildx_image
group: integration
pull: true
settings:
platforms: linux/amd64,linux/arm64
dockerfile: *dockerfile_root
registry:
from_secret: domain
tag: ${CI_COMMIT_TAG##v}
repo: *integration_image
build_args:
- GOPROXY=https://proxy.golang.org
password:
from_secret: releaseteamtoken
username:
from_secret: releaseteamuser
build-rootless:
image: *buildx_image
group: integration
pull: true
settings:
platforms: linux/amd64,linux/arm64
dockerfile: *dockerfile_rootless
registry:
from_secret: domain
tag: ${CI_COMMIT_TAG##v}-rootless
repo: *integration_image
build_args:
- GOPROXY=https://proxy.golang.org
password:
from_secret: releaseteamtoken
username:
from_secret: releaseteamuser

View file

@ -0,0 +1,34 @@
platform: linux/amd64
when:
event: push
variables:
- &dind_image 'docker:20.10-dind'
- &alpine_image 'alpine:3.17'
pipeline:
container-images-pull-verify-push:
image: *dind_image
group: integration
commands:
# arm64 would require qemu-user-static which is not available on alpline
# the test coverage does not change much and running the tests test locally
# is possible if there is a doubt
- ARCHS=amd64 ./releases/container-images-pull-verify-push-test.sh test_run
- ./releases/container-images-pull-verify-push-test.sh test_teardown
secrets:
- releaseteamuser
- releaseteamtoken
- domain
binaries-pull-push:
image: *alpine_image
group: integration
commands:
- ./releases/binaries-pull-push-test.sh test_run
- ./releases/binaries-pull-push-test.sh test_teardown
secrets:
- releaseteamuser
- releaseteamtoken
- domain

View file

@ -0,0 +1,36 @@
platform: linux/amd64
when:
event: tag
variables:
- &dind_image 'docker:20.10-dind'
- &gpg_sign_image 'plugins/gpgsign:1'
pipeline:
pull:
image: *dind_image
commands:
- ./releases/binaries-pull-push.sh pull
gpg-sign:
image: *gpg_sign_image
pull: true
settings:
detach_sign: true
excludes:
- "dist/release/*.sha256"
files:
- "dist/release/*"
key:
from_secret: releaseteamgpg
push:
image: *dind_image
commands:
- ./releases/binaries-pull-push.sh push
secrets:
- releaseteamtoken
- releaseteamuser
- domain

View file

@ -0,0 +1,27 @@
platform: linux/amd64
when:
event: tag
variables:
- &dind_image 'docker:20.10-dind'
- &integration_image 'codeberg.org/forgejo-integration/forgejo'
- &verify 'true'
# for testing purposes
# - &verify 'false'
- &archs 'amd64 arm64'
pipeline:
publish:
image: *dind_image
environment:
INTEGRATION_IMAGE: *integration_image
VERIFY: *verify
ARCHS: *archs
commands:
- ./releases/container-images-pull-verify-push.sh
secrets:
- releaseteamtoken
- releaseteamuser
- domain