diff --git a/.gitignore b/.gitignore index 2760a0ad58..fe8184f0dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Emacs +*~ + # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a diff --git a/Makefile b/Makefile index 3bcc893828..d69c2de3f7 100644 --- a/Makefile +++ b/Makefile @@ -415,8 +415,7 @@ lint-go-windows: .PHONY: lint-go-vet lint-go-vet: @echo "Running go vet..." - @GOOS= GOARCH= $(GO) build code.gitea.io/gitea-vet - @$(GO) vet -vettool=gitea-vet $(GO_PACKAGES) + @$(GO) vet $(GO_PACKAGES) .PHONY: lint-editorconfig lint-editorconfig: @@ -829,7 +828,7 @@ $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ) CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@ .PHONY: release -release: frontend generate release-windows release-linux release-darwin release-freebsd release-copy release-compress vendor release-sources release-docs release-check +release: frontend generate release-linux release-copy release-compress vendor release-sources release-check $(DIST_DIRS): mkdir -p $(DIST_DIRS) diff --git a/releases/Dockerfile b/releases/Dockerfile new file mode 100644 index 0000000000..bef4e4f6de --- /dev/null +++ b/releases/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.17 + +RUN echo root > state diff --git a/releases/Dockerfile-rootless b/releases/Dockerfile-rootless new file mode 100644 index 0000000000..561b67e9a8 --- /dev/null +++ b/releases/Dockerfile-rootless @@ -0,0 +1,3 @@ +FROM alpine:3.17 + +RUN echo rootless > state diff --git a/releases/binaries-pull-push-test.sh b/releases/binaries-pull-push-test.sh new file mode 100755 index 0000000000..d7299c8123 --- /dev/null +++ b/releases/binaries-pull-push-test.sh @@ -0,0 +1,69 @@ +#!/bin/sh + +set -ex + +test_teardown() { + setup_api + api DELETE repos/$PUSH_USER/forgejo/releases/tags/$TAG || true + api DELETE repos/$PUSH_USER/forgejo/tags/$TAG || true + rm -fr dist/release + setup_tea + $BIN_DIR/tea login delete $RELEASETEAMUSER || true +} + +test_setup() { + mkdir -p $RELEASE_DIR + touch $RELEASE_DIR/file-one.txt + touch $RELEASE_DIR/file-two.txt +} + +test_ensure_tag() { + api DELETE repos/$PUSH_USER/forgejo/tags/$TAG || true + # + # idempotent + # + ensure_tag + api GET repos/$PUSH_USER/forgejo/tags/$TAG > /tmp/tag1.json + ensure_tag + api GET repos/$PUSH_USER/forgejo/tags/$TAG > /tmp/tag2.json + diff -u /tmp/tag[12].json + # + # sanity check on the SHA of an existing tag + # + ( + CI_COMMIT_SHA=12345 + ! ensure_tag + ) + api DELETE repos/$PUSH_USER/forgejo/tags/$TAG +} + +# +# Running the test locally instead of within Woodpecker +# +# 1. Setup: obtain a token at https://codeberg.org/user/settings/applications +# 2. Run: RELEASETEAMUSER= RELEASETEAMTOKEn= binaries-pull-push-test.sh test_run +# 3. Verify: (optional) manual verification at https://codeberg.org//forgejo/releases +# 4. Cleanup: RELEASETEAMUSER= RELEASETEAMTOKEn= binaries-pull-push-test.sh test_teardown +# +test_run() { + test_teardown + to_push=/tmp/binaries-releases-to-push + pulled=/tmp/binaries-releases-pulled + RELEASE_DIR=$to_push + test_setup + test_ensure_tag + echo "================================ TEST BEGIN" + push + RELEASE_DIR=$pulled + pull + diff -r $to_push $pulled + echo "================================ TEST END" +} + +: ${CI_REPO_OWNER:=dachary} +: ${PULL_USER=$CI_REPO_OWNER} +: ${PUSH_USER=$CI_REPO_OWNER} +: ${CI_COMMIT_TAG:=W17.8.20-1} +: ${CI_COMMIT_SHA:=$(git rev-parse HEAD)} + +. $(dirname $0)/binaries-pull-push.sh diff --git a/releases/binaries-pull-push.sh b/releases/binaries-pull-push.sh new file mode 100755 index 0000000000..1ffb5d8567 --- /dev/null +++ b/releases/binaries-pull-push.sh @@ -0,0 +1,88 @@ +#!/bin/sh + +set -ex + +: ${PULL_USER:=forgejo-integration} +if test "$CI_REPO" = "forgejo/release" ; then + : ${PUSH_USER:=forgejo} +else + : ${PUSH_USER:=forgejo-experimental} +fi +: ${TAG:=${CI_COMMIT_TAG}} +: ${DOMAIN:=codeberg.org} +: ${RELEASE_DIR:=dist/release} +: ${BIN_DIR:=/tmp} +: ${TEA_VERSION:=0.9.0} + + +setup_tea() { + if ! test -f $BIN_DIR/tea ; then + curl -sL https://dl.gitea.io/tea/$TEA_VERSION/tea-$TEA_VERSION-linux-amd64 > $BIN_DIR/tea + chmod +x $BIN_DIR/tea + fi +} + +ensure_tag() { + if api GET repos/$PUSH_USER/forgejo/tags/$TAG > /tmp/tag.json ; then + local sha=$(jq --raw-output .commit.sha < /tmp/tag.json) + if test "$sha" != "$CI_COMMIT_SHA" ; then + cat /tmp/tag.json + echo "the tag SHA in the $PUSH_USER repository does not match the tag SHA that triggered the build: $CI_COMMIT_SHA" + false + fi + else + api POST repos/$PUSH_USER/forgejo/tags --data-raw '{"tag_name": "'$CI_COMMIT_TAG'", "target": "'$CI_COMMIT_SHA'"}' + fi +} + +upload() { + ASSETS=$(ls $RELEASE_DIR/* | sed -e 's/^/-a /') + echo "${CI_COMMIT_TAG}" | grep -qi '\-rc' && export RELEASETYPE="--prerelease" && echo "Uploading as Pre-Release" + echo "${CI_COMMIT_TAG}" | grep -qi '\-test' && export RELEASETYPE="--draft" && echo "Uploading as Draft" + test ${RELEASETYPE+false} || echo "Uploading as Stable" + ensure_tag + anchor=$(echo $CI_COMMIT_TAG | sed -e 's/^v//' -e 's/[^a-zA-Z0-9]/-/g') + $BIN_DIR/tea release create $ASSETS --repo $PUSH_USER/forgejo --note "See https://codeberg.org/forgejo/forgejo/src/branch/forgejo/RELEASE-NOTES.md#${anchor}" --tag $CI_COMMIT_TAG --title $CI_COMMIT_TAG ${RELEASETYPE} +} + +push() { + setup_api + setup_tea + GITEA_SERVER_TOKEN=$RELEASETEAMTOKEN $BIN_DIR/tea login add --name $RELEASETEAMUSER --url $DOMAIN + upload +} + +setup_api() { + if ! which jq || ! which curl ; then + apk --update --no-cache add jq curl + fi +} + +api() { + method=$1 + shift + path=$1 + shift + + curl --fail -X $method -sS -H "Content-Type: application/json" -H "Authorization: token $RELEASETEAMTOKEN" "$@" https://$DOMAIN/api/v1/$path +} + +pull() { + setup_api + ( + mkdir -p $RELEASE_DIR + cd $RELEASE_DIR + api GET repos/$PULL_USER/forgejo/releases/tags/$TAG > /tmp/assets.json + jq --raw-output '.assets[] | "\(.name) \(.browser_download_url)"' < /tmp/assets.json | while read name url ; do + wget --quiet -O $name $url + done + ) +} + + +missing() { + echo need pull or push argument got nothing + exit 1 +} + +${@:-missing} diff --git a/releases/container-images-pull-verify-push-test.sh b/releases/container-images-pull-verify-push-test.sh new file mode 100755 index 0000000000..96e60b33f9 --- /dev/null +++ b/releases/container-images-pull-verify-push-test.sh @@ -0,0 +1,70 @@ +#!/bin/sh + +set -ex + +image_delete() { + curl -sS -H @$TOKEN_HEADER -X DELETE https://$DOMAIN/v2/$1/forgejo/manifests/$2 +} + +# +# Create the same set of images that buildx would +# +test_setup() { + dir=$(dirname $0) + + for suffix in '' '-rootless' ; do + ( + cd $dir + manifests="" + for arch in $ARCHS ; do + image=$(arch_image_name $PULL_USER $arch $suffix) + docker build -f Dockerfile$suffix --platform linux/$arch -t $image . + docker push $image + images="$images $image" + done + manifest=$(image_name $PULL_USER $suffix) + docker manifest rm $manifest || true + docker manifest create $manifest $images + image_put $PULL_USER $(image_tag $suffix) $manifest + ) + done +} + +test_teardown() { + authenticate + for suffix in '' '-rootless' ; do + image_delete $PULL_USER $(image_tag $suffix) + image_delete $PUSH_USER $(image_tag $suffix) + image_delete $PUSH_USER $(short_image_tag $suffix) + for arch in $ARCHS ; do + image_delete $PULL_USER $(arch_image_tag $arch $suffix) + image_delete $PUSH_USER $(arch_image_tag $arch $suffix) + done + done +} + +# +# Running the test locally instead of within Woodpecker +# +# 1. Setup: obtain a token at https://codeberg.org/user/settings/applications +# 2. Run: RELEASETEAMUSER= RELEASETEAMTOKEn= container-images-pull-verify-push-test.sh test_run +# 3. Verify: (optional) manual verification at https://codeberg.org//-/packages/container/forgejo/versions +# 4. Cleanup: RELEASETEAMUSER= RELEASETEAMTOKEn= container-images-pull-verify-push-test.sh test_teardown +# +test_run() { + boot + test_teardown + test_setup + VERIFY_STRING=something + VERIFY_COMMAND="echo $VERIFY_STRING" + echo "================================ TEST BEGIN" + main + echo "================================ TEST END" +} + +: ${CI_REPO_OWNER:=dachary} +: ${PULL_USER:=$CI_REPO_OWNER} +: ${PUSH_USER:=$CI_REPO_OWNER} +: ${CI_COMMIT_TAG:=v17.1.42-2} + +. $(dirname $0)/container-images-pull-verify-push.sh diff --git a/releases/container-images-pull-verify-push.sh b/releases/container-images-pull-verify-push.sh new file mode 100755 index 0000000000..9b2ccc203c --- /dev/null +++ b/releases/container-images-pull-verify-push.sh @@ -0,0 +1,122 @@ +#!/bin/sh + +set -ex + +: ${DOCKER_HOST:=unix:///var/run/docker.sock} +: ${ARCHS:=amd64 arm64} +: ${PULL_USER:=forgejo-integration} +if test "$CI_REPO" = "forgejo/release" ; then + : ${PUSH_USER:=forgejo} +else + : ${PUSH_USER:=forgejo-experimental} +fi +: ${INTEGRATION_IMAGE:=codeberg.org/$PULL_USER/forgejo} +: ${TAG:=${CI_COMMIT_TAG##v}} +: ${SHORT_TAG=${TAG%.*-*}} +: ${DOMAIN:=codeberg.org} +: ${TOKEN_HEADER:=/tmp/token$$} +trap "rm -f ${TOKEN_HEADER}" EXIT + +: ${VERIFY:=true} +VERIFY_COMMAND='gitea --version' +VERIFY_STRING='built with' + +publish() { + for suffix in '' '-rootless' ; do + images="" + for arch in $ARCHS ; do + # + # Get the image from the integration user + # + image=$(image_name $PULL_USER $suffix) + docker pull --platform linux/$arch $image + # + # Verify it is usable + # + if $VERIFY ; then + docker run --platform linux/$arch --rm $image $VERIFY_COMMAND | grep "$VERIFY_STRING" + fi + # + # Push the image with a tag reflecting the architecture to the repo owner + # + arch_image=$(arch_image_name $PUSH_USER $arch $suffix) + docker tag $image $arch_image + docker push $arch_image + images="$images $arch_image" + done + + # + # Push a manifest with all the architectures to the repo owner + # + manifest=$(image_name $PUSH_USER $suffix) + docker manifest rm $manifest || true + docker manifest create $manifest $images + image_put $PUSH_USER $(image_tag $suffix) $manifest + image_put $PUSH_USER $(short_image_tag $suffix) $manifest + # + # Sanity check to ensure the manifest that are published can actualy + # be used. + # + for arch in $ARCHS ; do + docker pull --platform linux/$arch $(image_name $PUSH_USER $suffix) + docker pull --platform linux/$arch $(short_image_name $PUSH_USER $suffix) + done + done +} + +boot() { + if docker version ; then + return + fi + apk --update --no-cache add coredns jq curl + ( echo ".:53 {" ; echo " forward . /etc/resolv.conf"; echo "}" ) > /etc/coredns/Corefile + coredns -conf /etc/coredns/Corefile & + /usr/local/bin/dockerd --data-root /var/lib/docker --host=$DOCKER_HOST --dns 172.17.0.3 & + for i in $(seq 60) ; do + docker version && break + sleep 1 + done + docker version || exit 1 +} + +authenticate() { + echo "$RELEASETEAMTOKEN" | docker login --password-stdin --username "$RELEASETEAMUSER" $DOMAIN + curl -u$RELEASETEAMUSER:$RELEASETEAMTOKEN -sS https://$DOMAIN/v2/token | jq --raw-output '"Authorization: token \(.token)"' > $TOKEN_HEADER +} + +image_put() { + docker manifest inspect $3 > /tmp/manifest.json + curl -sS -H @$TOKEN_HEADER -X PUT --data-binary @/tmp/manifest.json https://$DOMAIN/v2/$1/forgejo/manifests/$2 +} + +main() { + boot + authenticate + publish +} + +image_name() { + echo $DOMAIN/$1/forgejo:$(image_tag $2) +} + +image_tag() { + echo $TAG$1 +} + +short_image_name() { + echo $DOMAIN/$1/forgejo:$(short_image_tag $2) +} + +short_image_tag() { + echo $SHORT_TAG$1 +} + +arch_image_name() { + echo $DOMAIN/$1/forgejo:$(arch_image_tag $2 $3) +} + +arch_image_tag() { + echo $TAG-$1$2 +} + +${@:-main} diff --git a/releases/woodpecker-build/binaries.yml b/releases/woodpecker-build/binaries.yml new file mode 100644 index 0000000000..ca84e6b27b --- /dev/null +++ b/releases/woodpecker-build/binaries.yml @@ -0,0 +1,107 @@ +platform: linux/amd64 + +when: + event: tag + tag: v* + +variables: + - &node_image 'node:18' + - &golang_image 'golang:1.20' + - &alpine_image 'alpine:3.18' + - &gpg_sign_image 'plugins/gpgsign:1' + - &xgo_image 'techknowlogick/xgo:go-1.20.x' + - &gpg_sign_image 'plugins/gpgsign:1' + - &goproxy_override '' + - &goproxy_setup |- + if [ -n "$${GOPROXY_OVERRIDE:-}" ]; then + export GOPROXY="$${GOPROXY_OVERRIDE}"; + echo "Using goproxy from goproxy_override \"$${GOPROXY}\""; + elif [ -n "$${GOPROXY_DEFAULT:-}" ]; then + export GOPROXY="$${GOPROXY_DEFAULT}"; + echo "Using goproxy from goproxy_default (secret) not displaying"; + else + export GOPROXY="https://proxy.golang.org,direct"; + echo "No goproxy overrides or defaults given, using \"$${GOPROXY}\""; + fi + +workspace: + base: /source + path: / + +pipeline: + fetch-tags: + image: *golang_image + pull: true + group: deps + commands: + - git config --add safe.directory '*' + - git fetch --tags --force + + deps-frontend: + image: *node_image + pull: true + group: deps + commands: + - make deps-frontend + + deps-backend: + image: *golang_image + pull: true + group: deps + environment: + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - goproxy_default + commands: + - *goproxy_setup + - make deps-backend + + static: + image: *xgo_image + pull: true + commands: + - *goproxy_setup + - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get -qqy install nodejs + - export PATH=$PATH:$GOPATH/bin + - make CI=true LINUX_ARCHS=linux/amd64,linux/arm64,linux/arm-6 release + environment: + TAGS: 'bindata sqlite sqlite_unlock_notify' + DEBIAN_FRONTEND: 'noninteractive' + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - goproxy_default + + # + # See https://codeberg.org/forgejo/forgejo/issues/230 for a discussion on this + # compilation stage. The goal is just to verify the build does not break, not that + # the binary produced actually works. + # + freebsd: + image: *xgo_image + group: build + commands: + - *goproxy_setup + - export PATH=$PATH:$GOPATH/bin + - make CI=false release-freebsd + environment: + TAGS: 'bindata sqlite sqlite_unlock_notify' + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - goproxy_default + + verifyruns: + image: *golang_image + commands: + - ./dist/release/forgejo-*-amd64 --version | grep 'built with' + - apt-get update + - apt-get install -y qemu-user-static + - /usr/bin/qemu-aarch64-static ./dist/release/forgejo-*-arm64 --version | grep 'built with' + - /usr/bin/qemu-arm-static ./dist/release/forgejo-*-arm-6 --version | grep 'built with' + + push-integration: + image: *alpine_image + commands: + - PUSH_USER=$CI_REPO_OWNER releases/binaries-pull-push.sh push + secrets: + - releaseteamtoken + - releaseteamuser diff --git a/releases/woodpecker-build/container-images.yml b/releases/woodpecker-build/container-images.yml new file mode 100644 index 0000000000..ebcd7cdc1c --- /dev/null +++ b/releases/woodpecker-build/container-images.yml @@ -0,0 +1,65 @@ +platform: linux/amd64 + +when: + event: tag + tag: v* + +variables: + - &golang_image 'golang:1.20' + - &dind_image 'docker:20.10-dind' + - &buildx_image 'woodpeckerci/plugin-docker-buildx:2.0.0' + - &integration_image 'codeberg.org/forgejo-integration/forgejo' + - &dockerfile_root 'Dockerfile' +# for testing purposes +# - &dockerfile_root 'releases/Dockerfile' + - &dockerfile_rootless 'Dockerfile.rootless' +# for testing purposes +# - &dockerfile_rootless 'releases/Dockerfile-rootless' + - &verify 'true' +# for testing purposes +# - &verify 'false' + - &archs 'amd64 arm64' + +pipeline: + fetch-tags: + image: *golang_image + pull: true + commands: + - git config --add safe.directory '*' + - git fetch --tags --force + + build-root: + image: *buildx_image + group: integration + pull: true + settings: + platforms: linux/amd64,linux/arm64 + dockerfile: *dockerfile_root + registry: + from_secret: domain + tag: ${CI_COMMIT_TAG##v} + repo: *integration_image + build_args: + - GOPROXY=https://proxy.golang.org + password: + from_secret: releaseteamtoken + username: + from_secret: releaseteamuser + + build-rootless: + image: *buildx_image + group: integration + pull: true + settings: + platforms: linux/amd64,linux/arm64 + dockerfile: *dockerfile_rootless + registry: + from_secret: domain + tag: ${CI_COMMIT_TAG##v}-rootless + repo: *integration_image + build_args: + - GOPROXY=https://proxy.golang.org + password: + from_secret: releaseteamtoken + username: + from_secret: releaseteamuser diff --git a/releases/woodpecker-build/releases-helper.yml b/releases/woodpecker-build/releases-helper.yml new file mode 100644 index 0000000000..e700a0f23c --- /dev/null +++ b/releases/woodpecker-build/releases-helper.yml @@ -0,0 +1,34 @@ +platform: linux/amd64 + +when: + event: push + +variables: + - &dind_image 'docker:20.10-dind' + - &alpine_image 'alpine:3.17' + +pipeline: + container-images-pull-verify-push: + image: *dind_image + group: integration + commands: +# arm64 would require qemu-user-static which is not available on alpline +# the test coverage does not change much and running the tests test locally +# is possible if there is a doubt + - ARCHS=amd64 ./releases/container-images-pull-verify-push-test.sh test_run + - ./releases/container-images-pull-verify-push-test.sh test_teardown + secrets: + - releaseteamuser + - releaseteamtoken + - domain + + binaries-pull-push: + image: *alpine_image + group: integration + commands: + - ./releases/binaries-pull-push-test.sh test_run + - ./releases/binaries-pull-push-test.sh test_teardown + secrets: + - releaseteamuser + - releaseteamtoken + - domain diff --git a/releases/woodpecker-publish/binaries.yml b/releases/woodpecker-publish/binaries.yml new file mode 100644 index 0000000000..c269903b85 --- /dev/null +++ b/releases/woodpecker-publish/binaries.yml @@ -0,0 +1,36 @@ +platform: linux/amd64 + +when: + event: tag + +variables: + - &dind_image 'docker:20.10-dind' + - &gpg_sign_image 'plugins/gpgsign:1' + +pipeline: + + pull: + image: *dind_image + commands: + - ./releases/binaries-pull-push.sh pull + + gpg-sign: + image: *gpg_sign_image + pull: true + settings: + detach_sign: true + excludes: + - "dist/release/*.sha256" + files: + - "dist/release/*" + key: + from_secret: releaseteamgpg + + push: + image: *dind_image + commands: + - ./releases/binaries-pull-push.sh push + secrets: + - releaseteamtoken + - releaseteamuser + - domain diff --git a/releases/woodpecker-publish/container-images.yml b/releases/woodpecker-publish/container-images.yml new file mode 100644 index 0000000000..77b1ac2933 --- /dev/null +++ b/releases/woodpecker-publish/container-images.yml @@ -0,0 +1,27 @@ +platform: linux/amd64 + +when: + event: tag + +variables: + - &dind_image 'docker:20.10-dind' + - &integration_image 'codeberg.org/forgejo-integration/forgejo' + - &verify 'true' +# for testing purposes +# - &verify 'false' + - &archs 'amd64 arm64' + +pipeline: + + publish: + image: *dind_image + environment: + INTEGRATION_IMAGE: *integration_image + VERIFY: *verify + ARCHS: *archs + commands: + - ./releases/container-images-pull-verify-push.sh + secrets: + - releaseteamtoken + - releaseteamuser + - domain