Reduce CI go module downloads, add make targets (#18708, #18475, #18443) (#18741)

Backport #18708 
Backport #18475 
Backport #18443 

The CI currently downloads all go modules in each pipeline step because go modules reside outside the project directory. Fix this by introducing a volume for the `/go` directory [1] so modules are only downloaded once per pipeline using a new `deps-backend` make target.

For completeness, I also included new `deps` and `deps-frontend` targets and the frontend one is also triggered explicitly on CI where needed.

[1] https://docs.drone.io/pipeline/kubernetes/examples/language/golang/#dependencies

* Also backports #18475 and #18443 so that is was able to merge cleanly.
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
silverwind 2022-02-12 11:43:08 +01:00 committed by GitHub
parent 65ad6362d7
commit c04a4afac1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 215 additions and 61 deletions

View file

@ -13,12 +13,25 @@ trigger:
- tag
- pull_request
volumes:
- name: deps
temp: {}
steps:
- name: deps-frontend
pull: always
image: node:16
pull: always
commands:
- make node_modules
- make deps-frontend
- name: deps-backend
image: golang:1.17
pull: always
commands:
- make deps-backend
volumes:
- name: deps
path: /go
- name: lint-frontend
image: node:16
@ -27,17 +40,17 @@ steps:
depends_on: [deps-frontend]
- name: lint-backend
pull: always
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
pull: always
commands:
- make lint-backend
environment:
GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata sqlite sqlite_unlock_notify
depends_on: [deps-backend]
- name: lint-backend-windows
pull: always
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
commands:
- make golangci-lint vet
@ -47,9 +60,9 @@ steps:
TAGS: bindata sqlite sqlite_unlock_notify
GOOS: windows
GOARCH: amd64
depends_on: [deps-backend]
- name: lint-backend-gogit
pull: always
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
commands:
- make lint-backend
@ -57,6 +70,7 @@ steps:
GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata gogit sqlite sqlite_unlock_notify
depends_on: [deps-backend]
- name: checks-frontend
image: node:16
@ -65,11 +79,13 @@ steps:
depends_on: [deps-frontend]
- name: checks-backend
pull: always
image: golang:1.17
commands:
- make checks-backend
depends_on: [lint-backend]
depends_on: [deps-backend]
volumes:
- name: deps
path: /go
- name: test-frontend
image: node:16
@ -84,14 +100,17 @@ steps:
depends_on: [test-frontend]
- name: build-backend-no-gcc
pull: always
image: golang:1.16 # this step is kept as the lowest version of golang that we support
pull: always
environment:
GO111MODULE: on
GOPROXY: https://goproxy.cn
commands:
- go build -o gitea_no_gcc # test if build succeeds without the sqlite tag
depends_on: [checks-backend]
depends_on: [deps-backend, checks-backend]
volumes:
- name: deps
path: /go
- name: build-backend-arm64
image: golang:1.17
@ -104,7 +123,10 @@ steps:
commands:
- make backend # test cross compile
- rm ./gitea # clean
depends_on: [checks-backend]
depends_on: [deps-backend, checks-backend]
volumes:
- name: deps
path: /go
- name: build-backend-windows
image: golang:1.17
@ -116,7 +138,10 @@ steps:
TAGS: bindata gogit
commands:
- go build -o gitea_windows
depends_on: [checks-backend]
depends_on: [deps-backend, checks-backend]
volumes:
- name: deps
path: /go
- name: build-backend-386
image: golang:1.17
@ -127,7 +152,10 @@ steps:
GOARCH: 386
commands:
- go build -o gitea_linux_386 # test if compatible with 32 bit
depends_on: [checks-backend]
depends_on: [deps-backend, checks-backend]
volumes:
- name: deps
path: /go
---
kind: pipeline
@ -147,21 +175,28 @@ trigger:
- tag
- pull_request
volumes:
- name: deps
temp: {}
services:
- name: mysql
image: mysql:5.7
pull: always
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
MYSQL_DATABASE: test
- name: mysql8
image: mysql:8
pull: always
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
MYSQL_DATABASE: testgitea
- name: mssql
image: mcr.microsoft.com/mssql/server:latest
pull: always
environment:
ACCEPT_EULA: Y
MSSQL_PID: Standard
@ -169,14 +204,17 @@ services:
- name: ldap
image: gitea/test-openldap:latest
pull: always
- name: elasticsearch
image: elasticsearch:7.5.0
pull: always
environment:
discovery.type: single-node
image: elasticsearch:7.5.0
- name: minio
image: minio/minio:RELEASE.2021-03-12T00-00-47Z
pull: always
commands:
- minio server /data
environment:
@ -186,6 +224,7 @@ services:
steps:
- name: fetch-tags
image: docker:git
pull: always
commands:
- git fetch --tags --force
when:
@ -193,19 +232,28 @@ steps:
exclude:
- pull_request
- name: tag-pre-condition
- name: deps-backend
image: golang:1.17
pull: always
commands:
- make deps-backend
volumes:
- name: deps
path: /go
- name: tag-pre-condition
image: drone/git
pull: always
commands:
- git update-ref refs/heads/tag_test ${DRONE_COMMIT_SHA}
- name: prepare-test-env
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
pull: always
commands:
- ./build/test-env-prepare.sh
- name: build
pull: always
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
@ -215,8 +263,10 @@ steps:
GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata sqlite sqlite_unlock_notify
depends_on:
- prepare-test-env
depends_on: [deps-backend, prepare-test-env]
volumes:
- name: deps
path: /go
- name: unit-test
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
@ -229,9 +279,12 @@ steps:
RACE_ENABLED: true
GITHUB_READ_TOKEN:
from_secret: github_read_token
depends_on: [deps-backend, prepare-test-env]
volumes:
- name: deps
path: /go
- name: unit-test-gogit
pull: always
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
@ -242,6 +295,10 @@ steps:
RACE_ENABLED: true
GITHUB_READ_TOKEN:
from_secret: github_read_token
depends_on: [deps-backend, prepare-test-env]
volumes:
- name: deps
path: /go
- name: test-mysql
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
@ -255,8 +312,10 @@ steps:
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200"
depends_on:
- build
depends_on: [build]
volumes:
- name: deps
path: /go
- name: test-mysql8
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
@ -269,8 +328,10 @@ steps:
RACE_ENABLED: true
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
depends_on:
- build
depends_on: [build]
volumes:
- name: deps
path: /go
- name: test-mssql
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
@ -283,8 +344,10 @@ steps:
RACE_ENABLED: true
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
depends_on:
- build
depends_on: [build]
volumes:
- name: deps
path: /go
- name: generate-coverage
image: golang:1.17
@ -293,9 +356,7 @@ steps:
environment:
GOPROXY: https://goproxy.cn
TAGS: bindata
depends_on:
- unit-test
- test-mysql
depends_on: [unit-test, test-mysql]
when:
branch:
- main
@ -304,15 +365,14 @@ steps:
- pull_request
- name: coverage-codecov
image: woodpeckerci/plugin-codecov:next-alpine
pull: always
image: plugins/codecov
settings:
files:
- coverage.all
token:
from_secret: codecov_token
depends_on:
- generate-coverage
depends_on: [generate-coverage]
when:
branch:
- main
@ -337,6 +397,10 @@ trigger:
- tag
- pull_request
volumes:
- name: deps
temp: {}
services:
- name: pgsql
pull: default
@ -352,6 +416,7 @@ services:
steps:
- name: fetch-tags
image: docker:git
pull: always
commands:
- git fetch --tags --force
when:
@ -359,13 +424,22 @@ steps:
exclude:
- pull_request
- name: deps-backend
image: golang:1.17
pull: always
commands:
- make deps-backend
volumes:
- name: deps
path: /go
- name: prepare-test-env
image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env
pull: always
commands:
- ./build/test-env-prepare.sh
- name: build
pull: always
image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env
user: gitea
commands:
@ -375,8 +449,10 @@ steps:
GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata gogit sqlite sqlite_unlock_notify
depends_on:
- prepare-test-env
depends_on: [deps-backend, prepare-test-env]
volumes:
- name: deps
path: /go
- name: test-sqlite
image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env
@ -389,8 +465,10 @@ steps:
RACE_ENABLED: true
TEST_TAGS: gogit sqlite sqlite_unlock_notify
USE_REPO_TEST_DIR: 1
depends_on:
- build
depends_on: [build]
volumes:
- name: deps
path: /go
- name: test-pgsql
image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env
@ -404,8 +482,10 @@ steps:
TEST_TAGS: gogit
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
depends_on:
- build
depends_on: [build]
volumes:
- name: deps
path: /go
---
kind: pipeline
@ -425,8 +505,8 @@ trigger:
steps:
- name: download
pull: always
image: jonasfranz/crowdin
pull: always
settings:
download: true
export_dir: options/locale/
@ -437,14 +517,14 @@ steps:
from_secret: crowdin_key
- name: update
pull: default
image: alpine:3.13
pull: always
commands:
- ./build/update-locales.sh
- name: push
pull: always
image: appleboy/drone-git-push
pull: always
settings:
author_email: "teabot@gitea.io"
author_name: GiteaBot
@ -457,8 +537,8 @@ steps:
from_secret: git_push_ssh_key
- name: upload_translations
pull: always
image: jonasfranz/crowdin
pull: always
settings:
files:
locale_en-US.ini: options/locale/locale_en-US.ini
@ -488,12 +568,13 @@ trigger:
steps:
- name: download
image: golang:1.17
pull: always
commands:
- timeout -s ABRT 40m make generate-license generate-gitignore
- name: push
pull: always
image: appleboy/drone-git-push
pull: always
settings:
author_email: "teabot@gitea.io"
author_name: GiteaBot
@ -529,15 +610,35 @@ depends_on:
- testing-amd64
- testing-arm64
volumes:
- name: deps
temp: {}
steps:
- name: fetch-tags
image: docker:git
pull: always
commands:
- git fetch --tags --force
- name: static
- name: deps-frontend
image: node:16
pull: always
commands:
- make deps-frontend
- name: deps-backend
image: golang:1.17
pull: always
commands:
- make deps-backend
volumes:
- name: deps
path: /go
- name: static
image: techknowlogick/xgo:go-1.17.x
pull: always
commands:
- curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get install -y nodejs
- export PATH=$PATH:$GOPATH/bin
@ -545,10 +646,13 @@ steps:
environment:
GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
TAGS: bindata sqlite sqlite_unlock_notify
volumes:
- name: deps
path: /go
- name: gpg-sign
pull: always
image: plugins/gpgsign:1
pull: always
settings:
detach_sign: true
excludes:
@ -562,8 +666,8 @@ steps:
from_secret: gpgsign_passphrase
- name: release-branch
pull: always
image: woodpeckerci/plugin-s3:latest
pull: always
settings:
acl: public-read
bucket: gitea-artifacts
@ -624,16 +728,35 @@ depends_on:
- testing-arm64
- testing-amd64
volumes:
- name: deps
temp: {}
steps:
- name: fetch-tags
pull: default
image: docker:git
pull: always
commands:
- git fetch --tags --force
- name: static
- name: deps-frontend
image: node:16
pull: always
commands:
- make deps-frontend
- name: deps-backend
image: golang:1.17
pull: always
commands:
- make deps-backend
volumes:
- name: deps
path: /go
- name: static
image: techknowlogick/xgo:go-1.17.x
pull: always
commands:
- curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get install -y nodejs
- export PATH=$PATH:$GOPATH/bin
@ -641,10 +764,14 @@ steps:
environment:
GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
TAGS: bindata sqlite sqlite_unlock_notify
depends_on: [fetch-tags]
volumes:
- name: deps
path: /go
- name: gpg-sign
pull: always
image: plugins/gpgsign:1
pull: always
settings:
detach_sign: true
excludes:
@ -656,10 +783,11 @@ steps:
from_secret: gpgsign_key
GPGSIGN_PASSPHRASE:
from_secret: gpgsign_passphrase
depends_on: [static]
- name: release-tag
pull: always
image: woodpeckerci/plugin-s3:latest
pull: always
settings:
acl: public-read
bucket: gitea-artifacts
@ -673,16 +801,18 @@ steps:
from_secret: aws_access_key_id
AWS_SECRET_ACCESS_KEY:
from_secret: aws_secret_access_key
depends_on: [gpg-sign]
- name: github
pull: always
image: plugins/github-release:1
pull: always
settings:
files:
- "dist/release/*"
environment:
GITHUB_TOKEN:
from_secret: github_token
depends_on: [gpg-sign]
---
kind: pipeline
@ -704,16 +834,16 @@ trigger:
steps:
- name: build-docs
pull: always
image: plugins/hugo:latest
pull: always
commands:
- apk add --no-cache make bash curl
- cd docs
- make trans-copy clean build
- name: publish-docs
pull: always
image: techknowlogick/drone-netlify:latest
pull: always
settings:
path: docs/public/
site_id: d2260bae-7861-4c02-8646-8f6440b12672
@ -749,12 +879,13 @@ trigger:
steps:
- name: fetch-tags
image: docker:git
pull: always
commands:
- git fetch --tags --force
- name: publish
pull: always
image: techknowlogick/drone-docker:latest
pull: always
settings:
auto_tag: true
auto_tag_suffix: linux-amd64
@ -811,12 +942,13 @@ trigger:
steps:
- name: fetch-tags
image: docker:git
pull: always
commands:
- git fetch --tags --force
- name: publish
pull: always
image: techknowlogick/drone-docker:latest
pull: always
settings:
auto_tag: false
tags: dev-linux-amd64
@ -872,12 +1004,13 @@ trigger:
steps:
- name: fetch-tags
image: docker:git
pull: always
commands:
- git fetch --tags --force
- name: publish
pull: always
image: techknowlogick/drone-docker:latest
pull: always
settings:
auto_tag: false
tags: ${DRONE_BRANCH##release/v}-dev-linux-amd64
@ -929,8 +1062,8 @@ trigger:
steps:
- name: dryrun
pull: always
image: techknowlogick/drone-docker:latest
pull: always
settings:
dry_run: true
repo: gitea/gitea
@ -967,12 +1100,13 @@ trigger:
steps:
- name: fetch-tags
image: docker:git
pull: always
commands:
- git fetch --tags --force
- name: publish
pull: always
image: techknowlogick/drone-docker:latest
pull: always
settings:
auto_tag: true
auto_tag_suffix: linux-arm64
@ -1029,12 +1163,13 @@ trigger:
steps:
- name: fetch-tags
image: docker:git
pull: always
commands:
- git fetch --tags --force
- name: publish
pull: always
image: techknowlogick/drone-docker:latest
pull: always
settings:
auto_tag: false
tags: dev-linux-arm64
@ -1090,12 +1225,13 @@ trigger:
steps:
- name: fetch-tags
image: docker:git
pull: always
commands:
- git fetch --tags --force
- name: publish
pull: always
image: techknowlogick/drone-docker:latest
pull: always
settings:
auto_tag: false
tags: ${DRONE_BRANCH##release/v}-dev-linux-arm64
@ -1140,8 +1276,8 @@ platform:
steps:
- name: manifest-rootless
pull: always
image: plugins/manifest
pull: always
settings:
auto_tag: true
ignore_missing: true
@ -1186,6 +1322,7 @@ steps:
- name: manifest-rootless
pull: always
image: plugins/manifest
pull: always
settings:
auto_tag: false
ignore_missing: true
@ -1260,8 +1397,8 @@ depends_on:
steps:
- name: discord
pull: always
image: appleboy/drone-discord:1.2.4
pull: always
settings:
message: "{{#success build.status}} ✅ Build #{{build.number}} of `{{repo.name}}` succeeded.\n\n📝 Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\n🌐 {{ build.link }} {{else}} ❌ Build #{{build.number}} of `{{repo.name}}` failed.\n\n📝 Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\n🌐 {{ build.link }} {{/success}}\n"
webhook_id:

View file

@ -23,6 +23,10 @@ linters:
run:
timeout: 3m
skip-dirs:
- node_modules
- public
- web_src
linters-settings:
gocritic:

View file

@ -166,6 +166,9 @@ help:
@echo " - watch-backend watch backend files and continuously rebuild"
@echo " - clean delete backend and integration files"
@echo " - clean-all delete backend, frontend and integration files"
@echo " - deps install dependencies"
@echo " - deps-frontend install frontend dependencies"
@echo " - deps-backend install backend dependencies"
@echo " - lint lint everything"
@echo " - lint-frontend lint frontend files"
@echo " - lint-backend lint backend files"
@ -656,6 +659,16 @@ docs:
fi
cd docs; make trans-copy clean build-offline;
.PHONY: deps
deps: deps-frontend deps-backend
.PHONY: deps-frontend
deps-frontend: node_modules
.PHONY: deps-backend
deps-backend:
$(GO) mod download
node_modules: package-lock.json
npm install --no-save
@touch node_modules