gitea.nulo.in: Use Alpine 3.16

Co-authored-by: John Olheiser <john+github@jolheiser.com>
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: delvh <dev.lh@web.de>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

CHANGELOG.md

@@ -4,6 +4,50 @@ This changelog goes through all the changes that have been made in each release
 without substantial changes to our git log; to see the highlights of what has
 been added to each release, please refer to the [blog](https://blog.gitea.io).
 
+## [1.17.2](https://github.com/go-gitea/gitea/releases/tag/v1.17.2) - 2022-09-06
+
+* SECURITY
+  * Double check CloneURL is acceptable (#20869) (#20892)
+  * Add more checks in migration code (#21011) (#21050)
+* ENHANCEMENTS
+  * Fix hard-coded timeout and error panic in API archive download endpoint (#20925) (#21051)
+  * Improve arc-green code theme (#21039) (#21042)
+  * Enable contenthash in filename for dynamic assets (#20813) (#20932)
+  * Don't open new page for ext wiki on same repository (#20725) (#20910)
+  * Disable doctor logging on panic (#20847) (#20898)
+  * Remove calls to load Mirrors in user.Dashboard (#20855) (#20897)
+  * Update codemirror to 5.65.8 (#20875)
+  * Rework repo buttons (#20602, #20718) (#20719)
+* BUGFIXES
+  * Ensure delete user deletes all comments (#21067) (#21068)
+  * Delete unreferenced packages when deleting a package version (#20977) (#21060)
+  * Redirect if user does not exist on admin pages (#20981) (#21059)
+  * Set uploadpack.allowFilter etc on gitea serv to enable partial clones with ssh (#20902) (#21058)
+  * Fix 500 on time in timeline API (#21052) (#21057)
+  * Fill the specified ref in webhook test payload (#20961) (#21055)
+  * Add another index for Action table on postgres (#21033) (#21054)
+  * Fix broken insecureskipverify handling in redis connection uris (#20967) (#21053)
+  * Add Dev, Peer and Optional dependencies to npm PackageMetadataVersion (#21017) (#21044)
+  * Do not add links to Posters or Assignees with ID < 0 (#20577) (#21037)
+  * Fix modified due date message (#20388) (#21032)
+  * Fix missed sort bug (#21006)
+  * Fix input.value attr for RequiredClaimName/Value (#20946) (#21001)
+  * Change review buttons to icons to make space for text (#20934) (#20978)
+  * Fix download archiver of a commit (#20962) (#20971)
+  * Return 404 NotFound if requested attachment does not exist (#20886) (#20941)
+  * Set no-tags in git fetch on compare (#20893) (#20936)
+  * Allow multiple metadata files for Maven packages (#20674) (#20916)
+  * Increase Content field size of gpg_key and public_key to MEDIUMTEXT (#20896) (#20911)
+  * Fix mirror address setting not working (#20850) (#20904)
+  * Fix push mirror address backend get error Address cause setting page display error (#20593) (#20901)
+  * Fix panic when an invalid oauth2 name is passed (#20820) (#20900)
+  * In PushMirrorsIterate and MirrorsIterate if limit is negative do not set it (#20837) (#20899)
+  * Ensure that graceful start-up is informed of unused SSH listener (#20877) (#20888)
+  * Pad GPG Key ID with preceding zeroes (#20878) (#20885)
+  * Fix SQL Query for `SearchTeam` (#20844) (#20872)
+  * Fix the mode of custom dir to 0700 in docker-rootless (#20861) (#20867)
+  * Fix UI mis-align for PR commit history (#20845) (#20859)
+
 ## [1.17.1](https://github.com/go-gitea/gitea/releases/tag/1.17.1) - 2022-08-17
 
 * SECURITY

cmd/doctor.go

@@ -5,6 +5,7 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	golog "log"
 	"os"
@@ -123,6 +124,47 @@ func runRecreateTable(ctx *cli.Context) error {
 	})
 }
 
+func setDoctorLogger(ctx *cli.Context) {
+	logFile := ctx.String("log-file")
+	if !ctx.IsSet("log-file") {
+		logFile = "doctor.log"
+	}
+	colorize := log.CanColorStdout
+	if ctx.IsSet("color") {
+		colorize = ctx.Bool("color")
+	}
+
+	if len(logFile) == 0 {
+		log.NewLogger(1000, "doctor", "console", fmt.Sprintf(`{"level":"NONE","stacktracelevel":"NONE","colorize":%t}`, colorize))
+		return
+	}
+
+	defer func() {
+		recovered := recover()
+		if recovered == nil {
+			return
+		}
+
+		err, ok := recovered.(error)
+		if !ok {
+			panic(recovered)
+		}
+		if errors.Is(err, os.ErrPermission) {
+			fmt.Fprintf(os.Stderr, "ERROR: Unable to write logs to provided file due to permissions error: %s\n       %v\n", logFile, err)
+		} else {
+			fmt.Fprintf(os.Stderr, "ERROR: Unable to write logs to provided file: %s\n       %v\n", logFile, err)
+		}
+		fmt.Fprintf(os.Stderr, "WARN: Logging will be disabled\n       Use `--log-file` to configure log file location\n")
+		log.NewLogger(1000, "doctor", "console", fmt.Sprintf(`{"level":"NONE","stacktracelevel":"NONE","colorize":%t}`, colorize))
+	}()
+
+	if logFile == "-" {
+		log.NewLogger(1000, "doctor", "console", fmt.Sprintf(`{"level":"trace","stacktracelevel":"NONE","colorize":%t}`, colorize))
+	} else {
+		log.NewLogger(1000, "doctor", "file", fmt.Sprintf(`{"filename":%q,"level":"trace","stacktracelevel":"NONE"}`, logFile))
+	}
+}
+
 func runDoctor(ctx *cli.Context) error {
 	// Silence the default loggers
 	log.DelNamedLogger("console")
@@ -132,24 +174,13 @@ func runDoctor(ctx *cli.Context) error {
 	defer cancel()
 
 	// Now setup our own
-	logFile := ctx.String("log-file")
-	if !ctx.IsSet("log-file") {
-		logFile = "doctor.log"
-	}
+	setDoctorLogger(ctx)
 
 	colorize := log.CanColorStdout
 	if ctx.IsSet("color") {
 		colorize = ctx.Bool("color")
 	}
 
-	if len(logFile) == 0 {
-		log.NewLogger(1000, "doctor", "console", fmt.Sprintf(`{"level":"NONE","stacktracelevel":"NONE","colorize":%t}`, colorize))
-	} else if logFile == "-" {
-		log.NewLogger(1000, "doctor", "console", fmt.Sprintf(`{"level":"trace","stacktracelevel":"NONE","colorize":%t}`, colorize))
-	} else {
-		log.NewLogger(1000, "doctor", "file", fmt.Sprintf(`{"filename":%q,"level":"trace","stacktracelevel":"NONE"}`, logFile))
-	}
-
 	// Finally redirect the default golog to here
 	golog.SetFlags(0)
 	golog.SetPrefix("")

cmd/migrate_storage.go

@@ -112,11 +112,8 @@ func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) e
 
 func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
 	return db.IterateObjects(ctx, func(archiver *repo_model.RepoArchiver) error {
-		p, err := archiver.RelativePath()
-		if err != nil {
-			return err
-		}
-		_, err = storage.Copy(dstStorage, p, storage.RepoArchives, p)
+		p := archiver.RelativePath()
+		_, err := storage.Copy(dstStorage, p, storage.RepoArchives, p)
 		return err
 	})
 }

docker/rootless/usr/local/bin/docker-setup.sh

@@ -5,7 +5,7 @@ mkdir -p ${HOME} && chmod 0700 ${HOME}
 if [ ! -w ${HOME} ]; then echo "${HOME} is not writable"; exit 1; fi
 
 # Prepare custom folder
-mkdir -p ${GITEA_CUSTOM} && chmod 0500 ${GITEA_CUSTOM}
+mkdir -p ${GITEA_CUSTOM} && chmod 0700 ${GITEA_CUSTOM}
 
 # Prepare temp folder
 mkdir -p ${GITEA_TEMP} && chmod 0700 ${GITEA_TEMP}

integrations/api_packages_maven_test.go

@@ -42,6 +42,7 @@ func TestPackageMaven(t *testing.T) {
 		defer PrintCurrentTest(t)()
 
 		putFile(t, fmt.Sprintf("/%s/%s", packageVersion, filename), "test", http.StatusCreated)
+		putFile(t, fmt.Sprintf("/%s/%s", packageVersion, filename), "test", http.StatusBadRequest)
 		putFile(t, "/maven-metadata.xml", "test", http.StatusOK)
 
 		pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeMaven)
@@ -135,12 +136,14 @@ func TestPackageMaven(t *testing.T) {
 		pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
 		assert.NoError(t, err)
 		assert.Len(t, pfs, 2)
-		i := 0
-		if strings.HasSuffix(pfs[1].Name, ".pom") {
-			i = 1
+		for _, pf := range pfs {
+			if strings.HasSuffix(pf.Name, ".pom") {
+				assert.Equal(t, filename+".pom", pf.Name)
+				assert.True(t, pf.IsLead)
+			} else {
+				assert.False(t, pf.IsLead)
+			}
 		}
-		assert.Equal(t, filename+".pom", pfs[i].Name)
-		assert.True(t, pfs[i].IsLead)
 	})
 
 	t.Run("DownloadPOM", func(t *testing.T) {
@@ -202,4 +205,13 @@ func TestPackageMaven(t *testing.T) {
 			assert.Equal(t, checksum, resp.Body.String())
 		}
 	})
+
+	t.Run("UploadSnapshot", func(t *testing.T) {
+		snapshotVersion := packageVersion + "-SNAPSHOT"
+
+		putFile(t, fmt.Sprintf("/%s/%s", snapshotVersion, filename), "test", http.StatusCreated)
+		putFile(t, "/maven-metadata.xml", "test", http.StatusOK)
+		putFile(t, fmt.Sprintf("/%s/maven-metadata.xml", snapshotVersion), "test", http.StatusCreated)
+		putFile(t, fmt.Sprintf("/%s/maven-metadata.xml", snapshotVersion), "test-overwrite", http.StatusCreated)
+	})
 }

integrations/api_team_test.go

@@ -223,7 +223,7 @@ func TestAPITeamSearch(t *testing.T) {
 	defer prepareTestEnv(t)()
 
 	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}).(*user_model.User)
-	org := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}).(*user_model.User)
+	org := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 17}).(*user_model.User)
 
 	var results TeamSearchResults
 

integrations/api_user_orgs_test.go

@@ -26,8 +26,19 @@ func TestUserOrgs(t *testing.T) {
 	orgs := getUserOrgs(t, adminUsername, normalUsername)
 
 	user3 := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user3"}).(*user_model.User)
+	user17 := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user17"}).(*user_model.User)
 
 	assert.Equal(t, []*api.Organization{
+		{
+			ID:          17,
+			UserName:    user17.Name,
+			FullName:    user17.FullName,
+			AvatarURL:   user17.AvatarLink(),
+			Description: "",
+			Website:     "",
+			Location:    "",
+			Visibility:  "public",
+		},
 		{
 			ID:          3,
 			UserName:    user3.Name,
@@ -82,8 +93,19 @@ func TestMyOrgs(t *testing.T) {
 	var orgs []*api.Organization
 	DecodeJSON(t, resp, &orgs)
 	user3 := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user3"}).(*user_model.User)
+	user17 := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user17"}).(*user_model.User)
 
 	assert.Equal(t, []*api.Organization{
+		{
+			ID:          17,
+			UserName:    user17.Name,
+			FullName:    user17.FullName,
+			AvatarURL:   user17.AvatarLink(),
+			Description: "",
+			Website:     "",
+			Location:    "",
+			Visibility:  "public",
+		},
 		{
 			ID:          3,
 			UserName:    user3.Name,

integrations/org_test.go

@@ -179,8 +179,8 @@ func TestOrgRestrictedUser(t *testing.T) {
 func TestTeamSearch(t *testing.T) {
 	defer prepareTestEnv(t)()
 
-	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}).(*user_model.User)
-	org := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}).(*user_model.User)
+	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 15}).(*user_model.User)
+	org := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 17}).(*user_model.User)
 
 	var results TeamSearchResults
 
@@ -190,9 +190,9 @@ func TestTeamSearch(t *testing.T) {
 	req.Header.Add("X-Csrf-Token", csrf)
 	resp := session.MakeRequest(t, req, http.StatusOK)
 	DecodeJSON(t, resp, &results)
-	assert.NotEmpty(t, results.Data)
-	assert.Len(t, results.Data, 1)
-	assert.Equal(t, "test_team", results.Data[0].Name)
+	assert.Len(t, results.Data, 2)
+	assert.Equal(t, "review_team", results.Data[0].Name)
+	assert.Equal(t, "test_team", results.Data[1].Name)
 
 	// no access if not organization member
 	user5 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 5}).(*user_model.User)

jest.config.js

@@ -1,7 +1,7 @@
 export default {
   rootDir: 'web_src',
   setupFilesAfterEnv: ['jest-extended/all'],
-  testEnvironment: '@happy-dom/jest-environment',
+  testEnvironment: 'jest-environment-jsdom',
   testMatch: ['<rootDir>/**/*.test.js'],
   testTimeout: 20000,
   transform: {

models/action.go

@@ -98,7 +98,14 @@ func (a *Action) TableIndices() []*schemas.Index {
 	actUserIndex := schemas.NewIndex("au_r_c_u_d", schemas.IndexType)
 	actUserIndex.AddColumn("act_user_id", "repo_id", "created_unix", "user_id", "is_deleted")
 
-	return []*schemas.Index{actUserIndex, repoIndex}
+	indices := []*schemas.Index{actUserIndex, repoIndex}
+	if setting.Database.UsePostgreSQL {
+		cudIndex := schemas.NewIndex("c_u_d", schemas.IndexType)
+		cudIndex.AddColumn("created_unix", "user_id", "is_deleted")
+		indices = append(indices, cudIndex)
+	}
+
+	return indices
 }
 
 // GetOpType gets the ActionType of this action.
@@ -275,7 +282,7 @@ func (a *Action) GetRefLink() string {
 		return a.GetRepoLink() + "/src/branch/" + util.PathEscapeSegments(strings.TrimPrefix(a.RefName, git.BranchPrefix))
 	case strings.HasPrefix(a.RefName, git.TagPrefix):
 		return a.GetRepoLink() + "/src/tag/" + util.PathEscapeSegments(strings.TrimPrefix(a.RefName, git.TagPrefix))
-	case len(a.RefName) == 40 && git.SHAPattern.MatchString(a.RefName):
+	case len(a.RefName) == 40 && git.IsValidSHAPattern(a.RefName):
 		return a.GetRepoLink() + "/src/commit/" + a.RefName
 	default:
 		// FIXME: we will just assume it's a branch - this was the old way - at some point we may want to enforce that there is always a ref here.

models/asymkey/gpg_key.go

@@ -33,7 +33,7 @@ type GPGKey struct {
 	OwnerID           int64              `xorm:"INDEX NOT NULL"`
 	KeyID             string             `xorm:"INDEX CHAR(16) NOT NULL"`
 	PrimaryKeyID      string             `xorm:"CHAR(16)"`
-	Content           string             `xorm:"TEXT NOT NULL"`
+	Content           string             `xorm:"MEDIUMTEXT NOT NULL"`
 	CreatedUnix       timeutil.TimeStamp `xorm:"created"`
 	ExpiredUnix       timeutil.TimeStamp
 	AddedUnix         timeutil.TimeStamp
@@ -63,6 +63,15 @@ func (key *GPGKey) AfterLoad(session *xorm.Session) {
 	}
 }
 
+// PaddedKeyID show KeyID padded to 16 characters
+func (key *GPGKey) PaddedKeyID() string {
+	if len(key.KeyID) > 15 {
+		return key.KeyID
+	}
+	zeros := "0000000000000000"
+	return zeros[0:16-len(key.KeyID)] + key.KeyID
+}
+
 // ListGPGKeys returns a list of public keys belongs to given user.
 func ListGPGKeys(ctx context.Context, uid int64, listOptions db.ListOptions) ([]*GPGKey, error) {
 	sess := db.GetEngine(ctx).Table(&GPGKey{}).Where("owner_id=? AND primary_key_id=''", uid)

models/asymkey/ssh_key.go

@@ -41,7 +41,7 @@ type PublicKey struct {
 	OwnerID       int64           `xorm:"INDEX NOT NULL"`
 	Name          string          `xorm:"NOT NULL"`
 	Fingerprint   string          `xorm:"INDEX NOT NULL"`
-	Content       string          `xorm:"TEXT NOT NULL"`
+	Content       string          `xorm:"MEDIUMTEXT NOT NULL"`
 	Mode          perm.AccessMode `xorm:"NOT NULL DEFAULT 2"`
 	Type          KeyType         `xorm:"NOT NULL DEFAULT 1"`
 	LoginSourceID int64           `xorm:"NOT NULL DEFAULT 0"`

models/auth/oauth2.go

@@ -512,10 +512,14 @@ func GetActiveOAuth2ProviderSources() ([]*Source, error) {
 func GetActiveOAuth2SourceByName(name string) (*Source, error) {
 	authSource := new(Source)
 	has, err := db.GetEngine(db.DefaultContext).Where("name = ? and type = ? and is_active = ?", name, OAuth2, true).Get(authSource)
-	if !has || err != nil {
+	if err != nil {
 		return nil, err
 	}
 
+	if !has {
+		return nil, fmt.Errorf("oauth2 source not found, name: %q", name)
+	}
+
 	return authSource, nil
 }
 

models/fixtures/org_user.yml

@@ -63,3 +63,9 @@
   uid: 29
   org_id: 17
   is_public: true
+
+-
+  id: 12
+  uid: 2
+  org_id: 17
+  is_public: true

models/fixtures/user.yml

@@ -309,7 +309,7 @@
   avatar_email: user17@example.com
   num_repos: 2
   is_active: true
-  num_members: 3
+  num_members: 4
   num_teams: 3
 
 -

models/issues/issue_project.go

@@ -68,6 +68,7 @@ func LoadIssuesFromBoard(b *project_model.Board) (IssueList, error) {
 		issues, err := Issues(&IssuesOptions{
 			ProjectBoardID: b.ID,
 			ProjectID:      b.ProjectID,
+			SortType:       "project-column-sorting",
 		})
 		if err != nil {
 			return nil, err
@@ -79,6 +80,7 @@ func LoadIssuesFromBoard(b *project_model.Board) (IssueList, error) {
 		issues, err := Issues(&IssuesOptions{
 			ProjectBoardID: -1, // Issues without ProjectBoardID
 			ProjectID:      b.ProjectID,
+			SortType:       "project-column-sorting",
 		})
 		if err != nil {
 			return nil, err

models/migrations/v218.go

@@ -5,6 +5,7 @@
 package migrations
 
 import (
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
 
 	"xorm.io/xorm"
@@ -37,8 +38,14 @@ func (*improveActionTableIndicesAction) TableIndices() []*schemas.Index {
 
 	actUserIndex := schemas.NewIndex("au_r_c_u_d", schemas.IndexType)
 	actUserIndex.AddColumn("act_user_id", "repo_id", "created_unix", "user_id", "is_deleted")
+	indices := []*schemas.Index{actUserIndex, repoIndex}
+	if setting.Database.UsePostgreSQL {
+		cudIndex := schemas.NewIndex("c_u_d", schemas.IndexType)
+		cudIndex.AddColumn("created_unix", "user_id", "is_deleted")
+		indices = append(indices, cudIndex)
+	}
 
-	return []*schemas.Index{actUserIndex, repoIndex}
+	return indices
 }
 
 func improveActionTableIndices(x *xorm.Engine) error {

models/organization/team.go

@@ -96,16 +96,7 @@ type SearchTeamOptions struct {
 	IncludeDesc bool
 }
 
-// SearchTeam search for teams. Caller is responsible to check permissions.
-func SearchTeam(opts *SearchTeamOptions) ([]*Team, int64, error) {
-	if opts.Page <= 0 {
-		opts.Page = 1
-	}
-	if opts.PageSize == 0 {
-		// Default limit
-		opts.PageSize = 10
-	}
-
+func (opts *SearchTeamOptions) toCond() builder.Cond {
 	cond := builder.NewCond()
 
 	if len(opts.Keyword) > 0 {
@@ -117,10 +108,28 @@ func SearchTeam(opts *SearchTeamOptions) ([]*Team, int64, error) {
 		cond = cond.And(keywordCond)
 	}
 
-	cond = cond.And(builder.Eq{"org_id": opts.OrgID})
+	if opts.OrgID > 0 {
+		cond = cond.And(builder.Eq{"`team`.org_id": opts.OrgID})
+	}
 
+	if opts.UserID > 0 {
+		cond = cond.And(builder.Eq{"team_user.uid": opts.UserID})
+	}
+
+	return cond
+}
+
+// SearchTeam search for teams. Caller is responsible to check permissions.
+func SearchTeam(opts *SearchTeamOptions) ([]*Team, int64, error) {
 	sess := db.GetEngine(db.DefaultContext)
 
+	opts.SetDefaultValues()
+	cond := opts.toCond()
+
+	if opts.UserID > 0 {
+		sess = sess.Join("INNER", "team_user", "team_user.team_id = team.id")
+	}
+
 	count, err := sess.
 		Where(cond).
 		Count(new(Team))
@@ -128,6 +137,10 @@ func SearchTeam(opts *SearchTeamOptions) ([]*Team, int64, error) {
 		return nil, 0, err
 	}
 
+	if opts.UserID > 0 {
+		sess = sess.Join("INNER", "team_user", "team_user.team_id = team.id")
+	}
+
 	sess = sess.Where(cond)
 	if opts.PageSize == -1 {
 		opts.PageSize = int(count)
@@ -137,6 +150,7 @@ func SearchTeam(opts *SearchTeamOptions) ([]*Team, int64, error) {
 
 	teams := make([]*Team, 0, opts.PageSize)
 	if err = sess.
+		Where(cond).
 		OrderBy("lower_name").
 		Find(&teams); err != nil {
 		return nil, 0, err

models/packages/package.go

@@ -214,9 +214,16 @@ func FindUnreferencedPackages(ctx context.Context) ([]*Package, error) {
 		Find(&ps)
 }
 
-// HasOwnerPackages tests if a user/org has packages
+// HasOwnerPackages tests if a user/org has accessible packages
 func HasOwnerPackages(ctx context.Context, ownerID int64) (bool, error) {
-	return db.GetEngine(ctx).Where("owner_id = ?", ownerID).Exist(&Package{})
+	return db.GetEngine(ctx).
+		Table("package_version").
+		Join("INNER", "package", "package.id = package_version.package_id").
+		Where(builder.Eq{
+			"package_version.is_internal": false,
+			"package.owner_id":            ownerID,
+		}).
+		Exist(&PackageVersion{})
 }
 
 // HasRepositoryPackages tests if a repository has packages

models/packages/package_test.go

@@ -0,0 +1,69 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package packages_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"code.gitea.io/gitea/models/db"
+	packages_model "code.gitea.io/gitea/models/packages"
+	"code.gitea.io/gitea/models/unittest"
+	user_model "code.gitea.io/gitea/models/user"
+
+	_ "code.gitea.io/gitea/models"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMain(m *testing.M) {
+	unittest.MainTest(m, &unittest.TestOptions{
+		GiteaRootPath: filepath.Join("..", ".."),
+	})
+}
+
+func TestHasOwnerPackages(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}).(*user_model.User)
+
+	p, err := packages_model.TryInsertPackage(db.DefaultContext, &packages_model.Package{
+		OwnerID:   owner.ID,
+		LowerName: "package",
+	})
+	assert.NotNil(t, p)
+	assert.NoError(t, err)
+
+	// A package without package versions gets automatically cleaned up and should return false
+	has, err := packages_model.HasOwnerPackages(db.DefaultContext, owner.ID)
+	assert.False(t, has)
+	assert.NoError(t, err)
+
+	pv, err := packages_model.GetOrInsertVersion(db.DefaultContext, &packages_model.PackageVersion{
+		PackageID:    p.ID,
+		LowerVersion: "internal",
+		IsInternal:   true,
+	})
+	assert.NotNil(t, pv)
+	assert.NoError(t, err)
+
+	// A package with an internal package version gets automaticaly cleaned up and should return false
+	has, err = packages_model.HasOwnerPackages(db.DefaultContext, owner.ID)
+	assert.False(t, has)
+	assert.NoError(t, err)
+
+	pv, err = packages_model.GetOrInsertVersion(db.DefaultContext, &packages_model.PackageVersion{
+		PackageID:    p.ID,
+		LowerVersion: "normal",
+		IsInternal:   false,
+	})
+	assert.NotNil(t, pv)
+	assert.NoError(t, err)
+
+	// A package with a normal package version should return true
+	has, err = packages_model.HasOwnerPackages(db.DefaultContext, owner.ID)
+	assert.True(t, has)
+	assert.NoError(t, err)
+}

models/repo.go

@@ -385,8 +385,7 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error {
 
 	archivePaths := make([]string, 0, len(archives))
 	for _, v := range archives {
-		p, _ := v.RelativePath()
-		archivePaths = append(archivePaths, p)
+		archivePaths = append(archivePaths, v.RelativePath())
 	}
 
 	if _, err := db.DeleteByBean(ctx, &repo_model.RepoArchiver{RepoID: repoID}); err != nil {

models/repo/archiver.go

@@ -39,9 +39,9 @@ func init() {
 	db.RegisterModel(new(RepoArchiver))
 }
 
-// RelativePath returns relative path
-func (archiver *RepoArchiver) RelativePath() (string, error) {
-	return fmt.Sprintf("%d/%s/%s.%s", archiver.RepoID, archiver.CommitID[:2], archiver.CommitID, archiver.Type.String()), nil
+// RelativePath returns the archive path relative to the archive storage root.
+func (archiver *RepoArchiver) RelativePath() string {
+	return fmt.Sprintf("%d/%s/%s.%s", archiver.RepoID, archiver.CommitID[:2], archiver.CommitID, archiver.Type.String())
 }
 
 var delRepoArchiver = new(RepoArchiver)

models/repo/mirror.go

@@ -8,7 +8,6 @@ package repo
 import (
 	"context"
 	"errors"
-	"fmt"
 	"time"
 
 	"code.gitea.io/gitea/models/db"
@@ -108,12 +107,14 @@ func DeleteMirrorByRepoID(repoID int64) error {
 
 // MirrorsIterate iterates all mirror repositories.
 func MirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
-	return db.GetEngine(db.DefaultContext).
+	sess := db.GetEngine(db.DefaultContext).
 		Where("next_update_unix<=?", time.Now().Unix()).
 		And("next_update_unix!=0").
-		OrderBy("updated_unix ASC").
-		Limit(limit).
-		Iterate(new(Mirror), f)
+		OrderBy("updated_unix ASC")
+	if limit > 0 {
+		sess = sess.Limit(limit)
+	}
+	return sess.Iterate(new(Mirror), f)
 }
 
 // InsertMirror inserts a mirror to database
@@ -121,55 +122,3 @@ func InsertMirror(ctx context.Context, mirror *Mirror) error {
 	_, err := db.GetEngine(ctx).Insert(mirror)
 	return err
 }
-
-// MirrorRepositoryList contains the mirror repositories
-type MirrorRepositoryList []*Repository
-
-func (repos MirrorRepositoryList) loadAttributes(ctx context.Context) error {
-	if len(repos) == 0 {
-		return nil
-	}
-
-	// Load mirrors.
-	repoIDs := make([]int64, 0, len(repos))
-	for i := range repos {
-		if !repos[i].IsMirror {
-			continue
-		}
-
-		repoIDs = append(repoIDs, repos[i].ID)
-	}
-	mirrors := make([]*Mirror, 0, len(repoIDs))
-	if err := db.GetEngine(ctx).
-		Where("id > 0").
-		In("repo_id", repoIDs).
-		Find(&mirrors); err != nil {
-		return fmt.Errorf("find mirrors: %v", err)
-	}
-
-	set := make(map[int64]*Mirror)
-	for i := range mirrors {
-		set[mirrors[i].RepoID] = mirrors[i]
-	}
-	for i := range repos {
-		repos[i].Mirror = set[repos[i].ID]
-		if repos[i].Mirror != nil {
-			repos[i].Mirror.Repo = repos[i]
-		}
-	}
-	return nil
-}
-
-// LoadAttributes loads the attributes for the given MirrorRepositoryList
-func (repos MirrorRepositoryList) LoadAttributes() error {
-	return repos.loadAttributes(db.DefaultContext)
-}
-
-// GetUserMirrorRepositories returns a list of mirror repositories of given user.
-func GetUserMirrorRepositories(userID int64) ([]*Repository, error) {
-	repos := make([]*Repository, 0, 10)
-	return repos, db.GetEngine(db.DefaultContext).
-		Where("owner_id = ?", userID).
-		And("is_mirror = ?", true).
-		Find(&repos)
-}

models/repo/pushmirror.go

@@ -95,10 +95,12 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) {
 
 // PushMirrorsIterate iterates all push-mirror repositories.
 func PushMirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
-	return db.GetEngine(db.DefaultContext).
+	sess := db.GetEngine(db.DefaultContext).
 		Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()).
 		And("`interval` != 0").
-		OrderBy("last_update ASC").
-		Limit(limit).
-		Iterate(new(PushMirror), f)
+		OrderBy("last_update ASC")
+	if limit > 0 {
+		sess = sess.Limit(limit)
+	}
+	return sess.Iterate(new(PushMirror), f)
 }

models/user.go

@@ -100,9 +100,9 @@ func DeleteUser(ctx context.Context, u *user_model.User) (err error) {
 
 		// Delete Comments
 		const batchSize = 50
-		for start := 0; ; start += batchSize {
+		for {
 			comments := make([]*issues_model.Comment, 0, batchSize)
-			if err = e.Where("type=? AND poster_id=?", issues_model.CommentTypeComment, u.ID).Limit(batchSize, start).Find(&comments); err != nil {
+			if err = e.Where("type=? AND poster_id=?", issues_model.CommentTypeComment, u.ID).Limit(batchSize, 0).Find(&comments); err != nil {
 				return err
 			}
 			if len(comments) == 0 {
@@ -200,7 +200,7 @@ func DeleteUser(ctx context.Context, u *user_model.User) (err error) {
 	// ***** END: ExternalLoginUser *****
 
 	if _, err = e.ID(u.ID).Delete(new(user_model.User)); err != nil {
-		return fmt.Errorf("Delete: %v", err)
+		return fmt.Errorf("delete: %v", err)
 	}
 
 	return nil

modules/context/repo.go

@@ -986,6 +986,7 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context
 		}
 
 		ctx.Data["BranchName"] = ctx.Repo.BranchName
+		ctx.Data["RefName"] = ctx.Repo.RefName
 		ctx.Data["BranchNameSubURL"] = ctx.Repo.BranchNameSubURL()
 		ctx.Data["TagName"] = ctx.Repo.TagName
 		ctx.Data["CommitID"] = ctx.Repo.CommitID

modules/convert/issue_comment.go

@@ -101,6 +101,12 @@ func ToTimelineComment(c *issues_model.Comment, doer *user_model.User) *api.Time
 	}
 
 	if c.Time != nil {
+		err = c.Time.LoadAttributes()
+		if err != nil {
+			log.Error("Time.LoadAttributes: %v", err)
+			return nil
+		}
+
 		comment.TrackedTime = ToTrackedTime(c.Time)
 	}
 

modules/git/git.go

@@ -287,7 +287,20 @@ func syncGitConfig() (err error) {
 		}
 	}
 
-	return nil
+	// By default partial clones are disabled, enable them from git v2.22
+	if !setting.Git.DisablePartialClone && CheckGitVersionAtLeast("2.22") == nil {
+		if err = configSet("uploadpack.allowfilter", "true"); err != nil {
+			return err
+		}
+		err = configSet("uploadpack.allowAnySHA1InWant", "true")
+	} else {
+		if err = configUnsetAll("uploadpack.allowfilter", "true"); err != nil {
+			return err
+		}
+		err = configUnsetAll("uploadpack.allowAnySHA1InWant", "true")
+	}
+
+	return err
 }
 
 // CheckGitVersionAtLeast check git version is at least the constraint version

modules/git/ref.go

@@ -4,7 +4,10 @@
 
 package git
 
-import "strings"
+import (
+	"regexp"
+	"strings"
+)
 
 const (
 	// RemotePrefix is the base directory of the remotes information of git.
@@ -15,6 +18,29 @@ const (
 	pullLen = len(PullPrefix)
 )
 
+// refNamePatternInvalid is regular expression with unallowed characters in git reference name
+// They cannot have ASCII control characters (i.e. bytes whose values are lower than \040, or \177 DEL), space, tilde ~, caret ^, or colon : anywhere.
+// They cannot have question-mark ?, asterisk *, or open bracket [ anywhere
+var refNamePatternInvalid = regexp.MustCompile(
+	`[\000-\037\177 \\~^:?*[]|` + // No absolutely invalid characters
+		`(?:^[/.])|` + // Not HasPrefix("/") or "."
+		`(?:/\.)|` + // no "/."
+		`(?:\.lock$)|(?:\.lock/)|` + // No ".lock/"" or ".lock" at the end
+		`(?:\.\.)|` + // no ".." anywhere
+		`(?://)|` + // no "//" anywhere
+		`(?:@{)|` + // no "@{"
+		`(?:[/.]$)|` + // no terminal '/' or '.'
+		`(?:^@$)`) // Not "@"
+
+// IsValidRefPattern ensures that the provided string could be a valid reference
+func IsValidRefPattern(name string) bool {
+	return !refNamePatternInvalid.MatchString(name)
+}
+
+func SanitizeRefPattern(name string) string {
+	return refNamePatternInvalid.ReplaceAllString(name, "_")
+}
+
 // Reference represents a Git ref.
 type Reference struct {
 	Name   string

modules/git/repo_commit_nogogit.go

@@ -138,7 +138,7 @@ func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id SHA1) (*Co
 
 // ConvertToSHA1 returns a Hash object from a potential ID string
 func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) {
-	if len(commitID) == 40 && SHAPattern.MatchString(commitID) {
+	if len(commitID) == 40 && IsValidSHAPattern(commitID) {
 		sha1, err := NewIDFromString(commitID)
 		if err == nil {
 			return sha1, nil

modules/git/repo_compare.go

@@ -40,7 +40,7 @@ func (repo *Repository) GetMergeBase(tmpRemote, base, head string) (string, stri
 	if tmpRemote != "origin" {
 		tmpBaseName := RemotePrefix + tmpRemote + "/tmp_" + base
 		// Fetch commit into a temporary branch in order to be able to handle commits and tags
-		_, _, err := NewCommand(repo.Ctx, "fetch", tmpRemote, base+":"+tmpBaseName).RunStdString(&RunOpts{Dir: repo.Path})
+		_, _, err := NewCommand(repo.Ctx, "fetch", "--no-tags", tmpRemote, "--", base+":"+tmpBaseName).RunStdString(&RunOpts{Dir: repo.Path})
 		if err == nil {
 			base = tmpBaseName
 		}

modules/git/sha1.go

@@ -19,7 +19,12 @@ const EmptySHA = "0000000000000000000000000000000000000000"
 const EmptyTreeSHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
 
 // SHAPattern can be used to determine if a string is an valid sha
-var SHAPattern = regexp.MustCompile(`^[0-9a-f]{4,40}$`)
+var shaPattern = regexp.MustCompile(`^[0-9a-f]{4,40}$`)
+
+// IsValidSHAPattern will check if the provided string matches the SHA Pattern
+func IsValidSHAPattern(sha string) bool {
+	return shaPattern.MatchString(sha)
+}
 
 // MustID always creates a new SHA1 from a [20]byte array with no validation of input.
 func MustID(b []byte) SHA1 {

modules/graceful/manager_windows.go

@@ -114,9 +114,9 @@ func (g *Manager) start() {
 // Execute makes Manager implement svc.Handler
 func (g *Manager) Execute(args []string, changes <-chan svc.ChangeRequest, status chan<- svc.Status) (svcSpecificEC bool, exitCode uint32) {
 	if setting.StartupTimeout > 0 {
-		status <- svc.Status{State: svc.StartPending}
-	} else {
 		status <- svc.Status{State: svc.StartPending, WaitHint: uint32(setting.StartupTimeout / time.Millisecond)}
+	} else {
+		status <- svc.Status{State: svc.StartPending}
 	}
 
 	log.Trace("Awaiting server start-up")

modules/log/multichannel.go

@@ -33,7 +33,7 @@ func newLogger(name string, buffer int64) *MultiChannelledLogger {
 func (l *MultiChannelledLogger) SetLogger(name, provider, config string) error {
 	eventLogger, err := NewChannelledLog(l.ctx, name, provider, config, l.bufferLength)
 	if err != nil {
-		return fmt.Errorf("Failed to create sublogger (%s): %v", name, err)
+		return fmt.Errorf("failed to create sublogger (%s): %w", name, err)
 	}
 
 	l.MultiChannelledLog.DelLogger(name)
@@ -41,9 +41,9 @@ func (l *MultiChannelledLogger) SetLogger(name, provider, config string) error {
 	err = l.MultiChannelledLog.AddLogger(eventLogger)
 	if err != nil {
 		if IsErrDuplicateName(err) {
-			return fmt.Errorf("Duplicate named sublogger %s %v", name, l.MultiChannelledLog.GetEventLoggerNames())
+			return fmt.Errorf("%w other names: %v", err, l.MultiChannelledLog.GetEventLoggerNames())
 		}
-		return fmt.Errorf("Failed to add sublogger (%s): %v", name, err)
+		return fmt.Errorf("failed to add sublogger (%s): %w", name, err)
 	}
 
 	return nil

modules/migration/pullrequest.go

@@ -26,7 +26,7 @@ type PullRequest struct {
 	Updated        time.Time
 	Closed         *time.Time
 	Labels         []*Label
-	PatchURL       string `yaml:"patch_url"`
+	PatchURL       string `yaml:"patch_url"` // SECURITY: This must be safe to download directly from
 	Merged         bool
 	MergedTime     *time.Time `yaml:"merged_time"`
 	MergeCommitSHA string     `yaml:"merge_commit_sha"`
@@ -37,6 +37,7 @@ type PullRequest struct {
 	Reactions      []*Reaction
 	ForeignIndex   int64
 	Context        DownloaderContext `yaml:"-"`
+	EnsuredSafe    bool              `yaml:"ensured_safe"`
 }
 
 func (p *PullRequest) GetLocalIndex() int64          { return p.Number }
@@ -55,9 +56,9 @@ func (p PullRequest) GetGitRefName() string {
 
 // PullRequestBranch represents a pull request branch
 type PullRequestBranch struct {
-	CloneURL  string `yaml:"clone_url"`
-	Ref       string
-	SHA       string
+	CloneURL  string `yaml:"clone_url"` // SECURITY: This must be safe to download from
+	Ref       string // SECURITY: this must be a git.IsValidRefPattern
+	SHA       string // SECURITY: this must be a git.IsValidSHAPattern
 	RepoName  string `yaml:"repo_name"`
 	OwnerName string `yaml:"owner_name"`
 }

modules/migration/release.go

@@ -18,15 +18,16 @@ type ReleaseAsset struct {
 	DownloadCount *int `yaml:"download_count"`
 	Created       time.Time
 	Updated       time.Time
-	DownloadURL   *string `yaml:"download_url"`
+
+	DownloadURL *string `yaml:"download_url"` // SECURITY: It is the responsibility of downloader to make sure this is safe
 	// if DownloadURL is nil, the function should be invoked
-	DownloadFunc func() (io.ReadCloser, error) `yaml:"-"`
+	DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` // SECURITY: It is the responsibility of downloader to make sure this is safe
 }
 
 // Release represents a release
 type Release struct {
-	TagName         string `yaml:"tag_name"`
-	TargetCommitish string `yaml:"target_commitish"`
+	TagName         string `yaml:"tag_name"`         // SECURITY: This must pass git.IsValidRefPattern
+	TargetCommitish string `yaml:"target_commitish"` // SECURITY: This must pass git.IsValidRefPattern
 	Name            string
 	Body            string
 	Draft           bool

modules/migration/repo.go

@@ -12,7 +12,7 @@ type Repository struct {
 	IsPrivate     bool `yaml:"is_private"`
 	IsMirror      bool `yaml:"is_mirror"`
 	Description   string
-	CloneURL      string `yaml:"clone_url"`
+	CloneURL      string `yaml:"clone_url"` // SECURITY: This must be checked to ensure that is safe to be used
 	OriginalURL   string `yaml:"original_url"`
 	DefaultBranch string
 }

modules/nosql/manager_redis.go

@@ -245,7 +245,7 @@ func getRedisTLSOptions(uri *url.URL) *tls.Config {
 
 	if len(skipverify) > 0 {
 		skipverify, err := strconv.ParseBool(skipverify)
-		if err != nil {
+		if err == nil {
 			tlsConfig.InsecureSkipVerify = skipverify
 		}
 	}
@@ -254,7 +254,7 @@ func getRedisTLSOptions(uri *url.URL) *tls.Config {
 
 	if len(insecureskipverify) > 0 {
 		insecureskipverify, err := strconv.ParseBool(insecureskipverify)
-		if err != nil {
+		if err == nil {
 			tlsConfig.InsecureSkipVerify = insecureskipverify
 		}
 	}

modules/nosql/manager_redis_test.go

@@ -27,6 +27,24 @@ func TestRedisPasswordOpt(t *testing.T) {
 	}
 }
 
+func TestSkipVerifyOpt(t *testing.T) {
+	uri, _ := url.Parse("rediss://myredis/0?skipverify=true")
+	tlsConfig := getRedisTLSOptions(uri)
+
+	if !tlsConfig.InsecureSkipVerify {
+		t.Fail()
+	}
+}
+
+func TestInsecureSkipVerifyOpt(t *testing.T) {
+	uri, _ := url.Parse("rediss://myredis/0?insecureskipverify=true")
+	tlsConfig := getRedisTLSOptions(uri)
+
+	if !tlsConfig.InsecureSkipVerify {
+		t.Fail()
+	}
+}
+
 func TestRedisSentinelUsernameOpt(t *testing.T) {
 	uri, _ := url.Parse("redis+sentinel://redis:password@myredis/0?sentinelusername=suser&sentinelpassword=spass")
 	opts := getRedisOptions(uri).Failover()

modules/setting/setting.go

@@ -91,6 +91,8 @@ var (
 	// LocalURL is the url for locally running applications to contact Gitea. It always has a '/' suffix
 	// It maps to ini:"LOCAL_ROOT_URL"
 	LocalURL string
+	// AssetVersion holds a opaque value that is used for cache-busting assets
+	AssetVersion string
 
 	// Server settings
 	Protocol             Scheme
@@ -749,6 +751,7 @@ func loadFromConf(allowEmpty bool, extraConfig string) {
 	}
 
 	AbsoluteAssetURL = MakeAbsoluteAssetURL(AppURL, StaticURLPrefix)
+	AssetVersion = strings.ReplaceAll(AppVer, "+", "~") // make sure the version string is clear (no real escaping is needed)
 
 	manifestBytes := MakeManifestData(AppName, AppURL, AbsoluteAssetURL)
 	ManifestData = `application/json;base64,` + base64.StdEncoding.EncodeToString(manifestBytes)

modules/ssh/init.go

@@ -18,6 +18,7 @@ import (
 
 func Init() error {
 	if setting.SSH.Disabled {
+		builtinUnused()
 		return nil
 	}
 

modules/templates/helper.go

@@ -81,6 +81,9 @@ func NewFuncMap() []template.FuncMap {
 		"AppDomain": func() string {
 			return setting.Domain
 		},
+		"AssetVersion": func() string {
+			return setting.AssetVersion
+		},
 		"DisableGravatar": func() bool {
 			return setting.DisableGravatar
 		},
@@ -151,7 +154,6 @@ func NewFuncMap() []template.FuncMap {
 		"DiffTypeToStr":                  DiffTypeToStr,
 		"DiffLineTypeToStr":              DiffLineTypeToStr,
 		"ShortSha":                       base.ShortSha,
-		"MD5":                            base.EncodeMD5,
 		"ActionContent2Commits":          ActionContent2Commits,
 		"PathEscape":                     url.PathEscape,
 		"PathEscapeSegments":             util.PathEscapeSegments,
@@ -454,6 +456,7 @@ func NewFuncMap() []template.FuncMap {
 			}
 			return items
 		},
+		"HasPrefix": strings.HasPrefix,
 	}}
 }
 
@@ -974,11 +977,11 @@ type remoteAddress struct {
 	Password string
 }
 
-func mirrorRemoteAddress(ctx context.Context, m *repo_model.Repository, remoteName string) remoteAddress {
+func mirrorRemoteAddress(ctx context.Context, m *repo_model.Repository, remoteName string, ignoreOriginalURL bool) remoteAddress {
 	a := remoteAddress{}
 
 	remoteURL := m.OriginalURL
-	if remoteURL == "" {
+	if ignoreOriginalURL || remoteURL == "" {
 		var err error
 		remoteURL, err = git.GetRemoteAddress(ctx, m.RepoPath(), remoteName)
 		if err != nil {

modules/timeutil/timestamp.go

@@ -54,6 +54,11 @@ func (ts TimeStamp) AsTime() (tm time.Time) {
 	return ts.AsTimeInLocation(setting.DefaultUILocation)
 }
 
+// AsLocalTime convert timestamp as time.Time in local location
+func (ts TimeStamp) AsLocalTime() time.Time {
+	return time.Unix(int64(ts), 0)
+}
+
 // AsTimeInLocation convert timestamp as time.Time in Local locale
 func (ts TimeStamp) AsTimeInLocation(loc *time.Location) (tm time.Time) {
 	tm = time.Unix(int64(ts), 0).In(loc)

modules/validation/binding.go

@@ -9,6 +9,8 @@ import (
 	"regexp"
 	"strings"
 
+	"code.gitea.io/gitea/modules/git"
+
 	"gitea.com/go-chi/binding"
 	"github.com/gobwas/glob"
 )
@@ -24,30 +26,6 @@ const (
 	ErrRegexPattern = "RegexPattern"
 )
 
-// GitRefNamePatternInvalid is regular expression with unallowed characters in git reference name
-// They cannot have ASCII control characters (i.e. bytes whose values are lower than \040, or \177 DEL), space, tilde ~, caret ^, or colon : anywhere.
-// They cannot have question-mark ?, asterisk *, or open bracket [ anywhere
-var GitRefNamePatternInvalid = regexp.MustCompile(`[\000-\037\177 \\~^:?*[]+`)
-
-// CheckGitRefAdditionalRulesValid check name is valid on additional rules
-func CheckGitRefAdditionalRulesValid(name string) bool {
-	// Additional rules as described at https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html
-	if strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") ||
-		strings.HasSuffix(name, ".") || strings.Contains(name, "..") ||
-		strings.Contains(name, "//") || strings.Contains(name, "@{") ||
-		name == "@" {
-		return false
-	}
-	parts := strings.Split(name, "/")
-	for _, part := range parts {
-		if strings.HasSuffix(part, ".lock") || strings.HasPrefix(part, ".") {
-			return false
-		}
-	}
-
-	return true
-}
-
 // AddBindingRules adds additional binding rules
 func AddBindingRules() {
 	addGitRefNameBindingRule()
@@ -67,16 +45,10 @@ func addGitRefNameBindingRule() {
 		IsValid: func(errs binding.Errors, name string, val interface{}) (bool, binding.Errors) {
 			str := fmt.Sprintf("%v", val)
 
-			if GitRefNamePatternInvalid.MatchString(str) {
+			if !git.IsValidRefPattern(str) {
 				errs.Add([]string{name}, ErrGitRefName, "GitRefName")
 				return false, errs
 			}
-
-			if !CheckGitRefAdditionalRulesValid(str) {
-				errs.Add([]string{name}, ErrGitRefName, "GitRefName")
-				return false, errs
-			}
-
 			return true, errs
 		},
 	})

options/locale/locale_de-DE.ini

@@ -1061,6 +1061,7 @@ normal_view=Normale Ansicht
 line=zeile
 lines=Zeilen
 
+editor.add_file=Datei hinzufügen
 editor.new_file=Neue Datei
 editor.upload_file=Datei hochladen
 editor.edit_file=Datei bearbeiten

options/locale/locale_en-US.ini

@@ -1061,6 +1061,7 @@ normal_view = Normal View
 line = line
 lines = lines
 
+editor.add_file = Add File
 editor.new_file = New File
 editor.upload_file = Upload File
 editor.edit_file = Edit File
@@ -1419,7 +1420,7 @@ issues.due_date_form_remove = "Remove"
 issues.due_date_not_writer = "You need repository write access to update an issue's due date."
 issues.due_date_not_set = "No due date set."
 issues.due_date_added = "added the due date %s %s"
-issues.due_date_modified = "modified the due date to %s from %s %s"
+issues.due_date_modified = "modified the due date from %[2]s to %[1]s %[3]s"
 issues.due_date_remove = "removed the due date %s %s"
 issues.due_date_overdue = "Overdue"
 issues.due_date_invalid = "The due date is invalid or out of range. Please use the format 'yyyy-mm-dd'."

options/locale/locale_es-ES.ini

@@ -1061,6 +1061,7 @@ normal_view=Vista normal
 line=línea
 lines=líneas
 
+editor.add_file=Añadir archivo
 editor.new_file=Nuevo Archivo
 editor.upload_file=Subir archivo
 editor.edit_file=Editar Archivo

options/locale/locale_it-IT.ini

@@ -820,6 +820,7 @@ normal_view=Vista normale
 line=riga
 lines=righe
 
+editor.add_file=Aggiungi file
 editor.new_file=Nuovo file
 editor.upload_file=Carica File
 editor.edit_file=Modifica File

options/locale/locale_lv-LV.ini

@@ -1061,6 +1061,7 @@ normal_view=Parastais skats
 line=rinda
 lines=rindas
 
+editor.add_file=Pievienot
 editor.new_file=Jauna datne
 editor.upload_file=Augšupielādēt failu
 editor.edit_file=Labot failu

options/locale/locale_nl-NL.ini

@@ -838,6 +838,7 @@ normal_view=Normale weergave
 line=regel
 lines=regels
 
+editor.add_file=Bestand toevoegen
 editor.new_file=Nieuw bestand
 editor.upload_file=Upload bestand
 editor.edit_file=Bewerk bestand

options/locale/locale_pt-PT.ini

@@ -1061,6 +1061,7 @@ normal_view=Vista normal
 line=linha
 lines=linhas
 
+editor.add_file=Adicionar ficheiro
 editor.new_file=Novo ficheiro
 editor.upload_file=Carregar ficheiro
 editor.edit_file=Editar ficheiro

options/locale/locale_tr-TR.ini

@@ -948,6 +948,7 @@ normal_view=Normal Görünüm
 line=satır
 lines=satır
 
+editor.add_file=Dosya Ekle
 editor.new_file=Yeni dosya
 editor.upload_file=Dosya Yükle
 editor.edit_file=Dosyayı Düzenle
@@ -1554,8 +1555,8 @@ activity.no_git_activity=Bu dönemde herhangi bir işleme yapılmamıştır.
 activity.git_stats_exclude_merges=Birleştirmeler hariç,
 activity.git_stats_author_1=%d yazar
 activity.git_stats_author_n=%d yazar
-activity.git_stats_pushed_1= 
-activity.git_stats_pushed_n= 
+activity.git_stats_pushed_1=
+activity.git_stats_pushed_n=
 activity.git_stats_commit_1=%d işlemeyi
 activity.git_stats_commit_n=%d işlemeyi
 activity.git_stats_push_to_branch=%s dalına ve
@@ -2033,7 +2034,7 @@ branch.create_success='%s' dalı oluşturuldu.
 branch.branch_already_exists='%s' dalı zaten bu depoda var.
 branch.branch_name_conflict='%s' dal adı zaten mevcut olan '%s' dalıyla çakışıyor.
 branch.tag_collision='%s' dalı, depoda aynı ada sahip bir etiket olduğundan oluşturulamıyor.
-branch.deleted_by=%s tarafından silindi 
+branch.deleted_by=%s tarafından silindi
 branch.restore_success='%s' dalı geri yüklendi.
 branch.restore_failed='%s' dalı geri yüklenemedi.
 branch.protected_deletion_failed='%s' dalı korunuyor. Silinemez.

options/locale/locale_zh-CN.ini

@@ -695,8 +695,8 @@ last_used=上次使用在
 no_activity=没有最近活动
 can_read_info=读取
 can_write_info=写入
-key_state_desc=7 天内使用过该密钥 
-token_state_desc=7 天内使用过该密钥 
+key_state_desc=7 天内使用过该密钥
+token_state_desc=7 天内使用过该密钥
 principal_state_desc=7 天内使用过该规则
 show_openid=在个人信息上显示
 hide_openid=在个人信息上隐藏
@@ -880,7 +880,7 @@ watchers=关注者
 stargazers=称赞者
 forks=派生仓库
 pick_reaction=选择你的表情
-reactions_more=再加载 %d 
+reactions_more=再加载 %d
 unit_disabled=站点管理员已禁用此仓库单元。
 language_other=其它
 adopt_search=输入用户名以搜索未被收录的仓库... (留空以查找全部)
@@ -1061,6 +1061,7 @@ normal_view=普通视图
 line=行
 lines=行
 
+editor.add_file=添加文件
 editor.new_file=新建文件
 editor.upload_file=上传文件
 editor.edit_file=编辑文件
@@ -1419,7 +1420,7 @@ issues.due_date_form_remove=删除
 issues.due_date_not_writer=你需要仓库写入权限来修改工单到期时间。
 issues.due_date_not_set=未设置到期时间。
 issues.due_date_added=于 %[2]s 设置到期时间为 %[1]s
-issues.due_date_modified=于 %[3]s 将到期时间从 %[2]s 修改为 %[1]s 
+issues.due_date_modified=于 %[3]s 将到期时间从 %[2]s 修改为 %[1]s
 issues.due_date_remove=于 %[2]s 删除了到期时间 %[1]s
 issues.due_date_overdue=过期
 issues.due_date_invalid=到期日期无效或超出范围。请使用 'yyyy-mm-dd' 格式。
@@ -3042,7 +3043,7 @@ filter.type.all=所有
 filter.no_result=您的过滤器没有产生任何结果。
 filter.container.tagged=已加标签
 filter.container.untagged=未加标签
-published_by=于 %[1]s 发布了 <a href="%[2]s">%[3]s</a> 
+published_by=于 %[1]s 发布了 <a href="%[2]s">%[3]s</a>
 published_by_in=<a href="%[2]s">%[3]s</a> 于 %[1]s 发布了 <a href="%[4]s"><strong>%[5]s</strong></a>
 installation=安装
 about=关于这个软件包

options/locale/locale_zh-TW.ini

@@ -1061,6 +1061,7 @@ normal_view=標準檢視
 line=行
 lines=行
 
+editor.add_file=加入檔案
 editor.new_file=新增文件
 editor.upload_file=上傳文件
 editor.edit_file=編輯文件

package.json

@@ -44,7 +44,6 @@
     "wrap-ansi": "8.0.1"
   },
   "devDependencies": {
-    "@happy-dom/jest-environment": "4.0.1",
     "eslint": "8.15.0",
     "eslint-plugin-html": "6.2.0",
     "eslint-plugin-import": "2.26.0",
@@ -52,6 +51,7 @@
     "eslint-plugin-unicorn": "42.0.0",
     "eslint-plugin-vue": "9.0.1",
     "jest": "28.1.0",
+    "jest-environment-jsdom": "28.1.3",
     "jest-extended": "2.0.0",
     "postcss-less": "6.0.0",
     "stylelint": "14.8.2",

public/img/svg/gitea-join.svg

@@ -0,0 +1 @@
+<svg viewBox="0 0 16 16" class="svg gitea-join" width="16" height="16" aria-hidden="true"><path d="M14 10.9V8.75h1.25a.75.75 0 0 0 0-1.5H14V5.1a.25.25 0 0 0-.43-.17l-2.9 2.9a.25.25 0 0 0 0 .35l2.9 2.9a.25.25 0 0 0 .43-.18ZM.75 8.75a.75.75 0 0 1 0-1.5H2V5.1a.25.25 0 0 1 .43-.17l2.9 2.9a.25.25 0 0 1 0 .35l-2.9 2.9A.25.25 0 0 1 2 10.9V8.75Zm6.5-6.5a.75.75 0 0 0 1.5 0v-.5a.75.75 0 0 0-1.5 0zM8 6a.75.75 0 0 1-.75-.75v-.5a.75.75 0 0 1 1.5 0v.5A.75.75 0 0 1 8 6Zm-.75 2.25a.75.75 0 0 0 1.5 0v-.5a.75.75 0 0 0-1.5 0zM8 12a.75.75 0 0 1-.75-.75v-.5a.75.75 0 0 1 1.5 0v.5A.75.75 0 0 1 8 12Zm-.75 2.25a.75.75 0 0 0 1.5 0v-.5a.75.75 0 0 0-1.5 0z"/></svg>

public/img/svg/gitea-split.svg

@@ -0,0 +1 @@
+<svg viewBox="0 0 16 16" class="svg gitea-split" width="16" height="16" aria-hidden="true"><path d="M7.25 14.25a.75.75 0 0 0 1.5 0v-.5a.75.75 0 0 0-1.5 0zM8 12a.75.75 0 0 1-.75-.75v-.5a.75.75 0 0 1 1.5 0v.5A.75.75 0 0 1 8 12Zm-.75-3.75a.75.75 0 0 0 1.5 0v-.5a.75.75 0 0 0-1.5 0zM8 6a.75.75 0 0 1-.75-.75v-.5a.75.75 0 0 1 1.5 0v.5A.75.75 0 0 1 8 6Zm-.75-3.75a.75.75 0 0 0 1.5 0v-.5a.75.75 0 0 0-1.5 0zm4.1 6.5a.75.75 0 0 1 0-1.5h1.25V5.1a.25.25 0 0 1 .43-.17l2.9 2.9a.25.25 0 0 1 0 .35l-2.9 2.9a.25.25 0 0 1-.43-.18V8.75ZM3.4 10.9V8.75h1.25a.75.75 0 0 0 0-1.5H3.4V5.1a.25.25 0 0 0-.43-.17l-2.9 2.9a.25.25 0 0 0 0 .35l2.9 2.9a.25.25 0 0 0 .43-.18z"/></svg>

public/img/svg/gitea-whitespace.svg

@@ -0,0 +1 @@
+<svg viewBox="0 0 15 15" class="svg gitea-whitespace" width="16" height="16" aria-hidden="true"><path d="m2.5 7.5.35.35a.5.5 0 0 0 0-.7l-.35.35ZM3 4h12V3H3v1Zm4 4h8V7H7v1Zm-4 4h12v-1H3v1ZM.85 9.85l2-2-.7-.7-2 2 .7.7Zm2-2.7-2-2-.7.7 2 2 .7-.7Z"/></svg>

routers/api/packages/maven/maven.go

@@ -266,8 +266,9 @@ func UploadPackageFile(ctx *context.Context) {
 		PackageFileInfo: packages_service.PackageFileInfo{
 			Filename: params.Filename,
 		},
-		Data:   buf,
-		IsLead: false,
+		Data:              buf,
+		IsLead:            false,
+		OverwriteExisting: params.IsMeta,
 	}
 
 	// If it's the package pom file extract the metadata

routers/api/packages/npm/api.go

@@ -55,15 +55,18 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package
 	metadata := pd.Metadata.(*npm_module.Metadata)
 
 	return &npm_module.PackageMetadataVersion{
-		ID:           fmt.Sprintf("%s@%s", pd.Package.Name, pd.Version.Version),
-		Name:         pd.Package.Name,
-		Version:      pd.Version.Version,
-		Description:  metadata.Description,
-		Author:       npm_module.User{Name: metadata.Author},
-		Homepage:     metadata.ProjectURL,
-		License:      metadata.License,
-		Dependencies: metadata.Dependencies,
-		Readme:       metadata.Readme,
+		ID:                   fmt.Sprintf("%s@%s", pd.Package.Name, pd.Version.Version),
+		Name:                 pd.Package.Name,
+		Version:              pd.Version.Version,
+		Description:          metadata.Description,
+		Author:               npm_module.User{Name: metadata.Author},
+		Homepage:             metadata.ProjectURL,
+		License:              metadata.License,
+		Dependencies:         metadata.Dependencies,
+		DevDependencies:      metadata.DevelopmentDependencies,
+		PeerDependencies:     metadata.PeerDependencies,
+		OptionalDependencies: metadata.OptionalDependencies,
+		Readme:               metadata.Readme,
 		Dist: npm_module.PackageDistribution{
 			Shasum:    pd.Files[0].Blob.HashSHA1,
 			Integrity: "sha512-" + base64.StdEncoding.EncodeToString(hashBytes),

routers/api/v1/repo/commits.go

@@ -17,7 +17,6 @@ import (
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/setting"
 	api "code.gitea.io/gitea/modules/structs"
-	"code.gitea.io/gitea/modules/validation"
 	"code.gitea.io/gitea/routers/api/v1/utils"
 )
 
@@ -53,7 +52,7 @@ func GetSingleCommit(ctx *context.APIContext) {
 	//     "$ref": "#/responses/notFound"
 
 	sha := ctx.Params(":sha")
-	if (validation.GitRefNamePatternInvalid.MatchString(sha) || !validation.CheckGitRefAdditionalRulesValid(sha)) && !git.SHAPattern.MatchString(sha) {
+	if !git.IsValidRefPattern(sha) {
 		ctx.Error(http.StatusUnprocessableEntity, "no valid ref or sha", fmt.Sprintf("no valid ref or sha: %s", sha))
 		return
 	}

routers/api/v1/repo/file.go

@@ -8,6 +8,7 @@ package repo
 import (
 	"bytes"
 	"encoding/base64"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -29,7 +30,7 @@ import (
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/web"
 	"code.gitea.io/gitea/routers/common"
-	"code.gitea.io/gitea/routers/web/repo"
+	archiver_service "code.gitea.io/gitea/services/repository/archiver"
 	files_service "code.gitea.io/gitea/services/repository/files"
 )
 
@@ -294,7 +295,53 @@ func GetArchive(ctx *context.APIContext) {
 		defer gitRepo.Close()
 	}
 
-	repo.Download(ctx.Context)
+	archiveDownload(ctx)
+}
+
+func archiveDownload(ctx *context.APIContext) {
+	uri := ctx.Params("*")
+	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri)
+	if err != nil {
+		if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) {
+			ctx.Error(http.StatusBadRequest, "unknown archive format", err)
+		} else if errors.Is(err, archiver_service.RepoRefNotFoundError{}) {
+			ctx.Error(http.StatusNotFound, "unrecognized reference", err)
+		} else {
+			ctx.ServerError("archiver_service.NewRequest", err)
+		}
+		return
+	}
+
+	archiver, err := aReq.Await(ctx)
+	if err != nil {
+		ctx.ServerError("archiver.Await", err)
+		return
+	}
+
+	download(ctx, aReq.GetArchiveName(), archiver)
+}
+
+func download(ctx *context.APIContext, archiveName string, archiver *repo_model.RepoArchiver) {
+	downloadName := ctx.Repo.Repository.Name + "-" + archiveName
+
+	rPath := archiver.RelativePath()
+	if setting.RepoArchive.ServeDirect {
+		// If we have a signed url (S3, object storage), redirect to this directly.
+		u, err := storage.RepoArchives.URL(rPath, downloadName)
+		if u != nil && err == nil {
+			ctx.Redirect(u.String())
+			return
+		}
+	}
+
+	// If we have matched and access to release or issue
+	fr, err := storage.RepoArchives.Open(rPath)
+	if err != nil {
+		ctx.ServerError("Open", err)
+		return
+	}
+	defer fr.Close()
+	ctx.ServeContent(downloadName, fr, archiver.CreatedUnix.AsLocalTime())
 }
 
 // GetEditorconfig get editor config of a repository

routers/api/v1/repo/hook.go

@@ -13,6 +13,7 @@ import (
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/convert"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/setting"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/web"
 	"code.gitea.io/gitea/routers/api/v1/utils"
@@ -140,7 +141,7 @@ func TestHook(ctx *context.APIContext) {
 	//   required: true
 	// - name: ref
 	//   in: query
-	//   description: "The name of the commit/branch/tag. Default the repository’s default branch (usually master)"
+	//   description: "The name of the commit/branch/tag, indicates which commit will be loaded to the webhook payload."
 	//   type: string
 	//   required: false
 	// responses:
@@ -153,6 +154,11 @@ func TestHook(ctx *context.APIContext) {
 		return
 	}
 
+	ref := git.BranchPrefix + ctx.Repo.Repository.DefaultBranch
+	if r := ctx.FormTrim("ref"); r != "" {
+		ref = r
+	}
+
 	hookID := ctx.ParamsInt64(":id")
 	hook, err := utils.GetRepoHook(ctx, ctx.Repo.Repository.ID, hookID)
 	if err != nil {
@@ -161,10 +167,12 @@ func TestHook(ctx *context.APIContext) {
 
 	commit := convert.ToPayloadCommit(ctx.Repo.Repository, ctx.Repo.Commit)
 
+	commitID := ctx.Repo.Commit.ID.String()
 	if err := webhook_service.PrepareWebhook(hook, ctx.Repo.Repository, webhook.HookEventPush, &api.PushPayload{
-		Ref:        git.BranchPrefix + ctx.Repo.Repository.DefaultBranch,
-		Before:     ctx.Repo.Commit.ID.String(),
-		After:      ctx.Repo.Commit.ID.String(),
+		Ref:        ref,
+		Before:     commitID,
+		After:      commitID,
+		CompareURL: setting.AppURL + ctx.Repo.Repository.ComposeCompareURL(commitID, commitID),
 		Commits:    []*api.PayloadCommit{commit},
 		HeadCommit: commit,
 		Repo:       convert.ToRepo(ctx.Repo.Repository, perm.AccessModeNone),

routers/api/v1/repo/notes.go

@@ -12,7 +12,6 @@ import (
 	"code.gitea.io/gitea/modules/convert"
 	"code.gitea.io/gitea/modules/git"
 	api "code.gitea.io/gitea/modules/structs"
-	"code.gitea.io/gitea/modules/validation"
 )
 
 // GetNote Get a note corresponding to a single commit from a repository
@@ -47,7 +46,7 @@ func GetNote(ctx *context.APIContext) {
 	//     "$ref": "#/responses/notFound"
 
 	sha := ctx.Params(":sha")
-	if (validation.GitRefNamePatternInvalid.MatchString(sha) || !validation.CheckGitRefAdditionalRulesValid(sha)) && !git.SHAPattern.MatchString(sha) {
+	if !git.IsValidRefPattern(sha) {
 		ctx.Error(http.StatusUnprocessableEntity, "no valid ref or sha", fmt.Sprintf("no valid ref or sha: %s", sha))
 		return
 	}

routers/api/v1/repo/release_attachment.go

@@ -57,6 +57,10 @@ func GetReleaseAttachment(ctx *context.APIContext) {
 	attachID := ctx.ParamsInt64(":asset")
 	attach, err := repo_model.GetAttachmentByID(ctx, attachID)
 	if err != nil {
+		if repo_model.IsErrAttachmentNotExist(err) {
+			ctx.NotFound()
+			return
+		}
 		ctx.Error(http.StatusInternalServerError, "GetAttachmentByID", err)
 		return
 	}
@@ -100,6 +104,10 @@ func ListReleaseAttachments(ctx *context.APIContext) {
 	releaseID := ctx.ParamsInt64(":id")
 	release, err := models.GetReleaseByID(ctx, releaseID)
 	if err != nil {
+		if models.IsErrReleaseNotExist(err) {
+			ctx.NotFound()
+			return
+		}
 		ctx.Error(http.StatusInternalServerError, "GetReleaseByID", err)
 		return
 	}
@@ -166,6 +174,10 @@ func CreateReleaseAttachment(ctx *context.APIContext) {
 	releaseID := ctx.ParamsInt64(":id")
 	release, err := models.GetReleaseByID(ctx, releaseID)
 	if err != nil {
+		if models.IsErrReleaseNotExist(err) {
+			ctx.NotFound()
+			return
+		}
 		ctx.Error(http.StatusInternalServerError, "GetReleaseByID", err)
 		return
 	}
@@ -244,6 +256,10 @@ func EditReleaseAttachment(ctx *context.APIContext) {
 	attachID := ctx.ParamsInt64(":asset")
 	attach, err := repo_model.GetAttachmentByID(ctx, attachID)
 	if err != nil {
+		if repo_model.IsErrAttachmentNotExist(err) {
+			ctx.NotFound()
+			return
+		}
 		ctx.Error(http.StatusInternalServerError, "GetAttachmentByID", err)
 		return
 	}
@@ -302,6 +318,10 @@ func DeleteReleaseAttachment(ctx *context.APIContext) {
 	attachID := ctx.ParamsInt64(":asset")
 	attach, err := repo_model.GetAttachmentByID(ctx, attachID)
 	if err != nil {
+		if repo_model.IsErrAttachmentNotExist(err) {
+			ctx.NotFound()
+			return
+		}
 		ctx.Error(http.StatusInternalServerError, "GetAttachmentByID", err)
 		return
 	}

routers/api/v1/user/gpg_key.go

@@ -7,6 +7,7 @@ package user
 import (
 	"fmt"
 	"net/http"
+	"strings"
 
 	asymkey_model "code.gitea.io/gitea/models/asymkey"
 	"code.gitea.io/gitea/models/db"
@@ -177,6 +178,12 @@ func VerifyUserGPGKey(ctx *context.APIContext) {
 	token := asymkey_model.VerificationToken(ctx.Doer, 1)
 	lastToken := asymkey_model.VerificationToken(ctx.Doer, 0)
 
+	form.KeyID = strings.TrimLeft(form.KeyID, "0")
+	if form.KeyID == "" {
+		ctx.NotFound()
+		return
+	}
+
 	_, err := asymkey_model.VerifyGPGKey(ctx.Doer.ID, form.KeyID, token, form.Signature)
 	if err != nil && asymkey_model.IsErrGPGInvalidTokenSignature(err) {
 		_, err = asymkey_model.VerifyGPGKey(ctx.Doer.ID, form.KeyID, lastToken, form.Signature)

routers/web/admin/users.go

@@ -209,7 +209,11 @@ func NewUserPost(ctx *context.Context) {
 func prepareUserInfo(ctx *context.Context) *user_model.User {
 	u, err := user_model.GetUserByID(ctx.ParamsInt64(":userid"))
 	if err != nil {
-		ctx.ServerError("GetUserByID", err)
+		if user_model.IsErrUserNotExist(err) {
+			ctx.Redirect(setting.AppSubURL + "/admin/users")
+		} else {
+			ctx.ServerError("GetUserByID", err)
+		}
 		return nil
 	}
 	ctx.Data["User"] = u

routers/web/org/teams.go

@@ -339,7 +339,7 @@ func SearchTeam(ctx *context.Context) {
 	}
 
 	opts := &organization.SearchTeamOptions{
-		UserID:      ctx.Doer.ID,
+		// UserID is not set because the router already requires the doer to be an org admin. Thus, we don't need to restrict to teams that the user belongs in
 		Keyword:     ctx.FormTrim("q"),
 		OrgID:       ctx.Org.Organization.ID,
 		IncludeDesc: ctx.FormString("include_desc") == "" || ctx.FormBool("include_desc"),

routers/web/repo/repo.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
-	"time"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/models/db"
@@ -21,7 +20,6 @@ import (
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/convert"
-	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
 	repo_module "code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
@@ -389,68 +387,27 @@ func Download(ctx *context.Context) {
 	if err != nil {
 		if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) {
 			ctx.Error(http.StatusBadRequest, err.Error())
+		} else if errors.Is(err, archiver_service.RepoRefNotFoundError{}) {
+			ctx.Error(http.StatusNotFound, err.Error())
 		} else {
 			ctx.ServerError("archiver_service.NewRequest", err)
 		}
 		return
 	}
-	if aReq == nil {
-		ctx.Error(http.StatusNotFound)
-		return
-	}
 
-	archiver, err := repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID)
+	archiver, err := aReq.Await(ctx)
 	if err != nil {
-		ctx.ServerError("models.GetRepoArchiver", err)
-		return
-	}
-	if archiver != nil && archiver.Status == repo_model.ArchiverReady {
-		download(ctx, aReq.GetArchiveName(), archiver)
+		ctx.ServerError("archiver.Await", err)
 		return
 	}
 
-	if err := archiver_service.StartArchive(aReq); err != nil {
-		ctx.ServerError("archiver_service.StartArchive", err)
-		return
-	}
-
-	var times int
-	t := time.NewTicker(time.Second * 1)
-	defer t.Stop()
-
-	for {
-		select {
-		case <-graceful.GetManager().HammerContext().Done():
-			log.Warn("exit archive download because system stop")
-			return
-		case <-t.C:
-			if times > 20 {
-				ctx.ServerError("wait download timeout", nil)
-				return
-			}
-			times++
-			archiver, err = repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID)
-			if err != nil {
-				ctx.ServerError("archiver_service.StartArchive", err)
-				return
-			}
-			if archiver != nil && archiver.Status == repo_model.ArchiverReady {
-				download(ctx, aReq.GetArchiveName(), archiver)
-				return
-			}
-		}
-	}
+	download(ctx, aReq.GetArchiveName(), archiver)
 }
 
 func download(ctx *context.Context, archiveName string, archiver *repo_model.RepoArchiver) {
 	downloadName := ctx.Repo.Repository.Name + "-" + archiveName
 
-	rPath, err := archiver.RelativePath()
-	if err != nil {
-		ctx.ServerError("archiver.RelativePath", err)
-		return
-	}
-
+	rPath := archiver.RelativePath()
 	if setting.RepoArchive.ServeDirect {
 		// If we have a signed url (S3, object storage), redirect to this directly.
 		u, err := storage.RepoArchives.URL(rPath, downloadName)

routers/web/repo/setting.go

@@ -227,14 +227,17 @@ func SettingsPost(ctx *context.Context) {
 			form.MirrorPassword, _ = u.User.Password()
 		}
 
-		err = migrations.IsMigrateURLAllowed(u.String(), ctx.Doer)
+		address, err := forms.ParseRemoteAddr(form.MirrorAddress, form.MirrorUsername, form.MirrorPassword)
+		if err == nil {
+			err = migrations.IsMigrateURLAllowed(address, ctx.Doer)
+		}
 		if err != nil {
 			ctx.Data["Err_MirrorAddress"] = true
 			handleSettingRemoteAddrError(ctx, err, form)
 			return
 		}
 
-		if err := mirror_service.UpdateAddress(ctx, ctx.Repo.Mirror, u.String()); err != nil {
+		if err := mirror_service.UpdateAddress(ctx, ctx.Repo.Mirror, address); err != nil {
 			ctx.ServerError("UpdateAddress", err)
 			return
 		}

routers/web/repo/webhook.go

@@ -1271,10 +1271,12 @@ func TestWebhook(ctx *context.Context) {
 		},
 	}
 
+	commitID := commit.ID.String()
 	p := &api.PushPayload{
 		Ref:        git.BranchPrefix + ctx.Repo.Repository.DefaultBranch,
-		Before:     commit.ID.String(),
-		After:      commit.ID.String(),
+		Before:     commitID,
+		After:      commitID,
+		CompareURL: setting.AppURL + ctx.Repo.Repository.ComposeCompareURL(commitID, commitID),
 		Commits:    []*api.PayloadCommit{apiCommit},
 		HeadCommit: apiCommit,
 		Repo:       convert.ToRepo(ctx.Repo.Repository, perm.AccessModeNone),

routers/web/user/home.go

@@ -100,39 +100,6 @@ func Dashboard(ctx *context.Context) {
 	}
 
 	var err error
-	var mirrors []*repo_model.Repository
-	if ctxUser.IsOrganization() {
-		var env organization.AccessibleReposEnvironment
-		if ctx.Org.Team != nil {
-			env = organization.OrgFromUser(ctxUser).AccessibleTeamReposEnv(ctx.Org.Team)
-		} else {
-			env, err = organization.AccessibleReposEnv(ctx, organization.OrgFromUser(ctxUser), ctx.Doer.ID)
-			if err != nil {
-				ctx.ServerError("AccessibleReposEnv", err)
-				return
-			}
-		}
-		mirrors, err = env.MirrorRepos()
-		if err != nil {
-			ctx.ServerError("env.MirrorRepos", err)
-			return
-		}
-	} else {
-		mirrors, err = repo_model.GetUserMirrorRepositories(ctxUser.ID)
-		if err != nil {
-			ctx.ServerError("GetUserMirrorRepositories", err)
-			return
-		}
-	}
-	ctx.Data["MaxShowRepoNum"] = setting.UI.User.RepoPagingNum
-
-	if err := repo_model.MirrorRepositoryList(mirrors).LoadAttributes(); err != nil {
-		ctx.ServerError("MirrorRepositoryList.LoadAttributes", err)
-		return
-	}
-	ctx.Data["MirrorCount"] = len(mirrors)
-	ctx.Data["Mirrors"] = mirrors
-
 	ctx.Data["Feeds"], err = models.GetFeeds(ctx, models.GetFeedsOptions{
 		RequestedUser:   ctxUser,
 		RequestedTeam:   ctx.Org.Team,

services/migrations/codebase.go

@@ -107,9 +107,24 @@ func NewCodebaseDownloader(ctx context.Context, projectURL *url.URL, project, re
 		commitMap: make(map[string]string),
 	}
 
+	log.Trace("Create Codebase downloader. BaseURL: %s Project: %s RepoName: %s", baseURL, project, repoName)
 	return downloader
 }
 
+// String implements Stringer
+func (d *CodebaseDownloader) String() string {
+	return fmt.Sprintf("migration from codebase server %s %s/%s", d.baseURL, d.project, d.repoName)
+}
+
+// ColorFormat provides a basic color format for a GogsDownloader
+func (d *CodebaseDownloader) ColorFormat(s fmt.State) {
+	if d == nil {
+		log.ColorFprintf(s, "<nil: CodebaseDownloader>")
+		return
+	}
+	log.ColorFprintf(s, "migration from codebase server %s %s/%s", d.baseURL, d.project, d.repoName)
+}
+
 // FormatCloneURL add authentication into remote URLs
 func (d *CodebaseDownloader) FormatCloneURL(opts base.MigrateOptions, remoteAddr string) (string, error) {
 	return opts.CloneAddr, nil
@@ -451,8 +466,8 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq
 				Value int64  `xml:",chardata"`
 				Type  string `xml:"type,attr"`
 			} `xml:"id"`
-			SourceRef string `xml:"source-ref"`
-			TargetRef string `xml:"target-ref"`
+			SourceRef string `xml:"source-ref"` // NOTE: from the documentation these are actually just branches NOT full refs
+			TargetRef string `xml:"target-ref"` // NOTE: from the documentation these are actually just branches NOT full refs
 			Subject   string `xml:"subject"`
 			Status    string `xml:"status"`
 			UserID    struct {
@@ -564,6 +579,9 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq
 				Comments: comments[1:],
 			},
 		})
+
+		// SECURITY: Ensure that the PR is safe
+		_ = CheckAndEnsureSafePR(pullRequests[len(pullRequests)-1], d.baseURL.String(), d)
 	}
 
 	return pullRequests, true, nil

services/migrations/common.go

@@ -0,0 +1,82 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+	"strings"
+
+	admin_model "code.gitea.io/gitea/models/admin"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+	base "code.gitea.io/gitea/modules/migration"
+)
+
+// WarnAndNotice will log the provided message and send a repository notice
+func WarnAndNotice(fmtStr string, args ...interface{}) {
+	log.Warn(fmtStr, args...)
+	if err := admin_model.CreateRepositoryNotice(fmt.Sprintf(fmtStr, args...)); err != nil {
+		log.Error("create repository notice failed: ", err)
+	}
+}
+
+func hasBaseURL(toCheck, baseURL string) bool {
+	if len(baseURL) > 0 && baseURL[len(baseURL)-1] != '/' {
+		baseURL += "/"
+	}
+	return strings.HasPrefix(toCheck, baseURL)
+}
+
+// CheckAndEnsureSafePR will check that a given PR is safe to download
+func CheckAndEnsureSafePR(pr *base.PullRequest, commonCloneBaseURL string, g base.Downloader) bool {
+	valid := true
+	// SECURITY: the patchURL must be checked to have the same baseURL as the current to prevent open redirect
+	if pr.PatchURL != "" && !hasBaseURL(pr.PatchURL, commonCloneBaseURL) {
+		// TODO: Should we check that this url has the expected format for a patch url?
+		WarnAndNotice("PR #%d in %s has invalid PatchURL: %s baseURL: %s", pr.Number, g, pr.PatchURL, commonCloneBaseURL)
+		pr.PatchURL = ""
+		valid = false
+	}
+
+	// SECURITY: the headCloneURL must be checked to have the same baseURL as the current to prevent open redirect
+	if pr.Head.CloneURL != "" && !hasBaseURL(pr.Head.CloneURL, commonCloneBaseURL) {
+		// TODO: Should we check that this url has the expected format for a patch url?
+		WarnAndNotice("PR #%d in %s has invalid HeadCloneURL: %s baseURL: %s", pr.Number, g, pr.Head.CloneURL, commonCloneBaseURL)
+		pr.Head.CloneURL = ""
+		valid = false
+	}
+
+	// SECURITY: SHAs Must be a SHA
+	if pr.MergeCommitSHA != "" && !git.IsValidSHAPattern(pr.MergeCommitSHA) {
+		WarnAndNotice("PR #%d in %s has invalid MergeCommitSHA: %s", pr.Number, g, pr.MergeCommitSHA)
+		pr.MergeCommitSHA = ""
+	}
+	if pr.Head.SHA != "" && !git.IsValidSHAPattern(pr.Head.SHA) {
+		WarnAndNotice("PR #%d in %s has invalid HeadSHA: %s", pr.Number, g, pr.Head.SHA)
+		pr.Head.SHA = ""
+		valid = false
+	}
+	if pr.Base.SHA != "" && !git.IsValidSHAPattern(pr.Base.SHA) {
+		WarnAndNotice("PR #%d in %s has invalid BaseSHA: %s", pr.Number, g, pr.Base.SHA)
+		pr.Base.SHA = ""
+		valid = false
+	}
+
+	// SECURITY: Refs must be valid refs or SHAs
+	if pr.Head.Ref != "" && !git.IsValidRefPattern(pr.Head.Ref) {
+		WarnAndNotice("PR #%d in %s has invalid HeadRef: %s", pr.Number, g, pr.Head.Ref)
+		pr.Head.Ref = ""
+		valid = false
+	}
+	if pr.Base.Ref != "" && !git.IsValidRefPattern(pr.Base.Ref) {
+		WarnAndNotice("PR #%d in %s has invalid BaseRef: %s", pr.Number, g, pr.Base.Ref)
+		pr.Base.Ref = ""
+		valid = false
+	}
+
+	pr.EnsuredSafe = true
+
+	return valid
+}

services/migrations/dump.go

@@ -12,7 +12,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -26,6 +25,8 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 
+	"github.com/google/uuid"
+
 	"gopkg.in/yaml.v2"
 )
 
@@ -47,7 +48,7 @@ type RepositoryDumper struct {
 	reviewFiles     map[int64]*os.File
 
 	gitRepo     *git.Repository
-	prHeadCache map[string]struct{}
+	prHeadCache map[string]string
 }
 
 // NewRepositoryDumper creates an gitea Uploader
@@ -62,7 +63,7 @@ func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName strin
 		baseDir:      baseDir,
 		repoOwner:    repoOwner,
 		repoName:     repoName,
-		prHeadCache:  make(map[string]struct{}),
+		prHeadCache:  make(map[string]string),
 		commentFiles: make(map[int64]*os.File),
 		reviewFiles:  make(map[int64]*os.File),
 	}, nil
@@ -296,8 +297,10 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error {
 			}
 			for _, asset := range release.Assets {
 				attachLocalPath := filepath.Join(attachDir, asset.Name)
-				// download attachment
 
+				// SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here
+				// ... we must assume that they are safe and simply download the attachment
+				// download attachment
 				err := func(attachPath string) error {
 					var rc io.ReadCloser
 					var err error
@@ -317,7 +320,7 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error {
 
 					fw, err := os.Create(attachPath)
 					if err != nil {
-						return fmt.Errorf("Create: %v", err)
+						return fmt.Errorf("create: %w", err)
 					}
 					defer fw.Close()
 
@@ -385,22 +388,7 @@ func (g *RepositoryDumper) createItems(dir string, itemFiles map[int64]*os.File,
 	}
 
 	for number, items := range itemsMap {
-		var err error
-		itemFile := itemFiles[number]
-		if itemFile == nil {
-			itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number)))
-			if err != nil {
-				return err
-			}
-			itemFiles[number] = itemFile
-		}
-
-		bs, err := yaml.Marshal(items)
-		if err != nil {
-			return err
-		}
-
-		if _, err := itemFile.Write(bs); err != nil {
+		if err := g.encodeItems(number, items, dir, itemFiles); err != nil {
 			return err
 		}
 	}
@@ -408,6 +396,23 @@ func (g *RepositoryDumper) createItems(dir string, itemFiles map[int64]*os.File,
 	return nil
 }
 
+func (g *RepositoryDumper) encodeItems(number int64, items []interface{}, dir string, itemFiles map[int64]*os.File) error {
+	itemFile := itemFiles[number]
+	if itemFile == nil {
+		var err error
+		itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number)))
+		if err != nil {
+			return err
+		}
+		itemFiles[number] = itemFile
+	}
+
+	encoder := yaml.NewEncoder(itemFile)
+	defer encoder.Close()
+
+	return encoder.Encode(items)
+}
+
 // CreateComments creates comments of issues
 func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error {
 	commentsMap := make(map[int64][]interface{}, len(comments))
@@ -418,102 +423,175 @@ func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error {
 	return g.createItems(g.commentDir(), g.commentFiles, commentsMap)
 }
 
-// CreatePullRequests creates pull requests
-func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
-	for _, pr := range prs {
-		// download patch file
-		err := func() error {
-			u, err := g.setURLToken(pr.PatchURL)
-			if err != nil {
-				return err
-			}
-			resp, err := http.Get(u)
-			if err != nil {
-				return err
-			}
-			defer resp.Body.Close()
-			pullDir := filepath.Join(g.gitPath(), "pulls")
-			if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
-				return err
-			}
-			fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))
-			f, err := os.Create(fPath)
-			if err != nil {
-				return err
-			}
-			defer f.Close()
-			if _, err = io.Copy(f, resp.Body); err != nil {
-				return err
-			}
-			pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number)
-
-			return nil
-		}()
-		if err != nil {
-			return err
-		}
-
-		// set head information
-		pullHead := filepath.Join(g.gitPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
-		if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
-			return err
-		}
-		p, err := os.Create(filepath.Join(pullHead, "head"))
-		if err != nil {
-			return err
-		}
-		_, err = p.WriteString(pr.Head.SHA)
-		p.Close()
-		if err != nil {
-			return err
-		}
-
-		if pr.IsForkPullRequest() && pr.State != "closed" {
-			if pr.Head.OwnerName != "" {
-				remote := pr.Head.OwnerName
-				_, ok := g.prHeadCache[remote]
-				if !ok {
-					// git remote add
-					// TODO: how to handle private CloneURL?
-					err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
-					if err != nil {
-						log.Error("AddRemote failed: %s", err)
-					} else {
-						g.prHeadCache[remote] = struct{}{}
-						ok = true
-					}
-				}
-
-				if ok {
-					_, _, err = git.NewCommand(g.ctx, "fetch", remote, pr.Head.Ref).RunStdString(&git.RunOpts{Dir: g.gitPath()})
-					if err != nil {
-						log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
-					} else {
-						// a new branch name with <original_owner_name/original_branchname> will be created to as new head branch
-						ref := path.Join(pr.Head.OwnerName, pr.Head.Ref)
-						headBranch := filepath.Join(g.gitPath(), "refs", "heads", ref)
-						if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
-							return err
-						}
-						b, err := os.Create(headBranch)
-						if err != nil {
-							return err
-						}
-						_, err = b.WriteString(pr.Head.SHA)
-						b.Close()
-						if err != nil {
-							return err
-						}
-						pr.Head.Ref = ref
-					}
-				}
-			}
-		}
-		// whatever it's a forked repo PR, we have to change head info as the same as the base info
-		pr.Head.OwnerName = pr.Base.OwnerName
-		pr.Head.RepoName = pr.Base.RepoName
+func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error {
+	// SECURITY: this pr must have been ensured safe
+	if !pr.EnsuredSafe {
+		log.Error("PR #%d in %s/%s has not been checked for safety ... We will ignore this.", pr.Number, g.repoOwner, g.repoName)
+		return fmt.Errorf("unsafe PR #%d", pr.Number)
 	}
 
+	// First we download the patch file
+	err := func() error {
+		// if the patchURL is empty there is nothing to download
+		if pr.PatchURL == "" {
+			return nil
+		}
+
+		// SECURITY: We will assume that the pr.PatchURL has been checked
+		// pr.PatchURL maybe a local file - but note EnsureSafe should be asserting that this safe
+		u, err := g.setURLToken(pr.PatchURL)
+		if err != nil {
+			return err
+		}
+
+		// SECURITY: We will assume that the pr.PatchURL has been checked
+		// pr.PatchURL maybe a local file - but note EnsureSafe should be asserting that this safe
+		resp, err := http.Get(u) // TODO: This probably needs to use the downloader as there may be rate limiting issues here
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		pullDir := filepath.Join(g.gitPath(), "pulls")
+		if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
+			return err
+		}
+		fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))
+		f, err := os.Create(fPath)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		// TODO: Should there be limits on the size of this file?
+		if _, err = io.Copy(f, resp.Body); err != nil {
+			return err
+		}
+		pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number)
+
+		return nil
+	}()
+	if err != nil {
+		log.Error("PR #%d in %s/%s unable to download patch: %v", pr.Number, g.repoOwner, g.repoName, err)
+		return err
+	}
+
+	isFork := pr.IsForkPullRequest()
+
+	// Even if it's a forked repo PR, we have to change head info as the same as the base info
+	oldHeadOwnerName := pr.Head.OwnerName
+	pr.Head.OwnerName, pr.Head.RepoName = pr.Base.OwnerName, pr.Base.RepoName
+
+	if !isFork || pr.State == "closed" {
+		return nil
+	}
+
+	// OK we want to fetch the current head as a branch from its CloneURL
+
+	// 1. Is there a head clone URL available?
+	// 2. Is there a head ref available?
+	if pr.Head.CloneURL == "" || pr.Head.Ref == "" {
+		// Set head information if pr.Head.SHA is available
+		if pr.Head.SHA != "" {
+			_, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref", pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()})
+			if err != nil {
+				log.Error("PR #%d in %s/%s unable to update-ref for pr HEAD: %v", pr.Number, g.repoOwner, g.repoName, err)
+			}
+		}
+		return nil
+	}
+
+	// 3. We need to create a remote for this clone url
+	// ... maybe we already have a name for this remote
+	remote, ok := g.prHeadCache[pr.Head.CloneURL+":"]
+	if !ok {
+		// ... let's try ownername as a reasonable name
+		remote = oldHeadOwnerName
+		if !git.IsValidRefPattern(remote) {
+			// ... let's try something less nice
+			remote = "head-pr-" + strconv.FormatInt(pr.Number, 10)
+		}
+		// ... now add the remote
+		err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
+		if err != nil {
+			log.Error("PR #%d in %s/%s AddRemote[%s] failed: %v", pr.Number, g.repoOwner, g.repoName, remote, err)
+		} else {
+			g.prHeadCache[pr.Head.CloneURL+":"] = remote
+			ok = true
+		}
+	}
+	if !ok {
+		// Set head information if pr.Head.SHA is available
+		if pr.Head.SHA != "" {
+			_, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref", pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()})
+			if err != nil {
+				log.Error("PR #%d in %s/%s unable to update-ref for pr HEAD: %v", pr.Number, g.repoOwner, g.repoName, err)
+			}
+		}
+
+		return nil
+	}
+
+	// 4. Check if we already have this ref?
+	localRef, ok := g.prHeadCache[pr.Head.CloneURL+":"+pr.Head.Ref]
+	if !ok {
+		// ... We would normally name this migrated branch as <OwnerName>/<HeadRef> but we need to ensure that is safe
+		localRef = git.SanitizeRefPattern(oldHeadOwnerName + "/" + pr.Head.Ref)
+
+		// ... Now we must assert that this does not exist
+		if g.gitRepo.IsBranchExist(localRef) {
+			localRef = "head-pr-" + strconv.FormatInt(pr.Number, 10) + "/" + localRef
+			i := 0
+			for g.gitRepo.IsBranchExist(localRef) {
+				if i > 5 {
+					// ... We tried, we really tried but this is just a seriously unfriendly repo
+					return fmt.Errorf("unable to create unique local reference from %s", pr.Head.Ref)
+				}
+				// OK just try some uuids!
+				localRef = git.SanitizeRefPattern("head-pr-" + strconv.FormatInt(pr.Number, 10) + uuid.New().String())
+				i++
+			}
+		}
+
+		fetchArg := pr.Head.Ref + ":" + git.BranchPrefix + localRef
+		if strings.HasPrefix(fetchArg, "-") {
+			fetchArg = git.BranchPrefix + fetchArg
+		}
+
+		_, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags", "--", remote, fetchArg).RunStdString(&git.RunOpts{Dir: g.gitPath()})
+		if err != nil {
+			log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
+			// We need to continue here so that the Head.Ref is reset and we attempt to set the gitref for the PR
+			// (This last step will likely fail but we should try to do as much as we can.)
+		} else {
+			// Cache the localRef as the Head.Ref - if we've failed we can always try again.
+			g.prHeadCache[pr.Head.CloneURL+":"+pr.Head.Ref] = localRef
+		}
+	}
+
+	// Set the pr.Head.Ref to the localRef
+	pr.Head.Ref = localRef
+
+	// 5. Now if pr.Head.SHA == "" we should recover this to the head of this branch
+	if pr.Head.SHA == "" {
+		headSha, err := g.gitRepo.GetBranchCommitID(localRef)
+		if err != nil {
+			log.Error("unable to get head SHA of local head for PR #%d from %s in %s/%s. Error: %v", pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err)
+			return nil
+		}
+		pr.Head.SHA = headSha
+	}
+	if pr.Head.SHA != "" {
+		_, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref", pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()})
+		if err != nil {
+			log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err)
+		}
+	}
+
+	return nil
+}
+
+// CreatePullRequests creates pull requests
+func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
 	var err error
 	if g.pullrequestFile == nil {
 		if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil {
@@ -525,16 +603,22 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
 		}
 	}
 
-	bs, err := yaml.Marshal(prs)
-	if err != nil {
-		return err
-	}
+	encoder := yaml.NewEncoder(g.pullrequestFile)
+	defer encoder.Close()
 
-	if _, err := g.pullrequestFile.Write(bs); err != nil {
-		return err
+	count := 0
+	for i := 0; i < len(prs); i++ {
+		pr := prs[i]
+		if err := g.handlePullRequest(pr); err != nil {
+			log.Error("PR #%d in %s/%s failed - skipping", pr.Number, g.repoOwner, g.repoName, err)
+			continue
+		}
+		prs[count] = pr
+		count++
 	}
+	prs = prs[:count]
 
-	return nil
+	return encoder.Encode(prs)
 }
 
 // CreateReviews create pull request reviews
@@ -560,6 +644,10 @@ func (g *RepositoryDumper) Finish() error {
 
 // DumpRepository dump repository according MigrateOptions to a local directory
 func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error {
+	doer, err := user_model.GetAdminUser()
+	if err != nil {
+		return err
+	}
 	downloader, err := newDownloader(ctx, ownerName, opts)
 	if err != nil {
 		return err
@@ -569,7 +657,7 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi
 		return err
 	}
 
-	if err := migrateRepository(downloader, uploader, opts, nil); err != nil {
+	if err := migrateRepository(doer, downloader, uploader, opts, nil); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)
 		}
@@ -641,7 +729,7 @@ func RestoreRepository(ctx context.Context, baseDir, ownerName, repoName string,
 		return err
 	}
 
-	if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil {
+	if err = migrateRepository(doer, downloader, uploader, migrateOpts, nil); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)
 		}

services/migrations/gitbucket.go

@@ -6,9 +6,11 @@ package migrations
 
 import (
 	"context"
+	"fmt"
 	"net/url"
 	"strings"
 
+	"code.gitea.io/gitea/modules/log"
 	base "code.gitea.io/gitea/modules/migration"
 	"code.gitea.io/gitea/modules/structs"
 )
@@ -37,6 +39,7 @@ func (f *GitBucketDownloaderFactory) New(ctx context.Context, opts base.MigrateO
 	oldOwner := fields[1]
 	oldName := strings.TrimSuffix(fields[2], ".git")
 
+	log.Trace("Create GitBucket downloader. BaseURL: %s RepoOwner: %s RepoName: %s", baseURL, oldOwner, oldName)
 	return NewGitBucketDownloader(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
 }
 
@@ -51,6 +54,20 @@ type GitBucketDownloader struct {
 	*GithubDownloaderV3
 }
 
+// String implements Stringer
+func (g *GitBucketDownloader) String() string {
+	return fmt.Sprintf("migration from gitbucket server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
+}
+
+// ColorFormat provides a basic color format for a GitBucketDownloader
+func (g *GitBucketDownloader) ColorFormat(s fmt.State) {
+	if g == nil {
+		log.ColorFprintf(s, "<nil: GitBucketDownloader>")
+		return
+	}
+	log.ColorFprintf(s, "migration from gitbucket server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
+}
+
 // NewGitBucketDownloader creates a GitBucket downloader
 func NewGitBucketDownloader(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GitBucketDownloader {
 	githubDownloader := NewGithubDownloaderV3(ctx, baseURL, userName, password, token, repoOwner, repoName)

services/migrations/gitea_downloader.go

@@ -14,7 +14,6 @@ import (
 	"strings"
 	"time"
 
-	admin_model "code.gitea.io/gitea/models/admin"
 	"code.gitea.io/gitea/modules/log"
 	base "code.gitea.io/gitea/modules/migration"
 	"code.gitea.io/gitea/modules/structs"
@@ -71,6 +70,7 @@ type GiteaDownloader struct {
 	base.NullDownloader
 	ctx        context.Context
 	client     *gitea_sdk.Client
+	baseURL    string
 	repoOwner  string
 	repoName   string
 	pagination bool
@@ -116,6 +116,7 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
 	return &GiteaDownloader{
 		ctx:        ctx,
 		client:     giteaClient,
+		baseURL:    baseURL,
 		repoOwner:  path[0],
 		repoName:   path[1],
 		pagination: paginationSupport,
@@ -128,6 +129,20 @@ func (g *GiteaDownloader) SetContext(ctx context.Context) {
 	g.ctx = ctx
 }
 
+// String implements Stringer
+func (g *GiteaDownloader) String() string {
+	return fmt.Sprintf("migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
+}
+
+// ColorFormat provides a basic color format for a GiteaDownloader
+func (g *GiteaDownloader) ColorFormat(s fmt.State) {
+	if g == nil {
+		log.ColorFprintf(s, "<nil: GiteaDownloader>")
+		return
+	}
+	log.ColorFprintf(s, "migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
+}
+
 // GetRepoInfo returns a repository information
 func (g *GiteaDownloader) GetRepoInfo() (*base.Repository, error) {
 	if g == nil {
@@ -283,6 +298,12 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
 				if err != nil {
 					return nil, err
 				}
+
+				if !hasBaseURL(asset.DownloadURL, g.baseURL) {
+					WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", asset.ID, g, asset.DownloadURL)
+					return io.NopCloser(strings.NewReader(asset.DownloadURL)), nil
+				}
+
 				// FIXME: for a private download?
 				req, err := http.NewRequest("GET", asset.DownloadURL, nil)
 				if err != nil {
@@ -402,11 +423,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err
 
 		reactions, err := g.getIssueReactions(issue.Index)
 		if err != nil {
-			log.Warn("Unable to load reactions during migrating issue #%d to %s/%s. Error: %v", issue.Index, g.repoOwner, g.repoName, err)
-			if err2 := admin_model.CreateRepositoryNotice(
-				fmt.Sprintf("Unable to load reactions during migrating issue #%d to %s/%s. Error: %v", issue.Index, g.repoOwner, g.repoName, err)); err2 != nil {
-				log.Error("create repository notice failed: ", err2)
-			}
+			WarnAndNotice("Unable to load reactions during migrating issue #%d in %s. Error: %v", issue.Index, g, err)
 		}
 
 		var assignees []string
@@ -464,11 +481,7 @@ func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Com
 		for _, comment := range comments {
 			reactions, err := g.getCommentReactions(comment.ID)
 			if err != nil {
-				log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", commentable.GetForeignIndex(), comment.ID, g.repoOwner, g.repoName, err)
-				if err2 := admin_model.CreateRepositoryNotice(
-					fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", commentable.GetForeignIndex(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil {
-					log.Error("create repository notice failed: ", err2)
-				}
+				WarnAndNotice("Unable to load comment reactions during migrating issue #%d for comment %d in %s. Error: %v", commentable.GetForeignIndex(), comment.ID, g, err)
 			}
 
 			allComments = append(allComments, &base.Comment{
@@ -543,11 +556,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques
 
 		reactions, err := g.getIssueReactions(pr.Index)
 		if err != nil {
-			log.Warn("Unable to load reactions during migrating pull #%d to %s/%s. Error: %v", pr.Index, g.repoOwner, g.repoName, err)
-			if err2 := admin_model.CreateRepositoryNotice(
-				fmt.Sprintf("Unable to load reactions during migrating pull #%d to %s/%s. Error: %v", pr.Index, g.repoOwner, g.repoName, err)); err2 != nil {
-				log.Error("create repository notice failed: ", err2)
-			}
+			WarnAndNotice("Unable to load reactions during migrating pull #%d in %s. Error: %v", pr.Index, g, err)
 		}
 
 		var assignees []string
@@ -604,6 +613,8 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques
 			},
 			ForeignIndex: pr.Index,
 		})
+		// SECURITY: Ensure that the PR is safe
+		_ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g)
 	}
 
 	isEnd := len(prs) < perPage

services/migrations/gitea_uploader.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -32,7 +33,7 @@ import (
 	"code.gitea.io/gitea/modules/uri"
 	"code.gitea.io/gitea/services/pull"
 
-	gouuid "github.com/google/uuid"
+	"github.com/google/uuid"
 )
 
 var _ base.Uploader = &GiteaLocalUploader{}
@@ -48,7 +49,7 @@ type GiteaLocalUploader struct {
 	milestones     map[string]int64
 	issues         map[int64]*issues_model.Issue
 	gitRepo        *git.Repository
-	prHeadCache    map[string]struct{}
+	prHeadCache    map[string]string
 	sameApp        bool
 	userMap        map[int64]int64 // external user id mapping to user id
 	prCache        map[int64]*issues_model.PullRequest
@@ -65,7 +66,7 @@ func NewGiteaLocalUploader(ctx context.Context, doer *user_model.User, repoOwner
 		labels:      make(map[string]*issues_model.Label),
 		milestones:  make(map[string]int64),
 		issues:      make(map[int64]*issues_model.Issue),
-		prHeadCache: make(map[string]struct{}),
+		prHeadCache: make(map[string]string),
 		userMap:     make(map[int64]int64),
 		prCache:     make(map[int64]*issues_model.PullRequest),
 	}
@@ -125,7 +126,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
 		Mirror:         repo.IsMirror,
 		LFS:            opts.LFS,
 		LFSEndpoint:    opts.LFSEndpoint,
-		CloneAddr:      repo.CloneURL,
+		CloneAddr:      repo.CloneURL, // SECURITY: we will assume that this has already been checked
 		Private:        repo.IsPrivate,
 		Wiki:           opts.Wiki,
 		Releases:       opts.Releases, // if didn't get releases, then sync them from tags
@@ -150,13 +151,15 @@ func (g *GiteaLocalUploader) Close() {
 
 // CreateTopics creates topics
 func (g *GiteaLocalUploader) CreateTopics(topics ...string) error {
-	// ignore topics to long for the db
+	// Ignore topics too long for the db
 	c := 0
-	for i := range topics {
-		if len(topics[i]) <= 50 {
-			topics[c] = topics[i]
-			c++
+	for _, topic := range topics {
+		if len(topic) > 50 {
+			continue
 		}
+
+		topics[c] = topic
+		c++
 	}
 	topics = topics[:c]
 	return repo_model.SaveTopics(g.repo.ID, topics...)
@@ -217,11 +220,17 @@ func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) err
 func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
 	lbs := make([]*issues_model.Label, 0, len(labels))
 	for _, label := range labels {
+		// We must validate color here:
+		if !issues_model.LabelColorPattern.MatchString("#" + label.Color) {
+			log.Warn("Invalid label color: #%s for label: %s in migration to %s/%s", label.Color, label.Name, g.repoOwner, g.repoName)
+			label.Color = "ffffff"
+		}
+
 		lbs = append(lbs, &issues_model.Label{
 			RepoID:      g.repo.ID,
 			Name:        label.Name,
 			Description: label.Description,
-			Color:       fmt.Sprintf("#%s", label.Color),
+			Color:       "#" + label.Color,
 		})
 	}
 
@@ -247,6 +256,16 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 			}
 		}
 
+		// SECURITY: The TagName must be a valid git ref
+		if release.TagName != "" && !git.IsValidRefPattern(release.TagName) {
+			release.TagName = ""
+		}
+
+		// SECURITY: The TargetCommitish must be a valid git ref
+		if release.TargetCommitish != "" && !git.IsValidRefPattern(release.TargetCommitish) {
+			release.TargetCommitish = ""
+		}
+
 		rel := models.Release{
 			RepoID:       g.repo.ID,
 			TagName:      release.TagName,
@@ -288,14 +307,15 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 				}
 			}
 			attach := repo_model.Attachment{
-				UUID:          gouuid.New().String(),
+				UUID:          uuid.New().String(),
 				Name:          asset.Name,
 				DownloadCount: int64(*asset.DownloadCount),
 				Size:          int64(*asset.Size),
 				CreatedUnix:   timeutil.TimeStamp(asset.Created.Unix()),
 			}
 
-			// download attachment
+			// SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here
+			// ... we must assume that they are safe and simply download the attachment
 			err := func() error {
 				// asset.DownloadURL maybe a local file
 				var rc io.ReadCloser
@@ -365,6 +385,12 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
 			}
 		}
 
+		// SECURITY: issue.Ref needs to be a valid reference
+		if !git.IsValidRefPattern(issue.Ref) {
+			log.Warn("Invalid issue.Ref[%s] in issue #%d in %s/%s", issue.Ref, issue.Number, g.repoOwner, g.repoName)
+			issue.Ref = ""
+		}
+
 		is := issues_model.Issue{
 			RepoID:      g.repo.ID,
 			Repo:        g.repo,
@@ -496,102 +522,151 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error
 }
 
 func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) {
-	// download patch file
+	// SECURITY: this pr must have been must have been ensured safe
+	if !pr.EnsuredSafe {
+		log.Error("PR #%d in %s/%s has not been checked for safety.", pr.Number, g.repoOwner, g.repoName)
+		return "", fmt.Errorf("the PR[%d] was not checked for safety", pr.Number)
+	}
+
+	// Anonymous function to download the patch file (allows us to use defer)
 	err = func() error {
+		// if the patchURL is empty there is nothing to download
 		if pr.PatchURL == "" {
 			return nil
 		}
-		// pr.PatchURL maybe a local file
-		ret, err := uri.Open(pr.PatchURL)
+
+		// SECURITY: We will assume that the pr.PatchURL has been checked
+		// pr.PatchURL maybe a local file - but note EnsureSafe should be asserting that this safe
+		ret, err := uri.Open(pr.PatchURL) // TODO: This probably needs to use the downloader as there may be rate limiting issues here
 		if err != nil {
 			return err
 		}
 		defer ret.Close()
+
 		pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
 		if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
 			return err
 		}
+
 		f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)))
 		if err != nil {
 			return err
 		}
 		defer f.Close()
+
+		// TODO: Should there be limits on the size of this file?
 		_, err = io.Copy(f, ret)
+
 		return err
 	}()
 	if err != nil {
 		return "", err
 	}
 
-	// set head information
-	pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
-	if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
-		return "", err
-	}
-	p, err := os.Create(filepath.Join(pullHead, "head"))
-	if err != nil {
-		return "", err
-	}
-	_, err = p.WriteString(pr.Head.SHA)
-	p.Close()
-	if err != nil {
-		return "", err
-	}
-
 	head = "unknown repository"
 	if pr.IsForkPullRequest() && pr.State != "closed" {
-		if pr.Head.OwnerName != "" {
-			remote := pr.Head.OwnerName
-			_, ok := g.prHeadCache[remote]
-			if !ok {
-				// git remote add
-				err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
-				if err != nil {
-					log.Error("AddRemote failed: %s", err)
-				} else {
-					g.prHeadCache[remote] = struct{}{}
-					ok = true
+		// OK we want to fetch the current head as a branch from its CloneURL
+
+		// 1. Is there a head clone URL available?
+		// 2. Is there a head ref available?
+		if pr.Head.CloneURL == "" || pr.Head.Ref == "" {
+			return head, nil
+		}
+
+		// 3. We need to create a remote for this clone url
+		// ... maybe we already have a name for this remote
+		remote, ok := g.prHeadCache[pr.Head.CloneURL+":"]
+		if !ok {
+			// ... let's try ownername as a reasonable name
+			remote = pr.Head.OwnerName
+			if !git.IsValidRefPattern(remote) {
+				// ... let's try something less nice
+				remote = "head-pr-" + strconv.FormatInt(pr.Number, 10)
+			}
+			// ... now add the remote
+			err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
+			if err != nil {
+				log.Error("PR #%d in %s/%s AddRemote[%s] failed: %v", pr.Number, g.repoOwner, g.repoName, remote, err)
+			} else {
+				g.prHeadCache[pr.Head.CloneURL+":"] = remote
+				ok = true
+			}
+		}
+		if !ok {
+			return head, nil
+		}
+
+		// 4. Check if we already have this ref?
+		localRef, ok := g.prHeadCache[pr.Head.CloneURL+":"+pr.Head.Ref]
+		if !ok {
+			// ... We would normally name this migrated branch as <OwnerName>/<HeadRef> but we need to ensure that is safe
+			localRef = git.SanitizeRefPattern(pr.Head.OwnerName + "/" + pr.Head.Ref)
+
+			// ... Now we must assert that this does not exist
+			if g.gitRepo.IsBranchExist(localRef) {
+				localRef = "head-pr-" + strconv.FormatInt(pr.Number, 10) + "/" + localRef
+				i := 0
+				for g.gitRepo.IsBranchExist(localRef) {
+					if i > 5 {
+						// ... We tried, we really tried but this is just a seriously unfriendly repo
+						return head, nil
+					}
+					// OK just try some uuids!
+					localRef = git.SanitizeRefPattern("head-pr-" + strconv.FormatInt(pr.Number, 10) + uuid.New().String())
+					i++
 				}
 			}
 
-			if ok {
-				_, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags", "--", remote, pr.Head.Ref).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()})
-				if err != nil {
-					log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
-				} else {
-					headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
-					if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
-						return "", err
-					}
-					b, err := os.Create(headBranch)
-					if err != nil {
-						return "", err
-					}
-					_, err = b.WriteString(pr.Head.SHA)
-					b.Close()
-					if err != nil {
-						return "", err
-					}
-					head = pr.Head.OwnerName + "/" + pr.Head.Ref
-				}
+			fetchArg := pr.Head.Ref + ":" + git.BranchPrefix + localRef
+			if strings.HasPrefix(fetchArg, "-") {
+				fetchArg = git.BranchPrefix + fetchArg
 			}
+
+			_, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags", "--", remote, fetchArg).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()})
+			if err != nil {
+				log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
+				return head, nil
+			}
+			g.prHeadCache[pr.Head.CloneURL+":"+pr.Head.Ref] = localRef
+			head = localRef
 		}
-	} else {
+
+		// 5. Now if pr.Head.SHA == "" we should recover this to the head of this branch
+		if pr.Head.SHA == "" {
+			headSha, err := g.gitRepo.GetBranchCommitID(localRef)
+			if err != nil {
+				log.Error("unable to get head SHA of local head for PR #%d from %s in %s/%s. Error: %v", pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err)
+				return head, nil
+			}
+			pr.Head.SHA = headSha
+		}
+
+		_, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref", pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()})
+		if err != nil {
+			return "", err
+		}
+
+		return head, nil
+	}
+
+	if pr.Head.Ref != "" {
 		head = pr.Head.Ref
-		// Ensure the closed PR SHA still points to an existing ref
+	}
+
+	// Ensure the closed PR SHA still points to an existing ref
+	if pr.Head.SHA == "" {
+		// The SHA is empty
+		log.Warn("Empty reference, no pull head for PR #%d in %s/%s", pr.Number, g.repoOwner, g.repoName)
+	} else {
 		_, _, err = git.NewCommand(g.ctx, "rev-list", "--quiet", "-1", pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()})
 		if err != nil {
-			if pr.Head.SHA != "" {
-				// Git update-ref remove bad references with a relative path
-				log.Warn("Deprecated local head, removing : %v", pr.Head.SHA)
-				err = g.gitRepo.RemoveReference(pr.GetGitRefName())
-			} else {
-				// The SHA is empty, remove the head file
-				log.Warn("Empty reference, removing : %v", pullHead)
-				err = os.Remove(filepath.Join(pullHead, "head"))
-			}
+			// Git update-ref remove bad references with a relative path
+			log.Warn("Deprecated local head %s for PR #%d in %s/%s, removing  %s", pr.Head.SHA, pr.Number, g.repoOwner, g.repoName, pr.GetGitRefName())
+		} else {
+			// set head information
+			_, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref", pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()})
 			if err != nil {
-				log.Error("Cannot remove local head ref, %v", err)
+				log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err)
 			}
 		}
 	}
@@ -615,6 +690,20 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model
 		return nil, fmt.Errorf("updateGitForPullRequest: %w", err)
 	}
 
+	// Now we may need to fix the mergebase
+	if pr.Base.SHA == "" {
+		if pr.Base.Ref != "" && pr.Head.SHA != "" {
+			// A PR against a tag base does not make sense - therefore pr.Base.Ref must be a branch
+			// TODO: should we be checking for the refs/heads/ prefix on the pr.Base.Ref? (i.e. are these actually branches or refs)
+			pr.Base.SHA, _, err = g.gitRepo.GetMergeBase("", git.BranchPrefix+pr.Base.Ref, pr.Head.SHA)
+			if err != nil {
+				log.Error("Cannot determine the merge base for PR #%d in %s/%s. Error: %v", pr.Number, g.repoOwner, g.repoName, err)
+			}
+		} else {
+			log.Error("Cannot determine the merge base for PR #%d in %s/%s. Not enough information", pr.Number, g.repoOwner, g.repoName)
+		}
+	}
+
 	if pr.Created.IsZero() {
 		if pr.Closed != nil {
 			pr.Created = *pr.Closed
@@ -728,6 +817,8 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
 			return err
 		}
 
+		cms = append(cms, &cm)
+
 		// get pr
 		pr, ok := g.prCache[issue.ID]
 		if !ok {
@@ -738,6 +829,17 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
 			}
 			g.prCache[issue.ID] = pr
 		}
+		if pr.MergeBase == "" {
+			// No mergebase -> no basis for any patches
+			log.Warn("PR #%d in %s/%s: does not have a merge base, all review comments will be ignored", pr.Index, g.repoOwner, g.repoName)
+			continue
+		}
+
+		headCommitID, err := g.gitRepo.GetRefCommitID(pr.GetGitRefName())
+		if err != nil {
+			log.Warn("PR #%d GetRefCommitID[%s] in %s/%s: %v, all review comments will be ignored", pr.Index, pr.GetGitRefName(), g.repoOwner, g.repoName, err)
+			continue
+		}
 
 		for _, comment := range review.Comments {
 			line := comment.Line
@@ -746,11 +848,9 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
 			} else {
 				_, _, line, _ = git.ParseDiffHunkString(comment.DiffHunk)
 			}
-			headCommitID, err := g.gitRepo.GetRefCommitID(pr.GetGitRefName())
-			if err != nil {
-				log.Warn("GetRefCommitID[%s]: %v, the review comment will be ignored", pr.GetGitRefName(), err)
-				continue
-			}
+
+			// SECURITY: The TreePath must be cleaned!
+			comment.TreePath = path.Clean("/" + comment.TreePath)[1:]
 
 			var patch string
 			reader, writer := io.Pipe()
@@ -775,6 +875,11 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
 				comment.UpdatedAt = comment.CreatedAt
 			}
 
+			if !git.IsValidSHAPattern(comment.CommitID) {
+				log.Warn("Invalid comment CommitID[%s] on comment[%d] in PR #%d of %s/%s replaced with %s", comment.CommitID, pr.Index, g.repoOwner, g.repoName, headCommitID)
+				comment.CommitID = headCommitID
+			}
+
 			c := issues_model.Comment{
 				Type:        issues_model.CommentTypeCode,
 				IssueID:     issue.ID,
@@ -793,8 +898,6 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
 
 			cm.Comments = append(cm.Comments, &c)
 		}
-
-		cms = append(cms, &cm)
 	}
 
 	return issues_model.InsertReviews(cms)

services/migrations/gitea_uploader_test.go

@@ -46,7 +46,7 @@ func TestGiteaUploadRepo(t *testing.T) {
 		uploader   = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
 	)
 
-	err := migrateRepository(downloader, uploader, base.MigrateOptions{
+	err := migrateRepository(user, downloader, uploader, base.MigrateOptions{
 		CloneAddr:    "https://github.com/go-xorm/builder",
 		RepoName:     repoName,
 		AuthUsername: "",
@@ -391,7 +391,7 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) {
 				},
 			},
 			assertContent: func(t *testing.T, content string) {
-				assert.Contains(t, content, "AddRemote failed")
+				assert.Contains(t, content, "AddRemote")
 			},
 		},
 		{
@@ -440,7 +440,7 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) {
 				},
 			},
 			assertContent: func(t *testing.T, content string) {
-				assert.Contains(t, content, "Empty reference, removing")
+				assert.Contains(t, content, "Empty reference")
 				assert.NotContains(t, content, "Cannot remove local head")
 			},
 		},
@@ -468,7 +468,6 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) {
 			},
 			assertContent: func(t *testing.T, content string) {
 				assert.Contains(t, content, "Deprecated local head")
-				assert.Contains(t, content, "Cannot remove local head")
 			},
 		},
 		{
@@ -505,6 +504,8 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) {
 			logger.SetLogger("buffer", "buffer", "{}")
 			defer logger.DelLogger("buffer")
 
+			testCase.pr.EnsuredSafe = true
+
 			head, err := uploader.updateGitForPullRequest(&testCase.pr)
 			assert.NoError(t, err)
 			assert.EqualValues(t, testCase.head, head)

services/migrations/github.go

@@ -51,7 +51,7 @@ func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOp
 	oldOwner := fields[1]
 	oldName := strings.TrimSuffix(fields[2], ".git")
 
-	log.Trace("Create github downloader: %s/%s", oldOwner, oldName)
+	log.Trace("Create github downloader BaseURL: %s %s/%s", baseURL, oldOwner, oldName)
 
 	return NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
 }
@@ -67,6 +67,7 @@ type GithubDownloaderV3 struct {
 	base.NullDownloader
 	ctx           context.Context
 	clients       []*github.Client
+	baseURL       string
 	repoOwner     string
 	repoName      string
 	userName      string
@@ -81,6 +82,7 @@ type GithubDownloaderV3 struct {
 func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
 	downloader := GithubDownloaderV3{
 		userName:   userName,
+		baseURL:    baseURL,
 		password:   password,
 		ctx:        ctx,
 		repoOwner:  repoOwner,
@@ -118,6 +120,20 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
 	return &downloader
 }
 
+// String implements Stringer
+func (g *GithubDownloaderV3) String() string {
+	return fmt.Sprintf("migration from github server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
+}
+
+// ColorFormat provides a basic color format for a GithubDownloader
+func (g *GithubDownloaderV3) ColorFormat(s fmt.State) {
+	if g == nil {
+		log.ColorFprintf(s, "<nil: GithubDownloaderV3>")
+		return
+	}
+	log.ColorFprintf(s, "migration from github server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
+}
+
 func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) {
 	githubClient := github.NewClient(client)
 	if baseURL != "https://github.com" {
@@ -322,33 +338,44 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
 			Updated:       asset.UpdatedAt.Time,
 			DownloadFunc: func() (io.ReadCloser, error) {
 				g.waitAndPickClient()
-				asset, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)
+				readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)
 				if err != nil {
 					return nil, err
 				}
 				if err := g.RefreshRate(); err != nil {
 					log.Error("g.getClient().RateLimits: %s", err)
 				}
-				if asset == nil {
-					if redirectURL != "" {
-						g.waitAndPickClient()
-						req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil)
-						if err != nil {
-							return nil, err
-						}
-						resp, err := httpClient.Do(req)
-						err1 := g.RefreshRate()
-						if err1 != nil {
-							log.Error("g.getClient().RateLimits: %s", err1)
-						}
-						if err != nil {
-							return nil, err
-						}
-						return resp.Body, nil
-					}
-					return nil, fmt.Errorf("No release asset found for %d", assetID)
+
+				if readCloser != nil {
+					return readCloser, nil
 				}
-				return asset, nil
+
+				if redirectURL == "" {
+					return nil, fmt.Errorf("no release asset found for %d", assetID)
+				}
+
+				// Prevent open redirect
+				if !hasBaseURL(redirectURL, g.baseURL) &&
+					!hasBaseURL(redirectURL, "https://objects.githubusercontent.com/") {
+					WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", asset.GetID(), g, redirectURL)
+
+					return io.NopCloser(strings.NewReader(redirectURL)), nil
+				}
+
+				g.waitAndPickClient()
+				req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil)
+				if err != nil {
+					return nil, err
+				}
+				resp, err := httpClient.Do(req)
+				err1 := g.RefreshRate()
+				if err1 != nil {
+					log.Error("g.RefreshRate(): %s", err1)
+				}
+				if err != nil {
+					return nil, err
+				}
+				return resp.Body, nil
 			},
 		})
 	}
@@ -697,7 +724,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
 				SHA:       pr.GetHead().GetSHA(),
 				OwnerName: pr.GetHead().GetUser().GetLogin(),
 				RepoName:  pr.GetHead().GetRepo().GetName(),
-				CloneURL:  pr.GetHead().GetRepo().GetCloneURL(),
+				CloneURL:  pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here
 			},
 			Base: base.PullRequestBranch{
 				Ref:       pr.GetBase().GetRef(),
@@ -705,10 +732,13 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
 				RepoName:  pr.GetBase().GetRepo().GetName(),
 				OwnerName: pr.GetBase().GetUser().GetLogin(),
 			},
-			PatchURL:     pr.GetPatchURL(),
+			PatchURL:     pr.GetPatchURL(), // see below for SECURITY related issues here
 			Reactions:    reactions,
 			ForeignIndex: int64(*pr.Number),
 		})
+
+		// SECURITY: Ensure that the PR is safe
+		_ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g)
 	}
 
 	return allPRs, len(prs) < perPage, nil

services/migrations/gitlab.go

@@ -63,6 +63,7 @@ type GitlabDownloader struct {
 	base.NullDownloader
 	ctx        context.Context
 	client     *gitlab.Client
+	baseURL    string
 	repoID     int
 	repoName   string
 	issueCount int64
@@ -124,12 +125,27 @@ func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, passw
 	return &GitlabDownloader{
 		ctx:        ctx,
 		client:     gitlabClient,
+		baseURL:    baseURL,
 		repoID:     gr.ID,
 		repoName:   gr.Name,
 		maxPerPage: 100,
 	}, nil
 }
 
+// String implements Stringer
+func (g *GitlabDownloader) String() string {
+	return fmt.Sprintf("migration from gitlab server %s [%d]/%s", g.baseURL, g.repoID, g.repoName)
+}
+
+// ColorFormat provides a basic color format for a GitlabDownloader
+func (g *GitlabDownloader) ColorFormat(s fmt.State) {
+	if g == nil {
+		log.ColorFprintf(s, "<nil: GitlabDownloader>")
+		return
+	}
+	log.ColorFprintf(s, "migration from gitlab server %s [%d]/%s", g.baseURL, g.repoID, g.repoName)
+}
+
 // SetContext set context
 func (g *GitlabDownloader) SetContext(ctx context.Context) {
 	g.ctx = ctx
@@ -307,6 +323,11 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
 					return nil, err
 				}
 
+				if !hasBaseURL(link.URL, g.baseURL) {
+					WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", asset.ID, g, link.URL)
+					return io.NopCloser(strings.NewReader(link.URL)), nil
+				}
+
 				req, err := http.NewRequest("GET", link.URL, nil)
 				if err != nil {
 					return nil, err
@@ -610,6 +631,9 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque
 			ForeignIndex: int64(pr.IID),
 			Context:      gitlabIssueContext{IsMergeRequest: true},
 		})
+
+		// SECURITY: Ensure that the PR is safe
+		_ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g)
 	}
 
 	return allPRs, len(prs) < perPage, nil

services/migrations/gogs.go

@@ -73,6 +73,20 @@ type GogsDownloader struct {
 	transport          http.RoundTripper
 }
 
+// String implements Stringer
+func (g *GogsDownloader) String() string {
+	return fmt.Sprintf("migration from gogs server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
+}
+
+// ColorFormat provides a basic color format for a GogsDownloader
+func (g *GogsDownloader) ColorFormat(s fmt.State) {
+	if g == nil {
+		log.ColorFprintf(s, "<nil: GogsDownloader>")
+		return
+	}
+	log.ColorFprintf(s, "migration from gogs server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
+}
+
 // SetContext set context
 func (g *GogsDownloader) SetContext(ctx context.Context) {
 	g.ctx = ctx

services/migrations/migrate.go

@@ -125,7 +125,7 @@ func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName str
 	uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
 	uploader.gitServiceType = opts.GitServiceType
 
-	if err := migrateRepository(downloader, uploader, opts, messenger); err != nil {
+	if err := migrateRepository(doer, downloader, uploader, opts, messenger); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)
 		}
@@ -174,7 +174,7 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio
 // migrateRepository will download information and then upload it to Uploader, this is a simple
 // process for small repository. For a big repository, save all the data to disk
 // before upload is better
-func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error {
+func migrateRepository(doer *user_model.User, downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error {
 	if messenger == nil {
 		messenger = base.NilMessenger
 	}
@@ -195,6 +195,27 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 		return err
 	}
 
+	// SECURITY: If the downloader is not a RepositoryRestorer then we need to recheck the CloneURL
+	if _, ok := downloader.(*RepositoryRestorer); !ok {
+		// Now the clone URL can be rewritten by the downloader so we must recheck
+		if err := IsMigrateURLAllowed(repo.CloneURL, doer); err != nil {
+			return err
+		}
+
+		// SECURITY: Ensure that we haven't been redirected from an external to a local filesystem
+		// Now we know all of these must parse
+		cloneAddrURL, _ := url.Parse(opts.CloneAddr)
+		cloneURL, _ := url.Parse(repo.CloneURL)
+
+		if cloneURL.Scheme == "file" || cloneURL.Scheme == "" {
+			if cloneAddrURL.Scheme != "file" && cloneAddrURL.Scheme != "" {
+				return fmt.Errorf("repo info has changed from external to local filesystem")
+			}
+		}
+
+		// We don't actually need to check the OriginalURL as it isn't used anywhere
+	}
+
 	log.Trace("migrating git data from %s", repo.CloneURL)
 	messenger("repo.migrate.migrating_git")
 	if err = uploader.CreateRepo(repo, opts); err != nil {

services/migrations/onedev.go

@@ -110,6 +110,20 @@ func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, passwo
 	return downloader
 }
 
+// String implements Stringer
+func (d *OneDevDownloader) String() string {
+	return fmt.Sprintf("migration from oneDev server %s [%d]/%s", d.baseURL, d.repoID, d.repoName)
+}
+
+// ColorFormat provides a basic color format for a OneDevDownloader
+func (d *OneDevDownloader) ColorFormat(s fmt.State) {
+	if d == nil {
+		log.ColorFprintf(s, "<nil: OneDevDownloader>")
+		return
+	}
+	log.ColorFprintf(s, "migration from oneDev server %s [%d]/%s", d.baseURL, d.repoID, d.repoName)
+}
+
 func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error {
 	u, err := d.baseURL.Parse(endpoint)
 	if err != nil {
@@ -542,6 +556,9 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque
 			ForeignIndex: pr.ID,
 			Context:      onedevIssueContext{IsPullRequest: true},
 		})
+
+		// SECURITY: Ensure that the PR is safe
+		_ = CheckAndEnsureSafePR(pullRequests[len(pullRequests)-1], d.baseURL.String(), d)
 	}
 
 	return pullRequests, len(pullRequests) == 0, nil

services/migrations/restore.go

@@ -243,6 +243,7 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq
 	}
 	for _, pr := range pulls {
 		pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL)
+		CheckAndEnsureSafePR(pr, "", r)
 	}
 	return pulls, true, nil
 }

services/release/release.go

@@ -37,6 +37,9 @@ func createTag(gitRepo *git.Repository, rel *models.Release, msg string) (bool,
 			if err != nil {
 				return false, fmt.Errorf("GetProtectedTags: %v", err)
 			}
+
+			// Trim '--' prefix to prevent command line argument vulnerability.
+			rel.TagName = strings.TrimPrefix(rel.TagName, "--")
 			isAllowed, err := git_model.IsUserAllowedToControlTag(protectedTags, rel.TagName, rel.PublisherID)
 			if err != nil {
 				return false, err
@@ -52,8 +55,6 @@ func createTag(gitRepo *git.Repository, rel *models.Release, msg string) (bool,
 				return false, fmt.Errorf("createTag::GetCommit[%v]: %v", rel.Target, err)
 			}
 
-			// Trim '--' prefix to prevent command line argument vulnerability.
-			rel.TagName = strings.TrimPrefix(rel.TagName, "--")
 			if len(msg) > 0 {
 				if err = gitRepo.CreateAnnotatedTag(rel.TagName, msg, commit.ID.String()); err != nil {
 					if strings.Contains(err.Error(), "is not a valid tag name") {
@@ -308,7 +309,7 @@ func DeleteReleaseByID(ctx context.Context, id int64, doer *user_model.User, del
 			}
 		}
 
-		if stdout, _, err := git.NewCommand(ctx, "tag", "-d", rel.TagName).
+		if stdout, _, err := git.NewCommand(ctx, "tag", "-d", "--", rel.TagName).
 			SetDescription(fmt.Sprintf("DeleteReleaseByID (git tag -d): %d", rel.ID)).
 			RunStdString(&git.RunOpts{Dir: repo.RepoPath()}); err != nil && !strings.Contains(err.Error(), "not found") {
 			log.Error("DeleteReleaseByID (git tag -d): %d in %v Failed:\nStdout: %s\nError: %v", rel.ID, repo, stdout, err)

services/repository/archiver/archiver.go

@@ -57,6 +57,21 @@ func (ErrUnknownArchiveFormat) Is(err error) bool {
 	return ok
 }
 
+// RepoRefNotFoundError is returned when a requested reference (commit, tag) was not found.
+type RepoRefNotFoundError struct {
+	RefName string
+}
+
+// Error implements error.
+func (e RepoRefNotFoundError) Error() string {
+	return fmt.Sprintf("unrecognized repository reference: %s", e.RefName)
+}
+
+func (e RepoRefNotFoundError) Is(err error) bool {
+	_, ok := err.(RepoRefNotFoundError)
+	return ok
+}
+
 // NewRequest creates an archival request, based on the URI.  The
 // resulting ArchiveRequest is suitable for being passed to ArchiveRepository()
 // if it's determined that the request still needs to be satisfied.
@@ -103,7 +118,7 @@ func NewRequest(repoID int64, repo *git.Repository, uri string) (*ArchiveRequest
 			}
 		}
 	} else {
-		return nil, fmt.Errorf("Unknow ref %s type", r.refName)
+		return nil, RepoRefNotFoundError{RefName: r.refName}
 	}
 
 	return r, nil
@@ -115,6 +130,49 @@ func (aReq *ArchiveRequest) GetArchiveName() string {
 	return strings.ReplaceAll(aReq.refName, "/", "-") + "." + aReq.Type.String()
 }
 
+// Await awaits the completion of an ArchiveRequest. If the archive has
+// already been prepared the method returns immediately. Otherwise an archiver
+// process will be started and its completion awaited. On success the returned
+// RepoArchiver may be used to download the archive. Note that even if the
+// context is cancelled/times out a started archiver will still continue to run
+// in the background.
+func (aReq *ArchiveRequest) Await(ctx context.Context) (*repo_model.RepoArchiver, error) {
+	archiver, err := repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID)
+	if err != nil {
+		return nil, fmt.Errorf("models.GetRepoArchiver: %v", err)
+	}
+
+	if archiver != nil && archiver.Status == repo_model.ArchiverReady {
+		// Archive already generated, we're done.
+		return archiver, nil
+	}
+
+	if err := StartArchive(aReq); err != nil {
+		return nil, fmt.Errorf("archiver.StartArchive: %v", err)
+	}
+
+	poll := time.NewTicker(time.Second * 1)
+	defer poll.Stop()
+
+	for {
+		select {
+		case <-graceful.GetManager().HammerContext().Done():
+			// System stopped.
+			return nil, graceful.GetManager().HammerContext().Err()
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-poll.C:
+			archiver, err = repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID)
+			if err != nil {
+				return nil, fmt.Errorf("repo_model.GetRepoArchiver: %v", err)
+			}
+			if archiver != nil && archiver.Status == repo_model.ArchiverReady {
+				return archiver, nil
+			}
+		}
+	}
+}
+
 func doArchive(r *ArchiveRequest) (*repo_model.RepoArchiver, error) {
 	txCtx, committer, err := db.TxContext()
 	if err != nil {
@@ -147,11 +205,7 @@ func doArchive(r *ArchiveRequest) (*repo_model.RepoArchiver, error) {
 		}
 	}
 
-	rPath, err := archiver.RelativePath()
-	if err != nil {
-		return nil, err
-	}
-
+	rPath := archiver.RelativePath()
 	_, err = storage.RepoArchives.Stat(rPath)
 	if err == nil {
 		if archiver.Status == repo_model.ArchiverGenerating {
@@ -284,13 +338,10 @@ func StartArchive(request *ArchiveRequest) error {
 }
 
 func deleteOldRepoArchiver(ctx context.Context, archiver *repo_model.RepoArchiver) error {
-	p, err := archiver.RelativePath()
-	if err != nil {
-		return err
-	}
 	if err := repo_model.DeleteRepoArchiver(ctx, archiver); err != nil {
 		return err
 	}
+	p := archiver.RelativePath()
 	if err := storage.RepoArchives.Delete(p); err != nil {
 		log.Error("delete repo archive file failed: %v", err)
 	}

templates/admin/auth/edit.tmpl

@@ -341,12 +341,12 @@
 					</div>
 					<div class="field">
 						<label for="oauth2_required_claim_name">{{.i18n.Tr "admin.auths.oauth2_required_claim_name"}}</label>
-						<input id="oauth2_required_claim_name" name="oauth2_required_claim_name" values="{{$cfg.RequiredClaimName}}">
+						<input id="oauth2_required_claim_name" name="oauth2_required_claim_name" value="{{$cfg.RequiredClaimName}}">
 						<p class="help">{{.i18n.Tr "admin.auths.oauth2_required_claim_name_helper"}}</p>
 					</div>
 					<div class="field">
 						<label for="oauth2_required_claim_value">{{.i18n.Tr "admin.auths.oauth2_required_claim_value"}}</label>
-						<input id="oauth2_required_claim_value" name="oauth2_required_claim_value" values="{{$cfg.RequiredClaimValue}}">
+						<input id="oauth2_required_claim_value" name="oauth2_required_claim_value" value="{{$cfg.RequiredClaimValue}}">
 						<p class="help">{{.i18n.Tr "admin.auths.oauth2_required_claim_value_helper"}}</p>
 					</div>
 					<div class="field">

templates/base/footer.tmpl

@@ -22,7 +22,7 @@
 		<script src='https://hcaptcha.com/1/api.js' async></script>
 	{{end}}
 {{end}}
-	<script src="{{AssetUrlPrefix}}/js/index.js?v={{MD5 AppVer}}" onerror="alert('Failed to load asset files from ' + this.src + ', please make sure the asset files can be accessed and the ROOT_URL setting in app.ini is correct.')"></script>
+	<script src="{{AssetUrlPrefix}}/js/index.js?v={{AssetVersion}}" onerror="alert('Failed to load asset files from ' + this.src + ', please make sure the asset files can be accessed and the ROOT_URL setting in app.ini is correct.')"></script>
 {{template "custom/footer" .}}
 </body>
 </html>

templates/base/head.tmpl

@@ -21,7 +21,7 @@
 {{end}}
 	<link rel="icon" href="{{AssetUrlPrefix}}/img/favicon.svg" type="image/svg+xml">
 	<link rel="alternate icon" href="{{AssetUrlPrefix}}/img/favicon.png" type="image/png">
-	<link rel="stylesheet" href="{{AssetUrlPrefix}}/css/index.css?v={{MD5 AppVer}}">
+	<link rel="stylesheet" href="{{AssetUrlPrefix}}/css/index.css?v={{AssetVersion}}">
 	{{template "base/head_script" .}}
 	<noscript>
 		<style>
@@ -67,10 +67,10 @@
 <meta property="og:site_name" content="{{AppName}}">
 {{if .IsSigned }}
 	{{ if ne .SignedUser.Theme "gitea" }}
-		<link rel="stylesheet" href="{{AssetUrlPrefix}}/css/theme-{{.SignedUser.Theme | PathEscape}}.css?v={{MD5 AppVer}}">
+		<link rel="stylesheet" href="{{AssetUrlPrefix}}/css/theme-{{.SignedUser.Theme | PathEscape}}.css?v={{AssetVersion}}">
 	{{end}}
 {{else if ne DefaultTheme "gitea"}}
-	<link rel="stylesheet" href="{{AssetUrlPrefix}}/css/theme-{{DefaultTheme | PathEscape}}.css?v={{MD5 AppVer}}">
+	<link rel="stylesheet" href="{{AssetUrlPrefix}}/css/theme-{{DefaultTheme | PathEscape}}.css?v={{AssetVersion}}">
 {{end}}
 {{template "custom/header" .}}
 </head>

templates/base/head_script.tmpl

@@ -10,6 +10,7 @@ If you introduce mistakes in it, Gitea JavaScript code wouldn't run correctly.
 		appVer: '{{AppVer}}',
 		appUrl: '{{AppUrl}}',
 		appSubUrl: '{{AppSubUrl}}',
+		assetVersionEncoded: encodeURIComponent('{{AssetVersion}}'), // will be used in URL construction directly
 		assetUrlPrefix: '{{AssetUrlPrefix}}',
 		runModeIsProd: {{.RunModeIsProd}},
 		customEmojis: {{CustomEmojis}},

templates/org/team/teams.tmpl

@@ -32,9 +32,7 @@
 					</div>
 					<div class="ui attached segment members">
 						{{range .Members}}
-							<a href="{{.HomeLink}}" title="{{.Name}}">
-								{{avatar .}}
-							</a>
+							{{template "shared/user/avatarlink" .}}
 						{{end}}
 					</div>
 					<div class="ui bottom attached header">

templates/repo/branch_dropdown.tmpl

@@ -1,5 +1,6 @@
 {{$release := .release}}
 {{$defaultBranch := $.root.BranchName}}{{if and .root.IsViewTag (not .noTag)}}{{$defaultBranch = .root.TagName}}{{end}}{{if eq $defaultBranch ""}}{{$defaultBranch = $.root.Repository.DefaultBranch}}{{end}}
+{{$type := ""}}{{if and .root.IsViewTag (not .noTag)}}{{$type = "tag"}}{{else if .root.IsViewBranch}}{{$type = "branch"}}{{else}}{{$type = "tree"}}{{end}}
 {{$showBranchesInDropdown := not .root.HideBranchesInDropdown}}
 <div class="fitted item choose reference{{if not $release}} mr-1{{end}}">
 	<div class="ui floating filter dropdown custom"
@@ -7,20 +8,20 @@
 		data-can-create-branch="{{if .canCreateBranch}}{{.canCreateBranch}}{{else}}{{.root.CanCreateBranch}}{{end}}"
 		data-no-results="{{.root.i18n.Tr "repo.pulls.no_results"}}"
 		data-set-action="{{.setAction}}" data-submit-form="{{.submitForm}}"
-		data-view-type="{{if and .root.IsViewTag (not .noTag)}}tag{{else if .root.IsViewBranch}}branch{{else}}tree{{end}}"
+		data-view-type="{{$type}}"
 		data-ref-name="{{if and .root.IsViewTag (not .noTag)}}{{.root.TagName}}{{else if .root.IsViewBranch}}{{.root.BranchName}}{{else}}{{ShortSha .root.CommitID}}{{end}}"
 		data-branch-url-prefix="{{if .branchURLPrefix}}{{.branchURLPrefix}}{{else}}{{$.root.RepoLink}}/{{if $.root.PageIsCommits}}commits{{else}}src{{end}}/branch/{{end}}"
 		data-branch-url-suffix="{{if .branchURLSuffix}}{{.branchURLSuffix}}{{else}}{{if $.root.TreePath}}/{{PathEscapeSegments $.root.TreePath}}{{end}}{{end}}"
 		data-tag-url-prefix="{{if .tagURLPrefix}}{{.tagURLPrefix}}{{else if $release}}{{$.root.RepoLink}}/compare/{{else}}{{$.root.RepoLink}}/{{if $.root.PageIsCommits}}commits{{else}}src{{end}}/tag/{{end}}"
 		data-tag-url-suffix="{{if .tagURLSuffix}}{{.tagURLSuffix}}{{else if $release}}...{{if $release.IsDraft}}{{PathEscapeSegments $release.Target}}{{else}}{{if $release.TagName}}{{PathEscapeSegments $release.TagName}}{{else}}{{PathEscapeSegments $release.Sha1}}{{end}}{{end}}{{else}}{{if $.root.TreePath}}/{{PathEscapeSegments $.root.TreePath}}{{end}}{{end}}">
-		<div class="ui basic small compact button" @click="menuVisible = !menuVisible" @keyup.enter="menuVisible = !menuVisible">
+		<div class="branch-dropdown-button ellipsis ui basic small compact button" @click="menuVisible = !menuVisible" @keyup.enter="menuVisible = !menuVisible">
 			<span class="text">
 				{{if $release}}
 					{{.root.i18n.Tr "repo.release.compare"}}
 				{{else}}
-					<span :class="{visible: isViewTag}" v-if="isViewTag" v-cloak>{{svg "octicon-tag"}} {{.root.i18n.Tr "repo.tag"}}:</span>
-					<span :class="{visible: isViewBranch}" v-if="isViewBranch" v-cloak>{{svg "octicon-git-branch"}} {{.root.i18n.Tr "repo.branch"}}:</span>
-					<span :class="{visible: isViewTree}" v-if="isViewTree" v-cloak>{{svg "octicon-git-branch"}} {{.root.i18n.Tr "repo.tree"}}:</span>
+					<span :class="{visible: isViewTag}" v-if="isViewTag" {{if not (eq $type "tag")}}v-cloak{{end}}>{{svg "octicon-tag"}} {{.root.i18n.Tr "repo.tag"}}:</span>
+					<span :class="{visible: isViewBranch}" v-if="isViewBranch" {{if not (eq $type "branch")}}v-cloak{{end}}>{{svg "octicon-git-branch"}} {{.root.i18n.Tr "repo.branch"}}:</span>
+					<span :class="{visible: isViewTree}" v-if="isViewTree" {{if not (eq $type "tree")}}v-cloak{{end}}>{{svg "octicon-git-branch"}} {{.root.i18n.Tr "repo.tree"}}:</span>
 					<strong ref="dropdownRefName">{{if and .root.IsViewTag (not .noTag)}}{{.root.TagName}}{{else if .root.IsViewBranch}}{{.root.BranchName}}{{else}}{{ShortSha .root.CommitID}}{{end}}</strong>
 				{{end}}
 			</span>

templates/repo/clone_buttons.tmpl

@@ -1,15 +1,15 @@
 <!-- there is always at least one button (by context/repo.go) -->
 {{if $.CloneButtonShowHTTPS}}
-	<button class="ui basic clone button no-transition" id="repo-clone-https" data-link="{{$.CloneButtonOriginLink.HTTPS}}">
+	<button class="ui basic small compact clone button no-transition" id="repo-clone-https" data-link="{{$.CloneButtonOriginLink.HTTPS}}">
 		{{if UseHTTPS}}HTTPS{{else}}HTTP{{end}}
 	</button>
 {{end}}
 {{if $.CloneButtonShowSSH}}
-	<button class="ui basic clone button no-transition" id="repo-clone-ssh" data-link="{{$.CloneButtonOriginLink.SSH}}">
+	<button class="ui basic small compact clone button no-transition" id="repo-clone-ssh" data-link="{{$.CloneButtonOriginLink.SSH}}">
 		SSH
 	</button>
 {{end}}
-<input id="repo-clone-url" class="js-clone-url" value="{{$.CloneButtonOriginLink.HTTPS}}" size="1" readonly>
-<button class="ui basic icon button tooltip" id="clipboard-btn" data-content="{{.i18n.Tr "copy_url"}}" data-clipboard-target="#repo-clone-url" aria-label="{{.i18n.Tr "copy_url"}}">
-	{{svg "octicon-paste"}}
+<input id="repo-clone-url" size="20" class="js-clone-url br-0" value="{{$.CloneButtonOriginLink.HTTPS}}" readonly>
+<button class="ui basic small compact icon button tooltip" id="clipboard-btn" data-content="{{.i18n.Tr "copy_url"}}" data-clipboard-target="#repo-clone-url" aria-label="{{.i18n.Tr "copy_url"}}">
+	{{svg "octicon-copy" 14}}
 </button>

templates/repo/commit_page.tmpl

@@ -222,7 +222,7 @@
 								{{.Verification.SigningSSHKey.Fingerprint}}
 							{{else}}
 								<span class="ui text mr-3">{{.i18n.Tr "repo.commits.gpg_key_id"}}:</span>
-								{{.Verification.SigningKey.KeyID}}
+								{{.Verification.SigningKey.PaddedKeyID}}
 							{{end}}
 						{{else}}
 							{{svg "octicon-shield-lock" 16 "mr-3"}}
@@ -231,7 +231,7 @@
 								{{.Verification.SigningSSHKey.Fingerprint}}
 							{{else}}
 								<span class="ui text mr-3 tooltip" data-content="{{.i18n.Tr "gpg.default_key"}}">{{.i18n.Tr "repo.commits.gpg_key_id"}}:</span>
-								{{.Verification.SigningKey.KeyID}}
+								{{.Verification.SigningKey.PaddedKeyID}}
 							{{end}}
 						{{end}}
 					{{else if .Verification.Warning}}
@@ -241,14 +241,14 @@
 							{{.Verification.SigningSSHKey.Fingerprint}}
 						{{else}}
 							<span class="ui text mr-3">{{.i18n.Tr "repo.commits.gpg_key_id"}}:</span>
-							{{.Verification.SigningKey.KeyID}}
+							{{.Verification.SigningKey.PaddedKeyID}}
 						{{end}}
 					{{else}}
 						{{if .Verification.SigningKey}}
 							{{if ne .Verification.SigningKey.KeyID ""}}
 								{{svg "octicon-shield" 16 "mr-3"}}
 								<span class="ui text mr-3">{{.i18n.Tr "repo.commits.gpg_key_id"}}:</span>
-								{{.Verification.SigningKey.KeyID}}
+								{{.Verification.SigningKey.PaddedKeyID}}
 							{{end}}
 						{{end}}
 						{{if .Verification.SigningSSHKey}}

templates/repo/commits.tmpl

@@ -3,9 +3,9 @@
 	{{template "repo/header" .}}
 	<div class="ui container">
 		{{template "repo/sub_menu" .}}
-		<div class="ui secondary stackable menu mobile--margin-between-items">
-		{{template "repo/branch_dropdown" dict "root" .}}
-			<div class="fitted item">
+		<div class="repo-button-row df ac sb fw mb-4 mt-3">
+			<div class="df ac">
+				{{template "repo/branch_dropdown" dict "root" .}}
 				<a href="{{.RepoLink}}/graph" class="ui basic small compact button">
 					<span class="text">
 						{{svg "octicon-git-branch"}}

templates/repo/diff/box.tmpl

@@ -13,14 +13,14 @@
 	<h4>{{.i18n.Tr "repo.diff.data_not_available"}}</h4>
 {{else}}
 	<div>
-		<div class="diff-detail-box diff-box sticky df sb ac">
+		<div class="diff-detail-box diff-box sticky df sb ac fw">
 			<div class="diff-detail-stats df ac">
 				{{svg "octicon-diff" 16 "mr-2"}}{{.i18n.Tr "repo.diff.stats_desc" .Diff.NumFiles .Diff.TotalAddition .Diff.TotalDeletion | Str2html}}
 			</div>
 			<div class="diff-detail-actions df ac">
 				{{if and .PageIsPullFiles $.SignedUserID (not .IsArchived)}}
 					<progress id="viewed-files-summary" class="mr-2" value="{{.Diff.NumViewedFiles}}" max="{{.Diff.NumFiles}}"></progress>
-					<label for="viewed-files-summary" id="viewed-files-summary-label" class="mr-2" data-text-changed-template="{{.i18n.Tr "repo.pulls.viewed_files_label"}}">
+					<label for="viewed-files-summary" id="viewed-files-summary-label" class="mr-3" data-text-changed-template="{{.i18n.Tr "repo.pulls.viewed_files_label"}}">
 						{{.i18n.Tr "repo.pulls.viewed_files_label" .Diff.NumViewedFiles .Diff.NumFiles}}
 					</label>
 				{{end}}

templates/repo/diff/comments.tmpl

@@ -5,9 +5,7 @@
 	{{if .OriginalAuthor }}
 		<span class="avatar"><img src="{{AppSubUrl}}/assets/img/avatar_default.png"></span>
 	{{else}}
-		<a class="avatar" {{if gt .Poster.ID 0}}href="{{.Poster.HomeLink}}"{{end}}>
-			{{avatar .Poster}}
-		</a>
+		{{template "shared/user/avatarlink" .Poster}}
 	{{end}}
 	<div class="content comment-container">
 		<div class="ui top attached header comment-header df ac sb">
@@ -27,9 +25,7 @@
 					</span>
 				{{else}}
 					<span class="text grey">
-						<a {{if gt .Poster.ID 0}}href="{{.Poster.HomeLink}}"{{end}}>
-							{{.Poster.GetDisplayName}}
-						</a>
+						{{template "shared/user/namelink" .Poster}}
 						{{$.root.i18n.Tr "repo.issues.commented_at" (.HashTag|Escape) $createdStr | Safe}}
 					</span>
 				{{end}}

templates/repo/diff/new_review.tmpl

@@ -1,5 +1,5 @@
 <div class="ui top right pointing dropdown custom" id="review-box">
-	<div class="ui tiny green button btn-review">
+	<div class="ui tiny green button btn-review ml-2 mr-0">
 		{{.i18n.Tr "repo.diff.review"}}
 		<span class="ui small label review-comments-counter" data-pending-comment-number="{{.PendingCodeCommentNumber}}">{{.PendingCodeCommentNumber}}</span>
 		{{svg "octicon-triangle-down" 14 "dropdown icon"}}