Nulo/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Move some repository methods from models to modules/repository (#9353)

Browse source 
* Move some repository methods from models to modules/repository

* fix test
This commit is contained in:
Lunny Xiao 2019-12-15 01:30:01 +08:00 committed by Antoine GIRARD
parent 2904abdc22
commit 7b2ff82d20
10 changed files with 245 additions and 209 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cmd/admin.go
View file

@ -16,6 +16,7 @@ import (
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
pwd "code.gitea.io/gitea/modules/password"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"github.com/urfave/cli"
@ -373,7 +374,7 @@ func runRepoSyncReleases(c *cli.Context) error {
}
log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum)
if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil {
if err = repository.SyncReleasesWithTags(repo, gitRepo); err != nil {
log.Warn(" SyncReleasesWithTags: %v", err)
gitRepo.Close()
continue

3
models/migrations/v39.go
View file

@ -10,6 +10,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/repository"
"xorm.io/xorm"
)
@ -44,7 +45,7 @@ func releaseAddColumnIsTagAndSyncTags(x *xorm.Engine) error {
continue
}
if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil {
if err = repository.SyncReleasesWithTags(repo, gitRepo); err != nil {
log.Warn("SyncReleasesWithTags: %v", err)
}
gitRepo.Close()

44
models/release.go
View file

@ -10,7 +10,6 @@ import (
"sort"
"strings"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
api "code.gitea.io/gitea/modules/structs"
@ -318,49 +317,6 @@ func DeleteReleaseByID(id int64) error {
return err
}
// SyncReleasesWithTags synchronizes release table with repository tags
func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error {
existingRelTags := make(map[string]struct{})
opts := FindReleasesOptions{IncludeDrafts: true, IncludeTags: true}
for page := 1; ; page++ {
rels, err := GetReleasesByRepoID(repo.ID, opts, page, 100)
if err != nil {
return fmt.Errorf("GetReleasesByRepoID: %v", err)
}
if len(rels) == 0 {
break
}
for _, rel := range rels {
if rel.IsDraft {
continue
}
commitID, err := gitRepo.GetTagCommitID(rel.TagName)
if err != nil && !git.IsErrNotExist(err) {
return fmt.Errorf("GetTagCommitID: %v", err)
}
if git.IsErrNotExist(err) || commitID != rel.Sha1 {
if err := PushUpdateDeleteTag(repo, rel.TagName); err != nil {
return fmt.Errorf("PushUpdateDeleteTag: %v", err)
}
} else {
existingRelTags[strings.ToLower(rel.TagName)] = struct{}{}
}
}
}
tags, err := gitRepo.GetTags()
if err != nil {
return fmt.Errorf("GetTags: %v", err)
}
for _, tagName := range tags {
if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok {
if err := PushUpdateAddTag(repo, gitRepo, tagName); err != nil {
return fmt.Errorf("pushUpdateAddTag: %v", err)
}
}
}
return nil
}
// UpdateReleasesMigrationsByType updates all migrated repositories' releases from gitServiceType to replace originalAuthorID to posterID
func UpdateReleasesMigrationsByType(gitServiceType structs.GitServiceType, originalAuthorID string, posterID int64) error {
_, err := x.Table("release").

161
models/repo.go
View file

@ -39,7 +39,6 @@ import (
"github.com/mcuadros/go-version"
"github.com/unknwon/com"
ini "gopkg.in/ini.v1"
"xorm.io/builder"
)
@ -941,25 +940,6 @@ func (repo *Repository) CloneLink() (cl *CloneLink) {
return repo.cloneLink(x, false)
}
/*
GitHub, GitLab, Gogs: *.wiki.git
BitBucket: *.git/wiki
*/
var commonWikiURLSuffixes = []string{".wiki.git", ".git/wiki"}
// wikiRemoteURL returns accessible repository URL for wiki if exists.
// Otherwise, it returns an empty string.
func wikiRemoteURL(remote string) string {
remote = strings.TrimSuffix(remote, ".git")
for _, suffix := range commonWikiURLSuffixes {
wikiURL := remote + suffix
if git.IsRepoURLAccessible(wikiURL) {
return wikiURL
}
}
return ""
}
// CheckCreateRepository check if could created a repository
func CheckCreateRepository(doer, u *User, name string) error {
if !doer.CanCreateRepo() {
@ -979,118 +959,9 @@ func CheckCreateRepository(doer, u *User, name string) error {
return nil
}
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
func MigrateRepositoryGitData(doer, u *User, repo *Repository, opts api.MigrateRepoOption) (*Repository, error) {
repoPath := RepoPath(u.Name, opts.RepoName)
if u.IsOrganization() {
t, err := u.GetOwnerTeam()
if err != nil {
return nil, err
}
repo.NumWatches = t.NumMembers
} else {
repo.NumWatches = 1
}
migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second
var err error
if err = os.RemoveAll(repoPath); err != nil {
return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err)
}
if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: migrateTimeout,
}); err != nil {
return repo, fmt.Errorf("Clone: %v", err)
}
if opts.Wiki {
wikiPath := WikiPath(u.Name, opts.RepoName)
wikiRemotePath := wikiRemoteURL(opts.CloneAddr)
if len(wikiRemotePath) > 0 {
if err := os.RemoveAll(wikiPath); err != nil {
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
}
if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: migrateTimeout,
Branch: "master",
}); err != nil {
log.Warn("Clone wiki: %v", err)
if err := os.RemoveAll(wikiPath); err != nil {
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
}
}
}
}
gitRepo, err := git.OpenRepository(repoPath)
if err != nil {
return repo, fmt.Errorf("OpenRepository: %v", err)
}
defer gitRepo.Close()
repo.IsEmpty, err = gitRepo.IsEmpty()
if err != nil {
return repo, fmt.Errorf("git.IsEmpty: %v", err)
}
if !opts.Releases && !repo.IsEmpty {
// Try to get HEAD branch and set it as default branch.
headBranch, err := gitRepo.GetHEADBranch()
if err != nil {
return repo, fmt.Errorf("GetHEADBranch: %v", err)
}
if headBranch != nil {
repo.DefaultBranch = headBranch.Name
}
if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
log.Error("Failed to synchronize tags to releases for repository: %v", err)
}
}
if err = repo.UpdateSize(); err != nil {
log.Error("Failed to update size for repository: %v", err)
}
if opts.Mirror {
if _, err = x.InsertOne(&Mirror{
RepoID: repo.ID,
Interval: setting.Mirror.DefaultInterval,
EnablePrune: true,
NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval),
}); err != nil {
return repo, fmt.Errorf("InsertOne: %v", err)
}
repo.IsMirror = true
err = UpdateRepository(repo, false)
} else {
repo, err = CleanUpMigrateInfo(repo)
}
return repo, err
}
// cleanUpMigrateGitConfig removes mirror info which prevents "push --all".
// This also removes possible user credentials.
func cleanUpMigrateGitConfig(configPath string) error {
cfg, err := ini.Load(configPath)
if err != nil {
return fmt.Errorf("open config file: %v", err)
}
cfg.DeleteSection("remote \"origin\"")
if err = cfg.SaveToIndent(configPath, "\t"); err != nil {
return fmt.Errorf("save config file: %v", err)
}
return nil
// CreateDelegateHooks creates all the hooks scripts for the repo
func CreateDelegateHooks(repoPath string) error {
return createDelegateHooks(repoPath)
}
// createDelegateHooks creates all the hooks scripts for the repo
@ -1132,32 +1003,6 @@ func createDelegateHooks(repoPath string) (err error) {
return nil
}
// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
func CleanUpMigrateInfo(repo *Repository) (*Repository, error) {
repoPath := repo.RepoPath()
if err := createDelegateHooks(repoPath); err != nil {
return repo, fmt.Errorf("createDelegateHooks: %v", err)
}
if repo.HasWiki() {
if err := createDelegateHooks(repo.WikiPath()); err != nil {
return repo, fmt.Errorf("createDelegateHooks.(wiki): %v", err)
}
}
_, err := git.NewCommand("remote", "rm", "origin").RunInDir(repoPath)
if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") {
return repo, fmt.Errorf("CleanUpMigrateInfo: %v", err)
}
if repo.HasWiki() {
if err := cleanUpMigrateGitConfig(path.Join(repo.WikiPath(), "config")); err != nil {
return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %v", err)
}
}
return repo, UpdateRepository(repo, false)
}
// initRepoCommit temporarily changes with work directory.
func initRepoCommit(tmpPath string, u *User) (err error) {
commitTimeStr := time.Now().Format(time.RFC3339)

6
models/repo_mirror.go
View file

@ -97,3 +97,9 @@ func MirrorsIterate(f func(idx int, bean interface{}) error) error {
And("next_update_unix!=0").
Iterate(new(Mirror), f)
}
// InsertMirror inserts a mirror to database
func InsertMirror(mirror *Mirror) error {
_, err := x.Insert(mirror)
return err
}

5
modules/migrations/gitea.go
View file

@ -21,6 +21,7 @@ import (
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
@ -111,7 +112,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
return err
}
r, err = models.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{
r, err = repository.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{
RepoName: g.repoName,
Description: repo.Description,
OriginalURL: repo.OriginalURL,
@ -294,7 +295,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
// SyncTags syncs releases with tags in the database
func (g *GiteaLocalUploader) SyncTags() error {
return models.SyncReleasesWithTags(g.repo, g.gitRepo)
return repository.SyncReleasesWithTags(g.repo, g.gitRepo)
}
// CreateIssues creates issues

223
modules/repository/repo.go Normal file
View file

@ -0,0 +1,223 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repository
import (
"fmt"
"os"
"path"
"strings"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"gopkg.in/ini.v1"
)
/*
GitHub, GitLab, Gogs: *.wiki.git
BitBucket: *.git/wiki
*/
var commonWikiURLSuffixes = []string{".wiki.git", ".git/wiki"}
// wikiRemoteURL returns accessible repository URL for wiki if exists.
// Otherwise, it returns an empty string.
func wikiRemoteURL(remote string) string {
remote = strings.TrimSuffix(remote, ".git")
for _, suffix := range commonWikiURLSuffixes {
wikiURL := remote + suffix
if git.IsRepoURLAccessible(wikiURL) {
return wikiURL
}
}
return ""
}
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opts api.MigrateRepoOption) (*models.Repository, error) {
repoPath := models.RepoPath(u.Name, opts.RepoName)
if u.IsOrganization() {
t, err := u.GetOwnerTeam()
if err != nil {
return nil, err
}
repo.NumWatches = t.NumMembers
} else {
repo.NumWatches = 1
}
migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second
var err error
if err = os.RemoveAll(repoPath); err != nil {
return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err)
}
if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: migrateTimeout,
}); err != nil {
return repo, fmt.Errorf("Clone: %v", err)
}
if opts.Wiki {
wikiPath := models.WikiPath(u.Name, opts.RepoName)
wikiRemotePath := wikiRemoteURL(opts.CloneAddr)
if len(wikiRemotePath) > 0 {
if err := os.RemoveAll(wikiPath); err != nil {
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
}
if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: migrateTimeout,
Branch: "master",
}); err != nil {
log.Warn("Clone wiki: %v", err)
if err := os.RemoveAll(wikiPath); err != nil {
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
}
}
}
}
gitRepo, err := git.OpenRepository(repoPath)
if err != nil {
return repo, fmt.Errorf("OpenRepository: %v", err)
}
defer gitRepo.Close()
repo.IsEmpty, err = gitRepo.IsEmpty()
if err != nil {
return repo, fmt.Errorf("git.IsEmpty: %v", err)
}
if !opts.Releases && !repo.IsEmpty {
// Try to get HEAD branch and set it as default branch.
headBranch, err := gitRepo.GetHEADBranch()
if err != nil {
return repo, fmt.Errorf("GetHEADBranch: %v", err)
}
if headBranch != nil {
repo.DefaultBranch = headBranch.Name
}
if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
log.Error("Failed to synchronize tags to releases for repository: %v", err)
}
}
if err = repo.UpdateSize(); err != nil {
log.Error("Failed to update size for repository: %v", err)
}
if opts.Mirror {
if err = models.InsertMirror(&models.Mirror{
RepoID: repo.ID,
Interval: setting.Mirror.DefaultInterval,
EnablePrune: true,
NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval),
}); err != nil {
return repo, fmt.Errorf("InsertOne: %v", err)
}
repo.IsMirror = true
err = models.UpdateRepository(repo, false)
} else {
repo, err = CleanUpMigrateInfo(repo)
}
return repo, err
}
// cleanUpMigrateGitConfig removes mirror info which prevents "push --all".
// This also removes possible user credentials.
func cleanUpMigrateGitConfig(configPath string) error {
cfg, err := ini.Load(configPath)
if err != nil {
return fmt.Errorf("open config file: %v", err)
}
cfg.DeleteSection("remote \"origin\"")
if err = cfg.SaveToIndent(configPath, "\t"); err != nil {
return fmt.Errorf("save config file: %v", err)
}
return nil
}
// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
func CleanUpMigrateInfo(repo *models.Repository) (*models.Repository, error) {
repoPath := repo.RepoPath()
if err := models.CreateDelegateHooks(repoPath); err != nil {
return repo, fmt.Errorf("createDelegateHooks: %v", err)
}
if repo.HasWiki() {
if err := models.CreateDelegateHooks(repo.WikiPath()); err != nil {
return repo, fmt.Errorf("createDelegateHooks.(wiki): %v", err)
}
}
_, err := git.NewCommand("remote", "rm", "origin").RunInDir(repoPath)
if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") {
return repo, fmt.Errorf("CleanUpMigrateInfo: %v", err)
}
if repo.HasWiki() {
if err := cleanUpMigrateGitConfig(path.Join(repo.WikiPath(), "config")); err != nil {
return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %v", err)
}
}
return repo, models.UpdateRepository(repo, false)
}
// SyncReleasesWithTags synchronizes release table with repository tags
func SyncReleasesWithTags(repo *models.Repository, gitRepo *git.Repository) error {
existingRelTags := make(map[string]struct{})
opts := models.FindReleasesOptions{IncludeDrafts: true, IncludeTags: true}
for page := 1; ; page++ {
rels, err := models.GetReleasesByRepoID(repo.ID, opts, page, 100)
if err != nil {
return fmt.Errorf("GetReleasesByRepoID: %v", err)
}
if len(rels) == 0 {
break
}
for _, rel := range rels {
if rel.IsDraft {
continue
}
commitID, err := gitRepo.GetTagCommitID(rel.TagName)
if err != nil && !git.IsErrNotExist(err) {
return fmt.Errorf("GetTagCommitID: %v", err)
}
if git.IsErrNotExist(err) || commitID != rel.Sha1 {
if err := models.PushUpdateDeleteTag(repo, rel.TagName); err != nil {
return fmt.Errorf("PushUpdateDeleteTag: %v", err)
}
} else {
existingRelTags[strings.ToLower(rel.TagName)] = struct{}{}
}
}
}
tags, err := gitRepo.GetTags()
if err != nil {
return fmt.Errorf("GetTags: %v", err)
}
for _, tagName := range tags {
if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok {
if err := models.PushUpdateAddTag(repo, gitRepo, tagName); err != nil {
return fmt.Errorf("pushUpdateAddTag: %v", err)
}
}
}
return nil
}

3
routers/repo/setting.go
View file

@ -20,6 +20,7 @@ import (
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/validation"
@ -338,7 +339,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) {
}
repo.IsMirror = false
if _, err := models.CleanUpMigrateInfo(repo); err != nil {
if _, err := repository.CleanUpMigrateInfo(repo); err != nil {
ctx.ServerError("CleanUpMigrateInfo", err)
return
} else if err = models.DeleteMirrorByRepoID(ctx.Repo.Repository.ID); err != nil {

3
services/mirror/mirror.go
View file

@ -15,6 +15,7 @@ import (
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/sync"
"code.gitea.io/gitea/modules/timeutil"
@ -207,7 +208,7 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) {
log.Error("OpenRepository: %v", err)
return nil, false
}
if err = models.SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
if err = repository.SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
gitRepo.Close()
log.Error("Failed to synchronize tags to releases for repository: %v", err)
}

3
services/mirror/mirror_test.go
View file

@ -10,6 +10,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/structs"
release_service "code.gitea.io/gitea/services/release"
@ -46,7 +47,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
})
assert.NoError(t, err)
mirror, err := models.MigrateRepositoryGitData(user, user, mirrorRepo, opts)
mirror, err := repository.MigrateRepositoryGitData(user, user, mirrorRepo, opts)
assert.NoError(t, err)
gitRepo, err := git.OpenRepository(repoPath)