Add repo-sync-releases admin command (#3254)
* Add repo-sync-releases admin command Will help recovering corrupted database, see #3247 * Load repos in chunks of 10, exit with error if unable to get a list, scan private repos, fix typo * Fix debug output about num releases * Introduce RepositoryListDefaultPageSize constant, set to 64 Use it from the new admin command * Use RepositoryListDefaultPageSize in more places * Document RepositoryListDefaultPageSize
This commit is contained in:
parent
ae9cc8f972
commit
8cd987af0c
5 changed files with 85 additions and 3 deletions
75
cmd/admin.go
75
cmd/admin.go
|
@ -8,7 +8,9 @@ package cmd
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
"code.gitea.io/git"
|
||||||
"code.gitea.io/gitea/models"
|
"code.gitea.io/gitea/models"
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
|
||||||
"github.com/urfave/cli"
|
"github.com/urfave/cli"
|
||||||
|
@ -24,6 +26,7 @@ to make automatic initialization process more smoothly`,
|
||||||
Subcommands: []cli.Command{
|
Subcommands: []cli.Command{
|
||||||
subcmdCreateUser,
|
subcmdCreateUser,
|
||||||
subcmdChangePassword,
|
subcmdChangePassword,
|
||||||
|
subcmdRepoSyncReleases,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -76,6 +79,12 @@ to make automatic initialization process more smoothly`,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
subcmdRepoSyncReleases = cli.Command{
|
||||||
|
Name: "repo-sync-releases",
|
||||||
|
Usage: "Synchronize repository releases with tags",
|
||||||
|
Action: runRepoSyncReleases,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
func runChangePassword(c *cli.Context) error {
|
func runChangePassword(c *cli.Context) error {
|
||||||
|
@ -145,3 +154,69 @@ func runCreateUser(c *cli.Context) error {
|
||||||
fmt.Printf("New user '%s' has been successfully created!\n", c.String("name"))
|
fmt.Printf("New user '%s' has been successfully created!\n", c.String("name"))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func runRepoSyncReleases(c *cli.Context) error {
|
||||||
|
|
||||||
|
setting.NewContext()
|
||||||
|
models.LoadConfigs()
|
||||||
|
|
||||||
|
setting.NewXORMLogService(false)
|
||||||
|
if err := models.SetEngine(); err != nil {
|
||||||
|
return fmt.Errorf("models.SetEngine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Trace("Synchronizing repository releases (this may take a while)")
|
||||||
|
for page := 1; ; page++ {
|
||||||
|
repos, count, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
|
||||||
|
Page: page,
|
||||||
|
PageSize: models.RepositoryListDefaultPageSize,
|
||||||
|
Private: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(4, "SearchRepositoryByName: %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(repos) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
log.Trace("Processing next %d repos of %d", len(repos), count)
|
||||||
|
for _, repo := range repos {
|
||||||
|
log.Trace("Synchronizing repo %s with path %s", repo.FullName(), repo.RepoPath())
|
||||||
|
gitRepo, err := git.OpenRepository(repo.RepoPath())
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("OpenRepository: %v", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
oldnum, err := models.GetReleaseCountByRepoID(repo.ID,
|
||||||
|
models.FindReleasesOptions{
|
||||||
|
IncludeDrafts: false,
|
||||||
|
IncludeTags: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Warn(" GetReleaseCountByRepoID: %v", err)
|
||||||
|
}
|
||||||
|
log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum)
|
||||||
|
|
||||||
|
if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil {
|
||||||
|
log.Warn(" SyncReleasesWithTags: %v", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
count, err = models.GetReleaseCountByRepoID(repo.ID,
|
||||||
|
models.FindReleasesOptions{
|
||||||
|
IncludeDrafts: false,
|
||||||
|
IncludeTags: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Warn(" GetReleaseCountByRepoID: %v", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Trace(" repo %s releases synchronized to tags: from %d to %d",
|
||||||
|
repo.FullName(), oldnum, count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
@ -29,7 +29,7 @@ func populateIssueIndexer() error {
|
||||||
for page := 1; ; page++ {
|
for page := 1; ; page++ {
|
||||||
repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
|
repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
|
||||||
Page: page,
|
Page: page,
|
||||||
PageSize: 10,
|
PageSize: RepositoryListDefaultPageSize,
|
||||||
OrderBy: SearchOrderByID,
|
OrderBy: SearchOrderByID,
|
||||||
Private: true,
|
Private: true,
|
||||||
Collaborate: util.OptionalBoolFalse,
|
Collaborate: util.OptionalBoolFalse,
|
||||||
|
|
|
@ -31,7 +31,7 @@ func releaseAddColumnIsTagAndSyncTags(x *xorm.Engine) error {
|
||||||
|
|
||||||
// For the sake of SQLite3, we can't use x.Iterate here.
|
// For the sake of SQLite3, we can't use x.Iterate here.
|
||||||
offset := 0
|
offset := 0
|
||||||
pageSize := 20
|
pageSize := models.RepositoryListDefaultPageSize
|
||||||
for {
|
for {
|
||||||
repos := make([]*models.Repository, 0, pageSize)
|
repos := make([]*models.Repository, 0, pageSize)
|
||||||
if err := x.Table("repository").Asc("id").Limit(pageSize, offset).Find(&repos); err != nil {
|
if err := x.Table("repository").Asc("id").Limit(pageSize, offset).Find(&repos); err != nil {
|
||||||
|
|
|
@ -81,7 +81,7 @@ func populateRepoIndexer() error {
|
||||||
for page := 1; ; page++ {
|
for page := 1; ; page++ {
|
||||||
repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
|
repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
|
||||||
Page: page,
|
Page: page,
|
||||||
PageSize: 10,
|
PageSize: RepositoryListDefaultPageSize,
|
||||||
OrderBy: SearchOrderByID,
|
OrderBy: SearchOrderByID,
|
||||||
Private: true,
|
Private: true,
|
||||||
})
|
})
|
||||||
|
|
|
@ -13,6 +13,13 @@ import (
|
||||||
"github.com/go-xorm/builder"
|
"github.com/go-xorm/builder"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// RepositoryListDefaultPageSize is the default number of repositories
|
||||||
|
// to load in memory when running administrative tasks on all (or almost
|
||||||
|
// all) of them.
|
||||||
|
// The number should be low enough to avoid filling up all RAM with
|
||||||
|
// repository data...
|
||||||
|
const RepositoryListDefaultPageSize = 64
|
||||||
|
|
||||||
// RepositoryList contains a list of repositories
|
// RepositoryList contains a list of repositories
|
||||||
type RepositoryList []*Repository
|
type RepositoryList []*Repository
|
||||||
|
|
||||||
|
|
Reference in a new issue