From 2b4a08e9621e052845062ecd71ebfd37453c0383 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 2 Dec 2020 19:36:06 +0100 Subject: [PATCH] Migrations: Use Process Manager to create own Context (#13792) --- .../git_helper_for_declarative_test.go | 2 +- modules/git/git.go | 1 + modules/git/repo.go | 12 +++++++--- modules/migrations/gitea_uploader.go | 2 +- modules/repository/repo.go | 7 +++--- modules/task/migrate.go | 22 ++++++++++++++----- services/mirror/mirror_test.go | 3 ++- 7 files changed, 34 insertions(+), 15 deletions(-) diff --git a/integrations/git_helper_for_declarative_test.go b/integrations/git_helper_for_declarative_test.go index 04b4c99013..f681cd143a 100644 --- a/integrations/git_helper_for_declarative_test.go +++ b/integrations/git_helper_for_declarative_test.go @@ -111,7 +111,7 @@ func onGiteaRun(t *testing.T, callback func(*testing.T, *url.URL), prepare ...bo func doGitClone(dstLocalPath string, u *url.URL) func(*testing.T) { return func(t *testing.T) { - assert.NoError(t, git.CloneWithArgs(u.String(), dstLocalPath, allowLFSFilters(), git.CloneRepoOptions{})) + assert.NoError(t, git.CloneWithArgs(context.Background(), u.String(), dstLocalPath, allowLFSFilters(), git.CloneRepoOptions{})) assert.True(t, com.IsExist(filepath.Join(dstLocalPath, "README.md"))) } } diff --git a/modules/git/git.go b/modules/git/git.go index cce5cc738d..6b15138a5c 100644 --- a/modules/git/git.go +++ b/modules/git/git.go @@ -32,6 +32,7 @@ var ( GitExecutable = "git" // DefaultContext is the default context to run git commands in + // will be overwritten by Init with HammerContext DefaultContext = context.Background() gitVersion *version.Version diff --git a/modules/git/repo.go b/modules/git/repo.go index ae370d3da9..9b1da87a32 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -8,6 +8,7 @@ package git import ( "bytes" "container/list" + "context" "errors" "fmt" "os" @@ -166,19 +167,24 @@ type CloneRepoOptions struct { // Clone clones original repository to target path. func Clone(from, to string, opts CloneRepoOptions) (err error) { + return CloneWithContext(DefaultContext, from, to, opts) +} + +// CloneWithContext clones original repository to target path. +func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) (err error) { cargs := make([]string, len(GlobalCommandArgs)) copy(cargs, GlobalCommandArgs) - return CloneWithArgs(from, to, cargs, opts) + return CloneWithArgs(ctx, from, to, cargs, opts) } // CloneWithArgs original repository to target path. -func CloneWithArgs(from, to string, args []string, opts CloneRepoOptions) (err error) { +func CloneWithArgs(ctx context.Context, from, to string, args []string, opts CloneRepoOptions) (err error) { toDir := path.Dir(to) if err = os.MkdirAll(toDir, os.ModePerm); err != nil { return err } - cmd := NewCommandNoGlobals(args...).AddArguments("clone") + cmd := NewCommandContextNoGlobals(ctx, args...).AddArguments("clone") if opts.Mirror { cmd.AddArguments("--mirror") } diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 466c832754..2cb19685fc 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -125,7 +125,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate } r.DefaultBranch = repo.DefaultBranch - r, err = repository.MigrateRepositoryGitData(g.doer, owner, r, base.MigrateOptions{ + r, err = repository.MigrateRepositoryGitData(g.ctx, owner, r, base.MigrateOptions{ RepoName: g.repoName, Description: repo.Description, OriginalURL: repo.OriginalURL, diff --git a/modules/repository/repo.go b/modules/repository/repo.go index b18dfddd2e..8ecb43ede6 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -5,6 +5,7 @@ package repository import ( + "context" "fmt" "path" "strings" @@ -41,7 +42,7 @@ func WikiRemoteURL(remote string) string { } // MigrateRepositoryGitData starts migrating git related data after created migrating repository -func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opts migration.MigrateOptions) (*models.Repository, error) { +func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.Repository, opts migration.MigrateOptions) (*models.Repository, error) { repoPath := models.RepoPath(u.Name, opts.RepoName) if u.IsOrganization() { @@ -61,7 +62,7 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err) } - if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{ + if err = git.CloneWithContext(ctx, opts.CloneAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, @@ -77,7 +78,7 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) } - if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ + if err = git.CloneWithContext(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, diff --git a/modules/task/migrate.go b/modules/task/migrate.go index 99f0435b28..57424abac3 100644 --- a/modules/task/migrate.go +++ b/modules/task/migrate.go @@ -5,6 +5,7 @@ package task import ( + "context" "errors" "fmt" "strings" @@ -15,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/migrations" migration "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" @@ -82,11 +84,6 @@ func runMigrateTask(t *models.Task) (err error) { if err = t.LoadOwner(); err != nil { return } - t.StartTime = timeutil.TimeStampNow() - t.Status = structs.TaskStatusRunning - if err = t.UpdateCols("start_time", "status"); err != nil { - return - } var opts *migration.MigrateOptions opts, err = t.MigrateConfig() @@ -96,7 +93,20 @@ func runMigrateTask(t *models.Task) (err error) { opts.MigrateToRepoID = t.RepoID var repo *models.Repository - repo, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), t.Doer, t.Owner.Name, *opts) + + ctx, cancel := context.WithCancel(graceful.GetManager().ShutdownContext()) + defer cancel() + pm := process.GetManager() + pid := pm.Add(fmt.Sprintf("MigrateTask: %s/%s", t.Owner.Name, opts.RepoName), cancel) + defer pm.Remove(pid) + + t.StartTime = timeutil.TimeStampNow() + t.Status = structs.TaskStatusRunning + if err = t.UpdateCols("start_time", "status"); err != nil { + return + } + + repo, err = migrations.MigrateRepository(ctx, t.Doer, t.Owner.Name, *opts) if err == nil { log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name) return diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_test.go index 79e18885b3..ddfb6c676b 100644 --- a/services/mirror/mirror_test.go +++ b/services/mirror/mirror_test.go @@ -5,6 +5,7 @@ package mirror import ( + "context" "path/filepath" "testing" @@ -47,7 +48,7 @@ func TestRelease_MirrorDelete(t *testing.T) { }) assert.NoError(t, err) - mirror, err := repository.MigrateRepositoryGitData(user, user, mirrorRepo, opts) + mirror, err := repository.MigrateRepositoryGitData(context.Background(), user, mirrorRepo, opts) assert.NoError(t, err) gitRepo, err := git.OpenRepository(repoPath)