Nulo/gitea
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Activity

Migrate reactions when migrating repository from github (#9599)

Browse source 
* Migrate reactions when migrating repository from github

* fix missed sleep

* fix tests

* update reactions when external user binding

* Fix test

* fix tests

* change the copy head

* fix test

* fix migrator add/delete reaction
This commit is contained in:
Lunny Xiao 2020-01-15 19:14:07 +08:00 committed by Antoine GIRARD
parent 4e566df1c6
commit 2b3e931cde
18 changed files with 329 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
models/external_login_user.go
View file

@ -177,5 +177,9 @@ func UpdateMigrationsByType(tp structs.GitServiceType, externalUserID string, us
return err
}
return UpdateReleasesMigrationsByType(tp, externalUserID, userID)
if err := UpdateReleasesMigrationsByType(tp, externalUserID, userID); err != nil {
return err
}
return UpdateReactionsMigrationsByType(tp, externalUserID, userID)
}

19
models/issue.go
View file

@ -218,8 +218,11 @@ func (issue *Issue) loadReactions(e Engine) (err error) {
if err != nil {
return err
}
if err = issue.loadRepo(e); err != nil {
return err
}
// Load reaction user data
if _, err := ReactionList(reactions).loadUsers(e); err != nil {
if _, err := ReactionList(reactions).loadUsers(e, issue.Repo); err != nil {
return err
}
@ -1836,3 +1839,17 @@ func UpdateIssuesMigrationsByType(gitServiceType structs.GitServiceType, origina
})
return err
}
// UpdateReactionsMigrationsByType updates all migrated repositories' reactions from gitServiceType to replace originalAuthorID to posterID
func UpdateReactionsMigrationsByType(gitServiceType structs.GitServiceType, originalAuthorID string, userID int64) error {
_, err := x.Table("reaction").
Join("INNER", "issue", "issue.id = reaction.issue_id").
Where("issue.repo_id IN (SELECT id FROM repository WHERE original_service_type = ?)", gitServiceType).
And("reaction.original_author_id = ?", originalAuthorID).
Update(map[string]interface{}{
"user_id": userID,
"original_author": "",
"original_author_id": 0,
})
return err
}

8
models/issue_comment.go
View file

@ -425,7 +425,7 @@ func (c *Comment) LoadDepIssueDetails() (err error) {
return err
}
func (c *Comment) loadReactions(e Engine) (err error) {
func (c *Comment) loadReactions(e Engine, repo *Repository) (err error) {
if c.Reactions != nil {
return nil
}
@ -437,15 +437,15 @@ func (c *Comment) loadReactions(e Engine) (err error) {
return err
}
// Load reaction user data
if _, err := c.Reactions.LoadUsers(); err != nil {
if _, err := c.Reactions.loadUsers(e, repo); err != nil {
return err
}
return nil
}
// LoadReactions loads comment reactions
func (c *Comment) LoadReactions() error {
return c.loadReactions(x)
func (c *Comment) LoadReactions(repo *Repository) error {
return c.loadReactions(x, repo)
}
func (c *Comment) loadReview(e Engine) (err error) {

24
models/issue_reaction.go
View file

@ -22,6 +22,8 @@ type Reaction struct {
IssueID int64 `xorm:"INDEX UNIQUE(s) NOT NULL"`
CommentID int64 `xorm:"INDEX UNIQUE(s)"`
UserID int64 `xorm:"INDEX UNIQUE(s) NOT NULL"`
OriginalAuthorID int64 `xorm:"INDEX UNIQUE(s) NOT NULL DEFAULT(0)"`
OriginalAuthor string
User *User `xorm:"-"`
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
}
@ -49,7 +51,10 @@ func (opts *FindReactionsOptions) toConds() builder.Cond {
cond = cond.And(builder.Eq{"reaction.comment_id": 0})
}
if opts.UserID > 0 {
cond = cond.And(builder.Eq{"reaction.user_id": opts.UserID})
cond = cond.And(builder.Eq{
"reaction.user_id": opts.UserID,
"reaction.original_author_id": 0,
})
}
if opts.Reaction != "" {
cond = cond.And(builder.Eq{"reaction.type": opts.Reaction})
@ -173,7 +178,7 @@ func deleteReaction(e *xorm.Session, opts *ReactionOptions) error {
if opts.Comment != nil {
reaction.CommentID = opts.Comment.ID
}
_, err := e.Delete(reaction)
_, err := e.Where("original_author_id = 0").Delete(reaction)
return err
}
@ -233,7 +238,7 @@ func (list ReactionList) HasUser(userID int64) bool {
return false
}
for _, reaction := range list {
if reaction.UserID == userID {
if reaction.OriginalAuthor == "" && reaction.UserID == userID {
return true
}
}
@ -252,6 +257,9 @@ func (list ReactionList) GroupByType() map[string]ReactionList {
func (list ReactionList) getUserIDs() []int64 {
userIDs := make(map[int64]struct{}, len(list))
for _, reaction := range list {
if reaction.OriginalAuthor != "" {
continue
}
if _, ok := userIDs[reaction.UserID]; !ok {
userIDs[reaction.UserID] = struct{}{}
}
@ -259,7 +267,7 @@ func (list ReactionList) getUserIDs() []int64 {
return keysInt64(userIDs)
}
func (list ReactionList) loadUsers(e Engine) ([]*User, error) {
func (list ReactionList) loadUsers(e Engine, repo *Repository) ([]*User, error) {
if len(list) == 0 {
return nil, nil
}
@ -274,7 +282,9 @@ func (list ReactionList) loadUsers(e Engine) ([]*User, error) {
}
for _, reaction := range list {
if user, ok := userMaps[reaction.UserID]; ok {
if reaction.OriginalAuthor != "" {
reaction.User = NewReplaceUser(fmt.Sprintf("%s(%s)", reaction.OriginalAuthor, repo.OriginalServiceType.Name()))
} else if user, ok := userMaps[reaction.UserID]; ok {
reaction.User = user
} else {
reaction.User = NewGhostUser()
@ -284,8 +294,8 @@ func (list ReactionList) loadUsers(e Engine) ([]*User, error) {
}
// LoadUsers loads reactions' all users
func (list ReactionList) LoadUsers() ([]*User, error) {
return list.loadUsers(x)
func (list ReactionList) LoadUsers(repo *Repository) ([]*User, error) {
return list.loadUsers(x, repo)
}
// GetFirstUsers returns first reacted user display names separated by comma

3
models/issue_reaction_test.go
View file

@ -132,6 +132,7 @@ func TestIssueCommentDeleteReaction(t *testing.T) {
user4 := AssertExistsAndLoadBean(t, &User{ID: 4}).(*User)
issue1 := AssertExistsAndLoadBean(t, &Issue{ID: 1}).(*Issue)
repo1 := AssertExistsAndLoadBean(t, &Repository{ID: issue1.RepoID}).(*Repository)
comment1 := AssertExistsAndLoadBean(t, &Comment{ID: 1}).(*Comment)
@ -140,7 +141,7 @@ func TestIssueCommentDeleteReaction(t *testing.T) {
addReaction(t, user3, issue1, comment1, "heart")
addReaction(t, user4, issue1, comment1, "+1")
err := comment1.LoadReactions()
err := comment1.LoadReactions(repo1)
assert.NoError(t, err)
assert.Len(t, comment1.Reactions, 4)

20
models/migrate.go
View file

@ -63,6 +63,13 @@ func insertIssue(sess *xorm.Session, issue *Issue) error {
return err
}
for _, reaction := range issue.Reactions {
reaction.IssueID = issue.ID
}
if _, err := sess.Insert(issue.Reactions); err != nil {
return err
}
cols := make([]string, 0)
if !issue.IsPull {
sess.ID(issue.RepoID).Incr("num_issues")
@ -130,9 +137,20 @@ func InsertIssueComments(comments []*Comment) error {
if err := sess.Begin(); err != nil {
return err
}
if _, err := sess.NoAutoTime().Insert(comments); err != nil {
for _, comment := range comments {
if _, err := sess.NoAutoTime().Insert(comment); err != nil {
return err
}
for _, reaction := range comment.Reactions {
reaction.IssueID = comment.IssueID
reaction.CommentID = comment.ID
}
if _, err := sess.Insert(comment.Reactions); err != nil {
return err
}
}
for issueID := range issueIDs {
if _, err := sess.Exec("UPDATE issue set num_comments = (SELECT count(*) FROM comment WHERE issue_id = ?) WHERE id = ?", issueID, issueID); err != nil {
return err

2
models/migrations/migrations.go
View file

@ -300,6 +300,8 @@ var migrations = []Migration{
NewMigration("add is_restricted column for users table", addIsRestricted),
// v122 -> v123
NewMigration("Add Require Signed Commits to ProtectedBranch", addRequireSignedCommits),
// v123 -> v124
NewMigration("Add original informations for reactions", addReactionOriginals),
}
// Migrate database to current version

18
models/migrations/v123.go Normal file
View file

@ -0,0 +1,18 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"xorm.io/xorm"
)
func addReactionOriginals(x *xorm.Engine) error {
type Reaction struct {
OriginalAuthorID int64 `xorm:"INDEX NOT NULL DEFAULT(0)"`
OriginalAuthor string
}
return x.Sync2(new(Reaction))
}

9
models/user.go
View file

@ -793,6 +793,15 @@ func NewGhostUser() *User {
}
}
// NewReplaceUser creates and returns a fake user for external user
func NewReplaceUser(name string) *User {
return &User{
ID: -1,
Name: name,
LowerName: strings.ToLower(name),
}
}
// IsGhost check if user is fake user for a deleted account
func (u *User) IsGhost() bool {
if u == nil {

2
modules/migrations/base/comment.go
View file

@ -16,5 +16,5 @@ type Comment struct {
Created time.Time
Updated time.Time
Content string
Reactions *Reactions
Reactions []*Reaction
}

2
modules/migrations/base/issue.go
View file

@ -22,5 +22,5 @@ type Issue struct {
Updated time.Time
Closed *time.Time
Labels []*Label
Reactions *Reactions
Reactions []*Reaction
}

1
modules/migrations/base/pullrequest.go
View file

@ -33,6 +33,7 @@ type PullRequest struct {
Assignee string
Assignees []string
IsLocked bool
Reactions []*Reaction
}
// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository

17
modules/migrations/base/reaction.go
View file

@ -1,17 +1,12 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
// Reactions represents a summary of reactions.
type Reactions struct {
TotalCount int
PlusOne int
MinusOne int
Laugh int
Confused int
Heart int
Hooray int
// Reaction represents a reaction to an issue/pr/comment.
type Reaction struct {
UserID int64
UserName string
Content string
}

90
modules/migrations/gitea.go
View file

@ -361,7 +361,32 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
if issue.Closed != nil {
is.ClosedUnix = timeutil.TimeStamp(issue.Closed.Unix())
}
// TODO: add reactions
// add reactions
for _, reaction := range issue.Reactions {
userid, ok := g.userMap[reaction.UserID]
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[reaction.UserID] = userid
}
}
var res = models.Reaction{
Type: reaction.Content,
CreatedUnix: timeutil.TimeStampNow(),
}
if userid > 0 {
res.UserID = userid
} else {
res.UserID = g.doer.ID
res.OriginalAuthorID = reaction.UserID
res.OriginalAuthor = reaction.UserName
}
is.Reactions = append(is.Reactions, &res)
}
iss = append(iss, &is)
}
@ -420,9 +445,34 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
cm.OriginalAuthorID = comment.PosterID
}
cms = append(cms, &cm)
// add reactions
for _, reaction := range comment.Reactions {
userid, ok := g.userMap[reaction.UserID]
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[reaction.UserID] = userid
}
}
var res = models.Reaction{
Type: reaction.Content,
CreatedUnix: timeutil.TimeStampNow(),
}
if userid > 0 {
res.UserID = userid
} else {
res.UserID = g.doer.ID
res.OriginalAuthorID = reaction.UserID
res.OriginalAuthor = reaction.UserName
}
cm.Reactions = append(cm.Reactions, &res)
}
// TODO: Reactions
cms = append(cms, &cm)
}
return models.InsertIssueComments(cms)
@ -581,10 +631,12 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
UpdatedUnix: timeutil.TimeStamp(pr.Updated.Unix()),
}
tp := g.gitServiceType.Name()
userid, ok := g.userMap[pr.PosterID]
if !ok {
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID("github", fmt.Sprintf("%v", pr.PosterID))
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
@ -601,6 +653,33 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
issue.OriginalAuthorID = pr.PosterID
}
// add reactions
for _, reaction := range pr.Reactions {
userid, ok := g.userMap[reaction.UserID]
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[reaction.UserID] = userid
}
}
var res = models.Reaction{
Type: reaction.Content,
CreatedUnix: timeutil.TimeStampNow(),
}
if userid > 0 {
res.UserID = userid
} else {
res.UserID = g.doer.ID
res.OriginalAuthorID = reaction.UserID
res.OriginalAuthor = reaction.UserName
}
issue.Reactions = append(issue.Reactions, &res)
}
var pullRequest = models.PullRequest{
HeadRepoID: g.repo.ID,
HeadBranch: head,
@ -622,7 +701,6 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
pullRequest.MergerID = g.doer.ID
}
// TODO: reactions
// TODO: assignees
return &pullRequest, nil

94
modules/migrations/github.go
View file

@ -319,18 +319,6 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
return releases, nil
}
func convertGithubReactions(reactions *github.Reactions) *base.Reactions {
return &base.Reactions{
TotalCount: *reactions.TotalCount,
PlusOne: *reactions.PlusOne,
MinusOne: *reactions.MinusOne,
Laugh: *reactions.Laugh,
Confused: *reactions.Confused,
Heart: *reactions.Heart,
Hooray: *reactions.Hooray,
}
}
// GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
opt := &github.IssueListByRepoOptions{
@ -366,15 +354,36 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
for _, l := range issue.Labels {
labels = append(labels, convertGithubLabel(&l))
}
var reactions *base.Reactions
if issue.Reactions != nil {
reactions = convertGithubReactions(issue.Reactions)
}
var email string
if issue.User.Email != nil {
email = *issue.User.Email
}
// get reactions
var reactions []*base.Reaction
for i := 1; ; i++ {
g.sleep()
res, resp, err := g.client.Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{
Page: i,
PerPage: perPage,
})
if err != nil {
return nil, false, err
}
g.rate = &resp.Rate
if len(res) == 0 {
break
}
for _, reaction := range res {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.GetID(),
UserName: reaction.User.GetLogin(),
Content: reaction.GetContent(),
})
}
}
allIssues = append(allIssues, &base.Issue{
Title: *issue.Title,
Number: int64(*issue.Number),
@ -418,9 +427,29 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
if comment.User.Email != nil {
email = *comment.User.Email
}
var reactions *base.Reactions
if comment.Reactions != nil {
reactions = convertGithubReactions(comment.Reactions)
// get reactions
var reactions []*base.Reaction
for i := 1; ; i++ {
g.sleep()
res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
Page: i,
PerPage: 100,
})
if err != nil {
return nil, err
}
g.rate = &resp.Rate
if len(res) == 0 {
break
}
for _, reaction := range res {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.GetID(),
UserName: reaction.User.GetLogin(),
Content: reaction.GetContent(),
})
}
}
allComments = append(allComments, &base.Comment{
IssueIndex: issueNumber,
@ -473,8 +502,6 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
labels = append(labels, convertGithubLabel(l))
}
// FIXME: This API missing reactions, we may need another extra request to get reactions
var email string
if pr.User.Email != nil {
email = *pr.User.Email
@ -515,6 +542,30 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
headUserName = *pr.Head.User.Login
}
// get reactions
var reactions []*base.Reaction
for i := 1; ; i++ {
g.sleep()
res, resp, err := g.client.Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{
Page: i,
PerPage: perPage,
})
if err != nil {
return nil, err
}
g.rate = &resp.Rate
if len(res) == 0 {
break
}
for _, reaction := range res {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.GetID(),
UserName: reaction.User.GetLogin(),
Content: reaction.GetContent(),
})
}
}
allPRs = append(allPRs, &base.PullRequest{
Title: *pr.Title,
Number: int64(*pr.Number),
@ -546,6 +597,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
OwnerName: *pr.Base.User.Login,
},
PatchURL: *pr.PatchURL,
Reactions: reactions,
})
}

89
modules/migrations/github_test.go
View file

@ -170,14 +170,12 @@ func TestGitHubDownloadRepo(t *testing.T) {
Description: "Good for newcomers",
},
},
Reactions: &base.Reactions{
TotalCount: 1,
PlusOne: 1,
MinusOne: 0,
Laugh: 0,
Confused: 0,
Heart: 0,
Hooray: 0,
Reactions: []*base.Reaction{
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "+1",
},
},
Closed: &closed1,
},
@ -198,14 +196,37 @@ func TestGitHubDownloadRepo(t *testing.T) {
Description: "This issue or pull request already exists",
},
},
Reactions: &base.Reactions{
TotalCount: 6,
PlusOne: 1,
MinusOne: 1,
Laugh: 1,
Confused: 1,
Heart: 1,
Hooray: 1,
Reactions: []*base.Reaction{
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "heart",
},
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "laugh",
},
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "-1",
},
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "confused",
},
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "hooray",
},
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "+1",
},
},
Closed: &closed2,
},
@ -223,14 +244,12 @@ func TestGitHubDownloadRepo(t *testing.T) {
Created: time.Date(2019, 11, 12, 21, 0, 13, 0, time.UTC),
Updated: time.Date(2019, 11, 12, 21, 0, 13, 0, time.UTC),
Content: "This is a comment",
Reactions: &base.Reactions{
TotalCount: 1,
PlusOne: 1,
MinusOne: 0,
Laugh: 0,
Confused: 0,
Heart: 0,
Hooray: 0,
Reactions: []*base.Reaction{
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "+1",
},
},
},
{
@ -240,15 +259,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
Created: time.Date(2019, 11, 12, 22, 7, 14, 0, time.UTC),
Updated: time.Date(2019, 11, 12, 22, 7, 14, 0, time.UTC),
Content: "A second comment",
Reactions: &base.Reactions{
TotalCount: 0,
PlusOne: 0,
MinusOne: 0,
Laugh: 0,
Confused: 0,
Heart: 0,
Hooray: 0,
},
Reactions: nil,
},
}, comments[:2])
@ -331,6 +342,18 @@ func TestGitHubDownloadRepo(t *testing.T) {
},
Merged: false,
MergeCommitSHA: "565d1208f5fffdc1c5ae1a2436491eb9a5e4ebae",
Reactions: []*base.Reaction{
{
UserID: 81045,
UserName: "lunny",
Content: "heart",
},
{
UserID: 81045,
UserName: "lunny",
Content: "+1",
},
},
},
}, prs)
}

4
routers/api/v1/repo/issue_reaction.go
View file

@ -65,7 +65,7 @@ func GetIssueCommentReactions(ctx *context.APIContext) {
ctx.Error(http.StatusInternalServerError, "FindIssueReactions", err)
return
}
_, err = reactions.LoadUsers()
_, err = reactions.LoadUsers(ctx.Repo.Repository)
if err != nil {
ctx.Error(http.StatusInternalServerError, "ReactionList.LoadUsers()", err)
return
@ -271,7 +271,7 @@ func GetIssueReactions(ctx *context.APIContext) {
ctx.Error(http.StatusInternalServerError, "FindIssueReactions", err)
return
}
_, err = reactions.LoadUsers()
_, err = reactions.LoadUsers(ctx.Repo.Repository)
if err != nil {
ctx.Error(http.StatusInternalServerError, "ReactionList.LoadUsers()", err)
return

4
routers/repo/issue.go
View file

@ -1608,7 +1608,7 @@ func ChangeCommentReaction(ctx *context.Context, form auth.ReactionForm) {
}
// Reload new reactions
comment.Reactions = nil
if err = comment.LoadReactions(); err != nil {
if err = comment.LoadReactions(ctx.Repo.Repository); err != nil {
log.Info("comment.LoadReactions: %s", err)
break
}
@ -1622,7 +1622,7 @@ func ChangeCommentReaction(ctx *context.Context, form auth.ReactionForm) {
// Reload new reactions
comment.Reactions = nil
if err = comment.LoadReactions(); err != nil {
if err = comment.LoadReactions(ctx.Repo.Repository); err != nil {
log.Info("comment.LoadReactions: %s", err)
break
}