improve github downloader on migrations (#7049)

* improve github downloader on migrations

* fix tests

* fix  uppercase function parameters
This commit is contained in:
Lunny Xiao 2019-05-31 04:26:57 +08:00 committed by techknowlogick
parent 43cf2f3b55
commit 7d12ec2abd
5 changed files with 141 additions and 151 deletions

View file

@ -11,9 +11,9 @@ type Downloader interface {
GetMilestones() ([]*Milestone, error) GetMilestones() ([]*Milestone, error)
GetReleases() ([]*Release, error) GetReleases() ([]*Release, error)
GetLabels() ([]*Label, error) GetLabels() ([]*Label, error)
GetIssues(start, limit int) ([]*Issue, error) GetIssues(page, perPage int) ([]*Issue, bool, error)
GetComments(issueNumber int64) ([]*Comment, error) GetComments(issueNumber int64) ([]*Comment, error)
GetPullRequests(start, limit int) ([]*PullRequest, error) GetPullRequests(page, perPage int) ([]*PullRequest, error)
} }
// DownloaderFactory defines an interface to match a downloader implementation and create a downloader // DownloaderFactory defines an interface to match a downloader implementation and create a downloader

View file

@ -53,9 +53,9 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
return nil, ErrNotSupported return nil, ErrNotSupported
} }
// GetIssues returns issues according start and limit // GetIssues returns issues according page and perPage
func (g *PlainGitDownloader) GetIssues(start, limit int) ([]*base.Issue, error) { func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
return nil, ErrNotSupported return nil, false, ErrNotSupported
} }
// GetComments returns comments according issueNumber // GetComments returns comments according issueNumber
@ -63,7 +63,7 @@ func (g *PlainGitDownloader) GetComments(issueNumber int64) ([]*base.Comment, er
return nil, ErrNotSupported return nil, ErrNotSupported
} }
// GetPullRequests returns pull requests according start and limit // GetPullRequests returns pull requests according page and perPage
func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) { func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
return nil, ErrNotSupported return nil, ErrNotSupported
} }

View file

@ -272,71 +272,65 @@ func convertGithubReactions(reactions *github.Reactions) *base.Reactions {
} }
// GetIssues returns issues according start and limit // GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(start, limit int) ([]*base.Issue, error) { func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
var perPage = 100
opt := &github.IssueListByRepoOptions{ opt := &github.IssueListByRepoOptions{
Sort: "created", Sort: "created",
Direction: "asc", Direction: "asc",
State: "all", State: "all",
ListOptions: github.ListOptions{ ListOptions: github.ListOptions{
PerPage: perPage, PerPage: perPage,
Page: page,
}, },
} }
var allIssues = make([]*base.Issue, 0, limit)
for {
issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
for _, issue := range issues {
if issue.IsPullRequest() {
continue
}
var body string
if issue.Body != nil {
body = *issue.Body
}
var milestone string
if issue.Milestone != nil {
milestone = *issue.Milestone.Title
}
var labels = make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, convertGithubLabel(&l))
}
var reactions *base.Reactions
if issue.Reactions != nil {
reactions = convertGithubReactions(issue.Reactions)
}
var email string var allIssues = make([]*base.Issue, 0, perPage)
if issue.User.Email != nil {
email = *issue.User.Email issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
} if err != nil {
allIssues = append(allIssues, &base.Issue{ return nil, false, fmt.Errorf("error while listing repos: %v", err)
Title: *issue.Title,
Number: int64(*issue.Number),
PosterName: *issue.User.Login,
PosterEmail: email,
Content: body,
Milestone: milestone,
State: *issue.State,
Created: *issue.CreatedAt,
Labels: labels,
Reactions: reactions,
Closed: issue.ClosedAt,
IsLocked: *issue.Locked,
})
if len(allIssues) >= limit {
return allIssues, nil
}
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
} }
return allIssues, nil for _, issue := range issues {
if issue.IsPullRequest() {
continue
}
var body string
if issue.Body != nil {
body = *issue.Body
}
var milestone string
if issue.Milestone != nil {
milestone = *issue.Milestone.Title
}
var labels = make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, convertGithubLabel(&l))
}
var reactions *base.Reactions
if issue.Reactions != nil {
reactions = convertGithubReactions(issue.Reactions)
}
var email string
if issue.User.Email != nil {
email = *issue.User.Email
}
allIssues = append(allIssues, &base.Issue{
Title: *issue.Title,
Number: int64(*issue.Number),
PosterName: *issue.User.Login,
PosterEmail: email,
Content: body,
Milestone: milestone,
State: *issue.State,
Created: *issue.CreatedAt,
Labels: labels,
Reactions: reactions,
Closed: issue.ClosedAt,
IsLocked: *issue.Locked,
})
}
return allIssues, len(issues) < perPage, nil
} }
// GetComments returns comments according issueNumber // GetComments returns comments according issueNumber
@ -379,97 +373,91 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
return allComments, nil return allComments, nil
} }
// GetPullRequests returns pull requests according start and limit // GetPullRequests returns pull requests according page and perPage
func (g *GithubDownloaderV3) GetPullRequests(start, limit int) ([]*base.PullRequest, error) { func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) {
opt := &github.PullRequestListOptions{ opt := &github.PullRequestListOptions{
Sort: "created", Sort: "created",
Direction: "asc", Direction: "asc",
State: "all", State: "all",
ListOptions: github.ListOptions{ ListOptions: github.ListOptions{
PerPage: 100, PerPage: perPage,
Page: page,
}, },
} }
var allPRs = make([]*base.PullRequest, 0, 100) var allPRs = make([]*base.PullRequest, 0, perPage)
for {
prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
for _, pr := range prs {
var body string
if pr.Body != nil {
body = *pr.Body
}
var milestone string
if pr.Milestone != nil {
milestone = *pr.Milestone.Title
}
var labels = make([]*base.Label, 0, len(pr.Labels))
for _, l := range pr.Labels {
labels = append(labels, convertGithubLabel(l))
}
// FIXME: This API missing reactions, we may need another extra request to get reactions prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
var email string return nil, fmt.Errorf("error while listing repos: %v", err)
if pr.User.Email != nil {
email = *pr.User.Email
}
var merged bool
// pr.Merged is not valid, so use MergedAt to test if it's merged
if pr.MergedAt != nil {
merged = true
}
var headRepoName string
var cloneURL string
if pr.Head.Repo != nil {
headRepoName = *pr.Head.Repo.Name
cloneURL = *pr.Head.Repo.CloneURL
}
var mergeCommitSHA string
if pr.MergeCommitSHA != nil {
mergeCommitSHA = *pr.MergeCommitSHA
}
allPRs = append(allPRs, &base.PullRequest{
Title: *pr.Title,
Number: int64(*pr.Number),
PosterName: *pr.User.Login,
PosterEmail: email,
Content: body,
Milestone: milestone,
State: *pr.State,
Created: *pr.CreatedAt,
Closed: pr.ClosedAt,
Labels: labels,
Merged: merged,
MergeCommitSHA: mergeCommitSHA,
MergedTime: pr.MergedAt,
IsLocked: pr.ActiveLockReason != nil,
Head: base.PullRequestBranch{
Ref: *pr.Head.Ref,
SHA: *pr.Head.SHA,
RepoName: headRepoName,
OwnerName: *pr.Head.User.Login,
CloneURL: cloneURL,
},
Base: base.PullRequestBranch{
Ref: *pr.Base.Ref,
SHA: *pr.Base.SHA,
RepoName: *pr.Base.Repo.Name,
OwnerName: *pr.Base.User.Login,
},
PatchURL: *pr.PatchURL,
})
if len(allPRs) >= limit {
return allPRs, nil
}
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
} }
for _, pr := range prs {
var body string
if pr.Body != nil {
body = *pr.Body
}
var milestone string
if pr.Milestone != nil {
milestone = *pr.Milestone.Title
}
var labels = make([]*base.Label, 0, len(pr.Labels))
for _, l := range pr.Labels {
labels = append(labels, convertGithubLabel(l))
}
// FIXME: This API missing reactions, we may need another extra request to get reactions
var email string
if pr.User.Email != nil {
email = *pr.User.Email
}
var merged bool
// pr.Merged is not valid, so use MergedAt to test if it's merged
if pr.MergedAt != nil {
merged = true
}
var headRepoName string
var cloneURL string
if pr.Head.Repo != nil {
headRepoName = *pr.Head.Repo.Name
cloneURL = *pr.Head.Repo.CloneURL
}
var mergeCommitSHA string
if pr.MergeCommitSHA != nil {
mergeCommitSHA = *pr.MergeCommitSHA
}
allPRs = append(allPRs, &base.PullRequest{
Title: *pr.Title,
Number: int64(*pr.Number),
PosterName: *pr.User.Login,
PosterEmail: email,
Content: body,
Milestone: milestone,
State: *pr.State,
Created: *pr.CreatedAt,
Closed: pr.ClosedAt,
Labels: labels,
Merged: merged,
MergeCommitSHA: mergeCommitSHA,
MergedTime: pr.MergedAt,
IsLocked: pr.ActiveLockReason != nil,
Head: base.PullRequestBranch{
Ref: *pr.Head.Ref,
SHA: *pr.Head.SHA,
RepoName: headRepoName,
OwnerName: *pr.Head.User.Login,
CloneURL: cloneURL,
},
Base: base.PullRequestBranch{
Ref: *pr.Base.Ref,
SHA: *pr.Base.SHA,
RepoName: *pr.Base.Repo.Name,
OwnerName: *pr.Base.User.Login,
},
PatchURL: *pr.PatchURL,
})
}
return allPRs, nil return allPRs, nil
} }

View file

@ -166,9 +166,11 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, releases[len(releases)-1:]) }, releases[len(releases)-1:])
// downloader.GetIssues() // downloader.GetIssues()
issues, err := downloader.GetIssues(0, 3) issues, isEnd, err := downloader.GetIssues(1, 8)
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, 3, len(issues)) assert.EqualValues(t, 3, len(issues))
assert.False(t, isEnd)
var ( var (
closed1 = time.Date(2018, 10, 23, 02, 57, 43, 0, time.UTC) closed1 = time.Date(2018, 10, 23, 02, 57, 43, 0, time.UTC)
) )
@ -319,7 +321,7 @@ something like in the latest 15days could be enough don't you think ?
}, comments[:3]) }, comments[:3])
// downloader.GetPullRequests() // downloader.GetPullRequests()
prs, err := downloader.GetPullRequests(0, 3) prs, err := downloader.GetPullRequests(1, 3)
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, 3, len(prs)) assert.EqualValues(t, 3, len(prs))

View file

@ -128,8 +128,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.Issues { if opts.Issues {
log.Trace("migrating issues and comments") log.Trace("migrating issues and comments")
for { for i := 1; ; i++ {
issues, err := downloader.GetIssues(0, 100) issues, isEnd, err := downloader.GetIssues(i, 100)
if err != nil { if err != nil {
return err return err
} }
@ -160,7 +160,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
} }
} }
if len(issues) < 100 { if isEnd {
break break
} }
} }
@ -168,8 +168,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.PullRequests { if opts.PullRequests {
log.Trace("migrating pull requests and comments") log.Trace("migrating pull requests and comments")
for { for i := 1; ; i++ {
prs, err := downloader.GetPullRequests(0, 100) prs, err := downloader.GetPullRequests(i, 100)
if err != nil { if err != nil {
return err return err
} }