Fix bug isEnd detection on getIssues/getPullRequests (#13299)
This commit is contained in:
Lunny Xiao
GitHub
parent
eab5837142
commit
d130cd147f
2 changed files with 50 additions and 32 deletions
|
|
@ -65,23 +65,25 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
|
|||
// GithubDownloaderV3 implements a Downloader interface to get repository informations
|
||||
// from github via APIv3
|
||||
type GithubDownloaderV3 struct {
|
||||
ctx context.Context
|
||||
client *github.Client
|
||||
repoOwner string
|
||||
repoName string
|
||||
userName string
|
||||
password string
|
||||
rate *github.Rate
|
||||
ctx context.Context
|
||||
client *github.Client
|
||||
repoOwner string
|
||||
repoName string
|
||||
userName string
|
||||
password string
|
||||
rate *github.Rate
|
||||
maxPerPage int
|
||||
}
|
||||
|
||||
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
|
||||
func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
|
||||
var downloader = GithubDownloaderV3{
|
||||
userName: userName,
|
||||
password: password,
|
||||
ctx: ctx,
|
||||
repoOwner: repoOwner,
|
||||
repoName: repoName,
|
||||
userName: userName,
|
||||
password: password,
|
||||
ctx: ctx,
|
||||
repoOwner: repoOwner,
|
||||
repoName: repoName,
|
||||
maxPerPage: 100,
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
|
|
@ -177,7 +179,7 @@ func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
|
|||
|
||||
// GetMilestones returns milestones
|
||||
func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
|
||||
var perPage = 100
|
||||
var perPage = g.maxPerPage
|
||||
var milestones = make([]*base.Milestone, 0, perPage)
|
||||
for i := 1; ; i++ {
|
||||
g.sleep()
|
||||
|
|
@ -233,7 +235,7 @@ func convertGithubLabel(label *github.Label) *base.Label {
|
|||
|
||||
// GetLabels returns labels
|
||||
func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
|
||||
var perPage = 100
|
||||
var perPage = g.maxPerPage
|
||||
var labels = make([]*base.Label, 0, perPage)
|
||||
for i := 1; ; i++ {
|
||||
g.sleep()
|
||||
|
|
@ -304,7 +306,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
|
|||
|
||||
// GetReleases returns releases
|
||||
func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
|
||||
var perPage = 100
|
||||
var perPage = g.maxPerPage
|
||||
var releases = make([]*base.Release, 0, perPage)
|
||||
for i := 1; ; i++ {
|
||||
g.sleep()
|
||||
|
|
@ -342,6 +344,9 @@ func (g *GithubDownloaderV3) GetAsset(_ string, _, id int64) (io.ReadCloser, err
|
|||
|
||||
// GetIssues returns issues according start and limit
|
||||
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
|
||||
if perPage > g.maxPerPage {
|
||||
perPage = g.maxPerPage
|
||||
}
|
||||
opt := &github.IssueListByRepoOptions{
|
||||
Sort: "created",
|
||||
Direction: "asc",
|
||||
|
|
@ -429,7 +434,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
|
|||
// GetComments returns comments according issueNumber
|
||||
func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, error) {
|
||||
var (
|
||||
allComments = make([]*base.Comment, 0, 100)
|
||||
allComments = make([]*base.Comment, 0, g.maxPerPage)
|
||||
created = "created"
|
||||
asc = "asc"
|
||||
)
|
||||
|
|
@ -437,7 +442,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
|
|||
Sort: &created,
|
||||
Direction: &asc,
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: 100,
|
||||
PerPage: g.maxPerPage,
|
||||
},
|
||||
}
|
||||
for {
|
||||
|
|
@ -459,7 +464,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
|
|||
g.sleep()
|
||||
res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
|
||||
Page: i,
|
||||
PerPage: 100,
|
||||
PerPage: g.maxPerPage,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -497,6 +502,9 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
|
|||
|
||||
// GetPullRequests returns pull requests according page and perPage
|
||||
func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
|
||||
if perPage > g.maxPerPage {
|
||||
perPage = g.maxPerPage
|
||||
}
|
||||
opt := &github.PullRequestListOptions{
|
||||
Sort: "created",
|
||||
Direction: "asc",
|
||||
|
|
@ -650,7 +658,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
|
|||
g.sleep()
|
||||
res, resp, err := g.client.Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
|
||||
Page: i,
|
||||
PerPage: 100,
|
||||
PerPage: g.maxPerPage,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -687,9 +695,9 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
|
|||
|
||||
// GetReviews returns pull requests review
|
||||
func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
|
||||
var allReviews = make([]*base.Review, 0, 100)
|
||||
var allReviews = make([]*base.Review, 0, g.maxPerPage)
|
||||
opt := &github.ListOptions{
|
||||
PerPage: 100,
|
||||
PerPage: g.maxPerPage,
|
||||
}
|
||||
for {
|
||||
g.sleep()
|
||||
|
|
@ -703,7 +711,7 @@ func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review
|
|||
r.IssueIndex = pullRequestNumber
|
||||
// retrieve all review comments
|
||||
opt2 := &github.ListOptions{
|
||||
PerPage: 100,
|
||||
PerPage: g.maxPerPage,
|
||||
}
|
||||
for {
|
||||
g.sleep()
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ type GitlabDownloader struct {
|
|||
repoName string
|
||||
issueCount int64
|
||||
fetchPRcomments bool
|
||||
maxPerPage int
|
||||
}
|
||||
|
||||
// NewGitlabDownloader creates a gitlab Downloader via gitlab API
|
||||
|
|
@ -99,10 +100,11 @@ func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, passw
|
|||
}
|
||||
|
||||
return &GitlabDownloader{
|
||||
ctx: ctx,
|
||||
client: gitlabClient,
|
||||
repoID: gr.ID,
|
||||
repoName: gr.Name,
|
||||
ctx: ctx,
|
||||
client: gitlabClient,
|
||||
repoID: gr.ID,
|
||||
repoName: gr.Name,
|
||||
maxPerPage: 100,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -159,7 +161,7 @@ func (g *GitlabDownloader) GetTopics() ([]string, error) {
|
|||
|
||||
// GetMilestones returns milestones
|
||||
func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
|
||||
var perPage = 100
|
||||
var perPage = g.maxPerPage
|
||||
var state = "all"
|
||||
var milestones = make([]*base.Milestone, 0, perPage)
|
||||
for i := 1; ; i++ {
|
||||
|
|
@ -230,7 +232,7 @@ func (g *GitlabDownloader) normalizeColor(val string) string {
|
|||
|
||||
// GetLabels returns labels
|
||||
func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
|
||||
var perPage = 100
|
||||
var perPage = g.maxPerPage
|
||||
var labels = make([]*base.Label, 0, perPage)
|
||||
for i := 1; ; i++ {
|
||||
ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{
|
||||
|
|
@ -281,7 +283,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
|
|||
|
||||
// GetReleases returns releases
|
||||
func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
|
||||
var perPage = 100
|
||||
var perPage = g.maxPerPage
|
||||
var releases = make([]*base.Release, 0, perPage)
|
||||
for i := 1; ; i++ {
|
||||
ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{
|
||||
|
|
@ -330,6 +332,10 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
|
|||
state := "all"
|
||||
sort := "asc"
|
||||
|
||||
if perPage > g.maxPerPage {
|
||||
perPage = g.maxPerPage
|
||||
}
|
||||
|
||||
opt := &gitlab.ListProjectIssuesOptions{
|
||||
State: &state,
|
||||
Sort: &sort,
|
||||
|
|
@ -401,7 +407,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
|
|||
// GetComments returns comments according issueNumber
|
||||
// TODO: figure out how to transfer comment reactions
|
||||
func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) {
|
||||
var allComments = make([]*base.Comment, 0, 100)
|
||||
var allComments = make([]*base.Comment, 0, g.maxPerPage)
|
||||
|
||||
var page = 1
|
||||
var realIssueNumber int64
|
||||
|
|
@ -415,14 +421,14 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro
|
|||
realIssueNumber = issueNumber
|
||||
comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListIssueDiscussionsOptions{
|
||||
Page: page,
|
||||
PerPage: 100,
|
||||
PerPage: g.maxPerPage,
|
||||
}, nil, gitlab.WithContext(g.ctx))
|
||||
} else {
|
||||
// If this is a PR, we need to figure out the Gitlab/original PR ID to be passed below
|
||||
realIssueNumber = issueNumber - g.issueCount
|
||||
comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListMergeRequestDiscussionsOptions{
|
||||
Page: page,
|
||||
PerPage: 100,
|
||||
PerPage: g.maxPerPage,
|
||||
}, nil, gitlab.WithContext(g.ctx))
|
||||
}
|
||||
|
||||
|
|
@ -465,6 +471,10 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro
|
|||
|
||||
// GetPullRequests returns pull requests according page and perPage
|
||||
func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
|
||||
if perPage > g.maxPerPage {
|
||||
perPage = g.maxPerPage
|
||||
}
|
||||
|
||||
opt := &gitlab.ListProjectMergeRequestsOptions{
|
||||
ListOptions: gitlab.ListOptions{
|
||||
PerPage: perPage,
|
||||
|
|
|
|||
Reference in a new issue