Add stat to ToCommit function for speed (#21337)

Calls to ToCommit are very slow due to fetching diffs, analyzing files.
This patch lets us supply `stat` as false to speed fetching a commit
when we don't need the diff.

/v1/repo/commits has a default `stat` set as true now. Set to false to
experience fetching thousands of commits per second instead of 2-5 per
second.
This commit is contained in:
Gennady Kovshenin 2022-10-06 06:21:04 +03:00 committed by GitHub
parent 8765f139c7
commit fd2d5f06b0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 49 additions and 31 deletions

View file

@ -73,7 +73,7 @@ func ToPayloadCommit(repo *repo_model.Repository, c *git.Commit) *api.PayloadCom
} }
// ToCommit convert a git.Commit to api.Commit // ToCommit convert a git.Commit to api.Commit
func ToCommit(repo *repo_model.Repository, gitRepo *git.Repository, commit *git.Commit, userCache map[string]*user_model.User) (*api.Commit, error) { func ToCommit(repo *repo_model.Repository, gitRepo *git.Repository, commit *git.Commit, userCache map[string]*user_model.User, stat bool) (*api.Commit, error) {
var apiAuthor, apiCommitter *api.User var apiAuthor, apiCommitter *api.User
// Retrieve author and committer information // Retrieve author and committer information
@ -133,28 +133,7 @@ func ToCommit(repo *repo_model.Repository, gitRepo *git.Repository, commit *git.
} }
} }
// Retrieve files affected by the commit res := &api.Commit{
fileStatus, err := git.GetCommitFileStatus(gitRepo.Ctx, repo.RepoPath(), commit.ID.String())
if err != nil {
return nil, err
}
affectedFileList := make([]*api.CommitAffectedFiles, 0, len(fileStatus.Added)+len(fileStatus.Removed)+len(fileStatus.Modified))
for _, files := range [][]string{fileStatus.Added, fileStatus.Removed, fileStatus.Modified} {
for _, filename := range files {
affectedFileList = append(affectedFileList, &api.CommitAffectedFiles{
Filename: filename,
})
}
}
diff, err := gitdiff.GetDiff(gitRepo, &gitdiff.DiffOptions{
AfterCommitID: commit.ID.String(),
})
if err != nil {
return nil, err
}
return &api.Commit{
CommitMeta: &api.CommitMeta{ CommitMeta: &api.CommitMeta{
URL: repo.APIURL() + "/git/commits/" + url.PathEscape(commit.ID.String()), URL: repo.APIURL() + "/git/commits/" + url.PathEscape(commit.ID.String()),
SHA: commit.ID.String(), SHA: commit.ID.String(),
@ -188,11 +167,37 @@ func ToCommit(repo *repo_model.Repository, gitRepo *git.Repository, commit *git.
Author: apiAuthor, Author: apiAuthor,
Committer: apiCommitter, Committer: apiCommitter,
Parents: apiParents, Parents: apiParents,
Files: affectedFileList, }
Stats: &api.CommitStats{
// Retrieve files affected by the commit
if stat {
fileStatus, err := git.GetCommitFileStatus(gitRepo.Ctx, repo.RepoPath(), commit.ID.String())
if err != nil {
return nil, err
}
affectedFileList := make([]*api.CommitAffectedFiles, 0, len(fileStatus.Added)+len(fileStatus.Removed)+len(fileStatus.Modified))
for _, files := range [][]string{fileStatus.Added, fileStatus.Removed, fileStatus.Modified} {
for _, filename := range files {
affectedFileList = append(affectedFileList, &api.CommitAffectedFiles{
Filename: filename,
})
}
}
diff, err := gitdiff.GetDiff(gitRepo, &gitdiff.DiffOptions{
AfterCommitID: commit.ID.String(),
})
if err != nil {
return nil, err
}
res.Files = affectedFileList
res.Stats = &api.CommitStats{
Total: diff.TotalAddition + diff.TotalDeletion, Total: diff.TotalAddition + diff.TotalDeletion,
Additions: diff.TotalAddition, Additions: diff.TotalAddition,
Deletions: diff.TotalDeletion, Deletions: diff.TotalDeletion,
}, }
}, nil }
return res, nil
} }

View file

@ -70,7 +70,7 @@ func getCommit(ctx *context.APIContext, identifier string) {
return return
} }
json, err := convert.ToCommit(ctx.Repo.Repository, ctx.Repo.GitRepo, commit, nil) json, err := convert.ToCommit(ctx.Repo.Repository, ctx.Repo.GitRepo, commit, nil, true)
if err != nil { if err != nil {
ctx.Error(http.StatusInternalServerError, "toCommit", err) ctx.Error(http.StatusInternalServerError, "toCommit", err)
return return
@ -104,6 +104,10 @@ func GetAllCommits(ctx *context.APIContext) {
// in: query // in: query
// description: filepath of a file/dir // description: filepath of a file/dir
// type: string // type: string
// - name: stat
// in: query
// description: include diff stats for every commit (disable for speedup, default 'true')
// type: boolean
// - name: page // - name: page
// in: query // in: query
// description: page number of results to return (1-based) // description: page number of results to return (1-based)
@ -209,9 +213,12 @@ func GetAllCommits(ctx *context.APIContext) {
userCache := make(map[string]*user_model.User) userCache := make(map[string]*user_model.User)
apiCommits := make([]*api.Commit, len(commits)) apiCommits := make([]*api.Commit, len(commits))
stat := ctx.FormString("stat") == "" || ctx.FormBool("stat")
for i, commit := range commits { for i, commit := range commits {
// Create json struct // Create json struct
apiCommits[i], err = convert.ToCommit(ctx.Repo.Repository, ctx.Repo.GitRepo, commit, userCache) apiCommits[i], err = convert.ToCommit(ctx.Repo.Repository, ctx.Repo.GitRepo, commit, userCache, stat)
if err != nil { if err != nil {
ctx.Error(http.StatusInternalServerError, "toCommit", err) ctx.Error(http.StatusInternalServerError, "toCommit", err)
return return

View file

@ -69,7 +69,7 @@ func getNote(ctx *context.APIContext, identifier string) {
return return
} }
cmt, err := convert.ToCommit(ctx.Repo.Repository, ctx.Repo.GitRepo, note.Commit, nil) cmt, err := convert.ToCommit(ctx.Repo.Repository, ctx.Repo.GitRepo, note.Commit, nil, true)
if err != nil { if err != nil {
ctx.Error(http.StatusInternalServerError, "ToCommit", err) ctx.Error(http.StatusInternalServerError, "ToCommit", err)
return return

View file

@ -1306,7 +1306,7 @@ func GetPullRequestCommits(ctx *context.APIContext) {
apiCommits := make([]*api.Commit, 0, end-start) apiCommits := make([]*api.Commit, 0, end-start)
for i := start; i < end; i++ { for i := start; i < end; i++ {
apiCommit, err := convert.ToCommit(ctx.Repo.Repository, baseGitRepo, commits[i], userCache) apiCommit, err := convert.ToCommit(ctx.Repo.Repository, baseGitRepo, commits[i], userCache, true)
if err != nil { if err != nil {
ctx.ServerError("toCommit", err) ctx.ServerError("toCommit", err)
return return

View file

@ -3264,6 +3264,12 @@
"name": "path", "name": "path",
"in": "query" "in": "query"
}, },
{
"type": "boolean",
"description": "include diff stats for every commit (disable for speedup, default 'true')",
"name": "stat",
"in": "query"
},
{ {
"type": "integer", "type": "integer",
"description": "page number of results to return (1-based)", "description": "page number of results to return (1-based)",