Memory usage improvements (#3013)
* govendor update code.gitea.io/git Signed-off-by: Duncan Ogilvie <mr.exodia.tpodt@gmail.com> * Greatly improve memory usage Signed-off-by: Duncan Ogilvie <mr.exodia.tpodt@gmail.com>
This commit is contained in:
parent
4035ab05fa
commit
551f3cbe42
9 changed files with 86 additions and 22 deletions
|
@ -143,6 +143,9 @@ func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if treeEntry.Blob().Size() >= setting.UI.MaxDisplayFileSize {
|
||||
return nil, git.ErrNotExist{ID: "", RelPath: ".editorconfig"}
|
||||
}
|
||||
reader, err := treeEntry.Blob().Data()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
@ -45,10 +45,11 @@ func ServeData(ctx *context.Context, name string, reader io.Reader) error {
|
|||
|
||||
// ServeBlob download a git.Blob
|
||||
func ServeBlob(ctx *context.Context, blob *git.Blob) error {
|
||||
dataRc, err := blob.Data()
|
||||
dataRc, err := blob.DataAsync()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dataRc.Close()
|
||||
|
||||
return ServeData(ctx, ctx.Repo.TreePath, dataRc)
|
||||
}
|
||||
|
|
|
@ -73,11 +73,16 @@ func editFile(ctx *context.Context, isNewFile bool) {
|
|||
|
||||
// No way to edit a directory online.
|
||||
if entry.IsDir() {
|
||||
ctx.Handle(404, "", nil)
|
||||
ctx.Handle(404, "entry.IsDir", nil)
|
||||
return
|
||||
}
|
||||
|
||||
blob := entry.Blob()
|
||||
if blob.Size() >= setting.UI.MaxDisplayFileSize {
|
||||
ctx.Handle(404, "blob.Size", err)
|
||||
return
|
||||
}
|
||||
|
||||
dataRc, err := blob.Data()
|
||||
if err != nil {
|
||||
ctx.Handle(404, "blob.Data", err)
|
||||
|
@ -93,7 +98,7 @@ func editFile(ctx *context.Context, isNewFile bool) {
|
|||
|
||||
// Only text file are editable online.
|
||||
if !base.IsTextFile(buf) {
|
||||
ctx.Handle(404, "", nil)
|
||||
ctx.Handle(404, "base.IsTextFile", nil)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
@ -319,6 +319,9 @@ func getFileContentFromDefaultBranch(ctx *context.Context, filename string) (str
|
|||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
if entry.Blob().Size() >= setting.UI.MaxDisplayFileSize {
|
||||
return "", false
|
||||
}
|
||||
r, err = entry.Blob().Data()
|
||||
if err != nil {
|
||||
return "", false
|
||||
|
|
|
@ -76,11 +76,12 @@ func renderDirectory(ctx *context.Context, treeLink string) {
|
|||
ctx.Data["ReadmeInList"] = true
|
||||
ctx.Data["ReadmeExist"] = true
|
||||
|
||||
dataRc, err := readmeFile.Data()
|
||||
dataRc, err := readmeFile.DataAsync()
|
||||
if err != nil {
|
||||
ctx.Handle(500, "Data", err)
|
||||
return
|
||||
}
|
||||
defer dataRc.Close()
|
||||
|
||||
buf := make([]byte, 1024)
|
||||
n, _ := dataRc.Read(buf)
|
||||
|
@ -91,14 +92,21 @@ func renderDirectory(ctx *context.Context, treeLink string) {
|
|||
ctx.Data["FileName"] = readmeFile.Name()
|
||||
// FIXME: what happens when README file is an image?
|
||||
if isTextFile {
|
||||
d, _ := ioutil.ReadAll(dataRc)
|
||||
buf = append(buf, d...)
|
||||
if markup.Type(readmeFile.Name()) != "" {
|
||||
ctx.Data["IsMarkup"] = true
|
||||
ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas()))
|
||||
if readmeFile.Size() >= setting.UI.MaxDisplayFileSize {
|
||||
// Pretend that this is a normal text file to display 'This file is too large to be shown'
|
||||
ctx.Data["IsFileTooLarge"] = true
|
||||
ctx.Data["IsTextFile"] = true
|
||||
ctx.Data["FileSize"] = readmeFile.Size()
|
||||
} else {
|
||||
ctx.Data["IsRenderedHTML"] = true
|
||||
ctx.Data["FileContent"] = string(bytes.Replace(buf, []byte("\n"), []byte(`<br>`), -1))
|
||||
d, _ := ioutil.ReadAll(dataRc)
|
||||
buf = append(buf, d...)
|
||||
if markup.Type(readmeFile.Name()) != "" {
|
||||
ctx.Data["IsMarkup"] = true
|
||||
ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas()))
|
||||
} else {
|
||||
ctx.Data["IsRenderedHTML"] = true
|
||||
ctx.Data["FileContent"] = string(bytes.Replace(buf, []byte("\n"), []byte(`<br>`), -1))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -135,11 +143,12 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
|
|||
ctx.Data["IsViewFile"] = true
|
||||
|
||||
blob := entry.Blob()
|
||||
dataRc, err := blob.Data()
|
||||
dataRc, err := blob.DataAsync()
|
||||
if err != nil {
|
||||
ctx.Handle(500, "Data", err)
|
||||
ctx.Handle(500, "DataAsync", err)
|
||||
return
|
||||
}
|
||||
defer dataRc.Close()
|
||||
|
||||
ctx.Data["FileSize"] = blob.Size()
|
||||
ctx.Data["FileName"] = blob.Name()
|
||||
|
|
50
vendor/code.gitea.io/git/blob.go
generated
vendored
50
vendor/code.gitea.io/git/blob.go
generated
vendored
|
@ -6,7 +6,11 @@ package git
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
// Blob represents a Git object.
|
||||
|
@ -18,14 +22,52 @@ type Blob struct {
|
|||
// Data gets content of blob all at once and wrap it as io.Reader.
|
||||
// This can be very slow and memory consuming for huge content.
|
||||
func (b *Blob) Data() (io.Reader, error) {
|
||||
stdout, err := NewCommand("show", b.ID.String()).RunInDirBytes(b.repo.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
stdout := new(bytes.Buffer)
|
||||
stderr := new(bytes.Buffer)
|
||||
|
||||
// Preallocate memory to save ~50% memory usage on big files.
|
||||
stdout.Grow(int(b.Size() + 2048))
|
||||
|
||||
if err := b.DataPipeline(stdout, stderr); err != nil {
|
||||
return nil, concatenateError(err, stderr.String())
|
||||
}
|
||||
return bytes.NewBuffer(stdout), nil
|
||||
return stdout, nil
|
||||
}
|
||||
|
||||
// DataPipeline gets content of blob and write the result or error to stdout or stderr
|
||||
func (b *Blob) DataPipeline(stdout, stderr io.Writer) error {
|
||||
return NewCommand("show", b.ID.String()).RunInDirPipeline(b.repo.Path, stdout, stderr)
|
||||
}
|
||||
|
||||
type cmdReadCloser struct {
|
||||
cmd *exec.Cmd
|
||||
stdout io.Reader
|
||||
}
|
||||
|
||||
func (c cmdReadCloser) Read(p []byte) (int, error) {
|
||||
return c.stdout.Read(p)
|
||||
}
|
||||
|
||||
func (c cmdReadCloser) Close() error {
|
||||
io.Copy(ioutil.Discard, c.stdout)
|
||||
return c.cmd.Wait()
|
||||
}
|
||||
|
||||
// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
|
||||
// Calling the Close function on the result will discard all unread output.
|
||||
func (b *Blob) DataAsync() (io.ReadCloser, error) {
|
||||
cmd := exec.Command("git", "show", b.ID.String())
|
||||
cmd.Dir = b.repo.Path
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("StdoutPipe: %v", err)
|
||||
}
|
||||
|
||||
if err = cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("Start: %v", err)
|
||||
}
|
||||
|
||||
return cmdReadCloser{stdout: stdout, cmd: cmd}, nil
|
||||
}
|
||||
|
|
3
vendor/code.gitea.io/git/commit.go
generated
vendored
3
vendor/code.gitea.io/git/commit.go
generated
vendored
|
@ -98,10 +98,11 @@ func (c *Commit) IsImageFile(name string) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
dataRc, err := blob.Data()
|
||||
dataRc, err := blob.DataAsync()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer dataRc.Close()
|
||||
buf := make([]byte, 1024)
|
||||
n, _ := dataRc.Read(buf)
|
||||
buf = buf[:n]
|
||||
|
|
2
vendor/code.gitea.io/git/git.go
generated
vendored
2
vendor/code.gitea.io/git/git.go
generated
vendored
|
@ -25,7 +25,7 @@ var (
|
|||
// Prefix the log prefix
|
||||
Prefix = "[git-module] "
|
||||
// GitVersionRequired is the minimum Git version required
|
||||
GitVersionRequired = "1.8.1.6"
|
||||
GitVersionRequired = "1.7.2"
|
||||
)
|
||||
|
||||
func log(format string, args ...interface{}) {
|
||||
|
|
6
vendor/vendor.json
vendored
6
vendor/vendor.json
vendored
|
@ -3,10 +3,10 @@
|
|||
"ignore": "test appengine",
|
||||
"package": [
|
||||
{
|
||||
"checksumSHA1": "JN/re4+x/hCzMLGHmieUcykVDAg=",
|
||||
"checksumSHA1": "vAVjAz7Wpjnu7GGba4JLIDTpQEw=",
|
||||
"path": "code.gitea.io/git",
|
||||
"revision": "d47b98c44c9a6472e44ab80efe65235e11c6da2a",
|
||||
"revisionTime": "2017-10-23T00:52:09Z"
|
||||
"revision": "f9dd6826bbb51c92c6964ce18176c304ea286e54",
|
||||
"revisionTime": "2017-11-28T15:25:05Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "QQ7g7B9+EIzGjO14KCGEs9TNEzM=",
|
||||
|
|
Reference in a new issue