Preserve file size when creating attachments (#23406) (#23426)

Backport #23406 by @baez90

When creating attachments (issue, release, repo) the file size (being
part of the multipart file header) is passed through the chain of
creating an attachment to ensure the MinIO client can stream the file
directly instead of having to read it to memory completely at first.

Fixes #23393

Co-authored-by: Peter <peter.kurfer@googlemail.com>
Co-authored-by: KN4CK3R <admin@oldschoolhack.me>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
Giteabot 2023-03-12 07:45:39 -04:00 committed by GitHub
parent e87f36e885
commit 0d9b44c0e3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 15 additions and 12 deletions

View file

@ -176,7 +176,7 @@ func CreateIssueAttachment(ctx *context.APIContext) {
filename = query filename = query
} }
attachment, err := attachment.UploadAttachment(file, setting.Attachment.AllowedTypes, &repo_model.Attachment{ attachment, err := attachment.UploadAttachment(file, setting.Attachment.AllowedTypes, header.Size, &repo_model.Attachment{
Name: filename, Name: filename,
UploaderID: ctx.Doer.ID, UploaderID: ctx.Doer.ID,
RepoID: ctx.Repo.Repository.ID, RepoID: ctx.Repo.Repository.ID,

View file

@ -180,7 +180,7 @@ func CreateIssueCommentAttachment(ctx *context.APIContext) {
filename = query filename = query
} }
attachment, err := attachment.UploadAttachment(file, setting.Attachment.AllowedTypes, &repo_model.Attachment{ attachment, err := attachment.UploadAttachment(file, setting.Attachment.AllowedTypes, header.Size, &repo_model.Attachment{
Name: filename, Name: filename,
UploaderID: ctx.Doer.ID, UploaderID: ctx.Doer.ID,
RepoID: ctx.Repo.Repository.ID, RepoID: ctx.Repo.Repository.ID,

View file

@ -194,7 +194,7 @@ func CreateReleaseAttachment(ctx *context.APIContext) {
} }
// Create a new attachment and save the file // Create a new attachment and save the file
attach, err := attachment.UploadAttachment(file, setting.Repository.Release.AllowedTypes, &repo_model.Attachment{ attach, err := attachment.UploadAttachment(file, setting.Repository.Release.AllowedTypes, header.Size, &repo_model.Attachment{
Name: filename, Name: filename,
UploaderID: ctx.Doer.ID, UploaderID: ctx.Doer.ID,
RepoID: release.RepoID, RepoID: release.RepoID,

View file

@ -44,7 +44,7 @@ func uploadAttachment(ctx *context.Context, repoID int64, allowedTypes string) {
} }
defer file.Close() defer file.Close()
attach, err := attachment.UploadAttachment(file, allowedTypes, &repo_model.Attachment{ attach, err := attachment.UploadAttachment(file, allowedTypes, header.Size, &repo_model.Attachment{
Name: header.Filename, Name: header.Filename,
UploaderID: ctx.Doer.ID, UploaderID: ctx.Doer.ID,
RepoID: repoID, RepoID: repoID,

View file

@ -19,14 +19,14 @@ import (
) )
// NewAttachment creates a new attachment object, but do not verify. // NewAttachment creates a new attachment object, but do not verify.
func NewAttachment(attach *repo_model.Attachment, file io.Reader) (*repo_model.Attachment, error) { func NewAttachment(attach *repo_model.Attachment, file io.Reader, size int64) (*repo_model.Attachment, error) {
if attach.RepoID == 0 { if attach.RepoID == 0 {
return nil, fmt.Errorf("attachment %s should belong to a repository", attach.Name) return nil, fmt.Errorf("attachment %s should belong to a repository", attach.Name)
} }
err := db.WithTx(db.DefaultContext, func(ctx context.Context) error { err := db.WithTx(db.DefaultContext, func(ctx context.Context) error {
attach.UUID = uuid.New().String() attach.UUID = uuid.New().String()
size, err := storage.Attachments.Save(attach.RelativePath(), file, -1) size, err := storage.Attachments.Save(attach.RelativePath(), file, size)
if err != nil { if err != nil {
return fmt.Errorf("Create: %w", err) return fmt.Errorf("Create: %w", err)
} }
@ -39,7 +39,7 @@ func NewAttachment(attach *repo_model.Attachment, file io.Reader) (*repo_model.A
} }
// UploadAttachment upload new attachment into storage and update database // UploadAttachment upload new attachment into storage and update database
func UploadAttachment(file io.Reader, allowedTypes string, opts *repo_model.Attachment) (*repo_model.Attachment, error) { func UploadAttachment(file io.Reader, allowedTypes string, fileSize int64, opts *repo_model.Attachment) (*repo_model.Attachment, error) {
buf := make([]byte, 1024) buf := make([]byte, 1024)
n, _ := util.ReadAtMost(file, buf) n, _ := util.ReadAtMost(file, buf)
buf = buf[:n] buf = buf[:n]
@ -48,5 +48,5 @@ func UploadAttachment(file io.Reader, allowedTypes string, opts *repo_model.Atta
return nil, err return nil, err
} }
return NewAttachment(opts, io.MultiReader(bytes.NewReader(buf), file)) return NewAttachment(opts, io.MultiReader(bytes.NewReader(buf), file), fileSize)
} }

View file

@ -36,7 +36,7 @@ func TestUploadAttachment(t *testing.T) {
RepoID: 1, RepoID: 1,
UploaderID: user.ID, UploaderID: user.ID,
Name: filepath.Base(fPath), Name: filepath.Base(fPath),
}, f) }, f, -1)
assert.NoError(t, err) assert.NoError(t, err)
attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, attach.UUID) attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, attach.UUID)

View file

@ -87,7 +87,7 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, doer *u
attachmentIDs := make([]string, 0, len(content.Attachments)) attachmentIDs := make([]string, 0, len(content.Attachments))
if setting.Attachment.Enabled { if setting.Attachment.Enabled {
for _, attachment := range content.Attachments { for _, attachment := range content.Attachments {
a, err := attachment_service.UploadAttachment(bytes.NewReader(attachment.Content), setting.Attachment.AllowedTypes, &repo_model.Attachment{ a, err := attachment_service.UploadAttachment(bytes.NewReader(attachment.Content), setting.Attachment.AllowedTypes, int64(len(attachment.Content)), &repo_model.Attachment{
Name: attachment.Name, Name: attachment.Name,
UploaderID: doer.ID, UploaderID: doer.ID,
RepoID: issue.Repo.ID, RepoID: issue.Repo.ID,

View file

@ -106,11 +106,13 @@ func TestRelease_Create(t *testing.T) {
IsTag: false, IsTag: false,
}, nil, "")) }, nil, ""))
testPlayload := "testtest"
attach, err := attachment.NewAttachment(&repo_model.Attachment{ attach, err := attachment.NewAttachment(&repo_model.Attachment{
RepoID: repo.ID, RepoID: repo.ID,
UploaderID: user.ID, UploaderID: user.ID,
Name: "test.txt", Name: "test.txt",
}, strings.NewReader("testtest")) }, strings.NewReader(testPlayload), int64(len([]byte(testPlayload))))
assert.NoError(t, err) assert.NoError(t, err)
release := repo_model.Release{ release := repo_model.Release{
@ -239,11 +241,12 @@ func TestRelease_Update(t *testing.T) {
assert.Equal(t, tagName, release.TagName) assert.Equal(t, tagName, release.TagName)
// Add new attachments // Add new attachments
samplePayload := "testtest"
attach, err := attachment.NewAttachment(&repo_model.Attachment{ attach, err := attachment.NewAttachment(&repo_model.Attachment{
RepoID: repo.ID, RepoID: repo.ID,
UploaderID: user.ID, UploaderID: user.ID,
Name: "test.txt", Name: "test.txt",
}, strings.NewReader("testtest")) }, strings.NewReader(samplePayload), int64(len([]byte(samplePayload))))
assert.NoError(t, err) assert.NoError(t, err)
assert.NoError(t, UpdateRelease(user, gitRepo, release, []string{attach.UUID}, nil, nil)) assert.NoError(t, UpdateRelease(user, gitRepo, release, []string{attach.UUID}, nil, nil))