From af73e1ee35a046f7fe2e856a286946303a8dcf82 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sun, 4 Apr 2021 17:04:55 +0100 Subject: [PATCH] Add size to Save function (#15264) (#15270) This PR proposes an alternative solution to #15255 - just add the size to the save function. Yes it is less apparently clean but it may be more correct. Close #15255 Fix #15253 Signed-off-by: Andrew Thornton Co-authored-by: Lunny Xiao --- integrations/attachment_test.go | 2 +- models/attachment.go | 2 +- modules/lfs/content_store.go | 2 +- modules/migrations/gitea_uploader.go | 2 +- modules/storage/local.go | 2 +- modules/storage/minio.go | 4 ++-- modules/storage/storage.go | 13 ++++++++++--- 7 files changed, 17 insertions(+), 10 deletions(-) diff --git a/integrations/attachment_test.go b/integrations/attachment_test.go index a28e38b99..052f27605 100644 --- a/integrations/attachment_test.go +++ b/integrations/attachment_test.go @@ -122,7 +122,7 @@ func TestGetAttachment(t *testing.T) { t.Run(tc.name, func(t *testing.T) { //Write empty file to be available for response if tc.createFile { - _, err := storage.Attachments.Save(models.AttachmentRelativePath(tc.uuid), strings.NewReader("hello world")) + _, err := storage.Attachments.Save(models.AttachmentRelativePath(tc.uuid), strings.NewReader("hello world"), -1) assert.NoError(t, err) } //Actual test diff --git a/models/attachment.go b/models/attachment.go index 478b4cd0d..972fe5680 100644 --- a/models/attachment.go +++ b/models/attachment.go @@ -85,7 +85,7 @@ func (a *Attachment) LinkedRepository() (*Repository, UnitType, error) { func NewAttachment(attach *Attachment, buf []byte, file io.Reader) (_ *Attachment, err error) { attach.UUID = gouuid.New().String() - size, err := storage.Attachments.Save(attach.RelativePath(), io.MultiReader(bytes.NewReader(buf), file)) + size, err := storage.Attachments.Save(attach.RelativePath(), io.MultiReader(bytes.NewReader(buf), file), -1) if err != nil { return nil, fmt.Errorf("Create: %v", err) } diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index 788ef5b9a..545d65c30 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -74,7 +74,7 @@ func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error { // now pass the wrapped reader to Save - if there is a size mismatch or hash mismatch then // the errors returned by the newHashingReader should percolate up to here - written, err := s.Save(p, wrappedRd) + written, err := s.Save(p, wrappedRd, meta.Size) if err != nil { log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err) return err diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index aa1ea4bc0..02f97c4ff 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -283,7 +283,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { } } defer rc.Close() - _, err = storage.Attachments.Save(attach.RelativePath(), rc) + _, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size)) return err }() if err != nil { diff --git a/modules/storage/local.go b/modules/storage/local.go index 982d2b88c..46e5d60e6 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -66,7 +66,7 @@ func (l *LocalStorage) Open(path string) (Object, error) { } // Save a file -func (l *LocalStorage) Save(path string, r io.Reader) (int64, error) { +func (l *LocalStorage) Save(path string, r io.Reader, size int64) (int64, error) { p := filepath.Join(l.dir, path) if err := os.MkdirAll(filepath.Dir(p), os.ModePerm); err != nil { return 0, err diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 0e0cb3690..724445c0a 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -131,13 +131,13 @@ func (m *MinioStorage) Open(path string) (Object, error) { } // Save save a file to minio -func (m *MinioStorage) Save(path string, r io.Reader) (int64, error) { +func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) { uploadInfo, err := m.client.PutObject( m.ctx, m.bucket, m.buildMinioPath(path), r, - -1, + size, minio.PutObjectOptions{ContentType: "application/octet-stream"}, ) if err != nil { diff --git a/modules/storage/storage.go b/modules/storage/storage.go index ec3a1c14a..65f8978e5 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -65,7 +65,8 @@ type Object interface { // ObjectStorage represents an object storage to handle a bucket and files type ObjectStorage interface { Open(path string) (Object, error) - Save(path string, r io.Reader) (int64, error) + // Save store a object, if size is unknown set -1 + Save(path string, r io.Reader, size int64) (int64, error) Stat(path string) (os.FileInfo, error) Delete(path string) error URL(path, name string) (*url.URL, error) @@ -80,7 +81,13 @@ func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, sr } defer f.Close() - return dstStorage.Save(dstPath, f) + size := int64(-1) + fsinfo, err := f.Stat() + if err == nil { + size = fsinfo.Size() + } + + return dstStorage.Save(dstPath, f, size) } // SaveFrom saves data to the ObjectStorage with path p from the callback @@ -94,7 +101,7 @@ func SaveFrom(objStorage ObjectStorage, p string, callback func(w io.Writer) err } }() - _, err := objStorage.Save(p, pr) + _, err := objStorage.Save(p, pr, -1) return err }