LFS support to be stored on minio (#12518)

* LFS support to be stored on minio

* Fix test

* Fix lint

* Fix lint

* Fix check

* Fix test

* Update documents and add migration for LFS

* Fix some bugs
This commit is contained in:
Lunny Xiao 2020-09-08 23:45:10 +08:00 committed by GitHub
parent e4b3f35b8d
commit 7a5465fc56
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 423 additions and 203 deletions

View file

@ -83,6 +83,13 @@ func migrateAttachments(dstStorage storage.ObjectStorage) error {
}) })
} }
func migrateLFS(dstStorage storage.ObjectStorage) error {
return models.IterateLFS(func(mo *models.LFSMetaObject) error {
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
return err
})
}
func runMigrateStorage(ctx *cli.Context) error { func runMigrateStorage(ctx *cli.Context) error {
if err := initDB(); err != nil { if err := initDB(); err != nil {
return err return err
@ -103,9 +110,6 @@ func runMigrateStorage(ctx *cli.Context) error {
return err return err
} }
tp := ctx.String("type")
switch tp {
case "attachments":
var dstStorage storage.ObjectStorage var dstStorage storage.ObjectStorage
var err error var err error
switch ctx.String("store") { switch ctx.String("store") {
@ -134,14 +138,22 @@ func runMigrateStorage(ctx *cli.Context) error {
if err != nil { if err != nil {
return err return err
} }
tp := ctx.String("type")
switch tp {
case "attachments":
if err := migrateAttachments(dstStorage); err != nil { if err := migrateAttachments(dstStorage); err != nil {
return err return err
} }
case "lfs":
if err := migrateLFS(dstStorage); err != nil {
return err
}
default:
return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
}
log.Warn("All files have been copied to the new placement but old files are still on the orignial placement.") log.Warn("All files have been copied to the new placement but old files are still on the orignial placement.")
return nil return nil
} }
return nil
}

View file

@ -206,12 +206,23 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
- `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars. - `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars.
- `ENABLE_GZIP`: **false**: Enables application-level GZIP support. - `ENABLE_GZIP`: **false**: Enables application-level GZIP support.
- `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\]. - `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\].
- `LFS_START_SERVER`: **false**: Enables git-lfs support. - `LFS_START_SERVER`: **false**: Enables git-lfs support.
- `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files. - `LFS_STORE_TYPE`: **local**: Storage type for lfs, `local` for local disk or `minio` for s3 compatible object storage service.
- `LFS_SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
- `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files, only available when `LFS_STORE_TYPE` is `local`.
- `LFS_MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `LFS_STORE_TYPE` is `minio`
- `LFS_MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `LFS_STORE_TYPE` is `minio`
- `LFS_MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `LFS_STORE_TYPE is` `minio`
- `LFS_MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `LFS_STORE_TYPE` is `minio`
- `LFS_MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `LFS_STORE_TYPE` is `minio`
- `LFS_MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `LFS_STORE_TYPE` is `minio`
- `LFS_MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `LFS_STORE_TYPE` is `minio`
- `LFS_JWT_SECRET`: **\<empty\>**: LFS authentication secret, change this a unique string. - `LFS_JWT_SECRET`: **\<empty\>**: LFS authentication secret, change this a unique string.
- `LFS_HTTP_AUTH_EXPIRY`: **20m**: LFS authentication validity period in time.Duration, pushes taking longer than this may fail. - `LFS_HTTP_AUTH_EXPIRY`: **20m**: LFS authentication validity period in time.Duration, pushes taking longer than this may fail.
- `LFS_MAX_FILE_SIZE`: **0**: Maximum allowed LFS file size in bytes (Set to 0 for no limit). - `LFS_MAX_FILE_SIZE`: **0**: Maximum allowed LFS file size in bytes (Set to 0 for no limit).
- `LFS_LOCK_PAGING_NUM`: **50**: Maximum number of LFS Locks returned per page. - `LFS_LOCK_PAGING_NUM`: **50**: Maximum number of LFS Locks returned per page.
- `REDIRECT_OTHER_PORT`: **false**: If true and `PROTOCOL` is https, allows redirecting http requests on `PORT_TO_REDIRECT` to the https port Gitea listens on. - `REDIRECT_OTHER_PORT`: **false**: If true and `PROTOCOL` is https, allows redirecting http requests on `PORT_TO_REDIRECT` to the https port Gitea listens on.
- `PORT_TO_REDIRECT`: **80**: Port for the http redirection service to listen on. Used when `REDIRECT_OTHER_PORT` is true. - `PORT_TO_REDIRECT`: **80**: Port for the http redirection service to listen on. Used when `REDIRECT_OTHER_PORT` is true.
- `ENABLE_LETSENCRYPT`: **false**: If enabled you must set `DOMAIN` to valid internet facing domain (ensure DNS is set and port 80 is accessible by letsencrypt validation server). - `ENABLE_LETSENCRYPT`: **false**: If enabled you must set `DOMAIN` to valid internet facing domain (ensure DNS is set and port 80 is accessible by letsencrypt validation server).

View file

@ -69,8 +69,18 @@ menu:
- `STATIC_CACHE_TIME`: **6h**: 静态资源文件,包括 `custom/`, `public/` 和所有上传的头像的浏览器缓存时间。 - `STATIC_CACHE_TIME`: **6h**: 静态资源文件,包括 `custom/`, `public/` 和所有上传的头像的浏览器缓存时间。
- `ENABLE_GZIP`: 启用应用级别的 GZIP 压缩。 - `ENABLE_GZIP`: 启用应用级别的 GZIP 压缩。
- `LANDING_PAGE`: 未登录用户的默认页面,可选 `home``explore` - `LANDING_PAGE`: 未登录用户的默认页面,可选 `home``explore`
- `LFS_START_SERVER`: 是否启用 git-lfs 支持. 可以为 `true``false` 默认是 `false` - `LFS_START_SERVER`: 是否启用 git-lfs 支持. 可以为 `true``false` 默认是 `false`
- `LFS_STORE_TYPE`: **local**: LFS 的存储类型,`local` 将存储到磁盘,`minio` 将存储到 s3 兼容的对象服务。
- `LFS_SERVE_DIRECT`: **false**: 允许直接重定向到存储系统。当前,仅 Minio/S3 是支持的。
- `LFS_CONTENT_PATH`: 存放 lfs 命令上传的文件的地方,默认是 `data/lfs` - `LFS_CONTENT_PATH`: 存放 lfs 命令上传的文件的地方,默认是 `data/lfs`
- `LFS_MINIO_ENDPOINT`: **localhost:9000**: Minio 地址,仅当 `LFS_STORE_TYPE``minio` 时有效。
- `LFS_MINIO_ACCESS_KEY_ID`: Minio accessKeyID仅当 `LFS_STORE_TYPE``minio` 时有效。
- `LFS_MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey仅当 `LFS_STORE_TYPE``minio` 时有效。
- `LFS_MINIO_BUCKET`: **gitea**: Minio bucket仅当 `LFS_STORE_TYPE``minio` 时有效。
- `LFS_MINIO_LOCATION`: **us-east-1**: Minio location ,仅当 `LFS_STORE_TYPE``minio` 时有效。
- `LFS_MINIO_BASE_PATH`: **lfs/**: Minio base path ,仅当 `LFS_STORE_TYPE``minio` 时有效。
- `LFS_MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ,仅当 `LFS_STORE_TYPE``minio` 时有效。
- `LFS_JWT_SECRET`: LFS 认证密钥,改成自己的。 - `LFS_JWT_SECRET`: LFS 认证密钥,改成自己的。
## Database (`database`) ## Database (`database`)

View file

@ -18,6 +18,7 @@ import (
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"gitea.com/macaron/gzip" "gitea.com/macaron/gzip"
gzipp "github.com/klauspost/compress/gzip" gzipp "github.com/klauspost/compress/gzip"
@ -49,8 +50,10 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string
lfsID++ lfsID++
lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject) lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
assert.NoError(t, err) assert.NoError(t, err)
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
if !contentStore.Exists(lfsMetaObject) { exist, err := contentStore.Exists(lfsMetaObject)
assert.NoError(t, err)
if !exist {
err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content)) err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content))
assert.NoError(t, err) assert.NoError(t, err)
} }

View file

@ -36,13 +36,23 @@ ROOT_URL = http://localhost:3001/
DISABLE_SSH = false DISABLE_SSH = false
SSH_LISTEN_HOST = localhost SSH_LISTEN_HOST = localhost
SSH_PORT = 2201 SSH_PORT = 2201
START_SSH_SERVER = true
LFS_START_SERVER = true
LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
OFFLINE_MODE = false
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
APP_DATA_PATH = integrations/gitea-integration-mysql/data APP_DATA_PATH = integrations/gitea-integration-mysql/data
BUILTIN_SSH_SERVER_USER = git BUILTIN_SSH_SERVER_USER = git
START_SSH_SERVER = true
OFFLINE_MODE = false
LFS_START_SERVER = true
LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
LFS_STORE_TYPE = minio
LFS_SERVE_DIRECT = false
LFS_MINIO_ENDPOINT = minio:9000
LFS_MINIO_ACCESS_KEY_ID = 123456
LFS_MINIO_SECRET_ACCESS_KEY = 12345678
LFS_MINIO_BUCKET = gitea
LFS_MINIO_LOCATION = us-east-1
LFS_MINIO_BASE_PATH = lfs/
LFS_MINIO_USE_SSL = false
[attachment] [attachment]
STORE_TYPE = minio STORE_TYPE = minio

View file

@ -10,6 +10,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"path"
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
@ -26,6 +27,15 @@ type LFSMetaObject struct {
CreatedUnix timeutil.TimeStamp `xorm:"created"` CreatedUnix timeutil.TimeStamp `xorm:"created"`
} }
// RelativePath returns the relative path of the lfs object
func (m *LFSMetaObject) RelativePath() string {
if len(m.Oid) < 5 {
return m.Oid
}
return path.Join(m.Oid[0:2], m.Oid[2:4], m.Oid[4:])
}
// Pointer returns the string representation of an LFS pointer file // Pointer returns the string representation of an LFS pointer file
func (m *LFSMetaObject) Pointer() string { func (m *LFSMetaObject) Pointer() string {
return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size) return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size)
@ -202,3 +212,25 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error {
return sess.Commit() return sess.Commit()
} }
// IterateLFS iterates lfs object
func IterateLFS(f func(mo *LFSMetaObject) error) error {
var start int
const batchSize = 100
for {
var mos = make([]*LFSMetaObject, 0, batchSize)
if err := x.Limit(batchSize, start).Find(&mos); err != nil {
return err
}
if len(mos) == 0 {
return nil
}
start += len(mos)
for _, mo := range mos {
if err := f(mo); err != nil {
return err
}
}
}
}

View file

@ -69,6 +69,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string) {
} }
setting.Attachment.Path = filepath.Join(setting.AppDataPath, "attachments") setting.Attachment.Path = filepath.Join(setting.AppDataPath, "attachments")
setting.LFS.ContentPath = filepath.Join(setting.AppDataPath, "lfs")
if err = storage.Init(); err != nil { if err = storage.Init(); err != nil {
fatalTestError("storage.Init: %v\n", err) fatalTestError("storage.Init: %v\n", err)
} }

View file

@ -10,11 +10,10 @@ import (
"errors" "errors"
"io" "io"
"os" "os"
"path/filepath"
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/storage"
) )
var ( var (
@ -24,17 +23,15 @@ var (
// ContentStore provides a simple file system based storage. // ContentStore provides a simple file system based storage.
type ContentStore struct { type ContentStore struct {
BasePath string storage.ObjectStorage
} }
// Get takes a Meta object and retrieves the content from the store, returning // Get takes a Meta object and retrieves the content from the store, returning
// it as an io.Reader. If fromByte > 0, the reader starts from that byte // it as an io.Reader. If fromByte > 0, the reader starts from that byte
func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) { func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
path := filepath.Join(s.BasePath, transformKey(meta.Oid)) f, err := s.Open(meta.RelativePath())
f, err := os.Open(path)
if err != nil { if err != nil {
log.Error("Whilst trying to read LFS OID[%s]: Unable to open %s Error: %v", meta.Oid, path, err) log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err)
return nil, err return nil, err
} }
if fromByte > 0 { if fromByte > 0 {
@ -48,82 +45,55 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
// Put takes a Meta object and an io.Reader and writes the content to the store. // Put takes a Meta object and an io.Reader and writes the content to the store.
func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error { func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
tmpPath := path + ".tmp"
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0750); err != nil {
log.Error("Whilst putting LFS OID[%s]: Unable to create the LFS directory: %s Error: %v", meta.Oid, dir, err)
return err
}
file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640)
if err != nil {
log.Error("Whilst putting LFS OID[%s]: Unable to open temporary file for writing: %s Error: %v", tmpPath, err)
return err
}
defer func() {
if err := util.Remove(tmpPath); err != nil {
log.Warn("Unable to remove temporary path: %s: Error: %v", tmpPath, err)
}
}()
hash := sha256.New() hash := sha256.New()
hw := io.MultiWriter(hash, file) rd := io.TeeReader(r, hash)
p := meta.RelativePath()
written, err := io.Copy(hw, r) written, err := s.Save(p, rd)
if err != nil { if err != nil {
log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, tmpPath, err) log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err)
file.Close()
return err return err
} }
file.Close()
if written != meta.Size { if written != meta.Size {
if err := s.Delete(p); err != nil {
log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
}
return errSizeMismatch return errSizeMismatch
} }
shaStr := hex.EncodeToString(hash.Sum(nil)) shaStr := hex.EncodeToString(hash.Sum(nil))
if shaStr != meta.Oid { if shaStr != meta.Oid {
return errHashMismatch if err := s.Delete(p); err != nil {
log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
} }
return errHashMismatch
if err := os.Rename(tmpPath, path); err != nil {
log.Error("Whilst putting LFS OID[%s]: Unable to move tmp file to final destination: %s Error: %v", meta.Oid, path, err)
return err
} }
return nil return nil
} }
// Exists returns true if the object exists in the content store. // Exists returns true if the object exists in the content store.
func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool { func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
path := filepath.Join(s.BasePath, transformKey(meta.Oid)) _, err := s.ObjectStorage.Stat(meta.RelativePath())
if _, err := os.Stat(path); os.IsNotExist(err) { if err != nil {
return false if os.IsNotExist(err) {
return false, nil
} }
return true return false, err
}
return true, nil
} }
// Verify returns true if the object exists in the content store and size is correct. // Verify returns true if the object exists in the content store and size is correct.
func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) { func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
path := filepath.Join(s.BasePath, transformKey(meta.Oid)) p := meta.RelativePath()
fi, err := s.ObjectStorage.Stat(p)
fi, err := os.Stat(path) if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) {
if os.IsNotExist(err) || err == nil && fi.Size() != meta.Size {
return false, nil return false, nil
} else if err != nil { } else if err != nil {
log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", path, meta.Oid, err) log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err)
return false, err return false, err
} }
return true, nil return true, nil
} }
func transformKey(key string) string {
if len(key) < 5 {
return key
}
return filepath.Join(key[0:2], key[2:4], key[4:])
}

View file

@ -12,6 +12,7 @@ import (
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
) )
// ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file // ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file
@ -53,9 +54,10 @@ func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
return nil return nil
} }
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} contentStore := &ContentStore{ObjectStorage: storage.LFS}
meta := &models.LFSMetaObject{Oid: oid, Size: size} meta := &models.LFSMetaObject{Oid: oid, Size: size}
if !contentStore.Exists(meta) { exist, err := contentStore.Exists(meta)
if err != nil || !exist {
return nil return nil
} }
@ -64,6 +66,6 @@ func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
// ReadMetaObject will read a models.LFSMetaObject and return a reader // ReadMetaObject will read a models.LFSMetaObject and return a reader
func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) { func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) {
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} contentStore := &ContentStore{ObjectStorage: storage.LFS}
return contentStore.Get(meta, 0) return contentStore.Get(meta, 0)
} }

View file

@ -20,6 +20,7 @@ import (
"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"gitea.com/macaron/macaron" "gitea.com/macaron/macaron"
"github.com/dgrijalva/jwt-go" "github.com/dgrijalva/jwt-go"
@ -187,7 +188,7 @@ func getContentHandler(ctx *context.Context) {
} }
} }
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} contentStore := &ContentStore{ObjectStorage: storage.LFS}
content, err := contentStore.Get(meta, fromByte) content, err := contentStore.Get(meta, fromByte)
if err != nil { if err != nil {
// Errors are logged in contentStore.Get // Errors are logged in contentStore.Get
@ -288,8 +289,14 @@ func PostHandler(ctx *context.Context) {
ctx.Resp.Header().Set("Content-Type", metaMediaType) ctx.Resp.Header().Set("Content-Type", metaMediaType)
sentStatus := 202 sentStatus := 202
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} contentStore := &ContentStore{ObjectStorage: storage.LFS}
if meta.Existing && contentStore.Exists(meta) { exist, err := contentStore.Exists(meta)
if err != nil {
log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err)
writeStatus(ctx, 500)
return
}
if meta.Existing && exist {
sentStatus = 200 sentStatus = 200
} }
ctx.Resp.WriteHeader(sentStatus) ctx.Resp.WriteHeader(sentStatus)
@ -343,13 +350,21 @@ func BatchHandler(ctx *context.Context) {
return return
} }
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} contentStore := &ContentStore{ObjectStorage: storage.LFS}
meta, err := repository.GetLFSMetaObjectByOid(object.Oid) meta, err := repository.GetLFSMetaObjectByOid(object.Oid)
if err == nil && contentStore.Exists(meta) { // Object is found and exists if err == nil { // Object is found and exists
exist, err := contentStore.Exists(meta)
if err != nil {
log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
writeStatus(ctx, 500)
return
}
if exist {
responseObjects = append(responseObjects, Represent(object, meta, true, false)) responseObjects = append(responseObjects, Represent(object, meta, true, false))
continue continue
} }
}
if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize { if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize {
log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", object.Oid, object.Size, object.User, object.Repo, setting.LFS.MaxFileSize) log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", object.Oid, object.Size, object.User, object.Repo, setting.LFS.MaxFileSize)
@ -360,7 +375,13 @@ func BatchHandler(ctx *context.Context) {
// Object is not found // Object is not found
meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID}) meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID})
if err == nil { if err == nil {
responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !contentStore.Exists(meta))) exist, err := contentStore.Exists(meta)
if err != nil {
log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
writeStatus(ctx, 500)
return
}
responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !exist))
} else { } else {
log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, object.User, object.Repo, err) log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, object.User, object.Repo, err)
} }
@ -387,7 +408,7 @@ func PutHandler(ctx *context.Context) {
return return
} }
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} contentStore := &ContentStore{ObjectStorage: storage.LFS}
bodyReader := ctx.Req.Body().ReadCloser() bodyReader := ctx.Req.Body().ReadCloser()
defer bodyReader.Close() defer bodyReader.Close()
if err := contentStore.Put(meta, bodyReader); err != nil { if err := contentStore.Put(meta, bodyReader); err != nil {
@ -429,7 +450,7 @@ func VerifyHandler(ctx *context.Context) {
return return
} }
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} contentStore := &ContentStore{ObjectStorage: storage.LFS}
ok, err := contentStore.Verify(meta) ok, err := contentStore.Verify(meta)
if err != nil { if err != nil {
// Error will be logged in Verify // Error will be logged in Verify

View file

@ -20,6 +20,7 @@ import (
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
repo_module "code.gitea.io/gitea/modules/repository" repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
pull_service "code.gitea.io/gitea/services/pull" pull_service "code.gitea.io/gitea/services/pull"
@ -433,8 +434,12 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up
if err != nil { if err != nil {
return nil, err return nil, err
} }
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
if !contentStore.Exists(lfsMetaObject) { exist, err := contentStore.Exists(lfsMetaObject)
if err != nil {
return nil, err
}
if !exist {
if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil { if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil {
if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil { if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err) return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err)

View file

@ -13,6 +13,7 @@ import (
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
) )
// UploadRepoFileOptions contains the uploaded repository file options // UploadRepoFileOptions contains the uploaded repository file options
@ -163,12 +164,16 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep
// OK now we can insert the data into the store - there's no way to clean up the store // OK now we can insert the data into the store - there's no way to clean up the store
// once it's in there, it's in there. // once it's in there, it's in there.
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
for _, uploadInfo := range infos { for _, uploadInfo := range infos {
if uploadInfo.lfsMetaObject == nil { if uploadInfo.lfsMetaObject == nil {
continue continue
} }
if !contentStore.Exists(uploadInfo.lfsMetaObject) { exist, err := contentStore.Exists(uploadInfo.lfsMetaObject)
if err != nil {
return cleanUpAfterFailure(&infos, t, err)
}
if !exist {
file, err := os.Open(uploadInfo.upload.LocalPath()) file, err := os.Open(uploadInfo.upload.LocalPath())
if err != nil { if err != nil {
return cleanUpAfterFailure(&infos, t, err) return cleanUpAfterFailure(&infos, t, err)

122
modules/setting/lfs.go Normal file
View file

@ -0,0 +1,122 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package setting
import (
"encoding/base64"
"os"
"path/filepath"
"time"
"code.gitea.io/gitea/modules/generate"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"github.com/unknwon/com"
ini "gopkg.in/ini.v1"
)
// LFS represents the configuration for Git LFS
var LFS = struct {
StartServer bool `ini:"LFS_START_SERVER"`
ContentPath string `ini:"LFS_CONTENT_PATH"`
JWTSecretBase64 string `ini:"LFS_JWT_SECRET"`
JWTSecretBytes []byte `ini:"-"`
HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"`
MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"`
LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"`
StoreType string
ServeDirect bool
Minio struct {
Endpoint string
AccessKeyID string
SecretAccessKey string
UseSSL bool
Bucket string
Location string
BasePath string
}
}{
StoreType: "local",
}
func newLFSService() {
sec := Cfg.Section("server")
if err := sec.MapTo(&LFS); err != nil {
log.Fatal("Failed to map LFS settings: %v", err)
}
LFS.ContentPath = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
if !filepath.IsAbs(LFS.ContentPath) {
LFS.ContentPath = filepath.Join(AppWorkPath, LFS.ContentPath)
}
if LFS.LocksPagingNum == 0 {
LFS.LocksPagingNum = 50
}
LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(20 * time.Minute)
if LFS.StartServer {
LFS.JWTSecretBytes = make([]byte, 32)
n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64))
if err != nil || n != 32 {
LFS.JWTSecretBase64, err = generate.NewJwtSecret()
if err != nil {
log.Fatal("Error generating JWT Secret for custom config: %v", err)
return
}
// Save secret
cfg := ini.Empty()
if com.IsFile(CustomConf) {
// Keeps custom settings if there is already something.
if err := cfg.Append(CustomConf); err != nil {
log.Error("Failed to load custom conf '%s': %v", CustomConf, err)
}
}
cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64)
if err := os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm); err != nil {
log.Fatal("Failed to create '%s': %v", CustomConf, err)
}
if err := cfg.SaveTo(CustomConf); err != nil {
log.Fatal("Error saving generated JWT Secret to custom config: %v", err)
return
}
}
}
}
func ensureLFSDirectory() {
if LFS.StartServer {
if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil {
log.Fatal("Failed to create '%s': %v", LFS.ContentPath, err)
}
}
}
// CheckLFSVersion will check lfs version, if not satisfied, then disable it.
func CheckLFSVersion() {
if LFS.StartServer {
//Disable LFS client hooks if installed for the current OS user
//Needs at least git v2.1.2
err := git.LoadGitVersion()
if err != nil {
log.Fatal("Error retrieving git version: %v", err)
}
if git.CheckGitVersionConstraint(">= 2.1.2") != nil {
LFS.StartServer = false
log.Error("LFS server support needs at least Git v2.1.2")
} else {
git.GlobalCommandArgs = append(git.GlobalCommandArgs, "-c", "filter.lfs.required=",
"-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=")
}
}
}

View file

@ -23,7 +23,6 @@ import (
"time" "time"
"code.gitea.io/gitea/modules/generate" "code.gitea.io/gitea/modules/generate"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/user" "code.gitea.io/gitea/modules/user"
@ -133,16 +132,6 @@ var (
MinimumKeySizes: map[string]int{"ed25519": 256, "ecdsa": 256, "rsa": 2048, "dsa": 1024}, MinimumKeySizes: map[string]int{"ed25519": 256, "ecdsa": 256, "rsa": 2048, "dsa": 1024},
} }
LFS struct {
StartServer bool `ini:"LFS_START_SERVER"`
ContentPath string `ini:"LFS_CONTENT_PATH"`
JWTSecretBase64 string `ini:"LFS_JWT_SECRET"`
JWTSecretBytes []byte `ini:"-"`
HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"`
MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"`
LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"`
}
// Security settings // Security settings
InstallLock bool InstallLock bool
SecretKey string SecretKey string
@ -472,27 +461,6 @@ func createPIDFile(pidPath string) {
} }
} }
// CheckLFSVersion will check lfs version, if not satisfied, then disable it.
func CheckLFSVersion() {
if LFS.StartServer {
//Disable LFS client hooks if installed for the current OS user
//Needs at least git v2.1.2
err := git.LoadGitVersion()
if err != nil {
log.Fatal("Error retrieving git version: %v", err)
}
if git.CheckGitVersionConstraint(">= 2.1.2") != nil {
LFS.StartServer = false
log.Error("LFS server support needs at least Git v2.1.2")
} else {
git.GlobalCommandArgs = append(git.GlobalCommandArgs, "-c", "filter.lfs.required=",
"-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=")
}
}
}
// SetCustomPathAndConf will set CustomPath and CustomConf with reference to the // SetCustomPathAndConf will set CustomPath and CustomConf with reference to the
// GITEA_CUSTOM environment variable and with provided overrides before stepping // GITEA_CUSTOM environment variable and with provided overrides before stepping
// back to the default // back to the default
@ -722,51 +690,7 @@ func NewContext() {
SSH.CreateAuthorizedKeysFile = sec.Key("SSH_CREATE_AUTHORIZED_KEYS_FILE").MustBool(true) SSH.CreateAuthorizedKeysFile = sec.Key("SSH_CREATE_AUTHORIZED_KEYS_FILE").MustBool(true)
SSH.ExposeAnonymous = sec.Key("SSH_EXPOSE_ANONYMOUS").MustBool(false) SSH.ExposeAnonymous = sec.Key("SSH_EXPOSE_ANONYMOUS").MustBool(false)
sec = Cfg.Section("server") newLFSService()
if err = sec.MapTo(&LFS); err != nil {
log.Fatal("Failed to map LFS settings: %v", err)
}
LFS.ContentPath = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
if !filepath.IsAbs(LFS.ContentPath) {
LFS.ContentPath = filepath.Join(AppWorkPath, LFS.ContentPath)
}
if LFS.LocksPagingNum == 0 {
LFS.LocksPagingNum = 50
}
LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(20 * time.Minute)
if LFS.StartServer {
LFS.JWTSecretBytes = make([]byte, 32)
n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64))
if err != nil || n != 32 {
LFS.JWTSecretBase64, err = generate.NewJwtSecret()
if err != nil {
log.Fatal("Error generating JWT Secret for custom config: %v", err)
return
}
// Save secret
cfg := ini.Empty()
if com.IsFile(CustomConf) {
// Keeps custom settings if there is already something.
if err := cfg.Append(CustomConf); err != nil {
log.Error("Failed to load custom conf '%s': %v", CustomConf, err)
}
}
cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64)
if err := os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm); err != nil {
log.Fatal("Failed to create '%s': %v", CustomConf, err)
}
if err := cfg.SaveTo(CustomConf); err != nil {
log.Fatal("Error saving generated JWT Secret to custom config: %v", err)
return
}
}
}
if err = Cfg.Section("oauth2").MapTo(&OAuth2); err != nil { if err = Cfg.Section("oauth2").MapTo(&OAuth2); err != nil {
log.Fatal("Failed to OAuth2 settings: %v", err) log.Fatal("Failed to OAuth2 settings: %v", err)
@ -1086,14 +1010,6 @@ func loadOrGenerateInternalToken(sec *ini.Section) string {
return token return token
} }
func ensureLFSDirectory() {
if LFS.StartServer {
if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil {
log.Fatal("Failed to create '%s': %v", LFS.ContentPath, err)
}
}
}
// NewServices initializes the services // NewServices initializes the services
func NewServices() { func NewServices() {
InitDBConfig() InitDBConfig()

View file

@ -34,7 +34,7 @@ func NewLocalStorage(bucket string) (*LocalStorage, error) {
} }
// Open a file // Open a file
func (l *LocalStorage) Open(path string) (io.ReadCloser, error) { func (l *LocalStorage) Open(path string) (Object, error) {
return os.Open(filepath.Join(l.dir, path)) return os.Open(filepath.Join(l.dir, path))
} }
@ -58,6 +58,11 @@ func (l *LocalStorage) Save(path string, r io.Reader) (int64, error) {
return io.Copy(f, r) return io.Copy(f, r)
} }
// Stat returns the info of the file
func (l *LocalStorage) Stat(path string) (ObjectInfo, error) {
return os.Stat(filepath.Join(l.dir, path))
}
// Delete delete a file // Delete delete a file
func (l *LocalStorage) Delete(path string) error { func (l *LocalStorage) Delete(path string) error {
p := filepath.Join(l.dir, path) p := filepath.Join(l.dir, path)

View file

@ -8,6 +8,7 @@ import (
"context" "context"
"io" "io"
"net/url" "net/url"
"os"
"path" "path"
"strings" "strings"
"time" "time"
@ -62,7 +63,7 @@ func (m *MinioStorage) buildMinioPath(p string) string {
} }
// Open open a file // Open open a file
func (m *MinioStorage) Open(path string) (io.ReadCloser, error) { func (m *MinioStorage) Open(path string) (Object, error) {
var opts = minio.GetObjectOptions{} var opts = minio.GetObjectOptions{}
object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts) object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
if err != nil { if err != nil {
@ -87,6 +88,41 @@ func (m *MinioStorage) Save(path string, r io.Reader) (int64, error) {
return uploadInfo.Size, nil return uploadInfo.Size, nil
} }
type minioFileInfo struct {
minio.ObjectInfo
}
func (m minioFileInfo) Name() string {
return m.ObjectInfo.Key
}
func (m minioFileInfo) Size() int64 {
return m.ObjectInfo.Size
}
func (m minioFileInfo) ModTime() time.Time {
return m.LastModified
}
// Stat returns the stat information of the object
func (m *MinioStorage) Stat(path string) (ObjectInfo, error) {
info, err := m.client.StatObject(
m.ctx,
m.bucket,
m.buildMinioPath(path),
minio.StatObjectOptions{},
)
if err != nil {
if errResp, ok := err.(minio.ErrorResponse); ok {
if errResp.Code == "NoSuchKey" {
return nil, os.ErrNotExist
}
}
return nil, err
}
return &minioFileInfo{info}, nil
}
// Delete delete a file // Delete delete a file
func (m *MinioStorage) Delete(path string) error { func (m *MinioStorage) Delete(path string) error {
return m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{}) return m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{})

View file

@ -10,6 +10,7 @@ import (
"fmt" "fmt"
"io" "io"
"net/url" "net/url"
"time"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
) )
@ -19,10 +20,24 @@ var (
ErrURLNotSupported = errors.New("url method not supported") ErrURLNotSupported = errors.New("url method not supported")
) )
// Object represents the object on the storage
type Object interface {
io.ReadCloser
io.Seeker
}
// ObjectInfo represents the object info on the storage
type ObjectInfo interface {
Name() string
Size() int64
ModTime() time.Time
}
// ObjectStorage represents an object storage to handle a bucket and files // ObjectStorage represents an object storage to handle a bucket and files
type ObjectStorage interface { type ObjectStorage interface {
Open(path string) (Object, error)
Save(path string, r io.Reader) (int64, error) Save(path string, r io.Reader) (int64, error)
Open(path string) (io.ReadCloser, error) Stat(path string) (ObjectInfo, error)
Delete(path string) error Delete(path string) error
URL(path, name string) (*url.URL, error) URL(path, name string) (*url.URL, error)
} }
@ -41,10 +56,21 @@ func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, sr
var ( var (
// Attachments represents attachments storage // Attachments represents attachments storage
Attachments ObjectStorage Attachments ObjectStorage
// LFS represents lfs storage
LFS ObjectStorage
) )
// Init init the stoarge // Init init the stoarge
func Init() error { func Init() error {
if err := initAttachments(); err != nil {
return err
}
return initLFS()
}
func initAttachments() error {
var err error var err error
switch setting.Attachment.StoreType { switch setting.Attachment.StoreType {
case "local": case "local":
@ -71,3 +97,31 @@ func Init() error {
return nil return nil
} }
func initLFS() error {
var err error
switch setting.LFS.StoreType {
case "local":
LFS, err = NewLocalStorage(setting.LFS.ContentPath)
case "minio":
minio := setting.LFS.Minio
LFS, err = NewMinioStorage(
context.Background(),
minio.Endpoint,
minio.AccessKeyID,
minio.SecretAccessKey,
minio.Bucket,
minio.Location,
minio.BasePath,
minio.UseSSL,
)
default:
return fmt.Errorf("Unsupported LFS store type: %s", setting.LFS.StoreType)
}
if err != nil {
return err
}
return nil
}

View file

@ -28,6 +28,7 @@ import (
"code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"
gogit "github.com/go-git/go-git/v5" gogit "github.com/go-git/go-git/v5"
@ -619,7 +620,7 @@ type pointerResult struct {
func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) { func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) {
defer wg.Done() defer wg.Done()
defer catFileBatchReader.Close() defer catFileBatchReader.Close()
contentStore := lfs.ContentStore{BasePath: setting.LFS.ContentPath} contentStore := lfs.ContentStore{ObjectStorage: storage.LFS}
bufferedReader := bufio.NewReader(catFileBatchReader) bufferedReader := bufio.NewReader(catFileBatchReader)
buf := make([]byte, 1025) buf := make([]byte, 1025)
@ -673,7 +674,11 @@ func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg
result.InRepo = true result.InRepo = true
} }
result.Exists = contentStore.Exists(pointer) result.Exists, err = contentStore.Exists(pointer)
if err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
if result.Exists { if result.Exists {
if !result.InRepo { if !result.InRepo {