LFS support to be stored on minio (#12518)
* LFS support to be stored on minio * Fix test * Fix lint * Fix lint * Fix check * Fix test * Update documents and add migration for LFS * Fix some bugs
This commit is contained in:
parent
e4b3f35b8d
commit
7a5465fc56
18 changed files with 423 additions and 203 deletions
|
@ -83,6 +83,13 @@ func migrateAttachments(dstStorage storage.ObjectStorage) error {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func migrateLFS(dstStorage storage.ObjectStorage) error {
|
||||||
|
return models.IterateLFS(func(mo *models.LFSMetaObject) error {
|
||||||
|
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func runMigrateStorage(ctx *cli.Context) error {
|
func runMigrateStorage(ctx *cli.Context) error {
|
||||||
if err := initDB(); err != nil {
|
if err := initDB(); err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -103,9 +110,6 @@ func runMigrateStorage(ctx *cli.Context) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
tp := ctx.String("type")
|
|
||||||
switch tp {
|
|
||||||
case "attachments":
|
|
||||||
var dstStorage storage.ObjectStorage
|
var dstStorage storage.ObjectStorage
|
||||||
var err error
|
var err error
|
||||||
switch ctx.String("store") {
|
switch ctx.String("store") {
|
||||||
|
@ -134,14 +138,22 @@ func runMigrateStorage(ctx *cli.Context) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
tp := ctx.String("type")
|
||||||
|
switch tp {
|
||||||
|
case "attachments":
|
||||||
if err := migrateAttachments(dstStorage); err != nil {
|
if err := migrateAttachments(dstStorage); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
case "lfs":
|
||||||
|
if err := migrateLFS(dstStorage); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
|
||||||
|
}
|
||||||
|
|
||||||
log.Warn("All files have been copied to the new placement but old files are still on the orignial placement.")
|
log.Warn("All files have been copied to the new placement but old files are still on the orignial placement.")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -206,12 +206,23 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
|
||||||
- `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars.
|
- `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars.
|
||||||
- `ENABLE_GZIP`: **false**: Enables application-level GZIP support.
|
- `ENABLE_GZIP`: **false**: Enables application-level GZIP support.
|
||||||
- `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\].
|
- `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\].
|
||||||
|
|
||||||
- `LFS_START_SERVER`: **false**: Enables git-lfs support.
|
- `LFS_START_SERVER`: **false**: Enables git-lfs support.
|
||||||
- `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files.
|
- `LFS_STORE_TYPE`: **local**: Storage type for lfs, `local` for local disk or `minio` for s3 compatible object storage service.
|
||||||
|
- `LFS_SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
|
||||||
|
- `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files, only available when `LFS_STORE_TYPE` is `local`.
|
||||||
|
- `LFS_MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `LFS_STORE_TYPE` is `minio`
|
||||||
|
- `LFS_MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `LFS_STORE_TYPE` is `minio`
|
||||||
|
- `LFS_MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `LFS_STORE_TYPE is` `minio`
|
||||||
|
- `LFS_MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `LFS_STORE_TYPE` is `minio`
|
||||||
|
- `LFS_MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `LFS_STORE_TYPE` is `minio`
|
||||||
|
- `LFS_MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `LFS_STORE_TYPE` is `minio`
|
||||||
|
- `LFS_MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `LFS_STORE_TYPE` is `minio`
|
||||||
- `LFS_JWT_SECRET`: **\<empty\>**: LFS authentication secret, change this a unique string.
|
- `LFS_JWT_SECRET`: **\<empty\>**: LFS authentication secret, change this a unique string.
|
||||||
- `LFS_HTTP_AUTH_EXPIRY`: **20m**: LFS authentication validity period in time.Duration, pushes taking longer than this may fail.
|
- `LFS_HTTP_AUTH_EXPIRY`: **20m**: LFS authentication validity period in time.Duration, pushes taking longer than this may fail.
|
||||||
- `LFS_MAX_FILE_SIZE`: **0**: Maximum allowed LFS file size in bytes (Set to 0 for no limit).
|
- `LFS_MAX_FILE_SIZE`: **0**: Maximum allowed LFS file size in bytes (Set to 0 for no limit).
|
||||||
- `LFS_LOCK_PAGING_NUM`: **50**: Maximum number of LFS Locks returned per page.
|
- `LFS_LOCK_PAGING_NUM`: **50**: Maximum number of LFS Locks returned per page.
|
||||||
|
|
||||||
- `REDIRECT_OTHER_PORT`: **false**: If true and `PROTOCOL` is https, allows redirecting http requests on `PORT_TO_REDIRECT` to the https port Gitea listens on.
|
- `REDIRECT_OTHER_PORT`: **false**: If true and `PROTOCOL` is https, allows redirecting http requests on `PORT_TO_REDIRECT` to the https port Gitea listens on.
|
||||||
- `PORT_TO_REDIRECT`: **80**: Port for the http redirection service to listen on. Used when `REDIRECT_OTHER_PORT` is true.
|
- `PORT_TO_REDIRECT`: **80**: Port for the http redirection service to listen on. Used when `REDIRECT_OTHER_PORT` is true.
|
||||||
- `ENABLE_LETSENCRYPT`: **false**: If enabled you must set `DOMAIN` to valid internet facing domain (ensure DNS is set and port 80 is accessible by letsencrypt validation server).
|
- `ENABLE_LETSENCRYPT`: **false**: If enabled you must set `DOMAIN` to valid internet facing domain (ensure DNS is set and port 80 is accessible by letsencrypt validation server).
|
||||||
|
|
|
@ -69,8 +69,18 @@ menu:
|
||||||
- `STATIC_CACHE_TIME`: **6h**: 静态资源文件,包括 `custom/`, `public/` 和所有上传的头像的浏览器缓存时间。
|
- `STATIC_CACHE_TIME`: **6h**: 静态资源文件,包括 `custom/`, `public/` 和所有上传的头像的浏览器缓存时间。
|
||||||
- `ENABLE_GZIP`: 启用应用级别的 GZIP 压缩。
|
- `ENABLE_GZIP`: 启用应用级别的 GZIP 压缩。
|
||||||
- `LANDING_PAGE`: 未登录用户的默认页面,可选 `home` 或 `explore`。
|
- `LANDING_PAGE`: 未登录用户的默认页面,可选 `home` 或 `explore`。
|
||||||
|
|
||||||
- `LFS_START_SERVER`: 是否启用 git-lfs 支持. 可以为 `true` 或 `false`, 默认是 `false`。
|
- `LFS_START_SERVER`: 是否启用 git-lfs 支持. 可以为 `true` 或 `false`, 默认是 `false`。
|
||||||
|
- `LFS_STORE_TYPE`: **local**: LFS 的存储类型,`local` 将存储到磁盘,`minio` 将存储到 s3 兼容的对象服务。
|
||||||
|
- `LFS_SERVE_DIRECT`: **false**: 允许直接重定向到存储系统。当前,仅 Minio/S3 是支持的。
|
||||||
- `LFS_CONTENT_PATH`: 存放 lfs 命令上传的文件的地方,默认是 `data/lfs`。
|
- `LFS_CONTENT_PATH`: 存放 lfs 命令上传的文件的地方,默认是 `data/lfs`。
|
||||||
|
- `LFS_MINIO_ENDPOINT`: **localhost:9000**: Minio 地址,仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
|
||||||
|
- `LFS_MINIO_ACCESS_KEY_ID`: Minio accessKeyID,仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
|
||||||
|
- `LFS_MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey,仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
|
||||||
|
- `LFS_MINIO_BUCKET`: **gitea**: Minio bucket,仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
|
||||||
|
- `LFS_MINIO_LOCATION`: **us-east-1**: Minio location ,仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
|
||||||
|
- `LFS_MINIO_BASE_PATH`: **lfs/**: Minio base path ,仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
|
||||||
|
- `LFS_MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ,仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
|
||||||
- `LFS_JWT_SECRET`: LFS 认证密钥,改成自己的。
|
- `LFS_JWT_SECRET`: LFS 认证密钥,改成自己的。
|
||||||
|
|
||||||
## Database (`database`)
|
## Database (`database`)
|
||||||
|
|
|
@ -18,6 +18,7 @@ import (
|
||||||
"code.gitea.io/gitea/models"
|
"code.gitea.io/gitea/models"
|
||||||
"code.gitea.io/gitea/modules/lfs"
|
"code.gitea.io/gitea/modules/lfs"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/storage"
|
||||||
|
|
||||||
"gitea.com/macaron/gzip"
|
"gitea.com/macaron/gzip"
|
||||||
gzipp "github.com/klauspost/compress/gzip"
|
gzipp "github.com/klauspost/compress/gzip"
|
||||||
|
@ -49,8 +50,10 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string
|
||||||
lfsID++
|
lfsID++
|
||||||
lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
|
lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
|
contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
|
||||||
if !contentStore.Exists(lfsMetaObject) {
|
exist, err := contentStore.Exists(lfsMetaObject)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
if !exist {
|
||||||
err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content))
|
err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content))
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,13 +36,23 @@ ROOT_URL = http://localhost:3001/
|
||||||
DISABLE_SSH = false
|
DISABLE_SSH = false
|
||||||
SSH_LISTEN_HOST = localhost
|
SSH_LISTEN_HOST = localhost
|
||||||
SSH_PORT = 2201
|
SSH_PORT = 2201
|
||||||
START_SSH_SERVER = true
|
|
||||||
LFS_START_SERVER = true
|
|
||||||
LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
|
|
||||||
OFFLINE_MODE = false
|
|
||||||
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
|
|
||||||
APP_DATA_PATH = integrations/gitea-integration-mysql/data
|
APP_DATA_PATH = integrations/gitea-integration-mysql/data
|
||||||
BUILTIN_SSH_SERVER_USER = git
|
BUILTIN_SSH_SERVER_USER = git
|
||||||
|
START_SSH_SERVER = true
|
||||||
|
OFFLINE_MODE = false
|
||||||
|
|
||||||
|
LFS_START_SERVER = true
|
||||||
|
LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
|
||||||
|
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
|
||||||
|
LFS_STORE_TYPE = minio
|
||||||
|
LFS_SERVE_DIRECT = false
|
||||||
|
LFS_MINIO_ENDPOINT = minio:9000
|
||||||
|
LFS_MINIO_ACCESS_KEY_ID = 123456
|
||||||
|
LFS_MINIO_SECRET_ACCESS_KEY = 12345678
|
||||||
|
LFS_MINIO_BUCKET = gitea
|
||||||
|
LFS_MINIO_LOCATION = us-east-1
|
||||||
|
LFS_MINIO_BASE_PATH = lfs/
|
||||||
|
LFS_MINIO_USE_SSL = false
|
||||||
|
|
||||||
[attachment]
|
[attachment]
|
||||||
STORE_TYPE = minio
|
STORE_TYPE = minio
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"path"
|
||||||
|
|
||||||
"code.gitea.io/gitea/modules/timeutil"
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
|
||||||
|
@ -26,6 +27,15 @@ type LFSMetaObject struct {
|
||||||
CreatedUnix timeutil.TimeStamp `xorm:"created"`
|
CreatedUnix timeutil.TimeStamp `xorm:"created"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RelativePath returns the relative path of the lfs object
|
||||||
|
func (m *LFSMetaObject) RelativePath() string {
|
||||||
|
if len(m.Oid) < 5 {
|
||||||
|
return m.Oid
|
||||||
|
}
|
||||||
|
|
||||||
|
return path.Join(m.Oid[0:2], m.Oid[2:4], m.Oid[4:])
|
||||||
|
}
|
||||||
|
|
||||||
// Pointer returns the string representation of an LFS pointer file
|
// Pointer returns the string representation of an LFS pointer file
|
||||||
func (m *LFSMetaObject) Pointer() string {
|
func (m *LFSMetaObject) Pointer() string {
|
||||||
return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size)
|
return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size)
|
||||||
|
@ -202,3 +212,25 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error {
|
||||||
|
|
||||||
return sess.Commit()
|
return sess.Commit()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IterateLFS iterates lfs object
|
||||||
|
func IterateLFS(f func(mo *LFSMetaObject) error) error {
|
||||||
|
var start int
|
||||||
|
const batchSize = 100
|
||||||
|
for {
|
||||||
|
var mos = make([]*LFSMetaObject, 0, batchSize)
|
||||||
|
if err := x.Limit(batchSize, start).Find(&mos); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(mos) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
start += len(mos)
|
||||||
|
|
||||||
|
for _, mo := range mos {
|
||||||
|
if err := f(mo); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -69,6 +69,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
setting.Attachment.Path = filepath.Join(setting.AppDataPath, "attachments")
|
setting.Attachment.Path = filepath.Join(setting.AppDataPath, "attachments")
|
||||||
|
setting.LFS.ContentPath = filepath.Join(setting.AppDataPath, "lfs")
|
||||||
if err = storage.Init(); err != nil {
|
if err = storage.Init(); err != nil {
|
||||||
fatalTestError("storage.Init: %v\n", err)
|
fatalTestError("storage.Init: %v\n", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,11 +10,10 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
|
|
||||||
"code.gitea.io/gitea/models"
|
"code.gitea.io/gitea/models"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
@ -24,17 +23,15 @@ var (
|
||||||
|
|
||||||
// ContentStore provides a simple file system based storage.
|
// ContentStore provides a simple file system based storage.
|
||||||
type ContentStore struct {
|
type ContentStore struct {
|
||||||
BasePath string
|
storage.ObjectStorage
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get takes a Meta object and retrieves the content from the store, returning
|
// Get takes a Meta object and retrieves the content from the store, returning
|
||||||
// it as an io.Reader. If fromByte > 0, the reader starts from that byte
|
// it as an io.Reader. If fromByte > 0, the reader starts from that byte
|
||||||
func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
|
func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
|
||||||
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
|
f, err := s.Open(meta.RelativePath())
|
||||||
|
|
||||||
f, err := os.Open(path)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("Whilst trying to read LFS OID[%s]: Unable to open %s Error: %v", meta.Oid, path, err)
|
log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if fromByte > 0 {
|
if fromByte > 0 {
|
||||||
|
@ -48,82 +45,55 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
|
||||||
|
|
||||||
// Put takes a Meta object and an io.Reader and writes the content to the store.
|
// Put takes a Meta object and an io.Reader and writes the content to the store.
|
||||||
func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
|
func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
|
||||||
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
|
|
||||||
tmpPath := path + ".tmp"
|
|
||||||
|
|
||||||
dir := filepath.Dir(path)
|
|
||||||
if err := os.MkdirAll(dir, 0750); err != nil {
|
|
||||||
log.Error("Whilst putting LFS OID[%s]: Unable to create the LFS directory: %s Error: %v", meta.Oid, dir, err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640)
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Whilst putting LFS OID[%s]: Unable to open temporary file for writing: %s Error: %v", tmpPath, err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
if err := util.Remove(tmpPath); err != nil {
|
|
||||||
log.Warn("Unable to remove temporary path: %s: Error: %v", tmpPath, err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
hash := sha256.New()
|
hash := sha256.New()
|
||||||
hw := io.MultiWriter(hash, file)
|
rd := io.TeeReader(r, hash)
|
||||||
|
p := meta.RelativePath()
|
||||||
written, err := io.Copy(hw, r)
|
written, err := s.Save(p, rd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, tmpPath, err)
|
log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err)
|
||||||
file.Close()
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
file.Close()
|
|
||||||
|
|
||||||
if written != meta.Size {
|
if written != meta.Size {
|
||||||
|
if err := s.Delete(p); err != nil {
|
||||||
|
log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
|
||||||
|
}
|
||||||
return errSizeMismatch
|
return errSizeMismatch
|
||||||
}
|
}
|
||||||
|
|
||||||
shaStr := hex.EncodeToString(hash.Sum(nil))
|
shaStr := hex.EncodeToString(hash.Sum(nil))
|
||||||
if shaStr != meta.Oid {
|
if shaStr != meta.Oid {
|
||||||
return errHashMismatch
|
if err := s.Delete(p); err != nil {
|
||||||
|
log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
|
||||||
}
|
}
|
||||||
|
return errHashMismatch
|
||||||
if err := os.Rename(tmpPath, path); err != nil {
|
|
||||||
log.Error("Whilst putting LFS OID[%s]: Unable to move tmp file to final destination: %s Error: %v", meta.Oid, path, err)
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Exists returns true if the object exists in the content store.
|
// Exists returns true if the object exists in the content store.
|
||||||
func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool {
|
func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
|
||||||
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
|
_, err := s.ObjectStorage.Stat(meta.RelativePath())
|
||||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
if err != nil {
|
||||||
return false
|
if os.IsNotExist(err) {
|
||||||
|
return false, nil
|
||||||
}
|
}
|
||||||
return true
|
return false, err
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify returns true if the object exists in the content store and size is correct.
|
// Verify returns true if the object exists in the content store and size is correct.
|
||||||
func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
|
func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
|
||||||
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
|
p := meta.RelativePath()
|
||||||
|
fi, err := s.ObjectStorage.Stat(p)
|
||||||
fi, err := os.Stat(path)
|
if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) {
|
||||||
if os.IsNotExist(err) || err == nil && fi.Size() != meta.Size {
|
|
||||||
return false, nil
|
return false, nil
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", path, meta.Oid, err)
|
log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err)
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func transformKey(key string) string {
|
|
||||||
if len(key) < 5 {
|
|
||||||
return key
|
|
||||||
}
|
|
||||||
|
|
||||||
return filepath.Join(key[0:2], key[2:4], key[4:])
|
|
||||||
}
|
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"code.gitea.io/gitea/models"
|
"code.gitea.io/gitea/models"
|
||||||
"code.gitea.io/gitea/modules/base"
|
"code.gitea.io/gitea/modules/base"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file
|
// ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file
|
||||||
|
@ -53,9 +54,10 @@ func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||||
meta := &models.LFSMetaObject{Oid: oid, Size: size}
|
meta := &models.LFSMetaObject{Oid: oid, Size: size}
|
||||||
if !contentStore.Exists(meta) {
|
exist, err := contentStore.Exists(meta)
|
||||||
|
if err != nil || !exist {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -64,6 +66,6 @@ func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
|
||||||
|
|
||||||
// ReadMetaObject will read a models.LFSMetaObject and return a reader
|
// ReadMetaObject will read a models.LFSMetaObject and return a reader
|
||||||
func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) {
|
func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) {
|
||||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||||
return contentStore.Get(meta, 0)
|
return contentStore.Get(meta, 0)
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,6 +20,7 @@ import (
|
||||||
"code.gitea.io/gitea/modules/context"
|
"code.gitea.io/gitea/modules/context"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/storage"
|
||||||
|
|
||||||
"gitea.com/macaron/macaron"
|
"gitea.com/macaron/macaron"
|
||||||
"github.com/dgrijalva/jwt-go"
|
"github.com/dgrijalva/jwt-go"
|
||||||
|
@ -187,7 +188,7 @@ func getContentHandler(ctx *context.Context) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||||
content, err := contentStore.Get(meta, fromByte)
|
content, err := contentStore.Get(meta, fromByte)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Errors are logged in contentStore.Get
|
// Errors are logged in contentStore.Get
|
||||||
|
@ -288,8 +289,14 @@ func PostHandler(ctx *context.Context) {
|
||||||
ctx.Resp.Header().Set("Content-Type", metaMediaType)
|
ctx.Resp.Header().Set("Content-Type", metaMediaType)
|
||||||
|
|
||||||
sentStatus := 202
|
sentStatus := 202
|
||||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||||
if meta.Existing && contentStore.Exists(meta) {
|
exist, err := contentStore.Exists(meta)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err)
|
||||||
|
writeStatus(ctx, 500)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if meta.Existing && exist {
|
||||||
sentStatus = 200
|
sentStatus = 200
|
||||||
}
|
}
|
||||||
ctx.Resp.WriteHeader(sentStatus)
|
ctx.Resp.WriteHeader(sentStatus)
|
||||||
|
@ -343,13 +350,21 @@ func BatchHandler(ctx *context.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||||
|
|
||||||
meta, err := repository.GetLFSMetaObjectByOid(object.Oid)
|
meta, err := repository.GetLFSMetaObjectByOid(object.Oid)
|
||||||
if err == nil && contentStore.Exists(meta) { // Object is found and exists
|
if err == nil { // Object is found and exists
|
||||||
|
exist, err := contentStore.Exists(meta)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
|
||||||
|
writeStatus(ctx, 500)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if exist {
|
||||||
responseObjects = append(responseObjects, Represent(object, meta, true, false))
|
responseObjects = append(responseObjects, Represent(object, meta, true, false))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize {
|
if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize {
|
||||||
log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", object.Oid, object.Size, object.User, object.Repo, setting.LFS.MaxFileSize)
|
log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", object.Oid, object.Size, object.User, object.Repo, setting.LFS.MaxFileSize)
|
||||||
|
@ -360,7 +375,13 @@ func BatchHandler(ctx *context.Context) {
|
||||||
// Object is not found
|
// Object is not found
|
||||||
meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID})
|
meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !contentStore.Exists(meta)))
|
exist, err := contentStore.Exists(meta)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
|
||||||
|
writeStatus(ctx, 500)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !exist))
|
||||||
} else {
|
} else {
|
||||||
log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, object.User, object.Repo, err)
|
log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, object.User, object.Repo, err)
|
||||||
}
|
}
|
||||||
|
@ -387,7 +408,7 @@ func PutHandler(ctx *context.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||||
bodyReader := ctx.Req.Body().ReadCloser()
|
bodyReader := ctx.Req.Body().ReadCloser()
|
||||||
defer bodyReader.Close()
|
defer bodyReader.Close()
|
||||||
if err := contentStore.Put(meta, bodyReader); err != nil {
|
if err := contentStore.Put(meta, bodyReader); err != nil {
|
||||||
|
@ -429,7 +450,7 @@ func VerifyHandler(ctx *context.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||||
ok, err := contentStore.Verify(meta)
|
ok, err := contentStore.Verify(meta)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Error will be logged in Verify
|
// Error will be logged in Verify
|
||||||
|
|
|
@ -20,6 +20,7 @@ import (
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
repo_module "code.gitea.io/gitea/modules/repository"
|
repo_module "code.gitea.io/gitea/modules/repository"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/storage"
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
pull_service "code.gitea.io/gitea/services/pull"
|
pull_service "code.gitea.io/gitea/services/pull"
|
||||||
|
|
||||||
|
@ -433,8 +434,12 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
|
contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
|
||||||
if !contentStore.Exists(lfsMetaObject) {
|
exist, err := contentStore.Exists(lfsMetaObject)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !exist {
|
||||||
if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil {
|
if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil {
|
||||||
if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
|
if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
|
||||||
return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err)
|
return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err)
|
||||||
|
|
|
@ -13,6 +13,7 @@ import (
|
||||||
"code.gitea.io/gitea/models"
|
"code.gitea.io/gitea/models"
|
||||||
"code.gitea.io/gitea/modules/lfs"
|
"code.gitea.io/gitea/modules/lfs"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
// UploadRepoFileOptions contains the uploaded repository file options
|
// UploadRepoFileOptions contains the uploaded repository file options
|
||||||
|
@ -163,12 +164,16 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep
|
||||||
|
|
||||||
// OK now we can insert the data into the store - there's no way to clean up the store
|
// OK now we can insert the data into the store - there's no way to clean up the store
|
||||||
// once it's in there, it's in there.
|
// once it's in there, it's in there.
|
||||||
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
|
contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
|
||||||
for _, uploadInfo := range infos {
|
for _, uploadInfo := range infos {
|
||||||
if uploadInfo.lfsMetaObject == nil {
|
if uploadInfo.lfsMetaObject == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if !contentStore.Exists(uploadInfo.lfsMetaObject) {
|
exist, err := contentStore.Exists(uploadInfo.lfsMetaObject)
|
||||||
|
if err != nil {
|
||||||
|
return cleanUpAfterFailure(&infos, t, err)
|
||||||
|
}
|
||||||
|
if !exist {
|
||||||
file, err := os.Open(uploadInfo.upload.LocalPath())
|
file, err := os.Open(uploadInfo.upload.LocalPath())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cleanUpAfterFailure(&infos, t, err)
|
return cleanUpAfterFailure(&infos, t, err)
|
||||||
|
|
122
modules/setting/lfs.go
Normal file
122
modules/setting/lfs.go
Normal file
|
@ -0,0 +1,122 @@
|
||||||
|
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package setting
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/generate"
|
||||||
|
"code.gitea.io/gitea/modules/git"
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
|
||||||
|
"github.com/unknwon/com"
|
||||||
|
ini "gopkg.in/ini.v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LFS represents the configuration for Git LFS
|
||||||
|
var LFS = struct {
|
||||||
|
StartServer bool `ini:"LFS_START_SERVER"`
|
||||||
|
ContentPath string `ini:"LFS_CONTENT_PATH"`
|
||||||
|
JWTSecretBase64 string `ini:"LFS_JWT_SECRET"`
|
||||||
|
JWTSecretBytes []byte `ini:"-"`
|
||||||
|
HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"`
|
||||||
|
MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"`
|
||||||
|
LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"`
|
||||||
|
|
||||||
|
StoreType string
|
||||||
|
ServeDirect bool
|
||||||
|
Minio struct {
|
||||||
|
Endpoint string
|
||||||
|
AccessKeyID string
|
||||||
|
SecretAccessKey string
|
||||||
|
UseSSL bool
|
||||||
|
Bucket string
|
||||||
|
Location string
|
||||||
|
BasePath string
|
||||||
|
}
|
||||||
|
}{
|
||||||
|
StoreType: "local",
|
||||||
|
}
|
||||||
|
|
||||||
|
func newLFSService() {
|
||||||
|
sec := Cfg.Section("server")
|
||||||
|
if err := sec.MapTo(&LFS); err != nil {
|
||||||
|
log.Fatal("Failed to map LFS settings: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
LFS.ContentPath = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
|
||||||
|
if !filepath.IsAbs(LFS.ContentPath) {
|
||||||
|
LFS.ContentPath = filepath.Join(AppWorkPath, LFS.ContentPath)
|
||||||
|
}
|
||||||
|
if LFS.LocksPagingNum == 0 {
|
||||||
|
LFS.LocksPagingNum = 50
|
||||||
|
}
|
||||||
|
|
||||||
|
LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(20 * time.Minute)
|
||||||
|
|
||||||
|
if LFS.StartServer {
|
||||||
|
LFS.JWTSecretBytes = make([]byte, 32)
|
||||||
|
n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64))
|
||||||
|
|
||||||
|
if err != nil || n != 32 {
|
||||||
|
LFS.JWTSecretBase64, err = generate.NewJwtSecret()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Error generating JWT Secret for custom config: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save secret
|
||||||
|
cfg := ini.Empty()
|
||||||
|
if com.IsFile(CustomConf) {
|
||||||
|
// Keeps custom settings if there is already something.
|
||||||
|
if err := cfg.Append(CustomConf); err != nil {
|
||||||
|
log.Error("Failed to load custom conf '%s': %v", CustomConf, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64)
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm); err != nil {
|
||||||
|
log.Fatal("Failed to create '%s': %v", CustomConf, err)
|
||||||
|
}
|
||||||
|
if err := cfg.SaveTo(CustomConf); err != nil {
|
||||||
|
log.Fatal("Error saving generated JWT Secret to custom config: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureLFSDirectory() {
|
||||||
|
if LFS.StartServer {
|
||||||
|
if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil {
|
||||||
|
log.Fatal("Failed to create '%s': %v", LFS.ContentPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckLFSVersion will check lfs version, if not satisfied, then disable it.
|
||||||
|
func CheckLFSVersion() {
|
||||||
|
if LFS.StartServer {
|
||||||
|
//Disable LFS client hooks if installed for the current OS user
|
||||||
|
//Needs at least git v2.1.2
|
||||||
|
|
||||||
|
err := git.LoadGitVersion()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Error retrieving git version: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if git.CheckGitVersionConstraint(">= 2.1.2") != nil {
|
||||||
|
LFS.StartServer = false
|
||||||
|
log.Error("LFS server support needs at least Git v2.1.2")
|
||||||
|
} else {
|
||||||
|
git.GlobalCommandArgs = append(git.GlobalCommandArgs, "-c", "filter.lfs.required=",
|
||||||
|
"-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -23,7 +23,6 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"code.gitea.io/gitea/modules/generate"
|
"code.gitea.io/gitea/modules/generate"
|
||||||
"code.gitea.io/gitea/modules/git"
|
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/user"
|
"code.gitea.io/gitea/modules/user"
|
||||||
|
|
||||||
|
@ -133,16 +132,6 @@ var (
|
||||||
MinimumKeySizes: map[string]int{"ed25519": 256, "ecdsa": 256, "rsa": 2048, "dsa": 1024},
|
MinimumKeySizes: map[string]int{"ed25519": 256, "ecdsa": 256, "rsa": 2048, "dsa": 1024},
|
||||||
}
|
}
|
||||||
|
|
||||||
LFS struct {
|
|
||||||
StartServer bool `ini:"LFS_START_SERVER"`
|
|
||||||
ContentPath string `ini:"LFS_CONTENT_PATH"`
|
|
||||||
JWTSecretBase64 string `ini:"LFS_JWT_SECRET"`
|
|
||||||
JWTSecretBytes []byte `ini:"-"`
|
|
||||||
HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"`
|
|
||||||
MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"`
|
|
||||||
LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Security settings
|
// Security settings
|
||||||
InstallLock bool
|
InstallLock bool
|
||||||
SecretKey string
|
SecretKey string
|
||||||
|
@ -472,27 +461,6 @@ func createPIDFile(pidPath string) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// CheckLFSVersion will check lfs version, if not satisfied, then disable it.
|
|
||||||
func CheckLFSVersion() {
|
|
||||||
if LFS.StartServer {
|
|
||||||
//Disable LFS client hooks if installed for the current OS user
|
|
||||||
//Needs at least git v2.1.2
|
|
||||||
|
|
||||||
err := git.LoadGitVersion()
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal("Error retrieving git version: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if git.CheckGitVersionConstraint(">= 2.1.2") != nil {
|
|
||||||
LFS.StartServer = false
|
|
||||||
log.Error("LFS server support needs at least Git v2.1.2")
|
|
||||||
} else {
|
|
||||||
git.GlobalCommandArgs = append(git.GlobalCommandArgs, "-c", "filter.lfs.required=",
|
|
||||||
"-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetCustomPathAndConf will set CustomPath and CustomConf with reference to the
|
// SetCustomPathAndConf will set CustomPath and CustomConf with reference to the
|
||||||
// GITEA_CUSTOM environment variable and with provided overrides before stepping
|
// GITEA_CUSTOM environment variable and with provided overrides before stepping
|
||||||
// back to the default
|
// back to the default
|
||||||
|
@ -722,51 +690,7 @@ func NewContext() {
|
||||||
SSH.CreateAuthorizedKeysFile = sec.Key("SSH_CREATE_AUTHORIZED_KEYS_FILE").MustBool(true)
|
SSH.CreateAuthorizedKeysFile = sec.Key("SSH_CREATE_AUTHORIZED_KEYS_FILE").MustBool(true)
|
||||||
SSH.ExposeAnonymous = sec.Key("SSH_EXPOSE_ANONYMOUS").MustBool(false)
|
SSH.ExposeAnonymous = sec.Key("SSH_EXPOSE_ANONYMOUS").MustBool(false)
|
||||||
|
|
||||||
sec = Cfg.Section("server")
|
newLFSService()
|
||||||
if err = sec.MapTo(&LFS); err != nil {
|
|
||||||
log.Fatal("Failed to map LFS settings: %v", err)
|
|
||||||
}
|
|
||||||
LFS.ContentPath = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
|
|
||||||
if !filepath.IsAbs(LFS.ContentPath) {
|
|
||||||
LFS.ContentPath = filepath.Join(AppWorkPath, LFS.ContentPath)
|
|
||||||
}
|
|
||||||
if LFS.LocksPagingNum == 0 {
|
|
||||||
LFS.LocksPagingNum = 50
|
|
||||||
}
|
|
||||||
|
|
||||||
LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(20 * time.Minute)
|
|
||||||
|
|
||||||
if LFS.StartServer {
|
|
||||||
LFS.JWTSecretBytes = make([]byte, 32)
|
|
||||||
n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64))
|
|
||||||
|
|
||||||
if err != nil || n != 32 {
|
|
||||||
LFS.JWTSecretBase64, err = generate.NewJwtSecret()
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal("Error generating JWT Secret for custom config: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Save secret
|
|
||||||
cfg := ini.Empty()
|
|
||||||
if com.IsFile(CustomConf) {
|
|
||||||
// Keeps custom settings if there is already something.
|
|
||||||
if err := cfg.Append(CustomConf); err != nil {
|
|
||||||
log.Error("Failed to load custom conf '%s': %v", CustomConf, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64)
|
|
||||||
|
|
||||||
if err := os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm); err != nil {
|
|
||||||
log.Fatal("Failed to create '%s': %v", CustomConf, err)
|
|
||||||
}
|
|
||||||
if err := cfg.SaveTo(CustomConf); err != nil {
|
|
||||||
log.Fatal("Error saving generated JWT Secret to custom config: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = Cfg.Section("oauth2").MapTo(&OAuth2); err != nil {
|
if err = Cfg.Section("oauth2").MapTo(&OAuth2); err != nil {
|
||||||
log.Fatal("Failed to OAuth2 settings: %v", err)
|
log.Fatal("Failed to OAuth2 settings: %v", err)
|
||||||
|
@ -1086,14 +1010,6 @@ func loadOrGenerateInternalToken(sec *ini.Section) string {
|
||||||
return token
|
return token
|
||||||
}
|
}
|
||||||
|
|
||||||
func ensureLFSDirectory() {
|
|
||||||
if LFS.StartServer {
|
|
||||||
if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil {
|
|
||||||
log.Fatal("Failed to create '%s': %v", LFS.ContentPath, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewServices initializes the services
|
// NewServices initializes the services
|
||||||
func NewServices() {
|
func NewServices() {
|
||||||
InitDBConfig()
|
InitDBConfig()
|
||||||
|
|
|
@ -34,7 +34,7 @@ func NewLocalStorage(bucket string) (*LocalStorage, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open a file
|
// Open a file
|
||||||
func (l *LocalStorage) Open(path string) (io.ReadCloser, error) {
|
func (l *LocalStorage) Open(path string) (Object, error) {
|
||||||
return os.Open(filepath.Join(l.dir, path))
|
return os.Open(filepath.Join(l.dir, path))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -58,6 +58,11 @@ func (l *LocalStorage) Save(path string, r io.Reader) (int64, error) {
|
||||||
return io.Copy(f, r)
|
return io.Copy(f, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stat returns the info of the file
|
||||||
|
func (l *LocalStorage) Stat(path string) (ObjectInfo, error) {
|
||||||
|
return os.Stat(filepath.Join(l.dir, path))
|
||||||
|
}
|
||||||
|
|
||||||
// Delete delete a file
|
// Delete delete a file
|
||||||
func (l *LocalStorage) Delete(path string) error {
|
func (l *LocalStorage) Delete(path string) error {
|
||||||
p := filepath.Join(l.dir, path)
|
p := filepath.Join(l.dir, path)
|
||||||
|
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
@ -62,7 +63,7 @@ func (m *MinioStorage) buildMinioPath(p string) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open open a file
|
// Open open a file
|
||||||
func (m *MinioStorage) Open(path string) (io.ReadCloser, error) {
|
func (m *MinioStorage) Open(path string) (Object, error) {
|
||||||
var opts = minio.GetObjectOptions{}
|
var opts = minio.GetObjectOptions{}
|
||||||
object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
|
object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -87,6 +88,41 @@ func (m *MinioStorage) Save(path string, r io.Reader) (int64, error) {
|
||||||
return uploadInfo.Size, nil
|
return uploadInfo.Size, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type minioFileInfo struct {
|
||||||
|
minio.ObjectInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m minioFileInfo) Name() string {
|
||||||
|
return m.ObjectInfo.Key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m minioFileInfo) Size() int64 {
|
||||||
|
return m.ObjectInfo.Size
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m minioFileInfo) ModTime() time.Time {
|
||||||
|
return m.LastModified
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stat returns the stat information of the object
|
||||||
|
func (m *MinioStorage) Stat(path string) (ObjectInfo, error) {
|
||||||
|
info, err := m.client.StatObject(
|
||||||
|
m.ctx,
|
||||||
|
m.bucket,
|
||||||
|
m.buildMinioPath(path),
|
||||||
|
minio.StatObjectOptions{},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
if errResp, ok := err.(minio.ErrorResponse); ok {
|
||||||
|
if errResp.Code == "NoSuchKey" {
|
||||||
|
return nil, os.ErrNotExist
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &minioFileInfo{info}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Delete delete a file
|
// Delete delete a file
|
||||||
func (m *MinioStorage) Delete(path string) error {
|
func (m *MinioStorage) Delete(path string) error {
|
||||||
return m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{})
|
return m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{})
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"time"
|
||||||
|
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
)
|
)
|
||||||
|
@ -19,10 +20,24 @@ var (
|
||||||
ErrURLNotSupported = errors.New("url method not supported")
|
ErrURLNotSupported = errors.New("url method not supported")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Object represents the object on the storage
|
||||||
|
type Object interface {
|
||||||
|
io.ReadCloser
|
||||||
|
io.Seeker
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObjectInfo represents the object info on the storage
|
||||||
|
type ObjectInfo interface {
|
||||||
|
Name() string
|
||||||
|
Size() int64
|
||||||
|
ModTime() time.Time
|
||||||
|
}
|
||||||
|
|
||||||
// ObjectStorage represents an object storage to handle a bucket and files
|
// ObjectStorage represents an object storage to handle a bucket and files
|
||||||
type ObjectStorage interface {
|
type ObjectStorage interface {
|
||||||
|
Open(path string) (Object, error)
|
||||||
Save(path string, r io.Reader) (int64, error)
|
Save(path string, r io.Reader) (int64, error)
|
||||||
Open(path string) (io.ReadCloser, error)
|
Stat(path string) (ObjectInfo, error)
|
||||||
Delete(path string) error
|
Delete(path string) error
|
||||||
URL(path, name string) (*url.URL, error)
|
URL(path, name string) (*url.URL, error)
|
||||||
}
|
}
|
||||||
|
@ -41,10 +56,21 @@ func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, sr
|
||||||
var (
|
var (
|
||||||
// Attachments represents attachments storage
|
// Attachments represents attachments storage
|
||||||
Attachments ObjectStorage
|
Attachments ObjectStorage
|
||||||
|
|
||||||
|
// LFS represents lfs storage
|
||||||
|
LFS ObjectStorage
|
||||||
)
|
)
|
||||||
|
|
||||||
// Init init the stoarge
|
// Init init the stoarge
|
||||||
func Init() error {
|
func Init() error {
|
||||||
|
if err := initAttachments(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return initLFS()
|
||||||
|
}
|
||||||
|
|
||||||
|
func initAttachments() error {
|
||||||
var err error
|
var err error
|
||||||
switch setting.Attachment.StoreType {
|
switch setting.Attachment.StoreType {
|
||||||
case "local":
|
case "local":
|
||||||
|
@ -71,3 +97,31 @@ func Init() error {
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func initLFS() error {
|
||||||
|
var err error
|
||||||
|
switch setting.LFS.StoreType {
|
||||||
|
case "local":
|
||||||
|
LFS, err = NewLocalStorage(setting.LFS.ContentPath)
|
||||||
|
case "minio":
|
||||||
|
minio := setting.LFS.Minio
|
||||||
|
LFS, err = NewMinioStorage(
|
||||||
|
context.Background(),
|
||||||
|
minio.Endpoint,
|
||||||
|
minio.AccessKeyID,
|
||||||
|
minio.SecretAccessKey,
|
||||||
|
minio.Bucket,
|
||||||
|
minio.Location,
|
||||||
|
minio.BasePath,
|
||||||
|
minio.UseSSL,
|
||||||
|
)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("Unsupported LFS store type: %s", setting.LFS.StoreType)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
@ -28,6 +28,7 @@ import (
|
||||||
"code.gitea.io/gitea/modules/lfs"
|
"code.gitea.io/gitea/modules/lfs"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/storage"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
|
|
||||||
gogit "github.com/go-git/go-git/v5"
|
gogit "github.com/go-git/go-git/v5"
|
||||||
|
@ -619,7 +620,7 @@ type pointerResult struct {
|
||||||
func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) {
|
func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
defer catFileBatchReader.Close()
|
defer catFileBatchReader.Close()
|
||||||
contentStore := lfs.ContentStore{BasePath: setting.LFS.ContentPath}
|
contentStore := lfs.ContentStore{ObjectStorage: storage.LFS}
|
||||||
|
|
||||||
bufferedReader := bufio.NewReader(catFileBatchReader)
|
bufferedReader := bufio.NewReader(catFileBatchReader)
|
||||||
buf := make([]byte, 1025)
|
buf := make([]byte, 1025)
|
||||||
|
@ -673,7 +674,11 @@ func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg
|
||||||
result.InRepo = true
|
result.InRepo = true
|
||||||
}
|
}
|
||||||
|
|
||||||
result.Exists = contentStore.Exists(pointer)
|
result.Exists, err = contentStore.Exists(pointer)
|
||||||
|
if err != nil {
|
||||||
|
_ = catFileBatchReader.CloseWithError(err)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
if result.Exists {
|
if result.Exists {
|
||||||
if !result.InRepo {
|
if !result.InRepo {
|
||||||
|
|
Reference in a new issue