Storage configuration support [storage]
(#13314)
* Fix minio bug * Add tests for storage configuration * Change the Seek flag to keep compitable minio? * Fix test when first-byte-pos of all ranges is greater than the resource length Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
parent
e7750e0f6a
commit
e4e85a3e51
6 changed files with 62 additions and 33 deletions
|
@ -11,7 +11,6 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/cookiejar"
|
"net/http/cookiejar"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
@ -27,8 +26,10 @@ import (
|
||||||
"code.gitea.io/gitea/models"
|
"code.gitea.io/gitea/models"
|
||||||
"code.gitea.io/gitea/modules/base"
|
"code.gitea.io/gitea/modules/base"
|
||||||
"code.gitea.io/gitea/modules/graceful"
|
"code.gitea.io/gitea/modules/graceful"
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/queue"
|
"code.gitea.io/gitea/modules/queue"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/storage"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
"code.gitea.io/gitea/routers"
|
"code.gitea.io/gitea/routers"
|
||||||
"code.gitea.io/gitea/routers/routes"
|
"code.gitea.io/gitea/routers/routes"
|
||||||
|
@ -59,6 +60,8 @@ func NewNilResponseRecorder() *NilResponseRecorder {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMain(m *testing.M) {
|
func TestMain(m *testing.M) {
|
||||||
|
defer log.Close()
|
||||||
|
|
||||||
managerCtx, cancel := context.WithCancel(context.Background())
|
managerCtx, cancel := context.WithCancel(context.Background())
|
||||||
graceful.InitManager(managerCtx)
|
graceful.InitManager(managerCtx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
@ -142,6 +145,10 @@ func initIntegrationTest() {
|
||||||
util.RemoveAll(models.LocalCopyPath())
|
util.RemoveAll(models.LocalCopyPath())
|
||||||
setting.CheckLFSVersion()
|
setting.CheckLFSVersion()
|
||||||
setting.InitDBConfig()
|
setting.InitDBConfig()
|
||||||
|
if err := storage.Init(); err != nil {
|
||||||
|
fmt.Printf("Init storage failed: %v", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case setting.Database.UseMySQL:
|
case setting.Database.UseMySQL:
|
||||||
|
@ -149,27 +156,27 @@ func initIntegrationTest() {
|
||||||
setting.Database.User, setting.Database.Passwd, setting.Database.Host))
|
setting.Database.User, setting.Database.Passwd, setting.Database.Host))
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("sql.Open: %v", err)
|
log.Fatal("sql.Open: %v", err)
|
||||||
}
|
}
|
||||||
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil {
|
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil {
|
||||||
log.Fatalf("db.Exec: %v", err)
|
log.Fatal("db.Exec: %v", err)
|
||||||
}
|
}
|
||||||
case setting.Database.UsePostgreSQL:
|
case setting.Database.UsePostgreSQL:
|
||||||
db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
|
db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
|
||||||
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
|
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("sql.Open: %v", err)
|
log.Fatal("sql.Open: %v", err)
|
||||||
}
|
}
|
||||||
dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
|
dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("db.Query: %v", err)
|
log.Fatal("db.Query: %v", err)
|
||||||
}
|
}
|
||||||
defer dbrows.Close()
|
defer dbrows.Close()
|
||||||
|
|
||||||
if !dbrows.Next() {
|
if !dbrows.Next() {
|
||||||
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
|
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
|
||||||
log.Fatalf("db.Exec: CREATE DATABASE: %v", err)
|
log.Fatal("db.Exec: CREATE DATABASE: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Check if we need to setup a specific schema
|
// Check if we need to setup a specific schema
|
||||||
|
@ -183,18 +190,18 @@ func initIntegrationTest() {
|
||||||
// This is a different db object; requires a different Close()
|
// This is a different db object; requires a different Close()
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("sql.Open: %v", err)
|
log.Fatal("sql.Open: %v", err)
|
||||||
}
|
}
|
||||||
schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
|
schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("db.Query: %v", err)
|
log.Fatal("db.Query: %v", err)
|
||||||
}
|
}
|
||||||
defer schrows.Close()
|
defer schrows.Close()
|
||||||
|
|
||||||
if !schrows.Next() {
|
if !schrows.Next() {
|
||||||
// Create and setup a DB schema
|
// Create and setup a DB schema
|
||||||
if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil {
|
if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil {
|
||||||
log.Fatalf("db.Exec: CREATE SCHEMA: %v", err)
|
log.Fatal("db.Exec: CREATE SCHEMA: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -203,10 +210,10 @@ func initIntegrationTest() {
|
||||||
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
|
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
|
||||||
host, port, "master", setting.Database.User, setting.Database.Passwd))
|
host, port, "master", setting.Database.User, setting.Database.Passwd))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("sql.Open: %v", err)
|
log.Fatal("sql.Open: %v", err)
|
||||||
}
|
}
|
||||||
if _, err := db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil {
|
if _, err := db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil {
|
||||||
log.Fatalf("db.Exec: %v", err)
|
log.Fatal("db.Exec: %v", err)
|
||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
}
|
}
|
||||||
|
|
|
@ -78,6 +78,7 @@ func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, exp
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
resp := session.MakeRequest(t, req, expectedStatus)
|
resp := session.MakeRequest(t, req, expectedStatus)
|
||||||
|
|
||||||
return resp
|
return resp
|
||||||
|
@ -210,7 +211,7 @@ func TestGetLFSRange(t *testing.T) {
|
||||||
{"bytes=0-10", "123456789\n", http.StatusPartialContent},
|
{"bytes=0-10", "123456789\n", http.StatusPartialContent},
|
||||||
// end-range bigger than length-1 is ignored
|
// end-range bigger than length-1 is ignored
|
||||||
{"bytes=0-11", "123456789\n", http.StatusPartialContent},
|
{"bytes=0-11", "123456789\n", http.StatusPartialContent},
|
||||||
{"bytes=11-", "", http.StatusPartialContent},
|
{"bytes=11-", "Requested Range Not Satisfiable", http.StatusRequestedRangeNotSatisfiable},
|
||||||
// incorrect header value cause whole header to be ignored
|
// incorrect header value cause whole header to be ignored
|
||||||
{"bytes=-", "123456789\n", http.StatusOK},
|
{"bytes=-", "123456789\n", http.StatusOK},
|
||||||
{"foobar", "123456789\n", http.StatusOK},
|
{"foobar", "123456789\n", http.StatusOK},
|
||||||
|
|
|
@ -45,19 +45,21 @@ START_SSH_SERVER = true
|
||||||
OFFLINE_MODE = false
|
OFFLINE_MODE = false
|
||||||
|
|
||||||
LFS_START_SERVER = true
|
LFS_START_SERVER = true
|
||||||
LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
|
|
||||||
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
|
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
|
||||||
LFS_STORE_TYPE = minio
|
|
||||||
LFS_SERVE_DIRECT = false
|
[lfs]
|
||||||
LFS_MINIO_ENDPOINT = minio:9000
|
MINIO_BASE_PATH = lfs/
|
||||||
LFS_MINIO_ACCESS_KEY_ID = 123456
|
|
||||||
LFS_MINIO_SECRET_ACCESS_KEY = 12345678
|
|
||||||
LFS_MINIO_BUCKET = gitea
|
|
||||||
LFS_MINIO_LOCATION = us-east-1
|
|
||||||
LFS_MINIO_BASE_PATH = lfs/
|
|
||||||
LFS_MINIO_USE_SSL = false
|
|
||||||
|
|
||||||
[attachment]
|
[attachment]
|
||||||
|
MINIO_BASE_PATH = attachments/
|
||||||
|
|
||||||
|
[avatars]
|
||||||
|
MINIO_BASE_PATH = avatars/
|
||||||
|
|
||||||
|
[repo-avatars]
|
||||||
|
MINIO_BASE_PATH = repo-avatars/
|
||||||
|
|
||||||
|
[storage]
|
||||||
STORAGE_TYPE = minio
|
STORAGE_TYPE = minio
|
||||||
SERVE_DIRECT = false
|
SERVE_DIRECT = false
|
||||||
MINIO_ENDPOINT = minio:9000
|
MINIO_ENDPOINT = minio:9000
|
||||||
|
@ -65,7 +67,6 @@ MINIO_ACCESS_KEY_ID = 123456
|
||||||
MINIO_SECRET_ACCESS_KEY = 12345678
|
MINIO_SECRET_ACCESS_KEY = 12345678
|
||||||
MINIO_BUCKET = gitea
|
MINIO_BUCKET = gitea
|
||||||
MINIO_LOCATION = us-east-1
|
MINIO_LOCATION = us-east-1
|
||||||
MINIO_BASE_PATH = attachments/
|
|
||||||
MINIO_USE_SSL = false
|
MINIO_USE_SSL = false
|
||||||
|
|
||||||
[mailer]
|
[mailer]
|
||||||
|
@ -88,9 +89,6 @@ ENABLE_NOTIFY_MAIL = true
|
||||||
DISABLE_GRAVATAR = false
|
DISABLE_GRAVATAR = false
|
||||||
ENABLE_FEDERATED_AVATAR = false
|
ENABLE_FEDERATED_AVATAR = false
|
||||||
|
|
||||||
AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/avatars
|
|
||||||
REPOSITORY_AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/repo-avatars
|
|
||||||
|
|
||||||
[session]
|
[session]
|
||||||
PROVIDER = file
|
PROVIDER = file
|
||||||
PROVIDER_CONFIG = integrations/gitea-integration-mysql/data/sessions
|
PROVIDER_CONFIG = integrations/gitea-integration-mysql/data/sessions
|
||||||
|
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
@ -21,6 +22,21 @@ var (
|
||||||
errSizeMismatch = errors.New("Content size does not match")
|
errSizeMismatch = errors.New("Content size does not match")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ErrRangeNotSatisfiable represents an error which request range is not satisfiable.
|
||||||
|
type ErrRangeNotSatisfiable struct {
|
||||||
|
FromByte int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (err ErrRangeNotSatisfiable) Error() string {
|
||||||
|
return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable
|
||||||
|
func IsErrRangeNotSatisfiable(err error) bool {
|
||||||
|
_, ok := err.(ErrRangeNotSatisfiable)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
// ContentStore provides a simple file system based storage.
|
// ContentStore provides a simple file system based storage.
|
||||||
type ContentStore struct {
|
type ContentStore struct {
|
||||||
storage.ObjectStorage
|
storage.ObjectStorage
|
||||||
|
@ -35,7 +51,12 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if fromByte > 0 {
|
if fromByte > 0 {
|
||||||
_, err = f.Seek(fromByte, os.SEEK_CUR)
|
if fromByte >= meta.Size {
|
||||||
|
return nil, ErrRangeNotSatisfiable{
|
||||||
|
FromByte: fromByte,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err = f.Seek(fromByte, io.SeekStart)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err)
|
log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -191,8 +191,12 @@ func getContentHandler(ctx *context.Context) {
|
||||||
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||||
content, err := contentStore.Get(meta, fromByte)
|
content, err := contentStore.Get(meta, fromByte)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Errors are logged in contentStore.Get
|
if IsErrRangeNotSatisfiable(err) {
|
||||||
writeStatus(ctx, 404)
|
writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable)
|
||||||
|
} else {
|
||||||
|
// Errors are logged in contentStore.Get
|
||||||
|
writeStatus(ctx, 404)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer content.Close()
|
defer content.Close()
|
||||||
|
|
|
@ -32,14 +32,12 @@ func (s *Storage) MapTo(v interface{}) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func getStorage(name, typ string, overrides ...*ini.Section) Storage {
|
func getStorage(name, typ string, overrides ...*ini.Section) Storage {
|
||||||
sectionName := "storage"
|
const sectionName = "storage"
|
||||||
if len(name) > 0 {
|
|
||||||
sectionName = sectionName + "." + typ
|
|
||||||
}
|
|
||||||
sec := Cfg.Section(sectionName)
|
sec := Cfg.Section(sectionName)
|
||||||
|
|
||||||
if len(overrides) == 0 {
|
if len(overrides) == 0 {
|
||||||
overrides = []*ini.Section{
|
overrides = []*ini.Section{
|
||||||
|
Cfg.Section(sectionName + "." + typ),
|
||||||
Cfg.Section(sectionName + "." + name),
|
Cfg.Section(sectionName + "." + name),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Reference in a new issue