Add package registry quota limits (#21584)

Related #20471

This PR adds global quota limits for the package registry. Settings for
individual users/orgs can be added in a seperate PR using the settings
table.

Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
KN4CK3R 2022-11-09 07:34:27 +01:00 committed by GitHub
parent cb83288530
commit 20674dd05d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 378 additions and 61 deletions

View file

@ -44,6 +44,7 @@ func TestMigratePackages(t *testing.T) {
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{
Filename: "a.go", Filename: "a.go",
}, },
Creator: creator,
Data: buf, Data: buf,
IsLead: true, IsLead: true,
}) })

View file

@ -2335,6 +2335,35 @@ ROUTER = console
;; ;;
;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload` ;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload`
;CHUNKED_UPLOAD_PATH = tmp/package-upload ;CHUNKED_UPLOAD_PATH = tmp/package-upload
;;
;; Maxmimum count of package versions a single owner can have (`-1` means no limits)
;LIMIT_TOTAL_OWNER_COUNT = -1
;; Maxmimum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_TOTAL_OWNER_SIZE = -1
;; Maxmimum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_COMPOSER = -1
;; Maxmimum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_CONAN = -1
;; Maxmimum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_CONTAINER = -1
;; Maxmimum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_GENERIC = -1
;; Maxmimum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_HELM = -1
;; Maxmimum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_MAVEN = -1
;; Maxmimum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_NPM = -1
;; Maxmimum size of a NuGet upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_NUGET = -1
;; Maxmimum size of a Pub upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_PUB = -1
;; Maxmimum size of a PyPI upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_PYPI = -1
;; Maxmimum size of a RubyGems upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_RUBYGEMS = -1
;; Maxmimum size of a Vagrant upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_VAGRANT = -1
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

View file

@ -1138,6 +1138,20 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
- `ENABLED`: **true**: Enable/Disable package registry capabilities - `ENABLED`: **true**: Enable/Disable package registry capabilities
- `CHUNKED_UPLOAD_PATH`: **tmp/package-upload**: Path for chunked uploads. Defaults to `APP_DATA_PATH` + `tmp/package-upload` - `CHUNKED_UPLOAD_PATH`: **tmp/package-upload**: Path for chunked uploads. Defaults to `APP_DATA_PATH` + `tmp/package-upload`
- `LIMIT_TOTAL_OWNER_COUNT`: **-1**: Maxmimum count of package versions a single owner can have (`-1` means no limits)
- `LIMIT_TOTAL_OWNER_SIZE`: **-1**: Maxmimum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_COMPOSER`: **-1**: Maxmimum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_CONAN`: **-1**: Maxmimum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_CONTAINER`: **-1**: Maxmimum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_GENERIC`: **-1**: Maxmimum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_HELM`: **-1**: Maxmimum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_MAVEN`: **-1**: Maxmimum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_NPM`: **-1**: Maxmimum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_NUGET`: **-1**: Maxmimum size of a NuGet upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_PUB`: **-1**: Maxmimum size of a Pub upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_PYPI`: **-1**: Maxmimum size of a PyPI upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_RUBYGEMS`: **-1**: Maxmimum size of a RubyGems upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_VAGRANT`: **-1**: Maxmimum size of a Vagrant upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
## Mirror (`mirror`) ## Mirror (`mirror`)

View file

@ -199,3 +199,13 @@ func SearchFiles(ctx context.Context, opts *PackageFileSearchOptions) ([]*Packag
count, err := sess.FindAndCount(&pfs) count, err := sess.FindAndCount(&pfs)
return pfs, count, err return pfs, count, err
} }
// CalculateBlobSize sums up all blob sizes matching the search options.
// It does NOT respect the deduplication of blobs.
func CalculateBlobSize(ctx context.Context, opts *PackageFileSearchOptions) (int64, error) {
return db.GetEngine(ctx).
Table("package_file").
Where(opts.toConds()).
Join("INNER", "package_blob", "package_blob.id = package_file.blob_id").
SumInt(new(PackageBlob), "size")
}

View file

@ -319,3 +319,12 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P
count, err := sess.FindAndCount(&pvs) count, err := sess.FindAndCount(&pvs)
return pvs, count, err return pvs, count, err
} }
// CountVersions counts all versions of packages matching the search options
func CountVersions(ctx context.Context, opts *PackageSearchOptions) (int64, error) {
return db.GetEngine(ctx).
Where(opts.toConds()).
Table("package_version").
Join("INNER", "package", "package.id = package_version.package_id").
Count(new(PackageVersion))
}

View file

@ -5,11 +5,15 @@
package setting package setting
import ( import (
"math"
"net/url" "net/url"
"os" "os"
"path/filepath" "path/filepath"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"github.com/dustin/go-humanize"
ini "gopkg.in/ini.v1"
) )
// Package registry settings // Package registry settings
@ -19,8 +23,24 @@ var (
Enabled bool Enabled bool
ChunkedUploadPath string ChunkedUploadPath string
RegistryHost string RegistryHost string
LimitTotalOwnerCount int64
LimitTotalOwnerSize int64
LimitSizeComposer int64
LimitSizeConan int64
LimitSizeContainer int64
LimitSizeGeneric int64
LimitSizeHelm int64
LimitSizeMaven int64
LimitSizeNpm int64
LimitSizeNuGet int64
LimitSizePub int64
LimitSizePyPI int64
LimitSizeRubyGems int64
LimitSizeVagrant int64
}{ }{
Enabled: true, Enabled: true,
LimitTotalOwnerCount: -1,
} }
) )
@ -43,4 +63,32 @@ func newPackages() {
if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil { if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil {
log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err)
} }
Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE")
Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER")
Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN")
Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER")
Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC")
Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM")
Packages.LimitSizeMaven = mustBytes(sec, "LIMIT_SIZE_MAVEN")
Packages.LimitSizeNpm = mustBytes(sec, "LIMIT_SIZE_NPM")
Packages.LimitSizeNuGet = mustBytes(sec, "LIMIT_SIZE_NUGET")
Packages.LimitSizePub = mustBytes(sec, "LIMIT_SIZE_PUB")
Packages.LimitSizePyPI = mustBytes(sec, "LIMIT_SIZE_PYPI")
Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS")
Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT")
}
func mustBytes(section *ini.Section, key string) int64 {
const noLimit = "-1"
value := section.Key(key).MustString(noLimit)
if value == noLimit {
return -1
}
bytes, err := humanize.ParseBytes(value)
if err != nil || bytes > math.MaxInt64 {
return -1
}
return int64(bytes)
} }

View file

@ -0,0 +1,31 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package setting
import (
"testing"
"github.com/stretchr/testify/assert"
ini "gopkg.in/ini.v1"
)
func TestMustBytes(t *testing.T) {
test := func(value string) int64 {
sec, _ := ini.Empty().NewSection("test")
sec.NewKey("VALUE", value)
return mustBytes(sec, "VALUE")
}
assert.EqualValues(t, -1, test(""))
assert.EqualValues(t, -1, test("-1"))
assert.EqualValues(t, 0, test("0"))
assert.EqualValues(t, 1, test("1"))
assert.EqualValues(t, 10000, test("10000"))
assert.EqualValues(t, 1000000, test("1 mb"))
assert.EqualValues(t, 1048576, test("1mib"))
assert.EqualValues(t, 1782579, test("1.7mib"))
assert.EqualValues(t, -1, test("1 yib")) // too large
}

View file

@ -235,16 +235,20 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{
Filename: strings.ToLower(fmt.Sprintf("%s.%s.zip", strings.ReplaceAll(cp.Name, "/", "-"), cp.Version)), Filename: strings.ToLower(fmt.Sprintf("%s.%s.zip", strings.ReplaceAll(cp.Name, "/", "-"), cp.Version)),
}, },
Creator: ctx.Doer,
Data: buf, Data: buf,
IsLead: true, IsLead: true,
}, },
) )
if err != nil { if err != nil {
if err == packages_model.ErrDuplicatePackageVersion { switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusBadRequest, err) apiError(ctx, http.StatusBadRequest, err)
return case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
} apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
}
return return
} }

View file

@ -348,6 +348,7 @@ func uploadFile(ctx *context.Context, fileFilter container.Set[string], fileKey
Filename: strings.ToLower(filename), Filename: strings.ToLower(filename),
CompositeKey: fileKey, CompositeKey: fileKey,
}, },
Creator: ctx.Doer,
Data: buf, Data: buf,
IsLead: isConanfileFile, IsLead: isConanfileFile,
Properties: map[string]string{ Properties: map[string]string{
@ -416,11 +417,14 @@ func uploadFile(ctx *context.Context, fileFilter container.Set[string], fileKey
pfci, pfci,
) )
if err != nil { if err != nil {
if err == packages_model.ErrDuplicatePackageFile { switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusBadRequest, err) apiError(ctx, http.StatusBadRequest, err)
return case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
} apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
}
return return
} }

View file

@ -104,16 +104,20 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{
Filename: filename, Filename: filename,
}, },
Creator: ctx.Doer,
Data: buf, Data: buf,
IsLead: true, IsLead: true,
}, },
) )
if err != nil { if err != nil {
if err == packages_model.ErrDuplicatePackageFile { switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusConflict, err) apiError(ctx, http.StatusConflict, err)
return case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
} apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
}
return return
} }

View file

@ -186,17 +186,21 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{
Filename: createFilename(metadata), Filename: createFilename(metadata),
}, },
Creator: ctx.Doer,
Data: buf, Data: buf,
IsLead: true, IsLead: true,
OverwriteExisting: true, OverwriteExisting: true,
}, },
) )
if err != nil { if err != nil {
if err == packages_model.ErrDuplicatePackageVersion { switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusConflict, err) apiError(ctx, http.StatusConflict, err)
return case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
} apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
}
return return
} }

View file

@ -266,6 +266,7 @@ func UploadPackageFile(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{
Filename: params.Filename, Filename: params.Filename,
}, },
Creator: ctx.Doer,
Data: buf, Data: buf,
IsLead: false, IsLead: false,
OverwriteExisting: params.IsMeta, OverwriteExisting: params.IsMeta,
@ -312,11 +313,14 @@ func UploadPackageFile(ctx *context.Context) {
pfci, pfci,
) )
if err != nil { if err != nil {
if err == packages_model.ErrDuplicatePackageFile { switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusBadRequest, err) apiError(ctx, http.StatusBadRequest, err)
return case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
} apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
}
return return
} }

View file

@ -180,16 +180,20 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{
Filename: npmPackage.Filename, Filename: npmPackage.Filename,
}, },
Creator: ctx.Doer,
Data: buf, Data: buf,
IsLead: true, IsLead: true,
}, },
) )
if err != nil { if err != nil {
if err == packages_model.ErrDuplicatePackageVersion { switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusBadRequest, err) apiError(ctx, http.StatusBadRequest, err)
return case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
} apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
}
return return
} }

View file

@ -374,16 +374,20 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{
Filename: strings.ToLower(fmt.Sprintf("%s.%s.nupkg", np.ID, np.Version)), Filename: strings.ToLower(fmt.Sprintf("%s.%s.nupkg", np.ID, np.Version)),
}, },
Creator: ctx.Doer,
Data: buf, Data: buf,
IsLead: true, IsLead: true,
}, },
) )
if err != nil { if err != nil {
if err == packages_model.ErrDuplicatePackageVersion { switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusConflict, err) apiError(ctx, http.StatusConflict, err)
return case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
} apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
}
return return
} }
@ -428,6 +432,7 @@ func UploadSymbolPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{
Filename: strings.ToLower(fmt.Sprintf("%s.%s.snupkg", np.ID, np.Version)), Filename: strings.ToLower(fmt.Sprintf("%s.%s.snupkg", np.ID, np.Version)),
}, },
Creator: ctx.Doer,
Data: buf, Data: buf,
IsLead: false, IsLead: false,
}, },
@ -438,6 +443,8 @@ func UploadSymbolPackage(ctx *context.Context) {
apiError(ctx, http.StatusNotFound, err) apiError(ctx, http.StatusNotFound, err)
case packages_model.ErrDuplicatePackageFile: case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusConflict, err) apiError(ctx, http.StatusConflict, err)
case packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default: default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
} }
@ -452,6 +459,7 @@ func UploadSymbolPackage(ctx *context.Context) {
Filename: strings.ToLower(pdb.Name), Filename: strings.ToLower(pdb.Name),
CompositeKey: strings.ToLower(pdb.ID), CompositeKey: strings.ToLower(pdb.ID),
}, },
Creator: ctx.Doer,
Data: pdb.Content, Data: pdb.Content,
IsLead: false, IsLead: false,
Properties: map[string]string{ Properties: map[string]string{
@ -463,6 +471,8 @@ func UploadSymbolPackage(ctx *context.Context) {
switch err { switch err {
case packages_model.ErrDuplicatePackageFile: case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusConflict, err) apiError(ctx, http.StatusConflict, err)
case packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default: default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
} }

View file

@ -199,16 +199,20 @@ func UploadPackageFile(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{
Filename: strings.ToLower(pck.Version + ".tar.gz"), Filename: strings.ToLower(pck.Version + ".tar.gz"),
}, },
Creator: ctx.Doer,
Data: buf, Data: buf,
IsLead: true, IsLead: true,
}, },
) )
if err != nil { if err != nil {
if err == packages_model.ErrDuplicatePackageVersion { switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusBadRequest, err) apiError(ctx, http.StatusBadRequest, err)
return case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
} apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
}
return return
} }

View file

@ -162,16 +162,20 @@ func UploadPackageFile(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{
Filename: fileHeader.Filename, Filename: fileHeader.Filename,
}, },
Creator: ctx.Doer,
Data: buf, Data: buf,
IsLead: true, IsLead: true,
}, },
) )
if err != nil { if err != nil {
if err == packages_model.ErrDuplicatePackageFile { switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusBadRequest, err) apiError(ctx, http.StatusBadRequest, err)
return case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
} apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
}
return return
} }

View file

@ -242,16 +242,20 @@ func UploadPackageFile(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{
Filename: filename, Filename: filename,
}, },
Creator: ctx.Doer,
Data: buf, Data: buf,
IsLead: true, IsLead: true,
}, },
) )
if err != nil { if err != nil {
if err == packages_model.ErrDuplicatePackageVersion { switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusBadRequest, err) apiError(ctx, http.StatusBadRequest, err)
return case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
} apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
}
return return
} }

View file

@ -193,6 +193,7 @@ func UploadPackageFile(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{
Filename: strings.ToLower(boxProvider), Filename: strings.ToLower(boxProvider),
}, },
Creator: ctx.Doer,
Data: buf, Data: buf,
IsLead: true, IsLead: true,
Properties: map[string]string{ Properties: map[string]string{
@ -201,11 +202,14 @@ func UploadPackageFile(ctx *context.Context) {
}, },
) )
if err != nil { if err != nil {
if err == packages_model.ErrDuplicatePackageFile { switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusConflict, err) apiError(ctx, http.StatusConflict, err)
return case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
} apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err) apiError(ctx, http.StatusInternalServerError, err)
}
return return
} }

View file

@ -6,6 +6,7 @@ package packages
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io" "io"
"strings" "strings"
@ -19,10 +20,17 @@ import (
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/notification"
packages_module "code.gitea.io/gitea/modules/packages" packages_module "code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"
container_service "code.gitea.io/gitea/services/packages/container" container_service "code.gitea.io/gitea/services/packages/container"
) )
var (
ErrQuotaTypeSize = errors.New("maximum allowed package type size exceeded")
ErrQuotaTotalSize = errors.New("maximum allowed package storage quota exceeded")
ErrQuotaTotalCount = errors.New("maximum allowed package count exceeded")
)
// PackageInfo describes a package // PackageInfo describes a package
type PackageInfo struct { type PackageInfo struct {
Owner *user_model.User Owner *user_model.User
@ -50,6 +58,7 @@ type PackageFileInfo struct {
// PackageFileCreationInfo describes a package file to create // PackageFileCreationInfo describes a package file to create
type PackageFileCreationInfo struct { type PackageFileCreationInfo struct {
PackageFileInfo PackageFileInfo
Creator *user_model.User
Data packages_module.HashedSizeReader Data packages_module.HashedSizeReader
IsLead bool IsLead bool
Properties map[string]string Properties map[string]string
@ -78,7 +87,7 @@ func createPackageAndAddFile(pvci *PackageCreationInfo, pfci *PackageFileCreatio
return nil, nil, err return nil, nil, err
} }
pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci) pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, &pvci.PackageInfo, pfci)
removeBlob := false removeBlob := false
defer func() { defer func() {
if blobCreated && removeBlob { if blobCreated && removeBlob {
@ -164,6 +173,10 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all
} }
if versionCreated { if versionCreated {
if err := checkCountQuotaExceeded(ctx, pvci.Creator, pvci.Owner); err != nil {
return nil, false, err
}
for name, value := range pvci.VersionProperties { for name, value := range pvci.VersionProperties {
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, name, value); err != nil { if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, name, value); err != nil {
log.Error("Error setting package version property: %v", err) log.Error("Error setting package version property: %v", err)
@ -188,7 +201,7 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (
return nil, nil, err return nil, nil, err
} }
pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci) pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pvi, pfci)
removeBlob := false removeBlob := false
defer func() { defer func() {
if removeBlob { if removeBlob {
@ -224,9 +237,13 @@ func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.Packag
} }
} }
func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename) log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename)
if err := checkSizeQuotaExceeded(ctx, pfci.Creator, pvi.Owner, pvi.PackageType, pfci.Data.Size()); err != nil {
return nil, nil, false, err
}
pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data)) pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data))
if err != nil { if err != nil {
log.Error("Error inserting package blob: %v", err) log.Error("Error inserting package blob: %v", err)
@ -285,6 +302,80 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers
return pf, pb, !exists, nil return pf, pb, !exists, nil
} }
func checkCountQuotaExceeded(ctx context.Context, doer, owner *user_model.User) error {
if doer.IsAdmin {
return nil
}
if setting.Packages.LimitTotalOwnerCount > -1 {
totalCount, err := packages_model.CountVersions(ctx, &packages_model.PackageSearchOptions{
OwnerID: owner.ID,
IsInternal: util.OptionalBoolFalse,
})
if err != nil {
log.Error("CountVersions failed: %v", err)
return err
}
if totalCount > setting.Packages.LimitTotalOwnerCount {
return ErrQuotaTotalCount
}
}
return nil
}
func checkSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, packageType packages_model.Type, uploadSize int64) error {
if doer.IsAdmin {
return nil
}
var typeSpecificSize int64
switch packageType {
case packages_model.TypeComposer:
typeSpecificSize = setting.Packages.LimitSizeComposer
case packages_model.TypeConan:
typeSpecificSize = setting.Packages.LimitSizeConan
case packages_model.TypeContainer:
typeSpecificSize = setting.Packages.LimitSizeContainer
case packages_model.TypeGeneric:
typeSpecificSize = setting.Packages.LimitSizeGeneric
case packages_model.TypeHelm:
typeSpecificSize = setting.Packages.LimitSizeHelm
case packages_model.TypeMaven:
typeSpecificSize = setting.Packages.LimitSizeMaven
case packages_model.TypeNpm:
typeSpecificSize = setting.Packages.LimitSizeNpm
case packages_model.TypeNuGet:
typeSpecificSize = setting.Packages.LimitSizeNuGet
case packages_model.TypePub:
typeSpecificSize = setting.Packages.LimitSizePub
case packages_model.TypePyPI:
typeSpecificSize = setting.Packages.LimitSizePyPI
case packages_model.TypeRubyGems:
typeSpecificSize = setting.Packages.LimitSizeRubyGems
case packages_model.TypeVagrant:
typeSpecificSize = setting.Packages.LimitSizeVagrant
}
if typeSpecificSize > -1 && typeSpecificSize < uploadSize {
return ErrQuotaTypeSize
}
if setting.Packages.LimitTotalOwnerSize > -1 {
totalSize, err := packages_model.CalculateBlobSize(ctx, &packages_model.PackageFileSearchOptions{
OwnerID: owner.ID,
})
if err != nil {
log.Error("CalculateBlobSize failed: %v", err)
return err
}
if totalSize+uploadSize > setting.Packages.LimitTotalOwnerSize {
return ErrQuotaTotalSize
}
}
return nil
}
// RemovePackageVersionByNameAndVersion deletes a package version and all associated files // RemovePackageVersionByNameAndVersion deletes a package version and all associated files
func RemovePackageVersionByNameAndVersion(doer *user_model.User, pvi *PackageInfo) error { func RemovePackageVersionByNameAndVersion(doer *user_model.User, pvi *PackageInfo) error {
pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)

View file

@ -16,6 +16,7 @@ import (
container_model "code.gitea.io/gitea/models/packages/container" container_model "code.gitea.io/gitea/models/packages/container"
"code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user" user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs"
packages_service "code.gitea.io/gitea/services/packages" packages_service "code.gitea.io/gitea/services/packages"
"code.gitea.io/gitea/tests" "code.gitea.io/gitea/tests"
@ -166,6 +167,39 @@ func TestPackageAccess(t *testing.T) {
uploadPackage(admin, user, http.StatusCreated) uploadPackage(admin, user, http.StatusCreated)
} }
func TestPackageQuota(t *testing.T) {
defer tests.PrepareTestEnv(t)()
limitTotalOwnerCount, limitTotalOwnerSize, limitSizeGeneric := setting.Packages.LimitTotalOwnerCount, setting.Packages.LimitTotalOwnerSize, setting.Packages.LimitSizeGeneric
admin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 10})
uploadPackage := func(doer *user_model.User, version string, expectedStatus int) {
url := fmt.Sprintf("/api/packages/%s/generic/test-package/%s/file.bin", user.Name, version)
req := NewRequestWithBody(t, "PUT", url, bytes.NewReader([]byte{1}))
AddBasicAuthHeader(req, doer.Name)
MakeRequest(t, req, expectedStatus)
}
// Exceeded quota result in StatusForbidden for normal users but admins are always allowed to upload.
setting.Packages.LimitTotalOwnerCount = 0
uploadPackage(user, "1.0", http.StatusForbidden)
uploadPackage(admin, "1.0", http.StatusCreated)
setting.Packages.LimitTotalOwnerCount = limitTotalOwnerCount
setting.Packages.LimitTotalOwnerSize = 0
uploadPackage(user, "1.1", http.StatusForbidden)
uploadPackage(admin, "1.1", http.StatusCreated)
setting.Packages.LimitTotalOwnerSize = limitTotalOwnerSize
setting.Packages.LimitSizeGeneric = 0
uploadPackage(user, "1.2", http.StatusForbidden)
uploadPackage(admin, "1.2", http.StatusCreated)
setting.Packages.LimitSizeGeneric = limitSizeGeneric
}
func TestPackageCleanup(t *testing.T) { func TestPackageCleanup(t *testing.T) {
defer tests.PrepareTestEnv(t)() defer tests.PrepareTestEnv(t)()