From 07629bd55cb9bf71459dcdf2eb3feaa5d24cfeca Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 2 Dec 2020 01:28:34 +0100 Subject: [PATCH] Add Allow-/Block-List for Migrate & Mirrors (#13610) (#13776) * add black list and white list support for migrating repositories * specify log message * use blocklist/allowlist * allways use lowercase to match url * Apply allow/block * Settings: use existing "migrations" section * convert domains lower case * dont store unused value * Block private addresses for migration by default * use proposed-upstream func to detect private IP addr * add own error for blocked migration, add tests, imprufe api * fix test * fix-if-localhost-is-ipv4 * rename error & error message * rename setting options * Apply suggestions from code review Co-authored-by: Lunny Xiao Co-authored-by: zeripath Co-authored-by: techknowlogick Co-authored-by: Lunny Xiao Co-authored-by: zeripath Co-authored-by: techknowlogick --- custom/conf/app.example.ini | 8 ++ .../doc/advanced/config-cheat-sheet.en-us.md | 3 + .../doc/advanced/config-cheat-sheet.zh-cn.md | 3 + integrations/api_repo_test.go | 12 ++- models/error.go | 23 ++++++ modules/matchlist/matchlist.go | 46 ++++++++++++ modules/migrations/migrate.go | 75 ++++++++++++++++++- modules/migrations/migrate_test.go | 34 +++++++++ modules/setting/migrations.go | 22 +++++- routers/api/v1/repo/migrate.go | 2 + routers/init.go | 5 ++ 11 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 modules/matchlist/matchlist.go create mode 100644 modules/migrations/migrate_test.go diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 2c083dd9e..529ccea93 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1188,6 +1188,14 @@ QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0" MAX_ATTEMPTS = 3 ; Backoff time per http/https request retry (seconds) RETRY_BACKOFF = 3 +; Allowed domains for migrating, default is blank. Blank means everything will be allowed. +; Multiple domains could be separated by commas. +ALLOWED_DOMAINS = +; Blocklist for migrating, default is blank. Multiple domains could be separated by commas. +; When ALLOWED_DOMAINS is not blank, this option will be ignored. +BLOCKED_DOMAINS = +; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 (false by default) +ALLOW_LOCALNETWORKS = false ; default storage for attachments, lfs and avatars [storage] diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 28c957c8d..ca09d4f03 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -811,6 +811,9 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations. - `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds) +- `ALLOWED_DOMAINS`: **\**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. +- `BLOCKED_DOMAINS`: **\**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored. +- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 ## Mirror (`mirror`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 505fdcdf7..597773a0a 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -313,6 +313,9 @@ IS_INPUT_FILE = false - `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。 - `RETRY_BACKOFF`: **3**: 等待下一次重试的时间，单位秒。 +- `ALLOWED_DOMAINS`: **\**: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。 +- `BLOCKED_DOMAINS`: **\**: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空，此选项将会被忽略。 +- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918 ## LFS (`lfs`) diff --git a/integrations/api_repo_test.go b/integrations/api_repo_test.go index c8afa73ae..1aa45984e 100644 --- a/integrations/api_repo_test.go +++ b/integrations/api_repo_test.go @@ -309,6 +309,8 @@ func TestAPIRepoMigrate(t *testing.T) { {ctxUserID: 2, userID: 1, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad", expectedStatus: http.StatusForbidden}, {ctxUserID: 2, userID: 3, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-org", expectedStatus: http.StatusCreated}, {ctxUserID: 2, userID: 6, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad-org", expectedStatus: http.StatusForbidden}, + {ctxUserID: 2, userID: 3, cloneURL: "https://localhost:3000/user/test_repo.git", repoName: "local-ip", expectedStatus: http.StatusUnprocessableEntity}, + {ctxUserID: 2, userID: 3, cloneURL: "https://10.0.0.1/user/test_repo.git", repoName: "private-ip", expectedStatus: http.StatusUnprocessableEntity}, } defer prepareTestEnv(t)() @@ -325,8 +327,16 @@ func TestAPIRepoMigrate(t *testing.T) { if resp.Code == http.StatusUnprocessableEntity { respJSON := map[string]string{} DecodeJSON(t, resp, &respJSON) - if assert.Equal(t, respJSON["message"], "Remote visit addressed rate limitation.") { + switch respJSON["message"] { + case "Remote visit addressed rate limitation.": t.Log("test hit github rate limitation") + case "migrate from '10.0.0.1' is not allowed: the host resolve to a private ip address '10.0.0.1'": + assert.EqualValues(t, "private-ip", testCase.repoName) + case "migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '::1'", + "migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '127.0.0.1'": + assert.EqualValues(t, "local-ip", testCase.repoName) + default: + t.Errorf("unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL) } } else { assert.EqualValues(t, testCase.expectedStatus, resp.Code) diff --git a/models/error.go b/models/error.go index 83354ff17..7f1eda1b1 100644 --- a/models/error.go +++ b/models/error.go @@ -1019,6 +1019,29 @@ func IsErrWontSign(err error) bool { return ok } +// ErrMigrationNotAllowed explains why a migration from an url is not allowed +type ErrMigrationNotAllowed struct { + Host string + NotResolvedIP bool + PrivateNet string +} + +func (e *ErrMigrationNotAllowed) Error() string { + if e.NotResolvedIP { + return fmt.Sprintf("migrate from '%s' is not allowed: unknown hostname", e.Host) + } + if len(e.PrivateNet) != 0 { + return fmt.Sprintf("migrate from '%s' is not allowed: the host resolve to a private ip address '%s'", e.Host, e.PrivateNet) + } + return fmt.Sprintf("migrate from '%s is not allowed'", e.Host) +} + +// IsErrMigrationNotAllowed checks if an error is a ErrMigrationNotAllowed +func IsErrMigrationNotAllowed(err error) bool { + _, ok := err.(*ErrMigrationNotAllowed) + return ok +} + // __________ .__ // \______ \____________ ____ ____ | |__ // | | _/\_ __ \__ \ / \_/ ___\| | \ diff --git a/modules/matchlist/matchlist.go b/modules/matchlist/matchlist.go new file mode 100644 index 000000000..b65ed909d --- /dev/null +++ b/modules/matchlist/matchlist.go @@ -0,0 +1,46 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package matchlist + +import ( + "strings" + + "github.com/gobwas/glob" +) + +// Matchlist represents a block or allow list +type Matchlist struct { + ruleGlobs []glob.Glob +} + +// NewMatchlist creates a new block or allow list +func NewMatchlist(rules ...string) (*Matchlist, error) { + for i := range rules { + rules[i] = strings.ToLower(rules[i]) + } + list := Matchlist{ + ruleGlobs: make([]glob.Glob, 0, len(rules)), + } + + for _, rule := range rules { + rg, err := glob.Compile(rule) + if err != nil { + return nil, err + } + list.ruleGlobs = append(list.ruleGlobs, rg) + } + + return &list, nil +} + +// Match will matches +func (b *Matchlist) Match(u string) bool { + for _, r := range b.ruleGlobs { + if r.Match(u) { + return true + } + } + return false +} diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 3c505d82b..b3ecb8114 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -8,9 +8,13 @@ package migrations import ( "context" "fmt" + "net" + "net/url" + "strings" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/matchlist" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/setting" ) @@ -20,6 +24,9 @@ type MigrateOptions = base.MigrateOptions var ( factories []base.DownloaderFactory + + allowList *matchlist.Matchlist + blockList *matchlist.Matchlist ) // RegisterDownloaderFactory registers a downloader factory @@ -27,12 +34,49 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) { factories = append(factories, factory) } +func isMigrateURLAllowed(remoteURL string) error { + u, err := url.Parse(strings.ToLower(remoteURL)) + if err != nil { + return err + } + + if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { + if len(setting.Migrations.AllowedDomains) > 0 { + if !allowList.Match(u.Host) { + return &models.ErrMigrationNotAllowed{Host: u.Host} + } + } else { + if blockList.Match(u.Host) { + return &models.ErrMigrationNotAllowed{Host: u.Host} + } + } + } + + if !setting.Migrations.AllowLocalNetworks { + addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0]) + if err != nil { + return &models.ErrMigrationNotAllowed{Host: u.Host, NotResolvedIP: true} + } + for _, addr := range addrList { + if isIPPrivate(addr) || !addr.IsGlobalUnicast() { + return &models.ErrMigrationNotAllowed{Host: u.Host, PrivateNet: addr.String()} + } + } + } + + return nil +} + // MigrateRepository migrate repository according MigrateOptions func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { + err := isMigrateURLAllowed(opts.CloneAddr) + if err != nil { + return nil, err + } + var ( downloader base.Downloader uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) - err error ) for _, factory := range factories { @@ -308,3 +352,32 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return nil } + +// Init migrations service +func Init() error { + var err error + allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...) + if err != nil { + return fmt.Errorf("init migration allowList domains failed: %v", err) + } + + blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...) + if err != nil { + return fmt.Errorf("init migration blockList domains failed: %v", err) + } + + return nil +} + +// isIPPrivate reports whether ip is a private address, according to +// RFC 1918 (IPv4 addresses) and RFC 4193 (IPv6 addresses). +// from https://github.com/golang/go/pull/42793 +// TODO remove if https://github.com/golang/go/issues/29146 got resolved +func isIPPrivate(ip net.IP) bool { + if ip4 := ip.To4(); ip4 != nil { + return ip4[0] == 10 || + (ip4[0] == 172 && ip4[1]&0xf0 == 16) || + (ip4[0] == 192 && ip4[1] == 168) + } + return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc +} diff --git a/modules/migrations/migrate_test.go b/modules/migrations/migrate_test.go new file mode 100644 index 000000000..3bad5cfd7 --- /dev/null +++ b/modules/migrations/migrate_test.go @@ -0,0 +1,34 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "testing" + + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func TestMigrateWhiteBlocklist(t *testing.T) { + setting.Migrations.AllowedDomains = []string{"github.com"} + assert.NoError(t, Init()) + + err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") + assert.Error(t, err) + + err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") + assert.NoError(t, err) + + setting.Migrations.AllowedDomains = []string{} + setting.Migrations.BlockedDomains = []string{"github.com"} + assert.NoError(t, Init()) + + err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") + assert.NoError(t, err) + + err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") + assert.Error(t, err) +} diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go index 51d6bbcf1..7808df528 100644 --- a/modules/setting/migrations.go +++ b/modules/setting/migrations.go @@ -4,11 +4,18 @@ package setting +import ( + "strings" +) + var ( // Migrations settings Migrations = struct { - MaxAttempts int - RetryBackoff int + MaxAttempts int + RetryBackoff int + AllowedDomains []string + BlockedDomains []string + AllowLocalNetworks bool }{ MaxAttempts: 3, RetryBackoff: 3, @@ -19,4 +26,15 @@ func newMigrationsService() { sec := Cfg.Section("migrations") Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) + + Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").Strings(",") + for i := range Migrations.AllowedDomains { + Migrations.AllowedDomains[i] = strings.ToLower(Migrations.AllowedDomains[i]) + } + Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").Strings(",") + for i := range Migrations.BlockedDomains { + Migrations.BlockedDomains[i] = strings.ToLower(Migrations.BlockedDomains[i]) + } + + Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false) } diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index f9cddbb7c..68ab7e489 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -212,6 +212,8 @@ func handleMigrateError(ctx *context.APIContext, repoOwner *models.User, remoteA ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The username '%s' contains invalid characters.", err.(models.ErrNameCharsNotAllowed).Name)) case models.IsErrNamePatternNotAllowed(err): ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The pattern '%s' is not allowed in a username.", err.(models.ErrNamePatternNotAllowed).Pattern)) + case models.IsErrMigrationNotAllowed(err): + ctx.Error(http.StatusUnprocessableEntity, "", err) default: err = util.URLSanitizedError(err, remoteAddr) if strings.Contains(err.Error(), "Authentication failed") || diff --git a/routers/init.go b/routers/init.go index faa84fb0c..34b94eb47 100644 --- a/routers/init.go +++ b/routers/init.go @@ -24,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/external" + repo_migrations "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" @@ -174,6 +175,10 @@ func GlobalInit(ctx context.Context) { } checkRunMode() + if err := repo_migrations.Init(); err != nil { + log.Fatal("Failed to initialize repository migrations: %v", err) + } + // Now because Install will re-run GlobalInit once it has set InstallLock // we can't tell if the ssh port will remain unused until that's done. // However, see FIXME comment in install.go