Use hostmatcher
to replace matchlist
, improve security (#17605)
Use hostmacher to replace matchlist. And we introduce a better DialContext to do a full host/IP check, otherwise the attackers can still bypass the allow/block list by a 302 redirection.
This commit is contained in:
parent
c96be0cd98
commit
013fb73068
33 changed files with 377 additions and 293 deletions
|
@ -2114,7 +2114,7 @@ PATH =
|
||||||
;ALLOWED_DOMAINS =
|
;ALLOWED_DOMAINS =
|
||||||
;;
|
;;
|
||||||
;; Blocklist for migrating, default is blank. Multiple domains could be separated by commas.
|
;; Blocklist for migrating, default is blank. Multiple domains could be separated by commas.
|
||||||
;; When ALLOWED_DOMAINS is not blank, this option will be ignored.
|
;; When ALLOWED_DOMAINS is not blank, this option has a higher priority to deny domains.
|
||||||
;BLOCKED_DOMAINS =
|
;BLOCKED_DOMAINS =
|
||||||
;;
|
;;
|
||||||
;; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 (false by default)
|
;; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 (false by default)
|
||||||
|
|
|
@ -1045,7 +1045,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
|
||||||
- `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations.
|
- `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations.
|
||||||
- `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds)
|
- `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds)
|
||||||
- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas.
|
- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas.
|
||||||
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored.
|
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option has a higher priority to deny domains.
|
||||||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291
|
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291
|
||||||
- `SKIP_TLS_VERIFY`: **false**: Allow skip tls verify
|
- `SKIP_TLS_VERIFY`: **false**: Allow skip tls verify
|
||||||
|
|
||||||
|
|
|
@ -335,7 +335,7 @@ IS_INPUT_FILE = false
|
||||||
- `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。
|
- `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。
|
||||||
- `RETRY_BACKOFF`: **3**: 等待下一次重试的时间,单位秒。
|
- `RETRY_BACKOFF`: **3**: 等待下一次重试的时间,单位秒。
|
||||||
- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单,默认为空,表示允许从任意域名迁移仓库,多个域名用逗号分隔。
|
- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单,默认为空,表示允许从任意域名迁移仓库,多个域名用逗号分隔。
|
||||||
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项将会被忽略。
|
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项有更高的优先级拒绝这里的域名。
|
||||||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918
|
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918
|
||||||
- `SKIP_TLS_VERIFY`: **false**: 允许忽略 TLS 认证
|
- `SKIP_TLS_VERIFY`: **false**: 允许忽略 TLS 认证
|
||||||
|
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
"code.gitea.io/gitea/modules/lfs"
|
"code.gitea.io/gitea/modules/lfs"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
api "code.gitea.io/gitea/modules/structs"
|
api "code.gitea.io/gitea/modules/structs"
|
||||||
|
"code.gitea.io/gitea/services/migrations"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
@ -25,6 +26,7 @@ func TestAPIRepoLFSMigrateLocal(t *testing.T) {
|
||||||
oldAllowLocalNetworks := setting.Migrations.AllowLocalNetworks
|
oldAllowLocalNetworks := setting.Migrations.AllowLocalNetworks
|
||||||
setting.ImportLocalPaths = true
|
setting.ImportLocalPaths = true
|
||||||
setting.Migrations.AllowLocalNetworks = true
|
setting.Migrations.AllowLocalNetworks = true
|
||||||
|
assert.NoError(t, migrations.Init())
|
||||||
|
|
||||||
user := unittest.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
|
user := unittest.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
|
||||||
session := loginUser(t, user.Name)
|
session := loginUser(t, user.Name)
|
||||||
|
@ -47,4 +49,5 @@ func TestAPIRepoLFSMigrateLocal(t *testing.T) {
|
||||||
|
|
||||||
setting.ImportLocalPaths = oldImportLocalPaths
|
setting.ImportLocalPaths = oldImportLocalPaths
|
||||||
setting.Migrations.AllowLocalNetworks = oldAllowLocalNetworks
|
setting.Migrations.AllowLocalNetworks = oldAllowLocalNetworks
|
||||||
|
assert.NoError(t, migrations.Init()) // reset old migration settings
|
||||||
}
|
}
|
||||||
|
|
|
@ -331,10 +331,10 @@ func TestAPIRepoMigrate(t *testing.T) {
|
||||||
switch respJSON["message"] {
|
switch respJSON["message"] {
|
||||||
case "Remote visit addressed rate limitation.":
|
case "Remote visit addressed rate limitation.":
|
||||||
t.Log("test hit github rate limitation")
|
t.Log("test hit github rate limitation")
|
||||||
case "You are not allowed to import from private IPs.":
|
case "You can not import from disallowed hosts.":
|
||||||
assert.EqualValues(t, "private-ip", testCase.repoName)
|
assert.EqualValues(t, "private-ip", testCase.repoName)
|
||||||
default:
|
default:
|
||||||
t.Errorf("unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL)
|
assert.Fail(t, "unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
assert.EqualValues(t, testCase.expectedStatus, resp.Code)
|
assert.EqualValues(t, testCase.expectedStatus, resp.Code)
|
||||||
|
|
|
@ -47,7 +47,7 @@ func TestMirrorPull(t *testing.T) {
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
mirror, err := repository.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts)
|
mirror, err := repository.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts, nil)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
gitRepo, err := git.OpenRepository(repoPath)
|
gitRepo, err := git.OpenRepository(repoPath)
|
||||||
|
|
|
@ -16,6 +16,7 @@ import (
|
||||||
"code.gitea.io/gitea/modules/git"
|
"code.gitea.io/gitea/modules/git"
|
||||||
"code.gitea.io/gitea/modules/repository"
|
"code.gitea.io/gitea/modules/repository"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/services/migrations"
|
||||||
mirror_service "code.gitea.io/gitea/services/mirror"
|
mirror_service "code.gitea.io/gitea/services/mirror"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
@ -29,6 +30,7 @@ func testMirrorPush(t *testing.T, u *url.URL) {
|
||||||
defer prepareTestEnv(t)()
|
defer prepareTestEnv(t)()
|
||||||
|
|
||||||
setting.Migrations.AllowLocalNetworks = true
|
setting.Migrations.AllowLocalNetworks = true
|
||||||
|
assert.NoError(t, migrations.Init())
|
||||||
|
|
||||||
user := unittest.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User)
|
user := unittest.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User)
|
||||||
srcRepo := unittest.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
|
srcRepo := unittest.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
|
||||||
|
|
|
@ -797,7 +797,6 @@ type ErrInvalidCloneAddr struct {
|
||||||
IsPermissionDenied bool
|
IsPermissionDenied bool
|
||||||
LocalPath bool
|
LocalPath bool
|
||||||
NotResolvedIP bool
|
NotResolvedIP bool
|
||||||
PrivateNet string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsErrInvalidCloneAddr checks if an error is a ErrInvalidCloneAddr.
|
// IsErrInvalidCloneAddr checks if an error is a ErrInvalidCloneAddr.
|
||||||
|
@ -810,9 +809,6 @@ func (err *ErrInvalidCloneAddr) Error() string {
|
||||||
if err.NotResolvedIP {
|
if err.NotResolvedIP {
|
||||||
return fmt.Sprintf("migration/cloning from '%s' is not allowed: unknown hostname", err.Host)
|
return fmt.Sprintf("migration/cloning from '%s' is not allowed: unknown hostname", err.Host)
|
||||||
}
|
}
|
||||||
if len(err.PrivateNet) != 0 {
|
|
||||||
return fmt.Sprintf("migration/cloning from '%s' is not allowed: the host resolve to a private ip address '%s'", err.Host, err.PrivateNet)
|
|
||||||
}
|
|
||||||
if err.IsInvalidPath {
|
if err.IsInvalidPath {
|
||||||
return fmt.Sprintf("migration/cloning from '%s' is not allowed: the provided path is invalid", err.Host)
|
return fmt.Sprintf("migration/cloning from '%s' is not allowed: the provided path is invalid", err.Host)
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,15 +13,18 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
// HostMatchList is used to check if a host or IP is in a list.
|
// HostMatchList is used to check if a host or IP is in a list.
|
||||||
// If you only need to do wildcard matching, consider to use modules/matchlist
|
|
||||||
type HostMatchList struct {
|
type HostMatchList struct {
|
||||||
hosts []string
|
SettingKeyHint string
|
||||||
|
SettingValue string
|
||||||
|
|
||||||
|
// builtins networks
|
||||||
|
builtins []string
|
||||||
|
// patterns for host names (with wildcard support)
|
||||||
|
patterns []string
|
||||||
|
// ipNets is the CIDR network list
|
||||||
ipNets []*net.IPNet
|
ipNets []*net.IPNet
|
||||||
}
|
}
|
||||||
|
|
||||||
// MatchBuiltinAll all hosts are matched
|
|
||||||
const MatchBuiltinAll = "*"
|
|
||||||
|
|
||||||
// MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched
|
// MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched
|
||||||
const MatchBuiltinExternal = "external"
|
const MatchBuiltinExternal = "external"
|
||||||
|
|
||||||
|
@ -31,9 +34,13 @@ const MatchBuiltinPrivate = "private"
|
||||||
// MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included.
|
// MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included.
|
||||||
const MatchBuiltinLoopback = "loopback"
|
const MatchBuiltinLoopback = "loopback"
|
||||||
|
|
||||||
|
func isBuiltin(s string) bool {
|
||||||
|
return s == MatchBuiltinExternal || s == MatchBuiltinPrivate || s == MatchBuiltinLoopback
|
||||||
|
}
|
||||||
|
|
||||||
// ParseHostMatchList parses the host list HostMatchList
|
// ParseHostMatchList parses the host list HostMatchList
|
||||||
func ParseHostMatchList(hostList string) *HostMatchList {
|
func ParseHostMatchList(settingKeyHint string, hostList string) *HostMatchList {
|
||||||
hl := &HostMatchList{}
|
hl := &HostMatchList{SettingKeyHint: settingKeyHint, SettingValue: hostList}
|
||||||
for _, s := range strings.Split(hostList, ",") {
|
for _, s := range strings.Split(hostList, ",") {
|
||||||
s = strings.ToLower(strings.TrimSpace(s))
|
s = strings.ToLower(strings.TrimSpace(s))
|
||||||
if s == "" {
|
if s == "" {
|
||||||
|
@ -42,53 +49,106 @@ func ParseHostMatchList(hostList string) *HostMatchList {
|
||||||
_, ipNet, err := net.ParseCIDR(s)
|
_, ipNet, err := net.ParseCIDR(s)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
hl.ipNets = append(hl.ipNets, ipNet)
|
hl.ipNets = append(hl.ipNets, ipNet)
|
||||||
|
} else if isBuiltin(s) {
|
||||||
|
hl.builtins = append(hl.builtins, s)
|
||||||
} else {
|
} else {
|
||||||
hl.hosts = append(hl.hosts, s)
|
hl.patterns = append(hl.patterns, s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return hl
|
return hl
|
||||||
}
|
}
|
||||||
|
|
||||||
// MatchesHostOrIP checks if the host or IP matches an allow/deny(block) list
|
// ParseSimpleMatchList parse a simple matchlist (no built-in networks, no CIDR support, only wildcard pattern match)
|
||||||
func (hl *HostMatchList) MatchesHostOrIP(host string, ip net.IP) bool {
|
func ParseSimpleMatchList(settingKeyHint string, matchList string) *HostMatchList {
|
||||||
var matched bool
|
hl := &HostMatchList{
|
||||||
host = strings.ToLower(host)
|
SettingKeyHint: settingKeyHint,
|
||||||
ipStr := ip.String()
|
SettingValue: matchList,
|
||||||
loop:
|
}
|
||||||
for _, hostInList := range hl.hosts {
|
for _, s := range strings.Split(matchList, ",") {
|
||||||
switch hostInList {
|
s = strings.ToLower(strings.TrimSpace(s))
|
||||||
case "":
|
if s == "" {
|
||||||
continue
|
continue
|
||||||
case MatchBuiltinAll:
|
}
|
||||||
matched = true
|
// we keep the same result as old `matchlist`, so no builtin/CIDR support here, we only match wildcard patterns
|
||||||
break loop
|
hl.patterns = append(hl.patterns, s)
|
||||||
|
}
|
||||||
|
return hl
|
||||||
|
}
|
||||||
|
|
||||||
|
// AppendBuiltin appends more builtins to match
|
||||||
|
func (hl *HostMatchList) AppendBuiltin(builtin string) {
|
||||||
|
hl.builtins = append(hl.builtins, builtin)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsEmpty checks if the checklist is empty
|
||||||
|
func (hl *HostMatchList) IsEmpty() bool {
|
||||||
|
return hl == nil || (len(hl.builtins) == 0 && len(hl.patterns) == 0 && len(hl.ipNets) == 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (hl *HostMatchList) checkPattern(host string) bool {
|
||||||
|
host = strings.ToLower(strings.TrimSpace(host))
|
||||||
|
for _, pattern := range hl.patterns {
|
||||||
|
if matched, _ := filepath.Match(pattern, host); matched {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (hl *HostMatchList) checkIP(ip net.IP) bool {
|
||||||
|
for _, pattern := range hl.patterns {
|
||||||
|
if pattern == "*" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, builtin := range hl.builtins {
|
||||||
|
switch builtin {
|
||||||
case MatchBuiltinExternal:
|
case MatchBuiltinExternal:
|
||||||
if matched = ip.IsGlobalUnicast() && !util.IsIPPrivate(ip); matched {
|
if ip.IsGlobalUnicast() && !util.IsIPPrivate(ip) {
|
||||||
break loop
|
return true
|
||||||
}
|
}
|
||||||
case MatchBuiltinPrivate:
|
case MatchBuiltinPrivate:
|
||||||
if matched = util.IsIPPrivate(ip); matched {
|
if util.IsIPPrivate(ip) {
|
||||||
break loop
|
return true
|
||||||
}
|
}
|
||||||
case MatchBuiltinLoopback:
|
case MatchBuiltinLoopback:
|
||||||
if matched = ip.IsLoopback(); matched {
|
if ip.IsLoopback() {
|
||||||
break loop
|
return true
|
||||||
}
|
|
||||||
default:
|
|
||||||
if matched, _ = filepath.Match(hostInList, host); matched {
|
|
||||||
break loop
|
|
||||||
}
|
|
||||||
if matched, _ = filepath.Match(hostInList, ipStr); matched {
|
|
||||||
break loop
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !matched {
|
for _, ipNet := range hl.ipNets {
|
||||||
for _, ipNet := range hl.ipNets {
|
if ipNet.Contains(ip) {
|
||||||
if matched = ipNet.Contains(ip); matched {
|
return true
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return matched
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// MatchHostName checks if the host matches an allow/deny(block) list
|
||||||
|
func (hl *HostMatchList) MatchHostName(host string) bool {
|
||||||
|
if hl == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if hl.checkPattern(host) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if ip := net.ParseIP(host); ip != nil {
|
||||||
|
return hl.checkIP(ip)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// MatchIPAddr checks if the IP matches an allow/deny(block) list, it's safe to pass `nil` to `ip`
|
||||||
|
func (hl *HostMatchList) MatchIPAddr(ip net.IP) bool {
|
||||||
|
if hl == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
host := ip.String() // nil-safe, we will get "<nil>" if ip is nil
|
||||||
|
return hl.checkPattern(host) || hl.checkIP(ip)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MatchHostOrIP checks if the host or IP matches an allow/deny(block) list
|
||||||
|
func (hl *HostMatchList) MatchHostOrIP(host string, ip net.IP) bool {
|
||||||
|
return hl.MatchHostName(host) || hl.MatchIPAddr(ip)
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,17 +20,28 @@ func TestHostOrIPMatchesList(t *testing.T) {
|
||||||
|
|
||||||
// for IPv6: "::1" is loopback, "fd00::/8" is private
|
// for IPv6: "::1" is loopback, "fd00::/8" is private
|
||||||
|
|
||||||
hl := ParseHostMatchList("private, External, *.myDomain.com, 169.254.1.0/24")
|
hl := ParseHostMatchList("", "private, External, *.myDomain.com, 169.254.1.0/24")
|
||||||
|
|
||||||
|
test := func(cases []tc) {
|
||||||
|
for _, c := range cases {
|
||||||
|
assert.Equalf(t, c.expected, hl.MatchHostOrIP(c.host, c.ip), "case domain=%s, ip=%v, expected=%v", c.host, c.ip, c.expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
cases := []tc{
|
cases := []tc{
|
||||||
{"", net.IPv4zero, false},
|
{"", net.IPv4zero, false},
|
||||||
{"", net.IPv6zero, false},
|
{"", net.IPv6zero, false},
|
||||||
|
|
||||||
{"", net.ParseIP("127.0.0.1"), false},
|
{"", net.ParseIP("127.0.0.1"), false},
|
||||||
|
{"127.0.0.1", nil, false},
|
||||||
{"", net.ParseIP("::1"), false},
|
{"", net.ParseIP("::1"), false},
|
||||||
|
|
||||||
{"", net.ParseIP("10.0.1.1"), true},
|
{"", net.ParseIP("10.0.1.1"), true},
|
||||||
|
{"10.0.1.1", nil, true},
|
||||||
{"", net.ParseIP("192.168.1.1"), true},
|
{"", net.ParseIP("192.168.1.1"), true},
|
||||||
|
{"192.168.1.1", nil, true},
|
||||||
{"", net.ParseIP("fd00::1"), true},
|
{"", net.ParseIP("fd00::1"), true},
|
||||||
|
{"fd00::1", nil, true},
|
||||||
|
|
||||||
{"", net.ParseIP("8.8.8.8"), true},
|
{"", net.ParseIP("8.8.8.8"), true},
|
||||||
{"", net.ParseIP("1001::1"), true},
|
{"", net.ParseIP("1001::1"), true},
|
||||||
|
@ -39,13 +50,13 @@ func TestHostOrIPMatchesList(t *testing.T) {
|
||||||
{"sub.mydomain.com", net.IPv4zero, true},
|
{"sub.mydomain.com", net.IPv4zero, true},
|
||||||
|
|
||||||
{"", net.ParseIP("169.254.1.1"), true},
|
{"", net.ParseIP("169.254.1.1"), true},
|
||||||
|
{"169.254.1.1", nil, true},
|
||||||
{"", net.ParseIP("169.254.2.2"), false},
|
{"", net.ParseIP("169.254.2.2"), false},
|
||||||
|
{"169.254.2.2", nil, false},
|
||||||
}
|
}
|
||||||
for _, c := range cases {
|
test(cases)
|
||||||
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
|
|
||||||
}
|
|
||||||
|
|
||||||
hl = ParseHostMatchList("loopback")
|
hl = ParseHostMatchList("", "loopback")
|
||||||
cases = []tc{
|
cases = []tc{
|
||||||
{"", net.IPv4zero, false},
|
{"", net.IPv4zero, false},
|
||||||
{"", net.ParseIP("127.0.0.1"), true},
|
{"", net.ParseIP("127.0.0.1"), true},
|
||||||
|
@ -59,11 +70,9 @@ func TestHostOrIPMatchesList(t *testing.T) {
|
||||||
|
|
||||||
{"mydomain.com", net.IPv4zero, false},
|
{"mydomain.com", net.IPv4zero, false},
|
||||||
}
|
}
|
||||||
for _, c := range cases {
|
test(cases)
|
||||||
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
|
|
||||||
}
|
|
||||||
|
|
||||||
hl = ParseHostMatchList("private")
|
hl = ParseHostMatchList("", "private")
|
||||||
cases = []tc{
|
cases = []tc{
|
||||||
{"", net.IPv4zero, false},
|
{"", net.IPv4zero, false},
|
||||||
{"", net.ParseIP("127.0.0.1"), false},
|
{"", net.ParseIP("127.0.0.1"), false},
|
||||||
|
@ -77,11 +86,9 @@ func TestHostOrIPMatchesList(t *testing.T) {
|
||||||
|
|
||||||
{"mydomain.com", net.IPv4zero, false},
|
{"mydomain.com", net.IPv4zero, false},
|
||||||
}
|
}
|
||||||
for _, c := range cases {
|
test(cases)
|
||||||
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
|
|
||||||
}
|
|
||||||
|
|
||||||
hl = ParseHostMatchList("external")
|
hl = ParseHostMatchList("", "external")
|
||||||
cases = []tc{
|
cases = []tc{
|
||||||
{"", net.IPv4zero, false},
|
{"", net.IPv4zero, false},
|
||||||
{"", net.ParseIP("127.0.0.1"), false},
|
{"", net.ParseIP("127.0.0.1"), false},
|
||||||
|
@ -95,11 +102,9 @@ func TestHostOrIPMatchesList(t *testing.T) {
|
||||||
|
|
||||||
{"mydomain.com", net.IPv4zero, false},
|
{"mydomain.com", net.IPv4zero, false},
|
||||||
}
|
}
|
||||||
for _, c := range cases {
|
test(cases)
|
||||||
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
|
|
||||||
}
|
|
||||||
|
|
||||||
hl = ParseHostMatchList("*")
|
hl = ParseHostMatchList("", "*")
|
||||||
cases = []tc{
|
cases = []tc{
|
||||||
{"", net.IPv4zero, true},
|
{"", net.IPv4zero, true},
|
||||||
{"", net.ParseIP("127.0.0.1"), true},
|
{"", net.ParseIP("127.0.0.1"), true},
|
||||||
|
@ -113,7 +118,43 @@ func TestHostOrIPMatchesList(t *testing.T) {
|
||||||
|
|
||||||
{"mydomain.com", net.IPv4zero, true},
|
{"mydomain.com", net.IPv4zero, true},
|
||||||
}
|
}
|
||||||
for _, c := range cases {
|
test(cases)
|
||||||
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
|
|
||||||
|
// built-in network names can be escaped (warping the first char with `[]`) to be used as a real host name
|
||||||
|
// this mechanism is reversed for internal usage only (maybe for some rare cases), it's not supposed to be used by end users
|
||||||
|
// a real user should never use loopback/private/external as their host names
|
||||||
|
hl = ParseHostMatchList("", "loopback, [p]rivate")
|
||||||
|
cases = []tc{
|
||||||
|
{"loopback", nil, false},
|
||||||
|
{"", net.ParseIP("127.0.0.1"), true},
|
||||||
|
{"private", nil, true},
|
||||||
|
{"", net.ParseIP("192.168.1.1"), false},
|
||||||
}
|
}
|
||||||
|
test(cases)
|
||||||
|
|
||||||
|
hl = ParseSimpleMatchList("", "loopback, *.domain.com")
|
||||||
|
cases = []tc{
|
||||||
|
{"loopback", nil, true},
|
||||||
|
{"", net.ParseIP("127.0.0.1"), false},
|
||||||
|
{"sub.domain.com", nil, true},
|
||||||
|
{"other.com", nil, false},
|
||||||
|
{"", net.ParseIP("1.1.1.1"), false},
|
||||||
|
}
|
||||||
|
test(cases)
|
||||||
|
|
||||||
|
hl = ParseSimpleMatchList("", "external")
|
||||||
|
cases = []tc{
|
||||||
|
{"", net.ParseIP("192.168.1.1"), false},
|
||||||
|
{"", net.ParseIP("1.1.1.1"), false},
|
||||||
|
{"external", nil, true},
|
||||||
|
}
|
||||||
|
test(cases)
|
||||||
|
|
||||||
|
hl = ParseSimpleMatchList("", "")
|
||||||
|
cases = []tc{
|
||||||
|
{"", net.ParseIP("192.168.1.1"), false},
|
||||||
|
{"", net.ParseIP("1.1.1.1"), false},
|
||||||
|
{"external", nil, false},
|
||||||
|
}
|
||||||
|
test(cases)
|
||||||
}
|
}
|
||||||
|
|
58
modules/hostmatcher/http.go
Normal file
58
modules/hostmatcher/http.go
Normal file
|
@ -0,0 +1,58 @@
|
||||||
|
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package hostmatcher
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewDialContext returns a DialContext for Transport, the DialContext will do allow/block list check
|
||||||
|
func NewDialContext(usage string, allowList *HostMatchList, blockList *HostMatchList) func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||||
|
// How Go HTTP Client works with redirection:
|
||||||
|
// transport.RoundTrip URL=http://domain.com, Host=domain.com
|
||||||
|
// transport.DialContext addrOrHost=domain.com:80
|
||||||
|
// dialer.Control tcp4:11.22.33.44:80
|
||||||
|
// transport.RoundTrip URL=http://www.domain.com/, Host=(empty here, in the direction, HTTP client doesn't fill the Host field)
|
||||||
|
// transport.DialContext addrOrHost=domain.com:80
|
||||||
|
// dialer.Control tcp4:11.22.33.44:80
|
||||||
|
return func(ctx context.Context, network, addrOrHost string) (net.Conn, error) {
|
||||||
|
dialer := net.Dialer{
|
||||||
|
// default values comes from http.DefaultTransport
|
||||||
|
Timeout: 30 * time.Second,
|
||||||
|
KeepAlive: 30 * time.Second,
|
||||||
|
|
||||||
|
Control: func(network, ipAddr string, c syscall.RawConn) (err error) {
|
||||||
|
var host string
|
||||||
|
if host, _, err = net.SplitHostPort(addrOrHost); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here
|
||||||
|
tcpAddr, err := net.ResolveTCPAddr(network, ipAddr)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", usage, host, network, ipAddr, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var blockedError error
|
||||||
|
if blockList.MatchHostOrIP(host, tcpAddr.IP) {
|
||||||
|
blockedError = fmt.Errorf("%s can not call blocked HTTP servers (check your %s setting), deny '%s(%s)'", usage, blockList.SettingKeyHint, host, ipAddr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// if we have an allow-list, check the allow-list first
|
||||||
|
if !allowList.IsEmpty() {
|
||||||
|
if !allowList.MatchHostOrIP(host, tcpAddr.IP) {
|
||||||
|
return fmt.Errorf("%s can only call allowed HTTP servers (check your %s setting), deny '%s(%s)'", usage, allowList.SettingKeyHint, host, ipAddr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// otherwise, we always follow the blocked list
|
||||||
|
return blockedError
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return dialer.DialContext(ctx, network, addrOrHost)
|
||||||
|
}
|
||||||
|
}
|
|
@ -7,6 +7,7 @@ package lfs
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -24,9 +25,9 @@ type Client interface {
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewClient creates a LFS client
|
// NewClient creates a LFS client
|
||||||
func NewClient(endpoint *url.URL, skipTLSVerify bool) Client {
|
func NewClient(endpoint *url.URL, httpTransport *http.Transport) Client {
|
||||||
if endpoint.Scheme == "file" {
|
if endpoint.Scheme == "file" {
|
||||||
return newFilesystemClient(endpoint)
|
return newFilesystemClient(endpoint)
|
||||||
}
|
}
|
||||||
return newHTTPClient(endpoint, skipTLSVerify)
|
return newHTTPClient(endpoint, httpTransport)
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,10 +13,10 @@ import (
|
||||||
|
|
||||||
func TestNewClient(t *testing.T) {
|
func TestNewClient(t *testing.T) {
|
||||||
u, _ := url.Parse("file:///test")
|
u, _ := url.Parse("file:///test")
|
||||||
c := NewClient(u, true)
|
c := NewClient(u, nil)
|
||||||
assert.IsType(t, &FilesystemClient{}, c)
|
assert.IsType(t, &FilesystemClient{}, c)
|
||||||
|
|
||||||
u, _ = url.Parse("https://test.com/lfs")
|
u, _ = url.Parse("https://test.com/lfs")
|
||||||
c = NewClient(u, true)
|
c = NewClient(u, nil)
|
||||||
assert.IsType(t, &HTTPClient{}, c)
|
assert.IsType(t, &HTTPClient{}, c)
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,6 @@ package lfs
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/tls"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
@ -34,12 +33,15 @@ func (c *HTTPClient) BatchSize() int {
|
||||||
return batchSize
|
return batchSize
|
||||||
}
|
}
|
||||||
|
|
||||||
func newHTTPClient(endpoint *url.URL, skipTLSVerify bool) *HTTPClient {
|
func newHTTPClient(endpoint *url.URL, httpTransport *http.Transport) *HTTPClient {
|
||||||
|
if httpTransport == nil {
|
||||||
|
httpTransport = &http.Transport{
|
||||||
|
Proxy: proxy.Proxy(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
hc := &http.Client{
|
hc := &http.Client{
|
||||||
Transport: &http.Transport{
|
Transport: httpTransport,
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify},
|
|
||||||
Proxy: proxy.Proxy(),
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
client := &HTTPClient{
|
client := &HTTPClient{
|
||||||
|
|
|
@ -1,46 +0,0 @@
|
||||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
|
||||||
// Use of this source code is governed by a MIT-style
|
|
||||||
// license that can be found in the LICENSE file.
|
|
||||||
|
|
||||||
package matchlist
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/gobwas/glob"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Matchlist represents a block or allow list
|
|
||||||
type Matchlist struct {
|
|
||||||
ruleGlobs []glob.Glob
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewMatchlist creates a new block or allow list
|
|
||||||
func NewMatchlist(rules ...string) (*Matchlist, error) {
|
|
||||||
for i := range rules {
|
|
||||||
rules[i] = strings.ToLower(rules[i])
|
|
||||||
}
|
|
||||||
list := Matchlist{
|
|
||||||
ruleGlobs: make([]glob.Glob, 0, len(rules)),
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, rule := range rules {
|
|
||||||
rg, err := glob.Compile(rule)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
list.ruleGlobs = append(list.ruleGlobs, rg)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &list, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Match will matches
|
|
||||||
func (b *Matchlist) Match(u string) bool {
|
|
||||||
for _, r := range b.ruleGlobs {
|
|
||||||
if r.Match(u) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
|
@ -8,7 +8,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/url"
|
"net/http"
|
||||||
"path"
|
"path"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
@ -46,7 +46,10 @@ func WikiRemoteURL(remote string) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
|
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
|
||||||
func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.Repository, opts migration.MigrateOptions) (*models.Repository, error) {
|
func MigrateRepositoryGitData(ctx context.Context, u *models.User,
|
||||||
|
repo *models.Repository, opts migration.MigrateOptions,
|
||||||
|
httpTransport *http.Transport,
|
||||||
|
) (*models.Repository, error) {
|
||||||
repoPath := models.RepoPath(u.Name, opts.RepoName)
|
repoPath := models.RepoPath(u.Name, opts.RepoName)
|
||||||
|
|
||||||
if u.IsOrganization() {
|
if u.IsOrganization() {
|
||||||
|
@ -141,8 +144,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.
|
||||||
}
|
}
|
||||||
|
|
||||||
if opts.LFS {
|
if opts.LFS {
|
||||||
ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint)
|
endpoint := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint)
|
||||||
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep, setting.Migrations.SkipTLSVerify); err != nil {
|
lfsClient := lfs.NewClient(endpoint, httpTransport)
|
||||||
|
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, lfsClient); err != nil {
|
||||||
log.Error("Failed to store missing LFS objects for repository: %v", err)
|
log.Error("Failed to store missing LFS objects for repository: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -336,8 +340,7 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName
|
||||||
}
|
}
|
||||||
|
|
||||||
// StoreMissingLfsObjectsInRepository downloads missing LFS objects
|
// StoreMissingLfsObjectsInRepository downloads missing LFS objects
|
||||||
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error {
|
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, lfsClient lfs.Client) error {
|
||||||
client := lfs.NewClient(endpoint, skipTLSVerify)
|
|
||||||
contentStore := lfs.NewContentStore()
|
contentStore := lfs.NewContentStore()
|
||||||
|
|
||||||
pointerChan := make(chan lfs.PointerBlob)
|
pointerChan := make(chan lfs.PointerBlob)
|
||||||
|
@ -345,7 +348,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi
|
||||||
go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan)
|
go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan)
|
||||||
|
|
||||||
downloadObjects := func(pointers []lfs.Pointer) error {
|
downloadObjects := func(pointers []lfs.Pointer) error {
|
||||||
err := client.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error {
|
err := lfsClient.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error {
|
||||||
if objectError != nil {
|
if objectError != nil {
|
||||||
return objectError
|
return objectError
|
||||||
}
|
}
|
||||||
|
@ -411,7 +414,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi
|
||||||
}
|
}
|
||||||
|
|
||||||
batch = append(batch, pointerBlob.Pointer)
|
batch = append(batch, pointerBlob.Pointer)
|
||||||
if len(batch) >= client.BatchSize() {
|
if len(batch) >= lfsClient.BatchSize() {
|
||||||
if err := downloadObjects(batch); err != nil {
|
if err := downloadObjects(batch); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,17 +4,13 @@
|
||||||
|
|
||||||
package setting
|
package setting
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
// Migrations settings
|
// Migrations settings
|
||||||
Migrations = struct {
|
Migrations = struct {
|
||||||
MaxAttempts int
|
MaxAttempts int
|
||||||
RetryBackoff int
|
RetryBackoff int
|
||||||
AllowedDomains []string
|
AllowedDomains string
|
||||||
BlockedDomains []string
|
BlockedDomains string
|
||||||
AllowLocalNetworks bool
|
AllowLocalNetworks bool
|
||||||
SkipTLSVerify bool
|
SkipTLSVerify bool
|
||||||
}{
|
}{
|
||||||
|
@ -28,15 +24,8 @@ func newMigrationsService() {
|
||||||
Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts)
|
Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts)
|
||||||
Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff)
|
Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff)
|
||||||
|
|
||||||
Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").Strings(",")
|
Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").MustString("")
|
||||||
for i := range Migrations.AllowedDomains {
|
Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").MustString("")
|
||||||
Migrations.AllowedDomains[i] = strings.ToLower(Migrations.AllowedDomains[i])
|
|
||||||
}
|
|
||||||
Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").Strings(",")
|
|
||||||
for i := range Migrations.BlockedDomains {
|
|
||||||
Migrations.BlockedDomains[i] = strings.ToLower(Migrations.BlockedDomains[i])
|
|
||||||
}
|
|
||||||
|
|
||||||
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
|
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
|
||||||
Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false)
|
Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false)
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,6 @@ package setting
|
||||||
import (
|
import (
|
||||||
"net/url"
|
"net/url"
|
||||||
|
|
||||||
"code.gitea.io/gitea/modules/hostmatcher"
|
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -17,7 +16,7 @@ var (
|
||||||
QueueLength int
|
QueueLength int
|
||||||
DeliverTimeout int
|
DeliverTimeout int
|
||||||
SkipTLSVerify bool
|
SkipTLSVerify bool
|
||||||
AllowedHostList *hostmatcher.HostMatchList
|
AllowedHostList string
|
||||||
Types []string
|
Types []string
|
||||||
PagingNum int
|
PagingNum int
|
||||||
ProxyURL string
|
ProxyURL string
|
||||||
|
@ -38,7 +37,7 @@ func newWebhookService() {
|
||||||
Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000)
|
Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000)
|
||||||
Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5)
|
Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5)
|
||||||
Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool()
|
Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool()
|
||||||
Webhook.AllowedHostList = hostmatcher.ParseHostMatchList(sec.Key("ALLOWED_HOST_LIST").MustString(hostmatcher.MatchBuiltinExternal))
|
Webhook.AllowedHostList = sec.Key("ALLOWED_HOST_LIST").MustString("")
|
||||||
Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix", "wechatwork"}
|
Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix", "wechatwork"}
|
||||||
Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10)
|
Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10)
|
||||||
Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("")
|
Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("")
|
||||||
|
|
|
@ -899,8 +899,7 @@ migrate.clone_address_desc = The HTTP(S) or Git 'clone' URL of an existing repos
|
||||||
migrate.github_token_desc = You can put one or more tokens with comma separated here to make migrating faster because of Github API rate limit. WARN: Abusing this feature may violate the service provider's policy and lead to account blocking.
|
migrate.github_token_desc = You can put one or more tokens with comma separated here to make migrating faster because of Github API rate limit. WARN: Abusing this feature may violate the service provider's policy and lead to account blocking.
|
||||||
migrate.clone_local_path = or a local server path
|
migrate.clone_local_path = or a local server path
|
||||||
migrate.permission_denied = You are not allowed to import local repositories.
|
migrate.permission_denied = You are not allowed to import local repositories.
|
||||||
migrate.permission_denied_blocked = You are not allowed to import from blocked hosts.
|
migrate.permission_denied_blocked = You can not import from disallowed hosts, please ask the admin to check ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS settings.
|
||||||
migrate.permission_denied_private_ip = You are not allowed to import from private IPs.
|
|
||||||
migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory."
|
migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory."
|
||||||
migrate.invalid_lfs_endpoint = The LFS endpoint is not valid.
|
migrate.invalid_lfs_endpoint = The LFS endpoint is not valid.
|
||||||
migrate.failed = Migration failed: %v
|
migrate.failed = Migration failed: %v
|
||||||
|
|
|
@ -253,10 +253,8 @@ func handleRemoteAddrError(ctx *context.APIContext, err error) {
|
||||||
case addrErr.IsPermissionDenied:
|
case addrErr.IsPermissionDenied:
|
||||||
if addrErr.LocalPath {
|
if addrErr.LocalPath {
|
||||||
ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import local repositories.")
|
ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import local repositories.")
|
||||||
} else if len(addrErr.PrivateNet) == 0 {
|
|
||||||
ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from blocked hosts.")
|
|
||||||
} else {
|
} else {
|
||||||
ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from private IPs.")
|
ctx.Error(http.StatusUnprocessableEntity, "", "You can not import from disallowed hosts.")
|
||||||
}
|
}
|
||||||
case addrErr.IsInvalidPath:
|
case addrErr.IsInvalidPath:
|
||||||
ctx.Error(http.StatusUnprocessableEntity, "", "Invalid local path, it does not exist or not a directory.")
|
ctx.Error(http.StatusUnprocessableEntity, "", "Invalid local path, it does not exist or not a directory.")
|
||||||
|
|
|
@ -128,10 +128,8 @@ func handleMigrateRemoteAddrError(ctx *context.Context, err error, tpl base.TplN
|
||||||
case addrErr.IsPermissionDenied:
|
case addrErr.IsPermissionDenied:
|
||||||
if addrErr.LocalPath {
|
if addrErr.LocalPath {
|
||||||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tpl, form)
|
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tpl, form)
|
||||||
} else if len(addrErr.PrivateNet) == 0 {
|
|
||||||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tpl, form)
|
|
||||||
} else {
|
} else {
|
||||||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tpl, form)
|
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tpl, form)
|
||||||
}
|
}
|
||||||
case addrErr.IsInvalidPath:
|
case addrErr.IsInvalidPath:
|
||||||
ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tpl, form)
|
ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tpl, form)
|
||||||
|
|
|
@ -750,10 +750,8 @@ func handleSettingRemoteAddrError(ctx *context.Context, err error, form *forms.R
|
||||||
case addrErr.IsPermissionDenied:
|
case addrErr.IsPermissionDenied:
|
||||||
if addrErr.LocalPath {
|
if addrErr.LocalPath {
|
||||||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tplSettingsOptions, form)
|
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tplSettingsOptions, form)
|
||||||
} else if len(addrErr.PrivateNet) == 0 {
|
|
||||||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tplSettingsOptions, form)
|
|
||||||
} else {
|
} else {
|
||||||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tplSettingsOptions, form)
|
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tplSettingsOptions, form)
|
||||||
}
|
}
|
||||||
case addrErr.IsInvalidPath:
|
case addrErr.IsInvalidPath:
|
||||||
ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tplSettingsOptions, form)
|
ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tplSettingsOptions, form)
|
||||||
|
|
|
@ -6,7 +6,6 @@ package migrations
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/tls"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
@ -18,8 +17,6 @@ import (
|
||||||
admin_model "code.gitea.io/gitea/models/admin"
|
admin_model "code.gitea.io/gitea/models/admin"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
base "code.gitea.io/gitea/modules/migration"
|
base "code.gitea.io/gitea/modules/migration"
|
||||||
"code.gitea.io/gitea/modules/proxy"
|
|
||||||
"code.gitea.io/gitea/modules/setting"
|
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
|
|
||||||
gitea_sdk "code.gitea.io/sdk/gitea"
|
gitea_sdk "code.gitea.io/sdk/gitea"
|
||||||
|
@ -90,12 +87,7 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
|
||||||
gitea_sdk.SetToken(token),
|
gitea_sdk.SetToken(token),
|
||||||
gitea_sdk.SetBasicAuth(username, password),
|
gitea_sdk.SetBasicAuth(username, password),
|
||||||
gitea_sdk.SetContext(ctx),
|
gitea_sdk.SetContext(ctx),
|
||||||
gitea_sdk.SetHTTPClient(&http.Client{
|
gitea_sdk.SetHTTPClient(NewMigrationHTTPClient()),
|
||||||
Transport: &http.Transport{
|
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
|
||||||
Proxy: proxy.Proxy(),
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
|
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
|
||||||
|
@ -275,12 +267,7 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
|
||||||
Created: rel.CreatedAt,
|
Created: rel.CreatedAt,
|
||||||
}
|
}
|
||||||
|
|
||||||
httpClient := &http.Client{
|
httpClient := NewMigrationHTTPClient()
|
||||||
Transport: &http.Transport{
|
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
|
||||||
Proxy: proxy.Proxy(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, asset := range rel.Attachments {
|
for _, asset := range rel.Attachments {
|
||||||
size := int(asset.Size)
|
size := int(asset.Size)
|
||||||
|
|
|
@ -125,7 +125,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
|
||||||
Wiki: opts.Wiki,
|
Wiki: opts.Wiki,
|
||||||
Releases: opts.Releases, // if didn't get releases, then sync them from tags
|
Releases: opts.Releases, // if didn't get releases, then sync them from tags
|
||||||
MirrorInterval: opts.MirrorInterval,
|
MirrorInterval: opts.MirrorInterval,
|
||||||
})
|
}, NewMigrationHTTPTransport())
|
||||||
|
|
||||||
g.repo = r
|
g.repo = r
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -7,7 +7,6 @@ package migrations
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/tls"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
@ -19,7 +18,6 @@ import (
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
base "code.gitea.io/gitea/modules/migration"
|
base "code.gitea.io/gitea/modules/migration"
|
||||||
"code.gitea.io/gitea/modules/proxy"
|
"code.gitea.io/gitea/modules/proxy"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
|
|
||||||
|
@ -100,12 +98,7 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
|
||||||
)
|
)
|
||||||
var client = &http.Client{
|
var client = &http.Client{
|
||||||
Transport: &oauth2.Transport{
|
Transport: &oauth2.Transport{
|
||||||
Base: &http.Transport{
|
Base: NewMigrationHTTPTransport(),
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
|
||||||
Proxy: func(req *http.Request) (*url.URL, error) {
|
|
||||||
return proxy.Proxy()(req)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Source: oauth2.ReuseTokenSource(nil, ts),
|
Source: oauth2.ReuseTokenSource(nil, ts),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -113,14 +106,13 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
|
||||||
downloader.addClient(client, baseURL)
|
downloader.addClient(client, baseURL)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
var transport = NewMigrationHTTPTransport()
|
||||||
|
transport.Proxy = func(req *http.Request) (*url.URL, error) {
|
||||||
|
req.SetBasicAuth(userName, password)
|
||||||
|
return proxy.Proxy()(req)
|
||||||
|
}
|
||||||
var client = &http.Client{
|
var client = &http.Client{
|
||||||
Transport: &http.Transport{
|
Transport: transport,
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
|
||||||
Proxy: func(req *http.Request) (*url.URL, error) {
|
|
||||||
req.SetBasicAuth(userName, password)
|
|
||||||
return proxy.Proxy()(req)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
downloader.addClient(client, baseURL)
|
downloader.addClient(client, baseURL)
|
||||||
}
|
}
|
||||||
|
@ -316,12 +308,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
|
||||||
r.Published = rel.PublishedAt.Time
|
r.Published = rel.PublishedAt.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
httpClient := &http.Client{
|
httpClient := NewMigrationHTTPClient()
|
||||||
Transport: &http.Transport{
|
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
|
||||||
Proxy: proxy.Proxy(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, asset := range rel.Assets {
|
for _, asset := range rel.Assets {
|
||||||
var assetID = *asset.ID // Don't optimize this, for closure we need a local variable
|
var assetID = *asset.ID // Don't optimize this, for closure we need a local variable
|
||||||
|
|
|
@ -6,7 +6,6 @@ package migrations
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/tls"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
@ -18,8 +17,6 @@ import (
|
||||||
|
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
base "code.gitea.io/gitea/modules/migration"
|
base "code.gitea.io/gitea/modules/migration"
|
||||||
"code.gitea.io/gitea/modules/proxy"
|
|
||||||
"code.gitea.io/gitea/modules/setting"
|
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
|
|
||||||
"github.com/xanzy/go-gitlab"
|
"github.com/xanzy/go-gitlab"
|
||||||
|
@ -77,16 +74,11 @@ type GitlabDownloader struct {
|
||||||
// Use either a username/password, personal token entered into the username field, or anonymous/public access
|
// Use either a username/password, personal token entered into the username field, or anonymous/public access
|
||||||
// Note: Public access only allows very basic access
|
// Note: Public access only allows very basic access
|
||||||
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
|
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
|
||||||
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{
|
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(NewMigrationHTTPClient()))
|
||||||
Transport: &http.Transport{
|
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
|
||||||
Proxy: proxy.Proxy(),
|
|
||||||
},
|
|
||||||
}))
|
|
||||||
// Only use basic auth if token is blank and password is NOT
|
// Only use basic auth if token is blank and password is NOT
|
||||||
// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
|
// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
|
||||||
if token == "" && password != "" {
|
if token == "" && password != "" {
|
||||||
gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL))
|
gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(NewMigrationHTTPClient()))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -300,12 +292,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
|
||||||
PublisherName: rel.Author.Username,
|
PublisherName: rel.Author.Username,
|
||||||
}
|
}
|
||||||
|
|
||||||
httpClient := &http.Client{
|
httpClient := NewMigrationHTTPClient()
|
||||||
Transport: &http.Transport{
|
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
|
||||||
Proxy: proxy.Proxy(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for k, asset := range rel.Assets.Links {
|
for k, asset := range rel.Assets.Links {
|
||||||
r.Assets = append(r.Assets, &base.ReleaseAsset{
|
r.Assets = append(r.Assets, &base.ReleaseAsset{
|
||||||
|
|
|
@ -6,7 +6,6 @@ package migrations
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/tls"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
@ -16,7 +15,6 @@ import (
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
base "code.gitea.io/gitea/modules/migration"
|
base "code.gitea.io/gitea/modules/migration"
|
||||||
"code.gitea.io/gitea/modules/proxy"
|
"code.gitea.io/gitea/modules/proxy"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
|
|
||||||
"github.com/gogs/go-gogs-client"
|
"github.com/gogs/go-gogs-client"
|
||||||
|
@ -97,13 +95,12 @@ func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token,
|
||||||
client = gogs.NewClient(baseURL, token)
|
client = gogs.NewClient(baseURL, token)
|
||||||
downloader.userName = token
|
downloader.userName = token
|
||||||
} else {
|
} else {
|
||||||
downloader.transport = &http.Transport{
|
var transport = NewMigrationHTTPTransport()
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
transport.Proxy = func(req *http.Request) (*url.URL, error) {
|
||||||
Proxy: func(req *http.Request) (*url.URL, error) {
|
req.SetBasicAuth(userName, password)
|
||||||
req.SetBasicAuth(userName, password)
|
return proxy.Proxy()(req)
|
||||||
return proxy.Proxy()(req)
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
downloader.transport = transport
|
||||||
|
|
||||||
client = gogs.NewClient(baseURL, "")
|
client = gogs.NewClient(baseURL, "")
|
||||||
client.SetHTTPClient(&http.Client{
|
client.SetHTTPClient(&http.Client{
|
||||||
|
|
30
services/migrations/http_client.go
Normal file
30
services/migrations/http_client.go
Normal file
|
@ -0,0 +1,30 @@
|
||||||
|
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package migrations
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/tls"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/hostmatcher"
|
||||||
|
"code.gitea.io/gitea/modules/proxy"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewMigrationHTTPClient returns a HTTP client for migration
|
||||||
|
func NewMigrationHTTPClient() *http.Client {
|
||||||
|
return &http.Client{
|
||||||
|
Transport: NewMigrationHTTPTransport(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewMigrationHTTPTransport returns a HTTP transport for migration
|
||||||
|
func NewMigrationHTTPTransport() *http.Transport {
|
||||||
|
return &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
||||||
|
Proxy: proxy.Proxy(),
|
||||||
|
DialContext: hostmatcher.NewDialContext("migration", allowList, blockList),
|
||||||
|
}
|
||||||
|
}
|
|
@ -15,8 +15,8 @@ import (
|
||||||
|
|
||||||
"code.gitea.io/gitea/models"
|
"code.gitea.io/gitea/models"
|
||||||
admin_model "code.gitea.io/gitea/models/admin"
|
admin_model "code.gitea.io/gitea/models/admin"
|
||||||
|
"code.gitea.io/gitea/modules/hostmatcher"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/matchlist"
|
|
||||||
base "code.gitea.io/gitea/modules/migration"
|
base "code.gitea.io/gitea/modules/migration"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
|
@ -28,8 +28,8 @@ type MigrateOptions = base.MigrateOptions
|
||||||
var (
|
var (
|
||||||
factories []base.DownloaderFactory
|
factories []base.DownloaderFactory
|
||||||
|
|
||||||
allowList *matchlist.Matchlist
|
allowList *hostmatcher.HostMatchList
|
||||||
blockList *matchlist.Matchlist
|
blockList *hostmatcher.HostMatchList
|
||||||
)
|
)
|
||||||
|
|
||||||
// RegisterDownloaderFactory registers a downloader factory
|
// RegisterDownloaderFactory registers a downloader factory
|
||||||
|
@ -73,30 +73,35 @@ func IsMigrateURLAllowed(remoteURL string, doer *models.User) error {
|
||||||
return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true}
|
return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true}
|
||||||
}
|
}
|
||||||
|
|
||||||
host := strings.ToLower(u.Host)
|
hostName, _, err := net.SplitHostPort(u.Host)
|
||||||
if len(setting.Migrations.AllowedDomains) > 0 {
|
if err != nil {
|
||||||
if !allowList.Match(host) {
|
// u.Host can be "host" or "host:port"
|
||||||
return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
|
err = nil //nolint
|
||||||
}
|
hostName = u.Host
|
||||||
} else {
|
}
|
||||||
if blockList.Match(host) {
|
addrList, err := net.LookupIP(hostName)
|
||||||
|
if err != nil {
|
||||||
|
return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true}
|
||||||
|
}
|
||||||
|
|
||||||
|
var ipAllowed bool
|
||||||
|
var ipBlocked bool
|
||||||
|
for _, addr := range addrList {
|
||||||
|
ipAllowed = ipAllowed || allowList.MatchIPAddr(addr)
|
||||||
|
ipBlocked = ipBlocked || blockList.MatchIPAddr(addr)
|
||||||
|
}
|
||||||
|
var blockedError error
|
||||||
|
if blockList.MatchHostName(hostName) || ipBlocked {
|
||||||
|
blockedError = &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
|
||||||
|
}
|
||||||
|
// if we have an allow-list, check the allow-list first
|
||||||
|
if !allowList.IsEmpty() {
|
||||||
|
if !allowList.MatchHostName(hostName) && !ipAllowed {
|
||||||
return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
|
return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// otherwise, we always follow the blocked list
|
||||||
if !setting.Migrations.AllowLocalNetworks {
|
return blockedError
|
||||||
addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0])
|
|
||||||
if err != nil {
|
|
||||||
return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true}
|
|
||||||
}
|
|
||||||
for _, addr := range addrList {
|
|
||||||
if util.IsIPPrivate(addr) || !addr.IsGlobalUnicast() {
|
|
||||||
return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// MigrateRepository migrate repository according MigrateOptions
|
// MigrateRepository migrate repository according MigrateOptions
|
||||||
|
@ -462,16 +467,18 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
|
||||||
|
|
||||||
// Init migrations service
|
// Init migrations service
|
||||||
func Init() error {
|
func Init() error {
|
||||||
var err error
|
// TODO: maybe we can deprecate these legacy ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS, use ALLOWED_HOST_LIST/BLOCKED_HOST_LIST instead
|
||||||
allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("init migration allowList domains failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...)
|
blockList = hostmatcher.ParseSimpleMatchList("migrations.BLOCKED_DOMAINS", setting.Migrations.BlockedDomains)
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("init migration blockList domains failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
allowList = hostmatcher.ParseSimpleMatchList("migrations.ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS", setting.Migrations.AllowedDomains)
|
||||||
|
if allowList.IsEmpty() {
|
||||||
|
// the default policy is that migration module can access external hosts
|
||||||
|
allowList.AppendBuiltin(hostmatcher.MatchBuiltinExternal)
|
||||||
|
}
|
||||||
|
if setting.Migrations.AllowLocalNetworks {
|
||||||
|
allowList.AppendBuiltin(hostmatcher.MatchBuiltinPrivate)
|
||||||
|
allowList.AppendBuiltin(hostmatcher.MatchBuiltinLoopback)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,7 +21,8 @@ func TestMigrateWhiteBlocklist(t *testing.T) {
|
||||||
adminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user1"}).(*models.User)
|
adminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user1"}).(*models.User)
|
||||||
nonAdminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user2"}).(*models.User)
|
nonAdminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user2"}).(*models.User)
|
||||||
|
|
||||||
setting.Migrations.AllowedDomains = []string{"github.com"}
|
setting.Migrations.AllowedDomains = "github.com"
|
||||||
|
setting.Migrations.AllowLocalNetworks = false
|
||||||
assert.NoError(t, Init())
|
assert.NoError(t, Init())
|
||||||
|
|
||||||
err := IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
|
err := IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
|
||||||
|
@ -33,8 +34,8 @@ func TestMigrateWhiteBlocklist(t *testing.T) {
|
||||||
err = IsMigrateURLAllowed("https://gITHUb.com/go-gitea/gitea.git", nonAdminUser)
|
err = IsMigrateURLAllowed("https://gITHUb.com/go-gitea/gitea.git", nonAdminUser)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
setting.Migrations.AllowedDomains = []string{}
|
setting.Migrations.AllowedDomains = ""
|
||||||
setting.Migrations.BlockedDomains = []string{"github.com"}
|
setting.Migrations.BlockedDomains = "github.com"
|
||||||
assert.NoError(t, Init())
|
assert.NoError(t, Init())
|
||||||
|
|
||||||
err = IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
|
err = IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
|
||||||
|
@ -47,6 +48,7 @@ func TestMigrateWhiteBlocklist(t *testing.T) {
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
|
|
||||||
setting.Migrations.AllowLocalNetworks = true
|
setting.Migrations.AllowLocalNetworks = true
|
||||||
|
assert.NoError(t, Init())
|
||||||
err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser)
|
err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
|
|
@ -261,8 +261,9 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool)
|
||||||
|
|
||||||
if m.LFS && setting.LFS.StartServer {
|
if m.LFS && setting.LFS.StartServer {
|
||||||
log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
|
log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
|
||||||
ep := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint)
|
endpoint := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint)
|
||||||
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep, false); err != nil {
|
lfsClient := lfs.NewClient(endpoint, nil)
|
||||||
|
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsClient); err != nil {
|
||||||
log.Error("Failed to synchronize LFS objects for repository: %v", err)
|
log.Error("Failed to synchronize LFS objects for repository: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,7 +8,6 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net/url"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
@ -133,8 +132,9 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error {
|
||||||
}
|
}
|
||||||
defer gitRepo.Close()
|
defer gitRepo.Close()
|
||||||
|
|
||||||
ep := lfs.DetermineEndpoint(remoteAddr.String(), "")
|
endpoint := lfs.DetermineEndpoint(remoteAddr.String(), "")
|
||||||
if err := pushAllLFSObjects(ctx, gitRepo, ep, false); err != nil {
|
lfsClient := lfs.NewClient(endpoint, nil)
|
||||||
|
if err := pushAllLFSObjects(ctx, gitRepo, lfsClient); err != nil {
|
||||||
return util.NewURLSanitizedError(err, remoteAddr, true)
|
return util.NewURLSanitizedError(err, remoteAddr, true)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -176,8 +176,7 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error {
|
func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, lfsClient lfs.Client) error {
|
||||||
client := lfs.NewClient(endpoint, skipTLSVerify)
|
|
||||||
contentStore := lfs.NewContentStore()
|
contentStore := lfs.NewContentStore()
|
||||||
|
|
||||||
pointerChan := make(chan lfs.PointerBlob)
|
pointerChan := make(chan lfs.PointerBlob)
|
||||||
|
@ -185,7 +184,7 @@ func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *u
|
||||||
go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan)
|
go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan)
|
||||||
|
|
||||||
uploadObjects := func(pointers []lfs.Pointer) error {
|
uploadObjects := func(pointers []lfs.Pointer) error {
|
||||||
err := client.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) {
|
err := lfsClient.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) {
|
||||||
if objectError != nil {
|
if objectError != nil {
|
||||||
return nil, objectError
|
return nil, objectError
|
||||||
}
|
}
|
||||||
|
@ -219,7 +218,7 @@ func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *u
|
||||||
}
|
}
|
||||||
|
|
||||||
batch = append(batch, pointerBlob.Pointer)
|
batch = append(batch, pointerBlob.Pointer)
|
||||||
if len(batch) >= client.BatchSize() {
|
if len(batch) >= lfsClient.BatchSize() {
|
||||||
if err := uploadObjects(batch); err != nil {
|
if err := uploadObjects(batch); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,17 +13,16 @@ import (
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"syscall"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
webhook_model "code.gitea.io/gitea/models/webhook"
|
webhook_model "code.gitea.io/gitea/models/webhook"
|
||||||
"code.gitea.io/gitea/modules/graceful"
|
"code.gitea.io/gitea/modules/graceful"
|
||||||
|
"code.gitea.io/gitea/modules/hostmatcher"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/proxy"
|
"code.gitea.io/gitea/modules/proxy"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
@ -31,8 +30,6 @@ import (
|
||||||
"github.com/gobwas/glob"
|
"github.com/gobwas/glob"
|
||||||
)
|
)
|
||||||
|
|
||||||
var contextKeyWebhookRequest interface{} = "contextKeyWebhookRequest"
|
|
||||||
|
|
||||||
// Deliver deliver hook task
|
// Deliver deliver hook task
|
||||||
func Deliver(t *webhook_model.HookTask) error {
|
func Deliver(t *webhook_model.HookTask) error {
|
||||||
w, err := webhook_model.GetWebhookByID(t.HookID)
|
w, err := webhook_model.GetWebhookByID(t.HookID)
|
||||||
|
@ -98,10 +95,10 @@ func Deliver(t *webhook_model.HookTask) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("Invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod)
|
return fmt.Errorf("invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod)
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("Invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod)
|
return fmt.Errorf("invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod)
|
||||||
}
|
}
|
||||||
|
|
||||||
var signatureSHA1 string
|
var signatureSHA1 string
|
||||||
|
@ -172,10 +169,10 @@ func Deliver(t *webhook_model.HookTask) error {
|
||||||
}()
|
}()
|
||||||
|
|
||||||
if setting.DisableWebhooks {
|
if setting.DisableWebhooks {
|
||||||
return fmt.Errorf("Webhook task skipped (webhooks disabled): [%d]", t.ID)
|
return fmt.Errorf("webhook task skipped (webhooks disabled): [%d]", t.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
resp, err := webhookHTTPClient.Do(req.WithContext(context.WithValue(req.Context(), contextKeyWebhookRequest, req)))
|
resp, err := webhookHTTPClient.Do(req.WithContext(graceful.GetManager().ShutdownContext()))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err)
|
t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err)
|
||||||
return err
|
return err
|
||||||
|
@ -296,29 +293,18 @@ func webhookProxy() func(req *http.Request) (*url.URL, error) {
|
||||||
func InitDeliverHooks() {
|
func InitDeliverHooks() {
|
||||||
timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second
|
timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second
|
||||||
|
|
||||||
|
allowedHostListValue := setting.Webhook.AllowedHostList
|
||||||
|
if allowedHostListValue == "" {
|
||||||
|
allowedHostListValue = hostmatcher.MatchBuiltinExternal
|
||||||
|
}
|
||||||
|
allowedHostMatcher := hostmatcher.ParseHostMatchList("webhook.ALLOWED_HOST_LIST", allowedHostListValue)
|
||||||
|
|
||||||
webhookHTTPClient = &http.Client{
|
webhookHTTPClient = &http.Client{
|
||||||
Timeout: timeout,
|
Timeout: timeout,
|
||||||
Transport: &http.Transport{
|
Transport: &http.Transport{
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify},
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify},
|
||||||
Proxy: webhookProxy(),
|
Proxy: webhookProxy(),
|
||||||
DialContext: func(ctx context.Context, network, addrOrHost string) (net.Conn, error) {
|
DialContext: hostmatcher.NewDialContext("webhook", allowedHostMatcher, nil),
|
||||||
dialer := net.Dialer{
|
|
||||||
Timeout: timeout,
|
|
||||||
Control: func(network, ipAddr string, c syscall.RawConn) error {
|
|
||||||
// in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here
|
|
||||||
tcpAddr, err := net.ResolveTCPAddr(network, ipAddr)
|
|
||||||
req := ctx.Value(contextKeyWebhookRequest).(*http.Request)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("webhook can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", req.Host, network, ipAddr, err)
|
|
||||||
}
|
|
||||||
if !setting.Webhook.AllowedHostList.MatchesHostOrIP(req.Host, tcpAddr.IP) {
|
|
||||||
return fmt.Errorf("webhook can only call allowed HTTP servers (check your webhook.ALLOWED_HOST_LIST setting), deny '%s(%s)'", req.Host, ipAddr)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return dialer.DialContext(ctx, network, addrOrHost)
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Reference in a new issue