diff --git a/go.sum b/go.sum
index 87446bd56..477199616 100644
--- a/go.sum
+++ b/go.sum
@@ -302,6 +302,7 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4=
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -447,9 +448,11 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn
github.com/oliamb/cutter v0.2.2 h1:Lfwkya0HHNU1YLnGv2hTkzHfasrSMkgv4Dn+5rmlk3k=
github.com/oliamb/cutter v0.2.2/go.mod h1:4BenG2/4GuRBDbVm/OPahDVqbrOemzpPiG5mi1iryBU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -527,6 +530,7 @@ github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY=
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
@@ -560,6 +564,7 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/syndtr/goleveldb v0.0.0-20190203031304-2f17a3356c66 h1:AwmkkZT+TucFotNCL+aNJ/0KCMsRtlXN9fs8uoOMSRk=
github.com/syndtr/goleveldb v0.0.0-20190203031304-2f17a3356c66/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
@@ -745,6 +750,7 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.0 h1:Tfd7cKwKbFRsI8RMAD3oqqw7JPFRrvFlOsfbgVkjOOw=
google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
diff --git a/models/repo.go b/models/repo.go
index ffd1c3060..eb7f286fe 100644
--- a/models/repo.go
+++ b/models/repo.go
@@ -1720,6 +1720,12 @@ func UpdateRepository(repo *Repository, visibilityChanged bool) (err error) {
return sess.Commit()
}
+// UpdateRepositoryUpdatedTime updates a repository's updated time
+func UpdateRepositoryUpdatedTime(repoID int64, updateTime time.Time) error {
+ _, err := x.Exec("UPDATE repository SET updated_unix = ? WHERE id = ?", updateTime.Unix(), repoID)
+ return err
+}
+
// UpdateRepositoryUnits updates a repository's units
func UpdateRepositoryUnits(repo *Repository, units []RepoUnit) (err error) {
sess := x.NewSession()
diff --git a/models/repo_mirror.go b/models/repo_mirror.go
index e1c1aefc3..4e91ea296 100644
--- a/models/repo_mirror.go
+++ b/models/repo_mirror.go
@@ -6,28 +6,14 @@
package models
import (
- "fmt"
- "net/url"
- "strings"
"time"
- "code.gitea.io/gitea/modules/cache"
- "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/process"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/sync"
"code.gitea.io/gitea/modules/timeutil"
- "code.gitea.io/gitea/modules/util"
"github.com/go-xorm/xorm"
- "github.com/mcuadros/go-version"
- "github.com/unknwon/com"
)
-// MirrorQueue holds an UniqueQueue object of the mirror
-var MirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength)
-
// Mirror represents mirror information of a repository.
type Mirror struct {
ID int64 `xorm:"pk autoincr"`
@@ -39,7 +25,7 @@ type Mirror struct {
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"`
NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"`
- address string `xorm:"-"`
+ Address string `xorm:"-"`
}
// BeforeInsert will be invoked by XORM before inserting a record
@@ -72,258 +58,6 @@ func (m *Mirror) ScheduleNextUpdate() {
}
}
-func remoteAddress(repoPath string) (string, error) {
- var cmd *git.Command
- binVersion, err := git.BinVersion()
- if err != nil {
- return "", err
- }
- if version.Compare(binVersion, "2.7", ">=") {
- cmd = git.NewCommand("remote", "get-url", "origin")
- } else {
- cmd = git.NewCommand("config", "--get", "remote.origin.url")
- }
-
- result, err := cmd.RunInDir(repoPath)
- if err != nil {
- if strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") {
- return "", nil
- }
- return "", err
- }
- if len(result) > 0 {
- return result[:len(result)-1], nil
- }
- return "", nil
-}
-
-func (m *Mirror) readAddress() {
- if len(m.address) > 0 {
- return
- }
- var err error
- m.address, err = remoteAddress(m.Repo.RepoPath())
- if err != nil {
- log.Error("remoteAddress: %v", err)
- }
-}
-
-// sanitizeOutput sanitizes output of a command, replacing occurrences of the
-// repository's remote address with a sanitized version.
-func sanitizeOutput(output, repoPath string) (string, error) {
- remoteAddr, err := remoteAddress(repoPath)
- if err != nil {
- // if we're unable to load the remote address, then we're unable to
- // sanitize.
- return "", err
- }
- return util.SanitizeMessage(output, remoteAddr), nil
-}
-
-// Address returns mirror address from Git repository config with credentials censored.
-func (m *Mirror) Address() string {
- m.readAddress()
- return util.SanitizeURLCredentials(m.address, false)
-}
-
-// FullAddress returns mirror address from Git repository config.
-func (m *Mirror) FullAddress() string {
- m.readAddress()
- return m.address
-}
-
-// AddressNoCredentials returns mirror address from Git repository config without credentials.
-func (m *Mirror) AddressNoCredentials() string {
- m.readAddress()
- u, err := url.Parse(m.address)
- if err != nil {
- // this shouldn't happen but just return it unsanitised
- return m.address
- }
- u.User = nil
- return u.String()
-}
-
-// Username returns the mirror address username
-func (m *Mirror) Username() string {
- m.readAddress()
- u, err := url.Parse(m.address)
- if err != nil {
- // this shouldn't happen but if it does return ""
- return ""
- }
- return u.User.Username()
-}
-
-// Password returns the mirror address password
-func (m *Mirror) Password() string {
- m.readAddress()
- u, err := url.Parse(m.address)
- if err != nil {
- // this shouldn't happen but if it does return ""
- return ""
- }
- password, _ := u.User.Password()
- return password
-}
-
-// SaveAddress writes new address to Git repository config.
-func (m *Mirror) SaveAddress(addr string) error {
- repoPath := m.Repo.RepoPath()
- // Remove old origin
- _, err := git.NewCommand("remote", "remove", "origin").RunInDir(repoPath)
- if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") {
- return err
- }
-
- _, err = git.NewCommand("remote", "add", "origin", "--mirror=fetch", addr).RunInDir(repoPath)
- return err
-}
-
-// gitShortEmptySha Git short empty SHA
-const gitShortEmptySha = "0000000"
-
-// mirrorSyncResult contains information of a updated reference.
-// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
-// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
-type mirrorSyncResult struct {
- refName string
- oldCommitID string
- newCommitID string
-}
-
-// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
-func parseRemoteUpdateOutput(output string) []*mirrorSyncResult {
- results := make([]*mirrorSyncResult, 0, 3)
- lines := strings.Split(output, "\n")
- for i := range lines {
- // Make sure reference name is presented before continue
- idx := strings.Index(lines[i], "-> ")
- if idx == -1 {
- continue
- }
-
- refName := lines[i][idx+3:]
-
- switch {
- case strings.HasPrefix(lines[i], " * "): // New reference
- results = append(results, &mirrorSyncResult{
- refName: refName,
- oldCommitID: gitShortEmptySha,
- })
- case strings.HasPrefix(lines[i], " - "): // Delete reference
- results = append(results, &mirrorSyncResult{
- refName: refName,
- newCommitID: gitShortEmptySha,
- })
- case strings.HasPrefix(lines[i], " "): // New commits of a reference
- delimIdx := strings.Index(lines[i][3:], " ")
- if delimIdx == -1 {
- log.Error("SHA delimiter not found: %q", lines[i])
- continue
- }
- shas := strings.Split(lines[i][3:delimIdx+3], "..")
- if len(shas) != 2 {
- log.Error("Expect two SHAs but not what found: %q", lines[i])
- continue
- }
- results = append(results, &mirrorSyncResult{
- refName: refName,
- oldCommitID: shas[0],
- newCommitID: shas[1],
- })
-
- default:
- log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i])
- }
- }
- return results
-}
-
-// runSync returns true if sync finished without error.
-func (m *Mirror) runSync() ([]*mirrorSyncResult, bool) {
- repoPath := m.Repo.RepoPath()
- wikiPath := m.Repo.WikiPath()
- timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
-
- gitArgs := []string{"remote", "update"}
- if m.EnablePrune {
- gitArgs = append(gitArgs, "--prune")
- }
-
- _, stderr, err := process.GetManager().ExecDir(
- timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
- git.GitExecutable, gitArgs...)
- if err != nil {
- // sanitize the output, since it may contain the remote address, which may
- // contain a password
- message, err := sanitizeOutput(stderr, repoPath)
- if err != nil {
- log.Error("sanitizeOutput: %v", err)
- return nil, false
- }
- desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, message)
- log.Error(desc)
- if err = CreateRepositoryNotice(desc); err != nil {
- log.Error("CreateRepositoryNotice: %v", err)
- }
- return nil, false
- }
- output := stderr
-
- gitRepo, err := git.OpenRepository(repoPath)
- if err != nil {
- log.Error("OpenRepository: %v", err)
- return nil, false
- }
- if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
- log.Error("Failed to synchronize tags to releases for repository: %v", err)
- }
-
- if err := m.Repo.UpdateSize(); err != nil {
- log.Error("Failed to update size for mirror repository: %v", err)
- }
-
- if m.Repo.HasWiki() {
- if _, stderr, err := process.GetManager().ExecDir(
- timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath),
- git.GitExecutable, "remote", "update", "--prune"); err != nil {
- // sanitize the output, since it may contain the remote address, which may
- // contain a password
- message, err := sanitizeOutput(stderr, wikiPath)
- if err != nil {
- log.Error("sanitizeOutput: %v", err)
- return nil, false
- }
- desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, message)
- log.Error(desc)
- if err = CreateRepositoryNotice(desc); err != nil {
- log.Error("CreateRepositoryNotice: %v", err)
- }
- return nil, false
- }
- }
-
- branches, err := m.Repo.GetBranches()
- if err != nil {
- log.Error("GetBranches: %v", err)
- return nil, false
- }
-
- for i := range branches {
- cache.Remove(m.Repo.GetCommitsCountCacheKey(branches[i].Name, true))
- }
-
- m.UpdatedUnix = timeutil.TimeStampNow()
- return parseRemoteUpdateOutput(output), true
-}
-
-// RunMirrorSync will invoke Mirror's runSync
-func RunMirrorSync(mirror *Mirror) bool {
- _, ok := mirror.runSync()
- return ok
-}
-
func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) {
m := &Mirror{RepoID: repoID}
has, err := e.Get(m)
@@ -356,129 +90,10 @@ func DeleteMirrorByRepoID(repoID int64) error {
return err
}
-// MirrorUpdate checks and updates mirror repositories.
-func MirrorUpdate() {
- log.Trace("Doing: MirrorUpdate")
-
- if err := x.
+// MirrorsIterate iterates all mirror repositories.
+func MirrorsIterate(f func(idx int, bean interface{}) error) error {
+ return x.
Where("next_update_unix<=?", time.Now().Unix()).
And("next_update_unix!=0").
- Iterate(new(Mirror), func(idx int, bean interface{}) error {
- m := bean.(*Mirror)
- if m.Repo == nil {
- log.Error("Disconnected mirror repository found: %d", m.ID)
- return nil
- }
-
- MirrorQueue.Add(m.RepoID)
- return nil
- }); err != nil {
- log.Error("MirrorUpdate: %v", err)
- }
-}
-
-// SyncMirrors checks and syncs mirrors.
-// TODO: sync more mirrors at same time.
-func SyncMirrors() {
- sess := x.NewSession()
- defer sess.Close()
- // Start listening on new sync requests.
- for repoID := range MirrorQueue.Queue() {
- log.Trace("SyncMirrors [repo_id: %v]", repoID)
- MirrorQueue.Remove(repoID)
-
- m, err := GetMirrorByRepoID(com.StrTo(repoID).MustInt64())
- if err != nil {
- log.Error("GetMirrorByRepoID [%s]: %v", repoID, err)
- continue
- }
-
- results, ok := m.runSync()
- if !ok {
- continue
- }
-
- m.ScheduleNextUpdate()
- if err = updateMirror(sess, m); err != nil {
- log.Error("UpdateMirror [%s]: %v", repoID, err)
- continue
- }
-
- var gitRepo *git.Repository
- if len(results) == 0 {
- log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID)
- } else {
- gitRepo, err = git.OpenRepository(m.Repo.RepoPath())
- if err != nil {
- log.Error("OpenRepository [%d]: %v", m.RepoID, err)
- continue
- }
- }
-
- for _, result := range results {
- // Discard GitHub pull requests, i.e. refs/pull/*
- if strings.HasPrefix(result.refName, "refs/pull/") {
- continue
- }
-
- // Create reference
- if result.oldCommitID == gitShortEmptySha {
- if err = MirrorSyncCreateAction(m.Repo, result.refName); err != nil {
- log.Error("MirrorSyncCreateAction [repo_id: %d]: %v", m.RepoID, err)
- }
- continue
- }
-
- // Delete reference
- if result.newCommitID == gitShortEmptySha {
- if err = MirrorSyncDeleteAction(m.Repo, result.refName); err != nil {
- log.Error("MirrorSyncDeleteAction [repo_id: %d]: %v", m.RepoID, err)
- }
- continue
- }
-
- // Push commits
- oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID)
- if err != nil {
- log.Error("GetFullCommitID [%d]: %v", m.RepoID, err)
- continue
- }
- newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID)
- if err != nil {
- log.Error("GetFullCommitID [%d]: %v", m.RepoID, err)
- continue
- }
- commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID)
- if err != nil {
- log.Error("CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err)
- continue
- }
- if err = MirrorSyncPushAction(m.Repo, MirrorSyncPushActionOptions{
- RefName: result.refName,
- OldCommitID: oldCommitID,
- NewCommitID: newCommitID,
- Commits: ListToPushCommits(commits),
- }); err != nil {
- log.Error("MirrorSyncPushAction [repo_id: %d]: %v", m.RepoID, err)
- continue
- }
- }
-
- // Get latest commit date and update to current repository updated time
- commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath())
- if err != nil {
- log.Error("GetLatestCommitDate [%d]: %v", m.RepoID, err)
- continue
- }
-
- if _, err = sess.Exec("UPDATE repository SET updated_unix = ? WHERE id = ?", commitDate.Unix(), m.RepoID); err != nil {
- log.Error("Update repository 'updated_unix' [%d]: %v", m.RepoID, err)
- continue
- }
- }
-}
-
-// InitSyncMirrors initializes a go routine to sync the mirrors
-func InitSyncMirrors() {
- go SyncMirrors()
+ Iterate(new(Mirror), f)
}
diff --git a/modules/cron/cron.go b/modules/cron/cron.go
index fe87f86e0..089f0fa76 100644
--- a/modules/cron/cron.go
+++ b/modules/cron/cron.go
@@ -12,6 +12,7 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/sync"
+ mirror_service "code.gitea.io/gitea/services/mirror"
"github.com/gogs/cron"
)
@@ -51,14 +52,14 @@ func NewContext() {
err error
)
if setting.Cron.UpdateMirror.Enabled {
- entry, err = c.AddFunc("Update mirrors", setting.Cron.UpdateMirror.Schedule, WithUnique(mirrorUpdate, models.MirrorUpdate))
+ entry, err = c.AddFunc("Update mirrors", setting.Cron.UpdateMirror.Schedule, WithUnique(mirrorUpdate, mirror_service.Update))
if err != nil {
log.Fatal("Cron[Update mirrors]: %v", err)
}
if setting.Cron.UpdateMirror.RunAtStart {
entry.Prev = time.Now()
entry.ExecTimes++
- go WithUnique(mirrorUpdate, models.MirrorUpdate)()
+ go WithUnique(mirrorUpdate, mirror_service.Update)()
}
}
if setting.Cron.RepoHealthCheck.Enabled {
diff --git a/modules/templates/helper.go b/modules/templates/helper.go
index ba61dd5ee..b40f7117f 100644
--- a/modules/templates/helper.go
+++ b/modules/templates/helper.go
@@ -28,6 +28,7 @@ import (
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/services/gitdiff"
+ mirror_service "code.gitea.io/gitea/services/mirror"
"gopkg.in/editorconfig/editorconfig-core-go.v1"
)
@@ -233,6 +234,8 @@ func NewFuncMap() []template.FuncMap {
return float32(n) * 100 / float32(sum)
},
"CommentMustAsDiff": gitdiff.CommentMustAsDiff,
+ "MirrorAddress": mirror_service.Address,
+ "MirrorFullAddress": mirror_service.AddressNoCredentials,
}}
}
diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go
index c838ba727..513e7a37b 100644
--- a/routers/api/v1/repo/repo.go
+++ b/routers/api/v1/repo/repo.go
@@ -20,6 +20,7 @@ import (
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/routers/api/v1/convert"
+ mirror_service "code.gitea.io/gitea/services/mirror"
)
var searchOrderByMap = map[string]map[string]models.SearchOrderBy{
@@ -869,6 +870,7 @@ func MirrorSync(ctx *context.APIContext) {
ctx.Error(403, "MirrorSync", "Must have write access")
}
- go models.MirrorQueue.Add(repo.ID)
+ mirror_service.StartToMirror(repo.ID)
+
ctx.Status(200)
}
diff --git a/routers/init.go b/routers/init.go
index 5c8741f24..1efddcfaa 100644
--- a/routers/init.go
+++ b/routers/init.go
@@ -21,6 +21,7 @@ import (
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/ssh"
"code.gitea.io/gitea/services/mailer"
+ mirror_service "code.gitea.io/gitea/services/mirror"
"gitea.com/macaron/macaron"
)
@@ -98,7 +99,7 @@ func GlobalInit() {
log.Fatal("Failed to initialize issue indexer: %v", err)
}
models.InitRepoIndexer()
- models.InitSyncMirrors()
+ mirror_service.InitSyncMirrors()
models.InitDeliverHooks()
models.InitTestPullRequests()
}
diff --git a/routers/repo/setting.go b/routers/repo/setting.go
index b0af396e7..c74b27342 100644
--- a/routers/repo/setting.go
+++ b/routers/repo/setting.go
@@ -25,6 +25,7 @@ import (
"code.gitea.io/gitea/modules/validation"
"code.gitea.io/gitea/routers/utils"
"code.gitea.io/gitea/services/mailer"
+ mirror_service "code.gitea.io/gitea/services/mirror"
"github.com/unknwon/com"
"mvdan.cc/xurls/v2"
@@ -190,7 +191,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) {
address = u.String()
- if err := ctx.Repo.Mirror.SaveAddress(address); err != nil {
+ if err := mirror_service.SaveAddress(ctx.Repo.Mirror, address); err != nil {
ctx.ServerError("SaveAddress", err)
return
}
@@ -204,7 +205,8 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) {
return
}
- go models.MirrorQueue.Add(repo.ID)
+ mirror_service.StartToMirror(repo.ID)
+
ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress"))
ctx.Redirect(repo.Link() + "/settings")
diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go
new file mode 100644
index 000000000..3339f7232
--- /dev/null
+++ b/services/mirror/mirror.go
@@ -0,0 +1,395 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package mirror
+
+import (
+ "fmt"
+ "net/url"
+ "strings"
+ "time"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/cache"
+ "code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/process"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/sync"
+ "code.gitea.io/gitea/modules/timeutil"
+ "code.gitea.io/gitea/modules/util"
+
+ "github.com/mcuadros/go-version"
+ "github.com/unknwon/com"
+)
+
+// mirrorQueue holds an UniqueQueue object of the mirror
+var mirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength)
+
+func readAddress(m *models.Mirror) {
+ if len(m.Address) > 0 {
+ return
+ }
+ var err error
+ m.Address, err = remoteAddress(m.Repo.RepoPath())
+ if err != nil {
+ log.Error("remoteAddress: %v", err)
+ }
+}
+
+func remoteAddress(repoPath string) (string, error) {
+ var cmd *git.Command
+ binVersion, err := git.BinVersion()
+ if err != nil {
+ return "", err
+ }
+ if version.Compare(binVersion, "2.7", ">=") {
+ cmd = git.NewCommand("remote", "get-url", "origin")
+ } else {
+ cmd = git.NewCommand("config", "--get", "remote.origin.url")
+ }
+
+ result, err := cmd.RunInDir(repoPath)
+ if err != nil {
+ if strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") {
+ return "", nil
+ }
+ return "", err
+ }
+ if len(result) > 0 {
+ return result[:len(result)-1], nil
+ }
+ return "", nil
+}
+
+// sanitizeOutput sanitizes output of a command, replacing occurrences of the
+// repository's remote address with a sanitized version.
+func sanitizeOutput(output, repoPath string) (string, error) {
+ remoteAddr, err := remoteAddress(repoPath)
+ if err != nil {
+ // if we're unable to load the remote address, then we're unable to
+ // sanitize.
+ return "", err
+ }
+ return util.SanitizeMessage(output, remoteAddr), nil
+}
+
+// AddressNoCredentials returns mirror address from Git repository config without credentials.
+func AddressNoCredentials(m *models.Mirror) string {
+ readAddress(m)
+ u, err := url.Parse(m.Address)
+ if err != nil {
+ // this shouldn't happen but just return it unsanitised
+ return m.Address
+ }
+ u.User = nil
+ return u.String()
+}
+
+// SaveAddress writes new address to Git repository config.
+func SaveAddress(m *models.Mirror, addr string) error {
+ repoPath := m.Repo.RepoPath()
+ // Remove old origin
+ _, err := git.NewCommand("remote", "remove", "origin").RunInDir(repoPath)
+ if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") {
+ return err
+ }
+
+ _, err = git.NewCommand("remote", "add", "origin", "--mirror=fetch", addr).RunInDir(repoPath)
+ return err
+}
+
+// gitShortEmptySha Git short empty SHA
+const gitShortEmptySha = "0000000"
+
+// mirrorSyncResult contains information of a updated reference.
+// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
+// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
+type mirrorSyncResult struct {
+ refName string
+ oldCommitID string
+ newCommitID string
+}
+
+// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
+func parseRemoteUpdateOutput(output string) []*mirrorSyncResult {
+ results := make([]*mirrorSyncResult, 0, 3)
+ lines := strings.Split(output, "\n")
+ for i := range lines {
+ // Make sure reference name is presented before continue
+ idx := strings.Index(lines[i], "-> ")
+ if idx == -1 {
+ continue
+ }
+
+ refName := lines[i][idx+3:]
+
+ switch {
+ case strings.HasPrefix(lines[i], " * "): // New reference
+ results = append(results, &mirrorSyncResult{
+ refName: refName,
+ oldCommitID: gitShortEmptySha,
+ })
+ case strings.HasPrefix(lines[i], " - "): // Delete reference
+ results = append(results, &mirrorSyncResult{
+ refName: refName,
+ newCommitID: gitShortEmptySha,
+ })
+ case strings.HasPrefix(lines[i], " "): // New commits of a reference
+ delimIdx := strings.Index(lines[i][3:], " ")
+ if delimIdx == -1 {
+ log.Error("SHA delimiter not found: %q", lines[i])
+ continue
+ }
+ shas := strings.Split(lines[i][3:delimIdx+3], "..")
+ if len(shas) != 2 {
+ log.Error("Expect two SHAs but not what found: %q", lines[i])
+ continue
+ }
+ results = append(results, &mirrorSyncResult{
+ refName: refName,
+ oldCommitID: shas[0],
+ newCommitID: shas[1],
+ })
+
+ default:
+ log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i])
+ }
+ }
+ return results
+}
+
+// runSync returns true if sync finished without error.
+func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) {
+ repoPath := m.Repo.RepoPath()
+ wikiPath := m.Repo.WikiPath()
+ timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
+
+ gitArgs := []string{"remote", "update"}
+ if m.EnablePrune {
+ gitArgs = append(gitArgs, "--prune")
+ }
+
+ _, stderr, err := process.GetManager().ExecDir(
+ timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
+ git.GitExecutable, gitArgs...)
+ if err != nil {
+ // sanitize the output, since it may contain the remote address, which may
+ // contain a password
+ message, err := sanitizeOutput(stderr, repoPath)
+ if err != nil {
+ log.Error("sanitizeOutput: %v", err)
+ return nil, false
+ }
+ desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, message)
+ log.Error(desc)
+ if err = models.CreateRepositoryNotice(desc); err != nil {
+ log.Error("CreateRepositoryNotice: %v", err)
+ }
+ return nil, false
+ }
+ output := stderr
+
+ gitRepo, err := git.OpenRepository(repoPath)
+ if err != nil {
+ log.Error("OpenRepository: %v", err)
+ return nil, false
+ }
+ if err = models.SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
+ log.Error("Failed to synchronize tags to releases for repository: %v", err)
+ }
+
+ if err := m.Repo.UpdateSize(); err != nil {
+ log.Error("Failed to update size for mirror repository: %v", err)
+ }
+
+ if m.Repo.HasWiki() {
+ if _, stderr, err := process.GetManager().ExecDir(
+ timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath),
+ git.GitExecutable, "remote", "update", "--prune"); err != nil {
+ // sanitize the output, since it may contain the remote address, which may
+ // contain a password
+ message, err := sanitizeOutput(stderr, wikiPath)
+ if err != nil {
+ log.Error("sanitizeOutput: %v", err)
+ return nil, false
+ }
+ desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, message)
+ log.Error(desc)
+ if err = models.CreateRepositoryNotice(desc); err != nil {
+ log.Error("CreateRepositoryNotice: %v", err)
+ }
+ return nil, false
+ }
+ }
+
+ branches, err := m.Repo.GetBranches()
+ if err != nil {
+ log.Error("GetBranches: %v", err)
+ return nil, false
+ }
+
+ for i := range branches {
+ cache.Remove(m.Repo.GetCommitsCountCacheKey(branches[i].Name, true))
+ }
+
+ m.UpdatedUnix = timeutil.TimeStampNow()
+ return parseRemoteUpdateOutput(output), true
+}
+
+// Address returns mirror address from Git repository config without credentials.
+func Address(m *models.Mirror) string {
+ readAddress(m)
+ return util.SanitizeURLCredentials(m.Address, false)
+}
+
+// Username returns the mirror address username
+func Username(m *models.Mirror) string {
+ readAddress(m)
+ u, err := url.Parse(m.Address)
+ if err != nil {
+ // this shouldn't happen but if it does return ""
+ return ""
+ }
+ return u.User.Username()
+}
+
+// Password returns the mirror address password
+func Password(m *models.Mirror) string {
+ readAddress(m)
+ u, err := url.Parse(m.Address)
+ if err != nil {
+ // this shouldn't happen but if it does return ""
+ return ""
+ }
+ password, _ := u.User.Password()
+ return password
+}
+
+// Update checks and updates mirror repositories.
+func Update() {
+ log.Trace("Doing: Update")
+
+ if err := models.MirrorsIterate(func(idx int, bean interface{}) error {
+ m := bean.(*models.Mirror)
+ if m.Repo == nil {
+ log.Error("Disconnected mirror repository found: %d", m.ID)
+ return nil
+ }
+
+ mirrorQueue.Add(m.RepoID)
+ return nil
+ }); err != nil {
+ log.Error("Update: %v", err)
+ }
+}
+
+// SyncMirrors checks and syncs mirrors.
+// TODO: sync more mirrors at same time.
+func SyncMirrors() {
+ // Start listening on new sync requests.
+ for repoID := range mirrorQueue.Queue() {
+ log.Trace("SyncMirrors [repo_id: %v]", repoID)
+ mirrorQueue.Remove(repoID)
+
+ m, err := models.GetMirrorByRepoID(com.StrTo(repoID).MustInt64())
+ if err != nil {
+ log.Error("GetMirrorByRepoID [%s]: %v", repoID, err)
+ continue
+ }
+
+ results, ok := runSync(m)
+ if !ok {
+ continue
+ }
+
+ m.ScheduleNextUpdate()
+ if err = models.UpdateMirror(m); err != nil {
+ log.Error("UpdateMirror [%s]: %v", repoID, err)
+ continue
+ }
+
+ var gitRepo *git.Repository
+ if len(results) == 0 {
+ log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID)
+ } else {
+ gitRepo, err = git.OpenRepository(m.Repo.RepoPath())
+ if err != nil {
+ log.Error("OpenRepository [%d]: %v", m.RepoID, err)
+ continue
+ }
+ }
+
+ for _, result := range results {
+ // Discard GitHub pull requests, i.e. refs/pull/*
+ if strings.HasPrefix(result.refName, "refs/pull/") {
+ continue
+ }
+
+ // Create reference
+ if result.oldCommitID == gitShortEmptySha {
+ if err = models.MirrorSyncCreateAction(m.Repo, result.refName); err != nil {
+ log.Error("MirrorSyncCreateAction [repo_id: %d]: %v", m.RepoID, err)
+ }
+ continue
+ }
+
+ // Delete reference
+ if result.newCommitID == gitShortEmptySha {
+ if err = models.MirrorSyncDeleteAction(m.Repo, result.refName); err != nil {
+ log.Error("MirrorSyncDeleteAction [repo_id: %d]: %v", m.RepoID, err)
+ }
+ continue
+ }
+
+ // Push commits
+ oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID)
+ if err != nil {
+ log.Error("GetFullCommitID [%d]: %v", m.RepoID, err)
+ continue
+ }
+ newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID)
+ if err != nil {
+ log.Error("GetFullCommitID [%d]: %v", m.RepoID, err)
+ continue
+ }
+ commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID)
+ if err != nil {
+ log.Error("CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err)
+ continue
+ }
+ if err = models.MirrorSyncPushAction(m.Repo, models.MirrorSyncPushActionOptions{
+ RefName: result.refName,
+ OldCommitID: oldCommitID,
+ NewCommitID: newCommitID,
+ Commits: models.ListToPushCommits(commits),
+ }); err != nil {
+ log.Error("MirrorSyncPushAction [repo_id: %d]: %v", m.RepoID, err)
+ continue
+ }
+ }
+
+ // Get latest commit date and update to current repository updated time
+ commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath())
+ if err != nil {
+ log.Error("GetLatestCommitDate [%d]: %v", m.RepoID, err)
+ continue
+ }
+
+ if err = models.UpdateRepositoryUpdatedTime(m.RepoID, commitDate); err != nil {
+ log.Error("Update repository 'updated_unix' [%d]: %v", m.RepoID, err)
+ continue
+ }
+ }
+}
+
+// InitSyncMirrors initializes a go routine to sync the mirrors
+func InitSyncMirrors() {
+ go SyncMirrors()
+}
+
+// StartToMirror adds repoID to mirror queue
+func StartToMirror(repoID int64) {
+ go mirrorQueue.Add(repoID)
+}
diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_test.go
new file mode 100644
index 000000000..76bd4c72f
--- /dev/null
+++ b/services/mirror/mirror_test.go
@@ -0,0 +1,78 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package mirror
+
+import (
+ "path/filepath"
+ "testing"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/git"
+ release_service "code.gitea.io/gitea/services/release"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestMain(m *testing.M) {
+ models.MainTest(m, filepath.Join("..", ".."))
+}
+
+func TestRelease_MirrorDelete(t *testing.T) {
+ assert.NoError(t, models.PrepareTestDatabase())
+
+ user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User)
+ repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
+ repoPath := models.RepoPath(user.Name, repo.Name)
+
+ migrationOptions := models.MigrateRepoOptions{
+ Name: "test_mirror",
+ Description: "Test mirror",
+ IsPrivate: false,
+ IsMirror: true,
+ RemoteAddr: repoPath,
+ Wiki: true,
+ SyncReleasesWithTags: true,
+ }
+ mirror, err := models.MigrateRepository(user, user, migrationOptions)
+ assert.NoError(t, err)
+
+ gitRepo, err := git.OpenRepository(repoPath)
+ assert.NoError(t, err)
+
+ findOptions := models.FindReleasesOptions{IncludeDrafts: true, IncludeTags: true}
+ initCount, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions)
+ assert.NoError(t, err)
+
+ assert.NoError(t, release_service.CreateRelease(gitRepo, &models.Release{
+ RepoID: repo.ID,
+ PublisherID: user.ID,
+ TagName: "v0.2",
+ Target: "master",
+ Title: "v0.2 is released",
+ Note: "v0.2 is released",
+ IsDraft: false,
+ IsPrerelease: false,
+ IsTag: true,
+ }, nil))
+
+ err = mirror.GetMirror()
+ assert.NoError(t, err)
+
+ _, ok := runSync(mirror.Mirror)
+ assert.True(t, ok)
+
+ count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions)
+ assert.EqualValues(t, initCount+1, count)
+
+ release, err := models.GetRelease(repo.ID, "v0.2")
+ assert.NoError(t, err)
+ assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true))
+
+ _, ok = runSync(mirror.Mirror)
+ assert.True(t, ok)
+
+ count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions)
+ assert.EqualValues(t, initCount, count)
+}
diff --git a/services/release/release_test.go b/services/release/release_test.go
index 97febeecb..d30dfee28 100644
--- a/services/release/release_test.go
+++ b/services/release/release_test.go
@@ -100,62 +100,3 @@ func TestRelease_Create(t *testing.T) {
IsTag: true,
}, nil))
}
-
-func TestRelease_MirrorDelete(t *testing.T) {
- assert.NoError(t, models.PrepareTestDatabase())
-
- user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User)
- repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
- repoPath := models.RepoPath(user.Name, repo.Name)
- migrationOptions := models.MigrateRepoOptions{
- Name: "test_mirror",
- Description: "Test mirror",
- IsPrivate: false,
- IsMirror: true,
- RemoteAddr: repoPath,
- Wiki: true,
- SyncReleasesWithTags: true,
- }
- mirror, err := models.MigrateRepository(user, user, migrationOptions)
- assert.NoError(t, err)
-
- gitRepo, err := git.OpenRepository(repoPath)
- assert.NoError(t, err)
-
- findOptions := models.FindReleasesOptions{IncludeDrafts: true, IncludeTags: true}
- initCount, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions)
- assert.NoError(t, err)
-
- assert.NoError(t, CreateRelease(gitRepo, &models.Release{
- RepoID: repo.ID,
- PublisherID: user.ID,
- TagName: "v0.2",
- Target: "master",
- Title: "v0.2 is released",
- Note: "v0.2 is released",
- IsDraft: false,
- IsPrerelease: false,
- IsTag: true,
- }, nil))
-
- err = mirror.GetMirror()
- assert.NoError(t, err)
-
- ok := models.RunMirrorSync(mirror.Mirror)
- assert.True(t, ok)
-
- count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions)
- assert.NoError(t, err)
- assert.EqualValues(t, initCount+1, count)
-
- release, err := models.GetRelease(repo.ID, "v0.2")
- assert.NoError(t, err)
- assert.NoError(t, DeleteReleaseByID(release.ID, user, true))
-
- ok = models.RunMirrorSync(mirror.Mirror)
- assert.True(t, ok)
-
- count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions)
- assert.NoError(t, err)
- assert.EqualValues(t, initCount, count)
-}
diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl
index e657042b6..fc7f1b660 100644
--- a/templates/repo/header.tmpl
+++ b/templates/repo/header.tmpl
@@ -13,7 +13,7 @@
{{.Name}}
{{if and .RelAvatarLink .IsPrivate}}{{end}}
{{if .IsArchived}}{{end}}
- {{if .IsMirror}}{{end}}
+ {{if .IsMirror}}{{end}}
{{if .IsFork}}{{end}}
diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl
index 87e23b693..4f7d32479 100644
--- a/templates/repo/settings/options.tmpl
+++ b/templates/repo/settings/options.tmpl
@@ -80,7 +80,7 @@