Add commit count caching (#2774)

* Add commit count caching

* Small refactoring

* Add different key prefix for refs and commits

* Add configuratuion option to allow to change caching time or disable it

conf/app.ini

@@ -339,6 +339,9 @@ INTERVAL = 60
 ; redis: network=tcp,addr=:6379,password=macaron,db=0,pool_size=100,idle_timeout=180
 ; memcache: `127.0.0.1:11211`
 HOST =
+; Time to keep items in cache if not used, default is 16 hours.
+; Setting it to 0 disables caching
+ITEM_TTL = 16h
 
 [session]
 ; Either "memory", "file", or "redis", default is "memory"

models/repo.go

@@ -258,6 +258,17 @@ func (repo *Repository) APIFormat(mode AccessMode) *api.Repository {
 	return repo.innerAPIFormat(mode, false)
 }
 
+// GetCommitsCountCacheKey returns cache key used for commits count caching.
+func (repo *Repository) GetCommitsCountCacheKey(contextName string, isRef bool) string {
+	var prefix string
+	if isRef {
+		prefix = "ref"
+	} else {
+		prefix = "commit"
+	}
+	return fmt.Sprintf("commits-count-%d-%s-%s", repo.ID, prefix, contextName)
+}
+
 func (repo *Repository) innerAPIFormat(mode AccessMode, isParent bool) *api.Repository {
 	var parent *api.Repository
 

models/update.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 
 	"code.gitea.io/git"
-
+	"code.gitea.io/gitea/modules/cache"
 	"code.gitea.io/gitea/modules/log"
 )
 
@@ -205,19 +205,26 @@ func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) {
 	var commits = &PushCommits{}
 	if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
 		// If is tag reference
+		tagName := opts.RefFullName[len(git.TagPrefix):]
 		if isDelRef {
-			err = pushUpdateDeleteTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
+			err = pushUpdateDeleteTag(repo, gitRepo, tagName)
 			if err != nil {
 				return nil, fmt.Errorf("pushUpdateDeleteTag: %v", err)
 			}
 		} else {
-			err = pushUpdateAddTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
+			// Clear cache for tag commit count
+			cache.Remove(repo.GetCommitsCountCacheKey(tagName, true))
+			err = pushUpdateAddTag(repo, gitRepo, tagName)
 			if err != nil {
 				return nil, fmt.Errorf("pushUpdateAddTag: %v", err)
 			}
 		}
 	} else if !isDelRef {
 		// If is branch reference
+
+		// Clear cache for branch commit count
+		cache.Remove(repo.GetCommitsCountCacheKey(opts.RefFullName[len(git.BranchPrefix):], true))
+
 		newCommit, err := gitRepo.GetCommit(opts.NewCommitID)
 		if err != nil {
 			return nil, fmt.Errorf("gitRepo.GetCommit: %v", err)

modules/cache/cache.go

@@ -0,0 +1,72 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+	"code.gitea.io/gitea/modules/setting"
+
+	mc "github.com/go-macaron/cache"
+)
+
+var conn mc.Cache
+
+// NewContext start cache service
+func NewContext() error {
+	if setting.CacheService == nil || conn != nil {
+		return nil
+	}
+
+	var err error
+	conn, err = mc.NewCacher(setting.CacheService.Adapter, mc.Options{
+		Adapter:       setting.CacheService.Adapter,
+		AdapterConfig: setting.CacheService.Conn,
+		Interval:      setting.CacheService.Interval,
+	})
+	return err
+}
+
+// GetInt returns key value from cache with callback when no key exists in cache
+func GetInt(key string, getFunc func() (int, error)) (int, error) {
+	if conn == nil || setting.CacheService.TTL == 0 {
+		return getFunc()
+	}
+	if !conn.IsExist(key) {
+		var (
+			value int
+			err   error
+		)
+		if value, err = getFunc(); err != nil {
+			return value, err
+		}
+		conn.Put(key, value, int64(setting.CacheService.TTL.Seconds()))
+	}
+	return conn.Get(key).(int), nil
+}
+
+// GetInt64 returns key value from cache with callback when no key exists in cache
+func GetInt64(key string, getFunc func() (int64, error)) (int64, error) {
+	if conn == nil || setting.CacheService.TTL == 0 {
+		return getFunc()
+	}
+	if !conn.IsExist(key) {
+		var (
+			value int64
+			err   error
+		)
+		if value, err = getFunc(); err != nil {
+			return value, err
+		}
+		conn.Put(key, value, int64(setting.CacheService.TTL.Seconds()))
+	}
+	return conn.Get(key).(int64), nil
+}
+
+// Remove key from cache
+func Remove(key string) {
+	if conn == nil {
+		return
+	}
+	conn.Delete(key)
+}

modules/context/repo.go

@@ -13,7 +13,9 @@ import (
 
 	"code.gitea.io/git"
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/cache"
 	"code.gitea.io/gitea/modules/setting"
+
 	"github.com/Unknwon/com"
 	"gopkg.in/editorconfig/editorconfig-core-go.v1"
 	"gopkg.in/macaron.v1"
@@ -100,6 +102,21 @@ func (r *Repository) CanUseTimetracker(issue *models.Issue, user *models.User) b
 		r.IsWriter() || issue.IsPoster(user.ID) || issue.AssigneeID == user.ID)
 }
 
+// GetCommitsCount returns cached commit count for current view
+func (r *Repository) GetCommitsCount() (int64, error) {
+	var contextName string
+	if r.IsViewBranch {
+		contextName = r.BranchName
+	} else if r.IsViewTag {
+		contextName = r.TagName
+	} else {
+		contextName = r.CommitID
+	}
+	return cache.GetInt64(r.Repository.GetCommitsCountCacheKey(contextName, r.IsViewBranch || r.IsViewTag), func() (int64, error) {
+		return r.Commit.CommitsCount()
+	})
+}
+
 // GetEditorconfig returns the .editorconfig definition if found in the
 // HEAD of the default repo branch.
 func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) {
@@ -535,9 +552,9 @@ func RepoRef() macaron.Handler {
 		ctx.Data["IsViewCommit"] = ctx.Repo.IsViewCommit
 		ctx.Data["CanCreateBranch"] = ctx.Repo.CanCreateBranch()
 
-		ctx.Repo.CommitsCount, err = ctx.Repo.Commit.CommitsCount()
+		ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount()
 		if err != nil {
-			ctx.Handle(500, "CommitsCount", err)
+			ctx.Handle(500, "GetCommitsCount", err)
 			return
 		}
 		ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount

modules/setting/setting.go

@@ -325,11 +325,6 @@ var (
 	// Time settings
 	TimeFormat string
 
-	// Cache settings
-	CacheAdapter  string
-	CacheInterval int
-	CacheConn     string
-
 	// Session settings
 	SessionConfig  session.Options
 	CSRFCookieName = "_csrf"
@@ -1295,17 +1290,34 @@ func NewXORMLogService(disableConsole bool) {
 	}
 }
 
-func newCacheService() {
+// Cache represents cache settings
-	CacheAdapter = Cfg.Section("cache").Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"})
+type Cache struct {
-	switch CacheAdapter {
+	Adapter  string
-	case "memory":
+	Interval int
-		CacheInterval = Cfg.Section("cache").Key("INTERVAL").MustInt(60)
+	Conn     string
-	case "redis", "memcache":
+	TTL      time.Duration
-		CacheConn = strings.Trim(Cfg.Section("cache").Key("HOST").String(), "\" ")
-	default:
-		log.Fatal(4, "Unknown cache adapter: %s", CacheAdapter)
 }
 
+var (
+	// CacheService the global cache
+	CacheService *Cache
+)
+
+func newCacheService() {
+	sec := Cfg.Section("cache")
+	CacheService = &Cache{
+		Adapter: sec.Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"}),
+	}
+	switch CacheService.Adapter {
+	case "memory":
+		CacheService.Interval = sec.Key("INTERVAL").MustInt(60)
+	case "redis", "memcache":
+		CacheService.Conn = strings.Trim(sec.Key("HOST").String(), "\" ")
+	default:
+		log.Fatal(4, "Unknown cache adapter: %s", CacheService.Adapter)
+	}
+	CacheService.TTL = sec.Key("ITEM_TTL").MustDuration(16 * time.Hour)
+
 	log.Info("Cache Service Enabled")
 }
 

routers/admin/admin.go

@@ -224,9 +224,9 @@ func Config(ctx *context.Context) {
 		ctx.Data["Mailer"] = setting.MailService
 	}
 
-	ctx.Data["CacheAdapter"] = setting.CacheAdapter
+	ctx.Data["CacheAdapter"] = setting.CacheService.Adapter
-	ctx.Data["CacheInterval"] = setting.CacheInterval
+	ctx.Data["CacheInterval"] = setting.CacheService.Interval
-	ctx.Data["CacheConn"] = setting.CacheConn
+	ctx.Data["CacheConn"] = setting.CacheService.Conn
 
 	ctx.Data["SessionConfig"] = setting.SessionConfig
 

routers/init.go

@@ -11,6 +11,7 @@ import (
 	"code.gitea.io/git"
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/models/migrations"
+	"code.gitea.io/gitea/modules/cache"
 	"code.gitea.io/gitea/modules/cron"
 	"code.gitea.io/gitea/modules/highlight"
 	"code.gitea.io/gitea/modules/log"
@@ -18,6 +19,7 @@ import (
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/ssh"
+
 	macaron "gopkg.in/macaron.v1"
 )
 
@@ -37,6 +39,7 @@ func checkRunMode() {
 func NewServices() {
 	setting.NewServices()
 	mailer.NewContext()
+	cache.NewContext()
 }
 
 // GlobalInit is for global configuration reload-able.

routers/repo/commit.go

@@ -55,7 +55,7 @@ func Commits(ctx *context.Context) {
 	}
 	ctx.Data["PageIsViewCode"] = true
 
-	commitsCount, err := ctx.Repo.Commit.CommitsCount()
+	commitsCount, err := ctx.Repo.GetCommitsCount()
 	if err != nil {
 		ctx.Handle(500, "GetCommitsCount", err)
 		return
@@ -91,7 +91,7 @@ func Graph(ctx *context.Context) {
 	ctx.Data["PageIsCommits"] = true
 	ctx.Data["PageIsViewCode"] = true
 
-	commitsCount, err := ctx.Repo.Commit.CommitsCount()
+	commitsCount, err := ctx.Repo.GetCommitsCount()
 	if err != nil {
 		ctx.Handle(500, "GetCommitsCount", err)
 		return

routers/routes/routes.go

@@ -99,9 +99,9 @@ func NewMacaron() *macaron.Macaron {
 		Redirect:    true,
 	}))
 	m.Use(cache.Cacher(cache.Options{
-		Adapter:       setting.CacheAdapter,
+		Adapter:       setting.CacheService.Adapter,
-		AdapterConfig: setting.CacheConn,
+		AdapterConfig: setting.CacheService.Conn,
-		Interval:      setting.CacheInterval,
+		Interval:      setting.CacheService.Interval,
 	}))
 	m.Use(captcha.Captchaer(captcha.Options{
 		SubURL: setting.AppSubURL,
@@ -576,9 +576,9 @@ func RegisterRoutes(m *macaron.Macaron) {
 				ctx.Handle(500, "GetBranchCommit", err)
 				return
 			}
-			ctx.Repo.CommitsCount, err = ctx.Repo.Commit.CommitsCount()
+			ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount()
 			if err != nil {
-				ctx.Handle(500, "CommitsCount", err)
+				ctx.Handle(500, "GetCommitsCount", err)
 				return
 			}
 			ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount