Sync gitea app path for git hooks and authorized keys when starting (#17335)

Gitea writes its own AppPath into git hook scripts. If Gitea's AppPath changes, then the git push will fail.

This PR:

* Introduce an AppState module, it can persist app states into database
* During GlobalInit, Gitea will check if the current AppPath is the same as last one. If they don't match, Gitea will sync git hooks.
* Refactor some code to make them more clear.
* Also, "Detect if gitea binary's name changed" #11341 is related, we call models.RewriteAllPublicKeys to update ssh authorized_keys file

contrib/pr/checkout.go

@@ -91,7 +91,7 @@ func runPR() {
 	dbCfg.NewKey("DB_TYPE", "sqlite3")
 	dbCfg.NewKey("PATH", ":memory:")
 
-	routers.NewServices()
+	routers.InitGitServices()
 	setting.Database.LogSQL = true
 	//x, err = xorm.NewEngine("sqlite3", "file::memory:?cache=shared")
 

models/appstate/appstate.go

@@ -0,0 +1,57 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package appstate
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/models/db"
+)
+
+// AppState represents a state record in database
+// if one day we would make Gitea run as a cluster,
+// we can introduce a new field `Scope` here to store different states for different nodes
+type AppState struct {
+	ID       string `xorm:"pk varchar(200)"`
+	Revision int64
+	Content  string `xorm:"LONGTEXT"`
+}
+
+func init() {
+	db.RegisterModel(new(AppState))
+}
+
+// SaveAppStateContent saves the app state item to database
+func SaveAppStateContent(key, content string) error {
+	return db.WithTx(func(ctx context.Context) error {
+		eng := db.GetEngine(ctx)
+		// try to update existing row
+		res, err := eng.Exec("UPDATE app_state SET revision=revision+1, content=? WHERE id=?", content, key)
+		if err != nil {
+			return err
+		}
+		rows, _ := res.RowsAffected()
+		if rows != 0 {
+			// the existing row is updated, so we can return
+			return nil
+		}
+		// if no existing row, insert a new row
+		_, err = eng.Insert(&AppState{ID: key, Content: content})
+		return err
+	})
+}
+
+// GetAppStateContent gets an app state from database
+func GetAppStateContent(key string) (content string, err error) {
+	e := db.GetEngine(db.DefaultContext)
+	appState := &AppState{ID: key}
+	has, err := e.Get(appState)
+	if err != nil {
+		return "", err
+	} else if !has {
+		return "", nil
+	}
+	return appState.Content, nil
+}

models/migrations/migrations.go

@@ -352,6 +352,8 @@ var migrations = []Migration{
 	NewMigration("Add issue content history table", addTableIssueContentHistory),
 	// v199 -> v200
 	NewMigration("Add remote version table", addRemoteVersionTable),
+	// v200 -> v201
+	NewMigration("Add table app_state", addTableAppState),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v200.go

@@ -0,0 +1,23 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"xorm.io/xorm"
+)
+
+func addTableAppState(x *xorm.Engine) error {
+	type AppState struct {
+		ID       string `xorm:"pk varchar(200)"`
+		Revision int64
+		Content  string `xorm:"LONGTEXT"`
+	}
+	if err := x.Sync2(new(AppState)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+	return nil
+}

modules/appstate/appstate.go

@@ -0,0 +1,25 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package appstate
+
+// StateStore is the interface to get/set app state items
+type StateStore interface {
+	Get(item StateItem) error
+	Set(item StateItem) error
+}
+
+// StateItem provides the name for a state item. the name will be used to generate filenames, etc
+type StateItem interface {
+	Name() string
+}
+
+// AppState contains the state items for the app
+var AppState StateStore
+
+// Init initialize AppState interface
+func Init() error {
+	AppState = &DBStore{}
+	return nil
+}

modules/appstate/appstate_test.go

@@ -0,0 +1,64 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package appstate
+
+import (
+	"path/filepath"
+	"testing"
+
+	"code.gitea.io/gitea/models/db"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMain(m *testing.M) {
+	db.MainTest(m, filepath.Join("..", ".."), "")
+}
+
+type testItem1 struct {
+	Val1 string
+	Val2 int
+}
+
+func (*testItem1) Name() string {
+	return "test-item1"
+}
+
+type testItem2 struct {
+	K string
+}
+
+func (*testItem2) Name() string {
+	return "test-item2"
+}
+
+func TestAppStateDB(t *testing.T) {
+	assert.NoError(t, db.PrepareTestDatabase())
+
+	as := &DBStore{}
+
+	item1 := new(testItem1)
+	assert.NoError(t, as.Get(item1))
+	assert.Equal(t, "", item1.Val1)
+	assert.EqualValues(t, 0, item1.Val2)
+
+	item1 = new(testItem1)
+	item1.Val1 = "a"
+	item1.Val2 = 2
+	assert.NoError(t, as.Set(item1))
+
+	item2 := new(testItem2)
+	item2.K = "V"
+	assert.NoError(t, as.Set(item2))
+
+	item1 = new(testItem1)
+	assert.NoError(t, as.Get(item1))
+	assert.Equal(t, "a", item1.Val1)
+	assert.EqualValues(t, 2, item1.Val2)
+
+	item2 = new(testItem2)
+	assert.NoError(t, as.Get(item2))
+	assert.Equal(t, "V", item2.K)
+}

modules/appstate/db.go

@@ -0,0 +1,37 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package appstate
+
+import (
+	"code.gitea.io/gitea/models/appstate"
+	"code.gitea.io/gitea/modules/json"
+
+	"github.com/yuin/goldmark/util"
+)
+
+// DBStore can be used to store app state items in local filesystem
+type DBStore struct {
+}
+
+// Get reads the state item
+func (f *DBStore) Get(item StateItem) error {
+	content, err := appstate.GetAppStateContent(item.Name())
+	if err != nil {
+		return err
+	}
+	if content == "" {
+		return nil
+	}
+	return json.Unmarshal(util.StringToReadOnlyBytes(content), item)
+}
+
+// Set saves the state item
+func (f *DBStore) Set(item StateItem) error {
+	b, err := json.Marshal(item)
+	if err != nil {
+		return err
+	}
+	return appstate.SaveAppStateContent(item.Name(), util.BytesToReadOnlyString(b))
+}

modules/appstate/item_runtime.go

@@ -0,0 +1,15 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package appstate
+
+// RuntimeState contains app state for runtime, and we can save remote version for update checker here in future
+type RuntimeState struct {
+	LastAppPath string `json:"last_app_path"`
+}
+
+// Name returns the item name
+func (a RuntimeState) Name() string {
+	return "runtime-state"
+}

modules/repository/hooks.go

@@ -23,64 +23,90 @@ import (
 func getHookTemplates() (hookNames, hookTpls, giteaHookTpls []string) {
 	hookNames = []string{"pre-receive", "update", "post-receive"}
 	hookTpls = []string{
+		// for pre-receive
 		fmt.Sprintf(`#!/usr/bin/env %s
+# AUTO GENERATED BY GITEA, DO NOT MODIFY
 data=$(cat)
 exitcodes=""
 hookname=$(basename $0)
 GIT_DIR=${GIT_DIR:-$(dirname $0)/..}
 
 for hook in ${GIT_DIR}/hooks/${hookname}.d/*; do
-test -x "${hook}" && test -f "${hook}" || continue
+  test -x "${hook}" && test -f "${hook}" || continue
-echo "${data}" | "${hook}"
+  echo "${data}" | "${hook}"
-exitcodes="${exitcodes} $?"
+  exitcodes="${exitcodes} $?"
 done
 
 for i in ${exitcodes}; do
-[ ${i} -eq 0 ] || exit ${i}
+  [ ${i} -eq 0 ] || exit ${i}
 done
 `, setting.ScriptType),
+
+		// for update
 		fmt.Sprintf(`#!/usr/bin/env %s
+# AUTO GENERATED BY GITEA, DO NOT MODIFY
 exitcodes=""
 hookname=$(basename $0)
 GIT_DIR=${GIT_DIR:-$(dirname $0/..)}
 
 for hook in ${GIT_DIR}/hooks/${hookname}.d/*; do
-test -x "${hook}" && test -f "${hook}" || continue
+  test -x "${hook}" && test -f "${hook}" || continue
-"${hook}" $1 $2 $3
+  "${hook}" $1 $2 $3
-exitcodes="${exitcodes} $?"
+  exitcodes="${exitcodes} $?"
 done
 
 for i in ${exitcodes}; do
-[ ${i} -eq 0 ] || exit ${i}
+  [ ${i} -eq 0 ] || exit ${i}
 done
 `, setting.ScriptType),
+
+		// for post-receive
 		fmt.Sprintf(`#!/usr/bin/env %s
+# AUTO GENERATED BY GITEA, DO NOT MODIFY
 data=$(cat)
 exitcodes=""
 hookname=$(basename $0)
 GIT_DIR=${GIT_DIR:-$(dirname $0)/..}
 
 for hook in ${GIT_DIR}/hooks/${hookname}.d/*; do
-test -x "${hook}" && test -f "${hook}" || continue
+  test -x "${hook}" && test -f "${hook}" || continue
-echo "${data}" | "${hook}"
+  echo "${data}" | "${hook}"
-exitcodes="${exitcodes} $?"
+  exitcodes="${exitcodes} $?"
 done
 
 for i in ${exitcodes}; do
-[ ${i} -eq 0 ] || exit ${i}
+  [ ${i} -eq 0 ] || exit ${i}
 done
 `, setting.ScriptType),
 	}
+
 	giteaHookTpls = []string{
-		fmt.Sprintf("#!/usr/bin/env %s\n%s hook --config=%s pre-receive\n", setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)),
+		// for pre-receive
-		fmt.Sprintf("#!/usr/bin/env %s\n%s hook --config=%s update $1 $2 $3\n", setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)),
+		fmt.Sprintf(`#!/usr/bin/env %s
-		fmt.Sprintf("#!/usr/bin/env %s\n%s hook --config=%s post-receive\n", setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)),
+# AUTO GENERATED BY GITEA, DO NOT MODIFY
+%s hook --config=%s pre-receive
+`, setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)),
+
+		// for update
+		fmt.Sprintf(`#!/usr/bin/env %s
+# AUTO GENERATED BY GITEA, DO NOT MODIFY
+%s hook --config=%s update $1 $2 $3
+`, setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)),
+
+		// for post-receive
+		fmt.Sprintf(`#!/usr/bin/env %s
+# AUTO GENERATED BY GITEA, DO NOT MODIFY
+%s hook --config=%s post-receive
+`, setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)),
 	}
 
 	if git.SupportProcReceive {
 		hookNames = append(hookNames, "proc-receive")
 		hookTpls = append(hookTpls,
-			fmt.Sprintf("#!/usr/bin/env %s\n%s hook --config=%s proc-receive\n", setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)))
+			fmt.Sprintf(`#!/usr/bin/env %s
+# AUTO GENERATED BY GITEA, DO NOT MODIFY
+%s hook --config=%s proc-receive
+`, setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)))
 		giteaHookTpls = append(giteaHookTpls, "")
 	}
 

modules/setting/setting.go

@@ -683,6 +683,18 @@ func NewContext() {
 	StaticRootPath = sec.Key("STATIC_ROOT_PATH").MustString(StaticRootPath)
 	StaticCacheTime = sec.Key("STATIC_CACHE_TIME").MustDuration(6 * time.Hour)
 	AppDataPath = sec.Key("APP_DATA_PATH").MustString(path.Join(AppWorkPath, "data"))
+	if _, err = os.Stat(AppDataPath); err != nil {
+		// FIXME: There are too many calls to MkdirAll in old code. It is incorrect.
+		// For example, if someDir=/mnt/vol1/gitea-home/data, if the mount point /mnt/vol1 is not mounted when Gitea runs,
+		// then gitea will make new empty directories in /mnt/vol1, all are stored in the root filesystem.
+		// The correct behavior should be: creating parent directories is end users' duty. We only create sub-directories in existing parent directories.
+		// For quickstart, the parent directories should be created automatically for first startup (eg: a flag or a check of INSTALL_LOCK).
+		// Now we can take the first step to do correctly (using Mkdir) in other packages, and prepare the AppDataPath here, then make a refactor in future.
+		err = os.MkdirAll(AppDataPath, os.ModePerm)
+		if err != nil {
+			log.Fatal("Failed to create the directory for app data path '%s'", AppDataPath)
+		}
+	}
 	EnableGzip = sec.Key("ENABLE_GZIP").MustBool()
 	EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false)
 	PprofDataPath = sec.Key("PPROF_DATA_PATH").MustString(path.Join(AppWorkPath, "data/tmp/pprof"))

routers/init.go

@@ -6,9 +6,12 @@ package routers
 
 import (
 	"context"
+	"reflect"
+	"runtime"
 	"strings"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/appstate"
 	"code.gitea.io/gitea/modules/cache"
 	"code.gitea.io/gitea/modules/cron"
 	"code.gitea.io/gitea/modules/eventsource"
@@ -22,6 +25,7 @@ import (
 	"code.gitea.io/gitea/modules/markup/external"
 	repo_migrations "code.gitea.io/gitea/modules/migrations"
 	"code.gitea.io/gitea/modules/notification"
+	repo_module "code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/ssh"
 	"code.gitea.io/gitea/modules/storage"
@@ -45,23 +49,49 @@ import (
 	"gitea.com/go-chi/session"
 )
 
-// NewServices init new services
+func mustInit(fn func() error) {
-func NewServices() {
+	err := fn()
+	if err != nil {
+		ptr := reflect.ValueOf(fn).Pointer()
+		fi := runtime.FuncForPC(ptr)
+		log.Fatal("%s failed: %v", fi.Name(), err)
+	}
+}
+
+func mustInitCtx(ctx context.Context, fn func(ctx context.Context) error) {
+	err := fn(ctx)
+	if err != nil {
+		ptr := reflect.ValueOf(fn).Pointer()
+		fi := runtime.FuncForPC(ptr)
+		log.Fatal("%s(ctx) failed: %v", fi.Name(), err)
+	}
+}
+
+// InitGitServices init new services for git, this is also called in `contrib/pr/checkout.go`
+func InitGitServices() {
 	setting.NewServices()
-	if err := storage.Init(); err != nil {
+	mustInit(storage.Init)
-		log.Fatal("storage init failed: %v", err)
+	mustInit(repository.NewContext)
+}
+
+func syncAppPathForGit(ctx context.Context) error {
+	runtimeState := new(appstate.RuntimeState)
+	if err := appstate.AppState.Get(runtimeState); err != nil {
+		return err
 	}
-	if err := repository.NewContext(); err != nil {
+	if runtimeState.LastAppPath != setting.AppPath {
-		log.Fatal("repository init failed: %v", err)
+		log.Info("AppPath changed from '%s' to '%s'", runtimeState.LastAppPath, setting.AppPath)
-	}
+
-	mailer.NewContext()
+		log.Info("re-sync repository hooks ...")
-	if err := cache.NewContext(); err != nil {
+		mustInitCtx(ctx, repo_module.SyncRepositoryHooks)
-		log.Fatal("Unable to start cache service: %v", err)
+
-	}
+		log.Info("re-write ssh public keys ...")
-	notification.NewContext()
+		mustInit(models.RewriteAllPublicKeys)
-	if err := archiver.Init(); err != nil {
+
-		log.Fatal("archiver init failed: %v", err)
+		runtimeState.LastAppPath = setting.AppPath
+		return appstate.AppState.Set(runtimeState)
 	}
+	return nil
 }
 
 // GlobalInit is for global configuration reload-able.
@@ -71,9 +101,7 @@ func GlobalInit(ctx context.Context) {
 		log.Fatal("Gitea is not installed")
 	}
 
-	if err := git.Init(ctx); err != nil {
+	mustInitCtx(ctx, git.Init)
-		log.Fatal("Git module init failed: %v", err)
-	}
 	log.Info(git.VersionInfo())
 
 	git.CheckLFSVersion()
@@ -87,7 +115,11 @@ func GlobalInit(ctx context.Context) {
 	// Setup i18n
 	translation.InitLocales()
 
-	NewServices()
+	InitGitServices()
+	mailer.NewContext()
+	mustInit(cache.NewContext)
+	notification.NewContext()
+	mustInit(archiver.Init)
 
 	highlight.NewContext()
 	external.RegisterRenderers()
@@ -98,15 +130,11 @@ func GlobalInit(ctx context.Context) {
 	} else if setting.Database.UseSQLite3 {
 		log.Fatal("SQLite3 is set in settings but NOT Supported")
 	}
-	if err := common.InitDBEngine(ctx); err == nil {
-		log.Info("ORM engine initialization successful!")
-	} else {
-		log.Fatal("ORM engine initialization failed: %v", err)
-	}
 
-	if err := oauth2.Init(); err != nil {
+	mustInitCtx(ctx, common.InitDBEngine)
-		log.Fatal("Failed to initialize OAuth2 support: %v", err)
+	log.Info("ORM engine initialization successful!")
-	}
+	mustInit(appstate.Init)
+	mustInit(oauth2.Init)
 
 	models.NewRepoContext()
 
@@ -114,22 +142,17 @@ func GlobalInit(ctx context.Context) {
 	cron.NewContext()
 	issue_indexer.InitIssueIndexer(false)
 	code_indexer.Init()
-	if err := stats_indexer.Init(); err != nil {
+	mustInit(stats_indexer.Init)
-		log.Fatal("Failed to initialize repository stats indexer queue: %v", err)
+
-	}
 	mirror_service.InitSyncMirrors()
 	webhook.InitDeliverHooks()
-	if err := pull_service.Init(); err != nil {
+	mustInit(pull_service.Init)
-		log.Fatal("Failed to initialize test pull requests queue: %v", err)
+	mustInit(task.Init)
-	}
+	mustInit(repo_migrations.Init)
-	if err := task.Init(); err != nil {
-		log.Fatal("Failed to initialize task scheduler: %v", err)
-	}
-	if err := repo_migrations.Init(); err != nil {
-		log.Fatal("Failed to initialize repository migrations: %v", err)
-	}
 	eventsource.GetManager().Init()
 
+	mustInitCtx(ctx, syncAppPathForGit)
+
 	if setting.SSH.StartBuiltinServer {
 		ssh.Listen(setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs)
 		log.Info("SSH server started on %s:%d. Cipher list (%v), key exchange algorithms (%v), MACs (%v)", setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs)
@@ -137,7 +160,6 @@ func GlobalInit(ctx context.Context) {
 		ssh.Unused()
 	}
 	auth.Init()
-
 	svg.Init()
 }