From ffbf35b7e9e2ea65525229f2c7d5a076c98e66ba Mon Sep 17 00:00:00 2001 From: zeripath Date: Wed, 16 Jun 2021 23:19:20 +0100 Subject: [PATCH] Clean-up the settings hierarchy for issue_indexer queue (#16001) There are a couple of settings in `[indexer]` relating to the `issue_indexer` queue which override settings in unpredictable ways. This PR adjusts this hierarchy and makes explicit that these settings are deprecated. Signed-off-by: Andrew Thornton Co-authored-by: techknowlogick --- custom/conf/app.example.ini | 10 +++--- .../doc/advanced/config-cheat-sheet.en-us.md | 10 +++--- modules/setting/indexer.go | 32 +++++++++---------- modules/setting/queue.go | 15 +++++---- 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index ea2fe982f..54320a58b 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1155,20 +1155,20 @@ PATH = ;STARTUP_TIMEOUT = 30s ;; ;; Issue indexer queue, currently support: channel, levelqueue or redis, default is levelqueue (deprecated - use [queue.issue_indexer]) -;ISSUE_INDEXER_QUEUE_TYPE = levelqueue +;ISSUE_INDEXER_QUEUE_TYPE = levelqueue; **DEPRECATED** use settings in `[queue.issue_indexer]`. ;; ;; When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the path where the queue will be saved. ;; This can be overridden by `ISSUE_INDEXER_QUEUE_CONN_STR`. ;; default is queues/common -;ISSUE_INDEXER_QUEUE_DIR = queues/common +;ISSUE_INDEXER_QUEUE_DIR = queues/common; **DEPRECATED** use settings in `[queue.issue_indexer]`. ;; ;; When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. ;; When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of ;; the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`. -;ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0" +;ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"; **DEPRECATED** use settings in `[queue.issue_indexer]`. ;; ;; Batch queue number, default is 20 -;ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20 +;ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Repository Indexer settings @@ -1197,7 +1197,7 @@ PATH = ;REPO_INDEXER_EXCLUDE = ;; ;; -;UPDATE_BUFFER_LEN = 20 +;UPDATE_BUFFER_LEN = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`. ;MAX_FILE_SIZE = 1048576 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 1fb05a425..f74011198 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -356,10 +356,10 @@ relation to port exhaustion. - `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. - The next 4 configuration values are deprecated and should be set in `queue.issue_indexer` however are kept for backwards compatibility: -- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`. -- `ISSUE_INDEXER_QUEUE_DIR`: **queues/common**: When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this will be the path where the queue will be saved. (Previously this was `indexers/issues.queue`.) -- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`. -- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number. +- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`. **DEPRECATED** use settings in `[queue.issue_indexer]`. +- `ISSUE_INDEXER_QUEUE_DIR`: **queues/common**: When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this will be the path where the queue will be saved. **DEPRECATED** use settings in `[queue.issue_indexer]`. +- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`. **DEPRECATED** use settings in `[queue.issue_indexer]`. +- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number. **DEPRECATED** use settings in `[queue.issue_indexer]`. - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size). - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`. @@ -370,7 +370,7 @@ relation to port exhaustion. - `REPO_INDEXER_INCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **include** in the index. Use `**.txt` to match any files with .txt extension. An empty list means include all files. - `REPO_INDEXER_EXCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **exclude** from the index. Files that match this list will not be indexed, even if they match in `REPO_INDEXER_INCLUDE`. - `REPO_INDEXER_EXCLUDE_VENDORED`: **true**: Exclude vendored files from index. -- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. +- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. **DEPRECATED** use settings in `[queue.issue_indexer]`. - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed. - `STARTUP_TIMEOUT`: **30s**: If the indexer takes longer than this timeout to start - fail. (This timeout will be added to the hammer time above for child processes - as bleve will not start until the previous parent is shutdown.) Set to zero to never timeout. diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 8f1d801a7..f170a8d3a 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -28,10 +28,10 @@ var ( IssuePath string IssueConnStr string IssueIndexerName string - IssueQueueType string - IssueQueueDir string - IssueQueueConnStr string - IssueQueueBatchNumber int + IssueQueueType string // DEPRECATED - replaced by queue.issue_indexer + IssueQueueDir string // DEPRECATED - replaced by queue.issue_indexer + IssueQueueConnStr string // DEPRECATED - replaced by queue.issue_indexer + IssueQueueBatchNumber int // DEPRECATED - replaced by queue.issue_indexer StartupTimeout time.Duration RepoIndexerEnabled bool @@ -39,20 +39,17 @@ var ( RepoPath string RepoConnStr string RepoIndexerName string - UpdateQueueLength int + UpdateQueueLength int // DEPRECATED - replaced by queue.issue_indexer MaxIndexerFileSize int64 IncludePatterns []glob.Glob ExcludePatterns []glob.Glob ExcludeVendored bool }{ - IssueType: "bleve", - IssuePath: "indexers/issues.bleve", - IssueConnStr: "", - IssueIndexerName: "gitea_issues", - IssueQueueType: LevelQueueType, - IssueQueueDir: "queues/common", - IssueQueueConnStr: "", - IssueQueueBatchNumber: 20, + IssueType: "bleve", + IssuePath: "indexers/issues.bleve", + IssueConnStr: "", + IssueIndexerName: "gitea_issues", + IssueQueueType: LevelQueueType, RepoIndexerEnabled: false, RepoType: "bleve", @@ -74,10 +71,12 @@ func newIndexerService() { Indexer.IssueConnStr = sec.Key("ISSUE_INDEXER_CONN_STR").MustString(Indexer.IssueConnStr) Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName) - Indexer.IssueQueueType = sec.Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(LevelQueueType) - Indexer.IssueQueueDir = filepath.ToSlash(sec.Key("ISSUE_INDEXER_QUEUE_DIR").MustString(filepath.ToSlash(filepath.Join(AppDataPath, "queues/common")))) + // The following settings are deprecated and can be overridden by settings in [queue] or [queue.issue_indexer] + Indexer.IssueQueueType = sec.Key("ISSUE_INDEXER_QUEUE_TYPE").MustString("") + Indexer.IssueQueueDir = filepath.ToSlash(sec.Key("ISSUE_INDEXER_QUEUE_DIR").MustString("")) Indexer.IssueQueueConnStr = sec.Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString("") - Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(20) + Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(0) + Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(0) Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false) Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve") @@ -91,7 +90,6 @@ func newIndexerService() { Indexer.IncludePatterns = IndexerGlobFromString(sec.Key("REPO_INDEXER_INCLUDE").MustString("")) Indexer.ExcludePatterns = IndexerGlobFromString(sec.Key("REPO_INDEXER_EXCLUDE").MustString("")) Indexer.ExcludeVendored = sec.Key("REPO_INDEXER_EXCLUDE_VENDORED").MustBool(true) - Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20) Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024) Indexer.StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(30 * time.Second) } diff --git a/modules/setting/queue.go b/modules/setting/queue.go index 41e95fbe5..4ff02b61e 100644 --- a/modules/setting/queue.go +++ b/modules/setting/queue.go @@ -65,7 +65,7 @@ func GetQueueSettings(name string) QueueSettings { q.SetName = q.QueueName + Queue.SetName } if !filepath.IsAbs(q.DataDir) { - q.DataDir = filepath.Join(AppDataPath, q.DataDir) + q.DataDir = filepath.ToSlash(filepath.Join(AppDataPath, q.DataDir)) } _, _ = sec.NewKey("DATADIR", q.DataDir) @@ -98,6 +98,7 @@ func NewQueueService() { Queue.QueueLength = sec.Key("LENGTH").MustInt(20) Queue.BatchLength = sec.Key("BATCH_LENGTH").MustInt(20) Queue.ConnectionString = sec.Key("CONN_STR").MustString("") + defaultType := sec.Key("TYPE").String() Queue.Type = sec.Key("TYPE").MustString("persistable-channel") Queue.Network, Queue.Addresses, Queue.Password, Queue.DBIndex, _ = ParseQueueConnStr(Queue.ConnectionString) Queue.WrapIfNecessary = sec.Key("WRAP_IF_NECESSARY").MustBool(true) @@ -117,7 +118,7 @@ func NewQueueService() { for _, key := range section.Keys() { sectionMap[key.Name()] = true } - if _, ok := sectionMap["TYPE"]; !ok { + if _, ok := sectionMap["TYPE"]; !ok && defaultType == "" { switch Indexer.IssueQueueType { case LevelQueueType: _, _ = section.NewKey("TYPE", "level") @@ -125,21 +126,23 @@ func NewQueueService() { _, _ = section.NewKey("TYPE", "persistable-channel") case RedisQueueType: _, _ = section.NewKey("TYPE", "redis") + case "": + _, _ = section.NewKey("TYPE", "level") default: log.Fatal("Unsupported indexer queue type: %v", Indexer.IssueQueueType) } } - if _, ok := sectionMap["LENGTH"]; !ok { + if _, ok := sectionMap["LENGTH"]; !ok && Indexer.UpdateQueueLength != 0 { _, _ = section.NewKey("LENGTH", fmt.Sprintf("%d", Indexer.UpdateQueueLength)) } - if _, ok := sectionMap["BATCH_LENGTH"]; !ok { + if _, ok := sectionMap["BATCH_LENGTH"]; !ok && Indexer.IssueQueueBatchNumber != 0 { _, _ = section.NewKey("BATCH_LENGTH", fmt.Sprintf("%d", Indexer.IssueQueueBatchNumber)) } - if _, ok := sectionMap["DATADIR"]; !ok { + if _, ok := sectionMap["DATADIR"]; !ok && Indexer.IssueQueueDir != "" { _, _ = section.NewKey("DATADIR", Indexer.IssueQueueDir) } - if _, ok := sectionMap["CONN_STR"]; !ok { + if _, ok := sectionMap["CONN_STR"]; !ok && Indexer.IssueQueueConnStr != "" { _, _ = section.NewKey("CONN_STR", Indexer.IssueQueueConnStr) }