Asynchronously populate the repo indexer (#3366)

* Populate repo indexer in background

* Check if no repos exist

* race cond

models/repo_indexer.go

@@ -70,34 +70,58 @@ func InitRepoIndexer() {
 	if !setting.Indexer.RepoIndexerEnabled {
 		return
 	}
-	indexer.InitRepoIndexer(populateRepoIndexer)
 	repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength)
+	indexer.InitRepoIndexer(populateRepoIndexerAsynchronously)
 	go processRepoIndexerOperationQueue()
 }
 
-// populateRepoIndexer populate the repo indexer with data
+// populateRepoIndexerAsynchronously asynchronously populates the repo indexer
-func populateRepoIndexer() error {
+// with pre-existing data. This should only be run when the indexer is created
-	log.Info("Populating repository indexer (this may take a while)")
+// for the first time.
-	for page := 1; ; page++ {
+func populateRepoIndexerAsynchronously() error {
-		repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
+	exist, err := x.Table("repository").Exist()
-			Page:     page,
-			PageSize: RepositoryListDefaultPageSize,
-			OrderBy:  SearchOrderByID,
-			Private:  true,
-		})
 	if err != nil {
 		return err
-		} else if len(repos) == 0 {
+	} else if !exist {
 		return nil
 	}
+
+	var maxRepoID int64
+	if _, err = x.Select("MAX(id)").Table("repository").Get(&maxRepoID); err != nil {
+		return err
+	}
+	go populateRepoIndexer(maxRepoID)
+	return nil
+}
+
+// populateRepoIndexer populate the repo indexer with pre-existing data. This
+// should only be run when the indexer is created for the first time.
+func populateRepoIndexer(maxRepoID int64) {
+	log.Info("Populating the repo indexer with existing repositories")
+	// start with the maximum existing repo ID and work backwards, so that we
+	// don't include repos that are created after gitea starts; such repos will
+	// already be added to the indexer, and we don't need to add them again.
+	for maxRepoID > 0 {
+		repos := make([]*Repository, 0, RepositoryListDefaultPageSize)
+		err := x.Where("id <= ?", maxRepoID).
+			OrderBy("id DESC").
+			Limit(RepositoryListDefaultPageSize).
+			Find(&repos)
+		if err != nil {
+			log.Error(4, "populateRepoIndexer: %v", err)
+			return
+		} else if len(repos) == 0 {
+			break
+		}
 		for _, repo := range repos {
-			if err = updateRepoIndexer(repo); err != nil {
+			repoIndexerOperationQueue <- repoIndexerOperation{
-				// only log error, since this should not prevent
+				repo:    repo,
-				// gitea from starting up
+				deleted: false,
-				log.Error(4, "updateRepoIndexer: repoID=%d, %v", repo.ID, err)
+			}
-			}
+			maxRepoID = repo.ID - 1
 		}
 	}
+	log.Info("Done populating the repo indexer with existing repositories")
 }
 
 func updateRepoIndexer(repo *Repository) error {