From 188fd2dd1a778ad140a569b31d2bf6c95e2a6bae Mon Sep 17 00:00:00 2001 From: zeripath Date: Tue, 23 Nov 2021 03:09:35 +0000 Subject: [PATCH] Add `PULL_LIMIT` and `PUSH_LIMIT` to cron.update_mirror task (#17568) --- custom/conf/app.example.ini | 6 +++ .../doc/advanced/config-cheat-sheet.en-us.md | 2 + models/repo_mirror.go | 1 + models/repo_pushmirror.go | 1 + services/cron/tasks_basic.go | 25 ++++++--- services/mirror/mirror.go | 52 +++++++++++++++---- 6 files changed, 70 insertions(+), 17 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 066e7e2cda..18985e04e9 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1719,6 +1719,12 @@ PATH = ;RUN_AT_START = false ;; Notice if not success ;NO_SUCCESS_NOTICE = true +;; Limit the number of mirrors added to the queue to this number +;; (negative values mean no limit, 0 will result in no result in no mirrors being queued effectively disabling pull mirror updating.) +;PULL_LIMIT=50 +;; Limit the number of mirrors added to the queue to this number +;; (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling push mirror updating) +;PUSH_LIMIT=50 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 10ba325ddc..3e9b279614 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -828,6 +828,8 @@ NB: You must have `DISABLE_ROUTER_LOG` set to `false` for this option to take ef - `SCHEDULE`: **@every 10m**: Cron syntax for scheduling update mirrors, e.g. `@every 3h`. - `NO_SUCCESS_NOTICE`: **true**: The cron task for update mirrors success report is not very useful - as it just means that the mirrors have been queued. Therefore this is turned off by default. +- `PULL_LIMIT`: **50**: Limit the number of mirrors added to the queue to this number (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling pull mirror updating). +- `PUSH_LIMIT`: **50**: Limit the number of mirrors added to the queue to this number (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling push mirror updating). #### Cron - Repository Health Check (`cron.repo_health_check`) diff --git a/models/repo_mirror.go b/models/repo_mirror.go index 35685b3220..e28d0d2517 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -119,6 +119,7 @@ func MirrorsIterate(f func(idx int, bean interface{}) error) error { return db.GetEngine(db.DefaultContext). Where("next_update_unix<=?", time.Now().Unix()). And("next_update_unix!=0"). + OrderBy("updated_unix ASC"). Iterate(new(Mirror), f) } diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go index c6207bae6d..38a1a66947 100644 --- a/models/repo_pushmirror.go +++ b/models/repo_pushmirror.go @@ -107,5 +107,6 @@ func PushMirrorsIterate(f func(idx int, bean interface{}) error) error { return db.GetEngine(db.DefaultContext). Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()). And("`interval` != 0"). + OrderBy("last_update ASC"). Iterate(new(PushMirror), f) } diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index 57fb399d4e..219173ccf0 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -18,13 +18,24 @@ import ( ) func registerUpdateMirrorTask() { - RegisterTaskFatal("update_mirrors", &BaseConfig{ - Enabled: true, - RunAtStart: false, - Schedule: "@every 10m", - NoSuccessNotice: true, - }, func(ctx context.Context, _ *models.User, _ Config) error { - return mirror_service.Update(ctx) + type UpdateMirrorTaskConfig struct { + BaseConfig + PullLimit int + PushLimit int + } + + RegisterTaskFatal("update_mirrors", &UpdateMirrorTaskConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 10m", + NoSuccessNotice: true, + }, + PullLimit: 50, + PushLimit: 50, + }, func(ctx context.Context, _ *models.User, cfg Config) error { + umtc := cfg.(*UpdateMirrorTaskConfig) + return mirror_service.Update(ctx, umtc.PullLimit, umtc.PushLimit) }) } diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index eb37639bef..dae6f2807b 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -45,15 +45,19 @@ func doMirrorSync(ctx context.Context, req *SyncRequest) { } } +var errLimit = fmt.Errorf("reached limit") + // Update checks and updates mirror repositories. -func Update(ctx context.Context) error { +func Update(ctx context.Context, pullLimit, pushLimit int) error { if !setting.Mirror.Enabled { log.Warn("Mirror feature disabled, but cron job enabled: skip update") return nil } log.Trace("Doing: Update") - handler := func(idx int, bean interface{}) error { + requested := 0 + + handler := func(idx int, bean interface{}, limit int) error { var item SyncRequest if m, ok := bean.(*models.Mirror); ok { if m.Repo == nil { @@ -78,21 +82,49 @@ func Update(ctx context.Context) error { return nil } + // Check we've not been cancelled select { case <-ctx.Done(): - return fmt.Errorf("Aborted") + return fmt.Errorf("aborted") default: - return mirrorQueue.Push(&item) } + + // Check if this request is already in the queue + has, err := mirrorQueue.Has(&item) + if err != nil { + return err + } + if has { + return nil + } + + // Push to the Queue + if err := mirrorQueue.Push(&item); err != nil { + return err + } + + requested++ + if limit > 0 && requested > limit { + return errLimit + } + return nil } - if err := models.MirrorsIterate(handler); err != nil { - log.Error("MirrorsIterate: %v", err) - return err + if pullLimit != 0 { + if err := models.MirrorsIterate(func(idx int, bean interface{}) error { + return handler(idx, bean, pullLimit) + }); err != nil && err != errLimit { + log.Error("MirrorsIterate: %v", err) + return err + } } - if err := models.PushMirrorsIterate(handler); err != nil { - log.Error("PushMirrorsIterate: %v", err) - return err + if pushLimit != 0 { + if err := models.PushMirrorsIterate(func(idx int, bean interface{}) error { + return handler(idx, bean, pushLimit) + }); err != nil && err != errLimit { + log.Error("PushMirrorsIterate: %v", err) + return err + } } log.Trace("Finished: Update") return nil