From 0e7f7df3cf176640c66ddf286ec052c7c13beb8a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 2 Nov 2019 06:51:22 +0800 Subject: [PATCH] Move webhook to a standalone package under modules (#8747) * Move webhook to a standalone package under modules * fix test * fix comments --- go.mod | 2 + models/webhook.go | 379 ++---------------------- models/webhook_test.go | 54 ---- modules/notification/webhook/webhook.go | 63 ++-- modules/repofiles/action.go | 13 +- modules/webhook/deliver.go | 208 +++++++++++++ modules/webhook/main_test.go | 16 + modules/webhook/webhook.go | 179 +++++++++++ modules/webhook/webhook_test.go | 67 +++++ routers/api/v1/repo/hook.go | 5 +- routers/init.go | 3 +- routers/repo/pull.go | 3 +- routers/repo/webhook.go | 5 +- services/issue/label.go | 7 +- services/milestone/milestone.go | 7 +- services/mirror/sync.go | 5 +- services/pull/merge.go | 5 +- services/pull/pull.go | 9 +- services/pull/review.go | 5 +- services/release/release.go | 13 +- 20 files changed, 570 insertions(+), 478 deletions(-) create mode 100644 modules/webhook/deliver.go create mode 100644 modules/webhook/main_test.go create mode 100644 modules/webhook/webhook.go create mode 100644 modules/webhook/webhook_test.go diff --git a/go.mod b/go.mod index 02f0c46f2..7e920ac03 100644 --- a/go.mod +++ b/go.mod @@ -69,6 +69,8 @@ require ( github.com/mattn/go-sqlite3 v1.11.0 github.com/mcuadros/go-version v0.0.0-20190308113854-92cdf37c5b75 github.com/microcosm-cc/bluemonday v0.0.0-20161012083705-f77f16ffc87a + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae // indirect github.com/msteinert/pam v0.0.0-20151204160544-02ccfbfaf0cc github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 diff --git a/models/webhook.go b/models/webhook.go index 6f2162c79..d3a8b52d8 100644 --- a/models/webhook.go +++ b/models/webhook.go @@ -6,34 +6,18 @@ package models import ( - "crypto/hmac" - "crypto/sha256" - "crypto/tls" - "encoding/hex" "encoding/json" "fmt" - "io/ioutil" - "net" - "net/http" - "net/url" - "strings" "time" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/timeutil" - "github.com/gobwas/glob" gouuid "github.com/satori/go.uuid" - "github.com/unknwon/com" ) -// HookQueue is a global queue of web hooks -var HookQueue = sync.NewUniqueQueue(setting.Webhook.QueueLength) - // HookContentType is the content type of a web hook type HookContentType int @@ -227,13 +211,14 @@ func (w *Webhook) HasRepositoryEvent() bool { (w.ChooseEvents && w.HookEvents.Repository) } -func (w *Webhook) eventCheckers() []struct { - has func() bool - typ HookEventType +// EventCheckers returns event checkers +func (w *Webhook) EventCheckers() []struct { + Has func() bool + Type HookEventType } { return []struct { - has func() bool - typ HookEventType + Has func() bool + Type HookEventType }{ {w.HasCreateEvent, HookEventCreate}, {w.HasDeleteEvent, HookEventDelete}, @@ -251,29 +236,14 @@ func (w *Webhook) eventCheckers() []struct { func (w *Webhook) EventsArray() []string { events := make([]string, 0, 7) - for _, c := range w.eventCheckers() { - if c.has() { - events = append(events, string(c.typ)) + for _, c := range w.EventCheckers() { + if c.Has() { + events = append(events, string(c.Type)) } } return events } -func (w *Webhook) checkBranch(branch string) bool { - if w.BranchFilter == "" || w.BranchFilter == "*" { - return true - } - - g, err := glob.Compile(w.BranchFilter) - if err != nil { - // should not really happen as BranchFilter is validated - log.Error("CheckBranch failed: %s", err) - return false - } - - return g.Match(branch) -} - // CreateWebhook creates a new web hook. func CreateWebhook(w *Webhook) error { return createWebhook(x, w) @@ -664,329 +634,20 @@ func UpdateHookTask(t *HookTask) error { return err } -// PrepareWebhook adds special webhook to task queue for given payload. -func PrepareWebhook(w *Webhook, repo *Repository, event HookEventType, p api.Payloader) error { - return prepareWebhook(x, w, repo, event, p) -} - -// getPayloadBranch returns branch for hook event, if applicable. -func getPayloadBranch(p api.Payloader) string { - switch pp := p.(type) { - case *api.CreatePayload: - if pp.RefType == "branch" { - return pp.Ref - } - case *api.DeletePayload: - if pp.RefType == "branch" { - return pp.Ref - } - case *api.PushPayload: - if strings.HasPrefix(pp.Ref, git.BranchPrefix) { - return pp.Ref[len(git.BranchPrefix):] - } - } - return "" -} - -func prepareWebhook(e Engine, w *Webhook, repo *Repository, event HookEventType, p api.Payloader) error { - for _, e := range w.eventCheckers() { - if event == e.typ { - if !e.has() { - return nil - } - } - } - - // If payload has no associated branch (e.g. it's a new tag, issue, etc.), - // branch filter has no effect. - if branch := getPayloadBranch(p); branch != "" { - if !w.checkBranch(branch) { - log.Info("Branch %q doesn't match branch filter %q, skipping", branch, w.BranchFilter) - return nil - } - } - - var payloader api.Payloader - var err error - // Use separate objects so modifications won't be made on payload on non-Gogs/Gitea type hooks. - switch w.HookTaskType { - case SLACK: - payloader, err = GetSlackPayload(p, event, w.Meta) - if err != nil { - return fmt.Errorf("GetSlackPayload: %v", err) - } - case DISCORD: - payloader, err = GetDiscordPayload(p, event, w.Meta) - if err != nil { - return fmt.Errorf("GetDiscordPayload: %v", err) - } - case DINGTALK: - payloader, err = GetDingtalkPayload(p, event, w.Meta) - if err != nil { - return fmt.Errorf("GetDingtalkPayload: %v", err) - } - case TELEGRAM: - payloader, err = GetTelegramPayload(p, event, w.Meta) - if err != nil { - return fmt.Errorf("GetTelegramPayload: %v", err) - } - case MSTEAMS: - payloader, err = GetMSTeamsPayload(p, event, w.Meta) - if err != nil { - return fmt.Errorf("GetMSTeamsPayload: %v", err) - } - default: - p.SetSecret(w.Secret) - payloader = p - } - - var signature string - if len(w.Secret) > 0 { - data, err := payloader.JSONPayload() - if err != nil { - log.Error("prepareWebhooks.JSONPayload: %v", err) - } - sig := hmac.New(sha256.New, []byte(w.Secret)) - _, err = sig.Write(data) - if err != nil { - log.Error("prepareWebhooks.sigWrite: %v", err) - } - signature = hex.EncodeToString(sig.Sum(nil)) - } - - if err = createHookTask(e, &HookTask{ - RepoID: repo.ID, - HookID: w.ID, - Type: w.HookTaskType, - URL: w.URL, - Signature: signature, - Payloader: payloader, - HTTPMethod: w.HTTPMethod, - ContentType: w.ContentType, - EventType: event, - IsSSL: w.IsSSL, - }); err != nil { - return fmt.Errorf("CreateHookTask: %v", err) - } - return nil -} - -// PrepareWebhooks adds new webhooks to task queue for given payload. -func PrepareWebhooks(repo *Repository, event HookEventType, p api.Payloader) error { - return prepareWebhooks(x, repo, event, p) -} - -func prepareWebhooks(e Engine, repo *Repository, event HookEventType, p api.Payloader) error { - ws, err := getActiveWebhooksByRepoID(e, repo.ID) - if err != nil { - return fmt.Errorf("GetActiveWebhooksByRepoID: %v", err) - } - - // check if repo belongs to org and append additional webhooks - if repo.mustOwner(e).IsOrganization() { - // get hooks for org - orgHooks, err := getActiveWebhooksByOrgID(e, repo.OwnerID) - if err != nil { - return fmt.Errorf("GetActiveWebhooksByOrgID: %v", err) - } - ws = append(ws, orgHooks...) - } - - if len(ws) == 0 { - return nil - } - - for _, w := range ws { - if err = prepareWebhook(e, w, repo, event, p); err != nil { - return err - } - } - return nil -} - -func (t *HookTask) deliver() error { - t.IsDelivered = true - - var req *http.Request - var err error - - switch t.HTTPMethod { - case "": - log.Info("HTTP Method for webhook %d empty, setting to POST as default", t.ID) - fallthrough - case http.MethodPost: - switch t.ContentType { - case ContentTypeJSON: - req, err = http.NewRequest("POST", t.URL, strings.NewReader(t.PayloadContent)) - if err != nil { - return err - } - - req.Header.Set("Content-Type", "application/json") - case ContentTypeForm: - var forms = url.Values{ - "payload": []string{t.PayloadContent}, - } - - req, err = http.NewRequest("POST", t.URL, strings.NewReader(forms.Encode())) - if err != nil { - - return err - } - - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - } - case http.MethodGet: - u, err := url.Parse(t.URL) - if err != nil { - return err - } - vals := u.Query() - vals["payload"] = []string{t.PayloadContent} - u.RawQuery = vals.Encode() - req, err = http.NewRequest("GET", u.String(), nil) - if err != nil { - return err - } - default: - return fmt.Errorf("Invalid http method for webhook: [%d] %v", t.ID, t.HTTPMethod) - } - - req.Header.Add("X-Gitea-Delivery", t.UUID) - req.Header.Add("X-Gitea-Event", string(t.EventType)) - req.Header.Add("X-Gitea-Signature", t.Signature) - req.Header.Add("X-Gogs-Delivery", t.UUID) - req.Header.Add("X-Gogs-Event", string(t.EventType)) - req.Header.Add("X-Gogs-Signature", t.Signature) - req.Header["X-GitHub-Delivery"] = []string{t.UUID} - req.Header["X-GitHub-Event"] = []string{string(t.EventType)} - - // Record delivery information. - t.RequestInfo = &HookRequest{ - Headers: map[string]string{}, - } - for k, vals := range req.Header { - t.RequestInfo.Headers[k] = strings.Join(vals, ",") - } - - t.ResponseInfo = &HookResponse{ - Headers: map[string]string{}, - } - - defer func() { - t.Delivered = time.Now().UnixNano() - if t.IsSucceed { - log.Trace("Hook delivered: %s", t.UUID) - } else { - log.Trace("Hook delivery failed: %s", t.UUID) - } - - if err := UpdateHookTask(t); err != nil { - log.Error("UpdateHookTask [%d]: %v", t.ID, err) - } - - // Update webhook last delivery status. - w, err := GetWebhookByID(t.HookID) - if err != nil { - log.Error("GetWebhookByID: %v", err) - return - } - if t.IsSucceed { - w.LastStatus = HookStatusSucceed - } else { - w.LastStatus = HookStatusFail - } - if err = UpdateWebhookLastStatus(w); err != nil { - log.Error("UpdateWebhookLastStatus: %v", err) - return - } - }() - - resp, err := webhookHTTPClient.Do(req) - if err != nil { - t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err) - return err - } - defer resp.Body.Close() - - // Status code is 20x can be seen as succeed. - t.IsSucceed = resp.StatusCode/100 == 2 - t.ResponseInfo.Status = resp.StatusCode - for k, vals := range resp.Header { - t.ResponseInfo.Headers[k] = strings.Join(vals, ",") - } - - p, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.ResponseInfo.Body = fmt.Sprintf("read body: %s", err) - return err - } - t.ResponseInfo.Body = string(p) - return nil -} - -// DeliverHooks checks and delivers undelivered hooks. -// TODO: shoot more hooks at same time. -func DeliverHooks() { +// FindUndeliveredHookTasks represents find the undelivered hook tasks +func FindUndeliveredHookTasks() ([]*HookTask, error) { tasks := make([]*HookTask, 0, 10) - err := x.Where("is_delivered=?", false).Find(&tasks) - if err != nil { - log.Error("DeliverHooks: %v", err) - return - } - - // Update hook task status. - for _, t := range tasks { - if err = t.deliver(); err != nil { - log.Error("deliver: %v", err) - } - } - - // Start listening on new hook requests. - for repoIDStr := range HookQueue.Queue() { - log.Trace("DeliverHooks [repo_id: %v]", repoIDStr) - HookQueue.Remove(repoIDStr) - - repoID, err := com.StrTo(repoIDStr).Int64() - if err != nil { - log.Error("Invalid repo ID: %s", repoIDStr) - continue - } - - tasks = make([]*HookTask, 0, 5) - if err := x.Where("repo_id=? AND is_delivered=?", repoID, false).Find(&tasks); err != nil { - log.Error("Get repository [%d] hook tasks: %v", repoID, err) - continue - } - for _, t := range tasks { - if err = t.deliver(); err != nil { - log.Error("deliver: %v", err) - } - } + if err := x.Where("is_delivered=?", false).Find(&tasks); err != nil { + return nil, err } + return tasks, nil } -var webhookHTTPClient *http.Client - -// InitDeliverHooks starts the hooks delivery thread -func InitDeliverHooks() { - timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second - - webhookHTTPClient = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify}, - Proxy: http.ProxyFromEnvironment, - Dial: func(netw, addr string) (net.Conn, error) { - conn, err := net.DialTimeout(netw, addr, timeout) - if err != nil { - return nil, err - } - - return conn, conn.SetDeadline(time.Now().Add(timeout)) - - }, - }, +// FindRepoUndeliveredHookTasks represents find the undelivered hook tasks of one repository +func FindRepoUndeliveredHookTasks(repoID int64) ([]*HookTask, error) { + tasks := make([]*HookTask, 0, 5) + if err := x.Where("repo_id=? AND is_delivered=?", repoID, false).Find(&tasks); err != nil { + return nil, err } - - go DeliverHooks() + return tasks, nil } diff --git a/models/webhook_test.go b/models/webhook_test.go index 343000f5b..7bdaadc5a 100644 --- a/models/webhook_test.go +++ b/models/webhook_test.go @@ -253,57 +253,3 @@ func TestUpdateHookTask(t *testing.T) { assert.NoError(t, UpdateHookTask(hook)) AssertExistsAndLoadBean(t, hook) } - -func TestPrepareWebhooks(t *testing.T) { - assert.NoError(t, PrepareTestDatabase()) - - repo := AssertExistsAndLoadBean(t, &Repository{ID: 1}).(*Repository) - hookTasks := []*HookTask{ - {RepoID: repo.ID, HookID: 1, EventType: HookEventPush}, - } - for _, hookTask := range hookTasks { - AssertNotExistsBean(t, hookTask) - } - assert.NoError(t, PrepareWebhooks(repo, HookEventPush, &api.PushPayload{})) - for _, hookTask := range hookTasks { - AssertExistsAndLoadBean(t, hookTask) - } -} - -func TestPrepareWebhooksBranchFilterMatch(t *testing.T) { - assert.NoError(t, PrepareTestDatabase()) - - repo := AssertExistsAndLoadBean(t, &Repository{ID: 2}).(*Repository) - hookTasks := []*HookTask{ - {RepoID: repo.ID, HookID: 4, EventType: HookEventPush}, - } - for _, hookTask := range hookTasks { - AssertNotExistsBean(t, hookTask) - } - // this test also ensures that * doesn't handle / in any special way (like shell would) - assert.NoError(t, PrepareWebhooks(repo, HookEventPush, &api.PushPayload{Ref: "refs/heads/feature/7791"})) - for _, hookTask := range hookTasks { - AssertExistsAndLoadBean(t, hookTask) - } -} - -func TestPrepareWebhooksBranchFilterNoMatch(t *testing.T) { - assert.NoError(t, PrepareTestDatabase()) - - repo := AssertExistsAndLoadBean(t, &Repository{ID: 2}).(*Repository) - hookTasks := []*HookTask{ - {RepoID: repo.ID, HookID: 4, EventType: HookEventPush}, - } - for _, hookTask := range hookTasks { - AssertNotExistsBean(t, hookTask) - } - assert.NoError(t, PrepareWebhooks(repo, HookEventPush, &api.PushPayload{Ref: "refs/heads/fix_weird_bug"})) - - for _, hookTask := range hookTasks { - AssertNotExistsBean(t, hookTask) - } -} - -// TODO TestHookTask_deliver - -// TODO TestDeliverHooks diff --git a/modules/notification/webhook/webhook.go b/modules/notification/webhook/webhook.go index b4629ac56..e2874fae7 100644 --- a/modules/notification/webhook/webhook.go +++ b/modules/notification/webhook/webhook.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification/base" api "code.gitea.io/gitea/modules/structs" + webhook_module "code.gitea.io/gitea/modules/webhook" ) type webhookNotifier struct { @@ -43,7 +44,7 @@ func (m *webhookNotifier) NotifyIssueClearLabels(doer *models.User, issue *model return } - err = models.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ + err = webhook_module.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ Action: api.HookIssueLabelCleared, Index: issue.Index, PullRequest: issue.PullRequest.APIFormat(), @@ -51,7 +52,7 @@ func (m *webhookNotifier) NotifyIssueClearLabels(doer *models.User, issue *model Sender: doer.APIFormat(), }) } else { - err = models.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{ + err = webhook_module.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{ Action: api.HookIssueLabelCleared, Index: issue.Index, Issue: issue.APIFormat(), @@ -62,7 +63,7 @@ func (m *webhookNotifier) NotifyIssueClearLabels(doer *models.User, issue *model if err != nil { log.Error("PrepareWebhooks [is_pull: %v]: %v", issue.IsPull, err) } else { - go models.HookQueue.Add(issue.RepoID) + go webhook_module.HookQueue.Add(issue.RepoID) } } @@ -71,21 +72,21 @@ func (m *webhookNotifier) NotifyForkRepository(doer *models.User, oldRepo, repo mode, _ := models.AccessLevel(doer, repo) // forked webhook - if err := models.PrepareWebhooks(oldRepo, models.HookEventFork, &api.ForkPayload{ + if err := webhook_module.PrepareWebhooks(oldRepo, models.HookEventFork, &api.ForkPayload{ Forkee: oldRepo.APIFormat(oldMode), Repo: repo.APIFormat(mode), Sender: doer.APIFormat(), }); err != nil { log.Error("PrepareWebhooks [repo_id: %d]: %v", oldRepo.ID, err) } else { - go models.HookQueue.Add(oldRepo.ID) + go webhook_module.HookQueue.Add(oldRepo.ID) } u := repo.MustOwner() // Add to hook queue for created repo after session commit. if u.IsOrganization() { - if err := models.PrepareWebhooks(repo, models.HookEventRepository, &api.RepositoryPayload{ + if err := webhook_module.PrepareWebhooks(repo, models.HookEventRepository, &api.RepositoryPayload{ Action: api.HookRepoCreated, Repository: repo.APIFormat(models.AccessModeOwner), Organization: u.APIFormat(), @@ -93,7 +94,7 @@ func (m *webhookNotifier) NotifyForkRepository(doer *models.User, oldRepo, repo }); err != nil { log.Error("PrepareWebhooks [repo_id: %d]: %v", repo.ID, err) } else { - go models.HookQueue.Add(repo.ID) + go webhook_module.HookQueue.Add(repo.ID) } } } @@ -101,7 +102,7 @@ func (m *webhookNotifier) NotifyForkRepository(doer *models.User, oldRepo, repo func (m *webhookNotifier) NotifyCreateRepository(doer *models.User, u *models.User, repo *models.Repository) { // Add to hook queue for created repo after session commit. if u.IsOrganization() { - if err := models.PrepareWebhooks(repo, models.HookEventRepository, &api.RepositoryPayload{ + if err := webhook_module.PrepareWebhooks(repo, models.HookEventRepository, &api.RepositoryPayload{ Action: api.HookRepoCreated, Repository: repo.APIFormat(models.AccessModeOwner), Organization: u.APIFormat(), @@ -109,7 +110,7 @@ func (m *webhookNotifier) NotifyCreateRepository(doer *models.User, u *models.Us }); err != nil { log.Error("PrepareWebhooks [repo_id: %d]: %v", repo.ID, err) } else { - go models.HookQueue.Add(repo.ID) + go webhook_module.HookQueue.Add(repo.ID) } } } @@ -118,7 +119,7 @@ func (m *webhookNotifier) NotifyDeleteRepository(doer *models.User, repo *models u := repo.MustOwner() if u.IsOrganization() { - if err := models.PrepareWebhooks(repo, models.HookEventRepository, &api.RepositoryPayload{ + if err := webhook_module.PrepareWebhooks(repo, models.HookEventRepository, &api.RepositoryPayload{ Action: api.HookRepoDeleted, Repository: repo.APIFormat(models.AccessModeOwner), Organization: u.APIFormat(), @@ -126,7 +127,7 @@ func (m *webhookNotifier) NotifyDeleteRepository(doer *models.User, repo *models }); err != nil { log.Error("PrepareWebhooks [repo_id: %d]: %v", repo.ID, err) } - go models.HookQueue.Add(repo.ID) + go webhook_module.HookQueue.Add(repo.ID) } } @@ -151,7 +152,7 @@ func (m *webhookNotifier) NotifyIssueChangeAssignee(doer *models.User, issue *mo apiPullRequest.Action = api.HookIssueAssigned } // Assignee comment triggers a webhook - if err := models.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, apiPullRequest); err != nil { + if err := webhook_module.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, apiPullRequest); err != nil { log.Error("PrepareWebhooks [is_pull: %v, remove_assignee: %v]: %v", issue.IsPull, removed, err) return } @@ -169,13 +170,13 @@ func (m *webhookNotifier) NotifyIssueChangeAssignee(doer *models.User, issue *mo apiIssue.Action = api.HookIssueAssigned } // Assignee comment triggers a webhook - if err := models.PrepareWebhooks(issue.Repo, models.HookEventIssues, apiIssue); err != nil { + if err := webhook_module.PrepareWebhooks(issue.Repo, models.HookEventIssues, apiIssue); err != nil { log.Error("PrepareWebhooks [is_pull: %v, remove_assignee: %v]: %v", issue.IsPull, removed, err) return } } - go models.HookQueue.Add(issue.RepoID) + go webhook_module.HookQueue.Add(issue.RepoID) } func (m *webhookNotifier) NotifyIssueChangeTitle(doer *models.User, issue *models.Issue, oldTitle string) { @@ -187,7 +188,7 @@ func (m *webhookNotifier) NotifyIssueChangeTitle(doer *models.User, issue *model return } issue.PullRequest.Issue = issue - err = models.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ + err = webhook_module.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ Action: api.HookIssueEdited, Index: issue.Index, Changes: &api.ChangesPayload{ @@ -200,7 +201,7 @@ func (m *webhookNotifier) NotifyIssueChangeTitle(doer *models.User, issue *model Sender: doer.APIFormat(), }) } else { - err = models.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{ + err = webhook_module.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{ Action: api.HookIssueEdited, Index: issue.Index, Changes: &api.ChangesPayload{ @@ -217,7 +218,7 @@ func (m *webhookNotifier) NotifyIssueChangeTitle(doer *models.User, issue *model if err != nil { log.Error("PrepareWebhooks [is_pull: %v]: %v", issue.IsPull, err) } else { - go models.HookQueue.Add(issue.RepoID) + go webhook_module.HookQueue.Add(issue.RepoID) } } @@ -241,7 +242,7 @@ func (m *webhookNotifier) NotifyIssueChangeStatus(doer *models.User, issue *mode } else { apiPullRequest.Action = api.HookIssueReOpened } - err = models.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, apiPullRequest) + err = webhook_module.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, apiPullRequest) } else { apiIssue := &api.IssuePayload{ Index: issue.Index, @@ -254,18 +255,18 @@ func (m *webhookNotifier) NotifyIssueChangeStatus(doer *models.User, issue *mode } else { apiIssue.Action = api.HookIssueReOpened } - err = models.PrepareWebhooks(issue.Repo, models.HookEventIssues, apiIssue) + err = webhook_module.PrepareWebhooks(issue.Repo, models.HookEventIssues, apiIssue) } if err != nil { log.Error("PrepareWebhooks [is_pull: %v, is_closed: %v]: %v", issue.IsPull, isClosed, err) } else { - go models.HookQueue.Add(issue.Repo.ID) + go webhook_module.HookQueue.Add(issue.Repo.ID) } } func (m *webhookNotifier) NotifyNewIssue(issue *models.Issue) { mode, _ := models.AccessLevel(issue.Poster, issue.Repo) - if err := models.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{ + if err := webhook_module.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{ Action: api.HookIssueOpened, Index: issue.Index, Issue: issue.APIFormat(), @@ -274,7 +275,7 @@ func (m *webhookNotifier) NotifyNewIssue(issue *models.Issue) { }); err != nil { log.Error("PrepareWebhooks: %v", err) } else { - go models.HookQueue.Add(issue.RepoID) + go webhook_module.HookQueue.Add(issue.RepoID) } } @@ -283,7 +284,7 @@ func (m *webhookNotifier) NotifyIssueChangeContent(doer *models.User, issue *mod var err error if issue.IsPull { issue.PullRequest.Issue = issue - err = models.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ + err = webhook_module.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ Action: api.HookIssueEdited, Index: issue.Index, Changes: &api.ChangesPayload{ @@ -296,7 +297,7 @@ func (m *webhookNotifier) NotifyIssueChangeContent(doer *models.User, issue *mod Sender: doer.APIFormat(), }) } else { - err = models.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{ + err = webhook_module.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{ Action: api.HookIssueEdited, Index: issue.Index, Changes: &api.ChangesPayload{ @@ -312,7 +313,7 @@ func (m *webhookNotifier) NotifyIssueChangeContent(doer *models.User, issue *mod if err != nil { log.Error("PrepareWebhooks [is_pull: %v]: %v", issue.IsPull, err) } else { - go models.HookQueue.Add(issue.RepoID) + go webhook_module.HookQueue.Add(issue.RepoID) } } @@ -332,7 +333,7 @@ func (m *webhookNotifier) NotifyUpdateComment(doer *models.User, c *models.Comme } mode, _ := models.AccessLevel(doer, c.Issue.Repo) - if err := models.PrepareWebhooks(c.Issue.Repo, models.HookEventIssueComment, &api.IssueCommentPayload{ + if err := webhook_module.PrepareWebhooks(c.Issue.Repo, models.HookEventIssueComment, &api.IssueCommentPayload{ Action: api.HookIssueCommentEdited, Issue: c.Issue.APIFormat(), Comment: c.APIFormat(), @@ -347,14 +348,14 @@ func (m *webhookNotifier) NotifyUpdateComment(doer *models.User, c *models.Comme }); err != nil { log.Error("PrepareWebhooks [comment_id: %d]: %v", c.ID, err) } else { - go models.HookQueue.Add(c.Issue.Repo.ID) + go webhook_module.HookQueue.Add(c.Issue.Repo.ID) } } func (m *webhookNotifier) NotifyCreateIssueComment(doer *models.User, repo *models.Repository, issue *models.Issue, comment *models.Comment) { mode, _ := models.AccessLevel(doer, repo) - if err := models.PrepareWebhooks(repo, models.HookEventIssueComment, &api.IssueCommentPayload{ + if err := webhook_module.PrepareWebhooks(repo, models.HookEventIssueComment, &api.IssueCommentPayload{ Action: api.HookIssueCommentCreated, Issue: issue.APIFormat(), Comment: comment.APIFormat(), @@ -364,7 +365,7 @@ func (m *webhookNotifier) NotifyCreateIssueComment(doer *models.User, repo *mode }); err != nil { log.Error("PrepareWebhooks [comment_id: %d]: %v", comment.ID, err) } else { - go models.HookQueue.Add(repo.ID) + go webhook_module.HookQueue.Add(repo.ID) } } @@ -385,7 +386,7 @@ func (m *webhookNotifier) NotifyDeleteComment(doer *models.User, comment *models mode, _ := models.AccessLevel(doer, comment.Issue.Repo) - if err := models.PrepareWebhooks(comment.Issue.Repo, models.HookEventIssueComment, &api.IssueCommentPayload{ + if err := webhook_module.PrepareWebhooks(comment.Issue.Repo, models.HookEventIssueComment, &api.IssueCommentPayload{ Action: api.HookIssueCommentDeleted, Issue: comment.Issue.APIFormat(), Comment: comment.APIFormat(), @@ -395,6 +396,6 @@ func (m *webhookNotifier) NotifyDeleteComment(doer *models.User, comment *models }); err != nil { log.Error("PrepareWebhooks [comment_id: %d]: %v", comment.ID, err) } else { - go models.HookQueue.Add(comment.Issue.Repo.ID) + go webhook_module.HookQueue.Add(comment.Issue.Repo.ID) } } diff --git a/modules/repofiles/action.go b/modules/repofiles/action.go index 9467e4fb7..addbdb464 100644 --- a/modules/repofiles/action.go +++ b/modules/repofiles/action.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/webhook" ) // CommitRepoActionOptions represent options of a new commit action. @@ -112,7 +113,7 @@ func CommitRepoAction(opts CommitRepoActionOptions) error { } defer func() { - go models.HookQueue.Add(repo.ID) + go webhook.HookQueue.Add(repo.ID) }() apiPusher := pusher.APIFormat() @@ -134,7 +135,7 @@ func CommitRepoAction(opts CommitRepoActionOptions) error { if err != nil { log.Error("GetBranchCommitID[%s]: %v", opts.RefFullName, err) } - if err = models.PrepareWebhooks(repo, models.HookEventCreate, &api.CreatePayload{ + if err = webhook.PrepareWebhooks(repo, models.HookEventCreate, &api.CreatePayload{ Ref: refName, Sha: shaSum, RefType: "branch", @@ -148,7 +149,7 @@ func CommitRepoAction(opts CommitRepoActionOptions) error { case models.ActionDeleteBranch: // Delete Branch isHookEventPush = true - if err = models.PrepareWebhooks(repo, models.HookEventDelete, &api.DeletePayload{ + if err = webhook.PrepareWebhooks(repo, models.HookEventDelete, &api.DeletePayload{ Ref: refName, RefType: "branch", PusherType: api.PusherTypeUser, @@ -169,7 +170,7 @@ func CommitRepoAction(opts CommitRepoActionOptions) error { if err != nil { log.Error("GetTagCommitID[%s]: %v", opts.RefFullName, err) } - if err = models.PrepareWebhooks(repo, models.HookEventCreate, &api.CreatePayload{ + if err = webhook.PrepareWebhooks(repo, models.HookEventCreate, &api.CreatePayload{ Ref: refName, Sha: shaSum, RefType: "tag", @@ -181,7 +182,7 @@ func CommitRepoAction(opts CommitRepoActionOptions) error { case models.ActionDeleteTag: // Delete Tag isHookEventPush = true - if err = models.PrepareWebhooks(repo, models.HookEventDelete, &api.DeletePayload{ + if err = webhook.PrepareWebhooks(repo, models.HookEventDelete, &api.DeletePayload{ Ref: refName, RefType: "tag", PusherType: api.PusherTypeUser, @@ -197,7 +198,7 @@ func CommitRepoAction(opts CommitRepoActionOptions) error { if err != nil { return err } - if err = models.PrepareWebhooks(repo, models.HookEventPush, &api.PushPayload{ + if err = webhook.PrepareWebhooks(repo, models.HookEventPush, &api.PushPayload{ Ref: opts.RefFullName, Before: opts.OldCommitID, After: opts.NewCommitID, diff --git a/modules/webhook/deliver.go b/modules/webhook/deliver.go new file mode 100644 index 000000000..1b3da25d5 --- /dev/null +++ b/modules/webhook/deliver.go @@ -0,0 +1,208 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package webhook + +import ( + "crypto/tls" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "github.com/unknwon/com" +) + +// Deliver deliver hook task +func Deliver(t *models.HookTask) error { + t.IsDelivered = true + + var req *http.Request + var err error + + switch t.HTTPMethod { + case "": + log.Info("HTTP Method for webhook %d empty, setting to POST as default", t.ID) + fallthrough + case http.MethodPost: + switch t.ContentType { + case models.ContentTypeJSON: + req, err = http.NewRequest("POST", t.URL, strings.NewReader(t.PayloadContent)) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/json") + case models.ContentTypeForm: + var forms = url.Values{ + "payload": []string{t.PayloadContent}, + } + + req, err = http.NewRequest("POST", t.URL, strings.NewReader(forms.Encode())) + if err != nil { + + return err + } + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + case http.MethodGet: + u, err := url.Parse(t.URL) + if err != nil { + return err + } + vals := u.Query() + vals["payload"] = []string{t.PayloadContent} + u.RawQuery = vals.Encode() + req, err = http.NewRequest("GET", u.String(), nil) + if err != nil { + return err + } + default: + return fmt.Errorf("Invalid http method for webhook: [%d] %v", t.ID, t.HTTPMethod) + } + + req.Header.Add("X-Gitea-Delivery", t.UUID) + req.Header.Add("X-Gitea-Event", string(t.EventType)) + req.Header.Add("X-Gitea-Signature", t.Signature) + req.Header.Add("X-Gogs-Delivery", t.UUID) + req.Header.Add("X-Gogs-Event", string(t.EventType)) + req.Header.Add("X-Gogs-Signature", t.Signature) + req.Header["X-GitHub-Delivery"] = []string{t.UUID} + req.Header["X-GitHub-Event"] = []string{string(t.EventType)} + + // Record delivery information. + t.RequestInfo = &models.HookRequest{ + Headers: map[string]string{}, + } + for k, vals := range req.Header { + t.RequestInfo.Headers[k] = strings.Join(vals, ",") + } + + t.ResponseInfo = &models.HookResponse{ + Headers: map[string]string{}, + } + + defer func() { + t.Delivered = time.Now().UnixNano() + if t.IsSucceed { + log.Trace("Hook delivered: %s", t.UUID) + } else { + log.Trace("Hook delivery failed: %s", t.UUID) + } + + if err := models.UpdateHookTask(t); err != nil { + log.Error("UpdateHookTask [%d]: %v", t.ID, err) + } + + // Update webhook last delivery status. + w, err := models.GetWebhookByID(t.HookID) + if err != nil { + log.Error("GetWebhookByID: %v", err) + return + } + if t.IsSucceed { + w.LastStatus = models.HookStatusSucceed + } else { + w.LastStatus = models.HookStatusFail + } + if err = models.UpdateWebhookLastStatus(w); err != nil { + log.Error("UpdateWebhookLastStatus: %v", err) + return + } + }() + + resp, err := webhookHTTPClient.Do(req) + if err != nil { + t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err) + return err + } + defer resp.Body.Close() + + // Status code is 20x can be seen as succeed. + t.IsSucceed = resp.StatusCode/100 == 2 + t.ResponseInfo.Status = resp.StatusCode + for k, vals := range resp.Header { + t.ResponseInfo.Headers[k] = strings.Join(vals, ",") + } + + p, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.ResponseInfo.Body = fmt.Sprintf("read body: %s", err) + return err + } + t.ResponseInfo.Body = string(p) + return nil +} + +// DeliverHooks checks and delivers undelivered hooks. +// TODO: shoot more hooks at same time. +func DeliverHooks() { + tasks, err := models.FindUndeliveredHookTasks() + if err != nil { + log.Error("DeliverHooks: %v", err) + return + } + + // Update hook task status. + for _, t := range tasks { + if err = Deliver(t); err != nil { + log.Error("deliver: %v", err) + } + } + + // Start listening on new hook requests. + for repoIDStr := range HookQueue.Queue() { + log.Trace("DeliverHooks [repo_id: %v]", repoIDStr) + HookQueue.Remove(repoIDStr) + + repoID, err := com.StrTo(repoIDStr).Int64() + if err != nil { + log.Error("Invalid repo ID: %s", repoIDStr) + continue + } + + tasks, err := models.FindRepoUndeliveredHookTasks(repoID) + if err != nil { + log.Error("Get repository [%d] hook tasks: %v", repoID, err) + continue + } + for _, t := range tasks { + if err = Deliver(t); err != nil { + log.Error("deliver: %v", err) + } + } + } +} + +var webhookHTTPClient *http.Client + +// InitDeliverHooks starts the hooks delivery thread +func InitDeliverHooks() { + timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second + + webhookHTTPClient = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify}, + Proxy: http.ProxyFromEnvironment, + Dial: func(netw, addr string) (net.Conn, error) { + conn, err := net.DialTimeout(netw, addr, timeout) + if err != nil { + return nil, err + } + + return conn, conn.SetDeadline(time.Now().Add(timeout)) + + }, + }, + } + + go DeliverHooks() +} diff --git a/modules/webhook/main_test.go b/modules/webhook/main_test.go new file mode 100644 index 000000000..6cb0cffe4 --- /dev/null +++ b/modules/webhook/main_test.go @@ -0,0 +1,16 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package webhook + +import ( + "path/filepath" + "testing" + + "code.gitea.io/gitea/models" +) + +func TestMain(m *testing.M) { + models.MainTest(m, filepath.Join("..", "..")) +} diff --git a/modules/webhook/webhook.go b/modules/webhook/webhook.go new file mode 100644 index 000000000..da88c58c8 --- /dev/null +++ b/modules/webhook/webhook.go @@ -0,0 +1,179 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package webhook + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/sync" + "github.com/gobwas/glob" +) + +// HookQueue is a global queue of web hooks +var HookQueue = sync.NewUniqueQueue(setting.Webhook.QueueLength) + +// getPayloadBranch returns branch for hook event, if applicable. +func getPayloadBranch(p api.Payloader) string { + switch pp := p.(type) { + case *api.CreatePayload: + if pp.RefType == "branch" { + return pp.Ref + } + case *api.DeletePayload: + if pp.RefType == "branch" { + return pp.Ref + } + case *api.PushPayload: + if strings.HasPrefix(pp.Ref, git.BranchPrefix) { + return pp.Ref[len(git.BranchPrefix):] + } + } + return "" +} + +// PrepareWebhook adds special webhook to task queue for given payload. +func PrepareWebhook(w *models.Webhook, repo *models.Repository, event models.HookEventType, p api.Payloader) error { + return prepareWebhook(w, repo, event, p) +} + +func checkBranch(w *models.Webhook, branch string) bool { + if w.BranchFilter == "" || w.BranchFilter == "*" { + return true + } + + g, err := glob.Compile(w.BranchFilter) + if err != nil { + // should not really happen as BranchFilter is validated + log.Error("CheckBranch failed: %s", err) + return false + } + + return g.Match(branch) +} + +func prepareWebhook(w *models.Webhook, repo *models.Repository, event models.HookEventType, p api.Payloader) error { + for _, e := range w.EventCheckers() { + if event == e.Type { + if !e.Has() { + return nil + } + } + } + + // If payload has no associated branch (e.g. it's a new tag, issue, etc.), + // branch filter has no effect. + if branch := getPayloadBranch(p); branch != "" { + if !checkBranch(w, branch) { + log.Info("Branch %q doesn't match branch filter %q, skipping", branch, w.BranchFilter) + return nil + } + } + + var payloader api.Payloader + var err error + // Use separate objects so modifications won't be made on payload on non-Gogs/Gitea type hooks. + switch w.HookTaskType { + case models.SLACK: + payloader, err = models.GetSlackPayload(p, event, w.Meta) + if err != nil { + return fmt.Errorf("GetSlackPayload: %v", err) + } + case models.DISCORD: + payloader, err = models.GetDiscordPayload(p, event, w.Meta) + if err != nil { + return fmt.Errorf("GetDiscordPayload: %v", err) + } + case models.DINGTALK: + payloader, err = models.GetDingtalkPayload(p, event, w.Meta) + if err != nil { + return fmt.Errorf("GetDingtalkPayload: %v", err) + } + case models.TELEGRAM: + payloader, err = models.GetTelegramPayload(p, event, w.Meta) + if err != nil { + return fmt.Errorf("GetTelegramPayload: %v", err) + } + case models.MSTEAMS: + payloader, err = models.GetMSTeamsPayload(p, event, w.Meta) + if err != nil { + return fmt.Errorf("GetMSTeamsPayload: %v", err) + } + default: + p.SetSecret(w.Secret) + payloader = p + } + + var signature string + if len(w.Secret) > 0 { + data, err := payloader.JSONPayload() + if err != nil { + log.Error("prepareWebhooks.JSONPayload: %v", err) + } + sig := hmac.New(sha256.New, []byte(w.Secret)) + _, err = sig.Write(data) + if err != nil { + log.Error("prepareWebhooks.sigWrite: %v", err) + } + signature = hex.EncodeToString(sig.Sum(nil)) + } + + if err = models.CreateHookTask(&models.HookTask{ + RepoID: repo.ID, + HookID: w.ID, + Type: w.HookTaskType, + URL: w.URL, + Signature: signature, + Payloader: payloader, + HTTPMethod: w.HTTPMethod, + ContentType: w.ContentType, + EventType: event, + IsSSL: w.IsSSL, + }); err != nil { + return fmt.Errorf("CreateHookTask: %v", err) + } + return nil +} + +// PrepareWebhooks adds new webhooks to task queue for given payload. +func PrepareWebhooks(repo *models.Repository, event models.HookEventType, p api.Payloader) error { + return prepareWebhooks(repo, event, p) +} + +func prepareWebhooks(repo *models.Repository, event models.HookEventType, p api.Payloader) error { + ws, err := models.GetActiveWebhooksByRepoID(repo.ID) + if err != nil { + return fmt.Errorf("GetActiveWebhooksByRepoID: %v", err) + } + + // check if repo belongs to org and append additional webhooks + if repo.MustOwner().IsOrganization() { + // get hooks for org + orgHooks, err := models.GetActiveWebhooksByOrgID(repo.OwnerID) + if err != nil { + return fmt.Errorf("GetActiveWebhooksByOrgID: %v", err) + } + ws = append(ws, orgHooks...) + } + + if len(ws) == 0 { + return nil + } + + for _, w := range ws { + if err = prepareWebhook(w, repo, event, p); err != nil { + return err + } + } + return nil +} diff --git a/modules/webhook/webhook_test.go b/modules/webhook/webhook_test.go new file mode 100644 index 000000000..c944bc477 --- /dev/null +++ b/modules/webhook/webhook_test.go @@ -0,0 +1,67 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package webhook + +import ( + "testing" + + "code.gitea.io/gitea/models" + api "code.gitea.io/gitea/modules/structs" + "github.com/stretchr/testify/assert" +) + +func TestPrepareWebhooks(t *testing.T) { + assert.NoError(t, models.PrepareTestDatabase()) + + repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) + hookTasks := []*models.HookTask{ + {RepoID: repo.ID, HookID: 1, EventType: models.HookEventPush}, + } + for _, hookTask := range hookTasks { + models.AssertNotExistsBean(t, hookTask) + } + assert.NoError(t, PrepareWebhooks(repo, models.HookEventPush, &api.PushPayload{})) + for _, hookTask := range hookTasks { + models.AssertExistsAndLoadBean(t, hookTask) + } +} + +func TestPrepareWebhooksBranchFilterMatch(t *testing.T) { + assert.NoError(t, models.PrepareTestDatabase()) + + repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) + hookTasks := []*models.HookTask{ + {RepoID: repo.ID, HookID: 4, EventType: models.HookEventPush}, + } + for _, hookTask := range hookTasks { + models.AssertNotExistsBean(t, hookTask) + } + // this test also ensures that * doesn't handle / in any special way (like shell would) + assert.NoError(t, PrepareWebhooks(repo, models.HookEventPush, &api.PushPayload{Ref: "refs/heads/feature/7791"})) + for _, hookTask := range hookTasks { + models.AssertExistsAndLoadBean(t, hookTask) + } +} + +func TestPrepareWebhooksBranchFilterNoMatch(t *testing.T) { + assert.NoError(t, models.PrepareTestDatabase()) + + repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) + hookTasks := []*models.HookTask{ + {RepoID: repo.ID, HookID: 4, EventType: models.HookEventPush}, + } + for _, hookTask := range hookTasks { + models.AssertNotExistsBean(t, hookTask) + } + assert.NoError(t, PrepareWebhooks(repo, models.HookEventPush, &api.PushPayload{Ref: "refs/heads/fix_weird_bug"})) + + for _, hookTask := range hookTasks { + models.AssertNotExistsBean(t, hookTask) + } +} + +// TODO TestHookTask_deliver + +// TODO TestDeliverHooks diff --git a/routers/api/v1/repo/hook.go b/routers/api/v1/repo/hook.go index 5c7126256..56fc51450 100644 --- a/routers/api/v1/repo/hook.go +++ b/routers/api/v1/repo/hook.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/webhook" "code.gitea.io/gitea/routers/api/v1/convert" "code.gitea.io/gitea/routers/api/v1/utils" ) @@ -122,7 +123,7 @@ func TestHook(ctx *context.APIContext) { return } - if err := models.PrepareWebhook(hook, ctx.Repo.Repository, models.HookEventPush, &api.PushPayload{ + if err := webhook.PrepareWebhook(hook, ctx.Repo.Repository, models.HookEventPush, &api.PushPayload{ Ref: git.BranchPrefix + ctx.Repo.Repository.DefaultBranch, Before: ctx.Repo.Commit.ID.String(), After: ctx.Repo.Commit.ID.String(), @@ -136,7 +137,7 @@ func TestHook(ctx *context.APIContext) { ctx.Error(500, "PrepareWebhook: ", err) return } - go models.HookQueue.Add(ctx.Repo.Repository.ID) + go webhook.HookQueue.Add(ctx.Repo.Repository.ID) ctx.Status(204) } diff --git a/routers/init.go b/routers/init.go index bdd978d71..4623f02e6 100644 --- a/routers/init.go +++ b/routers/init.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" "code.gitea.io/gitea/modules/task" + "code.gitea.io/gitea/modules/webhook" "code.gitea.io/gitea/services/mailer" mirror_service "code.gitea.io/gitea/services/mirror" @@ -101,7 +102,7 @@ func GlobalInit() { issue_indexer.InitIssueIndexer(false) models.InitRepoIndexer() mirror_service.InitSyncMirrors() - models.InitDeliverHooks() + webhook.InitDeliverHooks() models.InitTestPullRequests() if err := task.Init(); err != nil { log.Fatal("Failed to initialize task scheduler: %v", err) diff --git a/routers/repo/pull.go b/routers/repo/pull.go index 3defd04b1..d28ee0cfa 100644 --- a/routers/repo/pull.go +++ b/routers/repo/pull.go @@ -23,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/webhook" "code.gitea.io/gitea/services/gitdiff" pull_service "code.gitea.io/gitea/services/pull" repo_service "code.gitea.io/gitea/services/repository" @@ -823,7 +824,7 @@ func TriggerTask(ctx *context.Context) { log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) - go models.HookQueue.Add(repo.ID) + go webhook.HookQueue.Add(repo.ID) go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true) ctx.Status(202) } diff --git a/routers/repo/webhook.go b/routers/repo/webhook.go index 48b4e7aff..f89bce819 100644 --- a/routers/repo/webhook.go +++ b/routers/repo/webhook.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/webhook" "github.com/unknwon/com" ) @@ -864,11 +865,11 @@ func TestWebhook(ctx *context.Context) { Pusher: apiUser, Sender: apiUser, } - if err := models.PrepareWebhook(w, ctx.Repo.Repository, models.HookEventPush, p); err != nil { + if err := webhook.PrepareWebhook(w, ctx.Repo.Repository, models.HookEventPush, p); err != nil { ctx.Flash.Error("PrepareWebhook: " + err.Error()) ctx.Status(500) } else { - go models.HookQueue.Add(ctx.Repo.Repository.ID) + go webhook.HookQueue.Add(ctx.Repo.Repository.ID) ctx.Flash.Info(ctx.Tr("repo.settings.webhook.test_delivery_success")) ctx.Status(200) } diff --git a/services/issue/label.go b/services/issue/label.go index b393e5d38..bb36d5849 100644 --- a/services/issue/label.go +++ b/services/issue/label.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/webhook" ) func sendLabelUpdatedWebhook(issue *models.Issue, doer *models.User) { @@ -34,7 +35,7 @@ func sendLabelUpdatedWebhook(issue *models.Issue, doer *models.User) { log.Error("LoadIssue: %v", err) return } - err = models.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ + err = webhook.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ Action: api.HookIssueLabelUpdated, Index: issue.Index, PullRequest: issue.PullRequest.APIFormat(), @@ -42,7 +43,7 @@ func sendLabelUpdatedWebhook(issue *models.Issue, doer *models.User) { Sender: doer.APIFormat(), }) } else { - err = models.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{ + err = webhook.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{ Action: api.HookIssueLabelUpdated, Index: issue.Index, Issue: issue.APIFormat(), @@ -53,7 +54,7 @@ func sendLabelUpdatedWebhook(issue *models.Issue, doer *models.User) { if err != nil { log.Error("PrepareWebhooks [is_pull: %v]: %v", issue.IsPull, err) } else { - go models.HookQueue.Add(issue.RepoID) + go webhook.HookQueue.Add(issue.RepoID) } } diff --git a/services/milestone/milestone.go b/services/milestone/milestone.go index 68052e5a6..d6d18ad98 100644 --- a/services/milestone/milestone.go +++ b/services/milestone/milestone.go @@ -8,6 +8,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/webhook" ) // ChangeMilestoneAssign changes assignment of milestone for issue. @@ -34,7 +35,7 @@ func ChangeMilestoneAssign(issue *models.Issue, doer *models.User, oldMilestoneI log.Error("LoadIssue: %v", err) return } - err = models.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ + err = webhook.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ Action: hookAction, Index: issue.Index, PullRequest: issue.PullRequest.APIFormat(), @@ -42,7 +43,7 @@ func ChangeMilestoneAssign(issue *models.Issue, doer *models.User, oldMilestoneI Sender: doer.APIFormat(), }) } else { - err = models.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{ + err = webhook.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{ Action: hookAction, Index: issue.Index, Issue: issue.APIFormat(), @@ -53,7 +54,7 @@ func ChangeMilestoneAssign(issue *models.Issue, doer *models.User, oldMilestoneI if err != nil { log.Error("PrepareWebhooks [is_pull: %v]: %v", issue.IsPull, err) } else { - go models.HookQueue.Add(issue.RepoID) + go webhook.HookQueue.Add(issue.RepoID) } return nil } diff --git a/services/mirror/sync.go b/services/mirror/sync.go index 4bc153b47..1bf36f576 100644 --- a/services/mirror/sync.go +++ b/services/mirror/sync.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/webhook" ) func syncAction(opType models.ActionType, repo *models.Repository, refName string, data []byte) error { @@ -28,7 +29,7 @@ func syncAction(opType models.ActionType, repo *models.Repository, refName strin } defer func() { - go models.HookQueue.Add(repo.ID) + go webhook.HookQueue.Add(repo.ID) }() return nil @@ -55,7 +56,7 @@ func SyncPushAction(repo *models.Repository, opts SyncPushActionOptions) error { opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID) apiPusher := repo.MustOwner().APIFormat() - if err := models.PrepareWebhooks(repo, models.HookEventPush, &api.PushPayload{ + if err := webhook.PrepareWebhooks(repo, models.HookEventPush, &api.PushPayload{ Ref: opts.RefName, Before: opts.OldCommitID, After: opts.NewCommitID, diff --git a/services/pull/merge.go b/services/pull/merge.go index 2e093eef8..f13638c29 100644 --- a/services/pull/merge.go +++ b/services/pull/merge.go @@ -23,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/webhook" "github.com/mcuadros/go-version" ) @@ -360,7 +361,7 @@ func Merge(pr *models.PullRequest, doer *models.User, baseGitRepo *git.Repositor } mode, _ := models.AccessLevel(doer, pr.Issue.Repo) - if err = models.PrepareWebhooks(pr.Issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ + if err = webhook.PrepareWebhooks(pr.Issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ Action: api.HookIssueClosed, Index: pr.Index, PullRequest: pr.APIFormat(), @@ -369,7 +370,7 @@ func Merge(pr *models.PullRequest, doer *models.User, baseGitRepo *git.Repositor }); err != nil { log.Error("PrepareWebhooks: %v", err) } else { - go models.HookQueue.Add(pr.Issue.Repo.ID) + go webhook.HookQueue.Add(pr.Issue.Repo.ID) } return nil diff --git a/services/pull/pull.go b/services/pull/pull.go index 8e6110ac3..65b78f410 100644 --- a/services/pull/pull.go +++ b/services/pull/pull.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/webhook" issue_service "code.gitea.io/gitea/services/issue" ) @@ -41,7 +42,7 @@ func NewPullRequest(repo *models.Repository, pull *models.Issue, labelIDs []int6 pr.Issue = pull pull.PullRequest = pr mode, _ := models.AccessLevel(pull.Poster, repo) - if err := models.PrepareWebhooks(repo, models.HookEventPullRequest, &api.PullRequestPayload{ + if err := webhook.PrepareWebhooks(repo, models.HookEventPullRequest, &api.PullRequestPayload{ Action: api.HookIssueOpened, Index: pull.Index, PullRequest: pr.APIFormat(), @@ -50,7 +51,7 @@ func NewPullRequest(repo *models.Repository, pull *models.Issue, labelIDs []int6 }); err != nil { log.Error("PrepareWebhooks: %v", err) } else { - go models.HookQueue.Add(repo.ID) + go webhook.HookQueue.Add(repo.ID) } return nil @@ -114,7 +115,7 @@ func AddTestPullRequestTask(doer *models.User, repoID int64, branch string, isSy log.Error("LoadAttributes: %v", err) continue } - if err = models.PrepareWebhooks(pr.Issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ + if err = webhook.PrepareWebhooks(pr.Issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{ Action: api.HookIssueSynchronized, Index: pr.Issue.Index, PullRequest: pr.Issue.PullRequest.APIFormat(), @@ -124,7 +125,7 @@ func AddTestPullRequestTask(doer *models.User, repoID int64, branch string, isSy log.Error("PrepareWebhooks [pull_id: %v]: %v", pr.ID, err) continue } - go models.HookQueue.Add(pr.Issue.Repo.ID) + go webhook.HookQueue.Add(pr.Issue.Repo.ID) } } diff --git a/services/pull/review.go b/services/pull/review.go index 3388e4bb5..cffe5da4d 100644 --- a/services/pull/review.go +++ b/services/pull/review.go @@ -8,6 +8,7 @@ package pull import ( "code.gitea.io/gitea/models" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/webhook" ) // CreateReview creates a new review based on opts @@ -55,7 +56,7 @@ func reviewHook(review *models.Review) error { if err != nil { return err } - if err := models.PrepareWebhooks(review.Issue.Repo, reviewHookType, &api.PullRequestPayload{ + if err := webhook.PrepareWebhooks(review.Issue.Repo, reviewHookType, &api.PullRequestPayload{ Action: api.HookIssueSynchronized, Index: review.Issue.Index, PullRequest: pr.APIFormat(), @@ -68,7 +69,7 @@ func reviewHook(review *models.Review) error { }); err != nil { return err } - go models.HookQueue.Add(review.Issue.Repo.ID) + go webhook.HookQueue.Add(review.Issue.Repo.ID) return nil } diff --git a/services/release/release.go b/services/release/release.go index b4f814a77..42db1f341 100644 --- a/services/release/release.go +++ b/services/release/release.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/process" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/webhook" ) func createTag(gitRepo *git.Repository, rel *models.Release) error { @@ -84,7 +85,7 @@ func CreateRelease(gitRepo *git.Repository, rel *models.Release, attachmentUUIDs log.Error("LoadAttributes: %v", err) } else { mode, _ := models.AccessLevel(rel.Publisher, rel.Repo) - if err := models.PrepareWebhooks(rel.Repo, models.HookEventRelease, &api.ReleasePayload{ + if err := webhook.PrepareWebhooks(rel.Repo, models.HookEventRelease, &api.ReleasePayload{ Action: api.HookReleasePublished, Release: rel.APIFormat(), Repository: rel.Repo.APIFormat(mode), @@ -92,7 +93,7 @@ func CreateRelease(gitRepo *git.Repository, rel *models.Release, attachmentUUIDs }); err != nil { log.Error("PrepareWebhooks: %v", err) } else { - go models.HookQueue.Add(rel.Repo.ID) + go webhook.HookQueue.Add(rel.Repo.ID) } } } @@ -121,7 +122,7 @@ func UpdateRelease(doer *models.User, gitRepo *git.Repository, rel *models.Relea // even if attachments added failed, hooks will be still triggered mode, _ := models.AccessLevel(doer, rel.Repo) - if err1 := models.PrepareWebhooks(rel.Repo, models.HookEventRelease, &api.ReleasePayload{ + if err1 := webhook.PrepareWebhooks(rel.Repo, models.HookEventRelease, &api.ReleasePayload{ Action: api.HookReleaseUpdated, Release: rel.APIFormat(), Repository: rel.Repo.APIFormat(mode), @@ -129,7 +130,7 @@ func UpdateRelease(doer *models.User, gitRepo *git.Repository, rel *models.Relea }); err1 != nil { log.Error("PrepareWebhooks: %v", err) } else { - go models.HookQueue.Add(rel.Repo.ID) + go webhook.HookQueue.Add(rel.Repo.ID) } return err @@ -187,7 +188,7 @@ func DeleteReleaseByID(id int64, doer *models.User, delTag bool) error { } mode, _ := models.AccessLevel(doer, rel.Repo) - if err := models.PrepareWebhooks(rel.Repo, models.HookEventRelease, &api.ReleasePayload{ + if err := webhook.PrepareWebhooks(rel.Repo, models.HookEventRelease, &api.ReleasePayload{ Action: api.HookReleaseDeleted, Release: rel.APIFormat(), Repository: rel.Repo.APIFormat(mode), @@ -195,7 +196,7 @@ func DeleteReleaseByID(id int64, doer *models.User, delTag bool) error { }); err != nil { log.Error("PrepareWebhooks: %v", err) } else { - go models.HookQueue.Add(rel.Repo.ID) + go webhook.HookQueue.Add(rel.Repo.ID) } return nil