Custom regexp external issues (#17624)

* Implement custom regular expression for external issue tracking.

Signed-off-by: Alexander Beyn <malex@fatelectrons.org>

* Fix syntax/style

* Update repo.go

* Set metas['regexp']

* gofmt

* fix some tests

* fix more tests

* refactor frontend

* use LRU cache for regexp

* Update modules/markup/html_internal_test.go

Co-authored-by: Alexander Beyn <malex@fatelectrons.org>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

models/repo/repo.go

@@ -414,6 +414,9 @@ func (repo *Repository) ComposeMetas() map[string]string {
 			switch unit.ExternalTrackerConfig().ExternalTrackerStyle {
 			case markup.IssueNameStyleAlphanumeric:
 				metas["style"] = markup.IssueNameStyleAlphanumeric
+			case markup.IssueNameStyleRegexp:
+				metas["style"] = markup.IssueNameStyleRegexp
+				metas["regexp"] = unit.ExternalTrackerConfig().ExternalTrackerRegexpPattern
 			default:
 				metas["style"] = markup.IssueNameStyleNumeric
 			}

models/repo/repo_unit.go

@@ -76,9 +76,10 @@ func (cfg *ExternalWikiConfig) ToDB() ([]byte, error) {
 
 // ExternalTrackerConfig describes external tracker config
 type ExternalTrackerConfig struct {
-	ExternalTrackerURL    string
+	ExternalTrackerURL           string
-	ExternalTrackerFormat string
+	ExternalTrackerFormat        string
-	ExternalTrackerStyle  string
+	ExternalTrackerStyle         string
+	ExternalTrackerRegexpPattern string
 }
 
 // FromDB fills up a ExternalTrackerConfig from serialized format.

models/repo_test.go

@@ -74,6 +74,9 @@ func TestMetas(t *testing.T) {
 	externalTracker.ExternalTrackerConfig().ExternalTrackerStyle = markup.IssueNameStyleNumeric
 	testSuccess(markup.IssueNameStyleNumeric)
 
+	externalTracker.ExternalTrackerConfig().ExternalTrackerStyle = markup.IssueNameStyleRegexp
+	testSuccess(markup.IssueNameStyleRegexp)
+
 	repo, err := repo_model.GetRepositoryByID(3)
 	assert.NoError(t, err)
 

modules/markup/html.go

@@ -20,6 +20,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup/common"
 	"code.gitea.io/gitea/modules/references"
+	"code.gitea.io/gitea/modules/regexplru"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/templates/vars"
 	"code.gitea.io/gitea/modules/util"
@@ -33,6 +34,7 @@ import (
 const (
 	IssueNameStyleNumeric      = "numeric"
 	IssueNameStyleAlphanumeric = "alphanumeric"
+	IssueNameStyleRegexp       = "regexp"
 )
 
 var (
@@ -815,19 +817,35 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
 	)
 
 	next := node.NextSibling
+
 	for node != nil && node != next {
-		_, exttrack := ctx.Metas["format"]
+		_, hasExtTrackFormat := ctx.Metas["format"]
-		alphanum := ctx.Metas["style"] == IssueNameStyleAlphanumeric
 
 		// Repos with external issue trackers might still need to reference local PRs
 		// We need to concern with the first one that shows up in the text, whichever it is
-		found, ref = references.FindRenderizableReferenceNumeric(node.Data, exttrack && alphanum)
+		isNumericStyle := ctx.Metas["style"] == "" || ctx.Metas["style"] == IssueNameStyleNumeric
-		if exttrack && alphanum {
+		foundNumeric, refNumeric := references.FindRenderizableReferenceNumeric(node.Data, hasExtTrackFormat && !isNumericStyle)
-			if found2, ref2 := references.FindRenderizableReferenceAlphanumeric(node.Data); found2 {
+
-				if !found || ref2.RefLocation.Start < ref.RefLocation.Start {
+		switch ctx.Metas["style"] {
-					found = true
+		case "", IssueNameStyleNumeric:
-					ref = ref2
+			found, ref = foundNumeric, refNumeric
-				}
+		case IssueNameStyleAlphanumeric:
+			found, ref = references.FindRenderizableReferenceAlphanumeric(node.Data)
+		case IssueNameStyleRegexp:
+			pattern, err := regexplru.GetCompiled(ctx.Metas["regexp"])
+			if err != nil {
+				return
+			}
+			found, ref = references.FindRenderizableReferenceRegexp(node.Data, pattern)
+		}
+
+		// Repos with external issue trackers might still need to reference local PRs
+		// We need to concern with the first one that shows up in the text, whichever it is
+		if hasExtTrackFormat && !isNumericStyle {
+			// If numeric (PR) was found, and it was BEFORE the non-numeric pattern, use that
+			if foundNumeric && refNumeric.RefLocation.Start < ref.RefLocation.Start {
+				found = foundNumeric
+				ref = refNumeric
 			}
 		}
 		if !found {
@@ -836,7 +854,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
 
 		var link *html.Node
 		reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End]
-		if exttrack && !ref.IsPull {
+		if hasExtTrackFormat && !ref.IsPull {
 			ctx.Metas["index"] = ref.Issue
 
 			res, err := vars.Expand(ctx.Metas["format"], ctx.Metas)
@@ -869,7 +887,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
 
 		// Decorate action keywords if actionable
 		var keyword *html.Node
-		if references.IsXrefActionable(ref, exttrack, alphanum) {
+		if references.IsXrefActionable(ref, hasExtTrackFormat) {
 			keyword = createKeyword(node.Data[ref.ActionLocation.Start:ref.ActionLocation.End])
 		} else {
 			keyword = &html.Node{

modules/markup/html_internal_test.go

@@ -21,8 +21,8 @@ const (
 	TestRepoURL = TestAppURL + TestOrgRepo + "/"
 )
 
-// alphanumLink an HTML link to an alphanumeric-style issue
+// externalIssueLink an HTML link to an alphanumeric-style issue
-func alphanumIssueLink(baseURL, class, name string) string {
+func externalIssueLink(baseURL, class, name string) string {
 	return link(util.URLJoin(baseURL, name), class, name)
 }
 
@@ -54,6 +54,13 @@ var alphanumericMetas = map[string]string{
 	"style":  IssueNameStyleAlphanumeric,
 }
 
+var regexpMetas = map[string]string{
+	"format": "https://someurl.com/{user}/{repo}/{index}",
+	"user":   "someUser",
+	"repo":   "someRepo",
+	"style":  IssueNameStyleRegexp,
+}
+
 // these values should match the TestOrgRepo const above
 var localMetas = map[string]string{
 	"user": "gogits",
@@ -184,7 +191,7 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
 	test := func(s, expectedFmt string, names ...string) {
 		links := make([]interface{}, len(names))
 		for i, name := range names {
-			links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name)
+			links[i] = externalIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name)
 		}
 		expected := fmt.Sprintf(expectedFmt, links...)
 		testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: alphanumericMetas})
@@ -194,6 +201,43 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
 	test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890")
 }
 
+func TestRender_IssueIndexPattern5(t *testing.T) {
+	setting.AppURL = TestAppURL
+
+	// regexp: render inputs without valid mentions
+	test := func(s, expectedFmt, pattern string, ids, names []string) {
+		metas := regexpMetas
+		metas["regexp"] = pattern
+		links := make([]interface{}, len(ids))
+		for i, id := range ids {
+			links[i] = link(util.URLJoin("https://someurl.com/someUser/someRepo/", id), "ref-issue ref-external-issue", names[i])
+		}
+
+		expected := fmt.Sprintf(expectedFmt, links...)
+		testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: metas})
+	}
+
+	test("abc ISSUE-123 def", "abc %s def",
+		"ISSUE-(\\d+)",
+		[]string{"123"},
+		[]string{"ISSUE-123"},
+	)
+
+	test("abc (ISSUE 123) def", "abc %s def",
+		"\\(ISSUE (\\d+)\\)",
+		[]string{"123"},
+		[]string{"(ISSUE 123)"},
+	)
+
+	test("abc ISSUE-123 def", "abc %s def",
+		"(ISSUE-(\\d+))",
+		[]string{"ISSUE-123"},
+		[]string{"ISSUE-123"},
+	)
+
+	testRenderIssueIndexPattern(t, "will not match", "will not match", &RenderContext{Metas: regexpMetas})
+}
+
 func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) {
 	if ctx.URLPrefix == "" {
 		ctx.URLPrefix = TestAppURL
@@ -202,7 +246,7 @@ func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *Rend
 	var buf strings.Builder
 	err := postProcess(ctx, []processor{issueIndexPatternProcessor}, strings.NewReader(input), &buf)
 	assert.NoError(t, err)
-	assert.Equal(t, expected, buf.String())
+	assert.Equal(t, expected, buf.String(), "input=%q", input)
 }
 
 func TestRender_AutoLink(t *testing.T) {

modules/references/references.go

@@ -351,6 +351,24 @@ func FindRenderizableReferenceNumeric(content string, prOnly bool) (bool, *Rende
 	}
 }
 
+// FindRenderizableReferenceRegexp returns the first regexp unvalidated references found in a string.
+func FindRenderizableReferenceRegexp(content string, pattern *regexp.Regexp) (bool, *RenderizableReference) {
+	match := pattern.FindStringSubmatchIndex(content)
+	if len(match) < 4 {
+		return false, nil
+	}
+
+	action, location := findActionKeywords([]byte(content), match[2])
+
+	return true, &RenderizableReference{
+		Issue:          content[match[2]:match[3]],
+		RefLocation:    &RefSpan{Start: match[0], End: match[1]},
+		Action:         action,
+		ActionLocation: location,
+		IsPull:         false,
+	}
+}
+
 // FindRenderizableReferenceAlphanumeric returns the first alphanumeric unvalidated references found in a string.
 func FindRenderizableReferenceAlphanumeric(content string) (bool, *RenderizableReference) {
 	match := issueAlphanumericPattern.FindStringSubmatchIndex(content)
@@ -547,7 +565,7 @@ func findActionKeywords(content []byte, start int) (XRefAction, *RefSpan) {
 }
 
 // IsXrefActionable returns true if the xref action is actionable (i.e. produces a result when resolved)
-func IsXrefActionable(ref *RenderizableReference, extTracker, alphaNum bool) bool {
+func IsXrefActionable(ref *RenderizableReference, extTracker bool) bool {
 	if extTracker {
 		// External issues cannot be automatically closed
 		return false

modules/regexplru/regexplru.go

@@ -0,0 +1,45 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package regexplru
+
+import (
+	"regexp"
+
+	"code.gitea.io/gitea/modules/log"
+
+	lru "github.com/hashicorp/golang-lru"
+)
+
+var lruCache *lru.Cache
+
+func init() {
+	var err error
+	lruCache, err = lru.New(1000)
+	if err != nil {
+		log.Fatal("failed to new LRU cache, err: %v", err)
+	}
+}
+
+// GetCompiled works like regexp.Compile, the compiled expr or error is stored in LRU cache
+func GetCompiled(expr string) (r *regexp.Regexp, err error) {
+	v, ok := lruCache.Get(expr)
+	if !ok {
+		r, err = regexp.Compile(expr)
+		if err != nil {
+			lruCache.Add(expr, err)
+			return nil, err
+		}
+		lruCache.Add(expr, r)
+	} else {
+		r, ok = v.(*regexp.Regexp)
+		if !ok {
+			if err, ok = v.(error); ok {
+				return nil, err
+			}
+			panic("impossible")
+		}
+	}
+	return r, nil
+}

modules/regexplru/regexplru_test.go

@@ -0,0 +1,27 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package regexplru
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRegexpLru(t *testing.T) {
+	r, err := GetCompiled("a")
+	assert.NoError(t, err)
+	assert.True(t, r.MatchString("a"))
+
+	r, err = GetCompiled("a")
+	assert.NoError(t, err)
+	assert.True(t, r.MatchString("a"))
+
+	assert.EqualValues(t, 1, lruCache.Len())
+
+	_, err = GetCompiled("(")
+	assert.Error(t, err)
+	assert.EqualValues(t, 2, lruCache.Len())
+}

options/locale/locale_en-US.ini

@@ -1811,6 +1811,9 @@ settings.tracker_url_format_error = The external issue tracker URL format is not
 settings.tracker_issue_style = External Issue Tracker Number Format
 settings.tracker_issue_style.numeric = Numeric
 settings.tracker_issue_style.alphanumeric = Alphanumeric
+settings.tracker_issue_style.regexp = Regular Expression
+settings.tracker_issue_style.regexp_pattern = Regular Expression Pattern
+settings.tracker_issue_style.regexp_pattern_desc = The first captured group will be used in place of <code>{index}</code>.
 settings.tracker_url_format_desc = Use the placeholders <code>{user}</code>, <code>{repo}</code> and <code>{index}</code> for the username, repository name and issue index.
 settings.enable_timetracker = Enable Time Tracking
 settings.allow_only_contributors_to_track_time = Let Only Contributors Track Time

routers/web/repo/setting.go

@@ -434,9 +434,10 @@ func SettingsPost(ctx *context.Context) {
 				RepoID: repo.ID,
 				Type:   unit_model.TypeExternalTracker,
 				Config: &repo_model.ExternalTrackerConfig{
-					ExternalTrackerURL:    form.ExternalTrackerURL,
+					ExternalTrackerURL:           form.ExternalTrackerURL,
-					ExternalTrackerFormat: form.TrackerURLFormat,
+					ExternalTrackerFormat:        form.TrackerURLFormat,
-					ExternalTrackerStyle:  form.TrackerIssueStyle,
+					ExternalTrackerStyle:         form.TrackerIssueStyle,
+					ExternalTrackerRegexpPattern: form.ExternalTrackerRegexpPattern,
 				},
 			})
 			deleteUnitTypes = append(deleteUnitTypes, unit_model.TypeIssues)

services/forms/repo_form.go

@@ -141,6 +141,7 @@ type RepoSettingForm struct {
 	ExternalTrackerURL                    string
 	TrackerURLFormat                      string
 	TrackerIssueStyle                     string
+	ExternalTrackerRegexpPattern          string
 	EnableCloseIssuesViaCommitInAnyBranch bool
 	EnableProjects                        bool
 	EnablePackages                        bool

templates/repo/settings/options.tmpl

@@ -361,16 +361,27 @@
 								<div class="ui radio checkbox">
 								{{$externalTracker := (.Repository.MustGetUnit $.UnitTypeExternalTracker)}}
 								{{$externalTrackerStyle := $externalTracker.ExternalTrackerConfig.ExternalTrackerStyle}}
-									<input class="hidden" tabindex="0" name="tracker_issue_style" type="radio" value="numeric" {{if $externalTrackerStyle}}{{if eq $externalTrackerStyle "numeric"}}checked=""{{end}}{{end}}/>
+									<input class="js-tracker-issue-style" name="tracker_issue_style" type="radio" value="numeric" {{if eq $externalTrackerStyle "numeric"}}checked{{end}}>
-									<label>{{.i18n.Tr "repo.settings.tracker_issue_style.numeric"}} <span class="ui light grey text">(#1234)</span></label>
+									<label>{{.i18n.Tr "repo.settings.tracker_issue_style.numeric"}} <span class="ui light grey text">#1234</span></label>
 								</div>
 							</div>
 							<div class="field">
 								<div class="ui radio checkbox">
-									<input class="hidden" tabindex="0" name="tracker_issue_style" type="radio" value="alphanumeric" {{if $externalTrackerStyle}}{{if eq $externalTracker.ExternalTrackerConfig.ExternalTrackerStyle "alphanumeric"}}checked=""{{end}}{{end}} />
+									<input class="js-tracker-issue-style" name="tracker_issue_style" type="radio" value="alphanumeric" {{if eq $externalTrackerStyle "alphanumeric"}}checked{{end}}>
-									<label>{{.i18n.Tr "repo.settings.tracker_issue_style.alphanumeric"}} <span class="ui light grey text">(ABC-123, DEFG-234)</span></label>
+									<label>{{.i18n.Tr "repo.settings.tracker_issue_style.alphanumeric"}} <span class="ui light grey text">ABC-123 , DEFG-234</span></label>
 								</div>
 							</div>
+							<div class="field">
+								<div class="ui radio checkbox">
+									<input class="js-tracker-issue-style" name="tracker_issue_style" type="radio" value="regexp" {{if eq $externalTrackerStyle "regexp"}}checked{{end}}>
+									<label>{{.i18n.Tr "repo.settings.tracker_issue_style.regexp"}} <span class="ui light grey text">(ISSUE-\d+) , ISSUE-(\d+)</span></label>
+								</div>
+							</div>
+						</div>
+						<div class="field {{if ne $externalTrackerStyle "regexp"}}disabled{{end}}" id="tracker-issue-style-regex-box">
+							<label for="external_tracker_regexp_pattern">{{.i18n.Tr "repo.settings.tracker_issue_style.regexp_pattern"}}</label>
+							<input id="external_tracker_regexp_pattern" name="external_tracker_regexp_pattern" value="{{(.Repository.MustGetUnit $.UnitTypeExternalTracker).ExternalTrackerConfig.ExternalTrackerRegexpPattern}}">
+							<p class="help">{{.i18n.Tr "repo.settings.tracker_issue_style.regexp_pattern_desc" | Str2html}}</p>
 						</div>
 					</div>
 				</div>

web_src/js/features/repo-legacy.js

@@ -462,6 +462,11 @@ export function initRepository() {
         if (typeof $(this).data('context') !== 'undefined') $($(this).data('context')).addClass('disabled');
       }
     });
+    const $trackerIssueStyleRadios = $('.js-tracker-issue-style');
+    $trackerIssueStyleRadios.on('change input', () => {
+      const checkedVal = $trackerIssueStyleRadios.filter(':checked').val();
+      $('#tracker-issue-style-regex-box').toggleClass('disabled', checkedVal !== 'regexp');
+    });
   }
 
   // Labels