Determine fuzziness of bleve indexer by keyword length (#29706)
also bleve did match on fuzzy search and the other way around. this also fix that bug. (cherry picked from commit b9c57fb78e8e0d80d786d8e1da433b6c7ebf2f1c) Conflicts: tests/integration/repo_search_test.go simple conflict resolution in the tests
This commit is contained in:
parent
b73117127c
commit
ab5f0b7558
|
@ -39,6 +39,8 @@ import (
|
||||||
const (
|
const (
|
||||||
unicodeNormalizeName = "unicodeNormalize"
|
unicodeNormalizeName = "unicodeNormalize"
|
||||||
maxBatchSize = 16
|
maxBatchSize = 16
|
||||||
|
// fuzzyDenominator determines the levenshtein distance per each character of a keyword
|
||||||
|
fuzzyDenominator = 4
|
||||||
)
|
)
|
||||||
|
|
||||||
func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
|
func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
|
||||||
|
@ -239,15 +241,12 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
|
||||||
keywordQuery query.Query
|
keywordQuery query.Query
|
||||||
)
|
)
|
||||||
|
|
||||||
if opts.IsKeywordFuzzy {
|
|
||||||
phraseQuery := bleve.NewMatchPhraseQuery(opts.Keyword)
|
phraseQuery := bleve.NewMatchPhraseQuery(opts.Keyword)
|
||||||
phraseQuery.FieldVal = "Content"
|
phraseQuery.FieldVal = "Content"
|
||||||
phraseQuery.Analyzer = repoIndexerAnalyzer
|
phraseQuery.Analyzer = repoIndexerAnalyzer
|
||||||
keywordQuery = phraseQuery
|
keywordQuery = phraseQuery
|
||||||
} else {
|
if opts.IsKeywordFuzzy {
|
||||||
prefixQuery := bleve.NewPrefixQuery(opts.Keyword)
|
phraseQuery.Fuzziness = len(opts.Keyword) / fuzzyDenominator
|
||||||
prefixQuery.FieldVal = "Content"
|
|
||||||
keywordQuery = prefixQuery
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(opts.RepoIDs) > 0 {
|
if len(opts.RepoIDs) > 0 {
|
||||||
|
|
|
@ -20,17 +20,11 @@ func NumericEqualityQuery(value int64, field string) *query.NumericRangeQuery {
|
||||||
}
|
}
|
||||||
|
|
||||||
// MatchPhraseQuery generates a match phrase query for the given phrase, field and analyzer
|
// MatchPhraseQuery generates a match phrase query for the given phrase, field and analyzer
|
||||||
func MatchPhraseQuery(matchPhrase, field, analyzer string) *query.MatchPhraseQuery {
|
func MatchPhraseQuery(matchPhrase, field, analyzer string, fuzziness int) *query.MatchPhraseQuery {
|
||||||
q := bleve.NewMatchPhraseQuery(matchPhrase)
|
q := bleve.NewMatchPhraseQuery(matchPhrase)
|
||||||
q.FieldVal = field
|
q.FieldVal = field
|
||||||
q.Analyzer = analyzer
|
q.Analyzer = analyzer
|
||||||
return q
|
q.Fuzziness = fuzziness
|
||||||
}
|
|
||||||
|
|
||||||
// PrefixQuery generates a match prefix query for the given prefix and field
|
|
||||||
func PrefixQuery(matchPrefix, field string) *query.PrefixQuery {
|
|
||||||
q := bleve.NewPrefixQuery(matchPrefix)
|
|
||||||
q.FieldVal = field
|
|
||||||
return q
|
return q
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -35,7 +35,11 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
const maxBatchSize = 16
|
const (
|
||||||
|
maxBatchSize = 16
|
||||||
|
// fuzzyDenominator determines the levenshtein distance per each character of a keyword
|
||||||
|
fuzzyDenominator = 4
|
||||||
|
)
|
||||||
|
|
||||||
// IndexerData an update to the issue indexer
|
// IndexerData an update to the issue indexer
|
||||||
type IndexerData internal.IndexerData
|
type IndexerData internal.IndexerData
|
||||||
|
@ -156,19 +160,16 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
|
||||||
var queries []query.Query
|
var queries []query.Query
|
||||||
|
|
||||||
if options.Keyword != "" {
|
if options.Keyword != "" {
|
||||||
|
fuzziness := 0
|
||||||
if options.IsFuzzyKeyword {
|
if options.IsFuzzyKeyword {
|
||||||
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
|
fuzziness = len(options.Keyword) / fuzzyDenominator
|
||||||
inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer),
|
|
||||||
inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer),
|
|
||||||
inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer),
|
|
||||||
}...))
|
|
||||||
} else {
|
|
||||||
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
|
|
||||||
inner_bleve.PrefixQuery(options.Keyword, "title"),
|
|
||||||
inner_bleve.PrefixQuery(options.Keyword, "content"),
|
|
||||||
inner_bleve.PrefixQuery(options.Keyword, "comments"),
|
|
||||||
}...))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
|
||||||
|
inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness),
|
||||||
|
inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness),
|
||||||
|
inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness),
|
||||||
|
}...))
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(options.RepoIDs) > 0 || options.AllPublic {
|
if len(options.RepoIDs) > 0 || options.AllPublic {
|
||||||
|
|
|
@ -46,7 +46,7 @@ func testSearchRepo(t *testing.T, useExternalIndexer bool) {
|
||||||
|
|
||||||
if useExternalIndexer {
|
if useExternalIndexer {
|
||||||
gitReference = "/commit/"
|
gitReference = "/commit/"
|
||||||
executeIndexer(t, repo, code_indexer.UpdateRepoIndexer)
|
code_indexer.UpdateRepoIndexer(repo)
|
||||||
}
|
}
|
||||||
|
|
||||||
testSearch(t, "/user2/repo1/search?q=Description&page=1", gitReference, []string{"README.md"})
|
testSearch(t, "/user2/repo1/search?q=Description&page=1", gitReference, []string{"README.md"})
|
||||||
|
@ -58,12 +58,14 @@ func testSearchRepo(t *testing.T, useExternalIndexer bool) {
|
||||||
repo, err = repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "glob")
|
repo, err = repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "glob")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
executeIndexer(t, repo, code_indexer.UpdateRepoIndexer)
|
code_indexer.UpdateRepoIndexer(repo)
|
||||||
|
|
||||||
testSearch(t, "/user2/glob/search?q=loren&page=1", gitReference, []string{"a.txt"})
|
testSearch(t, "/user2/glob/search?q=loren&page=1", gitReference, []string{"a.txt"})
|
||||||
testSearch(t, "/user2/glob/search?q=file3&page=1", gitReference, []string{"x/b.txt"})
|
testSearch(t, "/user2/glob/search?q=loren&page=1&t=match", gitReference, []string{"a.txt"})
|
||||||
testSearch(t, "/user2/glob/search?q=file4&page=1", gitReference, []string{})
|
testSearch(t, "/user2/glob/search?q=file3&page=1", gitReference, []string{"x/b.txt", "a.txt"})
|
||||||
testSearch(t, "/user2/glob/search?q=file5&page=1", gitReference, []string{})
|
testSearch(t, "/user2/glob/search?q=file3&page=1&t=match", gitReference, []string{"x/b.txt", "a.txt"})
|
||||||
|
testSearch(t, "/user2/glob/search?q=file4&page=1&t=match", gitReference, []string{"x/b.txt", "a.txt"})
|
||||||
|
testSearch(t, "/user2/glob/search?q=file5&page=1&t=match", gitReference, []string{"x/b.txt", "a.txt"})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -88,7 +90,3 @@ func testSearch(t *testing.T, url, gitRef string, expected []string) {
|
||||||
|
|
||||||
checkResultLinks(t, gitRef, doc)
|
checkResultLinks(t, gitRef, doc)
|
||||||
}
|
}
|
||||||
|
|
||||||
func executeIndexer(t *testing.T, repo *repo_model.Repository, op func(*repo_model.Repository)) {
|
|
||||||
op(repo)
|
|
||||||
}
|
|
||||||
|
|
Loading…
Reference in a new issue