Fixes issue #2785 - Reporting does not return all tickets when using values with that "divide" values (e. g. some_value).
* Fixed SearchIndexBackend.selectors() in case if value if condition containing e. g. ::, _ or spaces. * Moved to standard elasticsearch field naming .keyword to save raw strings (in previous we used .raw which in not common) * Reindexing of elasticsearch is needed (zammad run rake searchindex:rebuild)
This commit is contained in:
parent
a343fb4f7a
commit
f204b89fe3
5 changed files with 445 additions and 22 deletions
|
@ -301,9 +301,9 @@ remove whole data from index
|
|||
next if value.blank?
|
||||
next if order_by&.at(index).blank?
|
||||
|
||||
# for sorting values use .raw values (no analyzer is used - plain values)
|
||||
# for sorting values use .keyword values (no analyzer is used - plain values)
|
||||
if value !~ /\./ && value !~ /_(time|date|till|id|ids|at)$/
|
||||
value += '.raw'
|
||||
value += '.keyword'
|
||||
end
|
||||
result.push(
|
||||
value => {
|
||||
|
@ -413,8 +413,15 @@ example for aggregations within one year
|
|||
response.data['hits']['hits'].each do |item|
|
||||
ticket_ids.push item['_id']
|
||||
end
|
||||
|
||||
# in lower ES 6 versions, we get total count directly, in higher
|
||||
# versions we need to pick it from total has
|
||||
count = response.data['hits']['total']
|
||||
if response.data['hits']['total'].class != Integer
|
||||
count = response.data['hits']['total']['value']
|
||||
end
|
||||
return {
|
||||
count: response.data['hits']['total'],
|
||||
count: count,
|
||||
ticket_ids: ticket_ids,
|
||||
}
|
||||
end
|
||||
|
@ -440,31 +447,47 @@ example for aggregations within one year
|
|||
if selector.present?
|
||||
selector.each do |key, data|
|
||||
key_tmp = key.sub(/^.+?\./, '')
|
||||
wildcard_or_term = 'term'
|
||||
t = {}
|
||||
|
||||
# use .raw in cases where query contains ::
|
||||
if data['value'].is_a?(Array)
|
||||
data['value'].each do |value|
|
||||
if value.is_a?(String) && value =~ /::/
|
||||
key_tmp += '.raw'
|
||||
# use .keyword in case of compare exact values
|
||||
if data['operator'] == 'is' || data['operator'] == 'is not'
|
||||
if data['value'].is_a?(Array)
|
||||
data['value'].each do |value|
|
||||
next if !value.is_a?(String) || value !~ /[A-z]/
|
||||
|
||||
wildcard_or_term = 'terms'
|
||||
key_tmp += '.keyword'
|
||||
break
|
||||
end
|
||||
elsif data['value'].is_a?(String) && /[A-z]/.match?(data['value'])
|
||||
key_tmp += '.keyword'
|
||||
wildcard_or_term = 'term'
|
||||
end
|
||||
elsif data['value'].is_a?(String)
|
||||
if /::/.match?(data['value'])
|
||||
key_tmp += '.raw'
|
||||
end
|
||||
|
||||
# use .keyword and wildcard search in cases where query contains non A-z chars
|
||||
if data['operator'] == 'contains' || data['operator'] == 'contains not'
|
||||
if data['value'].is_a?(Array)
|
||||
data['value'].each_with_index do |value, index|
|
||||
next if !value.is_a?(String) || value !~ /[A-z]/ || value !~ /\W/
|
||||
|
||||
data['value'][index] = "*#{value}*"
|
||||
key_tmp += '.keyword'
|
||||
wildcard_or_term = 'wildcards'
|
||||
break
|
||||
end
|
||||
elsif data['value'].is_a?(String) && /[A-z]/.match?(data['value']) && data['value'] =~ /\W/
|
||||
data['value'] = "*#{data['value']}*"
|
||||
key_tmp += '.keyword'
|
||||
wildcard_or_term = 'wildcard'
|
||||
end
|
||||
end
|
||||
|
||||
# is/is not/contains/contains not
|
||||
if data['operator'] == 'is' || data['operator'] == 'is not' || data['operator'] == 'contains' || data['operator'] == 'contains not'
|
||||
if data['value'].is_a?(Array)
|
||||
t[:terms] = {}
|
||||
t[:terms][key_tmp] = data['value']
|
||||
else
|
||||
t[:term] = {}
|
||||
t[:term][key_tmp] = data['value']
|
||||
end
|
||||
t[wildcard_or_term] = {}
|
||||
t[wildcard_or_term][key_tmp] = data['value']
|
||||
if data['operator'] == 'is' || data['operator'] == 'contains'
|
||||
query_must.push t
|
||||
elsif data['operator'] == 'is not' || data['operator'] == 'contains not'
|
||||
|
@ -581,6 +604,8 @@ example for aggregations within one year
|
|||
}
|
||||
sort[1] = '_score'
|
||||
data['sort'] = sort
|
||||
else
|
||||
data['sort'] = search_by_index_sort(options[:sort_by], options[:order_by])
|
||||
end
|
||||
|
||||
data
|
||||
|
|
|
@ -197,7 +197,7 @@ def get_mapping_properties_object(object)
|
|||
|
||||
# for elasticsearch 6.x and later
|
||||
string_type = 'text'
|
||||
string_raw = { 'type': 'keyword' }
|
||||
string_raw = { 'type': 'keyword', 'ignore_above': 5012 }
|
||||
boolean_raw = { 'type': 'boolean' }
|
||||
|
||||
# for elasticsearch 5.6 and lower
|
||||
|
@ -212,7 +212,7 @@ def get_mapping_properties_object(object)
|
|||
result[name][:properties][key] = {
|
||||
type: string_type,
|
||||
fields: {
|
||||
raw: string_raw,
|
||||
keyword: string_raw,
|
||||
}
|
||||
}
|
||||
elsif value.type == :integer
|
||||
|
@ -227,7 +227,7 @@ def get_mapping_properties_object(object)
|
|||
result[name][:properties][key] = {
|
||||
type: 'boolean',
|
||||
fields: {
|
||||
raw: boolean_raw,
|
||||
keyword: boolean_raw,
|
||||
}
|
||||
}
|
||||
elsif value.type == :binary
|
||||
|
|
|
@ -161,4 +161,309 @@ RSpec.describe SearchIndexBackend, searchindex: true do
|
|||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '.selectors' do
|
||||
|
||||
let(:ticket1) { create :ticket, title: 'some-title1', state_id: 1 }
|
||||
let(:ticket2) { create :ticket, title: 'some_title2', state_id: 4 }
|
||||
let(:ticket3) { create :ticket, title: 'some::title3', state_id: 1 }
|
||||
let(:ticket4) { create :ticket, title: 'phrase some-title4', state_id: 1 }
|
||||
let(:ticket5) { create :ticket, title: 'phrase some_title5', state_id: 1 }
|
||||
let(:ticket6) { create :ticket, title: 'phrase some::title6', state_id: 1 }
|
||||
let(:ticket7) { create :ticket, title: 'some title7', state_id: 1 }
|
||||
|
||||
before do
|
||||
Ticket.destroy_all # needed to remove not created tickets
|
||||
described_class.add('Ticket', ticket1)
|
||||
travel 1.second
|
||||
described_class.add('Ticket', ticket2)
|
||||
travel 1.second
|
||||
described_class.add('Ticket', ticket3)
|
||||
travel 1.second
|
||||
described_class.add('Ticket', ticket4)
|
||||
travel 1.second
|
||||
described_class.add('Ticket', ticket5)
|
||||
travel 1.second
|
||||
described_class.add('Ticket', ticket6)
|
||||
travel 1.second
|
||||
described_class.add('Ticket', ticket7)
|
||||
described_class.refresh
|
||||
end
|
||||
|
||||
context 'query with contains' do
|
||||
it 'finds records with containing phrase' do
|
||||
result = described_class.selectors('Ticket',
|
||||
{
|
||||
'title' => {
|
||||
'operator' => 'contains',
|
||||
'value' => 'phrase',
|
||||
},
|
||||
},
|
||||
{},
|
||||
{
|
||||
field: 'created_at', # sort to verify result
|
||||
})
|
||||
expect(result).to eq({ count: 3, ticket_ids: [ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with containing some title7' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'contains',
|
||||
'value' => 'some title7',
|
||||
})
|
||||
expect(result).to eq({ count: 1, ticket_ids: [ticket7.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with containing -' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'contains',
|
||||
'value' => 'some-title1',
|
||||
})
|
||||
expect(result).to eq({ count: 1, ticket_ids: [ticket1.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with containing _' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'contains',
|
||||
'value' => 'some_title2',
|
||||
})
|
||||
expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with containing ::' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'contains',
|
||||
'value' => 'some::title3',
|
||||
})
|
||||
expect(result).to eq({ count: 1, ticket_ids: [ticket3.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with containing 4' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'state_id' => {
|
||||
'operator' => 'contains',
|
||||
'value' => 4,
|
||||
})
|
||||
expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with containing "4"' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'state_id' => {
|
||||
'operator' => 'contains',
|
||||
'value' => '4',
|
||||
})
|
||||
expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
|
||||
end
|
||||
end
|
||||
|
||||
context 'query with contains not' do
|
||||
it 'finds records with containing not phrase' do
|
||||
result = described_class.selectors('Ticket',
|
||||
{
|
||||
'title' => {
|
||||
'operator' => 'contains not',
|
||||
'value' => 'phrase',
|
||||
},
|
||||
},
|
||||
{},
|
||||
{
|
||||
field: 'created_at', # sort to verify result
|
||||
})
|
||||
expect(result).to eq({ count: 4, ticket_ids: [ticket7.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with containing not some title7' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'contains not',
|
||||
'value' => 'some title7',
|
||||
})
|
||||
expect(result).to eq({ count: 6, ticket_ids: [ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with containing not -' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'contains not',
|
||||
'value' => 'some-title1',
|
||||
})
|
||||
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with containing not _' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'contains not',
|
||||
'value' => 'some_title2',
|
||||
})
|
||||
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with containing not ::' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'contains not',
|
||||
'value' => 'some::title3',
|
||||
})
|
||||
|
||||
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with containing not 4' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'state_id' => {
|
||||
'operator' => 'contains not',
|
||||
'value' => 4,
|
||||
})
|
||||
|
||||
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with containing not "4"' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'state_id' => {
|
||||
'operator' => 'contains not',
|
||||
'value' => '4',
|
||||
})
|
||||
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
|
||||
end
|
||||
end
|
||||
|
||||
context 'query with is' do
|
||||
it 'finds records with is phrase' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'is',
|
||||
'value' => 'phrase',
|
||||
})
|
||||
expect(result).to eq({ count: 0, ticket_ids: [] })
|
||||
end
|
||||
|
||||
it 'finds records with is some title7' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'is',
|
||||
'value' => 'some title7',
|
||||
})
|
||||
expect(result).to eq({ count: 1, ticket_ids: [ticket7.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with is -' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'is',
|
||||
'value' => 'some-title1',
|
||||
})
|
||||
expect(result).to eq({ count: 1, ticket_ids: [ticket1.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with is _' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'is',
|
||||
'value' => 'some_title2',
|
||||
})
|
||||
expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with is ::' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'is',
|
||||
'value' => 'some::title3',
|
||||
})
|
||||
expect(result).to eq({ count: 1, ticket_ids: [ticket3.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with is 4' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'state_id' => {
|
||||
'operator' => 'is',
|
||||
'value' => 4,
|
||||
})
|
||||
expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with is "4"' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'state_id' => {
|
||||
'operator' => 'is',
|
||||
'value' => '4',
|
||||
})
|
||||
|
||||
expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
|
||||
end
|
||||
end
|
||||
|
||||
context 'query with is not' do
|
||||
it 'finds records with is not phrase' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'is not',
|
||||
'value' => 'phrase',
|
||||
})
|
||||
expect(result).to eq({ count: 7, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with is not some title7' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'is not',
|
||||
'value' => 'some title7',
|
||||
})
|
||||
expect(result).to eq({ count: 6, ticket_ids: [ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with is not -' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'is not',
|
||||
'value' => 'some-title1',
|
||||
})
|
||||
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with is not _' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'is not',
|
||||
'value' => 'some_title2',
|
||||
})
|
||||
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with is not ::' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'title' => {
|
||||
'operator' => 'is not',
|
||||
'value' => 'some::title3',
|
||||
})
|
||||
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with is not 4' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'state_id' => {
|
||||
'operator' => 'is not',
|
||||
'value' => 4,
|
||||
})
|
||||
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
|
||||
end
|
||||
|
||||
it 'finds records with is not "4"' do
|
||||
result = described_class.selectors('Ticket',
|
||||
'state_id' => {
|
||||
'operator' => 'is not',
|
||||
'value' => '4',
|
||||
})
|
||||
|
||||
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -1380,7 +1380,7 @@ class ReportTest < ActiveSupport::TestCase
|
|||
assert(result)
|
||||
assert_nil(result[:ticket_ids][0])
|
||||
|
||||
# search for test_category.raw to find values with :: in query
|
||||
# search for test_category.keyword to find values with :: in query
|
||||
result = Report::TicketGenericTime.items(
|
||||
range_start: Time.zone.parse('2015-01-01T00:00:00Z'),
|
||||
range_end: Time.zone.parse('2015-12-31T23:59:59Z'),
|
||||
|
|
|
@ -1106,4 +1106,97 @@ class TicketSelectorTest < ActiveSupport::TestCase
|
|||
assert_equal(2, ticket_count)
|
||||
end
|
||||
|
||||
test 'ticket title with certain content' do
|
||||
Ticket.create!(
|
||||
title: 'some_title1',
|
||||
group: @group,
|
||||
customer_id: @customer1.id,
|
||||
owner_id: @agent1.id,
|
||||
state: Ticket::State.lookup(name: 'new'),
|
||||
priority: Ticket::Priority.lookup(name: '2 normal'),
|
||||
created_at: '2015-02-05 16:37:00',
|
||||
updated_by_id: 1,
|
||||
created_by_id: 1,
|
||||
)
|
||||
Ticket.create!(
|
||||
title: 'some::title2',
|
||||
group: @group,
|
||||
customer_id: @customer1.id,
|
||||
owner_id: @agent1.id,
|
||||
state: Ticket::State.lookup(name: 'new'),
|
||||
priority: Ticket::Priority.lookup(name: '2 normal'),
|
||||
created_at: '2015-02-05 16:37:00',
|
||||
updated_by_id: 1,
|
||||
created_by_id: 1,
|
||||
)
|
||||
Ticket.create!(
|
||||
title: 'some-title3',
|
||||
group: @group,
|
||||
customer_id: @customer1.id,
|
||||
owner_id: @agent1.id,
|
||||
state: Ticket::State.lookup(name: 'new'),
|
||||
priority: Ticket::Priority.lookup(name: '2 normal'),
|
||||
created_at: '2015-02-05 16:37:00',
|
||||
updated_by_id: 1,
|
||||
created_by_id: 1,
|
||||
)
|
||||
|
||||
# search all with contains
|
||||
condition = {
|
||||
'ticket.title' => {
|
||||
operator: 'contains',
|
||||
value: 'some_title1',
|
||||
},
|
||||
}
|
||||
ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
|
||||
assert_equal(1, ticket_count)
|
||||
|
||||
condition = {
|
||||
'ticket.title' => {
|
||||
operator: 'contains',
|
||||
value: 'some::title2',
|
||||
},
|
||||
}
|
||||
ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
|
||||
assert_equal(1, ticket_count)
|
||||
|
||||
condition = {
|
||||
'ticket.title' => {
|
||||
operator: 'contains',
|
||||
value: 'some-title3',
|
||||
},
|
||||
}
|
||||
ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
|
||||
assert_equal(1, ticket_count)
|
||||
|
||||
# search all with is
|
||||
condition = {
|
||||
'ticket.title' => {
|
||||
operator: 'is',
|
||||
value: 'some_title1',
|
||||
},
|
||||
}
|
||||
ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
|
||||
assert_equal(1, ticket_count)
|
||||
|
||||
condition = {
|
||||
'ticket.title' => {
|
||||
operator: 'is',
|
||||
value: 'some::title2',
|
||||
},
|
||||
}
|
||||
ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
|
||||
assert_equal(1, ticket_count)
|
||||
|
||||
condition = {
|
||||
'ticket.title' => {
|
||||
operator: 'is',
|
||||
value: 'some-title3',
|
||||
},
|
||||
}
|
||||
ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
|
||||
assert_equal(1, ticket_count)
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
|
|
Loading…
Reference in a new issue