Sutty/trabajo-afectivo
5
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fixes issue #2785 - Reporting does not return all tickets when using values with that "divide" values (e. g. some_value).

Browse source 
* Fixed SearchIndexBackend.selectors() in case if value if condition containing e. g. ::, _ or spaces.
* Moved to standard elasticsearch field naming .keyword to save raw strings (in previous we used .raw which in not common)
* Reindexing of elasticsearch is needed (zammad run rake searchindex:rebuild)
This commit is contained in:
Martin Edenhofer 2019-10-30 08:45:31 +01:00 committed by Thorsten Eckel
parent a343fb4f7a
commit f204b89fe3
5 changed files with 445 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
lib/search_index_backend.rb
View file

@ -301,9 +301,9 @@ remove whole data from index
next if value.blank?
next if order_by&.at(index).blank?
# for sorting values use .raw values (no analyzer is used - plain values)
# for sorting values use .keyword values (no analyzer is used - plain values)
if value !~ /\./ && value !~ /_(time|date|till|id|ids|at)$/
value += '.raw'
value += '.keyword'
end
result.push(
value => {
@ -413,8 +413,15 @@ example for aggregations within one year
response.data['hits']['hits'].each do |item|
ticket_ids.push item['_id']
end
# in lower ES 6 versions, we get total count directly, in higher
# versions we need to pick it from total has
count = response.data['hits']['total']
if response.data['hits']['total'].class != Integer
count = response.data['hits']['total']['value']
end
return {
count: response.data['hits']['total'],
count: count,
ticket_ids: ticket_ids,
}
end
@ -440,31 +447,47 @@ example for aggregations within one year
if selector.present?
selector.each do |key, data|
key_tmp = key.sub(/^.+?\./, '')
wildcard_or_term = 'term'
t = {}
# use .raw in cases where query contains ::
# use .keyword in case of compare exact values
if data['operator'] == 'is' || data['operator'] == 'is not'
if data['value'].is_a?(Array)
data['value'].each do |value|
if value.is_a?(String) && value =~ /::/
key_tmp += '.raw'
next if !value.is_a?(String) || value !~ /[A-z]/
wildcard_or_term = 'terms'
key_tmp += '.keyword'
break
end
elsif data['value'].is_a?(String) && /[A-z]/.match?(data['value'])
key_tmp += '.keyword'
wildcard_or_term = 'term'
end
elsif data['value'].is_a?(String)
if /::/.match?(data['value'])
key_tmp += '.raw'
end
# use .keyword and wildcard search in cases where query contains non A-z chars
if data['operator'] == 'contains' || data['operator'] == 'contains not'
if data['value'].is_a?(Array)
data['value'].each_with_index do |value, index|
next if !value.is_a?(String) || value !~ /[A-z]/ || value !~ /\W/
data['value'][index] = "*#{value}*"
key_tmp += '.keyword'
wildcard_or_term = 'wildcards'
break
end
elsif data['value'].is_a?(String) && /[A-z]/.match?(data['value']) && data['value'] =~ /\W/
data['value'] = "*#{data['value']}*"
key_tmp += '.keyword'
wildcard_or_term = 'wildcard'
end
end
# is/is not/contains/contains not
if data['operator'] == 'is' || data['operator'] == 'is not' || data['operator'] == 'contains' || data['operator'] == 'contains not'
if data['value'].is_a?(Array)
t[:terms] = {}
t[:terms][key_tmp] = data['value']
else
t[:term] = {}
t[:term][key_tmp] = data['value']
end
t[wildcard_or_term] = {}
t[wildcard_or_term][key_tmp] = data['value']
if data['operator'] == 'is' || data['operator'] == 'contains'
query_must.push t
elsif data['operator'] == 'is not' || data['operator'] == 'contains not'
@ -581,6 +604,8 @@ example for aggregations within one year
}
sort[1] = '_score'
data['sort'] = sort
else
data['sort'] = search_by_index_sort(options[:sort_by], options[:order_by])
end
data

6
lib/tasks/search_index_es.rake
View file

@ -197,7 +197,7 @@ def get_mapping_properties_object(object)
# for elasticsearch 6.x and later
string_type = 'text'
string_raw = { 'type': 'keyword' }
string_raw = { 'type': 'keyword', 'ignore_above': 5012 }
boolean_raw = { 'type': 'boolean' }
# for elasticsearch 5.6 and lower
@ -212,7 +212,7 @@ def get_mapping_properties_object(object)
result[name][:properties][key] = {
type: string_type,
fields: {
raw: string_raw,
keyword: string_raw,
}
}
elsif value.type == :integer
@ -227,7 +227,7 @@ def get_mapping_properties_object(object)
result[name][:properties][key] = {
type: 'boolean',
fields: {
raw: boolean_raw,
keyword: boolean_raw,
}
}
elsif value.type == :binary

305
spec/lib/search_index_backend_spec.rb
View file

@ -161,4 +161,309 @@ RSpec.describe SearchIndexBackend, searchindex: true do
end
end
end
describe '.selectors' do
let(:ticket1) { create :ticket, title: 'some-title1', state_id: 1 }
let(:ticket2) { create :ticket, title: 'some_title2', state_id: 4 }
let(:ticket3) { create :ticket, title: 'some::title3', state_id: 1 }
let(:ticket4) { create :ticket, title: 'phrase some-title4', state_id: 1 }
let(:ticket5) { create :ticket, title: 'phrase some_title5', state_id: 1 }
let(:ticket6) { create :ticket, title: 'phrase some::title6', state_id: 1 }
let(:ticket7) { create :ticket, title: 'some title7', state_id: 1 }
before do
Ticket.destroy_all # needed to remove not created tickets
described_class.add('Ticket', ticket1)
travel 1.second
described_class.add('Ticket', ticket2)
travel 1.second
described_class.add('Ticket', ticket3)
travel 1.second
described_class.add('Ticket', ticket4)
travel 1.second
described_class.add('Ticket', ticket5)
travel 1.second
described_class.add('Ticket', ticket6)
travel 1.second
described_class.add('Ticket', ticket7)
described_class.refresh
end
context 'query with contains' do
it 'finds records with containing phrase' do
result = described_class.selectors('Ticket',
{
'title' => {
'operator' => 'contains',
'value' => 'phrase',
},
},
{},
{
field: 'created_at', # sort to verify result
})
expect(result).to eq({ count: 3, ticket_ids: [ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s] })
end
it 'finds records with containing some title7' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'contains',
'value' => 'some title7',
})
expect(result).to eq({ count: 1, ticket_ids: [ticket7.id.to_s] })
end
it 'finds records with containing -' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'contains',
'value' => 'some-title1',
})
expect(result).to eq({ count: 1, ticket_ids: [ticket1.id.to_s] })
end
it 'finds records with containing _' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'contains',
'value' => 'some_title2',
})
expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
end
it 'finds records with containing ::' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'contains',
'value' => 'some::title3',
})
expect(result).to eq({ count: 1, ticket_ids: [ticket3.id.to_s] })
end
it 'finds records with containing 4' do
result = described_class.selectors('Ticket',
'state_id' => {
'operator' => 'contains',
'value' => 4,
})
expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
end
it 'finds records with containing "4"' do
result = described_class.selectors('Ticket',
'state_id' => {
'operator' => 'contains',
'value' => '4',
})
expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
end
end
context 'query with contains not' do
it 'finds records with containing not phrase' do
result = described_class.selectors('Ticket',
{
'title' => {
'operator' => 'contains not',
'value' => 'phrase',
},
},
{},
{
field: 'created_at', # sort to verify result
})
expect(result).to eq({ count: 4, ticket_ids: [ticket7.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
end
it 'finds records with containing not some title7' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'contains not',
'value' => 'some title7',
})
expect(result).to eq({ count: 6, ticket_ids: [ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
end
it 'finds records with containing not -' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'contains not',
'value' => 'some-title1',
})
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s] })
end
it 'finds records with containing not _' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'contains not',
'value' => 'some_title2',
})
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
end
it 'finds records with containing not ::' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'contains not',
'value' => 'some::title3',
})
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
end
it 'finds records with containing not 4' do
result = described_class.selectors('Ticket',
'state_id' => {
'operator' => 'contains not',
'value' => 4,
})
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
end
it 'finds records with containing not "4"' do
result = described_class.selectors('Ticket',
'state_id' => {
'operator' => 'contains not',
'value' => '4',
})
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
end
end
context 'query with is' do
it 'finds records with is phrase' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'is',
'value' => 'phrase',
})
expect(result).to eq({ count: 0, ticket_ids: [] })
end
it 'finds records with is some title7' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'is',
'value' => 'some title7',
})
expect(result).to eq({ count: 1, ticket_ids: [ticket7.id.to_s] })
end
it 'finds records with is -' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'is',
'value' => 'some-title1',
})
expect(result).to eq({ count: 1, ticket_ids: [ticket1.id.to_s] })
end
it 'finds records with is _' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'is',
'value' => 'some_title2',
})
expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
end
it 'finds records with is ::' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'is',
'value' => 'some::title3',
})
expect(result).to eq({ count: 1, ticket_ids: [ticket3.id.to_s] })
end
it 'finds records with is 4' do
result = described_class.selectors('Ticket',
'state_id' => {
'operator' => 'is',
'value' => 4,
})
expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
end
it 'finds records with is "4"' do
result = described_class.selectors('Ticket',
'state_id' => {
'operator' => 'is',
'value' => '4',
})
expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
end
end
context 'query with is not' do
it 'finds records with is not phrase' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'is not',
'value' => 'phrase',
})
expect(result).to eq({ count: 7, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
end
it 'finds records with is not some title7' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'is not',
'value' => 'some title7',
})
expect(result).to eq({ count: 6, ticket_ids: [ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
end
it 'finds records with is not -' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'is not',
'value' => 'some-title1',
})
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s] })
end
it 'finds records with is not _' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'is not',
'value' => 'some_title2',
})
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
end
it 'finds records with is not ::' do
result = described_class.selectors('Ticket',
'title' => {
'operator' => 'is not',
'value' => 'some::title3',
})
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
end
it 'finds records with is not 4' do
result = described_class.selectors('Ticket',
'state_id' => {
'operator' => 'is not',
'value' => 4,
})
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
end
it 'finds records with is not "4"' do
result = described_class.selectors('Ticket',
'state_id' => {
'operator' => 'is not',
'value' => '4',
})
expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
end
end
end
end

2
test/integration/report_test.rb
View file

@ -1380,7 +1380,7 @@ class ReportTest < ActiveSupport::TestCase
assert(result)
assert_nil(result[:ticket_ids][0])
# search for test_category.raw to find values with :: in query
# search for test_category.keyword to find values with :: in query
result = Report::TicketGenericTime.items(
range_start: Time.zone.parse('2015-01-01T00:00:00Z'),
range_end: Time.zone.parse('2015-12-31T23:59:59Z'),

93
test/unit/ticket_selector_test.rb
View file

@ -1106,4 +1106,97 @@ class TicketSelectorTest < ActiveSupport::TestCase
assert_equal(2, ticket_count)
end
test 'ticket title with certain content' do
Ticket.create!(
title: 'some_title1',
group: @group,
customer_id: @customer1.id,
owner_id: @agent1.id,
state: Ticket::State.lookup(name: 'new'),
priority: Ticket::Priority.lookup(name: '2 normal'),
created_at: '2015-02-05 16:37:00',
updated_by_id: 1,
created_by_id: 1,
)
Ticket.create!(
title: 'some::title2',
group: @group,
customer_id: @customer1.id,
owner_id: @agent1.id,
state: Ticket::State.lookup(name: 'new'),
priority: Ticket::Priority.lookup(name: '2 normal'),
created_at: '2015-02-05 16:37:00',
updated_by_id: 1,
created_by_id: 1,
)
Ticket.create!(
title: 'some-title3',
group: @group,
customer_id: @customer1.id,
owner_id: @agent1.id,
state: Ticket::State.lookup(name: 'new'),
priority: Ticket::Priority.lookup(name: '2 normal'),
created_at: '2015-02-05 16:37:00',
updated_by_id: 1,
created_by_id: 1,
)
# search all with contains
condition = {
'ticket.title' => {
operator: 'contains',
value: 'some_title1',
},
}
ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
assert_equal(1, ticket_count)
condition = {
'ticket.title' => {
operator: 'contains',
value: 'some::title2',
},
}
ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
assert_equal(1, ticket_count)
condition = {
'ticket.title' => {
operator: 'contains',
value: 'some-title3',
},
}
ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
assert_equal(1, ticket_count)
# search all with is
condition = {
'ticket.title' => {
operator: 'is',
value: 'some_title1',
},
}
ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
assert_equal(1, ticket_count)
condition = {
'ticket.title' => {
operator: 'is',
value: 'some::title2',
},
}
ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
assert_equal(1, ticket_count)
condition = {
'ticket.title' => {
operator: 'is',
value: 'some-title3',
},
}
ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
assert_equal(1, ticket_count)
end
end