diff --git a/lib/search_index_backend.rb b/lib/search_index_backend.rb index ede46b495..22d5ff366 100644 --- a/lib/search_index_backend.rb +++ b/lib/search_index_backend.rb @@ -301,9 +301,9 @@ remove whole data from index next if value.blank? next if order_by&.at(index).blank? - # for sorting values use .raw values (no analyzer is used - plain values) + # for sorting values use .keyword values (no analyzer is used - plain values) if value !~ /\./ && value !~ /_(time|date|till|id|ids|at)$/ - value += '.raw' + value += '.keyword' end result.push( value => { @@ -413,8 +413,15 @@ example for aggregations within one year response.data['hits']['hits'].each do |item| ticket_ids.push item['_id'] end + + # in lower ES 6 versions, we get total count directly, in higher + # versions we need to pick it from total has + count = response.data['hits']['total'] + if response.data['hits']['total'].class != Integer + count = response.data['hits']['total']['value'] + end return { - count: response.data['hits']['total'], + count: count, ticket_ids: ticket_ids, } end @@ -440,31 +447,47 @@ example for aggregations within one year if selector.present? selector.each do |key, data| key_tmp = key.sub(/^.+?\./, '') + wildcard_or_term = 'term' t = {} - # use .raw in cases where query contains :: - if data['value'].is_a?(Array) - data['value'].each do |value| - if value.is_a?(String) && value =~ /::/ - key_tmp += '.raw' + # use .keyword in case of compare exact values + if data['operator'] == 'is' || data['operator'] == 'is not' + if data['value'].is_a?(Array) + data['value'].each do |value| + next if !value.is_a?(String) || value !~ /[A-z]/ + + wildcard_or_term = 'terms' + key_tmp += '.keyword' break end + elsif data['value'].is_a?(String) && /[A-z]/.match?(data['value']) + key_tmp += '.keyword' + wildcard_or_term = 'term' end - elsif data['value'].is_a?(String) - if /::/.match?(data['value']) - key_tmp += '.raw' + end + + # use .keyword and wildcard search in cases where query contains non A-z chars + if data['operator'] == 'contains' || data['operator'] == 'contains not' + if data['value'].is_a?(Array) + data['value'].each_with_index do |value, index| + next if !value.is_a?(String) || value !~ /[A-z]/ || value !~ /\W/ + + data['value'][index] = "*#{value}*" + key_tmp += '.keyword' + wildcard_or_term = 'wildcards' + break + end + elsif data['value'].is_a?(String) && /[A-z]/.match?(data['value']) && data['value'] =~ /\W/ + data['value'] = "*#{data['value']}*" + key_tmp += '.keyword' + wildcard_or_term = 'wildcard' end end # is/is not/contains/contains not if data['operator'] == 'is' || data['operator'] == 'is not' || data['operator'] == 'contains' || data['operator'] == 'contains not' - if data['value'].is_a?(Array) - t[:terms] = {} - t[:terms][key_tmp] = data['value'] - else - t[:term] = {} - t[:term][key_tmp] = data['value'] - end + t[wildcard_or_term] = {} + t[wildcard_or_term][key_tmp] = data['value'] if data['operator'] == 'is' || data['operator'] == 'contains' query_must.push t elsif data['operator'] == 'is not' || data['operator'] == 'contains not' @@ -581,6 +604,8 @@ example for aggregations within one year } sort[1] = '_score' data['sort'] = sort + else + data['sort'] = search_by_index_sort(options[:sort_by], options[:order_by]) end data diff --git a/lib/tasks/search_index_es.rake b/lib/tasks/search_index_es.rake index 13298a63c..6d588d394 100644 --- a/lib/tasks/search_index_es.rake +++ b/lib/tasks/search_index_es.rake @@ -197,7 +197,7 @@ def get_mapping_properties_object(object) # for elasticsearch 6.x and later string_type = 'text' - string_raw = { 'type': 'keyword' } + string_raw = { 'type': 'keyword', 'ignore_above': 5012 } boolean_raw = { 'type': 'boolean' } # for elasticsearch 5.6 and lower @@ -212,7 +212,7 @@ def get_mapping_properties_object(object) result[name][:properties][key] = { type: string_type, fields: { - raw: string_raw, + keyword: string_raw, } } elsif value.type == :integer @@ -227,7 +227,7 @@ def get_mapping_properties_object(object) result[name][:properties][key] = { type: 'boolean', fields: { - raw: boolean_raw, + keyword: boolean_raw, } } elsif value.type == :binary diff --git a/spec/lib/search_index_backend_spec.rb b/spec/lib/search_index_backend_spec.rb index 9773e9bab..10eed4fee 100644 --- a/spec/lib/search_index_backend_spec.rb +++ b/spec/lib/search_index_backend_spec.rb @@ -161,4 +161,309 @@ RSpec.describe SearchIndexBackend, searchindex: true do end end end + + describe '.selectors' do + + let(:ticket1) { create :ticket, title: 'some-title1', state_id: 1 } + let(:ticket2) { create :ticket, title: 'some_title2', state_id: 4 } + let(:ticket3) { create :ticket, title: 'some::title3', state_id: 1 } + let(:ticket4) { create :ticket, title: 'phrase some-title4', state_id: 1 } + let(:ticket5) { create :ticket, title: 'phrase some_title5', state_id: 1 } + let(:ticket6) { create :ticket, title: 'phrase some::title6', state_id: 1 } + let(:ticket7) { create :ticket, title: 'some title7', state_id: 1 } + + before do + Ticket.destroy_all # needed to remove not created tickets + described_class.add('Ticket', ticket1) + travel 1.second + described_class.add('Ticket', ticket2) + travel 1.second + described_class.add('Ticket', ticket3) + travel 1.second + described_class.add('Ticket', ticket4) + travel 1.second + described_class.add('Ticket', ticket5) + travel 1.second + described_class.add('Ticket', ticket6) + travel 1.second + described_class.add('Ticket', ticket7) + described_class.refresh + end + + context 'query with contains' do + it 'finds records with containing phrase' do + result = described_class.selectors('Ticket', + { + 'title' => { + 'operator' => 'contains', + 'value' => 'phrase', + }, + }, + {}, + { + field: 'created_at', # sort to verify result + }) + expect(result).to eq({ count: 3, ticket_ids: [ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s] }) + end + + it 'finds records with containing some title7' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'contains', + 'value' => 'some title7', + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket7.id.to_s] }) + end + + it 'finds records with containing -' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'contains', + 'value' => 'some-title1', + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket1.id.to_s] }) + end + + it 'finds records with containing _' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'contains', + 'value' => 'some_title2', + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] }) + end + + it 'finds records with containing ::' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'contains', + 'value' => 'some::title3', + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket3.id.to_s] }) + end + + it 'finds records with containing 4' do + result = described_class.selectors('Ticket', + 'state_id' => { + 'operator' => 'contains', + 'value' => 4, + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] }) + end + + it 'finds records with containing "4"' do + result = described_class.selectors('Ticket', + 'state_id' => { + 'operator' => 'contains', + 'value' => '4', + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] }) + end + end + + context 'query with contains not' do + it 'finds records with containing not phrase' do + result = described_class.selectors('Ticket', + { + 'title' => { + 'operator' => 'contains not', + 'value' => 'phrase', + }, + }, + {}, + { + field: 'created_at', # sort to verify result + }) + expect(result).to eq({ count: 4, ticket_ids: [ticket7.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] }) + end + + it 'finds records with containing not some title7' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'contains not', + 'value' => 'some title7', + }) + expect(result).to eq({ count: 6, ticket_ids: [ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] }) + end + + it 'finds records with containing not -' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'contains not', + 'value' => 'some-title1', + }) + expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s] }) + end + + it 'finds records with containing not _' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'contains not', + 'value' => 'some_title2', + }) + expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] }) + end + + it 'finds records with containing not ::' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'contains not', + 'value' => 'some::title3', + }) + + expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket2.id.to_s, ticket1.id.to_s] }) + end + + it 'finds records with containing not 4' do + result = described_class.selectors('Ticket', + 'state_id' => { + 'operator' => 'contains not', + 'value' => 4, + }) + + expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] }) + end + + it 'finds records with containing not "4"' do + result = described_class.selectors('Ticket', + 'state_id' => { + 'operator' => 'contains not', + 'value' => '4', + }) + expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] }) + end + end + + context 'query with is' do + it 'finds records with is phrase' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'is', + 'value' => 'phrase', + }) + expect(result).to eq({ count: 0, ticket_ids: [] }) + end + + it 'finds records with is some title7' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'is', + 'value' => 'some title7', + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket7.id.to_s] }) + end + + it 'finds records with is -' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'is', + 'value' => 'some-title1', + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket1.id.to_s] }) + end + + it 'finds records with is _' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'is', + 'value' => 'some_title2', + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] }) + end + + it 'finds records with is ::' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'is', + 'value' => 'some::title3', + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket3.id.to_s] }) + end + + it 'finds records with is 4' do + result = described_class.selectors('Ticket', + 'state_id' => { + 'operator' => 'is', + 'value' => 4, + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] }) + end + + it 'finds records with is "4"' do + result = described_class.selectors('Ticket', + 'state_id' => { + 'operator' => 'is', + 'value' => '4', + }) + + expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] }) + end + end + + context 'query with is not' do + it 'finds records with is not phrase' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'is not', + 'value' => 'phrase', + }) + expect(result).to eq({ count: 7, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] }) + end + + it 'finds records with is not some title7' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'is not', + 'value' => 'some title7', + }) + expect(result).to eq({ count: 6, ticket_ids: [ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] }) + end + + it 'finds records with is not -' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'is not', + 'value' => 'some-title1', + }) + expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s] }) + end + + it 'finds records with is not _' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'is not', + 'value' => 'some_title2', + }) + expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] }) + end + + it 'finds records with is not ::' do + result = described_class.selectors('Ticket', + 'title' => { + 'operator' => 'is not', + 'value' => 'some::title3', + }) + expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket2.id.to_s, ticket1.id.to_s] }) + end + + it 'finds records with is not 4' do + result = described_class.selectors('Ticket', + 'state_id' => { + 'operator' => 'is not', + 'value' => 4, + }) + expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] }) + end + + it 'finds records with is not "4"' do + result = described_class.selectors('Ticket', + 'state_id' => { + 'operator' => 'is not', + 'value' => '4', + }) + + expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] }) + end + end + end end diff --git a/test/integration/report_test.rb b/test/integration/report_test.rb index 0e1638fc2..6deb899d3 100644 --- a/test/integration/report_test.rb +++ b/test/integration/report_test.rb @@ -1380,7 +1380,7 @@ class ReportTest < ActiveSupport::TestCase assert(result) assert_nil(result[:ticket_ids][0]) - # search for test_category.raw to find values with :: in query + # search for test_category.keyword to find values with :: in query result = Report::TicketGenericTime.items( range_start: Time.zone.parse('2015-01-01T00:00:00Z'), range_end: Time.zone.parse('2015-12-31T23:59:59Z'), diff --git a/test/unit/ticket_selector_test.rb b/test/unit/ticket_selector_test.rb index 2b9000684..6baec5784 100644 --- a/test/unit/ticket_selector_test.rb +++ b/test/unit/ticket_selector_test.rb @@ -1106,4 +1106,97 @@ class TicketSelectorTest < ActiveSupport::TestCase assert_equal(2, ticket_count) end + test 'ticket title with certain content' do + Ticket.create!( + title: 'some_title1', + group: @group, + customer_id: @customer1.id, + owner_id: @agent1.id, + state: Ticket::State.lookup(name: 'new'), + priority: Ticket::Priority.lookup(name: '2 normal'), + created_at: '2015-02-05 16:37:00', + updated_by_id: 1, + created_by_id: 1, + ) + Ticket.create!( + title: 'some::title2', + group: @group, + customer_id: @customer1.id, + owner_id: @agent1.id, + state: Ticket::State.lookup(name: 'new'), + priority: Ticket::Priority.lookup(name: '2 normal'), + created_at: '2015-02-05 16:37:00', + updated_by_id: 1, + created_by_id: 1, + ) + Ticket.create!( + title: 'some-title3', + group: @group, + customer_id: @customer1.id, + owner_id: @agent1.id, + state: Ticket::State.lookup(name: 'new'), + priority: Ticket::Priority.lookup(name: '2 normal'), + created_at: '2015-02-05 16:37:00', + updated_by_id: 1, + created_by_id: 1, + ) + + # search all with contains + condition = { + 'ticket.title' => { + operator: 'contains', + value: 'some_title1', + }, + } + ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1) + assert_equal(1, ticket_count) + + condition = { + 'ticket.title' => { + operator: 'contains', + value: 'some::title2', + }, + } + ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1) + assert_equal(1, ticket_count) + + condition = { + 'ticket.title' => { + operator: 'contains', + value: 'some-title3', + }, + } + ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1) + assert_equal(1, ticket_count) + + # search all with is + condition = { + 'ticket.title' => { + operator: 'is', + value: 'some_title1', + }, + } + ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1) + assert_equal(1, ticket_count) + + condition = { + 'ticket.title' => { + operator: 'is', + value: 'some::title2', + }, + } + ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1) + assert_equal(1, ticket_count) + + condition = { + 'ticket.title' => { + operator: 'is', + value: 'some-title3', + }, + } + ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1) + assert_equal(1, ticket_count) + + end + end