Fixes issue #2785 - Reporting does not return all tickets when using values with that "divide" values (e. g. some_value).

* Fixed SearchIndexBackend.selectors() in case if value if condition containing e. g. ::, _ or spaces.
* Moved to standard elasticsearch field naming .keyword to save raw strings (in previous we used .raw which in not common)
* Reindexing of elasticsearch is needed (zammad run rake searchindex:rebuild)

lib/search_index_backend.rb

@@ -301,9 +301,9 @@ remove whole data from index
       next if value.blank?
       next if order_by&.at(index).blank?
 
-      # for sorting values use .raw values (no analyzer is used - plain values)
+      # for sorting values use .keyword values (no analyzer is used - plain values)
       if value !~ /\./ && value !~ /_(time|date|till|id|ids|at)$/
-        value += '.raw'
+        value += '.keyword'
       end
       result.push(
         value => {
@@ -413,8 +413,15 @@ example for aggregations within one year
       response.data['hits']['hits'].each do |item|
         ticket_ids.push item['_id']
       end
+
+      # in lower ES 6 versions, we get total count directly, in higher
+      # versions we need to pick it from total has
+      count = response.data['hits']['total']
+      if response.data['hits']['total'].class != Integer
+        count = response.data['hits']['total']['value']
+      end
       return {
-        count:      response.data['hits']['total'],
+        count:      count,
         ticket_ids: ticket_ids,
       }
     end
@@ -440,31 +447,47 @@ example for aggregations within one year
     if selector.present?
       selector.each do |key, data|
         key_tmp = key.sub(/^.+?\./, '')
+        wildcard_or_term = 'term'
         t = {}
 
-        # use .raw in cases where query contains ::
+        # use .keyword in case of compare exact values
-        if data['value'].is_a?(Array)
+        if data['operator'] == 'is' || data['operator'] == 'is not'
-          data['value'].each do |value|
+          if data['value'].is_a?(Array)
-            if value.is_a?(String) && value =~ /::/
+            data['value'].each do |value|
-              key_tmp += '.raw'
+              next if !value.is_a?(String) || value !~ /[A-z]/
+
+              wildcard_or_term = 'terms'
+              key_tmp += '.keyword'
               break
             end
+          elsif data['value'].is_a?(String) && /[A-z]/.match?(data['value'])
+            key_tmp += '.keyword'
+            wildcard_or_term = 'term'
           end
-        elsif data['value'].is_a?(String)
+        end
-          if /::/.match?(data['value'])
+
-            key_tmp += '.raw'
+        # use .keyword and wildcard search in cases where query contains non A-z chars
+        if data['operator'] == 'contains' || data['operator'] == 'contains not'
+          if data['value'].is_a?(Array)
+            data['value'].each_with_index do |value, index|
+              next if !value.is_a?(String) || value !~ /[A-z]/ || value !~ /\W/
+
+              data['value'][index] = "*#{value}*"
+              key_tmp += '.keyword'
+              wildcard_or_term = 'wildcards'
+              break
+            end
+          elsif data['value'].is_a?(String) && /[A-z]/.match?(data['value']) && data['value'] =~ /\W/
+            data['value'] = "*#{data['value']}*"
+            key_tmp += '.keyword'
+            wildcard_or_term = 'wildcard'
           end
         end
 
         # is/is not/contains/contains not
         if data['operator'] == 'is' || data['operator'] == 'is not' || data['operator'] == 'contains' || data['operator'] == 'contains not'
-          if data['value'].is_a?(Array)
+          t[wildcard_or_term] = {}
-            t[:terms] = {}
+          t[wildcard_or_term][key_tmp] = data['value']
-            t[:terms][key_tmp] = data['value']
-          else
-            t[:term] = {}
-            t[:term][key_tmp] = data['value']
-          end
           if data['operator'] == 'is' || data['operator'] == 'contains'
             query_must.push t
           elsif data['operator'] == 'is not' || data['operator'] == 'contains not'
@@ -581,6 +604,8 @@ example for aggregations within one year
       }
       sort[1] = '_score'
       data['sort'] = sort
+    else
+      data['sort'] = search_by_index_sort(options[:sort_by], options[:order_by])
     end
 
     data

lib/tasks/search_index_es.rake

@@ -197,7 +197,7 @@ def get_mapping_properties_object(object)
 
   # for elasticsearch 6.x and later
   string_type = 'text'
-  string_raw  = { 'type': 'keyword' }
+  string_raw  = { 'type': 'keyword', 'ignore_above': 5012 }
   boolean_raw = { 'type': 'boolean' }
 
   # for elasticsearch 5.6 and lower
@@ -212,7 +212,7 @@ def get_mapping_properties_object(object)
       result[name][:properties][key] = {
         type:   string_type,
         fields: {
-          raw: string_raw,
+          keyword: string_raw,
         }
       }
     elsif value.type == :integer
@@ -227,7 +227,7 @@ def get_mapping_properties_object(object)
       result[name][:properties][key] = {
         type:   'boolean',
         fields: {
-          raw: boolean_raw,
+          keyword: boolean_raw,
         }
       }
     elsif value.type == :binary

spec/lib/search_index_backend_spec.rb

@@ -161,4 +161,309 @@ RSpec.describe SearchIndexBackend, searchindex: true do
       end
     end
   end
+
+  describe '.selectors' do
+
+    let(:ticket1) { create :ticket, title: 'some-title1', state_id: 1 }
+    let(:ticket2) { create :ticket, title: 'some_title2', state_id: 4 }
+    let(:ticket3) { create :ticket, title: 'some::title3', state_id: 1 }
+    let(:ticket4) { create :ticket, title: 'phrase some-title4', state_id: 1 }
+    let(:ticket5) { create :ticket, title: 'phrase some_title5', state_id: 1 }
+    let(:ticket6) { create :ticket, title: 'phrase some::title6', state_id: 1 }
+    let(:ticket7) { create :ticket, title: 'some title7', state_id: 1 }
+
+    before do
+      Ticket.destroy_all # needed to remove not created tickets
+      described_class.add('Ticket', ticket1)
+      travel 1.second
+      described_class.add('Ticket', ticket2)
+      travel 1.second
+      described_class.add('Ticket', ticket3)
+      travel 1.second
+      described_class.add('Ticket', ticket4)
+      travel 1.second
+      described_class.add('Ticket', ticket5)
+      travel 1.second
+      described_class.add('Ticket', ticket6)
+      travel 1.second
+      described_class.add('Ticket', ticket7)
+      described_class.refresh
+    end
+
+    context 'query with contains' do
+      it 'finds records with containing phrase' do
+        result = described_class.selectors('Ticket',
+                                           {
+                                             'title' => {
+                                               'operator' => 'contains',
+                                               'value'    => 'phrase',
+                                             },
+                                           },
+                                           {},
+                                           {
+                                             field: 'created_at', # sort to verify result
+                                           })
+        expect(result).to eq({ count: 3, ticket_ids: [ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s] })
+      end
+
+      it 'finds records with containing some title7' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'contains',
+                                             'value'    => 'some title7',
+                                           })
+        expect(result).to eq({ count: 1, ticket_ids: [ticket7.id.to_s] })
+      end
+
+      it 'finds records with containing -' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'contains',
+                                             'value'    => 'some-title1',
+                                           })
+        expect(result).to eq({ count: 1, ticket_ids: [ticket1.id.to_s] })
+      end
+
+      it 'finds records with containing _' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'contains',
+                                             'value'    => 'some_title2',
+                                           })
+        expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
+      end
+
+      it 'finds records with containing ::' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'contains',
+                                             'value'    => 'some::title3',
+                                           })
+        expect(result).to eq({ count: 1, ticket_ids: [ticket3.id.to_s] })
+      end
+
+      it 'finds records with containing 4' do
+        result = described_class.selectors('Ticket',
+                                           'state_id' => {
+                                             'operator' => 'contains',
+                                             'value'    => 4,
+                                           })
+        expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
+      end
+
+      it 'finds records with containing "4"' do
+        result = described_class.selectors('Ticket',
+                                           'state_id' => {
+                                             'operator' => 'contains',
+                                             'value'    => '4',
+                                           })
+        expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
+      end
+    end
+
+    context 'query with contains not' do
+      it 'finds records with containing not phrase' do
+        result = described_class.selectors('Ticket',
+                                           {
+                                             'title' => {
+                                               'operator' => 'contains not',
+                                               'value'    => 'phrase',
+                                             },
+                                           },
+                                           {},
+                                           {
+                                             field: 'created_at', # sort to verify result
+                                           })
+        expect(result).to eq({ count: 4, ticket_ids: [ticket7.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
+      end
+
+      it 'finds records with containing not some title7' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'contains not',
+                                             'value'    => 'some title7',
+                                           })
+        expect(result).to eq({ count: 6, ticket_ids: [ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
+      end
+
+      it 'finds records with containing not -' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'contains not',
+                                             'value'    => 'some-title1',
+                                           })
+        expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s] })
+      end
+
+      it 'finds records with containing not _' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'contains not',
+                                             'value'    => 'some_title2',
+                                           })
+        expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
+      end
+
+      it 'finds records with containing not ::' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'contains not',
+                                             'value'    => 'some::title3',
+                                           })
+
+        expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
+      end
+
+      it 'finds records with containing not 4' do
+        result = described_class.selectors('Ticket',
+                                           'state_id' => {
+                                             'operator' => 'contains not',
+                                             'value'    => 4,
+                                           })
+
+        expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
+      end
+
+      it 'finds records with containing not "4"' do
+        result = described_class.selectors('Ticket',
+                                           'state_id' => {
+                                             'operator' => 'contains not',
+                                             'value'    => '4',
+                                           })
+        expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
+      end
+    end
+
+    context 'query with is' do
+      it 'finds records with is phrase' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'is',
+                                             'value'    => 'phrase',
+                                           })
+        expect(result).to eq({ count: 0, ticket_ids: [] })
+      end
+
+      it 'finds records with is some title7' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'is',
+                                             'value'    => 'some title7',
+                                           })
+        expect(result).to eq({ count: 1, ticket_ids: [ticket7.id.to_s] })
+      end
+
+      it 'finds records with is -' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'is',
+                                             'value'    => 'some-title1',
+                                           })
+        expect(result).to eq({ count: 1, ticket_ids: [ticket1.id.to_s] })
+      end
+
+      it 'finds records with is _' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'is',
+                                             'value'    => 'some_title2',
+                                           })
+        expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
+      end
+
+      it 'finds records with is ::' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'is',
+                                             'value'    => 'some::title3',
+                                           })
+        expect(result).to eq({ count: 1, ticket_ids: [ticket3.id.to_s] })
+      end
+
+      it 'finds records with is 4' do
+        result = described_class.selectors('Ticket',
+                                           'state_id' => {
+                                             'operator' => 'is',
+                                             'value'    => 4,
+                                           })
+        expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
+      end
+
+      it 'finds records with is "4"' do
+        result = described_class.selectors('Ticket',
+                                           'state_id' => {
+                                             'operator' => 'is',
+                                             'value'    => '4',
+                                           })
+
+        expect(result).to eq({ count: 1, ticket_ids: [ticket2.id.to_s] })
+      end
+    end
+
+    context 'query with is not' do
+      it 'finds records with is not phrase' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'is not',
+                                             'value'    => 'phrase',
+                                           })
+        expect(result).to eq({ count: 7, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
+      end
+
+      it 'finds records with is not some title7' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'is not',
+                                             'value'    => 'some title7',
+                                           })
+        expect(result).to eq({ count: 6, ticket_ids: [ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
+      end
+
+      it 'finds records with is not -' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'is not',
+                                             'value'    => 'some-title1',
+                                           })
+        expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket2.id.to_s] })
+      end
+
+      it 'finds records with is not _' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'is not',
+                                             'value'    => 'some_title2',
+                                           })
+        expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
+      end
+
+      it 'finds records with is not ::' do
+        result = described_class.selectors('Ticket',
+                                           'title' => {
+                                             'operator' => 'is not',
+                                             'value'    => 'some::title3',
+                                           })
+        expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket2.id.to_s, ticket1.id.to_s] })
+      end
+
+      it 'finds records with is not 4' do
+        result = described_class.selectors('Ticket',
+                                           'state_id' => {
+                                             'operator' => 'is not',
+                                             'value'    => 4,
+                                           })
+        expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
+      end
+
+      it 'finds records with is not "4"' do
+        result = described_class.selectors('Ticket',
+                                           'state_id' => {
+                                             'operator' => 'is not',
+                                             'value'    => '4',
+                                           })
+
+        expect(result).to eq({ count: 6, ticket_ids: [ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket3.id.to_s, ticket1.id.to_s] })
+      end
+    end
+  end
 end

test/integration/report_test.rb

@@ -1380,7 +1380,7 @@ class ReportTest < ActiveSupport::TestCase
     assert(result)
     assert_nil(result[:ticket_ids][0])
 
-    # search for test_category.raw to find values with :: in query
+    # search for test_category.keyword to find values with :: in query
     result = Report::TicketGenericTime.items(
       range_start: Time.zone.parse('2015-01-01T00:00:00Z'),
       range_end:   Time.zone.parse('2015-12-31T23:59:59Z'),

test/unit/ticket_selector_test.rb

@@ -1106,4 +1106,97 @@ class TicketSelectorTest < ActiveSupport::TestCase
     assert_equal(2, ticket_count)
   end
 
+  test 'ticket title with certain content' do
+    Ticket.create!(
+      title:         'some_title1',
+      group:         @group,
+      customer_id:   @customer1.id,
+      owner_id:      @agent1.id,
+      state:         Ticket::State.lookup(name: 'new'),
+      priority:      Ticket::Priority.lookup(name: '2 normal'),
+      created_at:    '2015-02-05 16:37:00',
+      updated_by_id: 1,
+      created_by_id: 1,
+    )
+    Ticket.create!(
+      title:         'some::title2',
+      group:         @group,
+      customer_id:   @customer1.id,
+      owner_id:      @agent1.id,
+      state:         Ticket::State.lookup(name: 'new'),
+      priority:      Ticket::Priority.lookup(name: '2 normal'),
+      created_at:    '2015-02-05 16:37:00',
+      updated_by_id: 1,
+      created_by_id: 1,
+    )
+    Ticket.create!(
+      title:         'some-title3',
+      group:         @group,
+      customer_id:   @customer1.id,
+      owner_id:      @agent1.id,
+      state:         Ticket::State.lookup(name: 'new'),
+      priority:      Ticket::Priority.lookup(name: '2 normal'),
+      created_at:    '2015-02-05 16:37:00',
+      updated_by_id: 1,
+      created_by_id: 1,
+    )
+
+    # search all with contains
+    condition = {
+      'ticket.title' => {
+        operator: 'contains',
+        value:    'some_title1',
+      },
+    }
+    ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
+    assert_equal(1, ticket_count)
+
+    condition = {
+      'ticket.title' => {
+        operator: 'contains',
+        value:    'some::title2',
+      },
+    }
+    ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
+    assert_equal(1, ticket_count)
+
+    condition = {
+      'ticket.title' => {
+        operator: 'contains',
+        value:    'some-title3',
+      },
+    }
+    ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
+    assert_equal(1, ticket_count)
+
+    # search all with is
+    condition = {
+      'ticket.title' => {
+        operator: 'is',
+        value:    'some_title1',
+      },
+    }
+    ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
+    assert_equal(1, ticket_count)
+
+    condition = {
+      'ticket.title' => {
+        operator: 'is',
+        value:    'some::title2',
+      },
+    }
+    ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
+    assert_equal(1, ticket_count)
+
+    condition = {
+      'ticket.title' => {
+        operator: 'is',
+        value:    'some-title3',
+      },
+    }
+    ticket_count, _tickets = Ticket.selectors(condition, limit: 10, current_user: @agent1)
+    assert_equal(1, ticket_count)
+
+  end
+
 end