Refactoring search index backend.

app/models/application_model/search_index_base.rb

@@ -17,27 +17,8 @@ returns
   def search_index_update_backend
     return if !self.class.search_index_support_config
 
-    # default ignored attributes
-    ignore_attributes = {}
-    if self.class.search_index_support_config[:ignore_attributes]
-      self.class.search_index_support_config[:ignore_attributes].each { |key, value|
-        ignore_attributes[key] = value
-      }
-    end
-
-    # for performance reasons, Model.search_index_reload will only collect if of object
-    # get whole data here
-    data = self.class.find(id)
-
-    # remove ignored attributes
-    attributes = data.attributes
-    ignore_attributes.each { |key, value|
-      next if value != true
-      attributes.delete(key.to_s)
-    }
-
     # fill up with search data
-    attributes = search_index_attribute_lookup(attributes, data)
+    attributes = search_index_attribute_lookup
     return if !attributes
 
     # update backend
@@ -53,16 +34,20 @@ get data to store in search index
 
 returns
 
-  result = true # false
+  result = {
+    attribute1: 'some value',
+    attribute2: ['value 1', 'value 2'],
+    ...
+  }
 
 =end
 
   def search_index_data
     attributes = {}
     %w(name note).each { |key|
-      if self[key] && !self[key].empty?
+      next if !self[key]
+      next if self[key].respond_to?('empty?') && self[key].empty?
       attributes[key] = self[key]
-      end
     }
     return if attributes.empty?
     attributes
@@ -74,8 +59,8 @@ returns
 
 lookup name of ref. objects
 
-  ticket = Ticket.find(123)
+  ticket = Ticket.find(3)
-  attributes = ticket.search_index_attribute_lookup(attributes, Ticket)
+  attributes = ticket.search_index_attribute_lookup
 
 returns
 
@@ -83,10 +68,10 @@ returns
 
 =end
 
-  def search_index_attribute_lookup(attributes, ref_object)
+  def search_index_attribute_lookup
 
-    attributes_new = {}
+    attributes = self.attributes
-    attributes.each { |key, value|
+    self.attributes.each { |key, value|
       next if !value
 
       # get attribute name
@@ -96,10 +81,10 @@ returns
       attribute_name = attribute_name[ 0, attribute_name.length - 3 ]
 
       # check if attribute method exists
-      next if !ref_object.respond_to?(attribute_name)
+      next if !respond_to?(attribute_name)
 
       # check if method has own class
-      relation_class = ref_object.send(attribute_name).class
+      relation_class = send(attribute_name).class
       next if !relation_class
 
       # lookup ref object
@@ -111,11 +96,34 @@ returns
       if relation_model.respond_to?('search_index_data')
         value = relation_model.send('search_index_data')
       end
+
+      if relation_model.respond_to?('name')
+        value = relation_model.send('name')
+      end
+
       next if !value
 
       # save name of ref object
-      attributes_new[ attribute_name ] = value
+      attributes[ attribute_name ] = value
     }
-    attributes_new.merge(attributes)
+
+    # default ignored attributes
+    config = self.class.search_index_support_config
+    if config
+      ignore_attributes = {}
+      if config[:ignore_attributes]
+        config[:ignore_attributes].each { |key, value|
+          ignore_attributes[key] = value
+        }
+      end
+
+      # remove ignored attributes
+      ignore_attributes.each { |key, value|
+        next if value != true
+        attributes.delete(key.to_s)
+      }
+    end
+
+    attributes
   end
 end

app/models/observer/transaction.rb

@@ -1,7 +1,7 @@
 # Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/
 
 class Observer::Transaction < ActiveRecord::Observer
-  observe :ticket, 'ticket::_article', :user, :organization
+  observe :ticket, 'ticket::_article', :user, :organization, :tag
 
   def self.commit(params = {})
 
@@ -206,13 +206,20 @@ class Observer::Transaction < ActiveRecord::Observer
     # do not send anything if nothing has changed
     return if real_changes.empty?
 
+    changed_by_id = nil
+    if record.respond_to?('updated_by_id')
+      changed_by_id = record.updated_by_id
+    else
+      changed_by_id = record.created_by_id
+    end
+
     e = {
       object: record.class.name,
       type: 'update',
       data: record,
       changes: real_changes,
       id: record.id,
-      user_id: record.updated_by_id,
+      user_id: changed_by_id,
     }
     EventBuffer.add('transaction', e)
   end

app/models/organization.rb

@@ -5,6 +5,7 @@ class Organization < ApplicationModel
   load 'organization/assets.rb'
   include Organization::Assets
   extend Organization::Search
+  load 'organization/search_index.rb'
   include Organization::SearchIndex
 
   has_and_belongs_to_many :users

app/models/organization/search_index.rb

@@ -8,7 +8,7 @@ class Organization
 lookup name of ref. objects
 
   organization = Organization.find(123)
-  attributes = organization.search_index_attribute_lookup(attributes, Organization)
+  attributes = organization.search_index_attribute_lookup
 
 returns
 
@@ -16,47 +16,17 @@ returns
 
 =end
 
-    def search_index_attribute_lookup(attributes, ref_object)
+    def search_index_attribute_lookup
-      attributes_new = {}
+      attributes = super
-      attributes.each { |key, value|
-        next if !value
 
-        # get attribute name
+      # add org members for search index data
-        attribute_name = key.to_s
+      attributes['members'] = []
-        next if attribute_name[-3, 3] != '_id'
-        attribute_name = attribute_name[ 0, attribute_name.length - 3 ]
-
-        # check if attribute method exists
-        next if !ref_object.respond_to?(attribute_name)
-
-        # check if method has own class
-        relation_class = ref_object.send(attribute_name).class
-        next if !relation_class
-
-        # lookup ref object
-        relation_model = relation_class.lookup(id: value)
-        next if !relation_model
-
-        # get name of ref object
-        value = nil
-        if relation_model.respond_to?('search_index_data')
-          value = relation_model.send('search_index_data')
-        end
-        next if !value
-
-        # save name of ref object
-        attributes_new[ attribute_name ] = value
-        attributes.delete(key)
-      }
-
-      # add org member for search index data
-      attributes['member'] = []
       users = User.where(organization_id: id)
       users.each { |user|
-        attributes['member'].push user.search_index_data
+        attributes['members'].push user.search_index_data
       }
 
-      attributes_new.merge(attributes)
+      attributes
     end
   end
 end

app/models/ticket/search.rb

@@ -108,10 +108,10 @@ returns
                       .where('groups.active = ?', true)
         group_condition = []
         groups.each { |group|
-          group_condition.push group.name
+          group_condition.push group.id
         }
         access_condition = {
-          'query_string' => { 'default_field' => 'Ticket.group.name', 'query' => "\"#{group_condition.join('" OR "')}\"" }
+          'query_string' => { 'default_field' => 'Ticket.group_id', 'query' => "\"#{group_condition.join('" OR "')}\"" }
         }
       else
         access_condition = if !current_user.organization || ( !current_user.organization.shared || current_user.organization.shared == false )

app/models/ticket/search_index.rb

@@ -3,48 +3,28 @@ module Ticket::SearchIndex
 
 =begin
 
-build and send data for search index to backend
+lookup name of ref. objects
 
   ticket = Ticket.find(123)
-  result = ticket.search_index_update_backend
+  result = ticket.search_index_attribute_lookup
 
 returns
 
-  result = true # false
+  attributes # object with lookup data
 
 =end
 
-  def search_index_update_backend
+  def search_index_attribute_lookup
-    return if !self.class.search_index_support_config
+    attributes = super
-
+    return if !attributes
-    # default ignored attributes
-    ignore_attributes = {}
-    if self.class.search_index_support_config[:ignore_attributes]
-      self.class.search_index_support_config[:ignore_attributes].each { |key, value|
-        ignore_attributes[key] = value
-      }
-    end
-
-    # for performance reasons, Model.search_index_reload will only collect if of object
-    # get whole data here
-    ticket = self.class.find(id)
-
-    # remove ignored attributes
-    attributes = ticket.attributes
-    ignore_attributes.each { |key, value|
-      next if value != true
-      attributes.delete( key.to_s )
-    }
 
+    # collect article data
     # add tags
     tags = Tag.tag_list(object: 'Ticket', o_id: id)
     if tags && !tags.empty?
       attributes[:tag] = tags
     end
 
-    # lookup attributes of ref. objects (normally name and note)
-    attributes = search_index_attribute_lookup( attributes, ticket )
-
     # list ignored file extentions
     attachments_ignore = Setting.get('es_attachment_ignore') || [ '.png', '.jpg', '.jpeg', '.mpeg', '.mpg', '.mov', '.bin', '.exe' ]
 
@@ -64,7 +44,7 @@ returns
       }
 
       # lookup attributes of ref. objects (normally name and note)
-      article_attributes = search_index_attribute_lookup( article_attributes, article )
+      article_attributes = article.search_index_attribute_lookup
 
       # index raw text body
       if article_attributes['content_type'] && article_attributes['content_type'] == 'text/html' && article_attributes['body']
@@ -98,7 +78,7 @@ returns
       attributes['articles'].push article_attributes
     }
 
-    return if !attributes
+    attributes
-    SearchIndexBackend.add(self.class.to_s, attributes)
   end
+
 end

app/models/user.rb

@@ -28,6 +28,8 @@ class User < ApplicationModel
   load 'user/assets.rb'
   include User::Assets
   extend User::Search
+  load 'user/search_index.rb'
+  include User::SearchIndex
 
   before_create   :check_name, :check_email, :check_login, :check_password, :check_preferences_default
   before_update   :check_password, :check_email, :check_login, :check_preferences_default

app/models/user/search_index.rb

@@ -0,0 +1,50 @@
+# Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/
+
+class User
+  module SearchIndex
+
+=begin
+
+get data to store in search index
+
+  user = User.find(2)
+  result = user.search_index_data
+
+returns
+
+  result = {
+    attribute1: 'some value',
+    attribute2: ['value 1', 'value 2'],
+    ...
+  }
+
+=end
+
+    def search_index_data
+      attributes = {}
+      self.attributes.each { |key, value|
+        next if key == 'created_at'
+        next if key == 'updated_at'
+        next if key == 'created_by_id'
+        next if key == 'updated_by_id'
+        next if key == 'preferences'
+        next if key == 'password'
+        next if !value
+        next if value.respond_to?('empty?') && value.empty?
+        attributes[key] = value
+      }
+      return if attributes.empty?
+
+      if attributes['organization_id']
+        organization = Organization.lookup(id: attributes['organization_id'])
+        if organization
+          attributes['organization'] = organization.name
+          attributes['organization_ref'] = organization.search_index_data
+        end
+      end
+
+      attributes
+    end
+
+  end
+end

test/integration/elasticsearch_test.rb

@@ -52,6 +52,7 @@ class ElasticsearchTest < ActiveSupport::TestCase
   roles = Role.where(name: 'Customer')
   organization1 = Organization.create_if_not_exists(
     name: 'Customer Organization Update',
+    note: 'some note',
     updated_by_id: 1,
     created_by_id: 1,
   )
@@ -93,8 +94,87 @@ class ElasticsearchTest < ActiveSupport::TestCase
     created_by_id: 1,
   )
 
+  # check search attributes
+  test 'a - objects' do
+
+    # user
+    attributes = agent.search_index_data
+    assert_equal('E', attributes['firstname'])
+    assert_equal('S', attributes['lastname'])
+    assert_equal('es-agent@example.com', attributes['email'])
+    assert_not(attributes['password'])
+    assert_not(attributes['organization'])
+
+    attributes = agent.search_index_attribute_lookup
+    assert_equal('E', attributes['firstname'])
+    assert_equal('S', attributes['lastname'])
+    assert_equal('es-agent@example.com', attributes['email'])
+    assert_not(attributes['password'])
+    assert_not(attributes['organization'])
+
+    # organization
+    attributes = organization1.search_index_data
+    assert_equal('Customer Organization Update', attributes['name'])
+    assert_equal('some note', attributes['note'])
+    assert_not(attributes['members'])
+
+    attributes = organization1.search_index_attribute_lookup
+    assert_equal('Customer Organization Update', attributes['name'])
+    assert_equal('some note', attributes['note'])
+    assert(attributes['members'])
+
+    # ticket/article
+    ticket1 = Ticket.create(
+      title: 'some title äöüß',
+      group: Group.lookup(name: 'Users'),
+      customer_id: customer1.id,
+      state: Ticket::State.lookup(name: 'new'),
+      priority: Ticket::Priority.lookup(name: '2 normal'),
+      updated_by_id: 1,
+      created_by_id: 1,
+    )
+    article1 = Ticket::Article.create(
+      ticket_id: ticket1.id,
+      from: 'some_sender@example.com',
+      to: 'some_recipient@example.com',
+      subject: 'some subject',
+      message_id: 'some@id',
+      body: 'some message',
+      internal: false,
+      sender: Ticket::Article::Sender.where(name: 'Customer').first,
+      type: Ticket::Article::Type.where(name: 'email').first,
+      updated_by_id: 1,
+      created_by_id: 1,
+    )
+
+    attributes = ticket1.search_index_attribute_lookup
+
+    assert_equal('Users', attributes['group'])
+    assert_equal('new', attributes['state'])
+    assert_equal('2 normal', attributes['priority'])
+
+    assert_equal('ES', attributes['customer']['firstname'])
+    assert_equal('Customer1', attributes['customer']['lastname'])
+    assert_equal('es-customer1@example.com', attributes['customer']['email'])
+    assert_not(attributes['customer']['password'])
+    assert_equal('Customer Organization Update', attributes['customer']['organization'])
+
+    assert_equal('-', attributes['owner']['login'])
+    assert_equal('-', attributes['owner']['firstname'])
+    assert_not(attributes['owner']['password'])
+    assert_not(attributes['owner']['organization'])
+
+    ticket1.destroy
+
+    # execute background jobs
+    Scheduler.worker(true)
+
+  end
+
   # check tickets and search it
-  test 'a - tickets' do
+  test 'b - tickets' do
+
+    system('rake searchindex:rebuild')
 
     ticket1 = Ticket.create(
       title: "some title\n äöüß",
@@ -398,7 +478,7 @@ class ElasticsearchTest < ActiveSupport::TestCase
   end
 
   # check users and search it
-  test 'b - users' do
+  test 'c - users' do
 
     # search as agent
     result = User.search(