Sutty/trabajo-afectivo
5
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Refactoring search index backend.

Browse source
This commit is contained in:
Martin Edenhofer 2016-07-06 08:13:44 +02:00
parent f6d0fb3596
commit 7f1ecaa67e
10 changed files with 473 additions and 375 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
app/models/application_model/search_index_base.rb
View file

@ -17,27 +17,8 @@ returns
def search_index_update_backend
return if !self.class.search_index_support_config
# default ignored attributes
ignore_attributes = {}
if self.class.search_index_support_config[:ignore_attributes]
self.class.search_index_support_config[:ignore_attributes].each { |key, value|
ignore_attributes[key] = value
}
end
# for performance reasons, Model.search_index_reload will only collect if of object
# get whole data here
data = self.class.find(id)
# remove ignored attributes
attributes = data.attributes
ignore_attributes.each { |key, value|
next if value != true
attributes.delete(key.to_s)
}
# fill up with search data
attributes = search_index_attribute_lookup(attributes, data)
attributes = search_index_attribute_lookup
return if !attributes
# update backend
@ -53,16 +34,20 @@ get data to store in search index
returns
result = true # false
result = {
attribute1: 'some value',
attribute2: ['value 1', 'value 2'],
...
}
=end
def search_index_data
attributes = {}
%w(name note).each { |key|
if self[key] && !self[key].empty?
next if !self[key]
next if self[key].respond_to?('empty?') && self[key].empty?
attributes[key] = self[key]
end
}
return if attributes.empty?
attributes
@ -74,8 +59,8 @@ returns
lookup name of ref. objects
ticket = Ticket.find(123)
attributes = ticket.search_index_attribute_lookup(attributes, Ticket)
ticket = Ticket.find(3)
attributes = ticket.search_index_attribute_lookup
returns
@ -83,10 +68,10 @@ returns
=end
def search_index_attribute_lookup(attributes, ref_object)
def search_index_attribute_lookup
attributes_new = {}
attributes.each { |key, value|
attributes = self.attributes
self.attributes.each { |key, value|
next if !value
# get attribute name
@ -96,10 +81,10 @@ returns
attribute_name = attribute_name[ 0, attribute_name.length - 3 ]
# check if attribute method exists
next if !ref_object.respond_to?(attribute_name)
next if !respond_to?(attribute_name)
# check if method has own class
relation_class = ref_object.send(attribute_name).class
relation_class = send(attribute_name).class
next if !relation_class
# lookup ref object
@ -111,11 +96,34 @@ returns
if relation_model.respond_to?('search_index_data')
value = relation_model.send('search_index_data')
end
if relation_model.respond_to?('name')
value = relation_model.send('name')
end
next if !value
# save name of ref object
attributes_new[ attribute_name ] = value
attributes[ attribute_name ] = value
}
attributes_new.merge(attributes)
# default ignored attributes
config = self.class.search_index_support_config
if config
ignore_attributes = {}
if config[:ignore_attributes]
config[:ignore_attributes].each { |key, value|
ignore_attributes[key] = value
}
end
# remove ignored attributes
ignore_attributes.each { |key, value|
next if value != true
attributes.delete(key.to_s)
}
end
attributes
end
end

11
app/models/observer/transaction.rb
View file

@ -1,7 +1,7 @@
# Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/
class Observer::Transaction < ActiveRecord::Observer
observe :ticket, 'ticket::_article', :user, :organization
observe :ticket, 'ticket::_article', :user, :organization, :tag
def self.commit(params = {})
@ -206,13 +206,20 @@ class Observer::Transaction < ActiveRecord::Observer
# do not send anything if nothing has changed
return if real_changes.empty?
changed_by_id = nil
if record.respond_to?('updated_by_id')
changed_by_id = record.updated_by_id
else
changed_by_id = record.created_by_id
end
e = {
object: record.class.name,
type: 'update',
data: record,
changes: real_changes,
id: record.id,
user_id: record.updated_by_id,
user_id: changed_by_id,
}
EventBuffer.add('transaction', e)
end

1
app/models/organization.rb
View file

@ -5,6 +5,7 @@ class Organization < ApplicationModel
load 'organization/assets.rb'
include Organization::Assets
extend Organization::Search
load 'organization/search_index.rb'
include Organization::SearchIndex
has_and_belongs_to_many :users

44
app/models/organization/search_index.rb
View file

@ -8,7 +8,7 @@ class Organization
lookup name of ref. objects
organization = Organization.find(123)
attributes = organization.search_index_attribute_lookup(attributes, Organization)
attributes = organization.search_index_attribute_lookup
returns
@ -16,47 +16,17 @@ returns
=end
def search_index_attribute_lookup(attributes, ref_object)
attributes_new = {}
attributes.each { |key, value|
next if !value
def search_index_attribute_lookup
attributes = super
# get attribute name
attribute_name = key.to_s
next if attribute_name[-3, 3] != '_id'
attribute_name = attribute_name[ 0, attribute_name.length - 3 ]
# check if attribute method exists
next if !ref_object.respond_to?(attribute_name)
# check if method has own class
relation_class = ref_object.send(attribute_name).class
next if !relation_class
# lookup ref object
relation_model = relation_class.lookup(id: value)
next if !relation_model
# get name of ref object
value = nil
if relation_model.respond_to?('search_index_data')
value = relation_model.send('search_index_data')
end
next if !value
# save name of ref object
attributes_new[ attribute_name ] = value
attributes.delete(key)
}
# add org member for search index data
attributes['member'] = []
# add org members for search index data
attributes['members'] = []
users = User.where(organization_id: id)
users.each { |user|
attributes['member'].push user.search_index_data
attributes['members'].push user.search_index_data
}
attributes_new.merge(attributes)
attributes
end
end
end

4
app/models/ticket/search.rb
View file

@ -108,10 +108,10 @@ returns
.where('groups.active = ?', true)
group_condition = []
groups.each { |group|
group_condition.push group.name
group_condition.push group.id
}
access_condition = {
'query_string' => { 'default_field' => 'Ticket.group.name', 'query' => "\"#{group_condition.join('" OR "')}\"" }
'query_string' => { 'default_field' => 'Ticket.group_id', 'query' => "\"#{group_condition.join('" OR "')}\"" }
}
else
access_condition = if !current_user.organization || ( !current_user.organization.shared || current_user.organization.shared == false )

40
app/models/ticket/search_index.rb
View file

@ -3,48 +3,28 @@ module Ticket::SearchIndex
=begin
build and send data for search index to backend
lookup name of ref. objects
ticket = Ticket.find(123)
result = ticket.search_index_update_backend
result = ticket.search_index_attribute_lookup
returns
result = true # false
attributes # object with lookup data
=end
def search_index_update_backend
return if !self.class.search_index_support_config
# default ignored attributes
ignore_attributes = {}
if self.class.search_index_support_config[:ignore_attributes]
self.class.search_index_support_config[:ignore_attributes].each { |key, value|
ignore_attributes[key] = value
}
end
# for performance reasons, Model.search_index_reload will only collect if of object
# get whole data here
ticket = self.class.find(id)
# remove ignored attributes
attributes = ticket.attributes
ignore_attributes.each { |key, value|
next if value != true
attributes.delete( key.to_s )
}
def search_index_attribute_lookup
attributes = super
return if !attributes
# collect article data
# add tags
tags = Tag.tag_list(object: 'Ticket', o_id: id)
if tags && !tags.empty?
attributes[:tag] = tags
end
# lookup attributes of ref. objects (normally name and note)
attributes = search_index_attribute_lookup( attributes, ticket )
# list ignored file extentions
attachments_ignore = Setting.get('es_attachment_ignore') || [ '.png', '.jpg', '.jpeg', '.mpeg', '.mpg', '.mov', '.bin', '.exe' ]
@ -64,7 +44,7 @@ returns
}
# lookup attributes of ref. objects (normally name and note)
article_attributes = search_index_attribute_lookup( article_attributes, article )
article_attributes = article.search_index_attribute_lookup
# index raw text body
if article_attributes['content_type'] && article_attributes['content_type'] == 'text/html' && article_attributes['body']
@ -98,7 +78,7 @@ returns
attributes['articles'].push article_attributes
}
return if !attributes
SearchIndexBackend.add(self.class.to_s, attributes)
attributes
end
end

2
app/models/user.rb
View file

@ -28,6 +28,8 @@ class User < ApplicationModel
load 'user/assets.rb'
include User::Assets
extend User::Search
load 'user/search_index.rb'
include User::SearchIndex
before_create :check_name, :check_email, :check_login, :check_password, :check_preferences_default
before_update :check_password, :check_email, :check_login, :check_preferences_default

50
app/models/user/search_index.rb Normal file
View file

@ -0,0 +1,50 @@
# Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/
class User
module SearchIndex
=begin
get data to store in search index
user = User.find(2)
result = user.search_index_data
returns
result = {
attribute1: 'some value',
attribute2: ['value 1', 'value 2'],
...
}
=end
def search_index_data
attributes = {}
self.attributes.each { |key, value|
next if key == 'created_at'
next if key == 'updated_at'
next if key == 'created_by_id'
next if key == 'updated_by_id'
next if key == 'preferences'
next if key == 'password'
next if !value
next if value.respond_to?('empty?') && value.empty?
attributes[key] = value
}
return if attributes.empty?
if attributes['organization_id']
organization = Organization.lookup(id: attributes['organization_id'])
if organization
attributes['organization'] = organization.name
attributes['organization_ref'] = organization.search_index_data
end
end
attributes
end
end
end

84
test/integration/elasticsearch_test.rb
View file

@ -52,6 +52,7 @@ class ElasticsearchTest < ActiveSupport::TestCase
roles = Role.where(name: 'Customer')
organization1 = Organization.create_if_not_exists(
name: 'Customer Organization Update',
note: 'some note',
updated_by_id: 1,
created_by_id: 1,
)
@ -93,8 +94,87 @@ class ElasticsearchTest < ActiveSupport::TestCase
created_by_id: 1,
)
# check search attributes
test 'a - objects' do
# user
attributes = agent.search_index_data
assert_equal('E', attributes['firstname'])
assert_equal('S', attributes['lastname'])
assert_equal('es-agent@example.com', attributes['email'])
assert_not(attributes['password'])
assert_not(attributes['organization'])
attributes = agent.search_index_attribute_lookup
assert_equal('E', attributes['firstname'])
assert_equal('S', attributes['lastname'])
assert_equal('es-agent@example.com', attributes['email'])
assert_not(attributes['password'])
assert_not(attributes['organization'])
# organization
attributes = organization1.search_index_data
assert_equal('Customer Organization Update', attributes['name'])
assert_equal('some note', attributes['note'])
assert_not(attributes['members'])
attributes = organization1.search_index_attribute_lookup
assert_equal('Customer Organization Update', attributes['name'])
assert_equal('some note', attributes['note'])
assert(attributes['members'])
# ticket/article
ticket1 = Ticket.create(
title: 'some title äöüß',
group: Group.lookup(name: 'Users'),
customer_id: customer1.id,
state: Ticket::State.lookup(name: 'new'),
priority: Ticket::Priority.lookup(name: '2 normal'),
updated_by_id: 1,
created_by_id: 1,
)
article1 = Ticket::Article.create(
ticket_id: ticket1.id,
from: 'some_sender@example.com',
to: 'some_recipient@example.com',
subject: 'some subject',
message_id: 'some@id',
body: 'some message',
internal: false,
sender: Ticket::Article::Sender.where(name: 'Customer').first,
type: Ticket::Article::Type.where(name: 'email').first,
updated_by_id: 1,
created_by_id: 1,
)
attributes = ticket1.search_index_attribute_lookup
assert_equal('Users', attributes['group'])
assert_equal('new', attributes['state'])
assert_equal('2 normal', attributes['priority'])
assert_equal('ES', attributes['customer']['firstname'])
assert_equal('Customer1', attributes['customer']['lastname'])
assert_equal('es-customer1@example.com', attributes['customer']['email'])
assert_not(attributes['customer']['password'])
assert_equal('Customer Organization Update', attributes['customer']['organization'])
assert_equal('-', attributes['owner']['login'])
assert_equal('-', attributes['owner']['firstname'])
assert_not(attributes['owner']['password'])
assert_not(attributes['owner']['organization'])
ticket1.destroy
# execute background jobs
Scheduler.worker(true)
end
# check tickets and search it
test 'a - tickets' do
test 'b - tickets' do
system('rake searchindex:rebuild')
ticket1 = Ticket.create(
title: "some title\n äöüß",
@ -398,7 +478,7 @@ class ElasticsearchTest < ActiveSupport::TestCase
end
# check users and search it
test 'b - users' do
test 'c - users' do
# search as agent
result = User.search(