Fixes #2911 - Performance: Improved touching assets of users, organizations and tickets.

This commit is contained in:
Rolf Schmidt 2020-02-18 15:49:52 +01:00 committed by Thorsten Eckel
parent a01a2f3c91
commit ed8a152f28
23 changed files with 596 additions and 344 deletions

View file

@ -112,6 +112,8 @@ class ActionRow extends App.ObserverActionRow
class Object extends App.ObserverController class Object extends App.ObserverController
model: 'Organization' model: 'Organization'
observe:
member_ids: true
observeNot: observeNot:
cid: true cid: true
created_at: true created_at: true
@ -196,6 +198,7 @@ class Member extends App.ObserverController
login: true login: true
email: true email: true
active: true active: true
image: true
globalRerender: false globalRerender: false
render: (user) => render: (user) =>

View file

@ -43,15 +43,9 @@ class App.UserProfile extends App.Controller
new User( new User(
object_id: user.id object_id: user.id
el: elLocal.find('.js-name') el: elLocal.find('.js-profileName')
) )
if user.organization_id
new Organization(
object_id: user.organization_id
el: elLocal.find('.js-organization')
)
new Object( new Object(
el: elLocal.find('.js-object-container') el: elLocal.find('.js-object-container')
object_id: user.id object_id: user.id
@ -206,9 +200,19 @@ class User extends App.ObserverController
observe: observe:
firstname: true firstname: true
lastname: true lastname: true
organization_id: true
image: true
render: (user) => render: (user) =>
@html App.Utils.htmlEscape(user.displayName()) if user.organization_id
new Organization(
object_id: user.organization_id
el: @el.siblings('.js-organization')
)
@html App.view('user_profile/name')(
user: user
)
class Router extends App.ControllerPermanent class Router extends App.ControllerPermanent
requiredPermission: 'ticket.agent' requiredPermission: 'ticket.agent'

View file

@ -1,6 +1,4 @@
class App.WidgetOrganization extends App.Controller class App.WidgetOrganization extends App.Controller
@extend App.PopoverProvidable
@registerPopovers 'User'
events: events:
'focusout [contenteditable]': 'update' 'focusout [contenteditable]': 'update'
@ -44,10 +42,18 @@ class App.WidgetOrganization extends App.Controller
organizationData.push attributeConfig organizationData.push attributeConfig
# insert userData # insert userData
@html App.view('widget/organization')( elLocal = $(App.view('widget/organization')(
organization: organization organization: organization
organizationData: organizationData organizationData: organizationData
) ))
for user in organization.members
new User(
object_id: user.id
el: elLocal.find('div.userList-row[data-id=' + user.id + ']')
)
@html elLocal
@$('[contenteditable]').ce( @$('[contenteditable]').ce(
mode: 'textonly' mode: 'textonly'
@ -55,8 +61,6 @@ class App.WidgetOrganization extends App.Controller
maxlength: 250 maxlength: 250
) )
@renderPopovers()
update: (e) => update: (e) =>
name = $(e.target).attr('data-name') name = $(e.target).attr('data-name')
value = $(e.target).html() value = $(e.target).html()
@ -66,3 +70,21 @@ class App.WidgetOrganization extends App.Controller
data[name] = value data[name] = value
org.updateAttributes(data) org.updateAttributes(data)
@log 'notice', 'update', name, value, org @log 'notice', 'update', name, value, org
class User extends App.ObserverController
@extend App.PopoverProvidable
@registerPopovers 'User'
model: 'User'
observe:
firstname: true
lastname: true
image: true
render: (user) =>
@html App.view('organization_profile/member')(
user: user
el: @el,
)
@renderPopovers()

View file

@ -2,8 +2,7 @@
<div class="profile-window"> <div class="profile-window">
<div class="profile-section vertical centered"> <div class="profile-section vertical centered">
<div class="align-right profile-action js-action"></div> <div class="align-right profile-action js-action"></div>
<%- @user.avatar("80") %> <div class="profile-name js-profileName text-center"></div>
<h1 class="js-name"></h1>
<% if @user.organization: %> <% if @user.organization: %>
<div class="profile-organization js-organization"></div> <div class="profile-organization js-organization"></div>
<% end %> <% end %>

View file

@ -0,0 +1,2 @@
<%- @user.avatar("80") %>
<h1><%= @user.displayName() %></h1>

View file

@ -27,13 +27,8 @@
<label><%- @T('Members') %></label> <label><%- @T('Members') %></label>
<div class="userList"> <div class="userList">
<% for user in @organization.members: %> <% for user in @organization.members: %>
<div class="userList-entry"> <div class="userList-row" data-id="<%- user.id %>"></div>
<%- user.avatar("40") %>
<a href="<%- user.uiUrl() %>" class="userList-name user-popover" data-id="<%- user.id %>">
<%= user.displayName() %>
</a>
</div>
<% end %> <% end %>
</div> </div>
</div> </div>
<% end %> <% end %>

View file

@ -6,11 +6,11 @@ class SearchIndexJob < ApplicationJob
} }
def lock_key def lock_key
# "SearchIndexJob/User/42" # "SearchIndexJob/User/42/true"
"#{self.class.name}/#{arguments[0]}/#{arguments[1]}" "#{self.class.name}/#{arguments[0]}/#{arguments[1]}/#{arguments[2]}"
end end
def perform(object, o_id) def perform(object, o_id, update_associations = true)
@object = object @object = object
@o_id = o_id @o_id = o_id
@ -18,6 +18,11 @@ class SearchIndexJob < ApplicationJob
return if !exists?(record) return if !exists?(record)
record.search_index_update_backend record.search_index_update_backend
return if !update_associations
record.search_index_update_associations_delta
record.search_index_update_associations_full
end end
private private

View file

@ -19,48 +19,147 @@ returns
attributes = self.attributes attributes = self.attributes
self.attributes.each do |key, value| self.attributes.each do |key, value|
attribute_name = key.to_s
# ignore standard attribute if needed
if self.class.search_index_attribute_ignored?(attribute_name)
attributes.delete(attribute_name)
next
end
# need value for reference data
next if !value next if !value
# get attribute name # check if we have a referenced object which we could include here
attribute_name = key.to_s next if !search_index_attribute_method(attribute_name)
next if attribute_name[-3, 3] != '_id'
attribute_name = attribute_name[ 0, attribute_name.length - 3 ] # get referenced attribute name
attribute_ref_name = self.class.search_index_attribute_ref_name(attribute_name)
next if !attribute_ref_name
# check if attribute method exists # ignore referenced attributes if needed
next if !respond_to?(attribute_name) next if self.class.search_index_attribute_ignored?(attribute_ref_name)
# check if method has own class
relation_class = send(attribute_name).class
next if !relation_class
# lookup ref object
relation_model = relation_class.lookup(id: value)
next if !relation_model
# get name of ref object
value = nil
if relation_model.respond_to?('search_index_data')
value = relation_model.send('search_index_data')
end
if relation_model.respond_to?('name')
value = relation_model.send('name')
end
# get referenced attribute value
value = search_index_value_by_attribute(attribute_name)
next if !value next if !value
# save name of ref object # save name of ref object
attributes[ attribute_name ] = value attributes[ attribute_ref_name ] = value
end
ignored_attributes = self.class.instance_variable_get(:@search_index_attributes_ignored) || []
return attributes if ignored_attributes.blank?
ignored_attributes.each do |attribute|
attributes.delete(attribute.to_s)
end end
attributes attributes
end end
=begin
This function returns the relational search index value based on the attribute name.
organization = Organization.find(1)
value = organization.search_index_value_by_attribute('organization_id')
returns
value = {"name"=>"Zammad Foundation"}
=end
def search_index_value_by_attribute(attribute_name = '')
# get attribute name
relation_class = search_index_attribute_method(attribute_name)
return if !relation_class
# lookup ref object
relation_model = relation_class.lookup(id: attributes[attribute_name])
return if !relation_model
relation_model.search_index_value
end
=begin
This function returns the relational search value.
organization = Organization.find(1)
value = organization.search_index_value
returns
value = {"name"=>"Zammad Foundation"}
=end
def search_index_value
# get name of ref object
value = nil
if respond_to?('search_index_data')
value = send('search_index_data')
return if value == true
elsif respond_to?('name')
value = send('name')
end
value
end
=begin
This function returns the method for the relational search index attribute.
method = Ticket.new.search_index_attribute_method('organization_id')
returns
method = Organization (class)
=end
def search_index_attribute_method(attribute_name = '')
return if attribute_name[-3, 3] != '_id'
attribute_name = attribute_name[ 0, attribute_name.length - 3 ]
return if !respond_to?(attribute_name)
send(attribute_name).class
end
class_methods do
=begin
This function returns the relational search index attribute name for the given class.
attribute_ref_name = Organization.search_index_attribute_ref_name('user_id')
returns
attribute_ref_name = 'user'
=end
def search_index_attribute_ref_name(attribute_name)
attribute_name[ 0, attribute_name.length - 3 ]
end
=begin
This function returns if a search index attribute should be ignored.
ignored = Ticket.search_index_attribute_ignored?('organization_id')
returns
ignored = false
=end
def search_index_attribute_ignored?(attribute_name = '')
ignored_attributes = instance_variable_get(:@search_index_attributes_ignored) || []
return if ignored_attributes.blank?
ignored_attributes.include?(attribute_name.to_sym)
end
end
end end

View file

@ -30,6 +30,86 @@ update search index, if configured - will be executed automatically
=begin =begin
update search index, if configured - will be executed automatically
model = Organizations.find(123)
result = model.search_index_update_associations_full
returns
# Updates asscociation data for users and tickets of the organization in this example
result = true
=end
def search_index_update_associations_full
return if self.class.to_s != 'Organization'
# reindex all organization tickets for the given organization id
# we can not use the delta function for this because of the excluded
# ticket article attachments. see explain in delta function
Ticket.select('id').where(organization_id: id).order(id: :desc).limit(10_000).pluck(:id).each do |ticket_id|
SearchIndexJob.perform_later('Ticket', ticket_id, false)
end
end
=begin
update search index, if configured - will be executed automatically
model = Organizations.find(123)
result = model.search_index_update_associations_delta
returns
# Updates asscociation data for users and tickets of the organization in this example
result = true
=end
def search_index_update_associations_delta
# start background job to transfer data to search index
return true if !SearchIndexBackend.enabled?
return if search_index_value.blank?
Models.indexable.each do |local_object|
next if local_object == self.class
# delta update of associations is only possible for
# objects which are not containing modifications of the source
# https://github.com/zammad/zammad/blob/264853dcbe4e53addaf0f8e6df3735ceddc9de63/lib/tasks/search_index_es.rake#L266
# because of the exlusion of the article attachments for the ticket
# we dont have the attachment data available in the json store of the object.
# so the search index would lose the attachment information on the _update_by_query function
# https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
next if local_object.to_s == 'Ticket'
local_object.new.attributes.each do |key, _value|
attribute_name = key.to_s
attribute_ref_name = local_object.search_index_attribute_ref_name(attribute_name)
attribute_class = local_object.reflect_on_association(attribute_ref_name)&.klass
next if attribute_name.blank?
next if attribute_ref_name.blank?
next if attribute_class.blank?
next if attribute_class != self.class
data = {
attribute_ref_name => search_index_value
}
where = {
attribute_name => id
}
SearchIndexBackend.update_by_query(local_object.to_s, data, where)
end
end
true
end
=begin
delete search index object, will be executed automatically delete search index object, will be executed automatically
model = Model.find(123) model = Model.find(123)

View file

@ -1,43 +0,0 @@
# Copyright (C) 2012-2016 Zammad Foundation, http://zammad-foundation.org/
class Observer::Organization::RefObjectTouch < ActiveRecord::Observer
observe 'organization'
def after_create(record)
ref_object_touch(record)
end
def after_update(record)
ref_object_touch(record)
end
def after_destroy(record)
ref_object_touch(record)
end
def ref_object_touch(record)
# return if we run import mode
return true if Setting.get('import_mode')
# feature used for different purpose; do not touch references
return true if User.where(organization_id: record.id).count > 100
# touch organizations tickets
Ticket.select('id').where(organization_id: record.id).pluck(:id).each do |ticket_id|
ticket = Ticket.find(ticket_id)
ticket.with_lock do
ticket.touch # rubocop:disable Rails/SkipsModelValidations
end
end
# touch current members
User.select('id').where(organization_id: record.id).pluck(:id).each do |user_id|
user = User.find(user_id)
user.with_lock do
user.touch # rubocop:disable Rails/SkipsModelValidations
end
end
true
end
end

View file

@ -20,37 +20,22 @@ class Observer::User::RefObjectTouch < ActiveRecord::Observer
# return if we run import mode # return if we run import mode
return true if Setting.get('import_mode') return true if Setting.get('import_mode')
# touch old organization if changed
member_ids = []
organization_id_changed = record.saved_changes['organization_id'] organization_id_changed = record.saved_changes['organization_id']
if organization_id_changed && organization_id_changed[0] != organization_id_changed[1] return true if !organization_id_changed
if organization_id_changed[0]
# featrue used for different propose, do not touch references return true if organization_id_changed[0] == organization_id_changed[1]
if User.where(organization_id: organization_id_changed[0]).count < 100
organization = Organization.find(organization_id_changed[0]) # touch old organization
organization.touch # rubocop:disable Rails/SkipsModelValidations if organization_id_changed[0]
member_ids = organization.member_ids organization = Organization.find(organization_id_changed[0])
end organization.touch # rubocop:disable Rails/SkipsModelValidations
end
end end
# touch new/current organization # touch new/current organization
if record.organization if record&.organization
record.organization.touch # rubocop:disable Rails/SkipsModelValidations
# featrue used for different propose, do not touch references
if User.where(organization_id: record.organization_id).count < 100
record.organization.touch # rubocop:disable Rails/SkipsModelValidations
member_ids += record.organization.member_ids
end
end end
# touch old/current customer
member_ids.uniq.each do |user_id|
next if user_id == record.id
User.find(user_id).touch # rubocop:disable Rails/SkipsModelValidations
end
true true
end end
end end

View file

@ -48,7 +48,6 @@ module Zammad
'observer::_user::_ref_object_touch', 'observer::_user::_ref_object_touch',
'observer::_user::_ticket_organization', 'observer::_user::_ticket_organization',
'observer::_user::_geo', 'observer::_user::_geo',
'observer::_organization::_ref_object_touch',
'observer::_sla::_ticket_rebuild_escalation', 'observer::_sla::_ticket_rebuild_escalation',
'observer::_transaction' 'observer::_transaction'

View file

@ -119,7 +119,7 @@ create/update/delete index
def self.index(data) def self.index(data)
url = build_url(data[:name], nil, false, false) url = build_url(type: data[:name], with_pipeline: false, with_document_type: false)
return if url.blank? return if url.blank?
if data[:action] && data[:action] == 'delete' if data[:action] && data[:action] == 'delete'
@ -139,7 +139,7 @@ add new object to search index
def self.add(type, data) def self.add(type, data)
url = build_url(type, data['id']) url = build_url(type: type, object_id: data['id'])
return if url.blank? return if url.blank?
make_request_and_validate(url, data: data, method: :post) make_request_and_validate(url, data: data, method: :post)
@ -147,6 +147,48 @@ add new object to search index
=begin =begin
This function updates specifc attributes of an index based on a query.
data = {
organization: {
name: "Zammad Foundation"
}
}
where = {
organization_id: 1
}
SearchIndexBackend.update_by_query('Ticket', data, where)
=end
def self.update_by_query(type, data, where)
return if data.blank?
return if where.blank?
url = build_url(type: type, action: '_update_by_query', with_pipeline: false, with_document_type: false, url_params: { conflicts: 'proceed' })
return if url.blank?
script_list = []
data.each do |key, _value|
script_list.push("ctx._source.#{key}=params.#{key}")
end
data = {
script: {
lang: 'painless',
source: script_list.join(';'),
params: data,
},
query: {
term: where,
},
}
make_request_and_validate(url, data: data, method: :post, read_timeout: 10.minutes)
end
=begin
remove whole data from index remove whole data from index
SearchIndexBackend.remove('Ticket', 123) SearchIndexBackend.remove('Ticket', 123)
@ -157,9 +199,9 @@ remove whole data from index
def self.remove(type, o_id = nil) def self.remove(type, o_id = nil)
url = if o_id url = if o_id
build_url(type, o_id, false, true) build_url(type: type, object_id: o_id, with_pipeline: false, with_document_type: true)
else else
build_url(type, o_id, false, false) build_url(type: type, object_id: o_id, with_pipeline: false, with_document_type: false)
end end
return if url.blank? return if url.blank?
@ -237,11 +279,9 @@ remove whole data from index
def self.search_by_index(query, index, options = {}) def self.search_by_index(query, index, options = {})
return [] if query.blank? return [] if query.blank?
url = build_url url = build_url(type: index, action: '_search', with_pipeline: false, with_document_type: true)
return [] if url.blank? return [] if url.blank?
url += build_search_url(index)
# real search condition # real search condition
condition = { condition = {
'query_string' => { 'query_string' => {
@ -389,11 +429,9 @@ example for aggregations within one year
def self.selectors(index, selectors = nil, options = {}, aggs_interval = nil) def self.selectors(index, selectors = nil, options = {}, aggs_interval = nil)
raise 'no selectors given' if !selectors raise 'no selectors given' if !selectors
url = build_url(nil, nil, false, false) url = build_url(type: index, action: '_search', with_pipeline: false, with_document_type: true)
return if url.blank? return if url.blank?
url += build_search_url(index)
data = selector2query(selectors, options, aggs_interval) data = selector2query(selectors, options, aggs_interval)
response = make_request(url, data: data) response = make_request(url, data: data)
@ -626,107 +664,92 @@ return true if backend is configured
true true
end end
def self.build_index_name(index) def self.build_index_name(index = nil)
local_index = "#{Setting.get('es_index')}_#{Rails.env}" local_index = "#{Setting.get('es_index')}_#{Rails.env}"
return local_index if index.blank?
return "#{local_index}/#{index}" if lower_equal_es56?
"#{local_index}_#{index.underscore.tr('/', '_')}" "#{local_index}_#{index.underscore.tr('/', '_')}"
end end
=begin =begin
return true if the elastic search version is lower equal 5.6
result = SearchIndexBackend.lower_equal_es56?
returns
result = true
=end
def self.lower_equal_es56?
Setting.get('es_multi_index') == false
end
=begin
generate url for index or document access (only for internal use) generate url for index or document access (only for internal use)
# url to access single document in index (in case with_pipeline or not) # url to access single document in index (in case with_pipeline or not)
url = SearchIndexBackend.build_url('User', 123, with_pipeline) url = SearchIndexBackend.build_url(type: 'User', object_id: 123, with_pipeline: true)
# url to access whole index # url to access whole index
url = SearchIndexBackend.build_url('User') url = SearchIndexBackend.build_url(type: 'User')
# url to access document definition in index (only es6 and higher) # url to access document definition in index (only es6 and higher)
url = SearchIndexBackend.build_url('User', nil, false, true) url = SearchIndexBackend.build_url(type: 'User', with_pipeline: false, with_document_type: true)
# base url # base url
url = SearchIndexBackend.build_url url = SearchIndexBackend.build_url
=end =end
def self.build_url(type = nil, o_id = nil, with_pipeline = true, with_document_type = true) # rubocop:disable Metrics/ParameterLists
def self.build_url(type: nil, action: nil, object_id: nil, with_pipeline: true, with_document_type: true, url_params: {})
# rubocop:enable Metrics/ParameterLists
return if !SearchIndexBackend.enabled? return if !SearchIndexBackend.enabled?
# for elasticsearch 5.6 and lower # set index
index = "#{Setting.get('es_index')}_#{Rails.env}" index = build_index_name(type)
if Setting.get('es_multi_index') == false
url = Setting.get('es_url')
url = if type
if with_pipeline == true
url_pipline = Setting.get('es_pipeline')
if url_pipline.present?
url_pipline = "?pipeline=#{url_pipline}"
end
end
if o_id
"#{url}/#{index}/#{type}/#{o_id}#{url_pipline}"
else
"#{url}/#{index}/#{type}#{url_pipline}"
end
else
"#{url}/#{index}"
end
return url
end
# for elasticsearch 6.x and higher # add pipeline if needed
url = Setting.get('es_url') if index && with_pipeline == true
if with_pipeline == true
url_pipline = Setting.get('es_pipeline') url_pipline = Setting.get('es_pipeline')
if url_pipline.present? if url_pipline.present?
url_pipline = "?pipeline=#{url_pipline}" url_params['pipeline'] = url_pipline
end end
end end
if type
index = build_index_name(type)
# access (e. g. creating or dropping) whole index # prepare url params
if with_document_type == false params_string = ''
return "#{url}/#{index}" if url_params.present?
end params_string = '?' + url_params.map { |key, value| "#{key}=#{value}" }.join('&')
# access single document in index (e. g. drop or add document)
if o_id
return "#{url}/#{index}/_doc/#{o_id}#{url_pipline}"
end
# access document type (e. g. creating or dropping document mapping)
return "#{url}/#{index}/_doc#{url_pipline}"
end
"#{url}/"
end
=begin
generate url searchaccess (only for internal use)
# url search access with single index
url = SearchIndexBackend.build_search_url('User')
# url to access all over es
url = SearchIndexBackend.build_search_url
=end
def self.build_search_url(index = nil)
# for elasticsearch 5.6 and lower
if Setting.get('es_multi_index') == false
if index
return "/#{index}/_search"
end
return '/_search'
end end
# for elasticsearch 6.x and higher url = Setting.get('es_url')
"#{build_index_name(index)}/_doc/_search" return "#{url}#{params_string}" if index.blank?
# add type information
url = "#{url}/#{index}"
# add document type
if with_document_type && !lower_equal_es56?
url = "#{url}/_doc"
end
# add action
if action
url = "#{url}/#{action}"
end
# add object id
if object_id.present?
url = "#{url}/#{object_id}"
end
"#{url}#{params_string}"
end end
def self.humanized_error(verb:, url:, payload: nil, response:) def self.humanized_error(verb:, url:, payload: nil, response:)

View file

@ -2,7 +2,7 @@ $LOAD_PATH << './lib'
require 'rubygems' require 'rubygems'
namespace :searchindex do namespace :searchindex do
task :drop, [:opts] => :environment do |_t, _args| task :drop, [:opts] => %i[environment searchindex:version_supported] do |_t, _args|
print 'drop indexes...' print 'drop indexes...'
# drop indexes # drop indexes
@ -23,7 +23,7 @@ namespace :searchindex do
Rake::Task['searchindex:drop_pipeline'].execute Rake::Task['searchindex:drop_pipeline'].execute
end end
task :create, [:opts] => :environment do |_t, _args| task :create, [:opts] => %i[environment searchindex:version_supported] do |_t, _args|
print 'create indexes...' print 'create indexes...'
if es_multi_index? if es_multi_index?
@ -67,7 +67,7 @@ namespace :searchindex do
Rake::Task['searchindex:create_pipeline'].execute Rake::Task['searchindex:create_pipeline'].execute
end end
task :create_pipeline, [:opts] => :environment do |_t, _args| task :create_pipeline, [:opts] => %i[environment searchindex:version_supported] do |_t, _args|
if !es_pipeline? if !es_pipeline?
Setting.set('es_pipeline', '') Setting.set('es_pipeline', '')
next next
@ -124,7 +124,7 @@ namespace :searchindex do
puts 'done' puts 'done'
end end
task :drop_pipeline, [:opts] => :environment do |_t, _args| task :drop_pipeline, [:opts] => %i[environment searchindex:version_supported] do |_t, _args|
next if !es_pipeline? next if !es_pipeline?
# update processors # update processors
@ -142,7 +142,7 @@ namespace :searchindex do
puts 'done' puts 'done'
end end
task :reload, [:opts] => :environment do |_t, _args| task :reload, [:opts] => %i[environment searchindex:version_supported] do |_t, _args|
puts 'reload data...' puts 'reload data...'
Models.indexable.each do |model_class| Models.indexable.each do |model_class|
@ -156,17 +156,23 @@ namespace :searchindex do
end end
task :refresh, [:opts] => :environment do |_t, _args| task :refresh, [:opts] => %i[environment searchindex:version_supported] do |_t, _args|
print 'refresh all indexes...' print 'refresh all indexes...'
SearchIndexBackend.refresh SearchIndexBackend.refresh
end end
task :rebuild, [:opts] => :environment do |_t, _args| task :rebuild, [:opts] => %i[environment searchindex:version_supported] do |_t, _args|
Rake::Task['searchindex:drop'].execute Rake::Task['searchindex:drop'].execute
Rake::Task['searchindex:create'].execute Rake::Task['searchindex:create'].execute
Rake::Task['searchindex:reload'].execute Rake::Task['searchindex:reload'].execute
end end
task :version_supported, [:opts] => :environment do |_t, _args|
next if es_version_supported?
abort "Your elastic search version is not supported! Please update your version to a greater equal than 5.6.0 (Your current version: #{es_version})."
end
end end
=begin =begin
@ -300,6 +306,16 @@ def es_version
end end
end end
def es_version_supported?
version_split = es_version.split('.')
version = "#{version_split[0]}#{format('%03d', version_split[1])}#{format('%03d', version_split[2])}".to_i
# only versions greater/equal than 5.6.0 are supported
return if version < 5_006_000
true
end
# no es_pipeline for elasticsearch 5.5 and lower # no es_pipeline for elasticsearch 5.5 and lower
def es_pipeline? def es_pipeline?
number = es_version number = es_version

View file

@ -31,39 +31,12 @@ RSpec.describe Organization, type: :model do
let!(:member) { create(:customer_user, organization: organization) } let!(:member) { create(:customer_user, organization: organization) }
let!(:member_ticket) { create(:ticket, customer: member) } let!(:member_ticket) { create(:ticket, customer: member) }
context 'when basic attributes are updated' do
it 'touches its members and their tickets' do
expect { organization.update(name: 'foo') }
.to change { member.reload.updated_at }
.and change { member_ticket.reload.updated_at }
end
end
context 'when member associations are added' do context 'when member associations are added' do
let(:user) { create(:customer_user) } let(:user) { create(:customer_user) }
it 'is touched, and touches its other members (but not their tickets)' do it 'is touched, and touches its other members (but not their tickets)' do
expect { organization.members.push(user) } expect { organization.members.push(user) }
.to change { organization.reload.updated_at } .to change { organization.reload.updated_at }
.and change { member.reload.updated_at }
.and not_change { member_ticket.reload.updated_at }
end
end
context 'with 100+ members' do
let!(:members) { create_list(:user, 101, organization: organization) }
let!(:member_ticket) { create(:ticket, customer: members.first) }
# This _should_ be split into two separate examples,
# but setup is slow and expensive.
it 'does not perform any association updates' do
expect { organization.update(name: 'foo') }
.to not_change { members.map(&:reload).map(&:updated_at) }
.and not_change { member_ticket.reload.updated_at }
expect { organization.members.push(member) }
.to not_change { organization.reload.updated_at }
.and not_change { members.map(&:reload).map(&:updated_at) }
end end
end end
end end

View file

@ -852,20 +852,6 @@ RSpec.describe Ticket, type: :model do
.and change { other_organization.reload.updated_at } .and change { other_organization.reload.updated_at }
end end
end end
context 'when organization has 100+ members' do
let!(:other_members) { create_list(:user, 100, organization: organization) }
context 'and customer association is changed' do
it 'touches both old and new customer, and their organizations' do
expect { ticket.update(customer: other_customer) }
.to change { customer.reload.updated_at }
.and change { organization.reload.updated_at }
.and change { other_customer.reload.updated_at }
.and change { other_organization.reload.updated_at }
end
end
end
end end
describe 'Association & attachment management:' do describe 'Association & attachment management:' do

View file

@ -0,0 +1,38 @@
RSpec.shared_examples 'CanLookupSearchIndexAttributes' do
describe '.search_index_value_by_attribute' do
it 'returns hash of data' do
organization = create(:organization, name: 'Tomato42', note: 'special recipe')
user = create(:agent_user, organization: organization)
value = user.search_index_value_by_attribute('organization_id')
expect_value = { 'name' => 'Tomato42', 'note' => 'special recipe' }
expect(value).to be_a_kind_of(Hash)
expect(value).to eq(expect_value)
end
end
describe '.search_index_value' do
it 'returns correct value' do
organization = create(:organization, name: 'Tomato42', note: 'special recipe')
value = organization.search_index_value
expect_value = { 'name' => 'Tomato42', 'note' => 'special recipe' }
expect(value).to be_a_kind_of(Hash)
expect(value).to eq(expect_value)
end
end
describe '.search_index_attribute_ref_name' do
it 'returns correct value' do
attribute_ref_name = User.search_index_attribute_ref_name('organization_id')
expect(attribute_ref_name).to eq('organization')
end
end
describe '.search_index_attribute_ignored?' do
it 'returns correct value' do
ignored = User.search_index_attribute_ignored?('password')
expect(ignored).to be true
end
end
end

View file

@ -7,6 +7,7 @@ require 'models/concerns/has_groups_permissions_examples'
require 'models/concerns/has_xss_sanitized_note_examples' require 'models/concerns/has_xss_sanitized_note_examples'
require 'models/concerns/can_be_imported_examples' require 'models/concerns/can_be_imported_examples'
require 'models/concerns/has_object_manager_attributes_validation_examples' require 'models/concerns/has_object_manager_attributes_validation_examples'
require 'models/user/can_lookup_search_index_attributes_examples'
RSpec.describe User, type: :model do RSpec.describe User, type: :model do
subject(:user) { create(:user) } subject(:user) { create(:user) }
@ -23,6 +24,7 @@ RSpec.describe User, type: :model do
it_behaves_like 'HasGroups and Permissions', group_access_no_permission_factory: :user it_behaves_like 'HasGroups and Permissions', group_access_no_permission_factory: :user
it_behaves_like 'CanBeImported' it_behaves_like 'CanBeImported'
it_behaves_like 'HasObjectManagerAttributesValidation' it_behaves_like 'HasObjectManagerAttributesValidation'
it_behaves_like 'CanLookupSearchIndexAttributes'
describe 'Class methods:' do describe 'Class methods:' do
describe '.authenticate' do describe '.authenticate' do
@ -1142,28 +1144,16 @@ RSpec.describe User, type: :model do
end end
describe 'Touching associations on update:' do describe 'Touching associations on update:' do
subject(:user) { create(:customer_user, organization: organization) } subject!(:user) { create(:customer_user) }
let(:organization) { create(:organization) } let!(:organization) { create(:organization) }
let(:other_customer) { create(:customer_user) }
context 'when basic attributes are updated' do context 'when a customer gets a organization ' do
it 'touches its organization' do it 'touches its organization' do
expect { user.update(firstname: 'foo') } expect { user.update(organization: organization) }
.to change { organization.reload.updated_at } .to change { organization.reload.updated_at }
end end
end end
context 'when organization has 100+ other members' do
let!(:other_members) { create_list(:user, 100, organization: organization) }
context 'and basic attributes are updated' do
it 'does not touch its organization' do
expect { user.update(firstname: 'foo') }
.to not_change { organization.reload.updated_at }
end
end
end
end end
describe 'Cti::CallerId syncing:' do describe 'Cti::CallerId syncing:' do

View file

@ -106,24 +106,27 @@ RSpec.describe 'Organization', type: :request, searchindex: true do
get "/api/v1/organizations/search?query=#{CGI.escape('Zammad')}", params: {}, as: :json get "/api/v1/organizations/search?query=#{CGI.escape('Zammad')}", params: {}, as: :json
expect(response).to have_http_status(:ok) expect(response).to have_http_status(:ok)
expect(json_response).to be_a_kind_of(Array) expect(json_response).to be_a_kind_of(Array)
expect(json_response[0]['name']).to eq('Zammad Foundation') organization = json_response.detect { |object| object['name'] == 'Zammad Foundation' }
expect(json_response[0]['member_ids']).to be_truthy expect(organization['name']).to eq('Zammad Foundation')
expect(json_response[0]['members']).to be_falsey expect(organization['member_ids']).to be_truthy
expect(organization['members']).to be_falsey
get "/api/v1/organizations/search?query=#{CGI.escape('Zammad')}&expand=true", params: {}, as: :json get "/api/v1/organizations/search?query=#{CGI.escape('Zammad')}&expand=true", params: {}, as: :json
expect(response).to have_http_status(:ok) expect(response).to have_http_status(:ok)
expect(json_response).to be_a_kind_of(Array) expect(json_response).to be_a_kind_of(Array)
expect(json_response[0]['name']).to eq('Zammad Foundation') organization = json_response.detect { |object| object['name'] == 'Zammad Foundation' }
expect(json_response[0]['member_ids']).to be_truthy expect(organization['name']).to eq('Zammad Foundation')
expect(json_response[0]['members']).to be_truthy expect(organization['member_ids']).to be_truthy
expect(organization['members']).to be_truthy
get "/api/v1/organizations/search?query=#{CGI.escape('Zammad')}&label=true", params: {}, as: :json get "/api/v1/organizations/search?query=#{CGI.escape('Zammad')}&label=true", params: {}, as: :json
expect(response).to have_http_status(:ok) expect(response).to have_http_status(:ok)
expect(json_response).to be_a_kind_of(Array) expect(json_response).to be_a_kind_of(Array)
expect(json_response[0]['label']).to eq('Zammad Foundation') organization = json_response.detect { |object| object['label'] == 'Zammad Foundation' }
expect(json_response[0]['value']).to eq('Zammad Foundation') expect(organization['label']).to eq('Zammad Foundation')
expect(json_response[0]['member_ids']).to be_falsey expect(organization['value']).to eq('Zammad Foundation')
expect(json_response[0]['members']).to be_falsey expect(organization['member_ids']).to be_falsey
expect(organization['members']).to be_falsey
end end
it 'does index with customer1' do it 'does index with customer1' do

View file

@ -27,6 +27,12 @@ RSpec.describe 'Search', type: :request, searchindex: true do
let!(:organization5) do let!(:organization5) do
create(:organization, name: 'ABC_D Org') create(:organization, name: 'ABC_D Org')
end end
let!(:organization_nested) do
create(:organization, name: 'Tomato42 Ltd.', note: 'Tomato42 Ltd.')
end
let!(:customer_user_nested) do
create(:customer_user, organization: organization_nested)
end
let!(:customer_user2) do let!(:customer_user2) do
create(:customer_user, organization: organization1) create(:customer_user, organization: organization1)
end end
@ -42,6 +48,9 @@ RSpec.describe 'Search', type: :request, searchindex: true do
let!(:ticket3) do let!(:ticket3) do
create(:ticket, title: 'test 1234-2', customer: customer_user3, group: group) create(:ticket, title: 'test 1234-2', customer: customer_user3, group: group)
end end
let!(:ticket_nested) do
create(:ticket, title: 'vegetable request', customer: customer_user_nested, group: group)
end
let!(:article1) do let!(:article1) do
create(:ticket_article, ticket_id: ticket1.id) create(:ticket_article, ticket_id: ticket1.id)
end end
@ -51,6 +60,20 @@ RSpec.describe 'Search', type: :request, searchindex: true do
let!(:article3) do let!(:article3) do
create(:ticket_article, ticket_id: ticket3.id) create(:ticket_article, ticket_id: ticket3.id)
end end
let!(:article_nested) do
article = create(:ticket_article, ticket_id: ticket_nested.id)
Store.add(
object: 'Ticket::Article',
o_id: article.id,
data: File.binread(Rails.root.join('test', 'data', 'elasticsearch', 'es-normal.txt')),
filename: 'es-normal.txt',
preferences: {},
created_by_id: 1,
)
article
end
before do before do
configure_elasticsearch do configure_elasticsearch do
@ -327,5 +350,100 @@ RSpec.describe 'Search', type: :request, searchindex: true do
target_id = json_response['result'][0]['id'] target_id = json_response['result'][0]['id']
expect(json_response['assets']['Organization'][target_id.to_s]['name']).to eq('ABC_D Org') expect(json_response['assets']['Organization'][target_id.to_s]['name']).to eq('ABC_D Org')
end end
it 'does find the user of the nested organization and also even if the organization name changes' do
params = {
query: 'Tomato42',
limit: 10,
}
# because of the initial relation between user and organization
# both user and organization will be found as result
authenticated_as(agent_user)
post '/api/v1/search/User', params: params, as: :json
expect(response).to have_http_status(:ok)
expect(json_response).to be_a_kind_of(Hash)
expect(json_response).to be_truthy
expect(json_response['assets']['Organization'][organization_nested.id.to_s]).to be_truthy
expect(json_response['assets']['User'][customer_user_nested.id.to_s]).to be_truthy
organization_nested.update(name: 'Cucumber43 Ltd.')
Scheduler.worker(true)
SearchIndexBackend.refresh
params = {
query: 'Cucumber43',
limit: 10,
}
# even after a change of the organization name we should find
# the customer user because of the nested organization data
post '/api/v1/search/User', params: params, as: :json
expect(response).to have_http_status(:ok)
expect(json_response).to be_a_kind_of(Hash)
expect(json_response).to be_truthy
expect(json_response['assets']['Organization'][organization_nested.id.to_s]).to be_truthy
expect(json_response['assets']['User'][customer_user_nested.id.to_s]).to be_truthy
end
it 'does find the ticket by organization name even if the organization name changes' do
params = {
query: 'Tomato42',
limit: 10,
}
authenticated_as(agent_user)
post '/api/v1/search/Ticket', params: params, as: :json
expect(response).to have_http_status(:ok)
expect(json_response).to be_a_kind_of(Hash)
expect(json_response).to be_truthy
expect(json_response['assets']['Organization'][organization_nested.id.to_s]).to be_truthy
expect(json_response['assets']['Ticket'][ticket_nested.id.to_s]).to be_truthy
organization_nested.update(name: 'Cucumber43 Ltd.')
Scheduler.worker(true)
SearchIndexBackend.refresh
params = {
query: 'Cucumber43',
limit: 10,
}
post '/api/v1/search/Ticket', params: params, as: :json
expect(response).to have_http_status(:ok)
expect(json_response).to be_a_kind_of(Hash)
expect(json_response).to be_truthy
expect(json_response['assets']['Organization'][organization_nested.id.to_s]).to be_truthy
expect(json_response['assets']['Ticket'][ticket_nested.id.to_s]).to be_truthy
end
it 'does find the ticket by attachment even after ticket reindex' do
params = {
query: 'text66',
limit: 10,
}
authenticated_as(agent_user)
post '/api/v1/search/Ticket', params: params, as: :json
expect(response).to have_http_status(:ok)
expect(json_response).to be_a_kind_of(Hash)
expect(json_response).to be_truthy
expect(json_response['assets']['Ticket'][ticket_nested.id.to_s]).to be_truthy
organization_nested.update(name: 'Cucumber43 Ltd.')
Scheduler.worker(true)
SearchIndexBackend.refresh
params = {
query: 'text66',
limit: 10,
}
post '/api/v1/search/Ticket', params: params, as: :json
expect(response).to have_http_status(:ok)
expect(json_response).to be_a_kind_of(Hash)
expect(json_response).to be_truthy
expect(json_response['assets']['Ticket'][ticket_nested.id.to_s]).to be_truthy
end
end end
end end

View file

@ -89,24 +89,27 @@ RSpec.describe 'User Organization', type: :request, searchindex: true do
get "/api/v1/organizations/search?query=#{CGI.escape('Zammad')}", params: {}, as: :json get "/api/v1/organizations/search?query=#{CGI.escape('Zammad')}", params: {}, as: :json
expect(response).to have_http_status(:ok) expect(response).to have_http_status(:ok)
expect(json_response.class).to eq(Array) expect(json_response.class).to eq(Array)
expect(json_response[0]['name']).to eq('Zammad Foundation') organization = json_response.detect { |object| object['name'] == 'Zammad Foundation' }
expect(json_response[0]['member_ids']).to be_truthy expect(organization['name']).to eq('Zammad Foundation')
expect(json_response[0]['members']).to be_falsey expect(organization['member_ids']).to be_truthy
expect(organization['members']).to be_falsey
get "/api/v1/organizations/search?query=#{CGI.escape('Zammad')}&expand=true", params: {}, as: :json get "/api/v1/organizations/search?query=#{CGI.escape('Zammad')}&expand=true", params: {}, as: :json
expect(response).to have_http_status(:ok) expect(response).to have_http_status(:ok)
expect(json_response.class).to eq(Array) expect(json_response.class).to eq(Array)
expect(json_response[0]['name']).to eq('Zammad Foundation') organization = json_response.detect { |object| object['name'] == 'Zammad Foundation' }
expect(json_response[0]['member_ids']).to be_truthy expect(organization['name']).to eq('Zammad Foundation')
expect(json_response[0]['members']).to be_truthy expect(organization['member_ids']).to be_truthy
expect(organization['members']).to be_truthy
get "/api/v1/organizations/search?query=#{CGI.escape('Zammad')}&label=true", params: {}, as: :json get "/api/v1/organizations/search?query=#{CGI.escape('Zammad')}&label=true", params: {}, as: :json
expect(response).to have_http_status(:ok) expect(response).to have_http_status(:ok)
expect(json_response.class).to eq(Array) expect(json_response.class).to eq(Array)
expect(json_response[0]['label']).to eq('Zammad Foundation') organization = json_response.detect { |object| object['label'] == 'Zammad Foundation' }
expect(json_response[0]['value']).to eq('Zammad Foundation') expect(organization['label']).to eq('Zammad Foundation')
expect(json_response[0]['member_ids']).to be_falsey expect(organization['value']).to eq('Zammad Foundation')
expect(json_response[0]['members']).to be_falsey expect(organization['member_ids']).to be_falsey
expect(organization['members']).to be_falsey
end end
it 'does organization index with customer1' do it 'does organization index with customer1' do

View file

@ -74,6 +74,7 @@ class ElasticsearchTest < ActiveSupport::TestCase
# execute background jobs to index created/changed objects # execute background jobs to index created/changed objects
Scheduler.worker(true) Scheduler.worker(true)
SearchIndexBackend.refresh
end end
@ -103,7 +104,7 @@ class ElasticsearchTest < ActiveSupport::TestCase
assert_equal('es-customer1@example.com', attributes['email']) assert_equal('es-customer1@example.com', attributes['email'])
assert(attributes['preferences']) assert(attributes['preferences'])
assert_not(attributes['password']) assert_not(attributes['password'])
assert_equal('Customer Organization Update', attributes['organization']) assert_equal({ 'name' => 'Customer Organization Update', 'note' => 'some note' }, attributes['organization'])
# organization # organization
attributes = @organization1.search_index_data attributes = @organization1.search_index_data
@ -174,6 +175,7 @@ class ElasticsearchTest < ActiveSupport::TestCase
# execute background jobs # execute background jobs
Scheduler.worker(true) Scheduler.worker(true)
SearchIndexBackend.refresh
ticket1 = Ticket.create!( ticket1 = Ticket.create!(
title: "some title\n äöüß", title: "some title\n äöüß",
@ -295,7 +297,7 @@ class ElasticsearchTest < ActiveSupport::TestCase
# execute background jobs # execute background jobs
Scheduler.worker(true) Scheduler.worker(true)
sleep 2 # for ES to come ready/indexed SearchIndexBackend.refresh
# search as @agent # search as @agent
@ -433,7 +435,7 @@ class ElasticsearchTest < ActiveSupport::TestCase
# execute background jobs # execute background jobs
Scheduler.worker(true) Scheduler.worker(true)
sleep 2 # for ES to come ready/indexed SearchIndexBackend.refresh
# search for tags # search for tags
result = Ticket.search( result = Ticket.search(

View file

@ -94,60 +94,10 @@ class UserAssetsTest < ActiveSupport::TestCase
attributes.delete('authorization_ids') attributes.delete('authorization_ids')
assert(diff(attributes, assets[:User][user3.id]), 'check assets') assert(diff(attributes, assets[:User][user3.id]), 'check assets')
# touch org, check if user1 has changed
travel 2.seconds
org2 = Organization.find(org1.id)
org2.note = "some note...#{rand(9_999_999_999_999)}"
org2.save!
attributes = org2.attributes_with_association_ids
attributes.delete('user_ids')
assert_not(diff(attributes, assets[:Organization][org2.id]), 'check assets')
user1_new = User.find(user1.id)
attributes = user1_new.attributes_with_association_ids
attributes['accounts'] = {}
attributes.delete('password')
attributes.delete('token_ids')
attributes.delete('authorization_ids')
assert_not(diff(attributes, assets[:User][user1_new.id]), 'check assets')
# check new assets lookup
assets = user3.assets({})
attributes = org2.attributes_with_association_ids
attributes.delete('user_ids')
assert(diff(attributes, assets[:Organization][org1.id]), 'check assets')
user1 = User.find(user1.id)
attributes = user1.attributes_with_association_ids
attributes['accounts'] = {}
attributes.delete('password')
attributes.delete('token_ids')
attributes.delete('authorization_ids')
assert(diff(attributes, assets[:User][user1.id]), 'check assets')
user2 = User.find(user2.id)
attributes = user2.attributes_with_association_ids
attributes['accounts'] = {}
attributes.delete('password')
attributes.delete('token_ids')
attributes.delete('authorization_ids')
assert(diff(attributes, assets[:User][user2.id]), 'check assets')
user3 = User.find(user3.id)
attributes = user3.attributes_with_association_ids
attributes['accounts'] = {}
attributes.delete('password')
attributes.delete('token_ids')
attributes.delete('authorization_ids')
assert(diff(attributes, assets[:User][user3.id]), 'check assets')
travel_back
user3.destroy! user3.destroy!
user2.destroy! user2.destroy!
user1.destroy! user1.destroy!
org1.destroy! org1.destroy!
assert_not(Organization.find_by(id: org2.id))
end end
def diff(object1, object2) def diff(object1, object2)