From d0b5cbfe4b2aeccb5ace6a546a5104c298af891b Mon Sep 17 00:00:00 2001 From: Rolf Schmidt Date: Wed, 27 Jan 2021 10:58:35 +0100 Subject: [PATCH] Fixes #3243 - Relational report filters do not work (ticket.organization.*, ticket.article.*, ticket.customer.*). --- .gitlab/ci/integration/es.yml | 18 ++- .rubocop/todo.yml | 1 + .../can_lookup_search_index_attributes.rb | 39 +---- app/models/chat/session/search_index.rb | 2 +- .../concerns/has_search_index_backend.rb | 32 +--- .../knowledge_base/answer/translation.rb | 2 +- .../answer/translation/content.rb | 2 +- .../knowledge_base/category/translation.rb | 2 +- app/models/knowledge_base/translation.rb | 2 +- app/models/organization.rb | 4 +- app/models/organization/search_index.rb | 28 ++-- app/models/stats_store.rb | 1 - app/models/stats_store/search_index.rb | 20 --- app/models/ticket/search_index.rb | 17 +-- app/models/user/search_index.rb | 67 ++------ ...3095141_elastic_search_lower65_obsolete.rb | 9 ++ db/seeds/settings.rb | 9 -- lib/report/ticket_generic_time.rb | 4 +- lib/search_index_backend.rb | 42 +++-- lib/tasks/search_index_es.rake | 144 ++++-------------- spec/lib/search_index_backend_spec.rb | 116 +++++++++++++- spec/models/chat/session_spec.rb | 2 +- ...lookup_search_index_attributes_examples.rb | 11 +- spec/requests/search_spec.rb | 8 +- test/integration/elasticsearch_test.rb | 25 +-- 25 files changed, 227 insertions(+), 380 deletions(-) delete mode 100644 app/models/stats_store/search_index.rb create mode 100644 db/migrate/20201013095141_elastic_search_lower65_obsolete.rb diff --git a/.gitlab/ci/integration/es.yml b/.gitlab/ci/integration/es.yml index 6317dd090..80b73d9b8 100644 --- a/.gitlab/ci/integration/es.yml +++ b/.gitlab/ci/integration/es.yml @@ -13,20 +13,26 @@ - bundle exec rspec --tag searchindex --tag ~type:system - bundle exec rails test test/integration/report_test.rb -es:5.6: - <<: *template_integration_es - variables: - ELASTICSEARCH_TAG: '5.6' - RAILS_ENV: "test" - es:6: <<: *template_integration_es variables: ELASTICSEARCH_TAG: '6' RAILS_ENV: "test" +es:6.5: + <<: *template_integration_es + variables: + ELASTICSEARCH_TAG: '6.5' + RAILS_ENV: "test" + es:7: <<: *template_integration_es variables: ELASTICSEARCH_TAG: '7' RAILS_ENV: "test" + +es:7.0: + <<: *template_integration_es + variables: + ELASTICSEARCH_TAG: '7.0' + RAILS_ENV: "test" diff --git a/.rubocop/todo.yml b/.rubocop/todo.yml index b4afbc8ea..6d96503cb 100644 --- a/.rubocop/todo.yml +++ b/.rubocop/todo.yml @@ -733,6 +733,7 @@ Metrics/PerceivedComplexity: - 'app/models/application_model/checks_attribute_values_and_length.rb' - 'app/models/application_model/checks_user_columns_fillup.rb' - 'app/models/application_model/has_attachments.rb' + - 'app/models/application_model/can_lookup_search_index_attributes.rb' - 'app/models/authorization.rb' - 'app/models/avatar.rb' - 'app/models/calendar.rb' diff --git a/app/models/application_model/can_lookup_search_index_attributes.rb b/app/models/application_model/can_lookup_search_index_attributes.rb index 6d57ffbc5..68937b4be 100644 --- a/app/models/application_model/can_lookup_search_index_attributes.rb +++ b/app/models/application_model/can_lookup_search_index_attributes.rb @@ -4,10 +4,13 @@ module ApplicationModel::CanLookupSearchIndexAttributes =begin -lookup name of ref. objects +This function return the attributes for the elastic search with relation hash values. + +It can be run with parameter include_references: false to skip the relational hashes to prevent endless loops. ticket = Ticket.find(3) attributes = ticket.search_index_attribute_lookup + attributes = ticket.search_index_attribute_lookup(include_references: false) returns @@ -15,10 +18,11 @@ returns =end - def search_index_attribute_lookup - + def search_index_attribute_lookup(include_references: true) attributes = self.attributes self.attributes.each do |key, value| + break if !include_references + attribute_name = key.to_s # ignore standard attribute if needed @@ -74,34 +78,7 @@ returns relation_model = relation_class.lookup(id: attributes[attribute_name]) return if !relation_model - relation_model.search_index_value - end - -=begin - -This function returns the relational search value. - - organization = Organization.find(1) - value = organization.search_index_value - -returns - - value = {"name"=>"Zammad Foundation"} - -=end - - def search_index_value - - # get name of ref object - value = nil - if respond_to?('name') - value = send('name') - elsif respond_to?('search_index_data') - value = send('search_index_data') - return if value == true - end - - value + relation_model.search_index_attribute_lookup(include_references: false) end =begin diff --git a/app/models/chat/session/search_index.rb b/app/models/chat/session/search_index.rb index a76f977ae..ec1fbaefc 100644 --- a/app/models/chat/session/search_index.rb +++ b/app/models/chat/session/search_index.rb @@ -15,7 +15,7 @@ returns =end - def search_index_attribute_lookup + def search_index_attribute_lookup(include_references: true) attributes = super return if !attributes diff --git a/app/models/concerns/has_search_index_backend.rb b/app/models/concerns/has_search_index_backend.rb index b6b9a4e1b..626279dd5 100644 --- a/app/models/concerns/has_search_index_backend.rb +++ b/app/models/concerns/has_search_index_backend.rb @@ -92,7 +92,7 @@ returns # start background job to transfer data to search index return true if !SearchIndexBackend.enabled? - new_search_index_value = search_index_value + new_search_index_value = search_index_attribute_lookup(include_references: false) return if new_search_index_value.blank? Models.indexable.each do |local_object| @@ -174,36 +174,6 @@ returns true end -=begin - -get data to store in search index - - ticket = Ticket.find(123) - result = ticket.search_index_data - -returns - - result = { - attribute1: 'some value', - attribute2: ['value 1', 'value 2'], - ... - } - -=end - - def search_index_data - attributes = {} - %w[name note].each do |key| - next if !self[key] - next if self[key].respond_to?('blank?') && self[key].blank? - - attributes[key] = self[key] - end - return true if attributes.blank? - - attributes - end - def ignore_search_indexing?(_action) false end diff --git a/app/models/knowledge_base/answer/translation.rb b/app/models/knowledge_base/answer/translation.rb index 8e96df97e..0e1dfe7c0 100644 --- a/app/models/knowledge_base/answer/translation.rb +++ b/app/models/knowledge_base/answer/translation.rb @@ -51,7 +51,7 @@ class KnowledgeBase::Answer::Translation < ApplicationModel [answer_id, title.parameterize].join('-') end - def search_index_attribute_lookup + def search_index_attribute_lookup(include_references: true) attrs = super attrs['title'] = ActionController::Base.helpers.strip_tags attrs['title'] diff --git a/app/models/knowledge_base/answer/translation/content.rb b/app/models/knowledge_base/answer/translation/content.rb index b2d925e97..e0ca0bb1e 100644 --- a/app/models/knowledge_base/answer/translation/content.rb +++ b/app/models/knowledge_base/answer/translation/content.rb @@ -39,7 +39,7 @@ class KnowledgeBase::Answer::Translation::Content < ApplicationModel attributes end - def search_index_attribute_lookup + def search_index_attribute_lookup(include_references: true) attrs = super attrs['body'] = ActionController::Base.helpers.strip_tags attrs['body'] attrs diff --git a/app/models/knowledge_base/category/translation.rb b/app/models/knowledge_base/category/translation.rb index d6022c75d..ca2f8394b 100644 --- a/app/models/knowledge_base/category/translation.rb +++ b/app/models/knowledge_base/category/translation.rb @@ -29,7 +29,7 @@ class KnowledgeBase::Category::Translation < ApplicationModel category.assets(data) end - def search_index_attribute_lookup + def search_index_attribute_lookup(include_references: true) attrs = super attrs['title'] = ActionController::Base.helpers.strip_tags attrs['title'] diff --git a/app/models/knowledge_base/translation.rb b/app/models/knowledge_base/translation.rb index 938448af3..b29a99963 100644 --- a/app/models/knowledge_base/translation.rb +++ b/app/models/knowledge_base/translation.rb @@ -19,7 +19,7 @@ class KnowledgeBase::Translation < ApplicationModel knowledge_base.assets(data) end - def search_index_attribute_lookup + def search_index_attribute_lookup(include_references: true) attrs = super attrs['title'] = ActionController::Base.helpers.strip_tags attrs['title'] diff --git a/app/models/organization.rb b/app/models/organization.rb index 2b83c7efa..9ac1ce3a8 100644 --- a/app/models/organization.rb +++ b/app/models/organization.rb @@ -17,6 +17,8 @@ class Organization < ApplicationModel has_many :members, class_name: 'User' has_many :tickets, class_name: 'Ticket' + belongs_to :created_by, class_name: 'User' + belongs_to :updated_by, class_name: 'User' before_create :domain_cleanup before_update :domain_cleanup @@ -25,7 +27,7 @@ class Organization < ApplicationModel validates :name, presence: true validates :domain, presence: { message: 'required when Domain Based Assignment is enabled' }, if: :domain_assignment - association_attributes_ignored :tickets + association_attributes_ignored :tickets, :created_by, :updated_by activity_stream_permission 'admin.role' diff --git a/app/models/organization/search_index.rb b/app/models/organization/search_index.rb index 139b1b43b..1b373c930 100644 --- a/app/models/organization/search_index.rb +++ b/app/models/organization/search_index.rb @@ -4,27 +4,17 @@ class Organization module SearchIndex extend ActiveSupport::Concern -=begin - -lookup name of ref. objects - - organization = Organization.find(123) - attributes = organization.search_index_attribute_lookup - -returns - - attributes # object with lookup data - -=end - - def search_index_attribute_lookup + def search_index_attribute_lookup(include_references: true) attributes = super - # add org members for search index data - attributes['members'] = [] - users = User.where(organization_id: id) - users.each do |user| - attributes['members'].push user.search_index_data + if include_references + + # add org members for search index data + attributes['members'] = [] + users = User.where(organization_id: id) + users.each do |user| + attributes['members'].push user.search_index_attribute_lookup(include_references: false) + end end attributes diff --git a/app/models/stats_store.rb b/app/models/stats_store.rb index 7f3fbcf17..a30ea0209 100644 --- a/app/models/stats_store.rb +++ b/app/models/stats_store.rb @@ -2,7 +2,6 @@ class StatsStore < ApplicationModel include HasSearchIndexBackend - include StatsStore::SearchIndex belongs_to :stats_storable, polymorphic: true diff --git a/app/models/stats_store/search_index.rb b/app/models/stats_store/search_index.rb deleted file mode 100644 index 386fdadfc..000000000 --- a/app/models/stats_store/search_index.rb +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (C) 2012-2016 Zammad Foundation, http://zammad-foundation.org/ -module StatsStore::SearchIndex - - def search_index_attribute_lookup - attributes = super - return if !attributes - - begin - record = attributes['stats_store_object'].constantize.lookup(id: o_id) - return if !record - - attributes['stats_store_object_ref'] = record.search_index_attribute_lookup - rescue - return - end - - attributes - end - -end diff --git a/app/models/ticket/search_index.rb b/app/models/ticket/search_index.rb index e8217bf1c..96cc4ad78 100644 --- a/app/models/ticket/search_index.rb +++ b/app/models/ticket/search_index.rb @@ -2,20 +2,7 @@ module Ticket::SearchIndex extend ActiveSupport::Concern -=begin - -lookup name of ref. objects - - ticket = Ticket.find(123) - result = ticket.search_index_attribute_lookup - -returns - - attributes # object with lookup data - -=end - - def search_index_attribute_lookup + def search_index_attribute_lookup(include_references: true) attributes = super return if !attributes @@ -40,7 +27,7 @@ returns articles.each do |article| # lookup attributes of ref. objects (normally name and note) - article_attributes = article.search_index_attribute_lookup + article_attributes = article.search_index_attribute_lookup(include_references: false) # remove note needed attributes ignore = %w[message_id_md5 ticket] diff --git a/app/models/user/search_index.rb b/app/models/user/search_index.rb index 26b1c87a6..c91c87a60 100644 --- a/app/models/user/search_index.rb +++ b/app/models/user/search_index.rb @@ -4,68 +4,19 @@ class User module SearchIndex extend ActiveSupport::Concern -=begin - -lookup name of ref. objects - - user = User.find(123) - attributes = user.search_index_attribute_lookup - -returns - - attributes # object with lookup data - -=end - - def search_index_attribute_lookup + def search_index_attribute_lookup(include_references: true) attributes = super + attributes.delete('password') - attributes['permissions'] = [] - permissions_with_child_ids.each do |permission_id| - permission = ::Permission.lookup(id: permission_id) - next if !permission + if include_references + attributes['permissions'] = [] + permissions_with_child_ids.each do |permission_id| + permission = ::Permission.lookup(id: permission_id) + next if !permission - attributes['permissions'].push permission.name - end - attributes['role_ids'] = role_ids - - attributes - end - -=begin - -get data to store in search index - - user = User.find(2) - result = user.search_index_data - -returns - - result = { - attribute1: 'some value', - attribute2: ['value 1', 'value 2'], - ... - } - -=end - - def search_index_data - attributes = {} - self.attributes.each do |key, value| - next if key == 'password' - next if !value - next if value.respond_to?('blank?') && value.blank? - - attributes[key] = value - end - return if attributes.blank? - - if attributes['organization_id'].present? - organization = Organization.lookup(id: attributes['organization_id']) - if organization - attributes['organization'] = organization.name - attributes['organization_ref'] = organization.search_index_data + attributes['permissions'].push permission.name end + attributes['role_ids'] = role_ids end attributes diff --git a/db/migrate/20201013095141_elastic_search_lower65_obsolete.rb b/db/migrate/20201013095141_elastic_search_lower65_obsolete.rb new file mode 100644 index 000000000..3ef8f97df --- /dev/null +++ b/db/migrate/20201013095141_elastic_search_lower65_obsolete.rb @@ -0,0 +1,9 @@ +class ElasticSearchLower65Obsolete < ActiveRecord::Migration[5.2] + def change + + # return if it's a new setup + return if !Setting.exists?(name: 'system_init_done') + + Setting.find_by(name: 'es_multi_index').destroy + end +end diff --git a/db/seeds/settings.rb b/db/seeds/settings.rb index 2fa159eee..a89e8f63b 100644 --- a/db/seeds/settings.rb +++ b/db/seeds/settings.rb @@ -2980,15 +2980,6 @@ Setting.create_if_not_exists( preferences: { online_service_disable: true }, frontend: false ) -Setting.create_if_not_exists( - title: 'Elasticsearch Multi Index', - name: 'es_multi_index', - area: 'SearchIndex::Elasticsearch', - description: 'Define if Elasticsearch is using multiple indexes.', - state: false, - preferences: { online_service_disable: true }, - frontend: false -) Setting.create_if_not_exists( title: 'Import Mode', diff --git a/lib/report/ticket_generic_time.rb b/lib/report/ticket_generic_time.rb index e508f00a8..a12fb9e12 100644 --- a/lib/report/ticket_generic_time.rb +++ b/lib/report/ticket_generic_time.rb @@ -33,7 +33,7 @@ returns } without_merged_tickets = { - 'state' => { + 'state.name' => { 'operator' => 'is not', 'value' => 'merged' } @@ -156,7 +156,7 @@ returns end without_merged_tickets = { - 'state' => { + 'state.name' => { 'operator' => 'is not', 'value' => 'merged' } diff --git a/lib/search_index_backend.rb b/lib/search_index_backend.rb index ae5bc233c..252e79297 100644 --- a/lib/search_index_backend.rb +++ b/lib/search_index_backend.rb @@ -479,9 +479,9 @@ example for aggregations within one year current_user_id = current_user.id end - query_must = [] + query_must = [] query_must_not = [] - relative_map = { + relative_map = { day: 'd', year: 'y', month: 'M', @@ -490,7 +490,13 @@ example for aggregations within one year } if selector.present? selector.each do |key, data| - key_tmp = key.sub(/^.+?\./, '') + data = data.clone + table, key_tmp = key.split(/\./) + if key_tmp.blank? + key_tmp = table + table = 'ticket' + end + wildcard_or_term = 'term' if data['value'].is_a?(Array) wildcard_or_term = 'terms' @@ -539,22 +545,27 @@ example for aggregations within one year # use .keyword and wildcard search in cases where query contains non A-z chars if data['operator'] == 'contains' || data['operator'] == 'contains not' + if data['value'].is_a?(Array) data['value'].each_with_index do |value, index| - next if !value.is_a?(String) || value !~ /[A-z]/ || value !~ /\W/ + next if !value.is_a?(String) || value !~ /[A-z]/ data['value'][index] = "*#{value}*" key_tmp += '.keyword' wildcard_or_term = 'wildcards' break end - elsif data['value'].is_a?(String) && /[A-z]/.match?(data['value']) && data['value'] =~ /\W/ + elsif data['value'].is_a?(String) && /[A-z]/.match?(data['value']) data['value'] = "*#{data['value']}*" key_tmp += '.keyword' wildcard_or_term = 'wildcard' end end + if table != 'ticket' + key_tmp = "#{table}.#{key_tmp}" + end + # is/is not/contains/contains not case data['operator'] when 'is', 'is not', 'contains', 'contains not' @@ -702,29 +713,12 @@ return true if backend is configured def self.build_index_name(index = nil) local_index = "#{Setting.get('es_index')}_#{Rails.env}" return local_index if index.blank? - return "#{local_index}/#{index}" if lower_equal_es56? "#{local_index}_#{index.underscore.tr('/', '_')}" end =begin -return true if the elastic search version is lower equal 5.6 - - result = SearchIndexBackend.lower_equal_es56? - -returns - - result = true - -=end - - def self.lower_equal_es56? - Setting.get('es_multi_index') == false - end - -=begin - generate url for index or document access (only for internal use) # url to access single document in index (in case with_pipeline or not) @@ -770,7 +764,7 @@ generate url for index or document access (only for internal use) url = "#{url}/#{index}" # add document type - if with_document_type && !lower_equal_es56? + if with_document_type url = "#{url}/_doc" end @@ -801,7 +795,7 @@ generate url for index or document access (only for internal use) message = if response&.error&.match?('Connection refused') "Elasticsearch is not reachable, probably because it's not running or even installed." elsif url.end_with?('pipeline/zammad-attachment', 'pipeline=zammad-attachment') && response.code == 400 - 'The installed attachment plugin could not handle the request payload. Ensure that the correct attachment plugin is installed (5.6 => ingest-attachment, 2.4 - 5.5 => mapper-attachments).' + 'The installed attachment plugin could not handle the request payload. Ensure that the correct attachment plugin is installed (ingest-attachment).' else 'Check the response and payload for detailed information: ' end diff --git a/lib/tasks/search_index_es.rake b/lib/tasks/search_index_es.rake index 9c56fd4e1..0118c78d6 100644 --- a/lib/tasks/search_index_es.rake +++ b/lib/tasks/search_index_es.rake @@ -6,18 +6,13 @@ namespace :searchindex do print 'drop indexes...' # drop indexes - if es_multi_index? - Models.indexable.each do |local_object| - SearchIndexBackend.index( - action: 'delete', - name: local_object.name, - ) - end - else + Models.indexable.each do |local_object| SearchIndexBackend.index( action: 'delete', + name: local_object.name, ) end + puts 'done' Rake::Task['searchindex:drop_pipeline'].execute @@ -26,37 +21,17 @@ namespace :searchindex do task :create, [:opts] => %i[environment searchindex:configured searchindex:version_supported] do |_t, _args| print 'create indexes...' - if es_multi_index? - Setting.set('es_multi_index', true) - else - Setting.set('es_multi_index', false) - end - settings = { 'index.mapping.total_fields.limit': 2000, } # create indexes - if es_multi_index? - Models.indexable.each do |local_object| - SearchIndexBackend.index( - action: 'create', - name: local_object.name, - data: { - mappings: get_mapping_properties_object(local_object), - settings: settings, - } - ) - end - else - mapping = {} - Models.indexable.each do |local_object| - mapping.merge!(get_mapping_properties_object(local_object)) - end + Models.indexable.each do |local_object| SearchIndexBackend.index( action: 'create', + name: local_object.name, data: { - mappings: mapping, + mappings: get_mapping_properties_object(local_object), settings: settings, } ) @@ -68,10 +43,6 @@ namespace :searchindex do end task :create_pipeline, [:opts] => %i[environment searchindex:configured searchindex:version_supported] do |_t, _args| - if !es_pipeline? - Setting.set('es_pipeline', '') - next - end # update processors pipeline = Setting.get('es_pipeline') @@ -80,17 +51,11 @@ namespace :searchindex do Setting.set('es_pipeline', pipeline) end - # define pipeline_field_attributes - # ES 5.6 and nower has no ignore_missing support pipeline_field_attributes = { ignore_failure: true, + ignore_missing: true, } - if es_multi_index? - pipeline_field_attributes = { - ignore_failure: true, - ignore_missing: true, - } - end + print 'create pipeline (pipeline)... ' SearchIndexBackend.processors( "_ingest/pipeline/#{pipeline}": [ @@ -136,7 +101,6 @@ namespace :searchindex do end task :drop_pipeline, [:opts] => %i[environment searchindex:configured searchindex:version_supported] do |_t, _args| - next if !es_pipeline? # update processors pipeline = Setting.get('es_pipeline') @@ -181,7 +145,7 @@ namespace :searchindex do task :version_supported, [:opts] => :environment do |_t, _args| next if es_version_supported? - abort "Your Elasticsearch version is not supported! Please update your version to a greater equal than 5.6.0 (Your current version: #{es_version})." + abort "Your Elasticsearch version is not supported! Please update your version to a greater equal than 6.5.0 (Your current version: #{es_version})." end task :configured, [:opts] => :environment do |_t, _args| @@ -214,10 +178,7 @@ mapping = { def get_mapping_properties_object(object) - name = object.name - if es_multi_index? - name = '_doc' - end + name = '_doc' result = { name => { properties: {} @@ -231,13 +192,6 @@ def get_mapping_properties_object(object) string_raw = { 'type': 'keyword', 'ignore_above': 5012 } boolean_raw = { 'type': 'boolean' } - # for elasticsearch 5.6 and lower - if !es_multi_index? - string_type = 'string' - string_raw = { 'type': 'string', 'index': 'not_analyzed' } - boolean_raw = { 'type': 'boolean', 'index': 'not_analyzed' } - end - object.columns_hash.each do |key, value| if value.type == :string && value.limit && value.limit <= 5000 && store_columns.exclude?(key) result[name][:properties][key] = { @@ -276,46 +230,19 @@ def get_mapping_properties_object(object) end end - # es with mapper-attachments plugin - if object.name == 'Ticket' - - # do not server attachments if document is requested + case object.name + when 'Ticket' result[name][:_source] = { excludes: ['article.attachment'] } - - # for elasticsearch 5.5 and lower - if !es_pipeline? - result[name][:_source] = { - excludes: ['article.attachment'] - } - result[name][:properties][:article] = { - type: 'nested', - include_in_parent: true, - properties: { - attachment: { - type: 'attachment', - } - } - } - end - end - - if object.name == 'KnowledgeBase::Answer::Translation' - # do not server attachments if document is requested + result[name][:properties][:article] = { + type: 'nested', + include_in_parent: true, + } + when 'KnowledgeBase::Answer::Translation' result[name][:_source] = { excludes: ['attachment'] } - - # for elasticsearch 5.5 and lower - if !es_pipeline? - result[name][:_source] = { - excludes: ['attachment'] - } - result[name][:properties][:attachment] = { - type: 'attachment', - } - end end return result if es_type_in_mapping? @@ -335,40 +262,25 @@ def es_version end end +def es_version_int + number = es_version + return 0 if !number + + number_split = es_version.split('.') + "#{number_split[0]}#{format('%03d', minor: number_split[1])}#{format('%03d', patch: number_split[2])}".to_i +end + def es_version_supported? - version_split = es_version.split('.') - version = "#{version_split[0]}#{format('%03d', minor: version_split[1])}#{format('%03d', patch: version_split[2])}".to_i - # only versions greater/equal than 5.6.0 are supported - return if version < 5_006_000 - - true -end - -# no es_pipeline for elasticsearch 5.5 and lower -def es_pipeline? - number = es_version - return false if number.blank? - return false if number.match?(/^[2-4]\./) - return false if number.match?(/^5\.[0-5]\./) - - true -end - -# no multi index for elasticsearch 5.6 and lower -def es_multi_index? - number = es_version - return false if number.blank? - return false if number.match?(/^[2-5]\./) + # only versions greater/equal than 6.5.0 are supported + return if es_version_int < 6_005_000 true end # no type in mapping def es_type_in_mapping? - number = es_version - return true if number.blank? - return true if number.match?(/^[2-6]\./) + return true if es_version_int < 7_000_000 false end diff --git a/spec/lib/search_index_backend_spec.rb b/spec/lib/search_index_backend_spec.rb index 30ac440c2..7058cbeae 100644 --- a/spec/lib/search_index_backend_spec.rb +++ b/spec/lib/search_index_backend_spec.rb @@ -164,17 +164,32 @@ RSpec.describe SearchIndexBackend, searchindex: true do describe '.selectors' do - let(:organization1) { create :organization } - let(:agent1) { create :agent, organization: organization1 } - let(:customer1) { create :customer, organization: organization1 } - let(:ticket1) { create :ticket, title: 'some-title1', state_id: 1, created_by: agent1 } - let(:ticket2) { create :ticket, title: 'some_title2', state_id: 4 } - let(:ticket3) { create :ticket, title: 'some::title3', state_id: 1 } + let(:group1) { create :group } + let(:organization1) { create :organization, note: 'hihi' } + let(:agent1) { create :agent, organization: organization1, groups: [group1] } + let(:customer1) { create :customer, organization: organization1, firstname: 'special-first-name' } + let(:ticket1) do + ticket = create :ticket, title: 'some-title1', state_id: 1, created_by: agent1 + ticket.tag_add('t1', 1) + ticket + end + let(:ticket2) do + ticket = create :ticket, title: 'some_title2', state_id: 4 + ticket.tag_add('t2', 1) + ticket + end + let(:ticket3) do + ticket = create :ticket, title: 'some::title3', state_id: 1 + ticket.tag_add('t1', 1) + ticket.tag_add('t2', 1) + ticket + end let(:ticket4) { create :ticket, title: 'phrase some-title4', state_id: 1 } let(:ticket5) { create :ticket, title: 'phrase some_title5', state_id: 1 } let(:ticket6) { create :ticket, title: 'phrase some::title6', state_id: 1 } let(:ticket7) { create :ticket, title: 'some title7', state_id: 1 } - let(:ticket8) { create :ticket, title: 'sometitle', state_id: 1, owner: agent1, customer: customer1 } + let(:ticket8) { create :ticket, title: 'sometitle', group: group1, state_id: 1, owner: agent1, customer: customer1, organization: organization1 } + let(:article8) { create :ticket_article, ticket: ticket8, subject: 'lorem ipsum' } before do Ticket.destroy_all # needed to remove not created tickets @@ -192,11 +207,96 @@ RSpec.describe SearchIndexBackend, searchindex: true do travel 1.second ticket7.search_index_update_backend travel 1.second - ticket8.search_index_update_backend + article8.ticket.search_index_update_backend described_class.refresh end context 'query with contains' do + it 'finds records with tags which contains all' do + result = described_class.selectors('Ticket', + { 'ticket.tags'=>{ 'operator' => 'contains all', 'value' => 't1, t2' } }, + {}, + { + field: 'created_at', # sort to verify result + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket3.id.to_s] }) + end + + it 'finds records with tags which contains one' do + result = described_class.selectors('Ticket', + { 'ticket.tags'=>{ 'operator' => 'contains one', 'value' => 't1, t2' } }, + {}, + { + field: 'created_at', # sort to verify result + }) + expect(result).to eq({ count: 3, ticket_ids: [ticket3.id.to_s, ticket2.id.to_s, ticket1.id.to_s] }) + end + + it 'finds records with tags which contains all not' do + result = described_class.selectors('Ticket', + { 'ticket.tags'=>{ 'operator' => 'contains all not', 'value' => 't2' } }, + {}, + { + field: 'created_at', # sort to verify result + }) + expect(result).to eq({ count: 6, ticket_ids: [ticket8.id.to_s, ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket1.id.to_s] }) + end + + it 'finds records with tags which contains one not' do + result = described_class.selectors('Ticket', + { 'ticket.tags'=>{ 'operator' => 'contains one not', 'value' => 't1' } }, + {}, + { + field: 'created_at', # sort to verify result + }) + expect(result).to eq({ count: 6, ticket_ids: [ticket8.id.to_s, ticket7.id.to_s, ticket6.id.to_s, ticket5.id.to_s, ticket4.id.to_s, ticket2.id.to_s] }) + end + + it 'finds records with organization note' do + result = described_class.selectors('Ticket', + { + 'organization.note' => { + 'operator' => 'contains', + 'value' => 'hihi', + }, + }, + {}, + { + field: 'created_at', # sort to verify result + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket8.id.to_s] }) + end + + it 'finds records with customer firstname' do + result = described_class.selectors('Ticket', + { + 'customer.firstname' => { + 'operator' => 'contains', + 'value' => 'special', + }, + }, + {}, + { + field: 'created_at', # sort to verify result + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket8.id.to_s] }) + end + + it 'finds records with article subject' do + result = described_class.selectors('Ticket', + { + 'article.subject' => { + 'operator' => 'contains', + 'value' => 'ipsum', + }, + }, + {}, + { + field: 'created_at', # sort to verify result + }) + expect(result).to eq({ count: 1, ticket_ids: [ticket8.id.to_s] }) + end + it 'finds records with pre_condition not_set' do result = described_class.selectors('Ticket', { diff --git a/spec/models/chat/session_spec.rb b/spec/models/chat/session_spec.rb index 0a6c7ec58..e2cc9ab2f 100644 --- a/spec/models/chat/session_spec.rb +++ b/spec/models/chat/session_spec.rb @@ -17,7 +17,7 @@ RSpec.describe Chat::Session, type: :model do end it 'verify chat attribute' do - expect(chat_session.search_index_attribute_lookup['chat']).to eq chat.name + expect(chat_session.search_index_attribute_lookup['chat']['name']).to eq chat.name end end diff --git a/spec/models/user/can_lookup_search_index_attributes_examples.rb b/spec/models/user/can_lookup_search_index_attributes_examples.rb index 9417f06fe..11dc2f014 100644 --- a/spec/models/user/can_lookup_search_index_attributes_examples.rb +++ b/spec/models/user/can_lookup_search_index_attributes_examples.rb @@ -5,16 +5,7 @@ RSpec.shared_examples 'CanLookupSearchIndexAttributes' do user = create(:agent, organization: organization) value = user.search_index_value_by_attribute('organization_id') - expect(value).to eq('Tomato42') - end - end - - describe '.search_index_value' do - it 'returns correct value' do - organization = create(:organization, name: 'Tomato42', note: 'special recipe') - - value = organization.search_index_value - expect(value).to eq('Tomato42') + expect(value['name']).to eq('Tomato42') end end diff --git a/spec/requests/search_spec.rb b/spec/requests/search_spec.rb index 9f25eb2ad..6e029a982 100644 --- a/spec/requests/search_spec.rb +++ b/spec/requests/search_spec.rb @@ -363,7 +363,7 @@ RSpec.describe 'Search', type: :request, searchindex: true do expect(json_response['assets']['Organization'][organization_nested.id.to_s]).to be_truthy expect(json_response['assets']['User'][customer_nested.id.to_s]).to be_truthy - post '/api/v1/search/User', params: { query: 'organization:Tomato42' }, as: :json + post '/api/v1/search/User', params: { query: 'organization.name:Tomato42' }, as: :json expect(response).to have_http_status(:ok) expect(json_response).to be_a_kind_of(Hash) expect(json_response).to be_truthy @@ -383,7 +383,7 @@ RSpec.describe 'Search', type: :request, searchindex: true do expect(json_response['assets']['Organization'][organization_nested.id.to_s]).to be_truthy expect(json_response['assets']['User'][customer_nested.id.to_s]).to be_truthy - post '/api/v1/search/User', params: { query: 'organization:Cucumber43' }, as: :json + post '/api/v1/search/User', params: { query: 'organization.name:Cucumber43' }, as: :json expect(response).to have_http_status(:ok) expect(json_response).to be_a_kind_of(Hash) expect(json_response).to be_truthy @@ -400,7 +400,7 @@ RSpec.describe 'Search', type: :request, searchindex: true do expect(json_response['assets']['Organization'][organization_nested.id.to_s]).to be_truthy expect(json_response['assets']['Ticket'][ticket_nested.id.to_s]).to be_truthy - post '/api/v1/search/Ticket', params: { query: 'organization:Tomato42' }, as: :json + post '/api/v1/search/Ticket', params: { query: 'organization.name:Tomato42' }, as: :json expect(response).to have_http_status(:ok) expect(json_response).to be_a_kind_of(Hash) expect(json_response).to be_truthy @@ -418,7 +418,7 @@ RSpec.describe 'Search', type: :request, searchindex: true do expect(json_response['assets']['Organization'][organization_nested.id.to_s]).to be_truthy expect(json_response['assets']['Ticket'][ticket_nested.id.to_s]).to be_truthy - post '/api/v1/search/Ticket', params: { query: 'organization:Cucumber43' }, as: :json + post '/api/v1/search/Ticket', params: { query: 'organization.name:Cucumber43' }, as: :json expect(response).to have_http_status(:ok) expect(json_response).to be_a_kind_of(Hash) expect(json_response).to be_truthy diff --git a/test/integration/elasticsearch_test.rb b/test/integration/elasticsearch_test.rb index 0511bb454..4d4cc511e 100644 --- a/test/integration/elasticsearch_test.rb +++ b/test/integration/elasticsearch_test.rb @@ -82,14 +82,6 @@ class ElasticsearchTest < ActiveSupport::TestCase test 'a - objects' do # user - attributes = @agent.search_index_data - assert_equal('E', attributes['firstname']) - assert_equal('S', attributes['lastname']) - assert_equal('es-agent@example.com', attributes['email']) - assert(attributes['preferences']) - assert_not(attributes['password']) - assert_not(attributes['organization']) - attributes = @agent.search_index_attribute_lookup assert_equal('E', attributes['firstname']) assert_equal('S', attributes['lastname']) @@ -104,14 +96,9 @@ class ElasticsearchTest < ActiveSupport::TestCase assert_equal('es-customer1@example.com', attributes['email']) assert(attributes['preferences']) assert_not(attributes['password']) - assert_equal('Customer Organization Update', attributes['organization']) + assert_equal('Customer Organization Update', attributes['organization']['name']) # organization - attributes = @organization1.search_index_data - assert_equal('Customer Organization Update', attributes['name']) - assert_equal('some note', attributes['note']) - assert_not(attributes['members']) - attributes = @organization1.search_index_attribute_lookup assert_equal('Customer Organization Update', attributes['name']) assert_equal('some note', attributes['note']) @@ -150,15 +137,15 @@ class ElasticsearchTest < ActiveSupport::TestCase ) attributes = ticket1.search_index_attribute_lookup - assert_equal('Users', attributes['group']) - assert_equal('new', attributes['state']) - assert_equal('2 normal', attributes['priority']) + assert_equal('Users', attributes['group']['name']) + assert_equal('new', attributes['state']['name']) + assert_equal('2 normal', attributes['priority']['name']) assert_equal('ES', attributes['customer']['firstname']) assert_equal('Customer1', attributes['customer']['lastname']) assert_equal('es-customer1@example.com', attributes['customer']['email']) assert_not(attributes['customer']['password']) - assert_equal('Customer Organization Update', attributes['customer']['organization']) + assert_nil(attributes['customer']['organization']) assert_equal('-', attributes['owner']['login']) assert_equal('-', attributes['owner']['firstname']) @@ -466,7 +453,7 @@ class ElasticsearchTest < ActiveSupport::TestCase result = Ticket.search( current_user: @agent, - query: 'state:open', + query: 'state.name:open', limit: 15, ) assert(result[0], 'record 1')