From 604e0575ddab765e757703ffa8147fbcef56b2d9 Mon Sep 17 00:00:00 2001 From: Martin Edenhofer Date: Mon, 6 Apr 2015 01:28:52 +0200 Subject: [PATCH] Improved es behaviours. --- .../application_model/search_index_base.rb | 8 ++- app/models/ticket/search_index.rb | 66 ++++++++++++------- lib/tasks/search_index_es.rake | 2 +- test/integration/elasticsearch_test.rb | 6 +- 4 files changed, 53 insertions(+), 29 deletions(-) diff --git a/app/models/application_model/search_index_base.rb b/app/models/application_model/search_index_base.rb index 3865023ef..c357cd9c4 100644 --- a/app/models/application_model/search_index_base.rb +++ b/app/models/application_model/search_index_base.rb @@ -30,15 +30,19 @@ returns } end + # for performance reasons, Model.search_index_reload will only collect if of object + # get whole data here + data = self.class.find(self.id) + # remove ignored attributes - attributes = self.attributes + attributes = data.attributes ignore_attributes.each {|key, value| next if value != true attributes.delete( key.to_s ) } # fill up with search data - attributes = search_index_attribute_lookup(attributes, self) + attributes = search_index_attribute_lookup(attributes, data) return if !attributes # update backend diff --git a/app/models/ticket/search_index.rb b/app/models/ticket/search_index.rb index f97044414..380de6592 100644 --- a/app/models/ticket/search_index.rb +++ b/app/models/ticket/search_index.rb @@ -30,7 +30,12 @@ returns } end - attributes = self.attributes + # for performance reasons, Model.search_index_reload will only collect if of object + # get whole data here + ticket = self.class.find(self.id) + + # remove ignored attributes + attributes = ticket.attributes ignore_attributes.each {|key, value| next if value != true attributes.delete( key.to_s ) @@ -42,43 +47,58 @@ returns attributes[:tag] = tags end - attributes = search_index_attribute_lookup( attributes, self ) + # lookup attributes of ref. objects (normally name and note) + attributes = search_index_attribute_lookup( attributes, ticket ) - # add article data + # list ignored file extentions + ignore_attachments = [ '.png', '.jpg', '.jpeg', '.mpeg', '.mov' ] + + # collect article data articles = Ticket::Article.where( :ticket_id => self.id ) attributes['articles_all'] = [] - attributes['articles_external'] = [] articles.each {|article| article_attributes = article.attributes - article_attributes.delete('created_by_id') - article_attributes.delete('updated_by_id') - article_attributes.delete('updated_at') - article_attributes.delete('references') - article_attributes.delete('message_id_md5') - article_attributes.delete('message_id') - article_attributes.delete('in_reply_to') - article_attributes.delete('ticket_id') + + # remove note needed attributes + ignore = ['created_by_id', 'updated_by_id', 'updated_at', 'references', 'message_id_md5', 'message_id', 'in_reply_to', 'ticket_id'] + ignore.each {|attribute| + article_attributes.delete( attribute ) + } + + # lookup attributes of ref. objects (normally name and note) article_attributes = search_index_attribute_lookup( article_attributes, article ) + # index raw text body + if article_attributes['content_type'] && article_attributes['content_type'] == 'text/html' && article_attributes['body'] + article_attributes['body'] = article_attributes['body'].html2text + end + # lookup attachments article.attachments.each {|attachment| - if !article_attributes['attachments'] - article_attributes['attachments'] = [] + if !attributes['attachments'] + attributes['attachments'] = [] + end + + # check file size + + # check ignored files + if attachment.filename + filename_extention = attachment.filename.downcase + filename_extention.gsub!(/^.*(\..+?)$/, "\\1") + if !ignore_attachments.include?( filename_extention.downcase ) + data = { + "_name" => attachment.filename, + "content" => Base64.encode64( attachment.content ) + } + attributes['attachments'].push data + end end - data = { - "_name" => attachment.filename, - "content" => Base64.encode64( attachment.content ) - } - article_attributes['attachments'].push data } attributes['articles_all'].push article_attributes - if !article.internal - attributes['articles_external'].push article_attributes - end } return if !attributes SearchIndexBackend.add(self.class.to_s, attributes) end -end +end \ No newline at end of file diff --git a/lib/tasks/search_index_es.rake b/lib/tasks/search_index_es.rake index 64148fca9..b28a28121 100644 --- a/lib/tasks/search_index_es.rake +++ b/lib/tasks/search_index_es.rake @@ -21,7 +21,7 @@ namespace :searchindex do :data => { :mappings => { :Ticket => { - :_source => { :excludes => [ 'articles_all.attachments', 'articles_external.attachments' ] }, + :_source => { :excludes => [ 'articles_all.attachments' ] }, :properties => { :articles_all => { :type => 'nested', diff --git a/test/integration/elasticsearch_test.rb b/test/integration/elasticsearch_test.rb index ac03714f7..56d214431 100644 --- a/test/integration/elasticsearch_test.rb +++ b/test/integration/elasticsearch_test.rb @@ -56,10 +56,10 @@ class ElasticsearchTest < ActiveSupport::TestCase :created_by_id => 1, ) customer2 = User.create_or_update( - :login => 'es-customer1@example.com', + :login => 'es-customer2@example.com', :firstname => 'ES', - :lastname => 'Customer1', - :email => 'es-customer1@example.com', + :lastname => 'Customer2', + :email => 'es-customer2@example.com', :password => 'customerpw', :active => true, :organization_id => organization1.id,