From 695d05689bf7e000fb1ed126307595b3a42856dc Mon Sep 17 00:00:00 2001 From: Mantas Masalskis Date: Wed, 19 Feb 2020 17:41:25 +0100 Subject: [PATCH] Fixes #2810 - Attachments of Knowledge Base Answers don't get added to Elasticsearch --- .../answer/attachments_controller.rb | 37 ++--------------- .../application_model/has_attachments.rb | 40 +++++++++++++++++++ app/models/knowledge_base/answer.rb | 31 ++++++++++++++ .../knowledge_base/answer/translation.rb | 7 ++-- lib/tasks/search_index_es.rake | 28 +++++++++++++ spec/fixtures/upload/test.rtf | 10 +++++ .../search_with_attachment_spec.rb | 24 +++++++++++ 7 files changed, 141 insertions(+), 36 deletions(-) create mode 100644 spec/fixtures/upload/test.rtf create mode 100644 spec/models/knowledge_base/answer/translation/search_with_attachment_spec.rb diff --git a/app/controllers/knowledge_base/answer/attachments_controller.rb b/app/controllers/knowledge_base/answer/attachments_controller.rb index 636f4d284..def885962 100644 --- a/app/controllers/knowledge_base/answer/attachments_controller.rb +++ b/app/controllers/knowledge_base/answer/attachments_controller.rb @@ -7,27 +7,15 @@ class KnowledgeBase::Answer::AttachmentsController < ApplicationController before_action :fetch_answer def create - file = params[:file] + @answer.add_attachment params[:file] - Store.add( - object: @answer.class.name, - o_id: @answer.id, - data: file.read, - filename: file.original_filename, - preferences: headers_for_file(file) - ) - - output + render json: @answer.assets({}) end def destroy - attachment = @answer.attachments.find { |elem| elem.id == params[:id].to_i } + @answer.remove_attachment params[:id] - raise ActiveRecord::RecordNotFound if attachment.nil? - - Store.remove_item(attachment.id) - - output + render json: @answer.assets({}) end private @@ -35,21 +23,4 @@ class KnowledgeBase::Answer::AttachmentsController < ApplicationController def fetch_answer @answer = KnowledgeBase::Answer.find params[:answer_id] end - - def output - @answer.touch # rubocop:disable Rails/SkipsModelValidations - render json: @answer.assets({}) - end - - def headers_for_file(file) - content_type = file.content_type || 'application/octet-stream' - - if content_type == 'application/octet-stream' && (mime = MIME::Types.type_for(file.original_filename).first) - content_type = mime - end - - { - 'Content-Type': content_type - } - end end diff --git a/app/models/application_model/has_attachments.rb b/app/models/application_model/has_attachments.rb index d9770bb0f..3c6825c75 100644 --- a/app/models/application_model/has_attachments.rb +++ b/app/models/application_model/has_attachments.rb @@ -59,6 +59,46 @@ store attachments for this object attachments_buffer_check end +=begin + +Returns attachments in ElasticSearch-compatible format +For use in #search_index_attribute_lookup + +=end + + def attachments_for_search_index_attribute_lookup + # list ignored file extensions + attachments_ignore = Setting.get('es_attachment_ignore') || [ '.png', '.jpg', '.jpeg', '.mpeg', '.mpg', '.mov', '.bin', '.exe' ] + + # max attachment size + attachment_max_size_in_mb = (Setting.get('es_attachment_max_size_in_mb') || 10).megabytes + attachment_total_max_size_in_kb = 314_572.kilobytes + attachment_total_max_size_in_kb_current = 0.kilobytes + + attachments.each_with_object([]) do |attachment, memo| + # check if attachment exists + next if !attachment.content + + size_in_bytes = attachment.content.size.bytes + + # check file size + next if size_in_bytes > attachment_max_size_in_mb + + # check ignored files + next if !attachment.filename || attachments_ignore.include?(File.extname(attachment.filename).downcase) + + # check if fits into total size limit + next if attachment_total_max_size_in_kb_current + size_in_bytes > attachment_total_max_size_in_kb + + attachment_total_max_size_in_kb_current += size_in_bytes + + memo << { + '_name' => attachment.filename, + '_content' => Base64.encode64(attachment.content).delete("\n") + } + end + end + private def attachments_buffer diff --git a/app/models/knowledge_base/answer.rb b/app/models/knowledge_base/answer.rb index ed81f6190..d0504751e 100644 --- a/app/models/knowledge_base/answer.rb +++ b/app/models/knowledge_base/answer.rb @@ -61,6 +61,37 @@ class KnowledgeBase::Answer < ApplicationModel attachments.sort_by { |elem| elem.filename.downcase } end + def add_attachment(file) + filename = file.try(:original_filename) || File.basename(file.path) + content_type = file.try(:content_type) || MIME::Types.type_for(filename).first&.content_type || 'application/octet-stream' + + Store.add( + object: self.class.name, + o_id: id, + data: file.read, + filename: filename, + preferences: { 'Content-Type': content_type } + ) + + touch # rubocop:disable Rails/SkipsModelValidations + translations.each(&:touch) + + true + end + + def remove_attachment(attachment_id) + attachment = attachments.find { |elem| elem.id == attachment_id.to_i } + + raise ActiveRecord::RecordNotFound if attachment.nil? + + Store.remove_item(attachment.id) + + touch # rubocop:disable Rails/SkipsModelValidations + translations.each(&:touch) + + true + end + def api_url Rails.application.routes.url_helpers.knowledge_base_answer_path(category.knowledge_base, self) end diff --git a/app/models/knowledge_base/answer/translation.rb b/app/models/knowledge_base/answer/translation.rb index b04738931..f829121b2 100644 --- a/app/models/knowledge_base/answer/translation.rb +++ b/app/models/knowledge_base/answer/translation.rb @@ -54,9 +54,10 @@ class KnowledgeBase::Answer::Translation < ApplicationModel def search_index_attribute_lookup attrs = super - attrs['title'] = ActionController::Base.helpers.strip_tags attrs['title'] - attrs['content'] = content.search_index_attribute_lookup if content - attrs['scope_id'] = answer.category_id + attrs['title'] = ActionController::Base.helpers.strip_tags attrs['title'] + attrs['content'] = content.search_index_attribute_lookup if content + attrs['scope_id'] = answer.category_id + attrs['attachment'] = answer.attachments_for_search_index_attribute_lookup attrs end diff --git a/lib/tasks/search_index_es.rake b/lib/tasks/search_index_es.rake index 964334254..61f87d2fb 100644 --- a/lib/tasks/search_index_es.rake +++ b/lib/tasks/search_index_es.rake @@ -116,6 +116,17 @@ namespace :searchindex do }.merge(pipeline_field_attributes), } }.merge(pipeline_field_attributes), + }, + { + foreach: { + field: 'attachment', + processor: { + attachment: { + target_field: '_ingest._value', + field: '_ingest._value._content', + }.merge(pipeline_field_attributes), + } + }.merge(pipeline_field_attributes), } ] } @@ -289,6 +300,23 @@ def get_mapping_properties_object(object) end end + if object.name == 'KnowledgeBase::Answer::Translation' + # do not server attachments if document is requested + result[name][:_source] = { + excludes: ['attachment'] + } + + # for elasticsearch 5.5 and lower + if !es_pipeline? + result[name][:_source] = { + excludes: ['attachment'] + } + result[name][:properties][:attachment] = { + type: 'attachment', + } + end + end + return result if es_type_in_mapping? result[name] diff --git a/spec/fixtures/upload/test.rtf b/spec/fixtures/upload/test.rtf new file mode 100644 index 000000000..fbc55e8f9 --- /dev/null +++ b/spec/fixtures/upload/test.rtf @@ -0,0 +1,10 @@ +{\rtf1\ansi\ansicpg1252\cocoartf1671\cocoasubrtf600 +{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +{\*\expandedcolortbl;;} +\paperw11900\paperh16840\margl1440\margr1440\vieww10800\viewh8400\viewkind0 +\pard\tx566\tx1133\tx1700\tx2267\tx2834\tx3401\tx3968\tx4535\tx5102\tx5669\tx6236\tx6803\pardirnatural\partightenfactor0 + +\f0\fs24 \cf0 Hello world from Rich Text +\f1\b RTF +\f0\b0 document} \ No newline at end of file diff --git a/spec/models/knowledge_base/answer/translation/search_with_attachment_spec.rb b/spec/models/knowledge_base/answer/translation/search_with_attachment_spec.rb new file mode 100644 index 000000000..bfa24bb3d --- /dev/null +++ b/spec/models/knowledge_base/answer/translation/search_with_attachment_spec.rb @@ -0,0 +1,24 @@ +require 'rails_helper' +require 'models/concerns/checks_kb_client_notification_examples' +require 'models/contexts/factory_context' + +RSpec.describe KnowledgeBase::Answer::Translation, type: :model, current_user_id: 1, searchindex: 1 do + include_context 'basic Knowledge Base' + + let(:user) { create(:admin_user) } + let(:filename) { 'test.rtf' } + let(:query) { 'RTF document' } + + context 'search with attachment' do + before do + configure_elasticsearch(required: true, rebuild: true) do + published_answer.add_attachment File.open "spec/fixtures/upload/#{filename}" + end + end + + it do + expect(described_class.search(query: query, current_user: user)) + .to include published_answer.translations.first + end + end +end