diff --git a/lib/search_index_backend.rb b/lib/search_index_backend.rb index 252e79297..501349ef0 100644 --- a/lib/search_index_backend.rb +++ b/lib/search_index_backend.rb @@ -926,4 +926,249 @@ helper method for making HTTP calls and raising error if response was not succes ) end +=begin + + This function will return a index mapping based on the + attributes of the database table of the existing object. + + mapping = SearchIndexBackend.get_mapping_properties_object(Ticket) + + Returns: + + mapping = { + User: { + properties: { + firstname: { + type: 'keyword', + }, + } + } + } + +=end + + def self.get_mapping_properties_object(object) + name = '_doc' + result = { + name => { + properties: {} + } + } + + store_columns = %w[preferences data] + + # for elasticsearch 6.x and later + string_type = 'text' + string_raw = { 'type': 'keyword', 'ignore_above': 5012 } + boolean_raw = { 'type': 'boolean' } + + object.columns_hash.each do |key, value| + if value.type == :string && value.limit && value.limit <= 5000 && store_columns.exclude?(key) + result[name][:properties][key] = { + type: string_type, + fields: { + keyword: string_raw, + } + } + elsif value.type == :integer + result[name][:properties][key] = { + type: 'integer', + } + elsif value.type == :datetime || value.type == :date + result[name][:properties][key] = { + type: 'date', + } + elsif value.type == :boolean + result[name][:properties][key] = { + type: 'boolean', + fields: { + keyword: boolean_raw, + } + } + elsif value.type == :binary + result[name][:properties][key] = { + type: 'binary', + } + elsif value.type == :bigint + result[name][:properties][key] = { + type: 'long', + } + elsif value.type == :decimal + result[name][:properties][key] = { + type: 'float', + } + end + end + + case object.name + when 'Ticket' + result[name][:_source] = { + excludes: ['article.attachment'] + } + result[name][:properties][:article] = { + type: 'nested', + include_in_parent: true, + } + when 'KnowledgeBase::Answer::Translation' + result[name][:_source] = { + excludes: ['attachment'] + } + end + + return result if type_in_mapping? + + result[name] + end + + # get es version + def self.version + @version ||= begin + info = SearchIndexBackend.info + number = nil + if info.present? + number = info['version']['number'].to_s + end + number + end + end + + def self.version_int + number = version + return 0 if !number + + number_split = version.split('.') + "#{number_split[0]}#{format('%03d', minor: number_split[1])}#{format('%03d', patch: number_split[2])}".to_i + end + + def self.version_supported? + + # only versions greater/equal than 6.5.0 are supported + return if version_int < 6_005_000 + + true + end + + # no type in mapping + def self.type_in_mapping? + return true if version_int < 7_000_000 + + false + end + + # is es configured? + def self.configured? + return false if Setting.get('es_url').blank? + + true + end + + def self.settings + { + 'index.mapping.total_fields.limit': 2000, + } + end + + def self.create_index(models = Models.indexable) + models.each do |local_object| + SearchIndexBackend.index( + action: 'create', + name: local_object.name, + data: { + mappings: SearchIndexBackend.get_mapping_properties_object(local_object), + settings: SearchIndexBackend.settings, + } + ) + end + end + + def self.drop_index(models = Models.indexable) + models.each do |local_object| + SearchIndexBackend.index( + action: 'delete', + name: local_object.name, + ) + end + end + + def self.create_object_index(object) + models = Models.indexable.select { |c| c.to_s == object } + create_index(models) + end + + def self.drop_object_index(object) + models = Models.indexable.select { |c| c.to_s == object } + drop_index(models) + end + + def self.pipeline(create: false) + pipeline = Setting.get('es_pipeline') + if create && pipeline.blank? + pipeline = "zammad#{rand(999_999_999_999)}" + Setting.set('es_pipeline', pipeline) + end + pipeline + end + + def self.pipeline_settings + { + ignore_failure: true, + ignore_missing: true, + } + end + + def self.create_pipeline + SearchIndexBackend.processors( + "_ingest/pipeline/#{pipeline(create: true)}": [ + { + action: 'delete', + }, + { + action: 'create', + description: 'Extract zammad-attachment information from arrays', + processors: [ + { + foreach: { + field: 'article', + processor: { + foreach: { + field: '_ingest._value.attachment', + processor: { + attachment: { + target_field: '_ingest._value', + field: '_ingest._value._content', + }.merge(pipeline_settings), + } + }.merge(pipeline_settings), + } + }.merge(pipeline_settings), + }, + { + foreach: { + field: 'attachment', + processor: { + attachment: { + target_field: '_ingest._value', + field: '_ingest._value._content', + }.merge(pipeline_settings), + } + }.merge(pipeline_settings), + } + ] + } + ] + ) + end + + def self.drop_pipeline + return if pipeline.blank? + + SearchIndexBackend.processors( + "_ingest/pipeline/#{pipeline}": [ + { + action: 'delete', + }, + ] + ) + end + end diff --git a/lib/tasks/search_index_es.rake b/lib/tasks/search_index_es.rake index 0118c78d6..f77e7d371 100644 --- a/lib/tasks/search_index_es.rake +++ b/lib/tasks/search_index_es.rake @@ -6,12 +6,7 @@ namespace :searchindex do print 'drop indexes...' # drop indexes - Models.indexable.each do |local_object| - SearchIndexBackend.index( - action: 'delete', - name: local_object.name, - ) - end + SearchIndexBackend.drop_index puts 'done' @@ -21,21 +16,7 @@ namespace :searchindex do task :create, [:opts] => %i[environment searchindex:configured searchindex:version_supported] do |_t, _args| print 'create indexes...' - settings = { - 'index.mapping.total_fields.limit': 2000, - } - - # create indexes - Models.indexable.each do |local_object| - SearchIndexBackend.index( - action: 'create', - name: local_object.name, - data: { - mappings: get_mapping_properties_object(local_object), - settings: settings, - } - ) - end + SearchIndexBackend.create_index puts 'done' @@ -43,77 +24,18 @@ namespace :searchindex do end task :create_pipeline, [:opts] => %i[environment searchindex:configured searchindex:version_supported] do |_t, _args| - - # update processors - pipeline = Setting.get('es_pipeline') - if pipeline.blank? - pipeline = "zammad#{rand(999_999_999_999)}" - Setting.set('es_pipeline', pipeline) - end - - pipeline_field_attributes = { - ignore_failure: true, - ignore_missing: true, - } - print 'create pipeline (pipeline)... ' - SearchIndexBackend.processors( - "_ingest/pipeline/#{pipeline}": [ - { - action: 'delete', - }, - { - action: 'create', - description: 'Extract zammad-attachment information from arrays', - processors: [ - { - foreach: { - field: 'article', - processor: { - foreach: { - field: '_ingest._value.attachment', - processor: { - attachment: { - target_field: '_ingest._value', - field: '_ingest._value._content', - }.merge(pipeline_field_attributes), - } - }.merge(pipeline_field_attributes), - } - }.merge(pipeline_field_attributes), - }, - { - foreach: { - field: 'attachment', - processor: { - attachment: { - target_field: '_ingest._value', - field: '_ingest._value._content', - }.merge(pipeline_field_attributes), - } - }.merge(pipeline_field_attributes), - } - ] - } - ] - ) + + SearchIndexBackend.create_pipeline + puts 'done' end task :drop_pipeline, [:opts] => %i[environment searchindex:configured searchindex:version_supported] do |_t, _args| - - # update processors - pipeline = Setting.get('es_pipeline') - next if pipeline.blank? - print 'delete pipeline (pipeline)... ' - SearchIndexBackend.processors( - "_ingest/pipeline/#{pipeline}": [ - { - action: 'delete', - }, - ] - ) + + SearchIndexBackend.drop_pipeline + puts 'done' end @@ -127,7 +49,6 @@ namespace :searchindex do took = Time.zone.now - started_at puts " - took #{took.to_i} seconds" end - end task :refresh, [:opts] => %i[environment searchindex:configured searchindex:version_supported] do |_t, _args| @@ -143,151 +64,14 @@ namespace :searchindex do end task :version_supported, [:opts] => :environment do |_t, _args| - next if es_version_supported? + next if SearchIndexBackend.version_supported? - abort "Your Elasticsearch version is not supported! Please update your version to a greater equal than 6.5.0 (Your current version: #{es_version})." + abort "Your Elasticsearch version is not supported! Please update your version to a greater equal than 6.5.0 (Your current version: #{SearchIndexBackend.version})." end task :configured, [:opts] => :environment do |_t, _args| - next if es_configured? + next if SearchIndexBackend.configured? abort "You have not configured Elasticsearch (Setting.get('es_url'))." end end - -=begin - -This function will return a index mapping based on the -attributes of the database table of the existing object. - -mapping = get_mapping_properties_object(Ticket) - -Returns: - -mapping = { - User: { - properties: { - firstname: { - type: 'keyword', - }, - } - } -} - -=end - -def get_mapping_properties_object(object) - - name = '_doc' - result = { - name => { - properties: {} - } - } - - store_columns = %w[preferences data] - - # for elasticsearch 6.x and later - string_type = 'text' - string_raw = { 'type': 'keyword', 'ignore_above': 5012 } - boolean_raw = { 'type': 'boolean' } - - object.columns_hash.each do |key, value| - if value.type == :string && value.limit && value.limit <= 5000 && store_columns.exclude?(key) - result[name][:properties][key] = { - type: string_type, - fields: { - keyword: string_raw, - } - } - elsif value.type == :integer - result[name][:properties][key] = { - type: 'integer', - } - elsif value.type == :datetime || value.type == :date - result[name][:properties][key] = { - type: 'date', - } - elsif value.type == :boolean - result[name][:properties][key] = { - type: 'boolean', - fields: { - keyword: boolean_raw, - } - } - elsif value.type == :binary - result[name][:properties][key] = { - type: 'binary', - } - elsif value.type == :bigint - result[name][:properties][key] = { - type: 'long', - } - elsif value.type == :decimal - result[name][:properties][key] = { - type: 'float', - } - end - end - - case object.name - when 'Ticket' - result[name][:_source] = { - excludes: ['article.attachment'] - } - result[name][:properties][:article] = { - type: 'nested', - include_in_parent: true, - } - when 'KnowledgeBase::Answer::Translation' - result[name][:_source] = { - excludes: ['attachment'] - } - end - - return result if es_type_in_mapping? - - result[name] -end - -# get es version -def es_version - @es_version ||= begin - info = SearchIndexBackend.info - number = nil - if info.present? - number = info['version']['number'].to_s - end - number - end -end - -def es_version_int - number = es_version - return 0 if !number - - number_split = es_version.split('.') - "#{number_split[0]}#{format('%03d', minor: number_split[1])}#{format('%03d', patch: number_split[2])}".to_i -end - -def es_version_supported? - - # only versions greater/equal than 6.5.0 are supported - return if es_version_int < 6_005_000 - - true -end - -# no type in mapping -def es_type_in_mapping? - return true if es_version_int < 7_000_000 - - false -end - -# is es configured? -def es_configured? - return false if Setting.get('es_url').blank? - - true -end