Sutty/trabajo-afectivo
5
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Small refactoring of search index backend.

Browse source
This commit is contained in:
Martin Edenhofer 2017-12-15 14:58:41 +01:00
parent 34c14a22b5
commit 8ac7356184
3 changed files with 95 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lib/report/ticket_generic_time.rb
View file

@ -30,7 +30,7 @@ returns
} }
selector = params[:selector].clone selector = params[:selector].clone
if params[:params] && params[:params][:selector] if params[:params].present? && params[:params][:selector].present?
selector = selector.merge(params[:params][:selector]) selector = selector.merge(params[:params][:selector])
end end

38
lib/search_index_backend.rb
View file

@ -74,6 +74,7 @@ update processors
) )
Rails.logger.info "# #{response.code}" Rails.logger.info "# #{response.code}"
next if response.success? next if response.success?
next if response.code.to_s == '404'
raise "Unable to process DELETE at #{url}\n#{response.inspect}" raise "Unable to process DELETE at #{url}\n#{response.inspect}"
end end
Rails.logger.info "# curl -X PUT \"#{url}\" \\" Rails.logger.info "# curl -X PUT \"#{url}\" \\"
@ -133,7 +134,7 @@ create/update/delete index
def self.index(data) def self.index(data)
url = build_url(data[:name]) url = build_url(data[:name])
return if !url return if url.blank?
if data[:action] && data[:action] == 'delete' if data[:action] && data[:action] == 'delete'
return SearchIndexBackend.remove(data[:name]) return SearchIndexBackend.remove(data[:name])
@ -169,7 +170,7 @@ add new object to search index
def self.add(type, data) def self.add(type, data)
url = build_url(type, data['id']) url = build_url(type, data['id'])
return if !url return if url.blank?
Rails.logger.info "# curl -X POST \"#{url}\" \\" Rails.logger.info "# curl -X POST \"#{url}\" \\"
Rails.logger.debug "-d '#{data.to_json}'" Rails.logger.debug "-d '#{data.to_json}'"
@ -202,7 +203,7 @@ remove whole data from index
def self.remove(type, o_id = nil) def self.remove(type, o_id = nil)
url = build_url(type, o_id) url = build_url(type, o_id)
return if !url return if url.blank?
Rails.logger.info "# curl -X DELETE \"#{url}\"" Rails.logger.info "# curl -X DELETE \"#{url}\""
@ -217,7 +218,8 @@ remove whole data from index
) )
Rails.logger.info "# #{response.code}" Rails.logger.info "# #{response.code}"
return true if response.success? return true if response.success?
#Rails.logger.info "NOTICE: can't delete index #{url}: " + response.inspect return true if response.code.to_s == '400'
Rails.logger.info "NOTICE: can't delete index #{url}: " + response.inspect
false false
end end
@ -247,7 +249,7 @@ return search result
=end =end
def self.search(query, limit = 10, index = nil, query_extention = {}) def self.search(query, limit = 10, index = nil, query_extention = {})
return [] if !query return [] if query.blank?
if index.class == Array if index.class == Array
ids = [] ids = []
index.each do |local_index| index.each do |local_index|
@ -260,10 +262,10 @@ return search result
end end
def self.search_by_index(query, limit = 10, index = nil, query_extention = {}) def self.search_by_index(query, limit = 10, index = nil, query_extention = {})
return [] if !query return [] if query.blank?
url = build_url url = build_url
return if !url return if url.blank?
url += if index url += if index
if index.class == Array if index.class == Array
"/#{index.join(',')}/_search" "/#{index.join(',')}/_search"
@ -287,12 +289,8 @@ return search result
] ]
data['query'] = query_extention || {} data['query'] = query_extention || {}
if !data['query']['bool'] data['query']['bool'] ||= {}
data['query']['bool'] = {} data['query']['bool']['must'] ||= []
end
if !data['query']['bool']['must']
data['query']['bool']['must'] = []
end
# add * on simple query like "somephrase23" or "attribute: somephrase23" # add * on simple query like "somephrase23" or "attribute: somephrase23"
if query.present? if query.present?
@ -391,7 +389,7 @@ get count of tickets and tickets which match on selector
raise 'no selectors given' if !selectors raise 'no selectors given' if !selectors
url = build_url url = build_url
return if !url return if url.blank?
url += if index url += if index
if index.class == Array if index.class == Array
"/#{index.join(',')}/_search" "/#{index.join(',')}/_search"
@ -425,7 +423,7 @@ get count of tickets and tickets which match on selector
end end
Rails.logger.debug response.data.to_json Rails.logger.debug response.data.to_json
if !aggs_interval || !aggs_interval[:interval] if aggs_interval.blank? || aggs_interval[:interval].blank?
ticket_ids = [] ticket_ids = []
response.data['hits']['hits'].each do |item| response.data['hits']['hits'].each do |item|
ticket_ids.push item['_id'] ticket_ids.push item['_id']
@ -471,8 +469,8 @@ get count of tickets and tickets which match on selector
} }
# add aggs to filter # add aggs to filter
if aggs_interval if aggs_interval.present?
if aggs_interval[:interval] if aggs_interval[:interval].present?
data[:size] = 0 data[:size] = 0
data[:aggs] = { data[:aggs] = {
time_buckets: { time_buckets: {
@ -492,9 +490,7 @@ get count of tickets and tickets which match on selector
query_must.push r query_must.push r
end end
if !data[:query][:bool] data[:query][:bool] ||= {}
data[:query][:bool] = {}
end
if query_must.present? if query_must.present?
data[:query][:bool][:must] = query_must data[:query][:bool][:must] = query_must
@ -504,7 +500,7 @@ get count of tickets and tickets which match on selector
end end
# add sort # add sort
if aggs_interval && aggs_interval[:field] && !aggs_interval[:interval] if aggs_interval.present? && aggs_interval[:field].present? && aggs_interval[:interval].blank?
sort = [] sort = []
sort[0] = {} sort[0] = {}
sort[0][aggs_interval[:field]] = { sort[0][aggs_interval[:field]] = {

109
lib/tasks/search_index_es.rake
View file

@ -5,15 +5,17 @@ namespace :searchindex do
task :drop, [:opts] => :environment do |_t, _args| task :drop, [:opts] => :environment do |_t, _args|
# drop indexes # drop indexes
puts 'drop indexes...' print 'drop indexes...'
SearchIndexBackend.index( SearchIndexBackend.index(
action: 'delete', action: 'delete',
) )
puts 'done'
Rake::Task['searchindex:drop_pipeline'].execute
end end
task :create, [:opts] => :environment do |_t, _args| task :create, [:opts] => :environment do |_t, _args|
puts 'create indexes...' print 'create indexes...'
# es with mapper-attachments plugin # es with mapper-attachments plugin
info = SearchIndexBackend.info info = SearchIndexBackend.info
@ -45,6 +47,7 @@ namespace :searchindex do
} }
} }
) )
puts 'done'
Setting.set('es_pipeline', '') Setting.set('es_pipeline', '')
# es with ingest-attachment plugin # es with ingest-attachment plugin
@ -61,44 +64,86 @@ namespace :searchindex do
} }
} }
) )
puts 'done'
end
# update processors Rake::Task['searchindex:create_pipeline'].execute
pipeline = 'zammad-attachment' end
task :create_pipeline, [:opts] => :environment do |_t, _args|
# es with mapper-attachments plugin
info = SearchIndexBackend.info
number = nil
if info.present?
number = info['version']['number'].to_s
end
next if number.blank? || number =~ /^[2-4]\./ || number =~ /^5\.[0-5]\./
# update processors
pipeline = Setting.get('es_pipeline')
if pipeline.blank?
pipeline = "zammad#{rand(999_999_999_999)}"
Setting.set('es_pipeline', pipeline) Setting.set('es_pipeline', pipeline)
SearchIndexBackend.processors( end
"_ingest/pipeline/#{pipeline}": [ print 'create pipeline (pipeline)... '
{ SearchIndexBackend.processors(
action: 'delete', "_ingest/pipeline/#{pipeline}": [
}, {
{ action: 'delete',
action: 'create', },
description: 'Extract zammad-attachment information from arrays', {
processors: [ action: 'create',
{ description: 'Extract zammad-attachment information from arrays',
foreach: { processors: [
field: 'article', {
ignore_failure: true, foreach: {
processor: { field: 'article',
foreach: { ignore_failure: true,
field: '_ingest._value.attachment', processor: {
ignore_failure: true, foreach: {
processor: { field: '_ingest._value.attachment',
attachment: { ignore_failure: true,
target_field: '_ingest._value', processor: {
field: '_ingest._value._content', attachment: {
ignore_failure: true, target_field: '_ingest._value',
} field: '_ingest._value._content',
ignore_failure: true,
} }
} }
} }
} }
} }
] }
} ]
] }
) ]
end )
puts 'done'
end
task :drop_pipeline, [:opts] => :environment do |_t, _args|
# es with mapper-attachments plugin
info = SearchIndexBackend.info
number = nil
if info.present?
number = info['version']['number'].to_s
end
next if number.blank? || number =~ /^[2-4]\./ || number =~ /^5\.[0-5]\./
# update processors
pipeline = Setting.get('es_pipeline')
next if pipeline.blank?
print 'delete pipeline (pipeline)... '
SearchIndexBackend.processors(
"_ingest/pipeline/#{pipeline}": [
{
action: 'delete',
},
]
)
puts 'done'
end end
task :reload, [:opts] => :environment do |_t, _args| task :reload, [:opts] => :environment do |_t, _args|