Sutty/trabajo-afectivo
5
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Small refactoring of search index backend.

Browse source
This commit is contained in:
Martin Edenhofer 2017-12-15 14:58:41 +01:00
parent 34c14a22b5
commit 8ac7356184
3 changed files with 95 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lib/report/ticket_generic_time.rb
View file

@ -30,7 +30,7 @@ returns
}
selector = params[:selector].clone
if params[:params] && params[:params][:selector]
if params[:params].present? && params[:params][:selector].present?
selector = selector.merge(params[:params][:selector])
end

38
lib/search_index_backend.rb
View file

@ -74,6 +74,7 @@ update processors
)
Rails.logger.info "# #{response.code}"
next if response.success?
next if response.code.to_s == '404'
raise "Unable to process DELETE at #{url}\n#{response.inspect}"
end
Rails.logger.info "# curl -X PUT \"#{url}\" \\"
@ -133,7 +134,7 @@ create/update/delete index
def self.index(data)
url = build_url(data[:name])
return if !url
return if url.blank?
if data[:action] && data[:action] == 'delete'
return SearchIndexBackend.remove(data[:name])
@ -169,7 +170,7 @@ add new object to search index
def self.add(type, data)
url = build_url(type, data['id'])
return if !url
return if url.blank?
Rails.logger.info "# curl -X POST \"#{url}\" \\"
Rails.logger.debug "-d '#{data.to_json}'"
@ -202,7 +203,7 @@ remove whole data from index
def self.remove(type, o_id = nil)
url = build_url(type, o_id)
return if !url
return if url.blank?
Rails.logger.info "# curl -X DELETE \"#{url}\""
@ -217,7 +218,8 @@ remove whole data from index
)
Rails.logger.info "# #{response.code}"
return true if response.success?
#Rails.logger.info "NOTICE: can't delete index #{url}: " + response.inspect
return true if response.code.to_s == '400'
Rails.logger.info "NOTICE: can't delete index #{url}: " + response.inspect
false
end
@ -247,7 +249,7 @@ return search result
=end
def self.search(query, limit = 10, index = nil, query_extention = {})
return [] if !query
return [] if query.blank?
if index.class == Array
ids = []
index.each do |local_index|
@ -260,10 +262,10 @@ return search result
end
def self.search_by_index(query, limit = 10, index = nil, query_extention = {})
return [] if !query
return [] if query.blank?
url = build_url
return if !url
return if url.blank?
url += if index
if index.class == Array
"/#{index.join(',')}/_search"
@ -287,12 +289,8 @@ return search result
]
data['query'] = query_extention || {}
if !data['query']['bool']
data['query']['bool'] = {}
end
if !data['query']['bool']['must']
data['query']['bool']['must'] = []
end
data['query']['bool'] ||= {}
data['query']['bool']['must'] ||= []
# add * on simple query like "somephrase23" or "attribute: somephrase23"
if query.present?
@ -391,7 +389,7 @@ get count of tickets and tickets which match on selector
raise 'no selectors given' if !selectors
url = build_url
return if !url
return if url.blank?
url += if index
if index.class == Array
"/#{index.join(',')}/_search"
@ -425,7 +423,7 @@ get count of tickets and tickets which match on selector
end
Rails.logger.debug response.data.to_json
if !aggs_interval || !aggs_interval[:interval]
if aggs_interval.blank? || aggs_interval[:interval].blank?
ticket_ids = []
response.data['hits']['hits'].each do |item|
ticket_ids.push item['_id']
@ -471,8 +469,8 @@ get count of tickets and tickets which match on selector
}
# add aggs to filter
if aggs_interval
if aggs_interval[:interval]
if aggs_interval.present?
if aggs_interval[:interval].present?
data[:size] = 0
data[:aggs] = {
time_buckets: {
@ -492,9 +490,7 @@ get count of tickets and tickets which match on selector
query_must.push r
end
if !data[:query][:bool]
data[:query][:bool] = {}
end
data[:query][:bool] ||= {}
if query_must.present?
data[:query][:bool][:must] = query_must
@ -504,7 +500,7 @@ get count of tickets and tickets which match on selector
end
# add sort
if aggs_interval && aggs_interval[:field] && !aggs_interval[:interval]
if aggs_interval.present? && aggs_interval[:field].present? && aggs_interval[:interval].blank?
sort = []
sort[0] = {}
sort[0][aggs_interval[:field]] = {

109
lib/tasks/search_index_es.rake
View file

@ -5,15 +5,17 @@ namespace :searchindex do
task :drop, [:opts] => :environment do |_t, _args|
# drop indexes
puts 'drop indexes...'
print 'drop indexes...'
SearchIndexBackend.index(
action: 'delete',
)
puts 'done'
Rake::Task['searchindex:drop_pipeline'].execute
end
task :create, [:opts] => :environment do |_t, _args|
puts 'create indexes...'
print 'create indexes...'
# es with mapper-attachments plugin
info = SearchIndexBackend.info
@ -45,6 +47,7 @@ namespace :searchindex do
}
}
)
puts 'done'
Setting.set('es_pipeline', '')
# es with ingest-attachment plugin
@ -61,44 +64,86 @@ namespace :searchindex do
}
}
)
puts 'done'
end
# update processors
pipeline = 'zammad-attachment'
Rake::Task['searchindex:create_pipeline'].execute
end
task :create_pipeline, [:opts] => :environment do |_t, _args|
# es with mapper-attachments plugin
info = SearchIndexBackend.info
number = nil
if info.present?
number = info['version']['number'].to_s
end
next if number.blank? || number =~ /^[2-4]\./ || number =~ /^5\.[0-5]\./
# update processors
pipeline = Setting.get('es_pipeline')
if pipeline.blank?
pipeline = "zammad#{rand(999_999_999_999)}"
Setting.set('es_pipeline', pipeline)
SearchIndexBackend.processors(
"_ingest/pipeline/#{pipeline}": [
{
action: 'delete',
},
{
action: 'create',
description: 'Extract zammad-attachment information from arrays',
processors: [
{
foreach: {
field: 'article',
ignore_failure: true,
processor: {
foreach: {
field: '_ingest._value.attachment',
ignore_failure: true,
processor: {
attachment: {
target_field: '_ingest._value',
field: '_ingest._value._content',
ignore_failure: true,
}
end
print 'create pipeline (pipeline)... '
SearchIndexBackend.processors(
"_ingest/pipeline/#{pipeline}": [
{
action: 'delete',
},
{
action: 'create',
description: 'Extract zammad-attachment information from arrays',
processors: [
{
foreach: {
field: 'article',
ignore_failure: true,
processor: {
foreach: {
field: '_ingest._value.attachment',
ignore_failure: true,
processor: {
attachment: {
target_field: '_ingest._value',
field: '_ingest._value._content',
ignore_failure: true,
}
}
}
}
}
]
}
]
)
end
}
]
}
]
)
puts 'done'
end
task :drop_pipeline, [:opts] => :environment do |_t, _args|
# es with mapper-attachments plugin
info = SearchIndexBackend.info
number = nil
if info.present?
number = info['version']['number'].to_s
end
next if number.blank? || number =~ /^[2-4]\./ || number =~ /^5\.[0-5]\./
# update processors
pipeline = Setting.get('es_pipeline')
next if pipeline.blank?
print 'delete pipeline (pipeline)... '
SearchIndexBackend.processors(
"_ingest/pipeline/#{pipeline}": [
{
action: 'delete',
},
]
)
puts 'done'
end
task :reload, [:opts] => :environment do |_t, _args|