trabajo-afectivo/lib/search_index_backend.rb

478 lines
11 KiB
Ruby
Raw Normal View History

2016-10-19 03:11:36 +00:00
# Copyright (C) 2012-2016 Zammad Foundation, http://zammad-foundation.org/
class SearchIndexBackend
=begin
create/update/delete index
SearchIndexBackend.index(
:action => 'create', # create/update/delete
:data => {
:mappings => {
:Ticket => {
:properties => {
2015-04-06 19:00:16 +00:00
:articles => {
:type => 'nested',
:properties => {
'attachments' => { :type => 'attachment' }
2014-04-28 15:30:06 +00:00
}
}
2014-04-28 15:30:06 +00:00
}
}
}
}
)
SearchIndexBackend.index(
:action => 'delete', # create/update/delete
:name => 'Ticket', # optional
)
SearchIndexBackend.index(
:action => 'delete', # create/update/delete
)
=end
def self.index(data)
2016-04-26 12:42:55 +00:00
url = build_url(data[:name])
return if !url
if data[:action] && data[:action] == 'delete'
2016-04-26 12:42:55 +00:00
return SearchIndexBackend.remove(data[:name])
end
2015-05-04 18:58:28 +00:00
Rails.logger.info "# curl -X PUT \"#{url}\" \\"
2015-10-20 08:48:43 +00:00
Rails.logger.debug "-d '#{data[:data].to_json}'"
response = UserAgent.put(
url,
data[:data],
{
json: true,
open_timeout: 5,
read_timeout: 20,
user: Setting.get('es_user'),
password: Setting.get('es_password'),
}
)
2015-05-05 08:26:53 +00:00
Rails.logger.info "# #{response.code}"
return true if response.success?
raise "Unable to process PUT at #{url}\n#{response.inspect}"
end
=begin
add new object to search index
2016-04-26 12:42:55 +00:00
SearchIndexBackend.add('Ticket', some_data_object)
=end
def self.add(type, data)
2016-04-26 12:42:55 +00:00
url = build_url(type, data['id'])
return if !url
2015-05-04 18:58:28 +00:00
Rails.logger.info "# curl -X POST \"#{url}\" \\"
2015-10-20 08:48:43 +00:00
Rails.logger.debug "-d '#{data.to_json}'"
response = UserAgent.post(
url,
data,
{
json: true,
open_timeout: 5,
read_timeout: 20,
user: Setting.get('es_user'),
password: Setting.get('es_password'),
}
)
Rails.logger.info "# #{response.code}"
return true if response.success?
raise "Unable to process POST at #{url} (size: #{data.to_json.bytesize / 1024 / 1024}M)\n#{response.inspect}"
end
=begin
remove whole data from index
2016-04-26 12:42:55 +00:00
SearchIndexBackend.remove('Ticket', 123)
2016-04-26 12:42:55 +00:00
SearchIndexBackend.remove('Ticket')
=end
2016-04-26 12:42:55 +00:00
def self.remove(type, o_id = nil)
url = build_url(type, o_id)
return if !url
2015-05-05 08:26:53 +00:00
Rails.logger.info "# curl -X DELETE \"#{url}\""
response = UserAgent.delete(
url,
{
open_timeout: 5,
read_timeout: 14,
user: Setting.get('es_user'),
password: Setting.get('es_password'),
}
)
2015-10-20 08:48:43 +00:00
Rails.logger.info "# #{response.code}"
return true if response.success?
#Rails.logger.info "NOTICE: can't delete index #{url}: " + response.inspect
2015-04-06 19:00:16 +00:00
false
end
=begin
2014-01-29 23:51:55 +00:00
return search result
2016-04-26 12:42:55 +00:00
result = SearchIndexBackend.search('search query', limit, ['User', 'Organization'])
2014-09-19 21:35:40 +00:00
2016-04-26 12:42:55 +00:00
result = SearchIndexBackend.search('search query', limit, 'User')
2014-09-19 21:35:40 +00:00
result = [
{
:id => 123,
:type => 'User',
},
{
:id => 125,
:type => 'User',
},
{
:id => 15,
:type => 'Organization',
}
]
=end
2016-04-26 12:42:55 +00:00
def self.search(query, limit = 10, index = nil, query_extention = {})
2014-01-29 23:51:55 +00:00
return [] if !query
if index.class == Array
ids = []
index.each { |local_index|
local_ids = search_by_index(query, limit, local_index, query_extention)
ids = ids.concat(local_ids)
}
return ids
end
search_by_index(query, limit, index, query_extention)
end
def self.search_by_index(query, limit = 10, index = nil, query_extention = {})
return [] if !query
2014-01-29 23:51:55 +00:00
url = build_url()
return if !url
url += if index
if index.class == Array
"/#{index.join(',')}/_search"
else
"/#{index}/_search"
end
else
'/_search'
end
2014-01-29 23:51:55 +00:00
data = {}
data['from'] = 0
data['size'] = limit
2014-04-28 15:30:06 +00:00
data['sort'] =
[
{
updated_at: {
order: 'desc'
}
},
'_score'
]
2014-02-02 18:58:31 +00:00
data['query'] = query_extention || {}
if !data['query']['bool']
data['query']['bool'] = {}
end
if !data['query']['bool']['must']
data['query']['bool']['must'] = []
end
2014-01-29 23:51:55 +00:00
2016-03-14 22:29:39 +00:00
# add * on simple query search
if query && query =~ /^\w+$/
query += '*'
end
# real search condition
2014-02-02 18:58:31 +00:00
condition = {
'query_string' => {
'query' => query
}
}
data['query']['bool']['must'].push condition
2014-01-29 23:51:55 +00:00
2015-05-04 18:58:28 +00:00
Rails.logger.info "# curl -X POST \"#{url}\" \\"
2015-10-20 08:48:43 +00:00
Rails.logger.debug " -d'#{data.to_json}'"
2014-01-29 23:51:55 +00:00
response = UserAgent.get(
url,
data,
{
json: true,
open_timeout: 5,
read_timeout: 14,
user: Setting.get('es_user'),
password: Setting.get('es_password'),
}
)
Rails.logger.info "# #{response.code}"
2014-02-03 18:26:22 +00:00
if !response.success?
Rails.logger.error "ERROR: POST on #{url}\n#{response.inspect}"
return []
2014-02-03 18:26:22 +00:00
end
data = response.data
2014-02-03 18:26:22 +00:00
2014-01-29 23:51:55 +00:00
ids = []
return ids if !data
return ids if !data['hits']
return ids if !data['hits']['hits']
data['hits']['hits'].each { |item|
Rails.logger.info "... #{item['_type']} #{item['_id']}"
2014-09-19 21:35:40 +00:00
data = {
id: item['_id'],
type: item['_type'],
2014-09-19 21:35:40 +00:00
}
ids.push data
2014-01-29 23:51:55 +00:00
}
2014-02-03 18:26:22 +00:00
ids
end
2014-02-02 18:58:31 +00:00
=begin
2015-10-29 02:33:36 +00:00
get count of tickets and tickets which match on selector
2015-10-20 08:48:43 +00:00
2015-10-29 02:33:36 +00:00
aggs_interval = {
from: '2015-01-01',
to: '2015-12-31',
interval: 'month', # year, quarter, month, week, day, hour, minute, second
field: 'created_at',
}
2015-10-20 08:48:43 +00:00
2015-10-29 02:33:36 +00:00
result = SearchIndexBackend.selectors(index, params[:condition], limit, current_user, aggs_interval)
2015-10-20 08:48:43 +00:00
2015-10-29 02:33:36 +00:00
# for aggregations
2015-10-20 08:48:43 +00:00
result = {
hits:{
total:4819,
},
aggregations:{
time_buckets:{
buckets:[
{
key_as_string:"2014-10-01T00:00:00.000Z",
key:1412121600000,
doc_count:420
},
{
key_as_string:"2014-11-01T00:00:00.000Z",
key:1414800000000,
doc_count:561
},
...
]
}
}
}
=end
2015-10-29 18:20:22 +00:00
def self.selectors(index = nil, selectors = nil, limit = 10, current_user = nil, aggs_interval = nil)
2016-03-01 14:26:46 +00:00
raise 'no selectors given' if !selectors
2015-10-20 08:48:43 +00:00
url = build_url()
return if !url
url += if index
if index.class == Array
"/#{index.join(',')}/_search"
else
"/#{index}/_search"
end
else
'/_search'
end
2015-10-20 08:48:43 +00:00
2015-10-29 18:20:22 +00:00
data = selector2query(selectors, current_user, aggs_interval, limit)
2015-10-20 08:48:43 +00:00
Rails.logger.info "# curl -X POST \"#{url}\" \\"
Rails.logger.debug " -d'#{data.to_json}'"
response = UserAgent.get(
url,
data,
{
json: true,
open_timeout: 5,
read_timeout: 14,
user: Setting.get('es_user'),
password: Setting.get('es_password'),
}
)
Rails.logger.info "# #{response.code}"
if !response.success?
raise "Unable to process POST at #{url}\n#{response.inspect}"
2015-10-20 08:48:43 +00:00
end
Rails.logger.debug response.data.to_json
2015-10-29 02:33:36 +00:00
if !aggs_interval || !aggs_interval[:interval]
ticket_ids = []
2016-06-30 20:04:48 +00:00
response.data['hits']['hits'].each { |item|
2015-10-29 02:33:36 +00:00
ticket_ids.push item['_id']
}
return {
count: response.data['hits']['total'],
ticket_ids: ticket_ids,
}
end
2015-10-20 08:48:43 +00:00
response.data
end
2015-10-29 18:20:22 +00:00
def self.selector2query(selector, _current_user, aggs_interval, limit)
2015-10-29 02:33:36 +00:00
filter_must = []
filter_must_not = []
query_must = []
query_must_not = []
if selector && !selector.empty?
2016-06-30 20:04:48 +00:00
selector.each { |key, data|
2015-10-29 02:33:36 +00:00
key_tmp = key.sub(/^.+?\./, '')
t = {}
if data['value'].class == Array
t[:terms] = {}
t[:terms][key_tmp] = data['value']
else
t[:term] = {}
t[:term][key_tmp] = data['value']
end
if data['operator'] == 'is'
filter_must.push t
elsif data['operator'] == 'is not'
filter_must_not.push t
elsif data['operator'] == 'contains'
query_must.push t
elsif data['operator'] == 'contains not'
query_must_not.push t
else
2016-03-01 14:26:46 +00:00
raise "unknown operator '#{data['operator']}'"
2015-10-29 02:33:36 +00:00
end
}
end
data = {
query: {},
2015-10-29 18:20:22 +00:00
size: limit,
2015-10-29 02:33:36 +00:00
}
# add aggs to filter
if aggs_interval
if aggs_interval[:interval]
data[:size] = 0
data[:aggs] = {
time_buckets: {
date_histogram: {
field: aggs_interval[:field],
interval: aggs_interval[:interval],
}
}
}
end
r = {}
r[:range] = {}
r[:range][aggs_interval[:field]] = {
from: aggs_interval[:from],
to: aggs_interval[:to],
}
filter_must.push r
end
if !query_must.empty? || !query_must_not.empty?
if !data[:query][:filtered]
data[:query][:filtered] = {}
end
if !data[:query][:filtered][:query]
data[:query][:filtered][:query] = {}
end
if !data[:query][:filtered][:query][:bool]
data[:query][:filtered][:query][:bool] = {}
end
end
if !query_must.empty?
data[:query][:filtered][:query][:bool][:must] = query_must
end
if !query_must_not.empty?
data[:query][:filtered][:query][:bool][:must_not] = query_must_not
end
if !filter_must.empty? || !filter_must.empty?
if !data[:query][:filtered]
data[:query][:filtered] = {}
end
if !data[:query][:filtered][:filter]
data[:query][:filtered][:filter] = {}
end
if !data[:query][:filtered][:filter][:bool]
data[:query][:filtered][:filter][:bool] = {}
end
end
if !filter_must.empty?
data[:query][:filtered][:filter][:bool][:must] = filter_must
end
if !filter_must_not.empty?
data[:query][:filtered][:filter][:bool][:must_not] = filter_must_not
end
2015-10-29 09:07:45 +00:00
# add sort
if aggs_interval && aggs_interval[:field] && !aggs_interval[:interval]
sort = []
sort[0] = {}
sort[0][aggs_interval[:field]] = {
order: 'desc'
}
sort[1] = '_score'
data['sort'] = sort
end
2015-10-29 02:33:36 +00:00
data
end
2015-10-20 08:48:43 +00:00
=begin
2014-04-28 15:30:06 +00:00
return true if backend is configured
2014-02-02 18:58:31 +00:00
result = SearchIndexBackend.enabled?
=end
def self.enabled?
return if !Setting.get('es_url')
return if Setting.get('es_url').empty?
true
end
2016-04-26 12:42:55 +00:00
def self.build_url(type = nil, o_id = nil)
2014-02-02 18:58:31 +00:00
return if !SearchIndexBackend.enabled?
index = "#{Setting.get('es_index')}_#{Rails.env}"
url = Setting.get('es_url')
url = if type
if o_id
"#{url}/#{index}/#{type}/#{o_id}"
else
"#{url}/#{index}/#{type}"
end
else
"#{url}/#{index}"
end
url
end
end