Maintenance: Make ES connection establishment more stable by:

- performing a retry when Net::OpenTimeout exceptions occur
- cache the result of ES version check
This commit is contained in:
Thorsten Eckel 2019-09-16 11:15:12 +02:00
parent 256e7513b3
commit 429b95e65d
3 changed files with 87 additions and 56 deletions

View file

@ -22,6 +22,7 @@ info about used search index machine
json: true, json: true,
open_timeout: 8, open_timeout: 8,
read_timeout: 14, read_timeout: 14,
open_socket_tries: 3,
user: Setting.get('es_user'), user: Setting.get('es_user'),
password: Setting.get('es_password'), password: Setting.get('es_password'),
} }
@ -85,6 +86,7 @@ update processors
json: true, json: true,
open_timeout: 8, open_timeout: 8,
read_timeout: 60, read_timeout: 60,
open_socket_tries: 3,
user: Setting.get('es_user'), user: Setting.get('es_user'),
password: Setting.get('es_password'), password: Setting.get('es_password'),
} }
@ -109,6 +111,7 @@ update processors
json: true, json: true,
open_timeout: 8, open_timeout: 8,
read_timeout: 60, read_timeout: 60,
open_socket_tries: 3,
user: Setting.get('es_user'), user: Setting.get('es_user'),
password: Setting.get('es_password'), password: Setting.get('es_password'),
} }
@ -182,6 +185,7 @@ create/update/delete index
json: true, json: true,
open_timeout: 8, open_timeout: 8,
read_timeout: 60, read_timeout: 60,
open_socket_tries: 3,
user: Setting.get('es_user'), user: Setting.get('es_user'),
password: Setting.get('es_password'), password: Setting.get('es_password'),
} }
@ -220,6 +224,7 @@ add new object to search index
json: true, json: true,
open_timeout: 8, open_timeout: 8,
read_timeout: 60, read_timeout: 60,
open_socket_tries: 3,
user: Setting.get('es_user'), user: Setting.get('es_user'),
password: Setting.get('es_password'), password: Setting.get('es_password'),
} }
@ -256,6 +261,7 @@ remove whole data from index
{ {
open_timeout: 8, open_timeout: 8,
read_timeout: 60, read_timeout: 60,
open_socket_tries: 3,
user: Setting.get('es_user'), user: Setting.get('es_user'),
password: Setting.get('es_password'), password: Setting.get('es_password'),
} }
@ -368,6 +374,7 @@ remove whole data from index
json: true, json: true,
open_timeout: 5, open_timeout: 5,
read_timeout: 14, read_timeout: 14,
open_socket_tries: 3,
user: Setting.get('es_user'), user: Setting.get('es_user'),
password: Setting.get('es_password'), password: Setting.get('es_password'),
} }
@ -515,6 +522,7 @@ example for aggregations within one year
json: true, json: true,
open_timeout: 5, open_timeout: 5,
read_timeout: 14, read_timeout: 14,
open_socket_tries: 3,
user: Setting.get('es_user'), user: Setting.get('es_user'),
password: Setting.get('es_password'), password: Setting.get('es_password'),
} }

View file

@ -275,6 +275,7 @@ end
# get es version # get es version
def es_version def es_version
@es_version ||= begin
info = SearchIndexBackend.info info = SearchIndexBackend.info
number = nil number = nil
if info.present? if info.present?
@ -282,6 +283,7 @@ def es_version
end end
number number
end end
end
# no es_pipeline for elasticsearch 5.5 and lower # no es_pipeline for elasticsearch 5.5 and lower
def es_pipeline? def es_pipeline?

View file

@ -59,10 +59,13 @@ returns
# start http call # start http call
begin begin
total_timeout = options[:total_timeout] || 60 total_timeout = options[:total_timeout] || 60
handled_open_timeout(options[:open_socket_tries]) do
Timeout.timeout(total_timeout) do Timeout.timeout(total_timeout) do
response = http.request(request) response = http.request(request)
return process(request, response, uri, count, params, options) return process(request, response, uri, count, params, options)
end end
end
rescue => e rescue => e
log(url, request, nil, options) log(url, request, nil, options)
return Result.new( return Result.new(
@ -112,10 +115,13 @@ returns
# start http call # start http call
begin begin
total_timeout = options[:total_timeout] || 60 total_timeout = options[:total_timeout] || 60
handled_open_timeout(options[:open_socket_tries]) do
Timeout.timeout(total_timeout) do Timeout.timeout(total_timeout) do
response = http.request(request) response = http.request(request)
return process(request, response, uri, count, params, options) return process(request, response, uri, count, params, options)
end end
end
rescue => e rescue => e
log(url, request, nil, options) log(url, request, nil, options)
return Result.new( return Result.new(
@ -164,10 +170,13 @@ returns
# start http call # start http call
begin begin
total_timeout = options[:total_timeout] || 60 total_timeout = options[:total_timeout] || 60
handled_open_timeout(options[:open_socket_tries]) do
Timeout.timeout(total_timeout) do Timeout.timeout(total_timeout) do
response = http.request(request) response = http.request(request)
return process(request, response, uri, count, params, options) return process(request, response, uri, count, params, options)
end end
end
rescue => e rescue => e
log(url, request, nil, options) log(url, request, nil, options)
return Result.new( return Result.new(
@ -209,10 +218,12 @@ returns
# start http call # start http call
begin begin
total_timeout = options[:total_timeout] || 60 total_timeout = options[:total_timeout] || 60
handled_open_timeout(options[:open_socket_tries]) do
Timeout.timeout(total_timeout) do Timeout.timeout(total_timeout) do
response = http.request(request) response = http.request(request)
return process(request, response, uri, count, {}, options) return process(request, response, uri, count, {}, options)
end end
end
rescue => e rescue => e
log(url, request, nil, options) log(url, request, nil, options)
return Result.new( return Result.new(
@ -488,6 +499,16 @@ returns
) )
end end
def self.handled_open_timeout(tries)
tries ||= 1
tries.times do |index|
yield
rescue Net::OpenTimeout
raise if (index + 1) == tries
end
end
class Result class Result
attr_reader :error attr_reader :error