Maintenance: Make ES connection establishment more stable by:

- performing a retry when Net::OpenTimeout exceptions occur
- cache the result of ES version check

lib/search_index_backend.rb

@@ -22,6 +22,7 @@ info about used search index machine
         json:              true,
         open_timeout:      8,
         read_timeout:      14,
+        open_socket_tries: 3,
         user:              Setting.get('es_user'),
         password:          Setting.get('es_password'),
       }
@@ -85,6 +86,7 @@ update processors
               json:              true,
               open_timeout:      8,
               read_timeout:      60,
+              open_socket_tries: 3,
               user:              Setting.get('es_user'),
               password:          Setting.get('es_password'),
             }
@@ -109,6 +111,7 @@ update processors
             json:              true,
             open_timeout:      8,
             read_timeout:      60,
+            open_socket_tries: 3,
             user:              Setting.get('es_user'),
             password:          Setting.get('es_password'),
           }
@@ -182,6 +185,7 @@ create/update/delete index
         json:              true,
         open_timeout:      8,
         read_timeout:      60,
+        open_socket_tries: 3,
         user:              Setting.get('es_user'),
         password:          Setting.get('es_password'),
       }
@@ -220,6 +224,7 @@ add new object to search index
         json:              true,
         open_timeout:      8,
         read_timeout:      60,
+        open_socket_tries: 3,
         user:              Setting.get('es_user'),
         password:          Setting.get('es_password'),
       }
@@ -256,6 +261,7 @@ remove whole data from index
       {
         open_timeout:      8,
         read_timeout:      60,
+        open_socket_tries: 3,
         user:              Setting.get('es_user'),
         password:          Setting.get('es_password'),
       }
@@ -368,6 +374,7 @@ remove whole data from index
         json:              true,
         open_timeout:      5,
         read_timeout:      14,
+        open_socket_tries: 3,
         user:              Setting.get('es_user'),
         password:          Setting.get('es_password'),
       }
@@ -515,6 +522,7 @@ example for aggregations within one year
         json:              true,
         open_timeout:      5,
         read_timeout:      14,
+        open_socket_tries: 3,
         user:              Setting.get('es_user'),
         password:          Setting.get('es_password'),
       }

lib/tasks/search_index_es.rake

@@ -275,6 +275,7 @@ end
 
 # get es version
 def es_version
+  @es_version ||= begin
     info = SearchIndexBackend.info
     number = nil
     if info.present?
@@ -282,6 +283,7 @@ def es_version
     end
     number
   end
+end
 
 # no es_pipeline for elasticsearch 5.5 and lower
 def es_pipeline?

lib/user_agent.rb

@@ -59,10 +59,13 @@ returns
     # start http call
     begin
       total_timeout = options[:total_timeout] || 60
+
+      handled_open_timeout(options[:open_socket_tries]) do
         Timeout.timeout(total_timeout) do
           response = http.request(request)
           return process(request, response, uri, count, params, options)
         end
+      end
     rescue => e
       log(url, request, nil, options)
       return Result.new(
@@ -112,10 +115,13 @@ returns
     # start http call
     begin
       total_timeout = options[:total_timeout] || 60
+
+      handled_open_timeout(options[:open_socket_tries]) do
         Timeout.timeout(total_timeout) do
           response = http.request(request)
           return process(request, response, uri, count, params, options)
         end
+      end
     rescue => e
       log(url, request, nil, options)
       return Result.new(
@@ -164,10 +170,13 @@ returns
     # start http call
     begin
       total_timeout = options[:total_timeout] || 60
+
+      handled_open_timeout(options[:open_socket_tries]) do
         Timeout.timeout(total_timeout) do
           response = http.request(request)
           return process(request, response, uri, count, params, options)
         end
+      end
     rescue => e
       log(url, request, nil, options)
       return Result.new(
@@ -209,10 +218,12 @@ returns
     # start http call
     begin
       total_timeout = options[:total_timeout] || 60
+      handled_open_timeout(options[:open_socket_tries]) do
         Timeout.timeout(total_timeout) do
           response = http.request(request)
           return process(request, response, uri, count, {}, options)
         end
+      end
     rescue => e
       log(url, request, nil, options)
       return Result.new(
@@ -488,6 +499,16 @@ returns
     )
   end
 
+  def self.handled_open_timeout(tries)
+    tries ||= 1
+
+    tries.times do |index|
+      yield
+    rescue Net::OpenTimeout
+      raise if (index + 1) == tries
+    end
+  end
+
   class Result
 
     attr_reader :error