From 81dfe88d993793dccf9c6221af49d9a67ea5ccb0 Mon Sep 17 00:00:00 2001 From: Thorsten Eckel Date: Mon, 26 Mar 2018 17:43:52 +0200 Subject: [PATCH] @muhammadn fixed issue #1866 - Failing Scheduler Delayed::Job - jobs are not listed in the monitoring result. --- .gitlab-ci.yml | 2 + app/controllers/monitoring_controller.rb | 17 ++ .../monitoring_controller_test.rb | 179 +++++++++++++++++- 3 files changed, 197 insertions(+), 1 deletion(-) rename test/{controllers => integration}/monitoring_controller_test.rb (72%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c4201c35..62a24f7dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -82,6 +82,7 @@ test:unit:mysql: - ruby -I test/ test/integration/object_manager_test.rb - ruby -I test/ test/integration/object_manager_attributes_controller_test.rb - ruby -I test/ test/integration/package_test.rb + - ruby -I test/ test/integration/monitoring_controller_test.rb - rake db:drop test:unit:postgresql: @@ -99,6 +100,7 @@ test:unit:postgresql: - ruby -I test/ test/integration/object_manager_test.rb - ruby -I test/ test/integration/object_manager_attributes_controller_test.rb - ruby -I test/ test/integration/package_test.rb + - ruby -I test/ test/integration/monitoring_controller_test.rb - rake db:drop test:integration:autowizard: diff --git a/app/controllers/monitoring_controller.rb b/app/controllers/monitoring_controller.rb index c407c7a83..cdc18fec0 100644 --- a/app/controllers/monitoring_controller.rb +++ b/app/controllers/monitoring_controller.rb @@ -87,6 +87,23 @@ curl http://localhost/api/v1/monitoring/health_check?token=XXX actions.add(:restart_failed_jobs) end + # failed jobs check + failed_jobs = Delayed::Job.where('attempts > 0') + count_failed_jobs = failed_jobs.count + + if count_failed_jobs > 10 + issues.push "#{count_failed_jobs} failing background jobs." + end + + listed_failed_jobs = failed_jobs.select(:handler, :attempts).limit(10) + listed_failed_jobs.group_by(&:name).each_with_index do |(name, jobs), index| + + attempts = jobs.map(&:attempts).sum + + issues.push "Failed to run background job ##{index += 1} '#{name}' #{jobs.count} time(s) with #{attempts} attempt(s)." + end + + # import jobs import_backends = ImportJob.backends # failed import jobs diff --git a/test/controllers/monitoring_controller_test.rb b/test/integration/monitoring_controller_test.rb similarity index 72% rename from test/controllers/monitoring_controller_test.rb rename to test/integration/monitoring_controller_test.rb index e4a721f8e..9d591f635 100644 --- a/test/controllers/monitoring_controller_test.rb +++ b/test/integration/monitoring_controller_test.rb @@ -1,7 +1,7 @@ - require 'test_helper' class MonitoringControllerTest < ActionDispatch::IntegrationTest + self.use_transactional_tests = false setup do # set accept header @@ -439,4 +439,181 @@ class MonitoringControllerTest < ActionDispatch::IntegrationTest assert_response(200) end + test '10 check failed delayed job' do + credentials = ActionController::HttpAuthentication::Basic.encode_credentials('monitoring-admin@example.com', 'adminpw') + + # disable elasticsearch + prev_es_config = Setting.get('es_url') + Setting.set('es_url', 'http://127.0.0.1:92001') + + # add a new object + object = ObjectManager::Attribute.add( + name: 'test3', + object: 'Ticket', + display: 'Test 3', + active: true, + data_type: 'input', + data_option: { + default: 'test', + type: 'text', + maxlength: 120, + null: true + }, + screens: { + create_middle: { + 'ticket.customer' => { + shown: true, + item_class: 'column' + }, + 'ticket.agent' => { + shown: true, + item_class: 'column' + } + }, + edit: { + 'ticket.customer' => { + shown: true + }, + 'ticket.agent' => { + shown: true + } + } + }, + position: 1550, + editable: true + ) + + migration = ObjectManager::Attribute.migration_execute + assert_equal(migration, true) + + post "/api/v1/object_manager_attributes/#{object.id}", params: {}, headers: @headers + token = @response.headers['CSRF-TOKEN'] + + # parameters for updating + params = { + 'name': 'test4', + 'object': 'Ticket', + 'display': 'Test 4', + 'active': true, + 'data_type': 'input', + 'data_option': { + 'default': 'test', + 'type': 'text', + 'maxlength': 120 + }, + 'screens': { + 'create_middle': { + 'ticket.customer': { + 'shown': true, + 'item_class': 'column' + }, + 'ticket.agent': { + 'shown': true, + 'item_class': 'column' + } + }, + 'edit': { + 'ticket.customer': { + 'shown': true + }, + 'ticket.agent': { + 'shown': true + } + } + }, + 'id': 'c-196' + } + + # update the object + put "/api/v1/object_manager_attributes/#{object.id}", params: params.to_json, headers: @headers.merge('Authorization' => credentials) + + migration = ObjectManager::Attribute.migration_execute + assert_equal(migration, true) + + assert_response(200) + result = JSON.parse(@response.body) + assert(result) + assert(result['data_option']['null']) + assert_equal(result['name'], 'test4') + assert_equal(result['display'], 'Test 4') + + jobs = Delayed::Job.all + + 4.times do + jobs.each do |job| + Delayed::Worker.new.run(job) + end + end + + # health_check + get "/api/v1/monitoring/health_check?token=#{@token}", params: {}, headers: @headers + assert_response(200) + + result = JSON.parse(@response.body) + assert_equal(Hash, result.class) + assert(result['message']) + assert(result['issues']) + assert_equal(false, result['healthy']) + assert_equal("Failed to run background job #1 'BackgroundJobSearchIndex' 1 time(s) with 4 attempt(s).", result['message']) + + # add another job + manual_added = Delayed::Job.enqueue( BackgroundJobSearchIndex.new('Ticket', 1)) + manual_added.update!(attempts: 10) + + # health_check + get "/api/v1/monitoring/health_check?token=#{@token}", params: {}, headers: @headers + assert_response(200) + + result = JSON.parse(@response.body) + assert_equal(Hash, result.class) + assert(result['message']) + assert(result['issues']) + assert_equal(false, result['healthy']) + assert_equal("Failed to run background job #1 'BackgroundJobSearchIndex' 2 time(s) with 14 attempt(s).", result['message']) + + # add another job + dummy_class = Class.new do + + def perform + puts 'work work' + end + end + + manual_added = Delayed::Job.enqueue( dummy_class.new ) + manual_added.update!(attempts: 5) + + # health_check + get "/api/v1/monitoring/health_check?token=#{@token}", params: {}, headers: @headers + assert_response(200) + + result = JSON.parse(@response.body) + assert_equal(Hash, result.class) + assert(result['message']) + assert(result['issues']) + assert_equal(false, result['healthy']) + assert_equal("Failed to run background job #1 'BackgroundJobSearchIndex' 2 time(s) with 14 attempt(s).;Failed to run background job #2 'Object' 1 time(s) with 5 attempt(s).", result['message']) + + # reset settings + Setting.set('es_url', prev_es_config) + + # add some more failing job + 10.times do + manual_added = Delayed::Job.enqueue( dummy_class.new ) + manual_added.update!(attempts: 5) + end + + # health_check + get "/api/v1/monitoring/health_check?token=#{@token}", params: {}, headers: @headers + assert_response(200) + + result = JSON.parse(@response.body) + assert_equal(Hash, result.class) + assert(result['message']) + assert(result['issues']) + assert_equal(false, result['healthy']) + assert_equal("13 failing background jobs.;Failed to run background job #1 'BackgroundJobSearchIndex' 2 time(s) with 14 attempt(s).;Failed to run background job #2 'Object' 8 time(s) with 40 attempt(s).", result['message']) + + # cleanup + Delayed::Job.delete_all + end end