@muhammadn fixed issue #1866 - Failing Scheduler Delayed::Job - jobs are not listed in the monitoring result.
This commit is contained in:
Thorsten Eckel
parent
c300fdd4af
commit
81dfe88d99
3 changed files with 197 additions and 1 deletions
|
|
@ -82,6 +82,7 @@ test:unit:mysql:
|
|||
- ruby -I test/ test/integration/object_manager_test.rb
|
||||
- ruby -I test/ test/integration/object_manager_attributes_controller_test.rb
|
||||
- ruby -I test/ test/integration/package_test.rb
|
||||
- ruby -I test/ test/integration/monitoring_controller_test.rb
|
||||
- rake db:drop
|
||||
|
||||
test:unit:postgresql:
|
||||
|
|
@ -99,6 +100,7 @@ test:unit:postgresql:
|
|||
- ruby -I test/ test/integration/object_manager_test.rb
|
||||
- ruby -I test/ test/integration/object_manager_attributes_controller_test.rb
|
||||
- ruby -I test/ test/integration/package_test.rb
|
||||
- ruby -I test/ test/integration/monitoring_controller_test.rb
|
||||
- rake db:drop
|
||||
|
||||
test:integration:autowizard:
|
||||
|
|
|
|||
|
|
@ -87,6 +87,23 @@ curl http://localhost/api/v1/monitoring/health_check?token=XXX
|
|||
actions.add(:restart_failed_jobs)
|
||||
end
|
||||
|
||||
# failed jobs check
|
||||
failed_jobs = Delayed::Job.where('attempts > 0')
|
||||
count_failed_jobs = failed_jobs.count
|
||||
|
||||
if count_failed_jobs > 10
|
||||
issues.push "#{count_failed_jobs} failing background jobs."
|
||||
end
|
||||
|
||||
listed_failed_jobs = failed_jobs.select(:handler, :attempts).limit(10)
|
||||
listed_failed_jobs.group_by(&:name).each_with_index do |(name, jobs), index|
|
||||
|
||||
attempts = jobs.map(&:attempts).sum
|
||||
|
||||
issues.push "Failed to run background job ##{index += 1} '#{name}' #{jobs.count} time(s) with #{attempts} attempt(s)."
|
||||
end
|
||||
|
||||
# import jobs
|
||||
import_backends = ImportJob.backends
|
||||
|
||||
# failed import jobs
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
|
||||
require 'test_helper'
|
||||
|
||||
class MonitoringControllerTest < ActionDispatch::IntegrationTest
|
||||
self.use_transactional_tests = false
|
||||
setup do
|
||||
|
||||
# set accept header
|
||||
|
|
@ -439,4 +439,181 @@ class MonitoringControllerTest < ActionDispatch::IntegrationTest
|
|||
assert_response(200)
|
||||
end
|
||||
|
||||
test '10 check failed delayed job' do
|
||||
credentials = ActionController::HttpAuthentication::Basic.encode_credentials('monitoring-admin@example.com', 'adminpw')
|
||||
|
||||
# disable elasticsearch
|
||||
prev_es_config = Setting.get('es_url')
|
||||
Setting.set('es_url', 'http://127.0.0.1:92001')
|
||||
|
||||
# add a new object
|
||||
object = ObjectManager::Attribute.add(
|
||||
name: 'test3',
|
||||
object: 'Ticket',
|
||||
display: 'Test 3',
|
||||
active: true,
|
||||
data_type: 'input',
|
||||
data_option: {
|
||||
default: 'test',
|
||||
type: 'text',
|
||||
maxlength: 120,
|
||||
null: true
|
||||
},
|
||||
screens: {
|
||||
create_middle: {
|
||||
'ticket.customer' => {
|
||||
shown: true,
|
||||
item_class: 'column'
|
||||
},
|
||||
'ticket.agent' => {
|
||||
shown: true,
|
||||
item_class: 'column'
|
||||
}
|
||||
},
|
||||
edit: {
|
||||
'ticket.customer' => {
|
||||
shown: true
|
||||
},
|
||||
'ticket.agent' => {
|
||||
shown: true
|
||||
}
|
||||
}
|
||||
},
|
||||
position: 1550,
|
||||
editable: true
|
||||
)
|
||||
|
||||
migration = ObjectManager::Attribute.migration_execute
|
||||
assert_equal(migration, true)
|
||||
|
||||
post "/api/v1/object_manager_attributes/#{object.id}", params: {}, headers: @headers
|
||||
token = @response.headers['CSRF-TOKEN']
|
||||
|
||||
# parameters for updating
|
||||
params = {
|
||||
'name': 'test4',
|
||||
'object': 'Ticket',
|
||||
'display': 'Test 4',
|
||||
'active': true,
|
||||
'data_type': 'input',
|
||||
'data_option': {
|
||||
'default': 'test',
|
||||
'type': 'text',
|
||||
'maxlength': 120
|
||||
},
|
||||
'screens': {
|
||||
'create_middle': {
|
||||
'ticket.customer': {
|
||||
'shown': true,
|
||||
'item_class': 'column'
|
||||
},
|
||||
'ticket.agent': {
|
||||
'shown': true,
|
||||
'item_class': 'column'
|
||||
}
|
||||
},
|
||||
'edit': {
|
||||
'ticket.customer': {
|
||||
'shown': true
|
||||
},
|
||||
'ticket.agent': {
|
||||
'shown': true
|
||||
}
|
||||
}
|
||||
},
|
||||
'id': 'c-196'
|
||||
}
|
||||
|
||||
# update the object
|
||||
put "/api/v1/object_manager_attributes/#{object.id}", params: params.to_json, headers: @headers.merge('Authorization' => credentials)
|
||||
|
||||
migration = ObjectManager::Attribute.migration_execute
|
||||
assert_equal(migration, true)
|
||||
|
||||
assert_response(200)
|
||||
result = JSON.parse(@response.body)
|
||||
assert(result)
|
||||
assert(result['data_option']['null'])
|
||||
assert_equal(result['name'], 'test4')
|
||||
assert_equal(result['display'], 'Test 4')
|
||||
|
||||
jobs = Delayed::Job.all
|
||||
|
||||
4.times do
|
||||
jobs.each do |job|
|
||||
Delayed::Worker.new.run(job)
|
||||
end
|
||||
end
|
||||
|
||||
# health_check
|
||||
get "/api/v1/monitoring/health_check?token=#{@token}", params: {}, headers: @headers
|
||||
assert_response(200)
|
||||
|
||||
result = JSON.parse(@response.body)
|
||||
assert_equal(Hash, result.class)
|
||||
assert(result['message'])
|
||||
assert(result['issues'])
|
||||
assert_equal(false, result['healthy'])
|
||||
assert_equal("Failed to run background job #1 'BackgroundJobSearchIndex' 1 time(s) with 4 attempt(s).", result['message'])
|
||||
|
||||
# add another job
|
||||
manual_added = Delayed::Job.enqueue( BackgroundJobSearchIndex.new('Ticket', 1))
|
||||
manual_added.update!(attempts: 10)
|
||||
|
||||
# health_check
|
||||
get "/api/v1/monitoring/health_check?token=#{@token}", params: {}, headers: @headers
|
||||
assert_response(200)
|
||||
|
||||
result = JSON.parse(@response.body)
|
||||
assert_equal(Hash, result.class)
|
||||
assert(result['message'])
|
||||
assert(result['issues'])
|
||||
assert_equal(false, result['healthy'])
|
||||
assert_equal("Failed to run background job #1 'BackgroundJobSearchIndex' 2 time(s) with 14 attempt(s).", result['message'])
|
||||
|
||||
# add another job
|
||||
dummy_class = Class.new do
|
||||
|
||||
def perform
|
||||
puts 'work work'
|
||||
end
|
||||
end
|
||||
|
||||
manual_added = Delayed::Job.enqueue( dummy_class.new )
|
||||
manual_added.update!(attempts: 5)
|
||||
|
||||
# health_check
|
||||
get "/api/v1/monitoring/health_check?token=#{@token}", params: {}, headers: @headers
|
||||
assert_response(200)
|
||||
|
||||
result = JSON.parse(@response.body)
|
||||
assert_equal(Hash, result.class)
|
||||
assert(result['message'])
|
||||
assert(result['issues'])
|
||||
assert_equal(false, result['healthy'])
|
||||
assert_equal("Failed to run background job #1 'BackgroundJobSearchIndex' 2 time(s) with 14 attempt(s).;Failed to run background job #2 'Object' 1 time(s) with 5 attempt(s).", result['message'])
|
||||
|
||||
# reset settings
|
||||
Setting.set('es_url', prev_es_config)
|
||||
|
||||
# add some more failing job
|
||||
10.times do
|
||||
manual_added = Delayed::Job.enqueue( dummy_class.new )
|
||||
manual_added.update!(attempts: 5)
|
||||
end
|
||||
|
||||
# health_check
|
||||
get "/api/v1/monitoring/health_check?token=#{@token}", params: {}, headers: @headers
|
||||
assert_response(200)
|
||||
|
||||
result = JSON.parse(@response.body)
|
||||
assert_equal(Hash, result.class)
|
||||
assert(result['message'])
|
||||
assert(result['issues'])
|
||||
assert_equal(false, result['healthy'])
|
||||
assert_equal("13 failing background jobs.;Failed to run background job #1 'BackgroundJobSearchIndex' 2 time(s) with 14 attempt(s).;Failed to run background job #2 'Object' 8 time(s) with 40 attempt(s).", result['message'])
|
||||
|
||||
# cleanup
|
||||
Delayed::Job.delete_all
|
||||
end
|
||||
end
|
||||
Loading…
Reference in a new issue