Added monitoring check for stuck ImportJob backends.

This commit is contained in:
Thorsten Eckel 2018-02-12 12:22:16 +01:00
parent bc11d4c2b0
commit 3fd1c2de7a
4 changed files with 87 additions and 8 deletions

View file

@ -87,15 +87,10 @@ curl http://localhost/api/v1/monitoring/health_check?token=XXX
actions.add(:restart_failed_jobs) actions.add(:restart_failed_jobs)
end end
Setting.get('import_backends')&.each do |backend| import_backends = ImportJob.backends
if !ImportJob.backend_valid?(backend) # failed import jobs
logger.error "Invalid import backend '#{backend}'" import_backends.each do |backend|
next
end
# skip deactivated backends
next if !backend.constantize.active?
job = ImportJob.where( job = ImportJob.where(
name: backend, name: backend,
@ -111,6 +106,20 @@ curl http://localhost/api/v1/monitoring/health_check?token=XXX
issues.push "Failed to run import backend '#{backend}'. Cause: #{error_message}" issues.push "Failed to run import backend '#{backend}'. Cause: #{error_message}"
end end
# stuck import jobs
import_backends.each do |backend|
job = ImportJob.where(
name: backend,
dry_run: false,
finished_at: nil,
).where('updated_at <= ?', 5.minutes.ago).limit(1).first
next if job.blank?
issues.push "Stuck import backend '#{backend}' detected. Last update: #{job.updated_at}"
end
token = Setting.get('monitoring_token') token = Setting.get('monitoring_token')
if issues.blank? if issues.blank?

View file

@ -143,4 +143,26 @@ class ImportJob < ApplicationModel
rescue NameError rescue NameError
false false
end end
# Returns a list of valid import backends.
#
# @example
# ImportJob.backends
# # => ['Import::Ldap', 'Import::Exchange', ...]
#
# return [Boolean]
def self.backends
Setting.get('import_backends')&.select do |backend|
if !backend_valid?(backend)
logger.error "Invalid import backend '#{backend}'"
next
end
# skip deactivated backends
next if !backend.constantize.active?
true
end || []
end
end end

View file

@ -176,6 +176,35 @@ RSpec.describe ImportJob do
end end
end end
describe '#backends' do
it 'returns list of backend namespaces' do
expect(Setting).to receive(:get).with('import_backends').and_return(['Import::Ldap'])
expect(Import::Ldap).to receive(:active?).and_return(true)
backends = described_class.backends
expect(backends).to be_a(Array)
expect(backends).not_to be_blank
end
it 'returns blank array if none are found' do
expect(Setting).to receive(:get).with('import_backends')
expect(described_class.backends).to eq([])
end
it "doesn't return invalid backends" do
expect(Setting).to receive(:get).with('import_backends').and_return(['Import::InvalidBackend'])
expect(described_class.backends).to eq([])
end
it "doesn't return inactive backends" do
expect(Setting).to receive(:get).with('import_backends').and_return(['Import::Ldap'])
expect(Import::Ldap).to receive(:active?).and_return(false)
expect(described_class.backends).to eq([])
end
end
describe '.start' do describe '.start' do
it 'runs import backend and updates started_at and finished_at' do it 'runs import backend and updates started_at and finished_at' do

View file

@ -411,6 +411,25 @@ class MonitoringControllerTest < ActionDispatch::IntegrationTest
assert_equal(false, result['healthy']) assert_equal(false, result['healthy'])
assert_equal("Channel: Email::Notification out ;unprocessable mails: 1;scheduler not running;Failed to run import backend 'Import::Ldap'. Cause: Some bad error", result['message']) assert_equal("Channel: Email::Notification out ;unprocessable mails: 1;scheduler not running;Failed to run import backend 'Import::Ldap'. Cause: Some bad error", result['message'])
stuck_updated_at_timestamp = 15.minutes.ago
ImportJob.create(
name: 'Import::Ldap',
started_at: Time.zone.now,
finished_at: nil,
updated_at: stuck_updated_at_timestamp,
)
# health_check
get "/api/v1/monitoring/health_check?token=#{@token}", params: {}, headers: @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(Hash, result.class)
assert(result['message'])
assert(result['issues'])
assert_equal(false, result['healthy'])
assert_equal("Channel: Email::Notification out ;unprocessable mails: 1;scheduler not running;Failed to run import backend 'Import::Ldap'. Cause: Some bad error;Stuck import backend 'Import::Ldap' detected. Last update: #{15.minutes.ago}", result['message'])
Setting.set('ldap_integration', false) Setting.set('ldap_integration', false)
end end