diff --git a/app/assets/javascripts/app/controllers/monitoring.coffee b/app/assets/javascripts/app/controllers/monitoring.coffee index c43b29142..2601628b8 100644 --- a/app/assets/javascripts/app/controllers/monitoring.coffee +++ b/app/assets/javascripts/app/controllers/monitoring.coffee @@ -4,6 +4,7 @@ class Index extends App.ControllerSubContent events: 'click .js-resetToken': 'resetToken' 'click .js-select': 'selectAll' + 'click .js-restartDeadJobs': 'restartDeadJobs' constructor: -> super @@ -29,7 +30,10 @@ class Index extends App.ControllerSubContent ) render: => - @html App.view('monitoring')(data: @data) + @html App.view('monitoring')( + data: @data + job_restart_count: @job_restart_count + ) resetToken: (e) => e.preventDefault() @@ -42,4 +46,15 @@ class Index extends App.ControllerSubContent @load() ) + restartDeadJobs: (e) => + e.preventDefault() + @ajax( + id: 'restart_dead_jobs_request' + type: 'POST' + url: "#{@apiPath}/monitoring/restart_dead_jobs" + success: (data) => + @job_restart_count = data.job_restart_count + @render() + ) + App.Config.set('Monitoring', { prio: 3600, name: 'Monitoring', parent: '#system', target: '#system/monitoring', controller: Index, permission: ['admin.monitoring'] }, 'NavBarAdmin') diff --git a/app/assets/javascripts/app/views/monitoring.jst.eco b/app/assets/javascripts/app/views/monitoring.jst.eco index 053c128ed..247513122 100644 --- a/app/assets/javascripts/app/views/monitoring.jst.eco +++ b/app/assets/javascripts/app/views/monitoring.jst.eco @@ -34,6 +34,15 @@ <% end %> <% end %> + <% if !_.isEmpty(@data.issues): %> + + <% if !_.isUndefined(@job_restart_count): %> +

+ <%- @T('Detected %s dead job(s) available for restart', @job_restart_count) %> + <%- ', restarting...' if @job_restart_count > 0 %> +

+ <% end %> + <% end %> \ No newline at end of file diff --git a/app/controllers/monitoring_controller.rb b/app/controllers/monitoring_controller.rb index ece7e4a5d..c29a0762e 100644 --- a/app/controllers/monitoring_controller.rb +++ b/app/controllers/monitoring_controller.rb @@ -81,6 +81,10 @@ curl http://localhost/api/v1/monitoring/health_check?token=XXX issues.push 'scheduler not running' end + Scheduler.where(status: 'error').each { |scheduler| + issues.push "Failed to run scheduled job \'#{scheduler.name}\'. Cause: #{scheduler.error_message}" + } + token = Setting.get('monitoring_token') if issues.empty? @@ -173,6 +177,22 @@ curl http://localhost/api/v1/monitoring/status?token=XXX render json: result, status: :created end + def restart_dead_jobs + access_check + + count = 0 + Scheduler.where(status: 'error').where(active: false).each { |scheduler| + scheduler.active = true + scheduler.save + count += 1 + } + + result = { + job_restart_count: count + } + render json: result + end + private def token_or_permission_check diff --git a/app/models/scheduler.rb b/app/models/scheduler.rb index 0ce63ea30..7311c40d9 100644 --- a/app/models/scheduler.rb +++ b/app/models/scheduler.rb @@ -169,8 +169,10 @@ class Scheduler < ApplicationModel end def self._start_job(job, try_count = 0, try_run_time = Time.zone.now) - job.last_run = Time.zone.now - job.pid = Thread.current.object_id + job.last_run = Time.zone.now + job.pid = Thread.current.object_id + job.status = 'ok' + job.error_message = '' job.save logger.info "execute #{job.method} (try_count #{try_count})..." eval job.method() # rubocop:disable Lint/Eval @@ -197,7 +199,14 @@ class Scheduler < ApplicationModel if try_run_max > try_count _start_job(job, try_count, try_run_time) else - raise "STOP thread for #{job.method} after #{try_count} tries (#{e.inspect})" + @@jobs_started[ job.id ] = false + error = "Failed to run #{job.method} after #{try_count} tries #{e.inspect}" + logger.error error + + job.error_message = error + job.status = 'error' + job.active = false + job.save end end diff --git a/config/routes/monitoring.rb b/config/routes/monitoring.rb index 18f2e6c93..bf7205054 100644 --- a/config/routes/monitoring.rb +++ b/config/routes/monitoring.rb @@ -1,8 +1,9 @@ Zammad::Application.routes.draw do api_path = Rails.configuration.api_path - match api_path + '/monitoring/health_check', to: 'monitoring#health_check', via: :get - match api_path + '/monitoring/status', to: 'monitoring#status', via: :get - match api_path + '/monitoring/token', to: 'monitoring#token', via: :post + match api_path + '/monitoring/health_check', to: 'monitoring#health_check', via: :get + match api_path + '/monitoring/status', to: 'monitoring#status', via: :get + match api_path + '/monitoring/token', to: 'monitoring#token', via: :post + match api_path + '/monitoring/restart_dead_jobs', to: 'monitoring#restart_dead_jobs', via: :post end diff --git a/db/migrate/20170515000001_scheduler_status.rb b/db/migrate/20170515000001_scheduler_status.rb new file mode 100644 index 000000000..c6be3481a --- /dev/null +++ b/db/migrate/20170515000001_scheduler_status.rb @@ -0,0 +1,8 @@ +class SchedulerStatus < ActiveRecord::Migration + def up + change_table :schedulers do |t| + t.string :error_message + t.string :status + end + end +end