From fc7c2f31ddcc368d29ba3a23107c1a758a915d68 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 29 May 2021 17:42:45 -0300 Subject: [PATCH 1/4] enviar reportes a gitlab usando la api en lugar del correo --- Gemfile | 1 + Gemfile.lock | 1 + app/jobs/backtrace_job.rb | 2 +- app/jobs/gitlab_notifier_job.rb | 142 ++++++++++++++++++ app/lib/exception_notifier/gitlab_notifier.rb | 17 +++ app/lib/gitlab_api_client.rb | 61 ++++++++ .../exception_notifier/_backtrace.text.erb | 2 +- app/views/exception_notifier/_data.text.erb | 2 +- config/environments/production.rb | 9 +- 9 files changed, 226 insertions(+), 11 deletions(-) create mode 100644 app/jobs/gitlab_notifier_job.rb create mode 100644 app/lib/exception_notifier/gitlab_notifier.rb create mode 100644 app/lib/gitlab_api_client.rb diff --git a/Gemfile b/Gemfile index 3f100e6b..5f1cdc4f 100644 --- a/Gemfile +++ b/Gemfile @@ -41,6 +41,7 @@ gem 'hiredis' gem 'image_processing' gem 'icalendar' gem 'inline_svg' +gem 'httparty' gem 'safe_yaml', source: 'https://gems.sutty.nl' gem 'jekyll', '~> 4.2' gem 'jekyll-data', source: 'https://gems.sutty.nl' diff --git a/Gemfile.lock b/Gemfile.lock index 942d7d6f..5484ad88 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -646,6 +646,7 @@ DEPENDENCIES haml-lint hamlit-rails hiredis + httparty icalendar image_processing inline_svg diff --git a/app/jobs/backtrace_job.rb b/app/jobs/backtrace_job.rb index eab9f226..86a9b2a6 100644 --- a/app/jobs/backtrace_job.rb +++ b/app/jobs/backtrace_job.rb @@ -40,7 +40,7 @@ class BacktraceJob < ApplicationJob begin raise BacktraceException, "#{origin}: #{message}" rescue BacktraceException => e - ExceptionNotifier.notify_exception(e, data: { site: site.name, params: params, _backtrace: true }) + ExceptionNotifier.notify_exception(e, data: { site: site.name, params: params, javascript_backtrace: true }) end end diff --git a/app/jobs/gitlab_notifier_job.rb b/app/jobs/gitlab_notifier_job.rb new file mode 100644 index 00000000..4d676b8d --- /dev/null +++ b/app/jobs/gitlab_notifier_job.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +class GitlabNotifierJob < ApplicationJob + include ExceptionNotifier::BacktraceCleaner + + attr_reader :exception, :options + + queue_as :low_priority + + def perform(exception, **options) + @exception = exception + @options = options + + Rails.logger.info 'Enviando reporte a Gitlab' + + i = client.new_issue confidential: true, title: title, description: description + + Rails.logger.info "Enviado reporte a Gitlab: #{i['iid']}" + rescue Exception => e + Rails.logger.info 'No entrar en loop' + end + + private + + # Define si es una excepción de javascript o local + # + # @see BacktraceJob + def javascript? + @javascript ||= options.dig(:data, :javascript_backtrace).present? + end + + # @return [String] + def title + @title ||= ''.dup.tap do |t| + t << "[#{exception.class}] " unless javascript? + t << exception.message + end + end + + # @return [String] + def description + @description ||= ''.dup.tap do |d| + d << request_section + d << javascript_section + d << javascript_footer + d << backtrace_section + d << data_section + end + end + + # @return [String,Nil] + def backtrace + @backtrace ||= exception.backtrace ? clean_backtrace(exception) : nil + end + + def env + options[:env] + end + + def request + @request ||= ActionDispatch::Request.new(env) if env.present? + end + + # @return [GitlabApiClient] + def client + @client ||= GitlabApiClient.new + end + + def request_section + return '' unless request + + <<~REQUEST + + # Request + + ``` + #{request.request_method} #{request.url}#{' '} + + #{pp request.filtered_parameters} + ``` + + REQUEST + end + + def javascript_section + return '' unless javascript? + + options.dig(:data, :params, 'errors')&.map do |error| + <<~JAVASCRIPT + + ## #{error['type'] || 'NoError'}: #{error['message']} + + + ``` + #{Terminal::Table.new headings: error['backtrace'].first.keys, rows: error['backtrace'].map(&:values)} + ``` + + JAVASCRIPT + end&.join + end + + def javascript_footer + return '' unless javascript? + + <<~JAVASCRIPT + + #{options.dig(:data, :params, 'context', 'userAgent')} + + <#{options.dig(:data, :params, 'context', 'url')}> + + JAVASCRIPT + end + + def backtrace_section + return '' if javascript? + return '' unless backtrace + + <<~BACKTRACE + + ## Backtrace + + ``` + #{backtrace.join("\n")} + ``` + + BACKTRACE + end + + def data_section + return '' unless options[:data] + + <<~DATA + + ## Data + + ``` + #{pp options[:data]} + ``` + + DATA + end +end diff --git a/app/lib/exception_notifier/gitlab_notifier.rb b/app/lib/exception_notifier/gitlab_notifier.rb new file mode 100644 index 00000000..18bfc6d4 --- /dev/null +++ b/app/lib/exception_notifier/gitlab_notifier.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module ExceptionNotifier + # Notifica las excepciones como incidencias en Gitlab + class GitlabNotifier + def initialize(_); end + + # Recibe la excepción y empieza la tarea de notificación en segundo + # plano. + # + # @param [Exception] + # @param [Hash] + def call(exception, **options) + GitlabNotifierJob.perform_async(exception, **options) + end + end +end diff --git a/app/lib/gitlab_api_client.rb b/app/lib/gitlab_api_client.rb new file mode 100644 index 00000000..c8157b47 --- /dev/null +++ b/app/lib/gitlab_api_client.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'httparty' + +class GitlabApiClient + include HTTParty + + # TODO: Hacer configurable por sitio + base_uri ENV.fetch('GITLAB_URI', 'https://0xacab.org') + no_follow true + + # Trae todos los proyectos. Como estamos usando un Project Token, + # siempre va a traer uno solo. + # + # @return [HTTParty::Response] + def projects + self.class.get('/api/v4/projects', { query: params(membership: true) }) + end + + # Obtiene el identificador del proyecto + # + # @return [Integer] + def project_id + @project_id ||= ENV['GITLAB_PROJECT'] || projects&.first&.dig('id') + end + + # Crea un issue + # + # @see https://docs.gitlab.com/ee/api/issues.html#new-issue + # @return [HTTParty::Response] + def new_issue(**args) + self.class.post("/api/v4/projects/#{project_id}/issues", { query: params(**args) }) + end + + # Modifica un issue + # + # @see https://docs.gitlab.com/ee/api/issues.html#edit-issue + # @return [HTTParty::Response] + def edit_issue(**args) + self.class.put("/api/v4/projects/#{project_id}/issues", { query: params(**args) }) + end + + # Crea un comentario + # + # @see https://docs.gitlab.com/ee/api/notes.html#create-new-issue-note + # @return [HTTParty::Response] + def new_note(iid:, **args) + self.class.post("/api/v4/projects/#{project_id}/issues/#{iid}/notes", { query: params(**args) }) + end + + private + + def params(**args) + default_params.merge(args) + end + + # TODO: Que cada sitio tenga su propio token y uri + def default_params + { private_token: ENV['GITLAB_TOKEN'] } + end +end diff --git a/app/views/exception_notifier/_backtrace.text.erb b/app/views/exception_notifier/_backtrace.text.erb index d62b5719..aed7adbe 100644 --- a/app/views/exception_notifier/_backtrace.text.erb +++ b/app/views/exception_notifier/_backtrace.text.erb @@ -1,4 +1,4 @@ -<% unless @data[:_backtrace] %> +<% unless @data[:javascript_backtrace] %> ``` <%= raw @backtrace.join("\n") %> ``` diff --git a/app/views/exception_notifier/_data.text.erb b/app/views/exception_notifier/_data.text.erb index 09313f4c..acb94b89 100644 --- a/app/views/exception_notifier/_data.text.erb +++ b/app/views/exception_notifier/_data.text.erb @@ -1,4 +1,4 @@ -<% if @data[:_backtrace] %> +<% if @data[:javascript_backtrace] %> <% @data.dig(:params, 'errors')&.each do |error| %> # <%= error['type'] %>: <%= error['message'] %> diff --git a/config/environments/production.rb b/config/environments/production.rb index c1269fb2..d121bdbd 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -147,14 +147,7 @@ Rails.application.configure do } config.action_mailer.default_options = { from: ENV.fetch('DEFAULT_FROM', "noreply@sutty.nl") } - config.middleware.use ExceptionNotification::Rack, - error_grouping: true, - email: { - email_prefix: '', - sender_address: ENV.fetch('DEFAULT_FROM', "noreply@sutty.nl"), - exception_recipients: ENV.fetch('EXCEPTION_TO', "errors@sutty.nl"), - normalize_subject: true - } + config.middleware.use ExceptionNotification::Rack, gitlab: {} Rails.application.routes.default_url_options[:host] = "panel.#{ENV.fetch('SUTTY', 'sutty.nl')}" Rails.application.routes.default_url_options[:protocol] = 'https' From 5468a118851b92271e1d081ca386438970101d31 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 31 May 2021 12:08:19 -0300 Subject: [PATCH 2/4] reportar el issue una sola vez y luego actualizarlo --- app/jobs/gitlab_notifier_job.rb | 130 +++++++++++++++++++++++++++++--- app/lib/gitlab_api_client.rb | 22 +++--- 2 files changed, 130 insertions(+), 22 deletions(-) diff --git a/app/jobs/gitlab_notifier_job.rb b/app/jobs/gitlab_notifier_job.rb index 4d676b8d..ac712ab2 100644 --- a/app/jobs/gitlab_notifier_job.rb +++ b/app/jobs/gitlab_notifier_job.rb @@ -1,42 +1,98 @@ # frozen_string_literal: true +# Notifica excepciones a una instancia de Gitlab, como incidencias +# nuevas o como comentarios a las incidencias pre-existentes. class GitlabNotifierJob < ApplicationJob include ExceptionNotifier::BacktraceCleaner - attr_reader :exception, :options + # Variables que vamos a acceder luego + attr_reader :exception, :options, :issue_data, :cached queue_as :low_priority + # @param [Exception] la excepción lanzada + # @param [Hash] opciones de ExceptionNotifier def perform(exception, **options) @exception = exception @options = options + @issue_data = { count: 1 } + # Necesitamos saber si el issue ya existía + @cached = false - Rails.logger.info 'Enviando reporte a Gitlab' + # Traemos los datos desde la caché si existen, sino generamos un + # issue nuevo e inicializamos la caché + @issue_data = Rails.cache.fetch(cache_key) do + issue = client.new_issue confidential: true, title: title, description: description, issue_type: 'incident' + @cached = true - i = client.new_issue confidential: true, title: title, description: description + { + count: 1, + issue: issue['iid'], + user_agents: [user_agent].compact, + params: [request&.filtered_parameters].compact, + urls: [url].compact + } + end - Rails.logger.info "Enviado reporte a Gitlab: #{i['iid']}" + # No seguimos actualizando si acabamos de generar el issue + return if cached + + # Incrementar la cuenta de veces que ocurrió + issue_data[:count] += 1 + # Guardar información útil + issue_data[:urls] << url unless issue_data[:urls].include? url + issue_data[:user_agents] << user_agent unless issue_data[:user_agents].include? user_agent + + # Editar el título para que incluya la cuenta de eventos + client.edit_issue(iid: issue_data[:issue], title: title, state_event: 'reopen') + + # Agregar un comentario con la información posiblemente nueva + client.new_note(iid: issue_data[:issue], body: body) + + # Guardar para después + Rails.cache.write(cache_key, issue_data) + # Si este trabajo genera una excepción va a entrar en un loop + # TODO: Notificarnos por otros medios (mail) rescue Exception => e Rails.logger.info 'No entrar en loop' end private + # La llave en la cache tiene en cuenta la excepción, el mensaje, la + # ruta del backtrace y los errores de JS + # + # @return [String] + def cache_key + @cache_key ||= [ + exception.class.name, + Digest::SHA1.hexdigest(exception.message), + Digest::SHA1.hexdigest(backtrace&.first.to_s), + Digest::SHA1.hexdigest(options.dig(:data, :params, 'errors').to_s) + ].join('/') + end + # Define si es una excepción de javascript o local # # @see BacktraceJob + # @return [Boolean] def javascript? @javascript ||= options.dig(:data, :javascript_backtrace).present? end + # Título + # # @return [String] def title @title ||= ''.dup.tap do |t| t << "[#{exception.class}] " unless javascript? t << exception.message + t << " [#{issue_data[:count]}]" end end + # Descripción + # # @return [String] def description @description ||= ''.dup.tap do |d| @@ -48,24 +104,48 @@ class GitlabNotifierJob < ApplicationJob end end - # @return [String,Nil] + # Comentario + # + # @return [String] + def body + @body ||= ''.dup.tap do |b| + b << request_section + b << javascript_footer + b << data_section + end + end + + # Cadena de archivos donde se produjo el error + # + # @return [Array,Nil] def backtrace @backtrace ||= exception.backtrace ? clean_backtrace(exception) : nil end + # Entorno del error + # + # @return [Hash] def env options[:env] end + # Genera una petición a partir del entorno + # + # @return [ActionDispatch::Request] def request @request ||= ActionDispatch::Request.new(env) if env.present? end + # Cliente de la API de Gitlab + # # @return [GitlabApiClient] def client @client ||= GitlabApiClient.new end + # Muestra información de la petición + # + # @return [String] def request_section return '' unless request @@ -74,7 +154,7 @@ class GitlabNotifierJob < ApplicationJob # Request ``` - #{request.request_method} #{request.url}#{' '} + #{request.request_method} #{url} #{pp request.filtered_parameters} ``` @@ -82,14 +162,19 @@ class GitlabNotifierJob < ApplicationJob REQUEST end + # Muestra información de JavaScript + # + # @return [String] def javascript_section return '' unless javascript? options.dig(:data, :params, 'errors')&.map do |error| + # Algunos errores no son excepciones (?) + error['type'] = 'undefined' if error['type'].blank? + <<~JAVASCRIPT - ## #{error['type'] || 'NoError'}: #{error['message']} - + ## #{error['type']}: #{error['message']} ``` #{Terminal::Table.new headings: error['backtrace'].first.keys, rows: error['backtrace'].map(&:values)} @@ -99,18 +184,24 @@ class GitlabNotifierJob < ApplicationJob end&.join end + # Muestra información de la visita que generó el error en JS + # + # @return [String] def javascript_footer return '' unless javascript? <<~JAVASCRIPT - #{options.dig(:data, :params, 'context', 'userAgent')} + #{user_agent} - <#{options.dig(:data, :params, 'context', 'url')}> + <#{url}> JAVASCRIPT end + # Muestra el historial del error en Ruby + # + # @return [String] def backtrace_section return '' if javascript? return '' unless backtrace @@ -126,6 +217,9 @@ class GitlabNotifierJob < ApplicationJob BACKTRACE end + # Muestra datos extra de la visita + # + # @return [String] def data_section return '' unless options[:data] @@ -139,4 +233,20 @@ class GitlabNotifierJob < ApplicationJob DATA end + + # Obtiene el UA de este error + # + # @return [String] + def user_agent + @user_agent ||= options.dig(:data, :params, 'context', 'userAgent') if javascript? + @user_agent ||= request.headers['user-agent'] if request + @user_agent + end + + # Obtiene la URL actual + # + # @return [String] + def url + @url ||= request&.url || options.dig(:data, :params, 'context', 'url') + end end diff --git a/app/lib/gitlab_api_client.rb b/app/lib/gitlab_api_client.rb index c8157b47..5b1287d6 100644 --- a/app/lib/gitlab_api_client.rb +++ b/app/lib/gitlab_api_client.rb @@ -7,6 +7,9 @@ class GitlabApiClient # TODO: Hacer configurable por sitio base_uri ENV.fetch('GITLAB_URI', 'https://0xacab.org') + # No seguir redirecciones. Si nos olvidamos https:// en la dirección, + # las redirecciones nos pueden llevar a cualquier lado y obtener + # resultados diferentes. no_follow true # Trae todos los proyectos. Como estamos usando un Project Token, @@ -14,7 +17,7 @@ class GitlabApiClient # # @return [HTTParty::Response] def projects - self.class.get('/api/v4/projects', { query: params(membership: true) }) + self.class.get('/api/v4/projects', { query: { membership: true }, headers: headers }) end # Obtiene el identificador del proyecto @@ -29,15 +32,15 @@ class GitlabApiClient # @see https://docs.gitlab.com/ee/api/issues.html#new-issue # @return [HTTParty::Response] def new_issue(**args) - self.class.post("/api/v4/projects/#{project_id}/issues", { query: params(**args) }) + self.class.post("/api/v4/projects/#{project_id}/issues", { body: args, headers: headers }) end # Modifica un issue # # @see https://docs.gitlab.com/ee/api/issues.html#edit-issue # @return [HTTParty::Response] - def edit_issue(**args) - self.class.put("/api/v4/projects/#{project_id}/issues", { query: params(**args) }) + def edit_issue(iid:, **args) + self.class.put("/api/v4/projects/#{project_id}/issues/#{iid}", { body: args, headers: headers }) end # Crea un comentario @@ -45,17 +48,12 @@ class GitlabApiClient # @see https://docs.gitlab.com/ee/api/notes.html#create-new-issue-note # @return [HTTParty::Response] def new_note(iid:, **args) - self.class.post("/api/v4/projects/#{project_id}/issues/#{iid}/notes", { query: params(**args) }) + self.class.post("/api/v4/projects/#{project_id}/issues/#{iid}/notes", { body: args, headers: headers }) end private - def params(**args) - default_params.merge(args) - end - - # TODO: Que cada sitio tenga su propio token y uri - def default_params - { private_token: ENV['GITLAB_TOKEN'] } + def headers(extra = {}) + { 'Authorization' => "Bearer #{ENV['GITLAB_TOKEN']}" }.merge(extra) end end From 4540ea73ad6f3353f63deae5506e1e47991bb8e3 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 31 May 2021 12:18:06 -0300 Subject: [PATCH 3/4] variables de entorno nuevas --- .env.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.env.example b/.env.example index 2ba46219..a1348593 100644 --- a/.env.example +++ b/.env.example @@ -29,3 +29,6 @@ TIENDA=tienda.sutty.local PANEL_URL=https://panel.sutty.nl AIRBRAKE_SITE_ID=1 AIRBRAKE_API_KEY= +GITLAB_URI=https://0xacab.org +GITLAB_PROJECT= +GITLAB_TOKEN= From c6bcf95f345fccc3d7519b8ec98699b33fb9cc42 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 31 May 2021 12:18:23 -0300 Subject: [PATCH 4/4] =?UTF-8?q?si=20la=20notificaci=C3=B3n=20produce=20una?= =?UTF-8?q?=20excepci=C3=B3n,=20capturarla=20por=20correo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/gitlab_notifier_job.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/app/jobs/gitlab_notifier_job.rb b/app/jobs/gitlab_notifier_job.rb index ac712ab2..7218f68a 100644 --- a/app/jobs/gitlab_notifier_job.rb +++ b/app/jobs/gitlab_notifier_job.rb @@ -51,14 +51,22 @@ class GitlabNotifierJob < ApplicationJob # Guardar para después Rails.cache.write(cache_key, issue_data) - # Si este trabajo genera una excepción va a entrar en un loop - # TODO: Notificarnos por otros medios (mail) + # Si este trabajo genera una excepción va a entrar en un loop, así que + # la notificamos por correo rescue Exception => e - Rails.logger.info 'No entrar en loop' + email_notification.call(e) + email_notification.call(exception, options) end private + # Notificar por correo + # + # @return [ExceptionNotifier::EmailNotifier] + def email_notification + @email_notification ||= ExceptionNotifier::EmailNotifier.new(email_prefix: '[ERROR] ', sender_address: ENV['DEFAULT_FROM'], exception_recipients: ENV['EXCEPTION_TO']) + end + # La llave en la cache tiene en cuenta la excepción, el mensaje, la # ruta del backtrace y los errores de JS #