From 77e209ac9704185fad57271df9a5f058e0583f4e Mon Sep 17 00:00:00 2001 From: f Date: Tue, 14 Mar 2023 19:23:17 -0300 Subject: [PATCH 01/28] fix: traducir el asunto #9941 --- app/mailers/deploy_mailer.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/mailers/deploy_mailer.rb b/app/mailers/deploy_mailer.rb index 7d939940..25efb18d 100644 --- a/app/mailers/deploy_mailer.rb +++ b/app/mailers/deploy_mailer.rb @@ -15,12 +15,13 @@ class DeployMailer < ApplicationMailer def deployed(deploys) usuarie = Usuarie.find(params[:usuarie]) site = usuarie.sites.find(params[:site]) - subject = t('.subject', site: site.name) hostname = site.hostname # Informamos a cada quien en su idioma y damos una dirección de # respuesta porque a veces les usuaries nos escriben I18n.with_locale(usuarie.lang) do + subject = t('.subject', site: site.name) + @hi = t('.hi') @explanation = t('.explanation', fqdn: hostname) @help = t('.help') From 3230f71166cd47ffddfabb112cf9d1dfb204761d Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 18:38:47 -0300 Subject: [PATCH 02/28] feat: los deploys son interdependientes --- app/models/deploy.rb | 3 +++ app/models/deploy_alternative_domain.rb | 2 ++ app/models/deploy_rsync.rb | 8 ++++++++ app/models/deploy_www.rb | 2 ++ app/models/deploy_zip.rb | 2 ++ 5 files changed, 17 insertions(+) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 3f034ad5..40cb79b5 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'open3' + # Este modelo implementa los distintos tipos de alojamiento que provee # Sutty. # @@ -11,6 +12,8 @@ class Deploy < ApplicationRecord belongs_to :site has_many :build_stats, dependent: :destroy + DEPENDENCIES = [] + def deploy raise NotImplementedError end diff --git a/app/models/deploy_alternative_domain.rb b/app/models/deploy_alternative_domain.rb index e4960e65..fce69159 100644 --- a/app/models/deploy_alternative_domain.rb +++ b/app/models/deploy_alternative_domain.rb @@ -4,6 +4,8 @@ class DeployAlternativeDomain < Deploy store :values, accessors: %i[hostname], coder: JSON + DEPENDENCIES = %i[local] + # Generar un link simbólico del sitio principal al alternativo def deploy File.symlink?(destination) || diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 996f8cdd..ed71ec37 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -5,6 +5,14 @@ class DeployRsync < Deploy store :values, accessors: %i[destination host_keys], coder: JSON + DEPENDENCIES = %i[ + alternative_domain + hidden_service + local + www + zip + ] + def deploy ssh? && rsync end diff --git a/app/models/deploy_www.rb b/app/models/deploy_www.rb index 5602b0fc..088b0179 100644 --- a/app/models/deploy_www.rb +++ b/app/models/deploy_www.rb @@ -4,6 +4,8 @@ class DeployWww < Deploy store :values, accessors: %i[], coder: JSON + DEPENDENCIES = %i[local] + before_destroy :remove_destination! def deploy diff --git a/app/models/deploy_zip.rb b/app/models/deploy_zip.rb index ec8973d1..4bc9a240 100644 --- a/app/models/deploy_zip.rb +++ b/app/models/deploy_zip.rb @@ -8,6 +8,8 @@ require 'zip' class DeployZip < Deploy store :values, accessors: %i[], coder: JSON + DEPENDENCIES = %i[local] + # Una vez que el sitio está generado, tomar todos los archivos y # y generar un zip accesible públicamente. # From 2d3e4300c3bbefbd5a9a063f1c8ef40071a03dc6 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 18:54:38 -0300 Subject: [PATCH 03/28] feat: usar un grafo dirigido para ordenar las dependencias #10464 --- Gemfile | 1 + Gemfile.lock | 7 +++++++ app/models/site/deploy_dependencies.rb | 28 ++++++++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 app/models/site/deploy_dependencies.rb diff --git a/Gemfile b/Gemfile index 78eb020c..0fba35cc 100644 --- a/Gemfile +++ b/Gemfile @@ -23,6 +23,7 @@ if ENV['RAILS_GROUPS']&.split(',')&.include? 'assets' end gem 'nokogiri' +gem 'rgl' # Turbolinks makes navigating your web application faster. Read more: # https://github.com/turbolinks/turbolinks diff --git a/Gemfile.lock b/Gemfile.lock index dffe90bf..2eab3574 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -353,6 +353,7 @@ GEM mini_portile2 (~> 2.6.1) racc (~> 1.4) orm_adapter (0.5.0) + pairing_heap (3.0.0) parallel (1.21.0) parser (3.0.2.0) ast (~> 2.4.1) @@ -443,6 +444,10 @@ GEM actionpack (>= 5.0) railties (>= 5.0) rexml (3.2.5) + rgl (0.6.2) + pairing_heap (>= 0.3.0) + rexml (~> 3.2, >= 3.2.4) + stream (~> 0.5.3) rouge (3.26.1) rubocop (1.23.0) parallel (~> 1.10) @@ -510,6 +515,7 @@ GEM sprockets (>= 3.0.0) sqlite3 (1.4.2-x86_64-linux-musl) stackprof (0.2.17-x86_64-linux-musl) + stream (0.5.5) sucker_punch (3.0.1) concurrent-ruby (~> 1.0) sutty-archives (2.5.4) @@ -626,6 +632,7 @@ DEPENDENCIES rails_warden redis redis-rails + rgl rollups! rubocop-rails rubyzip diff --git a/app/models/site/deploy_dependencies.rb b/app/models/site/deploy_dependencies.rb new file mode 100644 index 00000000..472bbdf2 --- /dev/null +++ b/app/models/site/deploy_dependencies.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'rgl/adjacent' +require 'rgl/topsort' + +class Site + module DeployDependencies + extend ActiveSupport::Concern + + included do + def deployment_graph + @deployment_graph ||= RGL::AdjacencyGraph.new.tap do |graph| + deploys.each do |deploy| + graph.add_vertex deploy + end + + deploys.each do |deploy| + deploy.class::DEPENDENCIES.each do |dependency| + deploys.where(type: "Deploy#{dependency.to_s.classify}").each do |deploy_dependency| + graph.add_edge deploy_dependency, deploy + end + end + end + end + end + end + end +end From 4528da5bcb4d365439403823f0b01d999fc31c12 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 18:57:50 -0300 Subject: [PATCH 04/28] fixup! feat: usar un grafo dirigido para ordenar las dependencias #10464 --- app/models/site/deploy_dependencies.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/models/site/deploy_dependencies.rb b/app/models/site/deploy_dependencies.rb index 472bbdf2..b807f0e8 100644 --- a/app/models/site/deploy_dependencies.rb +++ b/app/models/site/deploy_dependencies.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true -require 'rgl/adjacent' -require 'rgl/topsort' +require 'rgl/adjacency' class Site module DeployDependencies From 5440ecc2c24145a1b52846dfc585f19f9f74f1ab Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 21:32:05 -0300 Subject: [PATCH 05/28] fixup! fixup! feat: usar un grafo dirigido para ordenar las dependencias #10464 --- app/models/site/deploy_dependencies.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/site/deploy_dependencies.rb b/app/models/site/deploy_dependencies.rb index b807f0e8..b40d8d04 100644 --- a/app/models/site/deploy_dependencies.rb +++ b/app/models/site/deploy_dependencies.rb @@ -8,7 +8,7 @@ class Site included do def deployment_graph - @deployment_graph ||= RGL::AdjacencyGraph.new.tap do |graph| + @deployment_graph ||= RGL::DirectedAdjacencyGraph.new.tap do |graph| deploys.each do |deploy| graph.add_vertex deploy end From 2453a105a3697230432d709576ee2e37a0580dd9 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 21:37:49 -0300 Subject: [PATCH 06/28] fixup! fixup! fixup! feat: usar un grafo dirigido para ordenar las dependencias #10464 --- app/models/site/deploy_dependencies.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/models/site/deploy_dependencies.rb b/app/models/site/deploy_dependencies.rb index b40d8d04..33ed9e37 100644 --- a/app/models/site/deploy_dependencies.rb +++ b/app/models/site/deploy_dependencies.rb @@ -7,6 +7,9 @@ class Site extend ActiveSupport::Concern included do + # Genera un grafo dirigido de todos los métodos de publicación + # + # @return [RGL::DirectedAdjacencyGraph] def deployment_graph @deployment_graph ||= RGL::DirectedAdjacencyGraph.new.tap do |graph| deploys.each do |deploy| From e67f67e175da8e391d00092375494dada88377bd Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 21:37:58 -0300 Subject: [PATCH 07/28] feat: devolver una lista ordenada de metodos de publicacion --- app/models/site/deploy_dependencies.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models/site/deploy_dependencies.rb b/app/models/site/deploy_dependencies.rb index 33ed9e37..05939320 100644 --- a/app/models/site/deploy_dependencies.rb +++ b/app/models/site/deploy_dependencies.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'rgl/adjacency' +require 'rgl/topsort' class Site module DeployDependencies @@ -25,6 +26,13 @@ class Site end end end + + # Devuelve una lista ordenada de todos los métodos de publicación + # + # @return [Array] + def deployment_list + @deployment_list ||= deployment_graph.topsort_iterator.to_a + end end end end From 358d2f3a3a41880f9f95d29791e14451618c0cc0 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 21:38:31 -0300 Subject: [PATCH 08/28] feat: activar ordenamiento de metodos de publicacion --- app/models/site.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/site.rb b/app/models/site.rb index 8cab0ae0..f5c48d36 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -7,6 +7,7 @@ class Site < ApplicationRecord include Site::Forms include Site::FindAndReplace include Site::Api + include Site::DeployDependencies include Tienda # Cifrar la llave privada que cifra y decifra campos ocultos. Sutty From 57719c8f44c420635ee080e90bf84be7312faa4d Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 13:42:47 -0300 Subject: [PATCH 09/28] feat: generar el sitio en orden #10464 --- app/jobs/deploy_job.rb | 83 +++++++++++-------------- app/models/deploy_alternative_domain.rb | 2 +- app/models/deploy_rsync.rb | 10 +-- app/models/deploy_www.rb | 2 +- app/models/deploy_zip.rb | 2 +- app/models/site/deploy_dependencies.rb | 2 +- 6 files changed, 46 insertions(+), 55 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index b4bb0097..d7eee49d 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -26,28 +26,30 @@ class DeployJob < ApplicationJob return end + @deployed = {} @site.update status: 'building' - # Asegurarse que DeployLocal sea el primero! - @deployed = { - deploy_local: { - status: deploy_locally, - seconds: deploy_local.build_stats.last.seconds, - size: deploy_local.size, - urls: [deploy_local.url] + @site.deployment_list.each do |d| + begin + raise DeployException, 'Una dependencia falló' if failed_dependencies? d + + status = d.deploy + seconds = d.build_stats.last.try(:seconds) + rescue StandardError => e + status = false + seconds = 0 + + notify_exception e, d + end + + @deployed[d.type.underscore.to_sym] = { + status: status, + seconds: seconds || 0, + size: d.size, + urls: d.respond_to?(:urls) ? d.urls : [d.url].compact } - } - - # No es opcional - unless @deployed[:deploy_local][:status] - # Hacer fallar la tarea - raise DeployException, "#{@site.name}: Falló la compilación" end - - deploy_others rescue DeployTimedOutException => e notify_exception e - rescue DeployException => e - notify_exception e, deploy_local ensure @site&.update status: 'waiting' @@ -58,6 +60,24 @@ class DeployJob < ApplicationJob private + # Detecta si un método de publicación tiene dependencias fallidas + # + # @param :deploy [Deploy] + # @return [Boolean] + def failed_dependencies?(deploy) + !(deploy.class::DEPENDENCIES - failed_dependencies(deploy)).empty? + end + + # Obtiene las dependencias fallidas de un deploy + # + # @param :deploy [Deploy] + # @return [Array] + def failed_dependencies(deploy) + @deployed.select do |_, v| + !v[:status] + end.keys + end + # @param :exception [StandardError] # @param :deploy [Deploy] def notify_exception(exception, deploy = nil) @@ -70,35 +90,6 @@ class DeployJob < ApplicationJob ExceptionNotifier.notify_exception(exception, data: data) end - def deploy_local - @deploy_local ||= @site.deploys.find_by(type: 'DeployLocal') - end - - def deploy_locally - deploy_local.deploy - end - - def deploy_others - @site.deploys.where.not(type: 'DeployLocal').find_each do |d| - begin - status = d.deploy - seconds = d.build_stats.last.try(:seconds) - rescue StandardError => e - status = false - seconds = 0 - - notify_exception e, d - end - - @deployed[d.type.underscore.to_sym] = { - status: status, - seconds: seconds || 0, - size: d.size, - urls: d.respond_to?(:urls) ? d.urls : [d.url].compact - } - end - end - def notify_usuaries @site.roles.where(rol: 'usuarie', temporal: false).pluck(:usuarie_id).each do |usuarie| DeployMailer.with(usuarie: usuarie, site: @site.id) diff --git a/app/models/deploy_alternative_domain.rb b/app/models/deploy_alternative_domain.rb index db7de872..a437b534 100644 --- a/app/models/deploy_alternative_domain.rb +++ b/app/models/deploy_alternative_domain.rb @@ -4,7 +4,7 @@ class DeployAlternativeDomain < Deploy store :values, accessors: %i[hostname], coder: JSON - DEPENDENCIES = %i[local] + DEPENDENCIES = %i[deploy_local] # Generar un link simbólico del sitio principal al alternativo def deploy diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index ed71ec37..01ef1990 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -6,11 +6,11 @@ class DeployRsync < Deploy store :values, accessors: %i[destination host_keys], coder: JSON DEPENDENCIES = %i[ - alternative_domain - hidden_service - local - www - zip + deploy_alternative_domain + deploy_hidden_service + deploy_local + deploy_www + deploy_zip ] def deploy diff --git a/app/models/deploy_www.rb b/app/models/deploy_www.rb index 2b604fe4..2e12a05f 100644 --- a/app/models/deploy_www.rb +++ b/app/models/deploy_www.rb @@ -4,7 +4,7 @@ class DeployWww < Deploy store :values, accessors: %i[], coder: JSON - DEPENDENCIES = %i[local] + DEPENDENCIES = %i[deploy_local] before_destroy :remove_destination! diff --git a/app/models/deploy_zip.rb b/app/models/deploy_zip.rb index 8e257041..e1632998 100644 --- a/app/models/deploy_zip.rb +++ b/app/models/deploy_zip.rb @@ -8,7 +8,7 @@ require 'zip' class DeployZip < Deploy store :values, accessors: %i[], coder: JSON - DEPENDENCIES = %i[local] + DEPENDENCIES = %i[deploy_local] # Una vez que el sitio está generado, tomar todos los archivos y # y generar un zip accesible públicamente. diff --git a/app/models/site/deploy_dependencies.rb b/app/models/site/deploy_dependencies.rb index 05939320..a01f99e7 100644 --- a/app/models/site/deploy_dependencies.rb +++ b/app/models/site/deploy_dependencies.rb @@ -19,7 +19,7 @@ class Site deploys.each do |deploy| deploy.class::DEPENDENCIES.each do |dependency| - deploys.where(type: "Deploy#{dependency.to_s.classify}").each do |deploy_dependency| + deploys.where(type: dependency.to_s.classify).each do |deploy_dependency| graph.add_edge deploy_dependency, deploy end end From 1059ab55014f33d0d92f3731df4a3d2c2c370dd6 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 13:47:34 -0300 Subject: [PATCH 10/28] =?UTF-8?q?fix:=20no=20pedir=20el=20tama=C3=B1o=20si?= =?UTF-8?q?=20fall=C3=B3=20la=20compilaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/deploy_job.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index d7eee49d..cc9aa429 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -34,9 +34,11 @@ class DeployJob < ApplicationJob status = d.deploy seconds = d.build_stats.last.try(:seconds) + size = d.size rescue StandardError => e status = false seconds = 0 + size = 0 notify_exception e, d end @@ -44,7 +46,7 @@ class DeployJob < ApplicationJob @deployed[d.type.underscore.to_sym] = { status: status, seconds: seconds || 0, - size: d.size, + size: size, urls: d.respond_to?(:urls) ? d.urls : [d.url].compact } end From 51e11d38e4227f0912c5265717311e4b5c4cd7c7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 13:54:19 -0300 Subject: [PATCH 11/28] =?UTF-8?q?fix:=20mejorar=20gesti=C3=B3n=20de=20zip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_zip.rb | 53 +++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/app/models/deploy_zip.rb b/app/models/deploy_zip.rb index e1632998..5f72a728 100644 --- a/app/models/deploy_zip.rb +++ b/app/models/deploy_zip.rb @@ -14,24 +14,43 @@ class DeployZip < Deploy # y generar un zip accesible públicamente. # # rubocop:disable Metrics/MethodLength - def deploy + def deploy(output: false) FileUtils.rm_f path - time_start - Dir.chdir(destination) do - Zip::File.open(path, Zip::File::CREATE) do |z| - Dir.glob('./**/**').each do |f| - File.directory?(f) ? z.mkdir(f) : z.add(f, f) + Zip::File.open(path, Zip::File::CREATE) do |zip| + Dir.glob(File.join(destination, '**', '**')).each do |file| + entry = Pathname.new(file).relative_path_from(destination).to_s + + if File.directory? file + log "Creando directorio #{entry}", output + + zip.mkdir(entry) + else + log "Comprimiendo #{entry}", output + zip.add(entry, file) end end end + time_stop - build_stats.create action: 'zip', - seconds: time_spent_in_seconds, - bytes: size + File.exist?(path).tap do |status| + build_stats.create action: 'zip', + seconds: time_spent_in_seconds, + bytes: size, + log: @log.join("\n"), + status: status + end + rescue Zip::Error => e + ExceptionNotifier.notify_exception(e, data: { site: site.name }) - File.exist? path + build_stats.create action: 'zip', + seconds: 0, + bytes: 0, + log: @log.join("\n"), + status: false + + false end # rubocop:enable Metrics/MethodLength @@ -43,8 +62,9 @@ class DeployZip < Deploy File.size path end + # @return [String] def destination - File.join(Rails.root, '_deploy', site.hostname) + Rails.root.join('_deploy', site.hostname).realpath.to_s end def file @@ -58,4 +78,15 @@ class DeployZip < Deploy def path File.join(destination, file) end + + private + + # @param :line [String] + # @param :output [Boolean] + def log(line, output) + @log ||= [] + @log << line + + puts line if output + end end From 56b2556447c4526a7a471f22b37c7d94b2c208f4 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 14:21:11 -0300 Subject: [PATCH 12/28] fixup! feat: generar el sitio en orden #10464 --- app/jobs/deploy_job.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index cc9aa429..37063ff3 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -67,7 +67,7 @@ class DeployJob < ApplicationJob # @param :deploy [Deploy] # @return [Boolean] def failed_dependencies?(deploy) - !(deploy.class::DEPENDENCIES - failed_dependencies(deploy)).empty? + failed_dependencies(deploy).present? end # Obtiene las dependencias fallidas de un deploy @@ -75,9 +75,9 @@ class DeployJob < ApplicationJob # @param :deploy [Deploy] # @return [Array] def failed_dependencies(deploy) - @deployed.select do |_, v| - !v[:status] - end.keys + deploy.class::DEPENDENCIES & (@deployed.reject do |_, v| + v[:status] + end.keys) end # @param :exception [StandardError] @@ -86,7 +86,8 @@ class DeployJob < ApplicationJob data = { site: @site.id, deploy: deploy&.type, - log: deploy&.build_stats&.last&.log + log: deploy&.build_stats&.last&.log, + failed_dependencies: (failed_dependencies(deploy) if deploy) } ExceptionNotifier.notify_exception(exception, data: data) From 2b4cf2bc753419a217642905839a2a5a67c1b0c8 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 14 Mar 2022 13:39:54 -0300 Subject: [PATCH 13/28] poder ver la salida opcionalmente --- app/models/deploy.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index aefd8090..269747fc 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -55,7 +55,7 @@ class Deploy < ApplicationRecord # # @param [String] # @return [Boolean] - def run(cmd) + def run(cmd, output: false) r = nil lines = [] @@ -68,6 +68,7 @@ class Deploy < ApplicationRecord # TODO: Enviar a un websocket para ver el proceso en vivo? o.each do |line| lines << line + puts line if output end end end From 878311abf6470d1b04beeba3cde5911987c90600 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 14 Mar 2022 13:40:17 -0300 Subject: [PATCH 14/28] leer la salida por separado para no bloquear el programa --- app/models/deploy.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 269747fc..a911ad84 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -62,14 +62,15 @@ class Deploy < ApplicationRecord time_start Dir.chdir(site.path) do Open3.popen2e(env, cmd, unsetenv_others: true) do |_, o, t| - r = t.value - # XXX: Tenemos que leer línea por línea porque en salidas largas - # se cuelga la IO # TODO: Enviar a un websocket para ver el proceso en vivo? - o.each do |line| - lines << line - puts line if output + Thread.new do + o.each do |line| + lines << line + + puts line if output + end end + r = t.value end end time_stop From b9b11b20b89d7ebd7172f06cdeb8ba40dc98004c Mon Sep 17 00:00:00 2001 From: f Date: Tue, 15 Mar 2022 16:03:16 -0300 Subject: [PATCH 15/28] poder ver la salida si lo ejecutamos desde la terminal --- app/jobs/deploy_job.rb | 6 ++++-- app/models/deploy_local.rb | 22 +++++++++++----------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 37063ff3..ccad1c5c 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -6,7 +6,9 @@ class DeployJob < ApplicationJob class DeployTimedOutException < DeployException; end # rubocop:disable Metrics/MethodLength - def perform(site, notify = true, time = Time.now) + def perform(site, notify: true, time: Time.now, output: false) + @output = output + ActiveRecord::Base.connection_pool.with_connection do @site = Site.find(site) @@ -22,7 +24,7 @@ class DeployJob < ApplicationJob "#{@site.name} la tarea estuvo más de 10 minutos esperando, volviendo al estado original" end - DeployJob.perform_in(60, site, notify, time) + DeployJob.perform_in(60, site, notify: notify, time: time, output: output) return end diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index c58fd9b3..c0da5430 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -12,12 +12,12 @@ class DeployLocal < Deploy # # Pasamos variables de entorno mínimas para no filtrar secretos de # Sutty - def deploy + def deploy(output: false) return false unless mkdir - return false unless yarn - return false unless bundle + return false unless yarn(output: output) + return false unless bundle(output: output) - jekyll_build + jekyll_build(output: output) end # Sólo permitimos un deploy local @@ -85,23 +85,23 @@ class DeployLocal < Deploy File.exist? yarn_lock end - def gem - run %(gem install bundler --no-document) + def gem(output: false) + run %(gem install bundler --no-document), output: output end # Corre yarn dentro del repositorio - def yarn + def yarn(output: false) return true unless yarn_lock? - run 'yarn install --production' + run 'yarn install --production', output: output end def bundle - run %(bundle install --no-cache --path="#{gems_dir}") + run %(bundle install --no-cache --path="#{gems_dir}"), output: output end - def jekyll_build - run %(bundle exec jekyll build --trace --profile --destination "#{escaped_destination}") + def jekyll_build(output: false) + run %(bundle exec jekyll build --trace --profile --destination "#{escaped_destination}"), output: output end # no debería haber espacios ni caracteres especiales, pero por si From 193ba69a5fdaf6ff4f4ce21e90ce6e58361bff32 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 16 Mar 2022 18:19:04 -0300 Subject: [PATCH 16/28] implementar la misma api en todos los deploys --- app/models/deploy.rb | 2 +- app/models/deploy_alternative_domain.rb | 2 +- app/models/deploy_hidden_service.rb | 2 +- app/models/deploy_localized_domain.rb | 12 ++++++++++++ app/models/deploy_private.rb | 4 ++-- app/models/deploy_www.rb | 2 +- 6 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 app/models/deploy_localized_domain.rb diff --git a/app/models/deploy.rb b/app/models/deploy.rb index a911ad84..967e798f 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -14,7 +14,7 @@ class Deploy < ApplicationRecord DEPENDENCIES = [] - def deploy + def deploy(**) raise NotImplementedError end diff --git a/app/models/deploy_alternative_domain.rb b/app/models/deploy_alternative_domain.rb index a437b534..9b1d63d7 100644 --- a/app/models/deploy_alternative_domain.rb +++ b/app/models/deploy_alternative_domain.rb @@ -7,7 +7,7 @@ class DeployAlternativeDomain < Deploy DEPENDENCIES = %i[deploy_local] # Generar un link simbólico del sitio principal al alternativo - def deploy + def deploy(**) File.symlink?(destination) || File.symlink(site.hostname, destination).zero? end diff --git a/app/models/deploy_hidden_service.rb b/app/models/deploy_hidden_service.rb index 8df46c2e..dc9549f5 100644 --- a/app/models/deploy_hidden_service.rb +++ b/app/models/deploy_hidden_service.rb @@ -2,7 +2,7 @@ # Genera una versión onion class DeployHiddenService < DeployWww - def deploy + def deploy(**) return true if fqdn.blank? super diff --git a/app/models/deploy_localized_domain.rb b/app/models/deploy_localized_domain.rb new file mode 100644 index 00000000..59e17dcd --- /dev/null +++ b/app/models/deploy_localized_domain.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Soportar dominios localizados +class DeployLocalizedDomain < DeployAlternativeDomain + store :values, accessors: %i[hostname locale], coder: JSON + + # Generar un link simbólico del sitio principal al alternativo + def deploy(**) + File.symlink?(destination) || + File.symlink(File.join(site.hostname, locale), destination).zero? + end +end diff --git a/app/models/deploy_private.rb b/app/models/deploy_private.rb index 0d79811d..d3bfb50d 100644 --- a/app/models/deploy_private.rb +++ b/app/models/deploy_private.rb @@ -7,8 +7,8 @@ # jekyll-private-data class DeployPrivate < DeployLocal # No es necesario volver a instalar dependencias - def deploy - jekyll_build + def deploy(output: false) + jekyll_build(output: output) end # Hacer el deploy a un directorio privado diff --git a/app/models/deploy_www.rb b/app/models/deploy_www.rb index 2e12a05f..d27f77cc 100644 --- a/app/models/deploy_www.rb +++ b/app/models/deploy_www.rb @@ -8,7 +8,7 @@ class DeployWww < Deploy before_destroy :remove_destination! - def deploy + def deploy(**) File.symlink?(destination) || File.symlink(site.hostname, destination).zero? end From 3c78f0dfb9e585470d56f560f4f9c8fb3a7bec86 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 18 Apr 2022 17:27:43 -0300 Subject: [PATCH 17/28] no fallar al cerrarse la salida --- app/models/deploy.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 967e798f..19d1f412 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -69,7 +69,10 @@ class Deploy < ApplicationRecord puts line if output end + rescue IOError => e + ExceptionNotifier.notify(e, data: { site: site.name }) end + r = t.value end end From 4de34ceab5a8d7add4b07bcf04ea0a63273bf5e1 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 28 Jun 2022 15:07:08 -0300 Subject: [PATCH 18/28] =?UTF-8?q?notificar=20correctamente=20la=20excepci?= =?UTF-8?q?=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 19d1f412..360b36d1 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -70,7 +70,7 @@ class Deploy < ApplicationRecord puts line if output end rescue IOError => e - ExceptionNotifier.notify(e, data: { site: site.name }) + ExceptionNotifier.notify_exception(e, data: { site: site.name }) end r = t.value From 709f3b29e53f608cc57659e92ef7ef3b3323f61b Mon Sep 17 00:00:00 2001 From: f Date: Wed, 29 Jun 2022 18:41:51 -0300 Subject: [PATCH 19/28] no generar issues para log --- app/models/deploy.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 360b36d1..5f2d3cca 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -70,7 +70,8 @@ class Deploy < ApplicationRecord puts line if output end rescue IOError => e - ExceptionNotifier.notify_exception(e, data: { site: site.name }) + lines << e.message + puts e.message if output end r = t.value From ba3e595f78677d20a005d2a75e45e647d4763e52 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 14:31:39 -0300 Subject: [PATCH 20/28] =?UTF-8?q?fix:=20mejorar=20gesti=C3=B3n=20de=20syml?= =?UTF-8?q?ink?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_www.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/deploy_www.rb b/app/models/deploy_www.rb index d27f77cc..bb25cc64 100644 --- a/app/models/deploy_www.rb +++ b/app/models/deploy_www.rb @@ -8,7 +8,9 @@ class DeployWww < Deploy before_destroy :remove_destination! - def deploy(**) + def deploy(output: false) + puts "Creando symlink #{site.hostname} => #{destination}" if output + File.symlink?(destination) || File.symlink(site.hostname, destination).zero? end @@ -30,7 +32,7 @@ class DeployWww < Deploy end def url - "https://www.#{site.hostname}/" + "https://#{fqdn}/" end private From dedf8979e484ee1445f2e0db43cd448651532b35 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 14:37:52 -0300 Subject: [PATCH 21/28] =?UTF-8?q?fix:=20rsync=20no=20deber=C3=ADa=20depend?= =?UTF-8?q?er=20de=20zip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aunque el zip no se genere igual queremos poder sincronizar el sitio --- app/models/deploy_rsync.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 01ef1990..de6f980b 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -10,7 +10,6 @@ class DeployRsync < Deploy deploy_hidden_service deploy_local deploy_www - deploy_zip ] def deploy From 753000736759cc697aa9838299e4d0ea9875f7e8 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 14:38:45 -0300 Subject: [PATCH 22/28] feat: eliminar archivos al sincronizar --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index de6f980b..a70b4a34 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -91,7 +91,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -aviH --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From fa04538669a93c79db7fc3244450c809d0a400d8 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 14:45:35 -0300 Subject: [PATCH 23/28] fix: output para rsync --- app/models/deploy_rsync.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index a70b4a34..c75f744d 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -12,8 +12,8 @@ class DeployRsync < Deploy deploy_www ] - def deploy - ssh? && rsync + def deploy(output: false) + ssh? && rsync(output: output) end # El espacio remoto es el mismo que el local @@ -91,7 +91,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/), output: output end # El origen es el destino de la compilación From 151aecc4d8f2598ccc6d5b313e6ef1deb5202ef9 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 14:59:34 -0300 Subject: [PATCH 24/28] feat: mostrar una tabla con el resumen en la consola --- app/jobs/deploy_job.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index ccad1c5c..5634108d 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -52,6 +52,16 @@ class DeployJob < ApplicationJob urls: d.respond_to?(:urls) ? d.urls : [d.url].compact } end + + return unless @output + + puts (Terminal::Table.new do |t| + t << (%w[type] + @deployed.values.first.keys) + t.add_separator + @deployed.each do |type, row| + t << ([type.to_s] + row.values) + end + end) rescue DeployTimedOutException => e notify_exception e ensure From 39a615c00c510f8303de91861832a7b29ed45086 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 15:10:32 -0300 Subject: [PATCH 25/28] =?UTF-8?q?fix:=20fallar=20el=20hidden=20service=20s?= =?UTF-8?q?i=20no=20se=20gener=C3=B3=20el=20onion=20aun?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_hidden_service.rb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/app/models/deploy_hidden_service.rb b/app/models/deploy_hidden_service.rb index dc9549f5..79ff1bae 100644 --- a/app/models/deploy_hidden_service.rb +++ b/app/models/deploy_hidden_service.rb @@ -2,14 +2,10 @@ # Genera una versión onion class DeployHiddenService < DeployWww - def deploy(**) - return true if fqdn.blank? - - super - end - def fqdn - values[:onion] + values[:onion].tap do |onion| + raise ArgumentError, 'Aun no se generó la dirección .onion' if onion.blank? + end end def url From 816773282b748469a0d94d3eb1b345f4d7616e63 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 15:22:10 -0300 Subject: [PATCH 26/28] fix: capturar todas las posibles excepciones --- app/jobs/deploy_job.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 5634108d..9ded7dc9 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -37,19 +37,21 @@ class DeployJob < ApplicationJob status = d.deploy seconds = d.build_stats.last.try(:seconds) size = d.size + urls = d.respond_to?(:urls) ? d.urls : [d.url].compact rescue StandardError => e status = false - seconds = 0 - size = 0 + seconds ||= 0 + size ||= 0 + urls ||= [] notify_exception e, d end @deployed[d.type.underscore.to_sym] = { status: status, - seconds: seconds || 0, + seconds: seconds, size: size, - urls: d.respond_to?(:urls) ? d.urls : [d.url].compact + urls: urls } end From d1a3ec25698deaf27af126abb172200fc5457d4b Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 15:23:13 -0300 Subject: [PATCH 27/28] feat: hacer sonar una campana --- app/jobs/deploy_job.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 9ded7dc9..9fbdb570 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -64,6 +64,8 @@ class DeployJob < ApplicationJob t << ([type.to_s] + row.values) end end) + + puts "\a" rescue DeployTimedOutException => e notify_exception e ensure From e623504665c924550c53369c476d5796c5acb3b9 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 15:50:05 -0300 Subject: [PATCH 28/28] feat: separar rsync de full rsync los segundos se usan para sincronizar todas las versiones de un sitio con otro servidor de sutty. los primeros solo sincronizan los archivos a otro servidor, no necesariamente bajo el mismo nombre. --- app/models/deploy_alternative_domain.rb | 6 ++- app/models/deploy_full_rsync.rb | 22 +++++++++++ app/models/deploy_rsync.rb | 9 +---- app/services/site_service.rb | 2 +- ...ename_deploy_rsync_to_deploy_full_rsync.rb | 37 +++++++++++++++++++ 5 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 app/models/deploy_full_rsync.rb create mode 100644 db/migrate/20230318183722_rename_deploy_rsync_to_deploy_full_rsync.rb diff --git a/app/models/deploy_alternative_domain.rb b/app/models/deploy_alternative_domain.rb index 9b1d63d7..75b69180 100644 --- a/app/models/deploy_alternative_domain.rb +++ b/app/models/deploy_alternative_domain.rb @@ -20,7 +20,11 @@ class DeployAlternativeDomain < Deploy end def destination - @destination ||= File.join(Rails.root, '_deploy', hostname.gsub(/\.\z/, '')) + @destination ||= File.join(Rails.root, '_deploy', fqdn) + end + + def fqdn + hostname.gsub(/\.\z/, '') end def url diff --git a/app/models/deploy_full_rsync.rb b/app/models/deploy_full_rsync.rb new file mode 100644 index 00000000..c0ff84c6 --- /dev/null +++ b/app/models/deploy_full_rsync.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class DeployFullRsync < DeployRsync + DEPENDENCIES = %i[ + deploy_alternative_domain + deploy_hidden_service + deploy_local + deploy_www + ] + + # Sincroniza las ubicaciones alternativas también + # + # @param :output [Boolean] + # @return [Boolean] + def rsync(output: false) + DEPENDENCIES.map(&:to_s).map(&:classify).map do |dependency| + site.deploys.where(type: dependency).find_each.map do |deploy| + run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape deploy.destination} #{Shellwords.escape destination}), output: output + end + end.flatten.all? + end +end diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index c75f744d..6a96a274 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -5,12 +5,7 @@ class DeployRsync < Deploy store :values, accessors: %i[destination host_keys], coder: JSON - DEPENDENCIES = %i[ - deploy_alternative_domain - deploy_hidden_service - deploy_local - deploy_www - ] + DEPENDENCIES = %i[deploy_local] def deploy(output: false) ssh? && rsync(output: output) @@ -90,7 +85,7 @@ class DeployRsync < Deploy # Sincroniza hacia el directorio remoto # # @return [Boolean] - def rsync + def rsync(output: false) run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/), output: output end diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 22423bb8..39e1c845 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -149,7 +149,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do # Crea los deploys necesarios para sincronizar a otros nodos de Sutty def sync_nodes Rails.application.nodes.each do |node| - site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") + site.deploys.build(type: 'DeployFullRsync', destination: "sutty@#{node}:") end end end diff --git a/db/migrate/20230318183722_rename_deploy_rsync_to_deploy_full_rsync.rb b/db/migrate/20230318183722_rename_deploy_rsync_to_deploy_full_rsync.rb new file mode 100644 index 00000000..689dc559 --- /dev/null +++ b/db/migrate/20230318183722_rename_deploy_rsync_to_deploy_full_rsync.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# Cambia todos los DeployRsync propios de Sutty a DeployFullRsync que se +# encarga de sincronizar todo. +class RenameDeployRsyncToDeployFullRsync < ActiveRecord::Migration[6.1] + def up + DeployRsync.all.find_each do |deploy| + dest = deploy.destination.split(':', 2).first + + next unless nodes.include? dest + + deploy.destination = "#{dest}:" + deploy.type = 'DeployFullRsync' + + deploy.save + end + end + + def down + DeployFullRsync.all.find_each do |deploy| + next unless nodes.include? deploy.destination.split(':', 2).first + + deploy.destination = "#{deploy.destination}#{deploy.site.hostname}" + deploy.type = 'DeployRsync' + + deploy.save + end + end + + private + + def nodes + @nodes ||= Rails.application.nodes.map do |node| + "sutty@#{node}" + end + end +end