diff --git a/Gemfile b/Gemfile index 45773b29..e3a46e34 100644 --- a/Gemfile +++ b/Gemfile @@ -25,6 +25,7 @@ if ENV['RAILS_GROUPS']&.split(',')&.include? 'assets' end gem 'nokogiri' +gem 'rgl' # Turbolinks makes navigating your web application faster. Read more: # https://github.com/turbolinks/turbolinks diff --git a/Gemfile.lock b/Gemfile.lock index 98630274..8bd4d470 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -361,6 +361,7 @@ GEM dry-schema httparty (~> 0.18) orm_adapter (0.5.0) + pairing_heap (3.0.0) parallel (1.22.1) parser (3.1.3.0) ast (~> 2.4.1) @@ -455,6 +456,10 @@ GEM actionpack (>= 5.0) railties (>= 5.0) rexml (3.2.5) + rgl (0.6.2) + pairing_heap (>= 0.3.0) + rexml (~> 3.2, >= 3.2.4) + stream (~> 0.5.3) rouge (3.30.0) rubocop (1.41.1) json (~> 2.3) @@ -518,6 +523,7 @@ GEM sqlite3 (1.5.4-x86_64-linux-musl) mini_portile2 (~> 2.8.0) stackprof (0.2.23-x86_64-linux-musl) + stream (0.5.5) sucker_punch (3.1.0) concurrent-ruby (~> 1.0) sutty-liquid (0.11.6) @@ -637,6 +643,7 @@ DEPENDENCIES redis (~> 4.0) redis-rails reek + rgl rollups! rubocop-rails rubyzip diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index b241f25c..9fbdb570 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -28,28 +28,46 @@ class DeployJob < ApplicationJob return end + @deployed = {} @site.update status: 'building' - # Asegurarse que DeployLocal sea el primero! - @deployed = { - deploy_local: { - status: deploy_locally, - seconds: deploy_local.build_stats.last.seconds, - size: deploy_local.size, - urls: [deploy_local.url] - } - } + @site.deployment_list.each do |d| + begin + raise DeployException, 'Una dependencia falló' if failed_dependencies? d - # No es opcional - unless @deployed[:deploy_local][:status] - # Hacer fallar la tarea - raise DeployException, "#{@site.name}: Falló la compilación" + status = d.deploy + seconds = d.build_stats.last.try(:seconds) + size = d.size + urls = d.respond_to?(:urls) ? d.urls : [d.url].compact + rescue StandardError => e + status = false + seconds ||= 0 + size ||= 0 + urls ||= [] + + notify_exception e, d + end + + @deployed[d.type.underscore.to_sym] = { + status: status, + seconds: seconds, + size: size, + urls: urls + } end - deploy_others + return unless @output + + puts (Terminal::Table.new do |t| + t << (%w[type] + @deployed.values.first.keys) + t.add_separator + @deployed.each do |type, row| + t << ([type.to_s] + row.values) + end + end) + + puts "\a" rescue DeployTimedOutException => e notify_exception e - rescue DeployException => e - notify_exception e, deploy_local ensure @site&.update status: 'waiting' @@ -60,47 +78,37 @@ class DeployJob < ApplicationJob private + # Detecta si un método de publicación tiene dependencias fallidas + # + # @param :deploy [Deploy] + # @return [Boolean] + def failed_dependencies?(deploy) + failed_dependencies(deploy).present? + end + + # Obtiene las dependencias fallidas de un deploy + # + # @param :deploy [Deploy] + # @return [Array] + def failed_dependencies(deploy) + deploy.class::DEPENDENCIES & (@deployed.reject do |_, v| + v[:status] + end.keys) + end + # @param :exception [StandardError] # @param :deploy [Deploy] def notify_exception(exception, deploy = nil) data = { site: @site.id, deploy: deploy&.type, - log: deploy&.build_stats&.last&.log + log: deploy&.build_stats&.last&.log, + failed_dependencies: (failed_dependencies(deploy) if deploy) } ExceptionNotifier.notify_exception(exception, data: data) end - def deploy_local - @deploy_local ||= @site.deploys.find_by(type: 'DeployLocal') - end - - def deploy_locally - deploy_local.deploy(output: @output) - end - - def deploy_others - @site.deploys.where.not(type: 'DeployLocal').find_each do |d| - begin - status = d.deploy(output: @output) - seconds = d.build_stats.last.try(:seconds) - rescue StandardError => e - status = false - seconds = 0 - - notify_exception e, d - end - - @deployed[d.type.underscore.to_sym] = { - status: status, - seconds: seconds || 0, - size: d.size, - urls: d.respond_to?(:urls) ? d.urls : [d.url].compact - } - end - end - def notify_usuaries @site.roles.where(rol: 'usuarie', temporal: false).pluck(:usuarie_id).each do |usuarie| DeployMailer.with(usuarie: usuarie, site: @site.id) diff --git a/app/mailers/deploy_mailer.rb b/app/mailers/deploy_mailer.rb index 7d939940..25efb18d 100644 --- a/app/mailers/deploy_mailer.rb +++ b/app/mailers/deploy_mailer.rb @@ -15,12 +15,13 @@ class DeployMailer < ApplicationMailer def deployed(deploys) usuarie = Usuarie.find(params[:usuarie]) site = usuarie.sites.find(params[:site]) - subject = t('.subject', site: site.name) hostname = site.hostname # Informamos a cada quien en su idioma y damos una dirección de # respuesta porque a veces les usuaries nos escriben I18n.with_locale(usuarie.lang) do + subject = t('.subject', site: site.name) + @hi = t('.hi') @explanation = t('.explanation', fqdn: hostname) @help = t('.help') diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 1c2683fc..52031d87 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'open3' + # Este modelo implementa los distintos tipos de alojamiento que provee # Sutty. # @@ -11,6 +12,8 @@ class Deploy < ApplicationRecord belongs_to :site has_many :build_stats, dependent: :destroy + DEPENDENCIES = [] + def deploy(**) raise NotImplementedError end diff --git a/app/models/deploy_alternative_domain.rb b/app/models/deploy_alternative_domain.rb index 5ad381d5..75b69180 100644 --- a/app/models/deploy_alternative_domain.rb +++ b/app/models/deploy_alternative_domain.rb @@ -4,6 +4,8 @@ class DeployAlternativeDomain < Deploy store :values, accessors: %i[hostname], coder: JSON + DEPENDENCIES = %i[deploy_local] + # Generar un link simbólico del sitio principal al alternativo def deploy(**) File.symlink?(destination) || @@ -18,7 +20,11 @@ class DeployAlternativeDomain < Deploy end def destination - @destination ||= File.join(Rails.root, '_deploy', hostname.gsub(/\.\z/, '')) + @destination ||= File.join(Rails.root, '_deploy', fqdn) + end + + def fqdn + hostname.gsub(/\.\z/, '') end def url diff --git a/app/models/deploy_full_rsync.rb b/app/models/deploy_full_rsync.rb new file mode 100644 index 00000000..c0ff84c6 --- /dev/null +++ b/app/models/deploy_full_rsync.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class DeployFullRsync < DeployRsync + DEPENDENCIES = %i[ + deploy_alternative_domain + deploy_hidden_service + deploy_local + deploy_www + ] + + # Sincroniza las ubicaciones alternativas también + # + # @param :output [Boolean] + # @return [Boolean] + def rsync(output: false) + DEPENDENCIES.map(&:to_s).map(&:classify).map do |dependency| + site.deploys.where(type: dependency).find_each.map do |deploy| + run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape deploy.destination} #{Shellwords.escape destination}), output: output + end + end.flatten.all? + end +end diff --git a/app/models/deploy_hidden_service.rb b/app/models/deploy_hidden_service.rb index dc9549f5..79ff1bae 100644 --- a/app/models/deploy_hidden_service.rb +++ b/app/models/deploy_hidden_service.rb @@ -2,14 +2,10 @@ # Genera una versión onion class DeployHiddenService < DeployWww - def deploy(**) - return true if fqdn.blank? - - super - end - def fqdn - values[:onion] + values[:onion].tap do |onion| + raise ArgumentError, 'Aun no se generó la dirección .onion' if onion.blank? + end end def url diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 76d69446..0020324c 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -5,6 +5,8 @@ class DeployRsync < Deploy store :values, accessors: %i[destination host_keys], coder: JSON + DEPENDENCIES = %i[deploy_local] + def deploy(output: false) ssh? && rsync(output: output) end @@ -88,7 +90,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync(output: false) - run %(rsync -aviH --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/), output: output + run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/), output: output end # El origen es el destino de la compilación diff --git a/app/models/deploy_www.rb b/app/models/deploy_www.rb index dff769a6..bb25cc64 100644 --- a/app/models/deploy_www.rb +++ b/app/models/deploy_www.rb @@ -4,9 +4,13 @@ class DeployWww < Deploy store :values, accessors: %i[], coder: JSON + DEPENDENCIES = %i[deploy_local] + before_destroy :remove_destination! - def deploy(**) + def deploy(output: false) + puts "Creando symlink #{site.hostname} => #{destination}" if output + File.symlink?(destination) || File.symlink(site.hostname, destination).zero? end @@ -28,7 +32,7 @@ class DeployWww < Deploy end def url - "https://www.#{site.hostname}/" + "https://#{fqdn}/" end private diff --git a/app/models/deploy_zip.rb b/app/models/deploy_zip.rb index f1c94083..5f72a728 100644 --- a/app/models/deploy_zip.rb +++ b/app/models/deploy_zip.rb @@ -8,28 +8,49 @@ require 'zip' class DeployZip < Deploy store :values, accessors: %i[], coder: JSON + DEPENDENCIES = %i[deploy_local] + # Una vez que el sitio está generado, tomar todos los archivos y # y generar un zip accesible públicamente. # # rubocop:disable Metrics/MethodLength - def deploy(**) + def deploy(output: false) FileUtils.rm_f path - time_start - Dir.chdir(destination) do - Zip::File.open(path, Zip::File::CREATE) do |z| - Dir.glob('./**/**').each do |f| - File.directory?(f) ? z.mkdir(f) : z.add(f, f) + Zip::File.open(path, Zip::File::CREATE) do |zip| + Dir.glob(File.join(destination, '**', '**')).each do |file| + entry = Pathname.new(file).relative_path_from(destination).to_s + + if File.directory? file + log "Creando directorio #{entry}", output + + zip.mkdir(entry) + else + log "Comprimiendo #{entry}", output + zip.add(entry, file) end end end + time_stop - build_stats.create action: 'zip', - seconds: time_spent_in_seconds, - bytes: size + File.exist?(path).tap do |status| + build_stats.create action: 'zip', + seconds: time_spent_in_seconds, + bytes: size, + log: @log.join("\n"), + status: status + end + rescue Zip::Error => e + ExceptionNotifier.notify_exception(e, data: { site: site.name }) - File.exist? path + build_stats.create action: 'zip', + seconds: 0, + bytes: 0, + log: @log.join("\n"), + status: false + + false end # rubocop:enable Metrics/MethodLength @@ -41,8 +62,9 @@ class DeployZip < Deploy File.size path end + # @return [String] def destination - File.join(Rails.root, '_deploy', site.hostname) + Rails.root.join('_deploy', site.hostname).realpath.to_s end def file @@ -56,4 +78,15 @@ class DeployZip < Deploy def path File.join(destination, file) end + + private + + # @param :line [String] + # @param :output [Boolean] + def log(line, output) + @log ||= [] + @log << line + + puts line if output + end end diff --git a/app/models/site.rb b/app/models/site.rb index 21453370..af0b2c53 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -7,6 +7,7 @@ class Site < ApplicationRecord include Site::Forms include Site::FindAndReplace include Site::Api + include Site::DeployDependencies include Tienda # Cifrar la llave privada que cifra y decifra campos ocultos. Sutty diff --git a/app/models/site/deploy_dependencies.rb b/app/models/site/deploy_dependencies.rb new file mode 100644 index 00000000..a01f99e7 --- /dev/null +++ b/app/models/site/deploy_dependencies.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require 'rgl/adjacency' +require 'rgl/topsort' + +class Site + module DeployDependencies + extend ActiveSupport::Concern + + included do + # Genera un grafo dirigido de todos los métodos de publicación + # + # @return [RGL::DirectedAdjacencyGraph] + def deployment_graph + @deployment_graph ||= RGL::DirectedAdjacencyGraph.new.tap do |graph| + deploys.each do |deploy| + graph.add_vertex deploy + end + + deploys.each do |deploy| + deploy.class::DEPENDENCIES.each do |dependency| + deploys.where(type: dependency.to_s.classify).each do |deploy_dependency| + graph.add_edge deploy_dependency, deploy + end + end + end + end + end + + # Devuelve una lista ordenada de todos los métodos de publicación + # + # @return [Array] + def deployment_list + @deployment_list ||= deployment_graph.topsort_iterator.to_a + end + end + end +end diff --git a/app/services/site_service.rb b/app/services/site_service.rb index ec32c932..3e8fe7c9 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -152,7 +152,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do # Crea los deploys necesarios para sincronizar a otros nodos de Sutty def sync_nodes Rails.application.nodes.each do |node| - site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") + site.deploys.build(type: 'DeployFullRsync', destination: "sutty@#{node}:") end end end diff --git a/db/migrate/20230318183722_rename_deploy_rsync_to_deploy_full_rsync.rb b/db/migrate/20230318183722_rename_deploy_rsync_to_deploy_full_rsync.rb new file mode 100644 index 00000000..689dc559 --- /dev/null +++ b/db/migrate/20230318183722_rename_deploy_rsync_to_deploy_full_rsync.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# Cambia todos los DeployRsync propios de Sutty a DeployFullRsync que se +# encarga de sincronizar todo. +class RenameDeployRsyncToDeployFullRsync < ActiveRecord::Migration[6.1] + def up + DeployRsync.all.find_each do |deploy| + dest = deploy.destination.split(':', 2).first + + next unless nodes.include? dest + + deploy.destination = "#{dest}:" + deploy.type = 'DeployFullRsync' + + deploy.save + end + end + + def down + DeployFullRsync.all.find_each do |deploy| + next unless nodes.include? deploy.destination.split(':', 2).first + + deploy.destination = "#{deploy.destination}#{deploy.site.hostname}" + deploy.type = 'DeployRsync' + + deploy.save + end + end + + private + + def nodes + @nodes ||= Rails.application.nodes.map do |node| + "sutty@#{node}" + end + end +end