From accb559f014ccc287ffcc5e6f15ce5ddae918a53 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 16:39:56 -0300 Subject: [PATCH 01/19] hacer deploys remotos --- app/models/deploy_rsync.rb | 88 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 app/models/deploy_rsync.rb diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb new file mode 100644 index 00000000..40018dbd --- /dev/null +++ b/app/models/deploy_rsync.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +# Sincroniza sitios a servidores remotos usando Rsync. El servidor +# remoto tiene que tener rsync instalado. +class DeployRsync < Deploy + store :values, accessors: %i[flags destination host_keys], coder: JSON + + def deploy + ssh? && rsync + end + + # No se ocupa espacio local + # + # @return [Integer] + def size + 0 + end + + private + + # Verificar la conexión SSH implementando Trust On First Use + # + # TODO: Medir el tiempo que tarda en iniciarse la conexión + # + # @return [Boolean] + def ssh? + user, host = user_host + ssh_available = false + + Net::SSH.start(host, user, verify_host_key: tofu) do |ssh| + if values[:host_keys].blank? + # Guardar las llaves que se encontraron en la primera conexión + values[:host_keys] = ssh.transport.host_keys.map do |host_key| + "#{host_key.ssh_type} #{host_key.fingerprint}" + end + + ssh_available = save + else + ssh_available = true + end + end + + ssh_available + rescue Exception => e + ExceptionNotifier.notify_exception(e, data: { site: site.id, hostname: host, user: user }) + + false + end + + # Confiar en la primera llave que encontremos, fallar si cambian + # + # @return [Symbol] + def tofu + values[:host_keys].present? ? :always : :accept_new + end + + # Devuelve el par user host + # + # @return [Array] + def user_host + destination.split(':', 2).first.split('@', 2).tap do |d| + next unless d.size == 1 + + d.insert(0, nil) + end + end + + # Sincroniza hacia el directorio remoto, usando las flags opcionales. + # + # @return [Boolean] + def rsync + run %(rsync -av #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + end + + # El origen es el destino de la compilación + # + # @return [String] + def source + site.deploys.find_by(type: 'DeployLocal').destination + end + + # Devolver el destino o lanzar un error si no está configurado + def destination + values[:destination].tap do |_d| + raise ArgumentError, 'destination no está configurado' + end + end +end From f10b65173a9542500e3d206a2167fb3524b59ed0 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 17:05:51 -0300 Subject: [PATCH 02/19] fixup! hacer deploys remotos --- app/models/deploy_rsync.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 40018dbd..9e855f57 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -81,8 +81,8 @@ class DeployRsync < Deploy # Devolver el destino o lanzar un error si no está configurado def destination - values[:destination].tap do |_d| - raise ArgumentError, 'destination no está configurado' + values[:destination].tap do |d| + raise(ArgumentError, 'destination no está configurado') if d.blank? end end end From 6fcdeb52f381bbde75b5156671f228016209f6ea Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 17:23:42 -0300 Subject: [PATCH 03/19] entorno --- app/models/deploy_rsync.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 9e855f57..4191b20d 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -47,6 +47,14 @@ class DeployRsync < Deploy false end + def env + { + 'HOME' => home_dir, + 'PATH' => '/usr/bin', + 'LANG' => ENV['LANG'] + } + end + # Confiar en la primera llave que encontremos, fallar si cambian # # @return [Symbol] From ea198f185b655a353755b097cdbc6c264378db34 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:07:45 -0300 Subject: [PATCH 04/19] bajar el timeout para no bloquear el deploy --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 4191b20d..2387366a 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -27,7 +27,7 @@ class DeployRsync < Deploy user, host = user_host ssh_available = false - Net::SSH.start(host, user, verify_host_key: tofu) do |ssh| + Net::SSH.start(host, user, verify_host_key: tofu, timeout: 5) do |ssh| if values[:host_keys].blank? # Guardar las llaves que se encontraron en la primera conexión values[:host_keys] = ssh.transport.host_keys.map do |host_key| From c49c63f77661413a208a47bd7a80f7b4b8d49535 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:11:21 -0300 Subject: [PATCH 05/19] timeout de rsync --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 2387366a..5d360c4c 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -77,7 +77,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -av #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -av --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From 8edc9b460af993ad4c52b1192962cb548c39a82a Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:17:55 -0300 Subject: [PATCH 06/19] obtener listado de nodos de un lugar central --- app/models/sutty.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 app/models/sutty.rb diff --git a/app/models/sutty.rb b/app/models/sutty.rb new file mode 100644 index 00000000..7ec8432c --- /dev/null +++ b/app/models/sutty.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Configuración general de Sutty +class Sutty + extend self + + # Los nodos son otros servidores de Sutty hacia los que se sincronizan + # sitios. + # + # @return [Array] + def nodes + @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') + end +end From dcbd1c02ac32c1b183d6c90255da693ad67bbc9e Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:26:41 -0300 Subject: [PATCH 07/19] =?UTF-8?q?el=20m=C3=A9todo=20es=20p=C3=BAblico?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_rsync.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 5d360c4c..50719cd1 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -16,6 +16,13 @@ class DeployRsync < Deploy 0 end + # Devolver el destino o lanzar un error si no está configurado + def destination + values[:destination].tap do |d| + raise(ArgumentError, 'destination no está configurado') if d.blank? + end + end + private # Verificar la conexión SSH implementando Trust On First Use @@ -86,11 +93,4 @@ class DeployRsync < Deploy def source site.deploys.find_by(type: 'DeployLocal').destination end - - # Devolver el destino o lanzar un error si no está configurado - def destination - values[:destination].tap do |d| - raise(ArgumentError, 'destination no está configurado') if d.blank? - end - end end From 5fac827ca09fd46231490c59f845b6ec6a2e6121 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:27:15 -0300 Subject: [PATCH 08/19] registrar los cambios --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 50719cd1..7733406e 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -84,7 +84,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -av --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -avi --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From 20915443d84d7ddb50ef8417d957d2a2dbb2dbcd Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:27:52 -0300 Subject: [PATCH 09/19] deprecar las flags por ahora --- app/models/deploy_rsync.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 7733406e..12f4ce0e 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -3,7 +3,7 @@ # Sincroniza sitios a servidores remotos usando Rsync. El servidor # remoto tiene que tener rsync instalado. class DeployRsync < Deploy - store :values, accessors: %i[flags destination host_keys], coder: JSON + store :values, accessors: %i[destination host_keys], coder: JSON def deploy ssh? && rsync @@ -80,11 +80,11 @@ class DeployRsync < Deploy end end - # Sincroniza hacia el directorio remoto, usando las flags opcionales. + # Sincroniza hacia el directorio remoto # # @return [Boolean] def rsync - run %(rsync -avi --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -avi --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From a5e90257a30566ecd00fc36a63bf8fcb5e789528 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:38:03 -0300 Subject: [PATCH 10/19] generar un deploy rsync para el sitio con todos los nodos --- app/services/site_service.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 5e2fc706..2ffb8cdd 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -144,4 +144,11 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do PostService.new(site: site, usuarie: usuarie, post: post, params: params).update end + + # Crea los deploys necesarios para sincronizar a otros nodos de Sutty + def sync_nodes + Sutty.nodes.each do |node| + site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") + end + end end From 10d950689b614f6ca2a99ace3cf14dce44b3a037 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:39:52 -0300 Subject: [PATCH 11/19] crear rsyncs al crear el sitio --- app/services/site_service.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 2ffb8cdd..89c5796d 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -9,6 +9,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do self.site = Site.new params add_role temporal: false, rol: 'usuarie' + sync_nodes I18n.with_locale(usuarie&.lang&.to_sym || I18n.default_locale) do site.save && From a0134cc052578e2ec66025b3d51b2879fc144217 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 19:05:50 -0300 Subject: [PATCH 12/19] =?UTF-8?q?ya=20existe=20un=20lugar=20donde=20guarda?= =?UTF-8?q?r=20la=20configuraci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/sutty.rb | 14 -------------- app/services/site_service.rb | 2 +- config/application.rb | 4 ++++ 3 files changed, 5 insertions(+), 15 deletions(-) delete mode 100644 app/models/sutty.rb diff --git a/app/models/sutty.rb b/app/models/sutty.rb deleted file mode 100644 index 7ec8432c..00000000 --- a/app/models/sutty.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -# Configuración general de Sutty -class Sutty - extend self - - # Los nodos son otros servidores de Sutty hacia los que se sincronizan - # sitios. - # - # @return [Array] - def nodes - @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') - end -end diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 89c5796d..22423bb8 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -148,7 +148,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do # Crea los deploys necesarios para sincronizar a otros nodos de Sutty def sync_nodes - Sutty.nodes.each do |node| + Rails.application.nodes.each do |node| site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") end end diff --git a/config/application.rb b/config/application.rb index 7326ae0f..bc948936 100644 --- a/config/application.rb +++ b/config/application.rb @@ -49,5 +49,9 @@ module Sutty EmailAddress::Config.error_messages translations.transform_keys(&:to_s), locale.to_s end end + + def nodes + @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') + end end end From aa0c359554917ef93547d3446e55ac73394587a8 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 19:06:11 -0300 Subject: [PATCH 13/19] crear un rsync por cada sitio existente --- .../20220406211042_add_deploy_rsync_to_sites.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 db/migrate/20220406211042_add_deploy_rsync_to_sites.rb diff --git a/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb b/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb new file mode 100644 index 00000000..92b6f17b --- /dev/null +++ b/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# Agrega un DeployRsync hacia los servidores alternativos para cada +# sitio +class AddDeployRsyncToSites < ActiveRecord::Migration[6.1] + def up + Site.find_each do |site| + SiteService.new(site: site).send :sync_nodes + site.save + end + end + + def down + DeployRsync.destroy_all + end +end From f51c051af81da89345e66f137be419b00800ebe5 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 19:35:35 -0300 Subject: [PATCH 14/19] mensajes --- config/locales/en.yml | 4 ++++ config/locales/es.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/config/locales/en.yml b/config/locales/en.yml index b814796d..647f3ee8 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -100,6 +100,10 @@ en: title: Alternative domain name success: Success! error: Error + deploy_rsync: + title: Synchronize to backup server + success: Success! + error: Error help: You can contact us by replying to this e-mail maintenance_mailer: notice: diff --git a/config/locales/es.yml b/config/locales/es.yml index a6fbd407..1ccf5047 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -100,6 +100,10 @@ es: title: Dominio alternativo success: ¡Éxito! error: Hubo un error + deploy_rsync: + title: Sincronizar al servidor alternativo + success: ¡Éxito! + error: Hubo un error help: Por cualquier duda, responde este correo para contactarte con nosotres. maintenance_mailer: notice: From 8c9bd6aa88c9c584896c9109d4c04aad5d9b1cfd Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:31:49 -0300 Subject: [PATCH 15/19] por ahora no es configurable --- app/views/deploys/_deploy_rsync.haml | 1 + 1 file changed, 1 insertion(+) create mode 100644 app/views/deploys/_deploy_rsync.haml diff --git a/app/views/deploys/_deploy_rsync.haml b/app/views/deploys/_deploy_rsync.haml new file mode 100644 index 00000000..0aab9802 --- /dev/null +++ b/app/views/deploys/_deploy_rsync.haml @@ -0,0 +1 @@ +-# nada From 35e41b729ad35e86b1b4d4ce63ba2d4dc0ad97d6 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:44:46 -0300 Subject: [PATCH 16/19] el espacio ocupado es el mismo que el local --- app/models/deploy_rsync.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 12f4ce0e..c19d279b 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -9,11 +9,11 @@ class DeployRsync < Deploy ssh? && rsync end - # No se ocupa espacio local + # El espacio remoto es el mismo que el local # # @return [Integer] def size - 0 + deploy_local.build_stats.last.size end # Devolver el destino o lanzar un error si no está configurado @@ -91,6 +91,10 @@ class DeployRsync < Deploy # # @return [String] def source - site.deploys.find_by(type: 'DeployLocal').destination + deploy_local.destination + end + + def deploy_local + @deploy_local ||= site.deploys.find_by(type: 'DeployLocal') end end From ea25d27f240cacad4fac4a1bf97455aa03c4c8f5 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:53:45 -0300 Subject: [PATCH 17/19] =?UTF-8?q?calcular=20el=20tama=C3=B1o=20a=20partir?= =?UTF-8?q?=20de=20deploy=20local?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index c19d279b..b2ffba7c 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -13,7 +13,7 @@ class DeployRsync < Deploy # # @return [Integer] def size - deploy_local.build_stats.last.size + deploy_local.size end # Devolver el destino o lanzar un error si no está configurado From bb7c50e24b4fb9a9ce16666d53a77fc666a22506 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:54:35 -0300 Subject: [PATCH 18/19] calcular el espacio una sola vez --- app/models/deploy_local.rb | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 4fa588f5..1b661059 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -28,15 +28,17 @@ class DeployLocal < Deploy # Obtener el tamaño de todos los archivos y directorios (los # directorios son archivos :) def size - paths = [destination, File.join(destination, '**', '**')] + @size ||= begin + paths = [destination, File.join(destination, '**', '**')] - Dir.glob(paths).map do |file| - if File.symlink? file - 0 - else - File.size(file) - end - end.inject(:+) + Dir.glob(paths).map do |file| + if File.symlink? file + 0 + else + File.size(file) + end + end.inject(:+) + end end def destination From dd34d176ea53cd24b157ce3aaf5da238f9c47217 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 15 Jun 2022 13:34:46 -0300 Subject: [PATCH 19/19] sincronizar links duros --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index b2ffba7c..996f8cdd 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -84,7 +84,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -avi --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -aviH --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación