From 4f8ea0d9a726b860edce4227939e7005c6b1107e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 18:38:47 -0300 Subject: [PATCH 01/77] feat: los deploys son interdependientes --- app/models/deploy.rb | 7 +++---- app/models/deploy_alternative_domain.rb | 2 ++ app/models/deploy_rsync.rb | 8 ++++++++ app/models/deploy_www.rb | 2 ++ app/models/deploy_zip.rb | 2 ++ 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 9d5c1d27..942336ad 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'open3' + # Este modelo implementa los distintos tipos de alojamiento que provee # Sutty. # @@ -11,11 +12,9 @@ class Deploy < ApplicationRecord belongs_to :site has_many :build_stats, dependent: :destroy - def deploy(**) - raise NotImplementedError - end + DEPENDENCIES = [] - def url + def deploy(**) raise NotImplementedError end diff --git a/app/models/deploy_alternative_domain.rb b/app/models/deploy_alternative_domain.rb index 5ad381d5..114f2aa5 100644 --- a/app/models/deploy_alternative_domain.rb +++ b/app/models/deploy_alternative_domain.rb @@ -4,6 +4,8 @@ class DeployAlternativeDomain < Deploy store :values, accessors: %i[hostname], coder: JSON + DEPENDENCIES = %i[local] + # Generar un link simbólico del sitio principal al alternativo def deploy(**) File.symlink?(destination) || diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index a658de6b..9ced0bb1 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -5,6 +5,14 @@ class DeployRsync < Deploy store :values, accessors: %i[destination host_keys], coder: JSON + DEPENDENCIES = %i[ + alternative_domain + hidden_service + local + www + zip + ] + def deploy(output: false) ssh? && rsync(output: output) end diff --git a/app/models/deploy_www.rb b/app/models/deploy_www.rb index dff769a6..e6cfe407 100644 --- a/app/models/deploy_www.rb +++ b/app/models/deploy_www.rb @@ -4,6 +4,8 @@ class DeployWww < Deploy store :values, accessors: %i[], coder: JSON + DEPENDENCIES = %i[local] + before_destroy :remove_destination! def deploy(**) diff --git a/app/models/deploy_zip.rb b/app/models/deploy_zip.rb index f1c94083..2f9ac173 100644 --- a/app/models/deploy_zip.rb +++ b/app/models/deploy_zip.rb @@ -8,6 +8,8 @@ require 'zip' class DeployZip < Deploy store :values, accessors: %i[], coder: JSON + DEPENDENCIES = %i[local] + # Una vez que el sitio está generado, tomar todos los archivos y # y generar un zip accesible públicamente. # From db25750ab0580f92d93e5194e4ba7d14ca883ddf Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 18:54:38 -0300 Subject: [PATCH 02/77] feat: usar un grafo dirigido para ordenar las dependencias #10464 --- Gemfile | 1 + Gemfile.lock | 7 ++++++ app/models/site/deploy_dependencies.rb | 30 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 app/models/site/deploy_dependencies.rb diff --git a/Gemfile b/Gemfile index f3799638..489097a6 100644 --- a/Gemfile +++ b/Gemfile @@ -23,6 +23,7 @@ if ENV['RAILS_GROUPS']&.split(',')&.include? 'assets' end gem 'nokogiri' +gem 'rgl' # Turbolinks makes navigating your web application faster. Read more: # https://github.com/turbolinks/turbolinks diff --git a/Gemfile.lock b/Gemfile.lock index abaf45c9..5df84963 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -353,6 +353,7 @@ GEM mini_portile2 (~> 2.6.1) racc (~> 1.4) orm_adapter (0.5.0) + pairing_heap (3.0.0) parallel (1.21.0) parser (3.0.2.0) ast (~> 2.4.1) @@ -443,6 +444,10 @@ GEM actionpack (>= 5.0) railties (>= 5.0) rexml (3.2.5) + rgl (0.6.2) + pairing_heap (>= 0.3.0) + rexml (~> 3.2, >= 3.2.4) + stream (~> 0.5.3) rouge (3.26.1) rubocop (1.23.0) parallel (~> 1.10) @@ -510,6 +515,7 @@ GEM sprockets (>= 3.0.0) sqlite3 (1.4.2-x86_64-linux-musl) stackprof (0.2.17-x86_64-linux-musl) + stream (0.5.5) sucker_punch (3.0.1) concurrent-ruby (~> 1.0) sutty-archives (2.5.4) @@ -626,6 +632,7 @@ DEPENDENCIES rails_warden redis redis-rails + rgl rollups! rubocop-rails rubyzip diff --git a/app/models/site/deploy_dependencies.rb b/app/models/site/deploy_dependencies.rb new file mode 100644 index 00000000..33ed9e37 --- /dev/null +++ b/app/models/site/deploy_dependencies.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'rgl/adjacency' + +class Site + module DeployDependencies + extend ActiveSupport::Concern + + included do + # Genera un grafo dirigido de todos los métodos de publicación + # + # @return [RGL::DirectedAdjacencyGraph] + def deployment_graph + @deployment_graph ||= RGL::DirectedAdjacencyGraph.new.tap do |graph| + deploys.each do |deploy| + graph.add_vertex deploy + end + + deploys.each do |deploy| + deploy.class::DEPENDENCIES.each do |dependency| + deploys.where(type: "Deploy#{dependency.to_s.classify}").each do |deploy_dependency| + graph.add_edge deploy_dependency, deploy + end + end + end + end + end + end + end +end From 0ed01a61f7e85c8ac2580ccf4bb526edb905262c Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 21:37:58 -0300 Subject: [PATCH 03/77] feat: devolver una lista ordenada de metodos de publicacion --- app/models/site/deploy_dependencies.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models/site/deploy_dependencies.rb b/app/models/site/deploy_dependencies.rb index 33ed9e37..05939320 100644 --- a/app/models/site/deploy_dependencies.rb +++ b/app/models/site/deploy_dependencies.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'rgl/adjacency' +require 'rgl/topsort' class Site module DeployDependencies @@ -25,6 +26,13 @@ class Site end end end + + # Devuelve una lista ordenada de todos los métodos de publicación + # + # @return [Array] + def deployment_list + @deployment_list ||= deployment_graph.topsort_iterator.to_a + end end end end From 607f19f6805512383358b80eea64197d5a20aafe Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 21:38:31 -0300 Subject: [PATCH 04/77] feat: activar ordenamiento de metodos de publicacion --- app/models/site.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/site.rb b/app/models/site.rb index 4b2a8eb9..f88e2e97 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -7,6 +7,7 @@ class Site < ApplicationRecord include Site::Forms include Site::FindAndReplace include Site::Api + include Site::DeployDependencies include Tienda # Cifrar la llave privada que cifra y decifra campos ocultos. Sutty From e5f96c435d8efec81f1a3cee4f5f657c3a32d7d3 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 13:42:47 -0300 Subject: [PATCH 05/77] feat: generar el sitio en orden #10464 --- app/jobs/deploy_job.rb | 86 +++++++++++-------------- app/models/deploy_alternative_domain.rb | 2 +- app/models/deploy_rsync.rb | 10 +-- app/models/deploy_www.rb | 2 +- app/models/deploy_zip.rb | 2 +- app/models/site/deploy_dependencies.rb | 2 +- 6 files changed, 48 insertions(+), 56 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index e018533c..f7d07349 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -30,28 +30,30 @@ class DeployJob < ApplicationJob return end + @deployed = {} @site.update status: 'building' - # Asegurarse que DeployLocal sea el primero! - @deployed = { - deploy_local: { - status: deploy_locally, - seconds: deploy_local.build_stats.last.seconds, - size: deploy_local.size, - urls: [deploy_local.url] + @site.deployment_list.each do |d| + begin + raise DeployException, 'Una dependencia falló' if failed_dependencies? d + + status = d.deploy + seconds = d.build_stats.last.try(:seconds) + rescue StandardError => e + status = false + seconds = 0 + + notify_exception e, d + end + + @deployed[d.type.underscore.to_sym] = { + status: status, + seconds: seconds || 0, + size: d.size, + urls: d.respond_to?(:urls) ? d.urls : [d.url].compact } - } - - # No es opcional - unless @deployed[:deploy_local][:status] - # Hacer fallar la tarea - raise DeployException, "#{@site.name}: Falló la compilación" end - - deploy_others rescue DeployTimedOutException => e notify_exception e - rescue DeployException => e - notify_exception e, deploy_local ensure @site&.update status: 'waiting' @@ -62,47 +64,37 @@ class DeployJob < ApplicationJob private + # Detecta si un método de publicación tiene dependencias fallidas + # + # @param :deploy [Deploy] + # @return [Boolean] + def failed_dependencies?(deploy) + failed_dependencies(deploy).present? + end + + # Obtiene las dependencias fallidas de un deploy + # + # @param :deploy [Deploy] + # @return [Array] + def failed_dependencies(deploy) + deploy.class::DEPENDENCIES & (@deployed.reject do |_, v| + v[:status] + end.keys) + end + # @param :exception [StandardError] # @param :deploy [Deploy] def notify_exception(exception, deploy = nil) data = { site: @site.id, deploy: deploy&.type, - log: deploy&.build_stats&.last&.log + log: deploy&.build_stats&.last&.log, + failed_dependencies: (failed_dependencies(deploy) if deploy) } ExceptionNotifier.notify_exception(exception, data: data) end - def deploy_local - @deploy_local ||= @site.deploys.find_by(type: 'DeployLocal') - end - - def deploy_locally - deploy_local.deploy(output: @output) - end - - def deploy_others - @site.deploys.where.not(type: 'DeployLocal').find_each do |d| - begin - status = d.deploy(output: @output) - seconds = d.build_stats.last.try(:seconds) - rescue StandardError => e - status = false - seconds = 0 - - notify_exception e, d - end - - @deployed[d.type.underscore.to_sym] = { - status: status, - seconds: seconds || 0, - size: d.size, - urls: d.respond_to?(:urls) ? d.urls : [d.url].compact - } - end - end - def notify_usuaries @site.roles.where(rol: 'usuarie', temporal: false).pluck(:usuarie_id).each do |usuarie| DeployMailer.with(usuarie: usuarie, site: @site.id) diff --git a/app/models/deploy_alternative_domain.rb b/app/models/deploy_alternative_domain.rb index 114f2aa5..9b1d63d7 100644 --- a/app/models/deploy_alternative_domain.rb +++ b/app/models/deploy_alternative_domain.rb @@ -4,7 +4,7 @@ class DeployAlternativeDomain < Deploy store :values, accessors: %i[hostname], coder: JSON - DEPENDENCIES = %i[local] + DEPENDENCIES = %i[deploy_local] # Generar un link simbólico del sitio principal al alternativo def deploy(**) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 9ced0bb1..dd7a492b 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -6,11 +6,11 @@ class DeployRsync < Deploy store :values, accessors: %i[destination host_keys], coder: JSON DEPENDENCIES = %i[ - alternative_domain - hidden_service - local - www - zip + deploy_alternative_domain + deploy_hidden_service + deploy_local + deploy_www + deploy_zip ] def deploy(output: false) diff --git a/app/models/deploy_www.rb b/app/models/deploy_www.rb index e6cfe407..d27f77cc 100644 --- a/app/models/deploy_www.rb +++ b/app/models/deploy_www.rb @@ -4,7 +4,7 @@ class DeployWww < Deploy store :values, accessors: %i[], coder: JSON - DEPENDENCIES = %i[local] + DEPENDENCIES = %i[deploy_local] before_destroy :remove_destination! diff --git a/app/models/deploy_zip.rb b/app/models/deploy_zip.rb index 2f9ac173..bba13686 100644 --- a/app/models/deploy_zip.rb +++ b/app/models/deploy_zip.rb @@ -8,7 +8,7 @@ require 'zip' class DeployZip < Deploy store :values, accessors: %i[], coder: JSON - DEPENDENCIES = %i[local] + DEPENDENCIES = %i[deploy_local] # Una vez que el sitio está generado, tomar todos los archivos y # y generar un zip accesible públicamente. diff --git a/app/models/site/deploy_dependencies.rb b/app/models/site/deploy_dependencies.rb index 05939320..a01f99e7 100644 --- a/app/models/site/deploy_dependencies.rb +++ b/app/models/site/deploy_dependencies.rb @@ -19,7 +19,7 @@ class Site deploys.each do |deploy| deploy.class::DEPENDENCIES.each do |dependency| - deploys.where(type: "Deploy#{dependency.to_s.classify}").each do |deploy_dependency| + deploys.where(type: dependency.to_s.classify).each do |deploy_dependency| graph.add_edge deploy_dependency, deploy end end From 3de6f14e63a7922e6670b45a87d4e079830e7244 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 13:47:34 -0300 Subject: [PATCH 06/77] =?UTF-8?q?fix:=20no=20pedir=20el=20tama=C3=B1o=20si?= =?UTF-8?q?=20fall=C3=B3=20la=20compilaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/deploy_job.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index f7d07349..fe344454 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -38,9 +38,11 @@ class DeployJob < ApplicationJob status = d.deploy seconds = d.build_stats.last.try(:seconds) + size = d.size rescue StandardError => e status = false seconds = 0 + size = 0 notify_exception e, d end @@ -48,7 +50,7 @@ class DeployJob < ApplicationJob @deployed[d.type.underscore.to_sym] = { status: status, seconds: seconds || 0, - size: d.size, + size: size, urls: d.respond_to?(:urls) ? d.urls : [d.url].compact } end From ea1ac09f9698be5cc615a4471a3d77b1bdbe9ca2 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 13:54:19 -0300 Subject: [PATCH 07/77] =?UTF-8?q?fix:=20mejorar=20gesti=C3=B3n=20de=20zip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_zip.rb | 53 +++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/app/models/deploy_zip.rb b/app/models/deploy_zip.rb index bba13686..5f72a728 100644 --- a/app/models/deploy_zip.rb +++ b/app/models/deploy_zip.rb @@ -14,24 +14,43 @@ class DeployZip < Deploy # y generar un zip accesible públicamente. # # rubocop:disable Metrics/MethodLength - def deploy(**) + def deploy(output: false) FileUtils.rm_f path - time_start - Dir.chdir(destination) do - Zip::File.open(path, Zip::File::CREATE) do |z| - Dir.glob('./**/**').each do |f| - File.directory?(f) ? z.mkdir(f) : z.add(f, f) + Zip::File.open(path, Zip::File::CREATE) do |zip| + Dir.glob(File.join(destination, '**', '**')).each do |file| + entry = Pathname.new(file).relative_path_from(destination).to_s + + if File.directory? file + log "Creando directorio #{entry}", output + + zip.mkdir(entry) + else + log "Comprimiendo #{entry}", output + zip.add(entry, file) end end end + time_stop - build_stats.create action: 'zip', - seconds: time_spent_in_seconds, - bytes: size + File.exist?(path).tap do |status| + build_stats.create action: 'zip', + seconds: time_spent_in_seconds, + bytes: size, + log: @log.join("\n"), + status: status + end + rescue Zip::Error => e + ExceptionNotifier.notify_exception(e, data: { site: site.name }) - File.exist? path + build_stats.create action: 'zip', + seconds: 0, + bytes: 0, + log: @log.join("\n"), + status: false + + false end # rubocop:enable Metrics/MethodLength @@ -43,8 +62,9 @@ class DeployZip < Deploy File.size path end + # @return [String] def destination - File.join(Rails.root, '_deploy', site.hostname) + Rails.root.join('_deploy', site.hostname).realpath.to_s end def file @@ -58,4 +78,15 @@ class DeployZip < Deploy def path File.join(destination, file) end + + private + + # @param :line [String] + # @param :output [Boolean] + def log(line, output) + @log ||= [] + @log << line + + puts line if output + end end From 199199117da9c70b666d851ebff29ea59f5c7ec0 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 16 Mar 2022 18:19:04 -0300 Subject: [PATCH 08/77] implementar la misma api en todos los deploys --- app/models/deploy.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 942336ad..7aedbb19 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -18,6 +18,10 @@ class Deploy < ApplicationRecord raise NotImplementedError end + def url + raise NotImplementedError + end + def limit raise NotImplementedError end From 875f1873d374d14b8fafbc5fb452afbf6ede9b3b Mon Sep 17 00:00:00 2001 From: f Date: Mon, 18 Apr 2022 17:27:43 -0300 Subject: [PATCH 09/77] no fallar al cerrarse la salida --- app/models/deploy.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 7aedbb19..922c7560 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -74,8 +74,7 @@ class Deploy < ApplicationRecord puts line if output end rescue IOError => e - lines << e.message - puts e.message if output + ExceptionNotifier.notify(e, data: { site: site.name }) end r = t.value From e8746ca385805b288f291866145ad8c005d7f4f0 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 28 Jun 2022 15:07:08 -0300 Subject: [PATCH 10/77] =?UTF-8?q?notificar=20correctamente=20la=20excepci?= =?UTF-8?q?=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 922c7560..e02602f8 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -74,7 +74,7 @@ class Deploy < ApplicationRecord puts line if output end rescue IOError => e - ExceptionNotifier.notify(e, data: { site: site.name }) + ExceptionNotifier.notify_exception(e, data: { site: site.name }) end r = t.value From 92b97b508b9414384adc2df6e697e7d537ef8d18 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 29 Jun 2022 18:41:51 -0300 Subject: [PATCH 11/77] no generar issues para log --- app/models/deploy.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index e02602f8..7aedbb19 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -74,7 +74,8 @@ class Deploy < ApplicationRecord puts line if output end rescue IOError => e - ExceptionNotifier.notify_exception(e, data: { site: site.name }) + lines << e.message + puts e.message if output end r = t.value From 4ebd9881f558bf443ae58cb566952b0bd91f792e Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 14:31:39 -0300 Subject: [PATCH 12/77] =?UTF-8?q?fix:=20mejorar=20gesti=C3=B3n=20de=20syml?= =?UTF-8?q?ink?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_www.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/deploy_www.rb b/app/models/deploy_www.rb index d27f77cc..bb25cc64 100644 --- a/app/models/deploy_www.rb +++ b/app/models/deploy_www.rb @@ -8,7 +8,9 @@ class DeployWww < Deploy before_destroy :remove_destination! - def deploy(**) + def deploy(output: false) + puts "Creando symlink #{site.hostname} => #{destination}" if output + File.symlink?(destination) || File.symlink(site.hostname, destination).zero? end @@ -30,7 +32,7 @@ class DeployWww < Deploy end def url - "https://www.#{site.hostname}/" + "https://#{fqdn}/" end private From 1e188bab0297ea1a21004239a59e7d118807ec69 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 14:37:52 -0300 Subject: [PATCH 13/77] =?UTF-8?q?fix:=20rsync=20no=20deber=C3=ADa=20depend?= =?UTF-8?q?er=20de=20zip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aunque el zip no se genere igual queremos poder sincronizar el sitio --- app/models/deploy_rsync.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index dd7a492b..04c33cb8 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -10,7 +10,6 @@ class DeployRsync < Deploy deploy_hidden_service deploy_local deploy_www - deploy_zip ] def deploy(output: false) From 431b7223b6b4c428a6edde72d74f5ab8d030abd7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 14:38:45 -0300 Subject: [PATCH 14/77] feat: eliminar archivos al sincronizar --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 04c33cb8..8702449a 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -91,7 +91,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync(output: output) - run %(rsync -aviH --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/), output: output + run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From f90ca9cd12d1e6019272393864c5ccbc7e203229 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 14:45:35 -0300 Subject: [PATCH 15/77] fix: output para rsync --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 8702449a..51d7b2d2 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -91,7 +91,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync(output: output) - run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/), output: output end # El origen es el destino de la compilación From c6f48f55fcaa602c960c602b70d503bf6e7969d1 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 14:59:34 -0300 Subject: [PATCH 16/77] feat: mostrar una tabla con el resumen en la consola --- app/jobs/deploy_job.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index fe344454..e7ce7ed0 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -54,6 +54,16 @@ class DeployJob < ApplicationJob urls: d.respond_to?(:urls) ? d.urls : [d.url].compact } end + + return unless @output + + puts (Terminal::Table.new do |t| + t << (%w[type] + @deployed.values.first.keys) + t.add_separator + @deployed.each do |type, row| + t << ([type.to_s] + row.values) + end + end) rescue DeployTimedOutException => e notify_exception e ensure From 58dffd00c9111ef2686264ae8f1d867a6fb46456 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 15:10:32 -0300 Subject: [PATCH 17/77] =?UTF-8?q?fix:=20fallar=20el=20hidden=20service=20s?= =?UTF-8?q?i=20no=20se=20gener=C3=B3=20el=20onion=20aun?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_hidden_service.rb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/app/models/deploy_hidden_service.rb b/app/models/deploy_hidden_service.rb index dc9549f5..79ff1bae 100644 --- a/app/models/deploy_hidden_service.rb +++ b/app/models/deploy_hidden_service.rb @@ -2,14 +2,10 @@ # Genera una versión onion class DeployHiddenService < DeployWww - def deploy(**) - return true if fqdn.blank? - - super - end - def fqdn - values[:onion] + values[:onion].tap do |onion| + raise ArgumentError, 'Aun no se generó la dirección .onion' if onion.blank? + end end def url From d4f4627dab695c73d58c63deb19def3cff5c2d5c Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 15:22:10 -0300 Subject: [PATCH 18/77] fix: capturar todas las posibles excepciones --- app/jobs/deploy_job.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index e7ce7ed0..67548e07 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -39,19 +39,21 @@ class DeployJob < ApplicationJob status = d.deploy seconds = d.build_stats.last.try(:seconds) size = d.size + urls = d.respond_to?(:urls) ? d.urls : [d.url].compact rescue StandardError => e status = false - seconds = 0 - size = 0 + seconds ||= 0 + size ||= 0 + urls ||= [] notify_exception e, d end @deployed[d.type.underscore.to_sym] = { status: status, - seconds: seconds || 0, + seconds: seconds, size: size, - urls: d.respond_to?(:urls) ? d.urls : [d.url].compact + urls: urls } end From 5974a40d027e1f7c6de11585591f7190918efa56 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 15:23:13 -0300 Subject: [PATCH 19/77] feat: hacer sonar una campana --- app/jobs/deploy_job.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 67548e07..3c9b2740 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -66,6 +66,8 @@ class DeployJob < ApplicationJob t << ([type.to_s] + row.values) end end) + + puts "\a" rescue DeployTimedOutException => e notify_exception e ensure From ba91ed56aa89546f494c14cd9dec8bcb1fe59ae9 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 15:50:05 -0300 Subject: [PATCH 20/77] feat: separar rsync de full rsync los segundos se usan para sincronizar todas las versiones de un sitio con otro servidor de sutty. los primeros solo sincronizan los archivos a otro servidor, no necesariamente bajo el mismo nombre. --- app/models/deploy_alternative_domain.rb | 6 ++- app/models/deploy_full_rsync.rb | 22 +++++++++++ app/models/deploy_rsync.rb | 9 +---- app/services/site_service.rb | 2 +- ...ename_deploy_rsync_to_deploy_full_rsync.rb | 37 +++++++++++++++++++ 5 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 app/models/deploy_full_rsync.rb create mode 100644 db/migrate/20230318183722_rename_deploy_rsync_to_deploy_full_rsync.rb diff --git a/app/models/deploy_alternative_domain.rb b/app/models/deploy_alternative_domain.rb index 9b1d63d7..75b69180 100644 --- a/app/models/deploy_alternative_domain.rb +++ b/app/models/deploy_alternative_domain.rb @@ -20,7 +20,11 @@ class DeployAlternativeDomain < Deploy end def destination - @destination ||= File.join(Rails.root, '_deploy', hostname.gsub(/\.\z/, '')) + @destination ||= File.join(Rails.root, '_deploy', fqdn) + end + + def fqdn + hostname.gsub(/\.\z/, '') end def url diff --git a/app/models/deploy_full_rsync.rb b/app/models/deploy_full_rsync.rb new file mode 100644 index 00000000..c0ff84c6 --- /dev/null +++ b/app/models/deploy_full_rsync.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class DeployFullRsync < DeployRsync + DEPENDENCIES = %i[ + deploy_alternative_domain + deploy_hidden_service + deploy_local + deploy_www + ] + + # Sincroniza las ubicaciones alternativas también + # + # @param :output [Boolean] + # @return [Boolean] + def rsync(output: false) + DEPENDENCIES.map(&:to_s).map(&:classify).map do |dependency| + site.deploys.where(type: dependency).find_each.map do |deploy| + run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape deploy.destination} #{Shellwords.escape destination}), output: output + end + end.flatten.all? + end +end diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 51d7b2d2..6a96a274 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -5,12 +5,7 @@ class DeployRsync < Deploy store :values, accessors: %i[destination host_keys], coder: JSON - DEPENDENCIES = %i[ - deploy_alternative_domain - deploy_hidden_service - deploy_local - deploy_www - ] + DEPENDENCIES = %i[deploy_local] def deploy(output: false) ssh? && rsync(output: output) @@ -90,7 +85,7 @@ class DeployRsync < Deploy # Sincroniza hacia el directorio remoto # # @return [Boolean] - def rsync(output: output) + def rsync(output: false) run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/), output: output end diff --git a/app/services/site_service.rb b/app/services/site_service.rb index f5f415e7..0ecccba4 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -163,7 +163,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do # Crea los deploys necesarios para sincronizar a otros nodos de Sutty def sync_nodes Rails.application.nodes.each do |node| - site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") + site.deploys.build(type: 'DeployFullRsync', destination: "sutty@#{node}:") end end end diff --git a/db/migrate/20230318183722_rename_deploy_rsync_to_deploy_full_rsync.rb b/db/migrate/20230318183722_rename_deploy_rsync_to_deploy_full_rsync.rb new file mode 100644 index 00000000..689dc559 --- /dev/null +++ b/db/migrate/20230318183722_rename_deploy_rsync_to_deploy_full_rsync.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# Cambia todos los DeployRsync propios de Sutty a DeployFullRsync que se +# encarga de sincronizar todo. +class RenameDeployRsyncToDeployFullRsync < ActiveRecord::Migration[6.1] + def up + DeployRsync.all.find_each do |deploy| + dest = deploy.destination.split(':', 2).first + + next unless nodes.include? dest + + deploy.destination = "#{dest}:" + deploy.type = 'DeployFullRsync' + + deploy.save + end + end + + def down + DeployFullRsync.all.find_each do |deploy| + next unless nodes.include? deploy.destination.split(':', 2).first + + deploy.destination = "#{deploy.destination}#{deploy.site.hostname}" + deploy.type = 'DeployRsync' + + deploy.save + end + end + + private + + def nodes + @nodes ||= Rails.application.nodes.map do |node| + "sutty@#{node}" + end + end +end From 6f4e91d3180005372a5ee9f7bb3e0db140d6c63f Mon Sep 17 00:00:00 2001 From: f Date: Wed, 5 Oct 2022 18:44:23 -0300 Subject: [PATCH 21/77] soportar distributed press apiv0 #7839 --- app/models/deploy.rb | 15 ++++++++++ app/models/deploy_distributed_press.rb | 34 +++++++++++++++++++++++ app/models/deploy_local.rb | 38 ++++++++++++++++++-------- 3 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 app/models/deploy_distributed_press.rb diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 7aedbb19..52031d87 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -92,6 +92,13 @@ class Deploy < ApplicationRecord r&.success? end + # Variables de entorno + # + # @return [Hash] + def local_env + @local_env ||= {} + end + private # @param [String] @@ -99,4 +106,12 @@ class Deploy < ApplicationRecord def readable_cmd(cmd) cmd.split(' -', 2).first.tr(' ', '_') end + + def deploy_local + @deploy_local ||= site.deploys.find_by(type: 'DeployLocal') + end + + def non_local_deploys + @non_local_deploys ||= site.deploys.where.not(type: 'DeployLocal') + end end diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb new file mode 100644 index 00000000..51788865 --- /dev/null +++ b/app/models/deploy_distributed_press.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +# Soportar Distributed Press APIv0 +# +# No se realiza ninguna acción porque el deploy se hace desde el plugin +# local. +class DeployDistributedPress < Deploy + store :values, accessors: %i[api_key hostname], coder: JSON + + def deploy; end + + def limit; end + + def size + deploy_local.size + end + + # TODO: Devolver hyper:// y otras + def url + "ipfs://#{hostname}/" + end + + # Devuelve variables de entorno para enviarle a DeployLocal + # + # @return [Hash] + def local_env + { + 'DISTRIBUTED_PRESS_PROJECT_DOMAIN' => hostname, + 'DISTRIBUTED_PRESS_API_KEY' => api_key + } + end + + def destination; end +end diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 00e0985d..2d77df75 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -67,21 +67,24 @@ class DeployLocal < Deploy end # Un entorno que solo tiene lo que necesitamos + # + # @return [Hash] def env # XXX: This doesn't support Windows paths :B paths = [File.dirname(`which bundle`), '/usr/bin', '/bin'] - { - 'HOME' => home_dir, - 'PATH' => paths.join(':'), - 'SPREE_API_KEY' => site.tienda_api_key, - 'SPREE_URL' => site.tienda_url, - 'AIRBRAKE_PROJECT_ID' => site.id.to_s, - 'AIRBRAKE_PROJECT_KEY' => site.airbrake_api_key, - 'JEKYLL_ENV' => Rails.env, - 'LANG' => ENV['LANG'], - 'YARN_CACHE_FOLDER' => yarn_cache_dir - } + # Las variables de entorno extra no pueden superponerse al local. + extra_env.merge({ + 'HOME' => home_dir, + 'PATH' => paths.join(':'), + 'SPREE_API_KEY' => site.tienda_api_key, + 'SPREE_URL' => site.tienda_url, + 'AIRBRAKE_PROJECT_ID' => site.id.to_s, + 'AIRBRAKE_PROJECT_KEY' => site.airbrake_api_key, + 'JEKYLL_ENV' => Rails.env, + 'LANG' => ENV['LANG'], + 'YARN_CACHE_FOLDER' => yarn_cache_dir + }) end def yarn_cache_dir @@ -125,4 +128,17 @@ class DeployLocal < Deploy def remove_destination! FileUtils.rm_rf destination end + + # Consigue todas las variables de entorno configuradas por otros + # deploys. + # + # @return [Hash] + def extra_env + @extra_env ||= + non_local_deploys.reduce({}) do |extra_env, deploy| + extra_env.tap do |e| + e.merge! deploy.local_env + end + end + end end From b7daa8e0eb4a111e880329b1ed57ccf6dc88063a Mon Sep 17 00:00:00 2001 From: f Date: Fri, 18 Nov 2022 15:56:59 -0300 Subject: [PATCH 22/77] feat: poder modificar la url de distributed press --- app/models/deploy_distributed_press.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 51788865..a24ae62c 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -5,7 +5,7 @@ # No se realiza ninguna acción porque el deploy se hace desde el plugin # local. class DeployDistributedPress < Deploy - store :values, accessors: %i[api_key hostname], coder: JSON + store :values, accessors: %i[api_url api_key hostname], coder: JSON def deploy; end @@ -26,7 +26,8 @@ class DeployDistributedPress < Deploy def local_env { 'DISTRIBUTED_PRESS_PROJECT_DOMAIN' => hostname, - 'DISTRIBUTED_PRESS_API_KEY' => api_key + 'DISTRIBUTED_PRESS_API_KEY' => api_key, + 'DISTRIBUTED_PRESS_API_URL' => api_url } end From ecdbfeb56863589f0c2b16031a17ecaf3da5769c Mon Sep 17 00:00:00 2001 From: Maki Date: Tue, 17 Jan 2023 18:31:54 -0300 Subject: [PATCH 23/77] agrego toggler para distributed press --- app/models/site.rb | 2 +- .../deploys/_deploy_distributed_press.haml | 21 +++++++++++++++++++ config/locales/en.yml | 4 ++++ config/locales/es.yml | 4 ++++ 4 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 app/views/deploys/_deploy_distributed_press.haml diff --git a/app/models/site.rb b/app/models/site.rb index f88e2e97..af0b2c53 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -18,7 +18,7 @@ class Site < ApplicationRecord # TODO: Hacer que los diferentes tipos de deploy se auto registren # @see app/services/site_service.rb - DEPLOYS = %i[local private www zip hidden_service].freeze + DEPLOYS = %i[local private www zip hidden_service distributed_press].freeze validates :name, uniqueness: true, hostname: { allow_root_label: true diff --git a/app/views/deploys/_deploy_distributed_press.haml b/app/views/deploys/_deploy_distributed_press.haml new file mode 100644 index 00000000..d7d54db0 --- /dev/null +++ b/app/views/deploys/_deploy_distributed_press.haml @@ -0,0 +1,21 @@ +-# Publicar a la web distribuida + +.row + .col + = deploy.hidden_field :id + = deploy.hidden_field :type + .custom-control.custom-switch + -# + El checkbox invierte la lógica de destrucción porque queremos + crear el deploy si está activado y destruirlo si está + desactivado. + = deploy.check_box :_destroy, + { checked: deploy.object.persisted?, class: 'custom-control-input' }, + '0', '1' + = deploy.label :_destroy, class: 'custom-control-label' do + %h3= t('.title') + = sanitize_markdown t('.help', public_url: deploy.object.site.url), + tags: %w[p strong em a] + + +%hr/ diff --git a/config/locales/en.yml b/config/locales/en.yml index 567ab6bb..595d6145 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -272,6 +272,10 @@ en: Only accessible through [Tor Browser](https://www.torproject.org/download/) + deploy_distributed_press: + title: 'Publish on Distributed Press' + help: | + ipfs stats: index: title: Statistics diff --git a/config/locales/es.yml b/config/locales/es.yml index 0a829e4a..3be67130 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -277,6 +277,10 @@ es: Sólo será accesible a través del [Navegador Tor](https://www.torproject.org/es/download/). + deploy_distributed_press: + title: 'Alojar en la web distribuida' + help: | + Es para que esté en la web distribuida stats: index: title: Estadísticas From 84b2968cdbf85c66a5a9ab1a4534f75a20463e3a Mon Sep 17 00:00:00 2001 From: f Date: Fri, 20 Jan 2023 18:22:08 -0300 Subject: [PATCH 24/77] feat: almacenar y renovar tokens de distributed press --- Gemfile | 1 + Gemfile.lock | 40 ++++++++++- Procfile | 1 + .../renew_distributed_press_tokens_job.rb | 17 +++++ app/models/distributed_press_publisher.rb | 68 +++++++++++++++++++ ...5420_create_distributed_press_publisher.rb | 14 ++++ lib/tasks/distributed_press.rake | 10 +++ monit.conf | 5 ++ 8 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 app/jobs/renew_distributed_press_tokens_job.rb create mode 100644 app/models/distributed_press_publisher.rb create mode 100644 db/migrate/20230119165420_create_distributed_press_publisher.rb create mode 100644 lib/tasks/distributed_press.rake diff --git a/Gemfile b/Gemfile index 489097a6..fb1f4953 100644 --- a/Gemfile +++ b/Gemfile @@ -39,6 +39,7 @@ gem 'commonmarker' gem 'devise' gem 'devise-i18n' gem 'devise_invitable' +gem 'distributed-press-api-client', '~> 0.2.0' gem 'email_address', git: 'https://github.com/fauno/email_address', branch: 'i18n' gem 'exception_notification' gem 'fast_blank' diff --git a/Gemfile.lock b/Gemfile.lock index 5df84963..9a754b83 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -115,6 +115,7 @@ GEM xpath (>= 2.0, < 4.0) chartkick (4.1.2) childprocess (4.1.0) + climate_control (1.2.0) coderay (1.1.3) colorator (1.1.0) commonmarker (0.21.2-x86_64-linux-musl) @@ -153,12 +154,45 @@ GEM devise_invitable (2.0.5) actionmailer (>= 5.0) devise (>= 4.6) + distributed-press-api-client (0.2.0) + addressable (~> 2.3, >= 2.3.0) + climate_control + dry-schema + httparty (~> 0.18) + json (~> 2.1, >= 2.1.0) + jwt (~> 2.6.0) dotenv (2.7.6) dotenv-rails (2.7.6) dotenv (= 2.7.6) railties (>= 3.2) down (5.2.4) addressable (~> 2.8) + dry-configurable (1.0.1) + dry-core (~> 1.0, < 2) + zeitwerk (~> 2.6) + dry-core (1.0.0) + concurrent-ruby (~> 1.0) + zeitwerk (~> 2.6) + dry-inflector (1.0.0) + dry-initializer (3.1.1) + dry-logic (1.5.0) + concurrent-ruby (~> 1.0) + dry-core (~> 1.0, < 2) + zeitwerk (~> 2.6) + dry-schema (1.13.0) + concurrent-ruby (~> 1.0) + dry-configurable (~> 1.0, >= 1.0.1) + dry-core (~> 1.0, < 2) + dry-initializer (~> 3.0) + dry-logic (>= 1.5, < 2) + dry-types (>= 1.7, < 2) + zeitwerk (~> 2.6) + dry-types (1.7.0) + concurrent-ruby (~> 1.0) + dry-core (~> 1.0, < 2) + dry-inflector (~> 1.0, < 2) + dry-logic (>= 1.4, < 2) + zeitwerk (~> 2.6) ed25519 (1.2.4-x86_64-linux-musl) em-websocket (0.5.3) eventmachine (>= 0.12.9) @@ -216,8 +250,8 @@ GEM thor hiredis (0.6.3-x86_64-linux-musl) http_parser.rb (0.8.0-x86_64-linux-musl) - httparty (0.18.1) - mime-types (~> 3.0) + httparty (0.21.0) + mini_mime (>= 1.0.0) multi_xml (>= 0.5.2) i18n (1.8.11) concurrent-ruby (~> 1.0) @@ -292,6 +326,7 @@ GEM jekyll-write-and-commit-changes (0.2.1) jekyll (~> 4) rugged (~> 1) + jwt (2.6.0) kaminari (1.2.1) activesupport (>= 4.1.0) kaminari-actionview (= 1.2.1) @@ -584,6 +619,7 @@ DEPENDENCIES devise devise-i18n devise_invitable + distributed-press-api-client (~> 0.2.0) dotenv-rails down ed25519 diff --git a/Procfile b/Procfile index 45fe1df7..79daa90b 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,3 @@ cleanup: bundle exec rake cleanup:everything stats: bundle exec rake stats:process_all +distributed_press_renew_tokens: bundle exec rake distributed_press:tokens:renew diff --git a/app/jobs/renew_distributed_press_tokens_job.rb b/app/jobs/renew_distributed_press_tokens_job.rb new file mode 100644 index 00000000..5664d9fa --- /dev/null +++ b/app/jobs/renew_distributed_press_tokens_job.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# Renueva los tokens de Distributed Press antes que se venzan, +# activando los callbacks que hacen que se refresque el token. +class RenewDistributedPressTokensJob < ApplicationJob + # Renueva todos los tokens a punto de vencer o informa el error sin + # detener la tarea si algo pasa. + def perform + DistributedPressPublisher.with_about_to_expire_tokens.find_each do |publisher| + publisher.touch + rescue DistributedPress::V1::Error => e + data = { instance: publisher.instance, expires_at: publisher.client.token.expires_at } + + ExceptionNotifier.notify_exception(e, data: data) + end + end +end diff --git a/app/models/distributed_press_publisher.rb b/app/models/distributed_press_publisher.rb new file mode 100644 index 00000000..610d698d --- /dev/null +++ b/app/models/distributed_press_publisher.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'distributed_press/v1' + +# Almacena el token de autenticación y la URL, por ahora solo vamos +# a tener uno, pero queda abierta la posibilidad de agregar más. +class DistributedPressPublisher < ApplicationRecord + # Cifrar la información del token en la base de datos + has_encrypted :token + + # La instancia es única + validates_uniqueness_of :instance + + # El token es necesario + validates_presence_of :token + + # Mantener la fecha de vencimiento actualizada + before_save :update_expires_at_from_token!, :update_token_from_client! + + # Devuelve todos los tokens que vencen en una hora + scope :with_about_to_expire_tokens, lambda { + where('expires_at > ? and expires_at < ?', Time.now, Time.now + 1.hour) + } + + # Al cambiar el token genera un cliente nuevo + # + # @return [String] + def token=(new_token) + @client = nil + super + end + + # Al cambiar la instancia genera un cliente nuevo + # + # @return [String] + def instance=(new_instance) + @client = nil + super + end + + # Instancia un cliente de Distributed Press a partir del token. Al + # cargar un token a punto de vencer se renueva automáticamente. + # + # @return [DistributedPress::V1::Client] + def client + @client ||= DistributedPress::V1::Client.new(url: instance, token: token) + end + + private + + # Actualiza o desactiva la fecha de vencimiento a partir de la + # información del token. + # + # @return [nil] + def update_expires_at_from_token! + self.expires_at = client.token.forever? ? nil : client.token.expires_at + nil + end + + # Actualiza el token a partir del cliente, que ya actualiza el token + # automáticamente. + # + # @return [nil] + def update_token_from_client! + self.token = client.token.to_s + nil + end +end diff --git a/db/migrate/20230119165420_create_distributed_press_publisher.rb b/db/migrate/20230119165420_create_distributed_press_publisher.rb new file mode 100644 index 00000000..8d8de37a --- /dev/null +++ b/db/migrate/20230119165420_create_distributed_press_publisher.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Crea la tabla de publishers de Distributed Press que contiene las +# instancias y tokens +class CreateDistributedPressPublisher < ActiveRecord::Migration[6.1] + def change + create_table :distributed_press_publishers do |t| + t.timestamps + t.string :instance, unique: true + t.text :token_ciphertext, null: false + t.datetime :expires_at, null: true + end + end +end diff --git a/lib/tasks/distributed_press.rake b/lib/tasks/distributed_press.rake new file mode 100644 index 00000000..8ba270ec --- /dev/null +++ b/lib/tasks/distributed_press.rake @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +namespace :distributed_press do + namespace :tokens do + desc 'Renew tokens' + task renew: :environment do + RenewDistributedPressTokensJob.perform_now + end + end +end diff --git a/monit.conf b/monit.conf index 39f45d6d..1d6bbae1 100644 --- a/monit.conf +++ b/monit.conf @@ -13,3 +13,8 @@ check program stats with path "/usr/bin/foreman run -f /srv/Procfile -d /srv stats" as uid "rails" gid "www-data" every "0 1 * * *" if status != 0 then alert + +check program distributed_press_tokens_renew + with path "/usr/bin/foreman run -f /srv/Procfile -d /srv distributed_press_tokens_renew" as uid "rails" gid "www-data" + every "0 3 * * *" + if status != 0 then alert From c44584596da1d3e23d8f986ec7d2b9a421bfb1b6 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 20 Jan 2023 20:34:27 -0300 Subject: [PATCH 25/77] feat: crear y publicar el sitio en distributed press v1 --- app/models/deploy_distributed_press.rb | 143 ++++++++++++++++++---- app/models/distributed_press_publisher.rb | 5 + 2 files changed, 125 insertions(+), 23 deletions(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index a24ae62c..09ba0364 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -1,13 +1,43 @@ # frozen_string_literal: true -# Soportar Distributed Press APIv0 -# -# No se realiza ninguna acción porque el deploy se hace desde el plugin -# local. -class DeployDistributedPress < Deploy - store :values, accessors: %i[api_url api_key hostname], coder: JSON +require 'distributed_press/v1/client/auth' +require 'distributed_press/v1/client/site' - def deploy; end +# Soportar Distributed Press APIv1 +# +# Usa tokens de publicación efímeros para todas las acciones. +# +# Al ser creado, genera el sitio en la instancia de Distributed Press +# configurada y almacena el ID. +# +# Al ser publicado, envía los archivos en un tarball y actualiza la +# información. +class DeployDistributedPress < Deploy + store :values, accessors: %i[hostname remote_site_id remote_info], coder: JSON + + before_create :create_remote_site! + + # Actualiza la información y luego envía los cambios + # + # @param :output [Bool] + # @return [Bool] + def deploy + time_start + + status = false + + site_client.tap do |c| + update remote_info: c.show(publishing_site).to_h + + status = c.publish(publishing_site, deploy_local.destination) + end + + time_stop + + create_stat! status + + status + end def limit; end @@ -15,21 +45,88 @@ class DeployDistributedPress < Deploy deploy_local.size end - # TODO: Devolver hyper:// y otras - def url - "ipfs://#{hostname}/" - end - - # Devuelve variables de entorno para enviarle a DeployLocal - # - # @return [Hash] - def local_env - { - 'DISTRIBUTED_PRESS_PROJECT_DOMAIN' => hostname, - 'DISTRIBUTED_PRESS_API_KEY' => api_key, - 'DISTRIBUTED_PRESS_API_URL' => api_url - } - end - def destination; end + + private + + # El cliente de la API + # + # TODO: cuando soportemos más, tiene que haber una relación entre + # DeployDistributedPress y DistributedPressPublisher. + # + # @return [DistributedPressPublisher] + def publisher + @publisher ||= DistributedPressPublisher.first + end + + # El cliente de autenticación se encarga de intercambiar un token por + # otro + # + # @return [DistributedPress::V1::Client::Auth] + def auth_client + DistributedPress::V1::Client::Auth.new(publisher.client) + end + + # Genera un token con permisos de publicación + # + # @return [DistributedPress::V1::Schemas::NewTokenPayload] + def publishing_token_payload + DistributedPress::V1::Schemas::NewTokenPayload.new.call(capabilities: %w[publisher]) + end + + # Genera un token efímero para publicar el sitio + # + # @return [DistributedPress::V1::Token] + def publishing_token + auth_client.exchange(publishing_token_payload) + end + + # Genera un cliente para publicar el sitio + # + # @return [DistributedPress::V1::Client] + def publishing_client + DistributedPress::V1::Client.new(url: publisher.instance, token: publishing_token) + end + + # El cliente para actualizar el sitio + # + # @return [DistributedPress::V1::Client::Site] + def site_client + DistributedPress::V1::Client::Site.new(publishing_client) + end + + # Genera el esquema de datos para poder publicar el sitio + # + # @return [DistributedPress::V1::Schemas::PublishingSite] + def publishing_site + DistributedPress::V1::Schemas::PublishingSite.new.call(id: remote_site_id) + end + + # Genera el esquema de datos para crear el sitio + # + # @return [DistributedPressPublisher::V1::Schemas::NewSite] + def create_site + DistributedPress::V1::Schemas::NewSite.new.call(domain: hostname, protocols: { http: true, ipfs: true, hyper: true }) + end + + # Crea el sitio en la instancia con el hostname especificado + # + # @return [nil] + def create_remote_site! + created_site = site_client.create(create_site) + + self.remote_site_id = created_site[:id] + self.remote_info = created_site.to_h + + nil + end + + # Registra lo que sucedió + # + # @param status [Bool] + # @return [nil] + def create_stat!(status) + build_stats.create action: publisher.to_s, seconds: time_spent_in_seconds, bytes: size, status: status + nil + end end diff --git a/app/models/distributed_press_publisher.rb b/app/models/distributed_press_publisher.rb index 610d698d..5358e738 100644 --- a/app/models/distributed_press_publisher.rb +++ b/app/models/distributed_press_publisher.rb @@ -46,6 +46,11 @@ class DistributedPressPublisher < ApplicationRecord @client ||= DistributedPress::V1::Client.new(url: instance, token: token) end + # @return [String] + def to_s + "Distributed Press <#{instance}>" + end + private # Actualiza o desactiva la fecha de vencimiento a partir de la From f190fb2dc82bc5f3a3e829ec1a10aa672f19f890 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 20 Jan 2023 20:35:39 -0300 Subject: [PATCH 26/77] fix: permitir modificar el token estos parches hacian que no se pueda modificar el token --- app/models/distributed_press_publisher.rb | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/app/models/distributed_press_publisher.rb b/app/models/distributed_press_publisher.rb index 5358e738..acfc4226 100644 --- a/app/models/distributed_press_publisher.rb +++ b/app/models/distributed_press_publisher.rb @@ -22,22 +22,6 @@ class DistributedPressPublisher < ApplicationRecord where('expires_at > ? and expires_at < ?', Time.now, Time.now + 1.hour) } - # Al cambiar el token genera un cliente nuevo - # - # @return [String] - def token=(new_token) - @client = nil - super - end - - # Al cambiar la instancia genera un cliente nuevo - # - # @return [String] - def instance=(new_instance) - @client = nil - super - end - # Instancia un cliente de Distributed Press a partir del token. Al # cargar un token a punto de vencer se renueva automáticamente. # From 4ebbd48fde8b8bf8203e163a48ddd9159f1a06df Mon Sep 17 00:00:00 2001 From: f Date: Mon, 23 Jan 2023 18:39:10 -0300 Subject: [PATCH 27/77] =?UTF-8?q?feat:=20no=20vamos=20a=20expedir=20tokens?= =?UTF-8?q?=20ef=C3=ADmeros=20por=20ahora?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_distributed_press.rb | 31 +------------------------- 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 09ba0364..38cb59e3 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -59,40 +59,11 @@ class DeployDistributedPress < Deploy @publisher ||= DistributedPressPublisher.first end - # El cliente de autenticación se encarga de intercambiar un token por - # otro - # - # @return [DistributedPress::V1::Client::Auth] - def auth_client - DistributedPress::V1::Client::Auth.new(publisher.client) - end - - # Genera un token con permisos de publicación - # - # @return [DistributedPress::V1::Schemas::NewTokenPayload] - def publishing_token_payload - DistributedPress::V1::Schemas::NewTokenPayload.new.call(capabilities: %w[publisher]) - end - - # Genera un token efímero para publicar el sitio - # - # @return [DistributedPress::V1::Token] - def publishing_token - auth_client.exchange(publishing_token_payload) - end - - # Genera un cliente para publicar el sitio - # - # @return [DistributedPress::V1::Client] - def publishing_client - DistributedPress::V1::Client.new(url: publisher.instance, token: publishing_token) - end - # El cliente para actualizar el sitio # # @return [DistributedPress::V1::Client::Site] def site_client - DistributedPress::V1::Client::Site.new(publishing_client) + DistributedPress::V1::Client::Site.new(publisher.client) end # Genera el esquema de datos para poder publicar el sitio From c789813a46834ee675f12c1a05bb7fe987dbb817 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 26 Jan 2023 15:41:54 -0300 Subject: [PATCH 28/77] fix: actualizar api --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9a754b83..c739d86b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -154,7 +154,7 @@ GEM devise_invitable (2.0.5) actionmailer (>= 5.0) devise (>= 4.6) - distributed-press-api-client (0.2.0) + distributed-press-api-client (0.2.1) addressable (~> 2.3, >= 2.3.0) climate_control dry-schema From ea0539bb1f2db3145f3a05b9ab067a9530872d84 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 6 Feb 2023 16:09:51 -0300 Subject: [PATCH 29/77] feat: usar pnpm si existe relacionado con sutty/jekyll/sutty-base-jekyll-theme#53 --- Dockerfile | 2 ++ app/models/deploy_local.rb | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/Dockerfile b/Dockerfile index 342c2750..71edc3d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,8 @@ RUN wget https://github.com/jgm/pandoc/releases/download/${PANDOC_VERSION}/pando COPY ./monit.conf /etc/monit.d/sutty.conf +RUN apk add npm && npm install -g pnpm && apk del npm + VOLUME "/srv" EXPOSE 3000 diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 2d77df75..0e4f6c1c 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -15,6 +15,7 @@ class DeployLocal < Deploy def deploy(output: false) return false unless mkdir return false unless yarn(output: output) + return false unless pnpm(output: output) return false unless bundle(output: output) jekyll_build(output: output) @@ -91,6 +92,10 @@ class DeployLocal < Deploy Rails.root.join('_yarn_cache').to_s end + def pnpm_cache_dir + Rails.root.join('_pnpm_cache').to_s + end + def yarn_lock File.join(site.path, 'yarn.lock') end @@ -103,6 +108,14 @@ class DeployLocal < Deploy run %(gem install bundler --no-document), output: output end + def pnpm_lock + File.join(site.path, 'pnpm-lock.yaml') + end + + def pnpm_lock? + File.exist? pnpm_lock + end + # Corre yarn dentro del repositorio def yarn(output: false) return true unless yarn_lock? @@ -114,6 +127,13 @@ class DeployLocal < Deploy run %(bundle install --no-cache --path="#{gems_dir}" --clean --without test development), output: output end + def pnpm + return true unless pnpm_lock? + + run %(pnpm config set store-dir "#{pnpm_cache_dir}") + run 'pnpm install --production' + end + def jekyll_build(output: false) run %(bundle exec jekyll build --trace --profile --destination "#{escaped_destination}"), output: output end From 0bb0698bd45cb2224d9d81e973f7b8cde1c19c90 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 6 Feb 2023 16:16:27 -0300 Subject: [PATCH 30/77] fix: pnpm se instala localmente --- app/models/deploy_local.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 0e4f6c1c..d6dc6c28 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -72,7 +72,7 @@ class DeployLocal < Deploy # @return [Hash] def env # XXX: This doesn't support Windows paths :B - paths = [File.dirname(`which bundle`), '/usr/bin', '/bin'] + paths = [File.dirname(`which bundle`), '/usr/local/bin', '/usr/bin', '/bin'] # Las variables de entorno extra no pueden superponerse al local. extra_env.merge({ From 436b55d3179beca19495a61b273ab2ed22a66474 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 7 Feb 2023 18:15:25 -0300 Subject: [PATCH 31/77] feat: textos para el panel --- config/locales/en.yml | 16 ++++++++++++++-- config/locales/es.yml | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 595d6145..66372077 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -273,9 +273,21 @@ en: Only accessible through [Tor Browser](https://www.torproject.org/download/) deploy_distributed_press: - title: 'Publish on Distributed Press' + title: 'Publish to the distributed Web' help: | - ipfs + Make your site available through peer-to-peer protocols, + Inter-Planetary File System (IPFS), Hypercore, and via + BitTorrent, so your site is more resilient and can be available + offline, including in community mesh networks. + + **Important:** Only use this option if you would like your data + to be permanently available. If you decide to undo this + selection, a cleared version of the site will be shared in its + place. However, it is possible that nodes on the distributed + storage network may continue retaining copies of the data + indefinitely. + + [Learn more](#) stats: index: title: Statistics diff --git a/config/locales/es.yml b/config/locales/es.yml index 3be67130..79736137 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -278,9 +278,21 @@ es: Sólo será accesible a través del [Navegador Tor](https://www.torproject.org/es/download/). deploy_distributed_press: - title: 'Alojar en la web distribuida' + title: 'Publicar a la Web distribuida' help: | - Es para que esté en la web distribuida + Utiliza protocolos de pares, Inter-Planetary File System (IPFS), + Hypercore y torrents, para que tu sitio sea más resiliente y + esté disponible _offline_, inclusive en redes _mesh_ + comunitarias. + + **Importante:** Sólo usa esta opción si te parece correcto que + tu contenido esté disponible permanentemente. Cuando elijas + des-hacer esta acción, una versión "vacía" del sitio será + compartida en su lugar. Sin embargo, es posible que algunos + nodos en la red de almacenamiento distribuida puedan retener + copias de tu contenido indefinidamente. + + [Saber más](#) stats: index: title: Estadísticas From 4c6818762d72cca41c3cf22118abbd5e7e163dba Mon Sep 17 00:00:00 2001 From: f Date: Wed, 8 Feb 2023 09:38:36 -0300 Subject: [PATCH 32/77] fix: faltaban algunos require --- Gemfile | 2 +- Gemfile.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index fb1f4953..f92a37bf 100644 --- a/Gemfile +++ b/Gemfile @@ -39,7 +39,7 @@ gem 'commonmarker' gem 'devise' gem 'devise-i18n' gem 'devise_invitable' -gem 'distributed-press-api-client', '~> 0.2.0' +gem 'distributed-press-api-client', '~> 0.2.1' gem 'email_address', git: 'https://github.com/fauno/email_address', branch: 'i18n' gem 'exception_notification' gem 'fast_blank' diff --git a/Gemfile.lock b/Gemfile.lock index c739d86b..244d5511 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -619,7 +619,7 @@ DEPENDENCIES devise devise-i18n devise_invitable - distributed-press-api-client (~> 0.2.0) + distributed-press-api-client (~> 0.2.1) dotenv-rails down ed25519 From 975bf39c2e233b6d78f0e968c43a34359c3aec7c Mon Sep 17 00:00:00 2001 From: f Date: Wed, 8 Feb 2023 11:55:07 -0300 Subject: [PATCH 33/77] fix: no fallar si tarda mucho dp --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index f92a37bf..64869a4b 100644 --- a/Gemfile +++ b/Gemfile @@ -39,7 +39,7 @@ gem 'commonmarker' gem 'devise' gem 'devise-i18n' gem 'devise_invitable' -gem 'distributed-press-api-client', '~> 0.2.1' +gem 'distributed-press-api-client', '~> 0.2.2' gem 'email_address', git: 'https://github.com/fauno/email_address', branch: 'i18n' gem 'exception_notification' gem 'fast_blank' diff --git a/Gemfile.lock b/Gemfile.lock index 244d5511..c214e144 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -154,7 +154,7 @@ GEM devise_invitable (2.0.5) actionmailer (>= 5.0) devise (>= 4.6) - distributed-press-api-client (0.2.1) + distributed-press-api-client (0.2.2) addressable (~> 2.3, >= 2.3.0) climate_control dry-schema @@ -619,7 +619,7 @@ DEPENDENCIES devise devise-i18n devise_invitable - distributed-press-api-client (~> 0.2.1) + distributed-press-api-client (~> 0.2.2) dotenv-rails down ed25519 From 4e1f14a528f44d98199b20c3969e8129e7e274ad Mon Sep 17 00:00:00 2001 From: f Date: Wed, 8 Feb 2023 17:08:02 -0300 Subject: [PATCH 34/77] =?UTF-8?q?feat:=20guardar=20un=20log=20y=20mostrarl?= =?UTF-8?q?o=20en=20output=20tambi=C3=A9n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_distributed_press.rb | 24 ++++++++++++++++++----- app/models/distributed_press_publisher.rb | 22 ++++++++++++++++++++- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 38cb59e3..d805e923 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -22,19 +22,32 @@ class DeployDistributedPress < Deploy # @param :output [Bool] # @return [Bool] def deploy + status = false + log = [] + time_start - status = false - site_client.tap do |c| + stdout = Thread.new(publisher.logger_out) do |io| + until io.eof? + line = io.gets + + puts line if output + log << line + end + end + update remote_info: c.show(publishing_site).to_h status = c.publish(publishing_site, deploy_local.destination) + + publisher.logger.close + stdout.join end time_stop - create_stat! status + create_stat! status, log.join status end @@ -95,9 +108,10 @@ class DeployDistributedPress < Deploy # Registra lo que sucedió # # @param status [Bool] + # @param log [String] # @return [nil] - def create_stat!(status) - build_stats.create action: publisher.to_s, seconds: time_spent_in_seconds, bytes: size, status: status + def create_stat!(status, log) + build_stats.create action: publisher.to_s,log: log, seconds: time_spent_in_seconds, bytes: size, status: status nil end end diff --git a/app/models/distributed_press_publisher.rb b/app/models/distributed_press_publisher.rb index acfc4226..089f63c6 100644 --- a/app/models/distributed_press_publisher.rb +++ b/app/models/distributed_press_publisher.rb @@ -8,6 +8,11 @@ class DistributedPressPublisher < ApplicationRecord # Cifrar la información del token en la base de datos has_encrypted :token + # La salida del log + # + # @return [IO] + attr_reader :logger_out + # La instancia es única validates_uniqueness_of :instance @@ -27,7 +32,7 @@ class DistributedPressPublisher < ApplicationRecord # # @return [DistributedPress::V1::Client] def client - @client ||= DistributedPress::V1::Client.new(url: instance, token: token) + @client ||= DistributedPress::V1::Client.new(url: instance, token: token, logger: logger) end # @return [String] @@ -35,8 +40,23 @@ class DistributedPressPublisher < ApplicationRecord "Distributed Press <#{instance}>" end + # @return [Logger] + def logger + @logger ||= + begin + @logger_out, @logger_in = IO.pipe + ::Logger.new @logger_in, formatter: formatter + end + end + private + def formatter + @formatter ||= lambda do |_, _, _, msg| + "#{msg}\n" + end + end + # Actualiza o desactiva la fecha de vencimiento a partir de la # información del token. # From c291fefa998110ecef8f53486169f504ab0f9bb2 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 8 Feb 2023 17:44:35 -0300 Subject: [PATCH 35/77] =?UTF-8?q?fix:=20link=20a=20saber=20m=C3=A1s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/locales/en.yml | 2 +- config/locales/es.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 66372077..531b492f 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -287,7 +287,7 @@ en: storage network may continue retaining copies of the data indefinitely. - [Learn more](#) + [Learn more](https://ffdweb.org/building-distributed-press-a-publishing-tool-for-the-decentralized-web/) stats: index: title: Statistics diff --git a/config/locales/es.yml b/config/locales/es.yml index 79736137..84723b81 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -292,7 +292,7 @@ es: nodos en la red de almacenamiento distribuida puedan retener copias de tu contenido indefinidamente. - [Saber más](#) + [Saber más (en inglés)](https://ffdweb.org/building-distributed-press-a-publishing-tool-for-the-decentralized-web/) stats: index: title: Estadísticas From 56a3f2becb0f09909e220dd0c62794b95e8aa9d3 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 8 Feb 2023 18:49:37 -0300 Subject: [PATCH 36/77] feat: un deploy puede tener varias urls --- app/models/deploy_distributed_press.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index d805e923..d3474d50 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -60,6 +60,15 @@ class DeployDistributedPress < Deploy def destination; end + # Devuelve las URLs de todos los protocolos + def urls + remote_info[:links].values.map do |protocol| + [ protocol[:link], protocol[:gateway] ] + end.flatten.compact.select do |link| + link.include? '://' + end + end + private # El cliente de la API From 0b96fc44d6ba7eedf93d9dcb4ee607c777d7970e Mon Sep 17 00:00:00 2001 From: f Date: Wed, 8 Feb 2023 18:52:48 -0300 Subject: [PATCH 37/77] =?UTF-8?q?feat:=20texto=20para=20el=20correo=20de?= =?UTF-8?q?=20notificaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/locales/en.yml | 4 ++++ config/locales/es.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/config/locales/en.yml b/config/locales/en.yml index 531b492f..e040b781 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -126,6 +126,10 @@ en: title: Synchronize to backup server success: Success! error: Error + deploy_distributed_press: + title: Distributed Web + success: Success! + error: Error help: You can contact us by replying to this e-mail maintenance_mailer: notice: diff --git a/config/locales/es.yml b/config/locales/es.yml index 84723b81..f9fdcf29 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -126,6 +126,10 @@ es: title: Sincronizar al servidor alternativo success: ¡Éxito! error: Hubo un error + deploy_distributed_press: + title: Web distribuida + success: ¡Éxito! + error: Hubo un error help: Por cualquier duda, responde este correo para contactarte con nosotres. maintenance_mailer: notice: From 1b5adfb712bf89342993d3b315f236830bf76c6c Mon Sep 17 00:00:00 2001 From: f Date: Thu, 9 Feb 2023 20:01:22 -0300 Subject: [PATCH 38/77] feat: crear temporalmente los dominios en njalla el problema es que no podemos delegar `_dnslink.*.sutty.nl` hacia distributed press, con lo que es necesario crear un subdominio por cada sitio que lo active. --- Gemfile | 1 + Gemfile.lock | 4 ++++ app/models/deploy_distributed_press.rb | 18 ++++++++++++++++++ app/models/distributed_press_publisher.rb | 7 +++++++ 4 files changed, 30 insertions(+) diff --git a/Gemfile b/Gemfile index 64869a4b..9d1b6d67 100644 --- a/Gemfile +++ b/Gemfile @@ -40,6 +40,7 @@ gem 'devise' gem 'devise-i18n' gem 'devise_invitable' gem 'distributed-press-api-client', '~> 0.2.2' +gem 'njalla-api-client' gem 'email_address', git: 'https://github.com/fauno/email_address', branch: 'i18n' gem 'exception_notification' gem 'fast_blank' diff --git a/Gemfile.lock b/Gemfile.lock index c214e144..c2c29531 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -387,6 +387,9 @@ GEM nokogiri (1.12.5-x86_64-linux-musl) mini_portile2 (~> 2.6.1) racc (~> 1.4) + njalla-api-client (0.1.0) + dry-schema + httparty (~> 0.18) orm_adapter (0.5.0) pairing_heap (3.0.0) parallel (1.21.0) @@ -654,6 +657,7 @@ DEPENDENCIES mini_magick mobility net-ssh + njalla-api-client nokogiri pg pg_search diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index d3474d50..6f2b2c78 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -111,6 +111,12 @@ class DeployDistributedPress < Deploy self.remote_site_id = created_site[:id] self.remote_info = created_site.to_h + # XXX: Esto depende de nuestro DNS actual, cuando lo migremos hay + # que eliminarlo. + self.remote_info['njalla'] = {} + self.remote_info['njalla']['a'] = njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h + self.remote_info['njalla']['ns'] = njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h + nil end @@ -123,4 +129,16 @@ class DeployDistributedPress < Deploy build_stats.create action: publisher.to_s,log: log, seconds: time_spent_in_seconds, bytes: size, status: status nil end + + # Actualizar registros en Njalla + # + # @return [Njalla::V1::Domain] + def njalla + @njalla ||= + begin + client = Njalla::V1::Client.new(token: ENV['NJALLA_TOKEN']) + + Njalla::V1::Domain.new(domain: Site.domain, client: client) + end + end end diff --git a/app/models/distributed_press_publisher.rb b/app/models/distributed_press_publisher.rb index 089f63c6..6139db93 100644 --- a/app/models/distributed_press_publisher.rb +++ b/app/models/distributed_press_publisher.rb @@ -40,6 +40,13 @@ class DistributedPressPublisher < ApplicationRecord "Distributed Press <#{instance}>" end + # Devuelve el hostname de la instancia + # + # @return [String] + def hostname + @hostname ||= URI.parse(instance).hostname + end + # @return [Logger] def logger @logger ||= From 4666e67b5da555380d340b7b263b0c735e65dd51 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 9 Feb 2023 20:08:56 -0300 Subject: [PATCH 39/77] fix: no crear subdominios para dominios personalizados esto es por retrocompatibilidad para sitios que no usan DeployAlternativeDomain --- app/models/deploy_distributed_press.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 6f2b2c78..1ae7a2f2 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -113,9 +113,11 @@ class DeployDistributedPress < Deploy # XXX: Esto depende de nuestro DNS actual, cuando lo migremos hay # que eliminarlo. - self.remote_info['njalla'] = {} - self.remote_info['njalla']['a'] = njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h - self.remote_info['njalla']['ns'] = njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h + unless site.name.end_with? '.' + self.remote_info['njalla'] = {} + self.remote_info['njalla']['a'] = njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h + self.remote_info['njalla']['ns'] = njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h + end nil end From 812fc0315cc2dfee183e87983894848b6ad03e57 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 9 Feb 2023 20:11:17 -0300 Subject: [PATCH 40/77] fix: requires --- app/models/deploy_distributed_press.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 1ae7a2f2..41054b99 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -require 'distributed_press/v1/client/auth' require 'distributed_press/v1/client/site' +require 'njalla/v1' # Soportar Distributed Press APIv1 # From debf45b726cfc1c50cfe379bd73e5365ee9e8faa Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 17:33:52 -0300 Subject: [PATCH 41/77] fix: no fallar si hay errores remotos closes #10467 closes #10506 closes #10509 --- app/models/deploy_distributed_press.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 41054b99..09410201 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -118,7 +118,9 @@ class DeployDistributedPress < Deploy self.remote_info['njalla']['a'] = njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h self.remote_info['njalla']['ns'] = njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h end - + rescue DistributedPress::V1::Error, HTTParty::Error => e + ExceptionNotifier.notify_exception(e, data: { site: site.name }) + ensure nil end From 2b340a4488da30a82fbdfb51bcdf8796a37ebc3d Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 17:37:48 -0300 Subject: [PATCH 42/77] fix: crear el sitio remoto si no se pudo crear antes --- app/models/deploy_distributed_press.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 09410201..bd80dca4 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -27,6 +27,11 @@ class DeployDistributedPress < Deploy time_start + if remote_site_id.blank? || remote_info['njalla'].blank? + create_remote_site! + save + end + site_client.tap do |c| stdout = Thread.new(publisher.logger_out) do |io| until io.eof? From 29e1d09986022af9a9257941cfbe9b53ddf6b8e9 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 17:39:41 -0300 Subject: [PATCH 43/77] =?UTF-8?q?feat:=20separar=20la=20creaci=C3=B3n=20de?= =?UTF-8?q?=20DP=20de=20njalla?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_distributed_press.rb | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index bd80dca4..9afc6a5d 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -15,7 +15,7 @@ require 'njalla/v1' class DeployDistributedPress < Deploy store :values, accessors: %i[hostname remote_site_id remote_info], coder: JSON - before_create :create_remote_site! + before_create :create_remote_site!, :create_njalla_records! # Actualiza la información y luego envía los cambios # @@ -27,10 +27,9 @@ class DeployDistributedPress < Deploy time_start - if remote_site_id.blank? || remote_info['njalla'].blank? - create_remote_site! - save - end + create_remote_site! if remote_site_id.blank? + create_njalla_records! if remote_info['njalla'].blank? + save site_client.tap do |c| stdout = Thread.new(publisher.logger_out) do |io| @@ -115,7 +114,16 @@ class DeployDistributedPress < Deploy self.remote_site_id = created_site[:id] self.remote_info = created_site.to_h + rescue DistributedPress::V1::Error + ExceptionNotifier.notify_exception(e, data: { site: site.name }) + ensure + nil + end + # Crea los registros en Njalla + # + # @return [nil] + def create_njalla_records! # XXX: Esto depende de nuestro DNS actual, cuando lo migremos hay # que eliminarlo. unless site.name.end_with? '.' @@ -123,7 +131,7 @@ class DeployDistributedPress < Deploy self.remote_info['njalla']['a'] = njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h self.remote_info['njalla']['ns'] = njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h end - rescue DistributedPress::V1::Error, HTTParty::Error => e + rescue HTTParty::Error => e ExceptionNotifier.notify_exception(e, data: { site: site.name }) ensure nil From f0eab9a7b340ca32842373c3937e35794d509602 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 17:44:13 -0300 Subject: [PATCH 44/77] fix: volver a fallar si al hacer deploy todavia estan caidos los servicios --- app/models/deploy_distributed_press.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 9afc6a5d..e8d8b095 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -31,6 +31,10 @@ class DeployDistributedPress < Deploy create_njalla_records! if remote_info['njalla'].blank? save + if remote_site_id.blank? || remote_info['njalla'].blank? + raise DeployJob::DeployException, '' + end + site_client.tap do |c| stdout = Thread.new(publisher.logger_out) do |io| until io.eof? @@ -133,6 +137,7 @@ class DeployDistributedPress < Deploy end rescue HTTParty::Error => e ExceptionNotifier.notify_exception(e, data: { site: site.name }) + self.remote_info['njalla'] = nil ensure nil end From 78c2fd0053f8491adf71889477e9a76ccb58dada Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 16:40:17 -0300 Subject: [PATCH 45/77] fix: agregar dependencias a distributed press --- app/models/deploy_distributed_press.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index e8d8b095..7f663a0b 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -17,6 +17,8 @@ class DeployDistributedPress < Deploy before_create :create_remote_site!, :create_njalla_records! + DEPENDENCIES = %i[deploy_local] + # Actualiza la información y luego envía los cambios # # @param :output [Bool] From 3dc10c46f80ae1a750a544351ecce69bc8dac9ce Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 16:41:32 -0300 Subject: [PATCH 46/77] =?UTF-8?q?feat:=20todav=C3=ADa=20no=20hay=20interfa?= =?UTF-8?q?z=20para=20estos=20deploys?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/deploys/_deploy_full_rsync.haml | 1 + app/views/deploys/_deploy_localized_domain.haml | 1 + 2 files changed, 2 insertions(+) create mode 100644 app/views/deploys/_deploy_full_rsync.haml create mode 100644 app/views/deploys/_deploy_localized_domain.haml diff --git a/app/views/deploys/_deploy_full_rsync.haml b/app/views/deploys/_deploy_full_rsync.haml new file mode 100644 index 00000000..0aab9802 --- /dev/null +++ b/app/views/deploys/_deploy_full_rsync.haml @@ -0,0 +1 @@ +-# nada diff --git a/app/views/deploys/_deploy_localized_domain.haml b/app/views/deploys/_deploy_localized_domain.haml new file mode 100644 index 00000000..0aab9802 --- /dev/null +++ b/app/views/deploys/_deploy_localized_domain.haml @@ -0,0 +1 @@ +-# nada From 8b10cfb524fd8874126070d1f307b423c881e09e Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 16:48:32 -0300 Subject: [PATCH 47/77] feat: darle url a deploy rsync --- app/models/deploy_rsync.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 6a96a274..ac9f19ec 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -3,7 +3,7 @@ # Sincroniza sitios a servidores remotos usando Rsync. El servidor # remoto tiene que tener rsync instalado. class DeployRsync < Deploy - store :values, accessors: %i[destination host_keys], coder: JSON + store :values, accessors: %i[hostname destination host_keys], coder: JSON DEPENDENCIES = %i[deploy_local] @@ -25,6 +25,11 @@ class DeployRsync < Deploy end end + # @return [String] + def url + "https://#{hostname}/" + end + private # Verificar la conexión SSH implementando Trust On First Use From 91750116a16ba0b64708f69435d7ae0216121881 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 16:52:36 -0300 Subject: [PATCH 48/77] feat: darle url a full rsync --- app/models/deploy_full_rsync.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/models/deploy_full_rsync.rb b/app/models/deploy_full_rsync.rb index c0ff84c6..8f1d306e 100644 --- a/app/models/deploy_full_rsync.rb +++ b/app/models/deploy_full_rsync.rb @@ -19,4 +19,8 @@ class DeployFullRsync < DeployRsync end end.flatten.all? end + + def url + "https://#{user_host.last}/" + end end From 75621fb5c553368f8c4013a8f4cd2dea856dc5ee Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 16:53:56 -0300 Subject: [PATCH 49/77] fix: traducir metodos de publicacion #10030 --- config/locales/en.yml | 4 ++++ config/locales/es.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/config/locales/en.yml b/config/locales/en.yml index e040b781..8ecdd6d3 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -126,6 +126,10 @@ en: title: Synchronize to backup server success: Success! error: Error + deploy_full_rsync: + title: Synchronize to another Sutty node + success: Success! + error: Error deploy_distributed_press: title: Distributed Web success: Success! diff --git a/config/locales/es.yml b/config/locales/es.yml index f9fdcf29..90c07e32 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -126,6 +126,10 @@ es: title: Sincronizar al servidor alternativo success: ¡Éxito! error: Hubo un error + deploy_full_rsync: + title: Sincronizar a otro nodo de Sutty + success: ¡Éxito! + error: Hubo un error deploy_distributed_press: title: Web distribuida success: ¡Éxito! From ad5724be297accc3dd439f0325b615dec9e119e1 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 17:00:29 -0300 Subject: [PATCH 50/77] fix: siempre hay segundos --- app/jobs/deploy_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 3c9b2740..7cc49157 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -37,7 +37,7 @@ class DeployJob < ApplicationJob raise DeployException, 'Una dependencia falló' if failed_dependencies? d status = d.deploy - seconds = d.build_stats.last.try(:seconds) + seconds = d.build_stats.last.try(:seconds) || 0 size = d.size urls = d.respond_to?(:urls) ? d.urls : [d.url].compact rescue StandardError => e From d40c401a7204c277214ddb57cdd955c0eef8e2c7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 17:01:38 -0300 Subject: [PATCH 51/77] fix: mostrar la salida --- app/jobs/deploy_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 7cc49157..83b17459 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -36,7 +36,7 @@ class DeployJob < ApplicationJob begin raise DeployException, 'Una dependencia falló' if failed_dependencies? d - status = d.deploy + status = d.deploy(output: @output) seconds = d.build_stats.last.try(:seconds) || 0 size = d.size urls = d.respond_to?(:urls) ? d.urls : [d.url].compact From 54575b3bc3e33f3b7a8425a4c9792957e6c9aa71 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 17:02:38 -0300 Subject: [PATCH 52/77] =?UTF-8?q?fix:=20siempre=20avisar=20cuando=20termin?= =?UTF-8?q?=C3=B3=20el=20deploy=20en=20la=20consola?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/deploy_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 83b17459..890f993a 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -66,14 +66,14 @@ class DeployJob < ApplicationJob t << ([type.to_s] + row.values) end end) - - puts "\a" rescue DeployTimedOutException => e notify_exception e ensure @site&.update status: 'waiting' notify_usuaries if notify + + puts "\a" if @output end end # rubocop:enable Metrics/MethodLength From 8b51eb99a3e0224fe0aace172735ebf9025618cd Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 19:01:13 -0300 Subject: [PATCH 53/77] fix: private depende de local para instalar las dependencias --- app/models/deploy_private.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/deploy_private.rb b/app/models/deploy_private.rb index d3bfb50d..1fa42648 100644 --- a/app/models/deploy_private.rb +++ b/app/models/deploy_private.rb @@ -6,6 +6,8 @@ # XXX: La plantilla tiene que soportar esto con el plugin # jekyll-private-data class DeployPrivate < DeployLocal + DEPENDENCIES = %i[deploy_local] + # No es necesario volver a instalar dependencias def deploy(output: false) jekyll_build(output: output) From 94980051b65f8ee9bf44fc2890edf02d89c679bb Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 19:33:25 -0300 Subject: [PATCH 54/77] fix: volver a hacer depender la sincronizacion del zip --- app/models/deploy_full_rsync.rb | 1 + app/models/deploy_rsync.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/deploy_full_rsync.rb b/app/models/deploy_full_rsync.rb index 8f1d306e..5bdfeeff 100644 --- a/app/models/deploy_full_rsync.rb +++ b/app/models/deploy_full_rsync.rb @@ -6,6 +6,7 @@ class DeployFullRsync < DeployRsync deploy_hidden_service deploy_local deploy_www + deploy_zip ] # Sincroniza las ubicaciones alternativas también diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index ac9f19ec..1dff2d99 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -5,7 +5,7 @@ class DeployRsync < Deploy store :values, accessors: %i[hostname destination host_keys], coder: JSON - DEPENDENCIES = %i[deploy_local] + DEPENDENCIES = %i[deploy_local deploy_zip] def deploy(output: false) ssh? && rsync(output: output) From 2a93218ef9d8f412b4116b8463e239cf2e413fb4 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 20 Mar 2023 12:20:50 -0300 Subject: [PATCH 55/77] fix: siempre devolver el destino closes ##10541 closes ##10540 closes ##10539 closes ##10538 closes ##10537 closes ##10536 closes ##10535 closes ##10534 closes ##10533 --- app/models/deploy_zip.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/deploy_zip.rb b/app/models/deploy_zip.rb index 5f72a728..85005470 100644 --- a/app/models/deploy_zip.rb +++ b/app/models/deploy_zip.rb @@ -65,6 +65,8 @@ class DeployZip < Deploy # @return [String] def destination Rails.root.join('_deploy', site.hostname).realpath.to_s + rescue Errno::ENOENT + Rails.root.join('_deploy', site.hostname).to_s end def file From f5274cf34224f3e5cef4ae72dbe1a4cb99d5f1a3 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 20 Mar 2023 12:26:11 -0300 Subject: [PATCH 56/77] =?UTF-8?q?fix:=20no=20hacer=20falla=20la=20configur?= =?UTF-8?q?aci=C3=B3n=20si=20todav=C3=ADa=20no=20se=20gener=C3=B3=20el=20o?= =?UTF-8?q?nion=20#10542?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/deploys/_deploy_hidden_service.haml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/views/deploys/_deploy_hidden_service.haml b/app/views/deploys/_deploy_hidden_service.haml index d6388123..9ebda012 100644 --- a/app/views/deploys/_deploy_hidden_service.haml +++ b/app/views/deploys/_deploy_hidden_service.haml @@ -17,7 +17,8 @@ = sanitize_markdown t('.help', public_url: deploy.object.site.url), tags: %w[p strong em a] - - if deploy.object.fqdn + - begin = sanitize_markdown t('.help_2', url: deploy.object.url), tags: %w[p strong em a] + - rescue ArgumentError %hr/ From ecd8c80e028ab572ea82546e386bce1d17ceb8ba Mon Sep 17 00:00:00 2001 From: f Date: Mon, 20 Mar 2023 12:13:51 -0300 Subject: [PATCH 57/77] =?UTF-8?q?fix:=20capturar=20la=20excepci=C3=B3n=20#?= =?UTF-8?q?10546?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_distributed_press.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 7f663a0b..9414abc6 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -120,7 +120,7 @@ class DeployDistributedPress < Deploy self.remote_site_id = created_site[:id] self.remote_info = created_site.to_h - rescue DistributedPress::V1::Error + rescue DistributedPress::V1::Error => e ExceptionNotifier.notify_exception(e, data: { site: site.name }) ensure nil From 3abd87e8e6ad18744d5b62a3386bdaeef31e258d Mon Sep 17 00:00:00 2001 From: f Date: Thu, 23 Mar 2023 17:11:51 -0300 Subject: [PATCH 58/77] feat: guardar la api key de njalla en las credenciales --- app/models/deploy_distributed_press.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 9414abc6..6a6b1d64 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -160,7 +160,7 @@ class DeployDistributedPress < Deploy def njalla @njalla ||= begin - client = Njalla::V1::Client.new(token: ENV['NJALLA_TOKEN']) + client = Njalla::V1::Client.new(token: Rails.application.credentials.njalla) Njalla::V1::Domain.new(domain: Site.domain, client: client) end From 1ff48bf4589e8da1b73323c646af9d95d01524b0 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 23 Mar 2023 18:02:43 -0300 Subject: [PATCH 59/77] fix: no intentar notificar si no hay sitio closes #12753 --- app/jobs/deploy_job.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 890f993a..530b69dd 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -69,11 +69,13 @@ class DeployJob < ApplicationJob rescue DeployTimedOutException => e notify_exception e ensure - @site&.update status: 'waiting' + if @site.present? + @site.update status: 'waiting' - notify_usuaries if notify + notify_usuaries if notify - puts "\a" if @output + puts "\a" if @output + end end end # rubocop:enable Metrics/MethodLength From 21dd6ffd55d7870eac6d682efa183a43aaf2deaa Mon Sep 17 00:00:00 2001 From: f Date: Thu, 23 Mar 2023 18:25:55 -0300 Subject: [PATCH 60/77] =?UTF-8?q?fix:=20no=20fallar=20njalla=20si=20fall?= =?UTF-8?q?=C3=B3=20dp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #12410 --- app/models/deploy_distributed_press.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 6a6b1d64..679da09d 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -133,6 +133,7 @@ class DeployDistributedPress < Deploy # XXX: Esto depende de nuestro DNS actual, cuando lo migremos hay # que eliminarlo. unless site.name.end_with? '.' + self.remote_info ||= {} self.remote_info['njalla'] = {} self.remote_info['njalla']['a'] = njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h self.remote_info['njalla']['ns'] = njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h From c53391b21cd881f4de246cf18eacd345d314c215 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 24 Mar 2023 13:30:14 -0300 Subject: [PATCH 61/77] =?UTF-8?q?fix:=20mostrar=20los=20metodos=20fallidos?= =?UTF-8?q?=20en=20el=20correo=20de=20notificaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit no salían porque la tabla se genera en base a expandir las urls y cuando las urls venían vacías no se generaba la fila. --- app/jobs/deploy_job.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 530b69dd..aeb0f4b6 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -44,7 +44,8 @@ class DeployJob < ApplicationJob status = false seconds ||= 0 size ||= 0 - urls ||= [] + # XXX: Hace que se vea la tabla + urls ||= [nil] notify_exception e, d end From da192ef14cf43ac136d22a9601073f3bd6f23682 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 24 Mar 2023 14:23:34 -0300 Subject: [PATCH 62/77] =?UTF-8?q?fix:=20sincronizar=20aunque=20algunos=20d?= =?UTF-8?q?estinos=20todav=C3=ADa=20no=20existan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit por ejemplo la dependencia en hidden service hace que no se sincronice el sitio principal. --- app/models/deploy_full_rsync.rb | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/app/models/deploy_full_rsync.rb b/app/models/deploy_full_rsync.rb index 5bdfeeff..a3c1840d 100644 --- a/app/models/deploy_full_rsync.rb +++ b/app/models/deploy_full_rsync.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class DeployFullRsync < DeployRsync - DEPENDENCIES = %i[ + SOFT_DEPENDENCIES = %i[ deploy_alternative_domain deploy_hidden_service deploy_local @@ -9,16 +9,24 @@ class DeployFullRsync < DeployRsync deploy_zip ] - # Sincroniza las ubicaciones alternativas también + # Sincroniza las ubicaciones alternativas también, ignorando las que + # todavía no se generaron. Solo falla si ningún sitio fue + # sincronizado o si alguna sincronización falló. # # @param :output [Boolean] # @return [Boolean] def rsync(output: false) - DEPENDENCIES.map(&:to_s).map(&:classify).map do |dependency| - site.deploys.where(type: dependency).find_each.map do |deploy| - run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape deploy.destination} #{Shellwords.escape destination}), output: output - end - end.flatten.all? + result = + SOFT_DEPENDENCIES.map(&:to_s).map(&:classify).map do |dependency| + site.deploys.where(type: dependency).find_each.map do |deploy| + next unless File.exist? deploy.destination + + run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape deploy.destination} #{Shellwords.escape destination}), output: output + rescue StandardError + end + end.flatten.compact + + result.present? && result.all? end def url From 57542c92a7fcf1548e93e87b0162e19497507037 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 27 Mar 2023 14:08:19 -0300 Subject: [PATCH 63/77] fix: typo --- app/models/deploy_local.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index d6dc6c28..5b48df29 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -123,15 +123,15 @@ class DeployLocal < Deploy run 'yarn install --production', output: output end - def bundle(output: false) - run %(bundle install --no-cache --path="#{gems_dir}" --clean --without test development), output: output - end - - def pnpm + def pnpm(output: false) return true unless pnpm_lock? - run %(pnpm config set store-dir "#{pnpm_cache_dir}") - run 'pnpm install --production' + run %(pnpm config set store-dir "#{pnpm_cache_dir}"), output: output + run 'pnpm install --production', output: output + end + + def bundle(output: false) + run %(bundle install --no-cache --path="#{gems_dir}" --clean --without test development), output: output end def jekyll_build(output: false) From 42c7e93ac96cff1681047cb9bfc9f4c5f9353945 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 27 Mar 2023 14:09:16 -0300 Subject: [PATCH 64/77] =?UTF-8?q?fix:=20crear=20un=20registro=20para=20www?= =?UTF-8?q?=20tambi=C3=A9n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_distributed_press.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 679da09d..f26a72b5 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -136,6 +136,7 @@ class DeployDistributedPress < Deploy self.remote_info ||= {} self.remote_info['njalla'] = {} self.remote_info['njalla']['a'] = njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h + self.remote_info['njalla']['cname'] = njalla.add_record(name: "www.#{site.name}", type: 'CNAME', content: "#{Site.domain}.").to_h self.remote_info['njalla']['ns'] = njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h end rescue HTTParty::Error => e From bef171d3ace090d2afcca7be5ea7be91930baf26 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 27 Mar 2023 14:10:35 -0300 Subject: [PATCH 65/77] fix: intentar crear siempre los registros en njalla --- app/models/deploy_distributed_press.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index f26a72b5..a2e2e20a 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -30,7 +30,7 @@ class DeployDistributedPress < Deploy time_start create_remote_site! if remote_site_id.blank? - create_njalla_records! if remote_info['njalla'].blank? + create_njalla_records! save if remote_site_id.blank? || remote_info['njalla'].blank? @@ -134,10 +134,10 @@ class DeployDistributedPress < Deploy # que eliminarlo. unless site.name.end_with? '.' self.remote_info ||= {} - self.remote_info['njalla'] = {} - self.remote_info['njalla']['a'] = njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h - self.remote_info['njalla']['cname'] = njalla.add_record(name: "www.#{site.name}", type: 'CNAME', content: "#{Site.domain}.").to_h - self.remote_info['njalla']['ns'] = njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h + self.remote_info['njalla'] ||= {} + self.remote_info['njalla']['a'] ||= njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h + self.remote_info['njalla']['cname'] ||= njalla.add_record(name: "www.#{site.name}", type: 'CNAME', content: "#{Site.domain}.").to_h + self.remote_info['njalla']['ns'] ||= njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h end rescue HTTParty::Error => e ExceptionNotifier.notify_exception(e, data: { site: site.name }) From c774337fd325b421fd53c3873b6c9ae0aaef6582 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 27 Mar 2023 14:43:01 -0300 Subject: [PATCH 66/77] feat: sincronizar dominios por idioma --- app/models/deploy_full_rsync.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/deploy_full_rsync.rb b/app/models/deploy_full_rsync.rb index a3c1840d..7bc0e788 100644 --- a/app/models/deploy_full_rsync.rb +++ b/app/models/deploy_full_rsync.rb @@ -3,6 +3,7 @@ class DeployFullRsync < DeployRsync SOFT_DEPENDENCIES = %i[ deploy_alternative_domain + deploy_localized_domain deploy_hidden_service deploy_local deploy_www From b2c6d40b0f0dbdb7ff26b763db8865b24c4294b0 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 27 Mar 2023 14:44:22 -0300 Subject: [PATCH 67/77] fix: deprecar DeployLocal#extra_env --- app/models/deploy.rb | 1 + app/models/deploy_local.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 52031d87..33dc9f2b 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -13,6 +13,7 @@ class Deploy < ApplicationRecord has_many :build_stats, dependent: :destroy DEPENDENCIES = [] + SOFT_DEPENDENCIES = [] def deploy(**) raise NotImplementedError diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 5b48df29..d1d93fe5 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -152,6 +152,7 @@ class DeployLocal < Deploy # Consigue todas las variables de entorno configuradas por otros # deploys. # + # @deprecated Solo tenía sentido para Distributed Press v0 # @return [Hash] def extra_env @extra_env ||= From ad0d7aac10736cc375a069e5172454c38950699a Mon Sep 17 00:00:00 2001 From: f Date: Mon, 27 Mar 2023 14:52:37 -0300 Subject: [PATCH 68/77] feat: tener en cuenta las dependencias suaves al armar el grafo --- app/models/deploy.rb | 7 +++++++ app/models/deploy_full_rsync.rb | 4 +--- app/models/site/deploy_dependencies.rb | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 33dc9f2b..a92708c0 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -100,6 +100,13 @@ class Deploy < ApplicationRecord @local_env ||= {} end + # Trae todas las dependencias + # + # @return [Array] + def self.all_dependencies + self::DEPENDENCIES | self::SOFT_DEPENDENCIES + end + private # @param [String] diff --git a/app/models/deploy_full_rsync.rb b/app/models/deploy_full_rsync.rb index 7bc0e788..b8c48eab 100644 --- a/app/models/deploy_full_rsync.rb +++ b/app/models/deploy_full_rsync.rb @@ -5,9 +5,7 @@ class DeployFullRsync < DeployRsync deploy_alternative_domain deploy_localized_domain deploy_hidden_service - deploy_local deploy_www - deploy_zip ] # Sincroniza las ubicaciones alternativas también, ignorando las que @@ -18,7 +16,7 @@ class DeployFullRsync < DeployRsync # @return [Boolean] def rsync(output: false) result = - SOFT_DEPENDENCIES.map(&:to_s).map(&:classify).map do |dependency| + self.class.all_dependencies.map(&:to_s).map(&:classify).map do |dependency| site.deploys.where(type: dependency).find_each.map do |deploy| next unless File.exist? deploy.destination diff --git a/app/models/site/deploy_dependencies.rb b/app/models/site/deploy_dependencies.rb index a01f99e7..ed80a8af 100644 --- a/app/models/site/deploy_dependencies.rb +++ b/app/models/site/deploy_dependencies.rb @@ -18,7 +18,7 @@ class Site end deploys.each do |deploy| - deploy.class::DEPENDENCIES.each do |dependency| + deploy.class.all_dependencies.each do |dependency| deploys.where(type: dependency.to_s.classify).each do |deploy_dependency| graph.add_edge deploy_dependency, deploy end From bee16538301b11a5b8c0bc3645ebe59a09816968 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 30 Mar 2023 14:45:49 -0300 Subject: [PATCH 69/77] =?UTF-8?q?fix:=20evitar=20m=C3=A9todos=20de=20publi?= =?UTF-8?q?caci=C3=B3n=20duplicados=20#12276?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/sites_controller.rb | 2 -- app/services/site_service.rb | 1 + app/views/sites/_form.haml | 3 --- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/app/controllers/sites_controller.rb b/app/controllers/sites_controller.rb index f0eff0dc..8d49fd8a 100644 --- a/app/controllers/sites_controller.rb +++ b/app/controllers/sites_controller.rb @@ -28,8 +28,6 @@ class SitesController < ApplicationController @site = Site.new authorize @site - - @site.deploys.build type: 'DeployLocal' end def create diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 0ecccba4..bb76c6ac 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -14,6 +14,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do self.site = Site.new params add_role temporal: false, rol: 'usuarie' + site.deploys.build type: 'DeployLocal' sync_nodes I18n.with_locale(usuarie.lang.to_sym || I18n.default_locale) do diff --git a/app/views/sites/_form.haml b/app/views/sites/_form.haml index 757b8b48..48e30e0c 100644 --- a/app/views/sites/_form.haml +++ b/app/views/sites/_form.haml @@ -158,9 +158,6 @@ = f.fields_for :deploys do |deploy| = render "deploys/#{deploy.object.type.underscore}", deploy: deploy, site: site - - else - = f.fields_for :deploys do |deploy| - = deploy.hidden_field :type .form-group = f.submit submit, class: 'btn btn-lg btn-block' From 95198832c4791134df2c8f11fd682219e4684780 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 30 Mar 2023 14:53:34 -0300 Subject: [PATCH 70/77] feat: configurar la fuente de gemas del skel desde el panel --- app/models/deploy_local.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index d1d93fe5..39cd9622 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -84,7 +84,8 @@ class DeployLocal < Deploy 'AIRBRAKE_PROJECT_KEY' => site.airbrake_api_key, 'JEKYLL_ENV' => Rails.env, 'LANG' => ENV['LANG'], - 'YARN_CACHE_FOLDER' => yarn_cache_dir + 'YARN_CACHE_FOLDER' => yarn_cache_dir, + 'GEMS_SOURCE' => ENV['GEMS_SOURCE'] }) end From b759398a4ee15beaa52972840dfc29ac27557a55 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 31 Mar 2023 12:01:00 -0300 Subject: [PATCH 71/77] =?UTF-8?q?BREAKING=20CHANGE:=20usar=20la=20=C3=BAlt?= =?UTF-8?q?ima=20instancia=20de=20Distributed=20Press?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_distributed_press.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index a2e2e20a..64c66cfc 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -88,7 +88,7 @@ class DeployDistributedPress < Deploy # # @return [DistributedPressPublisher] def publisher - @publisher ||= DistributedPressPublisher.first + @publisher ||= DistributedPressPublisher.last end # El cliente para actualizar el sitio From b818408f3e7f5b2c9a4a0983e26102af9e428073 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 31 Mar 2023 12:14:14 -0300 Subject: [PATCH 72/77] feat: indicarle a distributed press que elimine el sitio sutty/distributed-press-api-client#10 --- Gemfile | 2 +- Gemfile.lock | 2 +- app/models/deploy_distributed_press.rb | 11 ++++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 9d1b6d67..2ff1c68c 100644 --- a/Gemfile +++ b/Gemfile @@ -39,7 +39,7 @@ gem 'commonmarker' gem 'devise' gem 'devise-i18n' gem 'devise_invitable' -gem 'distributed-press-api-client', '~> 0.2.2' +gem 'distributed-press-api-client', '~> 0.2.3' gem 'njalla-api-client' gem 'email_address', git: 'https://github.com/fauno/email_address', branch: 'i18n' gem 'exception_notification' diff --git a/Gemfile.lock b/Gemfile.lock index c2c29531..30619795 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -622,7 +622,7 @@ DEPENDENCIES devise devise-i18n devise_invitable - distributed-press-api-client (~> 0.2.2) + distributed-press-api-client (~> 0.2.3) dotenv-rails down ed25519 diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 64c66cfc..ca10218a 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -16,6 +16,7 @@ class DeployDistributedPress < Deploy store :values, accessors: %i[hostname remote_site_id remote_info], coder: JSON before_create :create_remote_site!, :create_njalla_records! + before_destroy :delete_remote_site! DEPENDENCIES = %i[deploy_local] @@ -120,9 +121,9 @@ class DeployDistributedPress < Deploy self.remote_site_id = created_site[:id] self.remote_info = created_site.to_h + nil rescue DistributedPress::V1::Error => e ExceptionNotifier.notify_exception(e, data: { site: site.name }) - ensure nil end @@ -156,6 +157,14 @@ class DeployDistributedPress < Deploy nil end + def delete_remote_site! + site_client.delete(publishing_site) + nil + rescue DistributedPress::V1::Error => e + ExceptionNotifier.notify_exception(e, data: { site: site.name }) + nil + end + # Actualizar registros en Njalla # # @return [Njalla::V1::Domain] From f6a08f1073c834d2201b7cbcb5d067fa28140cd7 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 31 Mar 2023 13:35:42 -0300 Subject: [PATCH 73/77] feat: eliminar registros en njalla --- Gemfile | 2 +- Gemfile.lock | 2 +- app/models/deploy_distributed_press.rb | 10 +++++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 2ff1c68c..b2472035 100644 --- a/Gemfile +++ b/Gemfile @@ -40,7 +40,7 @@ gem 'devise' gem 'devise-i18n' gem 'devise_invitable' gem 'distributed-press-api-client', '~> 0.2.3' -gem 'njalla-api-client' +gem 'njalla-api-client', '~> 0.2.0' gem 'email_address', git: 'https://github.com/fauno/email_address', branch: 'i18n' gem 'exception_notification' gem 'fast_blank' diff --git a/Gemfile.lock b/Gemfile.lock index 30619795..67ce13e2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -387,7 +387,7 @@ GEM nokogiri (1.12.5-x86_64-linux-musl) mini_portile2 (~> 2.6.1) racc (~> 1.4) - njalla-api-client (0.1.0) + njalla-api-client (0.2.0) dry-schema httparty (~> 0.18) orm_adapter (0.5.0) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index ca10218a..9f78ba28 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -16,7 +16,7 @@ class DeployDistributedPress < Deploy store :values, accessors: %i[hostname remote_site_id remote_info], coder: JSON before_create :create_remote_site!, :create_njalla_records! - before_destroy :delete_remote_site! + before_destroy :delete_remote_site!, :delete_njalla_records! DEPENDENCIES = %i[deploy_local] @@ -165,6 +165,14 @@ class DeployDistributedPress < Deploy nil end + def delete_njalla_records! + %w[a ns cname].each do |type| + next if (id = remote_info.dig('njalla', type, 'id')).blank? + + njalla.remove_record(id: id.to_i) + end + end + # Actualizar registros en Njalla # # @return [Njalla::V1::Domain] From d2e4fe09646e7b337868bf6711e2a3e5062a426b Mon Sep 17 00:00:00 2001 From: f Date: Fri, 31 Mar 2023 13:58:10 -0300 Subject: [PATCH 74/77] fix: los registros en njalla no siempre son necesarios --- app/models/deploy_distributed_press.rb | 36 ++++++++++++++++++-------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 9f78ba28..b77dfe19 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -34,8 +34,12 @@ class DeployDistributedPress < Deploy create_njalla_records! save - if remote_site_id.blank? || remote_info['njalla'].blank? - raise DeployJob::DeployException, '' + if remote_site_id.blank? + raise DeployJob::DeployException, 'El sitio no se creó en Distributed Press' + end + + if create_njalla_records? && remote_info['njalla'].blank? + raise DeployJob::DeployException, 'No se pudieron crear los registros necesarios en Njalla' end site_client.tap do |c| @@ -129,17 +133,20 @@ class DeployDistributedPress < Deploy # Crea los registros en Njalla # + # XXX: Esto depende de nuestro DNS actual, cuando lo migremos hay + # que eliminarlo. + # # @return [nil] def create_njalla_records! - # XXX: Esto depende de nuestro DNS actual, cuando lo migremos hay - # que eliminarlo. - unless site.name.end_with? '.' - self.remote_info ||= {} - self.remote_info['njalla'] ||= {} - self.remote_info['njalla']['a'] ||= njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h - self.remote_info['njalla']['cname'] ||= njalla.add_record(name: "www.#{site.name}", type: 'CNAME', content: "#{Site.domain}.").to_h - self.remote_info['njalla']['ns'] ||= njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h - end + return unless create_njalla_records? + + self.remote_info ||= {} + self.remote_info['njalla'] ||= {} + self.remote_info['njalla']['a'] ||= njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h + self.remote_info['njalla']['cname'] ||= njalla.add_record(name: "www.#{site.name}", type: 'CNAME', content: "#{Site.domain}.").to_h + self.remote_info['njalla']['ns'] ||= njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h + + nil rescue HTTParty::Error => e ExceptionNotifier.notify_exception(e, data: { site: site.name }) self.remote_info['njalla'] = nil @@ -166,6 +173,8 @@ class DeployDistributedPress < Deploy end def delete_njalla_records! + return unless create_njalla_records? + %w[a ns cname].each do |type| next if (id = remote_info.dig('njalla', type, 'id')).blank? @@ -184,4 +193,9 @@ class DeployDistributedPress < Deploy Njalla::V1::Domain.new(domain: Site.domain, client: client) end end + + # Detecta si tenemos que crear registros en Njalla + def create_njalla_records? + !site.name.end_with?('.') + end end From 943bf7b5bdafd2080736d8cdf4774ee6632d5b91 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 31 Mar 2023 14:38:04 -0300 Subject: [PATCH 75/77] fix: no pisar las datos de njalla con los de distributed press --- app/models/deploy_distributed_press.rb | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index b77dfe19..081ceeb5 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -38,7 +38,7 @@ class DeployDistributedPress < Deploy raise DeployJob::DeployException, 'El sitio no se creó en Distributed Press' end - if create_njalla_records? && remote_info['njalla'].blank? + if create_njalla_records? && remote_info[:njalla].blank? raise DeployJob::DeployException, 'No se pudieron crear los registros necesarios en Njalla' end @@ -52,7 +52,8 @@ class DeployDistributedPress < Deploy end end - update remote_info: c.show(publishing_site).to_h + self.remote_info[:distributed_press] = c.show(publishing_site).to_h + save status = c.publish(publishing_site, deploy_local.destination) @@ -77,7 +78,7 @@ class DeployDistributedPress < Deploy # Devuelve las URLs de todos los protocolos def urls - remote_info[:links].values.map do |protocol| + remote_info.dig(:distributed_press, :links).values.map do |protocol| [ protocol[:link], protocol[:gateway] ] end.flatten.compact.select do |link| link.include? '://' @@ -124,7 +125,8 @@ class DeployDistributedPress < Deploy created_site = site_client.create(create_site) self.remote_site_id = created_site[:id] - self.remote_info = created_site.to_h + self.remote_info ||= {} + self.remote_info[:distributed_press] = created_site.to_h nil rescue DistributedPress::V1::Error => e ExceptionNotifier.notify_exception(e, data: { site: site.name }) @@ -141,15 +143,15 @@ class DeployDistributedPress < Deploy return unless create_njalla_records? self.remote_info ||= {} - self.remote_info['njalla'] ||= {} - self.remote_info['njalla']['a'] ||= njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h - self.remote_info['njalla']['cname'] ||= njalla.add_record(name: "www.#{site.name}", type: 'CNAME', content: "#{Site.domain}.").to_h - self.remote_info['njalla']['ns'] ||= njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h + self.remote_info[:njalla] ||= {} + self.remote_info[:njalla][:a] ||= njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h + self.remote_info[:njalla][:cname] ||= njalla.add_record(name: "www.#{site.name}", type: 'CNAME', content: "#{Site.domain}.").to_h + self.remote_info[:njalla][:ns] ||= njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h nil rescue HTTParty::Error => e ExceptionNotifier.notify_exception(e, data: { site: site.name }) - self.remote_info['njalla'] = nil + self.remote_info.delete :njalla ensure nil end From a56acfd6da2aec2c3e5b913c91bae4de229be5ec Mon Sep 17 00:00:00 2001 From: f Date: Fri, 31 Mar 2023 14:44:48 -0300 Subject: [PATCH 76/77] =?UTF-8?q?fix:=20obtener=20los=20links=20luego=20de?= =?UTF-8?q?=20la=20publicaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_distributed_press.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 081ceeb5..d8fd3f61 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -52,11 +52,13 @@ class DeployDistributedPress < Deploy end end - self.remote_info[:distributed_press] = c.show(publishing_site).to_h - save - status = c.publish(publishing_site, deploy_local.destination) + if status + self.remote_info[:distributed_press] = c.show(publishing_site).to_h + save + end + publisher.logger.close stdout.join end From ab126038afe4a9352c82002d58687bb5074142bf Mon Sep 17 00:00:00 2001 From: f Date: Fri, 31 Mar 2023 14:49:17 -0300 Subject: [PATCH 77/77] feat: devolver las urls con protocolos --- app/models/deploy_distributed_press.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index d8fd3f61..32a3049e 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -80,6 +80,12 @@ class DeployDistributedPress < Deploy # Devuelve las URLs de todos los protocolos def urls + protocol_urls + gateway_urls + end + + private + + def gateway_urls remote_info.dig(:distributed_press, :links).values.map do |protocol| [ protocol[:link], protocol[:gateway] ] end.flatten.compact.select do |link| @@ -87,7 +93,13 @@ class DeployDistributedPress < Deploy end end - private + def protocol_urls + remote_info.dig(:distributed_press, :protocols).select do |_, enabled| + enabled + end.map do |protocol, _| + "#{protocol}://#{site.hostname}" + end + end # El cliente de la API #