5
0
Fork 0
mirror of https://0xacab.org/sutty/sutty synced 2024-11-16 12:41:41 +00:00

Merge branch 'issue-10464' into 'rails'

Issue #10464

Closes #10509, #10506, #10467, #12753, and #12410

See merge request sutty/sutty!121
This commit is contained in:
fauno 2023-04-10 15:31:38 +00:00
commit 69c5d8d7bb
27 changed files with 377 additions and 103 deletions

View file

@ -22,6 +22,8 @@ RUN apk add npm && npm install -g pnpm@~7 && apk del npm
COPY ./monit.conf /etc/monit.d/sutty.conf
RUN apk add npm && npm install -g pnpm && apk del npm
VOLUME "/srv"
EXPOSE 3000

View file

@ -23,6 +23,7 @@ if ENV['RAILS_GROUPS']&.split(',')&.include? 'assets'
end
gem 'nokogiri'
gem 'rgl'
# Turbolinks makes navigating your web application faster. Read more:
# https://github.com/turbolinks/turbolinks
@ -38,8 +39,8 @@ gem 'commonmarker'
gem 'devise'
gem 'devise-i18n'
gem 'devise_invitable'
gem 'distributed-press-api-client', '~> 0.2.2'
gem 'njalla-api-client'
gem 'distributed-press-api-client', '~> 0.2.3'
gem 'njalla-api-client', '~> 0.2.0'
gem 'email_address', git: 'https://github.com/fauno/email_address', branch: 'i18n'
gem 'exception_notification'
gem 'fast_blank'

View file

@ -387,10 +387,11 @@ GEM
nokogiri (1.12.5-x86_64-linux-musl)
mini_portile2 (~> 2.6.1)
racc (~> 1.4)
njalla-api-client (0.1.0)
njalla-api-client (0.2.0)
dry-schema
httparty (~> 0.18)
orm_adapter (0.5.0)
pairing_heap (3.0.0)
parallel (1.21.0)
parser (3.0.2.0)
ast (~> 2.4.1)
@ -481,6 +482,10 @@ GEM
actionpack (>= 5.0)
railties (>= 5.0)
rexml (3.2.5)
rgl (0.6.2)
pairing_heap (>= 0.3.0)
rexml (~> 3.2, >= 3.2.4)
stream (~> 0.5.3)
rouge (3.26.1)
rubocop (1.23.0)
parallel (~> 1.10)
@ -548,6 +553,7 @@ GEM
sprockets (>= 3.0.0)
sqlite3 (1.4.2-x86_64-linux-musl)
stackprof (0.2.17-x86_64-linux-musl)
stream (0.5.5)
sucker_punch (3.0.1)
concurrent-ruby (~> 1.0)
sutty-archives (2.5.4)
@ -616,7 +622,7 @@ DEPENDENCIES
devise
devise-i18n
devise_invitable
distributed-press-api-client (~> 0.2.2)
distributed-press-api-client (~> 0.2.3)
dotenv-rails
down
ed25519
@ -666,6 +672,7 @@ DEPENDENCIES
rails_warden
redis
redis-rails
rgl
rollups!
rubocop-rails
rubyzip

View file

@ -8,3 +8,4 @@ prometheus: bundle exec prometheus_exporter -b 0.0.0.0 --prefix "sutty_"
distributed_press_tokens_renew: bundle exec rake distributed_press:tokens:renew
cleanup: bundle exec rake cleanup:everything
stats: bundle exec rake stats:process_all
distributed_press_renew_tokens: bundle exec rake distributed_press:tokens:renew

View file

@ -28,8 +28,6 @@ class SitesController < ApplicationController
@site = Site.new
authorize @site
@site.deploys.build type: 'DeployLocal'
end
def create

View file

@ -30,79 +30,90 @@ class DeployJob < ApplicationJob
return
end
@deployed = {}
@site.update status: 'building'
# Asegurarse que DeployLocal sea el primero!
@deployed = {
deploy_local: {
status: deploy_locally,
seconds: deploy_local.build_stats.last.seconds,
size: deploy_local.size,
urls: [deploy_local.url]
}
}
@site.deployment_list.each do |d|
begin
raise DeployException, 'Una dependencia falló' if failed_dependencies? d
# No es opcional
unless @deployed[:deploy_local][:status]
# Hacer fallar la tarea
raise DeployException, "#{@site.name}: Falló la compilación"
status = d.deploy(output: @output)
seconds = d.build_stats.last.try(:seconds) || 0
size = d.size
urls = d.respond_to?(:urls) ? d.urls : [d.url].compact
rescue StandardError => e
status = false
seconds ||= 0
size ||= 0
# XXX: Hace que se vea la tabla
urls ||= [nil]
notify_exception e, d
end
@deployed[d.type.underscore.to_sym] = {
status: status,
seconds: seconds,
size: size,
urls: urls
}
end
deploy_others
return unless @output
puts (Terminal::Table.new do |t|
t << (%w[type] + @deployed.values.first.keys)
t.add_separator
@deployed.each do |type, row|
t << ([type.to_s] + row.values)
end
end)
rescue DeployTimedOutException => e
notify_exception e
rescue DeployException => e
notify_exception e, deploy_local
ensure
@site&.update status: 'waiting'
if @site.present?
@site.update status: 'waiting'
notify_usuaries if notify
notify_usuaries if notify
puts "\a" if @output
end
end
end
# rubocop:enable Metrics/MethodLength
private
# Detecta si un método de publicación tiene dependencias fallidas
#
# @param :deploy [Deploy]
# @return [Boolean]
def failed_dependencies?(deploy)
failed_dependencies(deploy).present?
end
# Obtiene las dependencias fallidas de un deploy
#
# @param :deploy [Deploy]
# @return [Array]
def failed_dependencies(deploy)
deploy.class::DEPENDENCIES & (@deployed.reject do |_, v|
v[:status]
end.keys)
end
# @param :exception [StandardError]
# @param :deploy [Deploy]
def notify_exception(exception, deploy = nil)
data = {
site: @site.id,
deploy: deploy&.type,
log: deploy&.build_stats&.last&.log
log: deploy&.build_stats&.last&.log,
failed_dependencies: (failed_dependencies(deploy) if deploy)
}
ExceptionNotifier.notify_exception(exception, data: data)
end
def deploy_local
@deploy_local ||= @site.deploys.find_by(type: 'DeployLocal')
end
def deploy_locally
deploy_local.deploy(output: @output)
end
def deploy_others
@site.deploys.where.not(type: 'DeployLocal').find_each do |d|
begin
status = d.deploy(output: @output)
seconds = d.build_stats.last.try(:seconds)
rescue StandardError => e
status = false
seconds = 0
notify_exception e, d
end
@deployed[d.type.underscore.to_sym] = {
status: status,
seconds: seconds || 0,
size: d.size,
urls: d.respond_to?(:urls) ? d.urls : [d.url].compact
}
end
end
def notify_usuaries
@site.roles.where(rol: 'usuarie', temporal: false).pluck(:usuarie_id).each do |usuarie|
DeployMailer.with(usuarie: usuarie, site: @site.id)

View file

@ -1,6 +1,7 @@
# frozen_string_literal: true
require 'open3'
# Este modelo implementa los distintos tipos de alojamiento que provee
# Sutty.
#
@ -11,6 +12,9 @@ class Deploy < ApplicationRecord
belongs_to :site
has_many :build_stats, dependent: :destroy
DEPENDENCIES = []
SOFT_DEPENDENCIES = []
def deploy(**)
raise NotImplementedError
end
@ -96,6 +100,13 @@ class Deploy < ApplicationRecord
@local_env ||= {}
end
# Trae todas las dependencias
#
# @return [Array]
def self.all_dependencies
self::DEPENDENCIES | self::SOFT_DEPENDENCIES
end
private
# @param [String]

View file

@ -4,6 +4,8 @@
class DeployAlternativeDomain < Deploy
store :values, accessors: %i[hostname], coder: JSON
DEPENDENCIES = %i[deploy_local]
# Generar un link simbólico del sitio principal al alternativo
def deploy(**)
File.symlink?(destination) ||
@ -18,7 +20,11 @@ class DeployAlternativeDomain < Deploy
end
def destination
@destination ||= File.join(Rails.root, '_deploy', hostname.gsub(/\.\z/, ''))
@destination ||= File.join(Rails.root, '_deploy', fqdn)
end
def fqdn
hostname.gsub(/\.\z/, '')
end
def url

View file

@ -16,6 +16,9 @@ class DeployDistributedPress < Deploy
store :values, accessors: %i[hostname remote_site_id remote_info], coder: JSON
before_create :create_remote_site!, :create_njalla_records!
before_destroy :delete_remote_site!, :delete_njalla_records!
DEPENDENCIES = %i[deploy_local]
# Actualiza la información y luego envía los cambios
#
@ -28,11 +31,15 @@ class DeployDistributedPress < Deploy
time_start
create_remote_site! if remote_site_id.blank?
create_njalla_records! if remote_info['njalla'].blank?
create_njalla_records!
save
if remote_site_id.blank? || remote_info['njalla'].blank?
raise DeployJob::DeployException, ''
if remote_site_id.blank?
raise DeployJob::DeployException, 'El sitio no se creó en Distributed Press'
end
if create_njalla_records? && remote_info[:njalla].blank?
raise DeployJob::DeployException, 'No se pudieron crear los registros necesarios en Njalla'
end
site_client.tap do |c|
@ -45,10 +52,13 @@ class DeployDistributedPress < Deploy
end
end
update remote_info: c.show(publishing_site).to_h
status = c.publish(publishing_site, deploy_local.destination)
if status
self.remote_info[:distributed_press] = c.show(publishing_site).to_h
save
end
publisher.logger.close
stdout.join
end
@ -70,14 +80,26 @@ class DeployDistributedPress < Deploy
# Devuelve las URLs de todos los protocolos
def urls
remote_info[:links].values.map do |protocol|
protocol_urls + gateway_urls
end
private
def gateway_urls
remote_info.dig(:distributed_press, :links).values.map do |protocol|
[ protocol[:link], protocol[:gateway] ]
end.flatten.compact.select do |link|
link.include? '://'
end
end
private
def protocol_urls
remote_info.dig(:distributed_press, :protocols).select do |_, enabled|
enabled
end.map do |protocol, _|
"#{protocol}://#{site.hostname}"
end
end
# El cliente de la API
#
@ -86,7 +108,7 @@ class DeployDistributedPress < Deploy
#
# @return [DistributedPressPublisher]
def publisher
@publisher ||= DistributedPressPublisher.first
@publisher ||= DistributedPressPublisher.last
end
# El cliente para actualizar el sitio
@ -117,27 +139,33 @@ class DeployDistributedPress < Deploy
created_site = site_client.create(create_site)
self.remote_site_id = created_site[:id]
self.remote_info = created_site.to_h
self.remote_info ||= {}
self.remote_info[:distributed_press] = created_site.to_h
nil
rescue DistributedPress::V1::Error => e
ExceptionNotifier.notify_exception(e, data: { site: site.name })
ensure
nil
end
# Crea los registros en Njalla
#
# XXX: Esto depende de nuestro DNS actual, cuando lo migremos hay
# que eliminarlo.
#
# @return [nil]
def create_njalla_records!
# XXX: Esto depende de nuestro DNS actual, cuando lo migremos hay
# que eliminarlo.
unless site.name.end_with? '.'
self.remote_info['njalla'] = {}
self.remote_info['njalla']['a'] = njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h
self.remote_info['njalla']['ns'] = njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h
end
return unless create_njalla_records?
self.remote_info ||= {}
self.remote_info[:njalla] ||= {}
self.remote_info[:njalla][:a] ||= njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h
self.remote_info[:njalla][:cname] ||= njalla.add_record(name: "www.#{site.name}", type: 'CNAME', content: "#{Site.domain}.").to_h
self.remote_info[:njalla][:ns] ||= njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h
nil
rescue HTTParty::Error => e
ExceptionNotifier.notify_exception(e, data: { site: site.name })
self.remote_info['njalla'] = nil
self.remote_info.delete :njalla
ensure
nil
end
@ -152,15 +180,38 @@ class DeployDistributedPress < Deploy
nil
end
def delete_remote_site!
site_client.delete(publishing_site)
nil
rescue DistributedPress::V1::Error => e
ExceptionNotifier.notify_exception(e, data: { site: site.name })
nil
end
def delete_njalla_records!
return unless create_njalla_records?
%w[a ns cname].each do |type|
next if (id = remote_info.dig('njalla', type, 'id')).blank?
njalla.remove_record(id: id.to_i)
end
end
# Actualizar registros en Njalla
#
# @return [Njalla::V1::Domain]
def njalla
@njalla ||=
begin
client = Njalla::V1::Client.new(token: ENV['NJALLA_TOKEN'])
client = Njalla::V1::Client.new(token: Rails.application.credentials.njalla)
Njalla::V1::Domain.new(domain: Site.domain, client: client)
end
end
# Detecta si tenemos que crear registros en Njalla
def create_njalla_records?
!site.name.end_with?('.')
end
end

View file

@ -0,0 +1,34 @@
# frozen_string_literal: true
class DeployFullRsync < DeployRsync
SOFT_DEPENDENCIES = %i[
deploy_alternative_domain
deploy_localized_domain
deploy_hidden_service
deploy_www
]
# Sincroniza las ubicaciones alternativas también, ignorando las que
# todavía no se generaron. Solo falla si ningún sitio fue
# sincronizado o si alguna sincronización falló.
#
# @param :output [Boolean]
# @return [Boolean]
def rsync(output: false)
result =
self.class.all_dependencies.map(&:to_s).map(&:classify).map do |dependency|
site.deploys.where(type: dependency).find_each.map do |deploy|
next unless File.exist? deploy.destination
run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape deploy.destination} #{Shellwords.escape destination}), output: output
rescue StandardError
end
end.flatten.compact
result.present? && result.all?
end
def url
"https://#{user_host.last}/"
end
end

View file

@ -2,14 +2,10 @@
# Genera una versión onion
class DeployHiddenService < DeployWww
def deploy(**)
return true if fqdn.blank?
super
end
def fqdn
values[:onion]
values[:onion].tap do |onion|
raise ArgumentError, 'Aun no se generó la dirección .onion' if onion.blank?
end
end
def url

View file

@ -84,7 +84,8 @@ class DeployLocal < Deploy
'AIRBRAKE_PROJECT_KEY' => site.airbrake_api_key,
'JEKYLL_ENV' => Rails.env,
'LANG' => ENV['LANG'],
'YARN_CACHE_FOLDER' => yarn_cache_dir
'YARN_CACHE_FOLDER' => yarn_cache_dir,
'GEMS_SOURCE' => ENV['GEMS_SOURCE']
})
end
@ -116,6 +117,14 @@ class DeployLocal < Deploy
run %(gem install bundler --no-document), output: output
end
def pnpm_lock
File.join(site.path, 'pnpm-lock.yaml')
end
def pnpm_lock?
File.exist? pnpm_lock
end
# Corre yarn dentro del repositorio
def yarn(output: false)
return true unless yarn_lock?
@ -152,6 +161,7 @@ class DeployLocal < Deploy
# Consigue todas las variables de entorno configuradas por otros
# deploys.
#
# @deprecated Solo tenía sentido para Distributed Press v0
# @return [Hash]
def extra_env
@extra_env ||=

View file

@ -6,6 +6,8 @@
# XXX: La plantilla tiene que soportar esto con el plugin
# jekyll-private-data
class DeployPrivate < DeployLocal
DEPENDENCIES = %i[deploy_local]
# No es necesario volver a instalar dependencias
def deploy(output: false)
jekyll_build(output: output)

View file

@ -3,7 +3,9 @@
# Sincroniza sitios a servidores remotos usando Rsync. El servidor
# remoto tiene que tener rsync instalado.
class DeployRsync < Deploy
store :values, accessors: %i[destination host_keys], coder: JSON
store :values, accessors: %i[hostname destination host_keys], coder: JSON
DEPENDENCIES = %i[deploy_local deploy_zip]
def deploy(output: false)
ssh? && rsync(output: output)
@ -23,6 +25,11 @@ class DeployRsync < Deploy
end
end
# @return [String]
def url
"https://#{hostname}/"
end
private
# Verificar la conexión SSH implementando Trust On First Use
@ -83,8 +90,8 @@ class DeployRsync < Deploy
# Sincroniza hacia el directorio remoto
#
# @return [Boolean]
def rsync(output: output)
run %(rsync -aviH --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/), output: output
def rsync(output: false)
run %(rsync -aviH --delete-after --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/), output: output
end
# El origen es el destino de la compilación

View file

@ -4,9 +4,13 @@
class DeployWww < Deploy
store :values, accessors: %i[], coder: JSON
DEPENDENCIES = %i[deploy_local]
before_destroy :remove_destination!
def deploy(**)
def deploy(output: false)
puts "Creando symlink #{site.hostname} => #{destination}" if output
File.symlink?(destination) ||
File.symlink(site.hostname, destination).zero?
end
@ -28,7 +32,7 @@ class DeployWww < Deploy
end
def url
"https://www.#{site.hostname}/"
"https://#{fqdn}/"
end
private

View file

@ -8,28 +8,49 @@ require 'zip'
class DeployZip < Deploy
store :values, accessors: %i[], coder: JSON
DEPENDENCIES = %i[deploy_local]
# Una vez que el sitio está generado, tomar todos los archivos y
# y generar un zip accesible públicamente.
#
# rubocop:disable Metrics/MethodLength
def deploy(**)
def deploy(output: false)
FileUtils.rm_f path
time_start
Dir.chdir(destination) do
Zip::File.open(path, Zip::File::CREATE) do |z|
Dir.glob('./**/**').each do |f|
File.directory?(f) ? z.mkdir(f) : z.add(f, f)
Zip::File.open(path, Zip::File::CREATE) do |zip|
Dir.glob(File.join(destination, '**', '**')).each do |file|
entry = Pathname.new(file).relative_path_from(destination).to_s
if File.directory? file
log "Creando directorio #{entry}", output
zip.mkdir(entry)
else
log "Comprimiendo #{entry}", output
zip.add(entry, file)
end
end
end
time_stop
build_stats.create action: 'zip',
seconds: time_spent_in_seconds,
bytes: size
File.exist?(path).tap do |status|
build_stats.create action: 'zip',
seconds: time_spent_in_seconds,
bytes: size,
log: @log.join("\n"),
status: status
end
rescue Zip::Error => e
ExceptionNotifier.notify_exception(e, data: { site: site.name })
File.exist? path
build_stats.create action: 'zip',
seconds: 0,
bytes: 0,
log: @log.join("\n"),
status: false
false
end
# rubocop:enable Metrics/MethodLength
@ -41,8 +62,11 @@ class DeployZip < Deploy
File.size path
end
# @return [String]
def destination
File.join(Rails.root, '_deploy', site.hostname)
Rails.root.join('_deploy', site.hostname).realpath.to_s
rescue Errno::ENOENT
Rails.root.join('_deploy', site.hostname).to_s
end
def file
@ -56,4 +80,15 @@ class DeployZip < Deploy
def path
File.join(destination, file)
end
private
# @param :line [String]
# @param :output [Boolean]
def log(line, output)
@log ||= []
@log << line
puts line if output
end
end

View file

@ -7,6 +7,7 @@ class Site < ApplicationRecord
include Site::Forms
include Site::FindAndReplace
include Site::Api
include Site::DeployDependencies
include Site::BuildStats
include Tienda

View file

@ -0,0 +1,38 @@
# frozen_string_literal: true
require 'rgl/adjacency'
require 'rgl/topsort'
class Site
module DeployDependencies
extend ActiveSupport::Concern
included do
# Genera un grafo dirigido de todos los métodos de publicación
#
# @return [RGL::DirectedAdjacencyGraph]
def deployment_graph
@deployment_graph ||= RGL::DirectedAdjacencyGraph.new.tap do |graph|
deploys.each do |deploy|
graph.add_vertex deploy
end
deploys.each do |deploy|
deploy.class.all_dependencies.each do |dependency|
deploys.where(type: dependency.to_s.classify).each do |deploy_dependency|
graph.add_edge deploy_dependency, deploy
end
end
end
end
end
# Devuelve una lista ordenada de todos los métodos de publicación
#
# @return [Array]
def deployment_list
@deployment_list ||= deployment_graph.topsort_iterator.to_a
end
end
end
end

View file

@ -14,6 +14,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do
self.site = Site.new params
add_role temporal: false, rol: 'usuarie'
site.deploys.build type: 'DeployLocal'
sync_nodes
I18n.with_locale(usuarie.lang.to_sym || I18n.default_locale) do
@ -217,7 +218,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do
# Crea los deploys necesarios para sincronizar a otros nodos de Sutty
def sync_nodes
Rails.application.nodes.each do |node|
site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}")
site.deploys.build(type: 'DeployFullRsync', destination: "sutty@#{node}:")
end
end

View file

@ -0,0 +1 @@
-# nada

View file

@ -17,7 +17,8 @@
= sanitize_markdown t('.help', public_url: deploy.object.site.url),
tags: %w[p strong em a]
- if deploy.object.fqdn
- begin
= sanitize_markdown t('.help_2', url: deploy.object.url),
tags: %w[p strong em a]
- rescue ArgumentError
%hr/

View file

@ -0,0 +1 @@
-# nada

View file

@ -160,9 +160,6 @@
= f.fields_for :deploys do |deploy|
= render "deploys/#{deploy.object.type.underscore}",
deploy: deploy, site: site
- else
= f.fields_for :deploys do |deploy|
= deploy.hidden_field :type
.form-group
= f.submit submit, class: 'btn btn-lg btn-block'

View file

@ -133,6 +133,14 @@ en:
title: Synchronize to backup server
success: Success!
error: Error
deploy_full_rsync:
title: Synchronize to another Sutty node
success: Success!
error: Error
deploy_distributed_press:
title: Distributed Web
success: Success!
error: Error
help: You can contact us by replying to this e-mail
maintenance_mailer:
notice:

View file

@ -133,6 +133,14 @@ es:
title: Sincronizar al servidor alternativo
success: ¡Éxito!
error: Hubo un error
deploy_full_rsync:
title: Sincronizar a otro nodo de Sutty
success: ¡Éxito!
error: Hubo un error
deploy_distributed_press:
title: Web distribuida
success: ¡Éxito!
error: Hubo un error
help: Por cualquier duda, responde este correo para contactarte con nosotres.
maintenance_mailer:
notice:

View file

@ -0,0 +1,37 @@
# frozen_string_literal: true
# Cambia todos los DeployRsync propios de Sutty a DeployFullRsync que se
# encarga de sincronizar todo.
class RenameDeployRsyncToDeployFullRsync < ActiveRecord::Migration[6.1]
def up
DeployRsync.all.find_each do |deploy|
dest = deploy.destination.split(':', 2).first
next unless nodes.include? dest
deploy.destination = "#{dest}:"
deploy.type = 'DeployFullRsync'
deploy.save
end
end
def down
DeployFullRsync.all.find_each do |deploy|
next unless nodes.include? deploy.destination.split(':', 2).first
deploy.destination = "#{deploy.destination}#{deploy.site.hostname}"
deploy.type = 'DeployRsync'
deploy.save
end
end
private
def nodes
@nodes ||= Rails.application.nodes.map do |node|
"sutty@#{node}"
end
end
end

View file

@ -18,3 +18,8 @@ check program stats
with path "/usr/bin/foreman run -f /srv/Procfile -d /srv stats" as uid "rails" gid "www-data"
every "0 1 * * *"
if status != 0 then alert
check program distributed_press_tokens_renew
with path "/usr/bin/foreman run -f /srv/Procfile -d /srv distributed_press_tokens_renew" as uid "rails" gid "www-data"
every "0 3 * * *"
if status != 0 then alert