Merge branch 'distributed-press' into 'rails'

soportar distributed press apiv0 #7839

Closes #10509, #10506, and #10467

See merge request sutty/sutty!107

Dockerfile

@@ -14,6 +14,7 @@ RUN apk add --no-cache libxslt libxml2 postgresql-libs libssh2 \
 
 RUN gem install --no-document --no-user-install foreman
 RUN wget https://github.com/jgm/pandoc/releases/download/${PANDOC_VERSION}/pandoc-${PANDOC_VERSION}-linux-amd64.tar.gz -O - | tar --strip-components 1 -xvzf - pandoc-${PANDOC_VERSION}/bin/pandoc && mv /bin/pandoc /usr/bin/pandoc
+RUN apk add npm && npm install -g pnpm@~7 && apk del npm
 
 COPY ./monit.conf /etc/monit.d/sutty.conf
 

Gemfile

@@ -38,6 +38,8 @@ gem 'commonmarker'
 gem 'devise'
 gem 'devise-i18n'
 gem 'devise_invitable'
+gem 'distributed-press-api-client', '~> 0.2.2'
+gem 'njalla-api-client'
 gem 'email_address', git: 'https://github.com/fauno/email_address', branch: 'i18n'
 gem 'exception_notification'
 gem 'fast_blank'

Gemfile.lock

@@ -115,6 +115,7 @@ GEM
       xpath (>= 2.0, < 4.0)
     chartkick (4.1.2)
     childprocess (4.1.0)
+    climate_control (1.2.0)
     coderay (1.1.3)
     colorator (1.1.0)
     commonmarker (0.21.2-x86_64-linux-musl)
@@ -153,12 +154,45 @@ GEM
     devise_invitable (2.0.5)
       actionmailer (>= 5.0)
       devise (>= 4.6)
+    distributed-press-api-client (0.2.2)
+      addressable (~> 2.3, >= 2.3.0)
+      climate_control
+      dry-schema
+      httparty (~> 0.18)
+      json (~> 2.1, >= 2.1.0)
+      jwt (~> 2.6.0)
     dotenv (2.7.6)
     dotenv-rails (2.7.6)
       dotenv (= 2.7.6)
       railties (>= 3.2)
     down (5.2.4)
       addressable (~> 2.8)
+    dry-configurable (1.0.1)
+      dry-core (~> 1.0, < 2)
+      zeitwerk (~> 2.6)
+    dry-core (1.0.0)
+      concurrent-ruby (~> 1.0)
+      zeitwerk (~> 2.6)
+    dry-inflector (1.0.0)
+    dry-initializer (3.1.1)
+    dry-logic (1.5.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.0, < 2)
+      zeitwerk (~> 2.6)
+    dry-schema (1.13.0)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 1.0, >= 1.0.1)
+      dry-core (~> 1.0, < 2)
+      dry-initializer (~> 3.0)
+      dry-logic (>= 1.5, < 2)
+      dry-types (>= 1.7, < 2)
+      zeitwerk (~> 2.6)
+    dry-types (1.7.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.0, < 2)
+      dry-inflector (~> 1.0, < 2)
+      dry-logic (>= 1.4, < 2)
+      zeitwerk (~> 2.6)
     ed25519 (1.2.4-x86_64-linux-musl)
     em-websocket (0.5.3)
       eventmachine (>= 0.12.9)
@@ -216,8 +250,8 @@ GEM
       thor
     hiredis (0.6.3-x86_64-linux-musl)
     http_parser.rb (0.8.0-x86_64-linux-musl)
-    httparty (0.18.1)
+    httparty (0.21.0)
-      mime-types (~> 3.0)
+      mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
     i18n (1.8.11)
       concurrent-ruby (~> 1.0)
@@ -292,6 +326,7 @@ GEM
     jekyll-write-and-commit-changes (0.2.1)
       jekyll (~> 4)
       rugged (~> 1)
+    jwt (2.6.0)
     kaminari (1.2.1)
       activesupport (>= 4.1.0)
       kaminari-actionview (= 1.2.1)
@@ -352,6 +387,9 @@ GEM
     nokogiri (1.12.5-x86_64-linux-musl)
       mini_portile2 (~> 2.6.1)
       racc (~> 1.4)
+    njalla-api-client (0.1.0)
+      dry-schema
+      httparty (~> 0.18)
     orm_adapter (0.5.0)
     parallel (1.21.0)
     parser (3.0.2.0)
@@ -578,6 +616,7 @@ DEPENDENCIES
   devise
   devise-i18n
   devise_invitable
+  distributed-press-api-client (~> 0.2.2)
   dotenv-rails
   down
   ed25519
@@ -612,6 +651,7 @@ DEPENDENCIES
   mini_magick
   mobility
   net-ssh
+  njalla-api-client
   nokogiri
   pg
   pg_search

Procfile

@@ -1,2 +1,10 @@
+migrate: bundle exec rake db:prepare db:seed
+sutty: bundle exec puma config.ru
+blazer_5m: bundle exec rake blazer:run_checks SCHEDULE="5 minutes"
+blazer_1h: bundle exec rake blazer:run_checks SCHEDULE="1 hour"
+blazer_1d: bundle exec rake blazer:run_checks SCHEDULE="1 day"
+blazer: bundle exec rake blazer:send_failing_checks
+prometheus: bundle exec prometheus_exporter -b 0.0.0.0 --prefix "sutty_"
+distributed_press_tokens_renew: bundle exec rake distributed_press:tokens:renew
 cleanup: bundle exec rake cleanup:everything
 stats: bundle exec rake stats:process_all

app/jobs/renew_distributed_press_tokens_job.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Renueva los tokens de Distributed Press antes que se venzan,
+# activando los callbacks que hacen que se refresque el token.
+class RenewDistributedPressTokensJob < ApplicationJob
+  # Renueva todos los tokens a punto de vencer o informa el error sin
+  # detener la tarea si algo pasa.
+  def perform
+    DistributedPressPublisher.with_about_to_expire_tokens.find_each do |publisher|
+      publisher.touch
+    rescue DistributedPress::V1::Error => e
+      data = { instance: publisher.instance, expires_at: publisher.client.token.expires_at }
+
+      ExceptionNotifier.notify_exception(e, data: data)
+    end
+  end
+end

app/models/deploy.rb

@@ -89,6 +89,13 @@ class Deploy < ApplicationRecord
     r&.success?
   end
 
+  # Variables de entorno
+  #
+  # @return [Hash]
+  def local_env
+    @local_env ||= {}
+  end
+
   private
 
   # @param [String]
@@ -96,4 +103,12 @@ class Deploy < ApplicationRecord
   def readable_cmd(cmd)
     cmd.split(' -', 2).first.tr(' ', '_')
   end
+
+  def deploy_local
+    @deploy_local ||= site.deploys.find_by(type: 'DeployLocal')
+  end
+
+  def non_local_deploys
+    @non_local_deploys ||= site.deploys.where.not(type: 'DeployLocal')
+  end
 end

app/models/deploy_distributed_press.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require 'distributed_press/v1/client/site'
+require 'njalla/v1'
+
+# Soportar Distributed Press APIv1
+#
+# Usa tokens de publicación efímeros para todas las acciones.
+#
+# Al ser creado, genera el sitio en la instancia de Distributed Press
+# configurada y almacena el ID.
+#
+# Al ser publicado, envía los archivos en un tarball y actualiza la
+# información.
+class DeployDistributedPress < Deploy
+  store :values, accessors: %i[hostname remote_site_id remote_info], coder: JSON
+
+  before_create :create_remote_site!, :create_njalla_records!
+
+  # Actualiza la información y luego envía los cambios
+  #
+  # @param :output [Bool]
+  # @return [Bool]
+  def deploy
+    status = false
+    log = []
+
+    time_start
+
+    create_remote_site! if remote_site_id.blank?
+    create_njalla_records! if remote_info['njalla'].blank?
+    save
+
+    if remote_site_id.blank? || remote_info['njalla'].blank?
+      raise DeployJob::DeployException, ''
+    end
+
+    site_client.tap do |c|
+      stdout = Thread.new(publisher.logger_out) do |io|
+        until io.eof?
+          line = io.gets
+
+          puts line if output
+          log << line
+        end
+      end
+
+      update remote_info: c.show(publishing_site).to_h
+
+      status = c.publish(publishing_site, deploy_local.destination)
+
+      publisher.logger.close
+      stdout.join
+    end
+
+    time_stop
+
+    create_stat! status, log.join
+
+    status
+  end
+
+  def limit; end
+
+  def size
+    deploy_local.size
+  end
+
+  def destination; end
+
+  # Devuelve las URLs de todos los protocolos
+  def urls
+    remote_info[:links].values.map do |protocol|
+      [ protocol[:link], protocol[:gateway] ]
+    end.flatten.compact.select do |link|
+      link.include? '://'
+    end
+  end
+
+  private
+
+  # El cliente de la API
+  #
+  # TODO: cuando soportemos más, tiene que haber una relación entre
+  # DeployDistributedPress y DistributedPressPublisher.
+  #
+  # @return [DistributedPressPublisher]
+  def publisher
+    @publisher ||= DistributedPressPublisher.first
+  end
+
+  # El cliente para actualizar el sitio
+  #
+  # @return [DistributedPress::V1::Client::Site]
+  def site_client
+    DistributedPress::V1::Client::Site.new(publisher.client)
+  end
+
+  # Genera el esquema de datos para poder publicar el sitio
+  #
+  # @return [DistributedPress::V1::Schemas::PublishingSite]
+  def publishing_site
+    DistributedPress::V1::Schemas::PublishingSite.new.call(id: remote_site_id)
+  end
+
+  # Genera el esquema de datos para crear el sitio
+  #
+  # @return [DistributedPressPublisher::V1::Schemas::NewSite]
+  def create_site
+    DistributedPress::V1::Schemas::NewSite.new.call(domain: hostname, protocols: { http: true, ipfs: true, hyper: true })
+  end
+
+  # Crea el sitio en la instancia con el hostname especificado
+  #
+  # @return [nil]
+  def create_remote_site!
+    created_site = site_client.create(create_site)
+
+    self.remote_site_id = created_site[:id]
+    self.remote_info = created_site.to_h
+  rescue DistributedPress::V1::Error => e
+    ExceptionNotifier.notify_exception(e, data: { site: site.name })
+  ensure
+    nil
+  end
+
+  # Crea los registros en Njalla
+  #
+  # @return [nil]
+  def create_njalla_records!
+    # XXX: Esto depende de nuestro DNS actual, cuando lo migremos hay
+    # que eliminarlo.
+    unless site.name.end_with? '.'
+      self.remote_info['njalla'] = {}
+      self.remote_info['njalla']['a'] = njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h
+      self.remote_info['njalla']['ns'] = njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h
+    end
+  rescue HTTParty::Error => e
+    ExceptionNotifier.notify_exception(e, data: { site: site.name })
+    self.remote_info['njalla'] = nil
+  ensure
+    nil
+  end
+
+  # Registra lo que sucedió
+  #
+  # @param status [Bool]
+  # @param log [String]
+  # @return [nil]
+  def create_stat!(status, log)
+    build_stats.create action: publisher.to_s,log: log, seconds: time_spent_in_seconds, bytes: size, status: status
+    nil
+  end
+
+  # Actualizar registros en Njalla
+  #
+  # @return [Njalla::V1::Domain]
+  def njalla
+    @njalla ||=
+      begin
+        client = Njalla::V1::Client.new(token: ENV['NJALLA_TOKEN'])
+
+        Njalla::V1::Domain.new(domain: Site.domain, client: client)
+      end
+  end
+end

app/models/deploy_local.rb

@@ -15,6 +15,7 @@ class DeployLocal < Deploy
   def deploy(output: false)
     return false unless mkdir
     return false unless yarn(output: output)
+    return false unless pnpm(output: output)
     return false unless bundle(output: output)
 
     jekyll_build(output: output)
@@ -67,27 +68,34 @@ class DeployLocal < Deploy
   end
 
   # Un entorno que solo tiene lo que necesitamos
+  #
+  # @return [Hash]
   def env
     # XXX: This doesn't support Windows paths :B
-    paths = [File.dirname(`which bundle`), '/usr/bin', '/bin']
+    paths = [File.dirname(`which bundle`), '/usr/local/bin', '/usr/bin', '/bin']
 
-    {
+    # Las variables de entorno extra no pueden superponerse al local.
-      'HOME' => home_dir,
+    extra_env.merge({
-      'PATH' => paths.join(':'),
+                      'HOME' => home_dir,
-      'SPREE_API_KEY' => site.tienda_api_key,
+                      'PATH' => paths.join(':'),
-      'SPREE_URL' => site.tienda_url,
+                      'SPREE_API_KEY' => site.tienda_api_key,
-      'AIRBRAKE_PROJECT_ID' => site.id.to_s,
+                      'SPREE_URL' => site.tienda_url,
-      'AIRBRAKE_PROJECT_KEY' => site.airbrake_api_key,
+                      'AIRBRAKE_PROJECT_ID' => site.id.to_s,
-      'JEKYLL_ENV' => Rails.env,
+                      'AIRBRAKE_PROJECT_KEY' => site.airbrake_api_key,
-      'LANG' => ENV['LANG'],
+                      'JEKYLL_ENV' => Rails.env,
-      'YARN_CACHE_FOLDER' => yarn_cache_dir
+                      'LANG' => ENV['LANG'],
-    }
+                      'YARN_CACHE_FOLDER' => yarn_cache_dir
+                    })
   end
 
   def yarn_cache_dir
     Rails.root.join('_yarn_cache').to_s
   end
 
+  def pnpm_cache_dir
+    Rails.root.join('_pnpm_cache').to_s
+  end
+
   def yarn_lock
     File.join(site.path, 'yarn.lock')
   end
@@ -96,6 +104,14 @@ class DeployLocal < Deploy
     File.exist? yarn_lock
   end
 
+  def pnpm_lock
+    File.join(site.path, 'pnpm-lock.yaml')
+  end
+
+  def pnpm_lock?
+    File.exist? pnpm_lock
+  end
+
   def gem(output: false)
     run %(gem install bundler --no-document), output: output
   end
@@ -107,6 +123,13 @@ class DeployLocal < Deploy
     run 'yarn install --production', output: output
   end
 
+  def pnpm(output: false)
+    return true unless pnpm_lock?
+
+    run %(pnpm config set store-dir "#{pnpm_cache_dir}"), output: output
+    run 'pnpm install --production', output: output
+  end
+
   def bundle(output: false)
     run %(bundle install --no-cache --path="#{gems_dir}" --clean --without test development), output: output
   end
@@ -125,4 +148,17 @@ class DeployLocal < Deploy
   def remove_destination!
     FileUtils.rm_rf destination
   end
+
+  # Consigue todas las variables de entorno configuradas por otros
+  # deploys.
+  #
+  # @return [Hash]
+  def extra_env
+    @extra_env ||=
+      non_local_deploys.reduce({}) do |extra_env, deploy|
+        extra_env.tap do |e|
+          e.merge! deploy.local_env
+        end
+      end
+  end
 end

app/models/distributed_press_publisher.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require 'distributed_press/v1'
+
+# Almacena el token de autenticación y la URL, por ahora solo vamos
+# a tener uno, pero queda abierta la posibilidad de agregar más.
+class DistributedPressPublisher < ApplicationRecord
+  # Cifrar la información del token en la base de datos
+  has_encrypted :token
+
+  # La salida del log
+  #
+  # @return [IO]
+  attr_reader :logger_out
+
+  # La instancia es única
+  validates_uniqueness_of :instance
+
+  # El token es necesario
+  validates_presence_of :token
+
+  # Mantener la fecha de vencimiento actualizada
+  before_save :update_expires_at_from_token!, :update_token_from_client!
+
+  # Devuelve todos los tokens que vencen en una hora
+  scope :with_about_to_expire_tokens, lambda {
+    where('expires_at > ? and expires_at < ?', Time.now, Time.now + 1.hour)
+  }
+
+  # Instancia un cliente de Distributed Press a partir del token.  Al
+  # cargar un token a punto de vencer se renueva automáticamente.
+  #
+  # @return [DistributedPress::V1::Client]
+  def client
+    @client ||= DistributedPress::V1::Client.new(url: instance, token: token, logger: logger)
+  end
+
+  # @return [String]
+  def to_s
+    "Distributed Press <#{instance}>"
+  end
+
+  # Devuelve el hostname de la instancia
+  #
+  # @return [String]
+  def hostname
+    @hostname ||= URI.parse(instance).hostname
+  end
+
+  # @return [Logger]
+  def logger
+    @logger ||=
+      begin
+        @logger_out, @logger_in = IO.pipe
+        ::Logger.new @logger_in, formatter: formatter
+      end
+  end
+
+  private
+
+  def formatter
+    @formatter ||= lambda do |_, _, _, msg|
+      "#{msg}\n"
+    end
+  end
+
+  # Actualiza o desactiva la fecha de vencimiento a partir de la
+  # información del token.
+  #
+  # @return [nil]
+  def update_expires_at_from_token!
+    self.expires_at = client.token.forever? ? nil : client.token.expires_at
+    nil
+  end
+
+  # Actualiza el token a partir del cliente, que ya actualiza el token
+  # automáticamente.
+  #
+  # @return [nil]
+  def update_token_from_client!
+    self.token = client.token.to_s
+    nil
+  end
+end

app/models/site.rb

@@ -17,7 +17,7 @@ class Site < ApplicationRecord
 
   # TODO: Hacer que los diferentes tipos de deploy se auto registren
   # @see app/services/site_service.rb
-  DEPLOYS = %i[local private www zip hidden_service].freeze
+  DEPLOYS = %i[local private www zip hidden_service distributed_press].freeze
 
   validates :name, uniqueness: true, hostname: {
     allow_root_label: true

app/views/deploys/_deploy_distributed_press.haml

@@ -0,0 +1,21 @@
+-# Publicar a la web distribuida
+
+.row
+  .col
+    = deploy.hidden_field :id
+    = deploy.hidden_field :type
+    .custom-control.custom-switch
+      -#
+        El checkbox invierte la lógica de destrucción porque queremos
+        crear el deploy si está activado y destruirlo si está
+        desactivado.
+      = deploy.check_box :_destroy,
+        { checked: deploy.object.persisted?, class: 'custom-control-input' },
+        '0', '1'
+      = deploy.label :_destroy, class: 'custom-control-label' do
+        %h3= t('.title')
+    = sanitize_markdown t('.help', public_url: deploy.object.site.url),
+      tags: %w[p strong em a]
+
+
+%hr/

config/locales/en.yml

@@ -114,6 +114,10 @@ en:
         title: Alternative domain name
         success: Success!
         error: Error
+      deploy_distributed_press:
+        title: Distributed Web
+        success: Success!
+        error: Error
       deploy_reindex:
         title: Reindex
         success: Success!
@@ -272,6 +276,22 @@ en:
 
         Only accessible through [Tor
         Browser](https://www.torproject.org/download/)
+    deploy_distributed_press:
+      title: 'Publish to the distributed Web'
+      help: |
+        Make your site available through peer-to-peer protocols,
+        Inter-Planetary File System (IPFS), Hypercore, and via
+        BitTorrent, so your site is more resilient and can be available
+        offline, including in community mesh networks.
+
+        **Important:** Only use this option if you would like your data
+        to be permanently available. If you decide to undo this
+        selection, a cleared version of the site will be shared in its
+        place. However, it is possible that nodes on the distributed
+        storage network may continue retaining copies of the data
+        indefinitely.
+
+        [Learn more](https://ffdweb.org/building-distributed-press-a-publishing-tool-for-the-decentralized-web/)
   stats:
     index:
       title: Statistics

config/locales/es.yml

@@ -114,6 +114,10 @@ es:
         title: Dominio alternativo
         success: ¡Éxito!
         error: Hubo un error
+      deploy_distributed_press:
+        title: Web distribuida
+        success: ¡Éxito!
+        error: Hubo un error
       deploy_reindex:
         title: Reindexación
         success: ¡Éxito!
@@ -277,6 +281,22 @@ es:
 
         Sólo será accesible a través del [Navegador
         Tor](https://www.torproject.org/es/download/).
+    deploy_distributed_press:
+      title: 'Publicar a la Web distribuida'
+      help: |
+        Utiliza protocolos de pares, Inter-Planetary File System (IPFS),
+        Hypercore y torrents, para que tu sitio sea más resiliente y
+        esté disponible _offline_, inclusive en redes _mesh_
+        comunitarias.
+
+        **Importante:** Sólo usa esta opción si te parece correcto que
+        tu contenido esté disponible permanentemente. Cuando elijas
+        des-hacer esta acción, una versión "vacía" del sitio será
+        compartida en su lugar. Sin embargo, es posible que algunos
+        nodos en la red de almacenamiento distribuida puedan retener
+        copias de tu contenido indefinidamente.
+
+        [Saber más (en inglés)](https://ffdweb.org/building-distributed-press-a-publishing-tool-for-the-decentralized-web/)
   stats:
     index:
       title: Estadísticas

db/migrate/20230119165420_create_distributed_press_publisher.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Crea la tabla de publishers de Distributed Press que contiene las
+# instancias y tokens
+class CreateDistributedPressPublisher < ActiveRecord::Migration[6.1]
+  def change
+    create_table :distributed_press_publishers do |t|
+      t.timestamps
+      t.string :instance, unique: true
+      t.text :token_ciphertext, null: false
+      t.datetime :expires_at, null: true
+    end
+  end
+end

lib/tasks/distributed_press.rake

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+namespace :distributed_press do
+  namespace :tokens do
+    desc 'Renew tokens'
+    task renew: :environment do
+      RenewDistributedPressTokensJob.perform_now
+    end
+  end
+end

monit.conf

@@ -4,6 +4,11 @@ check program cleanup
   every "0 3 1 * *"
   if status != 0 then alert
 
+check program distributed_press_tokens_renew
+  with path "/usr/bin/foreman run -f /srv/Procfile -d /srv distributed_press_tokens_renew" as uid "rails" gid "www-data"
+  every "0 3 * * *"
+  if status != 0 then alert
+
 check program access_logs
   with path "/srv/http/bin/access_logs" as uid "app" and gid "www-data"
   every "0 0 * * *"