diff --git a/Dockerfile b/Dockerfile index 342c2750..3da9ffab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ RUN apk add --no-cache libxslt libxml2 postgresql-libs libssh2 \ RUN gem install --no-document --no-user-install foreman RUN wget https://github.com/jgm/pandoc/releases/download/${PANDOC_VERSION}/pandoc-${PANDOC_VERSION}-linux-amd64.tar.gz -O - | tar --strip-components 1 -xvzf - pandoc-${PANDOC_VERSION}/bin/pandoc && mv /bin/pandoc /usr/bin/pandoc +RUN apk add npm && npm install -g pnpm@~7 && apk del npm COPY ./monit.conf /etc/monit.d/sutty.conf diff --git a/Gemfile b/Gemfile index f3799638..560401a5 100644 --- a/Gemfile +++ b/Gemfile @@ -38,6 +38,8 @@ gem 'commonmarker' gem 'devise' gem 'devise-i18n' gem 'devise_invitable' +gem 'distributed-press-api-client', '~> 0.2.2' +gem 'njalla-api-client' gem 'email_address', git: 'https://github.com/fauno/email_address', branch: 'i18n' gem 'exception_notification' gem 'fast_blank' diff --git a/Gemfile.lock b/Gemfile.lock index abaf45c9..3f8be3a1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -115,6 +115,7 @@ GEM xpath (>= 2.0, < 4.0) chartkick (4.1.2) childprocess (4.1.0) + climate_control (1.2.0) coderay (1.1.3) colorator (1.1.0) commonmarker (0.21.2-x86_64-linux-musl) @@ -153,12 +154,45 @@ GEM devise_invitable (2.0.5) actionmailer (>= 5.0) devise (>= 4.6) + distributed-press-api-client (0.2.2) + addressable (~> 2.3, >= 2.3.0) + climate_control + dry-schema + httparty (~> 0.18) + json (~> 2.1, >= 2.1.0) + jwt (~> 2.6.0) dotenv (2.7.6) dotenv-rails (2.7.6) dotenv (= 2.7.6) railties (>= 3.2) down (5.2.4) addressable (~> 2.8) + dry-configurable (1.0.1) + dry-core (~> 1.0, < 2) + zeitwerk (~> 2.6) + dry-core (1.0.0) + concurrent-ruby (~> 1.0) + zeitwerk (~> 2.6) + dry-inflector (1.0.0) + dry-initializer (3.1.1) + dry-logic (1.5.0) + concurrent-ruby (~> 1.0) + dry-core (~> 1.0, < 2) + zeitwerk (~> 2.6) + dry-schema (1.13.0) + concurrent-ruby (~> 1.0) + dry-configurable (~> 1.0, >= 1.0.1) + dry-core (~> 1.0, < 2) + dry-initializer (~> 3.0) + dry-logic (>= 1.5, < 2) + dry-types (>= 1.7, < 2) + zeitwerk (~> 2.6) + dry-types (1.7.0) + concurrent-ruby (~> 1.0) + dry-core (~> 1.0, < 2) + dry-inflector (~> 1.0, < 2) + dry-logic (>= 1.4, < 2) + zeitwerk (~> 2.6) ed25519 (1.2.4-x86_64-linux-musl) em-websocket (0.5.3) eventmachine (>= 0.12.9) @@ -216,8 +250,8 @@ GEM thor hiredis (0.6.3-x86_64-linux-musl) http_parser.rb (0.8.0-x86_64-linux-musl) - httparty (0.18.1) - mime-types (~> 3.0) + httparty (0.21.0) + mini_mime (>= 1.0.0) multi_xml (>= 0.5.2) i18n (1.8.11) concurrent-ruby (~> 1.0) @@ -292,6 +326,7 @@ GEM jekyll-write-and-commit-changes (0.2.1) jekyll (~> 4) rugged (~> 1) + jwt (2.6.0) kaminari (1.2.1) activesupport (>= 4.1.0) kaminari-actionview (= 1.2.1) @@ -352,6 +387,9 @@ GEM nokogiri (1.12.5-x86_64-linux-musl) mini_portile2 (~> 2.6.1) racc (~> 1.4) + njalla-api-client (0.1.0) + dry-schema + httparty (~> 0.18) orm_adapter (0.5.0) parallel (1.21.0) parser (3.0.2.0) @@ -578,6 +616,7 @@ DEPENDENCIES devise devise-i18n devise_invitable + distributed-press-api-client (~> 0.2.2) dotenv-rails down ed25519 @@ -612,6 +651,7 @@ DEPENDENCIES mini_magick mobility net-ssh + njalla-api-client nokogiri pg pg_search diff --git a/Procfile b/Procfile index 45fe1df7..25a1639d 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,10 @@ +migrate: bundle exec rake db:prepare db:seed +sutty: bundle exec puma config.ru +blazer_5m: bundle exec rake blazer:run_checks SCHEDULE="5 minutes" +blazer_1h: bundle exec rake blazer:run_checks SCHEDULE="1 hour" +blazer_1d: bundle exec rake blazer:run_checks SCHEDULE="1 day" +blazer: bundle exec rake blazer:send_failing_checks +prometheus: bundle exec prometheus_exporter -b 0.0.0.0 --prefix "sutty_" +distributed_press_tokens_renew: bundle exec rake distributed_press:tokens:renew cleanup: bundle exec rake cleanup:everything stats: bundle exec rake stats:process_all diff --git a/app/jobs/renew_distributed_press_tokens_job.rb b/app/jobs/renew_distributed_press_tokens_job.rb new file mode 100644 index 00000000..5664d9fa --- /dev/null +++ b/app/jobs/renew_distributed_press_tokens_job.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# Renueva los tokens de Distributed Press antes que se venzan, +# activando los callbacks que hacen que se refresque el token. +class RenewDistributedPressTokensJob < ApplicationJob + # Renueva todos los tokens a punto de vencer o informa el error sin + # detener la tarea si algo pasa. + def perform + DistributedPressPublisher.with_about_to_expire_tokens.find_each do |publisher| + publisher.touch + rescue DistributedPress::V1::Error => e + data = { instance: publisher.instance, expires_at: publisher.client.token.expires_at } + + ExceptionNotifier.notify_exception(e, data: data) + end + end +end diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 9d5c1d27..1c2683fc 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -89,6 +89,13 @@ class Deploy < ApplicationRecord r&.success? end + # Variables de entorno + # + # @return [Hash] + def local_env + @local_env ||= {} + end + private # @param [String] @@ -96,4 +103,12 @@ class Deploy < ApplicationRecord def readable_cmd(cmd) cmd.split(' -', 2).first.tr(' ', '_') end + + def deploy_local + @deploy_local ||= site.deploys.find_by(type: 'DeployLocal') + end + + def non_local_deploys + @non_local_deploys ||= site.deploys.where.not(type: 'DeployLocal') + end end diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb new file mode 100644 index 00000000..6940a83e --- /dev/null +++ b/app/models/deploy_distributed_press.rb @@ -0,0 +1,166 @@ +# frozen_string_literal: true + +require 'distributed_press/v1/client/site' +require 'njalla/v1' + +# Soportar Distributed Press APIv1 +# +# Usa tokens de publicación efímeros para todas las acciones. +# +# Al ser creado, genera el sitio en la instancia de Distributed Press +# configurada y almacena el ID. +# +# Al ser publicado, envía los archivos en un tarball y actualiza la +# información. +class DeployDistributedPress < Deploy + store :values, accessors: %i[hostname remote_site_id remote_info], coder: JSON + + before_create :create_remote_site!, :create_njalla_records! + + # Actualiza la información y luego envía los cambios + # + # @param :output [Bool] + # @return [Bool] + def deploy + status = false + log = [] + + time_start + + create_remote_site! if remote_site_id.blank? + create_njalla_records! if remote_info['njalla'].blank? + save + + if remote_site_id.blank? || remote_info['njalla'].blank? + raise DeployJob::DeployException, '' + end + + site_client.tap do |c| + stdout = Thread.new(publisher.logger_out) do |io| + until io.eof? + line = io.gets + + puts line if output + log << line + end + end + + update remote_info: c.show(publishing_site).to_h + + status = c.publish(publishing_site, deploy_local.destination) + + publisher.logger.close + stdout.join + end + + time_stop + + create_stat! status, log.join + + status + end + + def limit; end + + def size + deploy_local.size + end + + def destination; end + + # Devuelve las URLs de todos los protocolos + def urls + remote_info[:links].values.map do |protocol| + [ protocol[:link], protocol[:gateway] ] + end.flatten.compact.select do |link| + link.include? '://' + end + end + + private + + # El cliente de la API + # + # TODO: cuando soportemos más, tiene que haber una relación entre + # DeployDistributedPress y DistributedPressPublisher. + # + # @return [DistributedPressPublisher] + def publisher + @publisher ||= DistributedPressPublisher.first + end + + # El cliente para actualizar el sitio + # + # @return [DistributedPress::V1::Client::Site] + def site_client + DistributedPress::V1::Client::Site.new(publisher.client) + end + + # Genera el esquema de datos para poder publicar el sitio + # + # @return [DistributedPress::V1::Schemas::PublishingSite] + def publishing_site + DistributedPress::V1::Schemas::PublishingSite.new.call(id: remote_site_id) + end + + # Genera el esquema de datos para crear el sitio + # + # @return [DistributedPressPublisher::V1::Schemas::NewSite] + def create_site + DistributedPress::V1::Schemas::NewSite.new.call(domain: hostname, protocols: { http: true, ipfs: true, hyper: true }) + end + + # Crea el sitio en la instancia con el hostname especificado + # + # @return [nil] + def create_remote_site! + created_site = site_client.create(create_site) + + self.remote_site_id = created_site[:id] + self.remote_info = created_site.to_h + rescue DistributedPress::V1::Error => e + ExceptionNotifier.notify_exception(e, data: { site: site.name }) + ensure + nil + end + + # Crea los registros en Njalla + # + # @return [nil] + def create_njalla_records! + # XXX: Esto depende de nuestro DNS actual, cuando lo migremos hay + # que eliminarlo. + unless site.name.end_with? '.' + self.remote_info['njalla'] = {} + self.remote_info['njalla']['a'] = njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h + self.remote_info['njalla']['ns'] = njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h + end + rescue HTTParty::Error => e + ExceptionNotifier.notify_exception(e, data: { site: site.name }) + self.remote_info['njalla'] = nil + ensure + nil + end + + # Registra lo que sucedió + # + # @param status [Bool] + # @param log [String] + # @return [nil] + def create_stat!(status, log) + build_stats.create action: publisher.to_s,log: log, seconds: time_spent_in_seconds, bytes: size, status: status + nil + end + + # Actualizar registros en Njalla + # + # @return [Njalla::V1::Domain] + def njalla + @njalla ||= + begin + client = Njalla::V1::Client.new(token: ENV['NJALLA_TOKEN']) + + Njalla::V1::Domain.new(domain: Site.domain, client: client) + end + end +end diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 00e0985d..9c0ba492 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -15,6 +15,7 @@ class DeployLocal < Deploy def deploy(output: false) return false unless mkdir return false unless yarn(output: output) + return false unless pnpm(output: output) return false unless bundle(output: output) jekyll_build(output: output) @@ -67,27 +68,34 @@ class DeployLocal < Deploy end # Un entorno que solo tiene lo que necesitamos + # + # @return [Hash] def env # XXX: This doesn't support Windows paths :B - paths = [File.dirname(`which bundle`), '/usr/bin', '/bin'] + paths = [File.dirname(`which bundle`), '/usr/local/bin', '/usr/bin', '/bin'] - { - 'HOME' => home_dir, - 'PATH' => paths.join(':'), - 'SPREE_API_KEY' => site.tienda_api_key, - 'SPREE_URL' => site.tienda_url, - 'AIRBRAKE_PROJECT_ID' => site.id.to_s, - 'AIRBRAKE_PROJECT_KEY' => site.airbrake_api_key, - 'JEKYLL_ENV' => Rails.env, - 'LANG' => ENV['LANG'], - 'YARN_CACHE_FOLDER' => yarn_cache_dir - } + # Las variables de entorno extra no pueden superponerse al local. + extra_env.merge({ + 'HOME' => home_dir, + 'PATH' => paths.join(':'), + 'SPREE_API_KEY' => site.tienda_api_key, + 'SPREE_URL' => site.tienda_url, + 'AIRBRAKE_PROJECT_ID' => site.id.to_s, + 'AIRBRAKE_PROJECT_KEY' => site.airbrake_api_key, + 'JEKYLL_ENV' => Rails.env, + 'LANG' => ENV['LANG'], + 'YARN_CACHE_FOLDER' => yarn_cache_dir + }) end def yarn_cache_dir Rails.root.join('_yarn_cache').to_s end + def pnpm_cache_dir + Rails.root.join('_pnpm_cache').to_s + end + def yarn_lock File.join(site.path, 'yarn.lock') end @@ -96,6 +104,14 @@ class DeployLocal < Deploy File.exist? yarn_lock end + def pnpm_lock + File.join(site.path, 'pnpm-lock.yaml') + end + + def pnpm_lock? + File.exist? pnpm_lock + end + def gem(output: false) run %(gem install bundler --no-document), output: output end @@ -107,6 +123,13 @@ class DeployLocal < Deploy run 'yarn install --production', output: output end + def pnpm(output: false) + return true unless pnpm_lock? + + run %(pnpm config set store-dir "#{pnpm_cache_dir}"), output: output + run 'pnpm install --production', output: output + end + def bundle(output: false) run %(bundle install --no-cache --path="#{gems_dir}" --clean --without test development), output: output end @@ -125,4 +148,17 @@ class DeployLocal < Deploy def remove_destination! FileUtils.rm_rf destination end + + # Consigue todas las variables de entorno configuradas por otros + # deploys. + # + # @return [Hash] + def extra_env + @extra_env ||= + non_local_deploys.reduce({}) do |extra_env, deploy| + extra_env.tap do |e| + e.merge! deploy.local_env + end + end + end end diff --git a/app/models/distributed_press_publisher.rb b/app/models/distributed_press_publisher.rb new file mode 100644 index 00000000..6139db93 --- /dev/null +++ b/app/models/distributed_press_publisher.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require 'distributed_press/v1' + +# Almacena el token de autenticación y la URL, por ahora solo vamos +# a tener uno, pero queda abierta la posibilidad de agregar más. +class DistributedPressPublisher < ApplicationRecord + # Cifrar la información del token en la base de datos + has_encrypted :token + + # La salida del log + # + # @return [IO] + attr_reader :logger_out + + # La instancia es única + validates_uniqueness_of :instance + + # El token es necesario + validates_presence_of :token + + # Mantener la fecha de vencimiento actualizada + before_save :update_expires_at_from_token!, :update_token_from_client! + + # Devuelve todos los tokens que vencen en una hora + scope :with_about_to_expire_tokens, lambda { + where('expires_at > ? and expires_at < ?', Time.now, Time.now + 1.hour) + } + + # Instancia un cliente de Distributed Press a partir del token. Al + # cargar un token a punto de vencer se renueva automáticamente. + # + # @return [DistributedPress::V1::Client] + def client + @client ||= DistributedPress::V1::Client.new(url: instance, token: token, logger: logger) + end + + # @return [String] + def to_s + "Distributed Press <#{instance}>" + end + + # Devuelve el hostname de la instancia + # + # @return [String] + def hostname + @hostname ||= URI.parse(instance).hostname + end + + # @return [Logger] + def logger + @logger ||= + begin + @logger_out, @logger_in = IO.pipe + ::Logger.new @logger_in, formatter: formatter + end + end + + private + + def formatter + @formatter ||= lambda do |_, _, _, msg| + "#{msg}\n" + end + end + + # Actualiza o desactiva la fecha de vencimiento a partir de la + # información del token. + # + # @return [nil] + def update_expires_at_from_token! + self.expires_at = client.token.forever? ? nil : client.token.expires_at + nil + end + + # Actualiza el token a partir del cliente, que ya actualiza el token + # automáticamente. + # + # @return [nil] + def update_token_from_client! + self.token = client.token.to_s + nil + end +end diff --git a/app/models/site.rb b/app/models/site.rb index 4b2a8eb9..21453370 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -17,7 +17,7 @@ class Site < ApplicationRecord # TODO: Hacer que los diferentes tipos de deploy se auto registren # @see app/services/site_service.rb - DEPLOYS = %i[local private www zip hidden_service].freeze + DEPLOYS = %i[local private www zip hidden_service distributed_press].freeze validates :name, uniqueness: true, hostname: { allow_root_label: true diff --git a/app/views/deploys/_deploy_distributed_press.haml b/app/views/deploys/_deploy_distributed_press.haml new file mode 100644 index 00000000..d7d54db0 --- /dev/null +++ b/app/views/deploys/_deploy_distributed_press.haml @@ -0,0 +1,21 @@ +-# Publicar a la web distribuida + +.row + .col + = deploy.hidden_field :id + = deploy.hidden_field :type + .custom-control.custom-switch + -# + El checkbox invierte la lógica de destrucción porque queremos + crear el deploy si está activado y destruirlo si está + desactivado. + = deploy.check_box :_destroy, + { checked: deploy.object.persisted?, class: 'custom-control-input' }, + '0', '1' + = deploy.label :_destroy, class: 'custom-control-label' do + %h3= t('.title') + = sanitize_markdown t('.help', public_url: deploy.object.site.url), + tags: %w[p strong em a] + + +%hr/ diff --git a/config/locales/en.yml b/config/locales/en.yml index 567ab6bb..5ca19452 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -114,6 +114,10 @@ en: title: Alternative domain name success: Success! error: Error + deploy_distributed_press: + title: Distributed Web + success: Success! + error: Error deploy_reindex: title: Reindex success: Success! @@ -272,6 +276,22 @@ en: Only accessible through [Tor Browser](https://www.torproject.org/download/) + deploy_distributed_press: + title: 'Publish to the distributed Web' + help: | + Make your site available through peer-to-peer protocols, + Inter-Planetary File System (IPFS), Hypercore, and via + BitTorrent, so your site is more resilient and can be available + offline, including in community mesh networks. + + **Important:** Only use this option if you would like your data + to be permanently available. If you decide to undo this + selection, a cleared version of the site will be shared in its + place. However, it is possible that nodes on the distributed + storage network may continue retaining copies of the data + indefinitely. + + [Learn more](https://ffdweb.org/building-distributed-press-a-publishing-tool-for-the-decentralized-web/) stats: index: title: Statistics diff --git a/config/locales/es.yml b/config/locales/es.yml index 0a829e4a..76ce479d 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -114,6 +114,10 @@ es: title: Dominio alternativo success: ¡Éxito! error: Hubo un error + deploy_distributed_press: + title: Web distribuida + success: ¡Éxito! + error: Hubo un error deploy_reindex: title: Reindexación success: ¡Éxito! @@ -277,6 +281,22 @@ es: Sólo será accesible a través del [Navegador Tor](https://www.torproject.org/es/download/). + deploy_distributed_press: + title: 'Publicar a la Web distribuida' + help: | + Utiliza protocolos de pares, Inter-Planetary File System (IPFS), + Hypercore y torrents, para que tu sitio sea más resiliente y + esté disponible _offline_, inclusive en redes _mesh_ + comunitarias. + + **Importante:** Sólo usa esta opción si te parece correcto que + tu contenido esté disponible permanentemente. Cuando elijas + des-hacer esta acción, una versión "vacía" del sitio será + compartida en su lugar. Sin embargo, es posible que algunos + nodos en la red de almacenamiento distribuida puedan retener + copias de tu contenido indefinidamente. + + [Saber más (en inglés)](https://ffdweb.org/building-distributed-press-a-publishing-tool-for-the-decentralized-web/) stats: index: title: Estadísticas diff --git a/db/migrate/20230119165420_create_distributed_press_publisher.rb b/db/migrate/20230119165420_create_distributed_press_publisher.rb new file mode 100644 index 00000000..8d8de37a --- /dev/null +++ b/db/migrate/20230119165420_create_distributed_press_publisher.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Crea la tabla de publishers de Distributed Press que contiene las +# instancias y tokens +class CreateDistributedPressPublisher < ActiveRecord::Migration[6.1] + def change + create_table :distributed_press_publishers do |t| + t.timestamps + t.string :instance, unique: true + t.text :token_ciphertext, null: false + t.datetime :expires_at, null: true + end + end +end diff --git a/lib/tasks/distributed_press.rake b/lib/tasks/distributed_press.rake new file mode 100644 index 00000000..8ba270ec --- /dev/null +++ b/lib/tasks/distributed_press.rake @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +namespace :distributed_press do + namespace :tokens do + desc 'Renew tokens' + task renew: :environment do + RenewDistributedPressTokensJob.perform_now + end + end +end diff --git a/monit.conf b/monit.conf index 39f45d6d..dc866517 100644 --- a/monit.conf +++ b/monit.conf @@ -4,6 +4,11 @@ check program cleanup every "0 3 1 * *" if status != 0 then alert +check program distributed_press_tokens_renew + with path "/usr/bin/foreman run -f /srv/Procfile -d /srv distributed_press_tokens_renew" as uid "rails" gid "www-data" + every "0 3 * * *" + if status != 0 then alert + check program access_logs with path "/srv/http/bin/access_logs" as uid "app" and gid "www-data" every "0 0 * * *"