From e9d33b625a8843f9fb9a20015b36045a4e7ba010 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 25 Apr 2023 14:51:54 -0300 Subject: [PATCH 01/14] =?UTF-8?q?feat:=20encontrar=20el=20post=20a=20parti?= =?UTF-8?q?r=20de=20su=20indexaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/indexed_post.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb index 7f6865f6..184cd05f 100644 --- a/app/models/indexed_post.rb +++ b/app/models/indexed_post.rb @@ -36,6 +36,15 @@ class IndexedPost < ApplicationRecord belongs_to :site + # Encuentra el post original + # + # @return [nil,Post] + def post + return if post_id.blank? + + @post ||= site.posts(lang: locale).find(post_id, uuid: true) + end + # Convertir locale a direccionario de PG # # @param [String,Symbol] From f38251af7a30ca65848c68971fe6aad10a7ef074 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 26 Apr 2023 15:15:56 -0300 Subject: [PATCH 02/14] fix: permisos para posts indexados #13266 --- app/policies/indexed_post_policy.rb | 66 +++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 app/policies/indexed_post_policy.rb diff --git a/app/policies/indexed_post_policy.rb b/app/policies/indexed_post_policy.rb new file mode 100644 index 00000000..e0151c7a --- /dev/null +++ b/app/policies/indexed_post_policy.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# Política de acceso a artículos +class IndexedPostPolicy + attr_reader :indexed_post, :usuarie, :site + + def initialize(usuarie, indexed_post) + @usuarie = usuarie + @indexed_post = indexed_post + @site = indexed_post.site + end + + def index? + true + end + + # Les invitades solo pueden ver sus propios posts + def show? + site.usuarie?(usuarie) || site.indexed_posts.by_usuarie(usuarie.id).find_by_post_id(indexed_post.post_id).present? + end + + def preview? + show? + end + + def new? + create? + end + + def create? + true + end + + def edit? + update? + end + + # Les invitades solo pueden modificar sus propios artículos + def update? + show? + end + + # Solo las usuarias pueden eliminar artículos. Les invitades pueden + # borrar sus propios artículos + def destroy? + update? + end + + # Las usuarias pueden ver todos los posts + # + # Les invitades solo pueden ver sus propios posts + class Scope + attr_reader :usuarie, :scope + + def initialize(usuarie, scope) + @usuarie = usuarie + @scope = scope + end + + def resolve + return scope if scope&.first&.site&.usuarie? usuarie + + scope.by_usuarie(usuarie.id) + end + end +end From 8376e663ced7c232d18e3575ce4b6d68abc4af89 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 15:39:12 -0300 Subject: [PATCH 03/14] =?UTF-8?q?feat:=20almacenar=20el=20=C3=BAltimo=20co?= =?UTF-8?q?mmit=20indexado=20#13780?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../20230927153926_add_last_indexed_commit_to_sites.rb | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb diff --git a/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb b/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb new file mode 100644 index 00000000..2d22cbd7 --- /dev/null +++ b/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# Almacenar el último commit indexado +class AddLastIndexedCommitToSites < ActiveRecord::Migration[6.1] + def change + add_column :sites, :last_indexed_commit, :string, null: true + end +end From f18d0213dfc4d769ce58fc8ad3d898fd8a65f508 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 15:40:14 -0300 Subject: [PATCH 04/14] =?UTF-8?q?feat:=20guardar=20el=20=C3=BAltimo=20comm?= =?UTF-8?q?it=20indexado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/site/index.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/models/site/index.rb b/app/models/site/index.rb index e11095e3..f728b48b 100644 --- a/app/models/site/index.rb +++ b/app/models/site/index.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -# Indexa todos los artículos de un sitio -# -# TODO: Hacer opcional class Site + # Indexa todos los artículos de un sitio + # + # TODO: Hacer opcional module Index extend ActiveSupport::Concern @@ -15,6 +15,8 @@ class Site def index_posts! Site.transaction do docs.each(&:index!) + + update(last_indexed_commit: repository.head_commit.oid) end end end From 074cb49752fcbbacd8a3ab9424a4210b062ffbc4 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 16:20:57 -0300 Subject: [PATCH 05/14] feat: reindexar cambios #13780 --- app/models/site/index.rb | 100 +++++++++++++++++++++++++++++++++++++++ config/locales/en.yml | 3 ++ config/locales/es.yml | 3 ++ 3 files changed, 106 insertions(+) diff --git a/app/models/site/index.rb b/app/models/site/index.rb index f728b48b..ed0932bb 100644 --- a/app/models/site/index.rb +++ b/app/models/site/index.rb @@ -12,6 +12,10 @@ class Site after_create :index_posts! has_many :indexed_posts, dependent: :destroy + MODIFIED_STATUSES = %i[added modified].freeze + DELETED_STATUSES = %i[deleted].freeze + LOCALE_FROM_PATH = /\A_/.freeze + def index_posts! Site.transaction do docs.each(&:index!) @@ -19,6 +23,102 @@ class Site update(last_indexed_commit: repository.head_commit.oid) end end + + # Encuentra los artículos modificados entre dos commits y los + # reindexa. + def reindex_changes! + return unless reindexable? + + Site.transaction do + remove_deleted_posts! + reindex_modified_posts! + + update(last_indexed_commit: repository.head_commit.oid) + end + end + + # No hacer nada si el repositorio no cambió o no hubo cambios + # necesarios + def reindexable? + return false if last_indexed_commit.blank? + return false if last_indexed_commit == repository.head_commit.oid + + !indexable_posts.empty? + end + + private + + # Trae el último commit indexado desde el repositorio + # + # @return [Rugged::Commit] + def indexed_commit + @indexed_commit ||= repository.rugged.lookup(last_indexed_commit) + end + + # Calcula la diferencia entre el último commit indexado y el + # actual + # + # XXX: Esto no tiene en cuenta modificaciones en la historia como + # cambio de ramas, reverts y etc, solo asume que se mueve hacia + # adelante en la misma rama o las dos ramas están relacionadas. + # + # @return [Rugged::Diff] + def diff_with_head + @diff_with_head ||= indexed_commit.diff(repository.head_commit) + end + + # Obtiene todos los archivos a reindexar + # + # @return [Array] + def indexable_posts + @indexable_posts ||= + diff_with_head.each_delta.select do |delta| + locales.any? do |locale| + delta.old_file[:path].start_with? "_#{locale}/" + end + end + end + + # Elimina los artículos eliminados o que cambiaron de ubicación + # del índice + def remove_deleted_posts! + indexable_posts.select do |delta| + DELETED_STATUSES.include? delta.status + end.each do |delta| + locale, path = locale_and_path_from(delta.old_file[:path]) + + indexed_posts.destroy_by(locale: locale, path: path).tap do |destroyed_posts| + next unless destroyed_posts.empty? + + Rails.logger.info I18n.t('indexed_posts.deleted', site: name, path: path, records: destroyed_posts.count) + end + end + end + + # Reindexa artículos que cambiaron de ubicación, se agregaron + # o fueron modificados + def reindex_modified_posts! + indexable_posts.select do |delta| + MODIFIED_STATUSES.include? delta.status + end.each do |delta| + locale, path = locale_and_path_from(delta.new_file[:path]) + + site.posts(lang: locale).find(path).index! + end + end + + # Obtiene el idioma y la ruta del post a partir de la ubicación en + # el disco + # + # @return [Array] + def locale_and_path_from(path) + locale, path = path.split(File::SEPARATOR, 2) + + [ + locale.sub(LOCALE_FROM_PATH, ''), + File.basename(path, '.*') + ] + end end end end diff --git a/config/locales/en.yml b/config/locales/en.yml index 5f97a8b9..f2c0d94c 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -167,6 +167,7 @@ en: usuarie: User licencia: License design: Design + indexed_post: Indexed post attributes: usuarie: email: 'E-mail address' @@ -709,3 +710,5 @@ en: build_stats: index: title: "Publications" + indexed_posts: + deleted: "Deleted indexed post %{path} from %{site} (records: %{records})" diff --git a/config/locales/es.yml b/config/locales/es.yml index 9e0b8945..73de7b18 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -167,6 +167,7 @@ es: usuarie: Usuarie licencia: Licencia design: Diseño + indexed_post: Artículo indexado attributes: usuarie: email: 'Correo electrónico' @@ -717,3 +718,5 @@ es: build_stats: index: title: "Publicaciones" + indexed_posts: + deleted: "Eliminado artículo %{path} de %{site} (filas: %{records})" From 1e13985ef82c0dde7068db2bd0e027e8f8419434 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 16:31:23 -0300 Subject: [PATCH 06/14] =?UTF-8?q?fix:=20asumir=20que=20todos=20los=20sitio?= =?UTF-8?q?s=20ya=20est=C3=A1n=20indexados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...0230927153926_add_last_indexed_commit_to_sites.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb b/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb index 2d22cbd7..71e08f37 100644 --- a/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb +++ b/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb @@ -2,7 +2,17 @@ # Almacenar el último commit indexado class AddLastIndexedCommitToSites < ActiveRecord::Migration[6.1] - def change + def up add_column :sites, :last_indexed_commit, :string, null: true + + Site.find_each do |site| + site.update_columns(last_indexed_commit: site.repository.head_commit.oid) + rescue Rugged::Error, Rugged::OSError => e + puts "Falló #{site.name}, ignorando: #{e.message}" + end + end + + def down + remove_column :sites, :last_indexed_commit end end From 997114a89688bb735347f2c12966c24097662a5c Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 16:38:19 -0300 Subject: [PATCH 07/14] fix: ya estamos en el contexto del sitio --- app/models/site/index.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/site/index.rb b/app/models/site/index.rb index ed0932bb..06c8821b 100644 --- a/app/models/site/index.rb +++ b/app/models/site/index.rb @@ -103,7 +103,7 @@ class Site end.each do |delta| locale, path = locale_and_path_from(delta.new_file[:path]) - site.posts(lang: locale).find(path).index! + posts(lang: locale).find(path).index! end end From 85e528e660b2482d6a3b71b39b0f57544135b749 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 16:46:33 -0300 Subject: [PATCH 08/14] fix: solo hacer merge si hubo cambios para traer --- app/jobs/git_pull_job.rb | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/app/jobs/git_pull_job.rb b/app/jobs/git_pull_job.rb index de263403..41a2ec2c 100644 --- a/app/jobs/git_pull_job.rb +++ b/app/jobs/git_pull_job.rb @@ -3,11 +3,12 @@ # Permite traer los cambios cada vez que se # hace un push al repositorio class GitPullJob < ApplicationJob - # @param :site [Site] - # @param :usuarie [Usuarie] - # @return [nil] - def perform(site, usuarie) - site.repository.fetch - site.repository.merge(usuarie) - end -end \ No newline at end of file + # @param :site [Site] + # @param :usuarie [Usuarie] + # @return [nil] + def perform(site, usuarie) + return unless site.repository.fetch.positive? + + site.repository.merge(usuarie) + end +end From b119cba7e10afcbb4a665c26b4e7ad4d61afff8f Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 16:47:00 -0300 Subject: [PATCH 09/14] feat: reindexar los cambios luego de traerlos #13780 --- app/jobs/git_pull_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/git_pull_job.rb b/app/jobs/git_pull_job.rb index 41a2ec2c..203e5698 100644 --- a/app/jobs/git_pull_job.rb +++ b/app/jobs/git_pull_job.rb @@ -10,5 +10,6 @@ class GitPullJob < ApplicationJob return unless site.repository.fetch.positive? site.repository.merge(usuarie) + site.reindex_changes! end end From 0538b3bbfd832a03bf108fd069ba23f442894244 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 16:54:38 -0300 Subject: [PATCH 10/14] fix: no fallar si no hay origin --- app/jobs/git_pull_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/git_pull_job.rb b/app/jobs/git_pull_job.rb index 203e5698..949a2cdd 100644 --- a/app/jobs/git_pull_job.rb +++ b/app/jobs/git_pull_job.rb @@ -7,6 +7,7 @@ class GitPullJob < ApplicationJob # @param :usuarie [Usuarie] # @return [nil] def perform(site, usuarie) + return unless site.repository.origin return unless site.repository.fetch.positive? site.repository.merge(usuarie) From cb2c1b0e7daeab32fdc3f01fca9f7158eaf6918e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Sep 2023 10:10:18 -0300 Subject: [PATCH 11/14] =?UTF-8?q?fix:=20usar=20la=20codificaci=C3=B3n=20co?= =?UTF-8?q?rrecta?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit los archivos con tildes fallaban --- app/models/site/index.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/site/index.rb b/app/models/site/index.rb index 06c8821b..6f4714c6 100644 --- a/app/models/site/index.rb +++ b/app/models/site/index.rb @@ -108,11 +108,14 @@ class Site end # Obtiene el idioma y la ruta del post a partir de la ubicación en - # el disco + # el disco. + # + # Las rutas vienen en ASCII-9BIT desde Rugged, pero en realidad + # son UTF-8 # # @return [Array] def locale_and_path_from(path) - locale, path = path.split(File::SEPARATOR, 2) + locale, path = path.force_encoding('utf-8').split(File::SEPARATOR, 2) [ locale.sub(LOCALE_FROM_PATH, ''), From 9709f419ca6756d21ba21bb41cad6fce0cb0bb08 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Nov 2023 18:19:13 -0300 Subject: [PATCH 12/14] =?UTF-8?q?fix:=20notificar=20cuando=20falla=20la=20?= =?UTF-8?q?=C3=BAltima=20compilaci=C3=B3n=20#14555?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/deploy_job.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 291991c7..adba1815 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -56,6 +56,10 @@ class DeployJob < ApplicationJob rescue URI::Error nil end.compact + + unless d == @site.deployment_list.last && !status + raise DeployException, 'Falló la compilación' + end rescue StandardError => e status = false seconds ||= 0 From c8e1d502324f427382e74dc97b7b227fccb7c493 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Nov 2023 18:40:34 -0300 Subject: [PATCH 13/14] fix: reversed logic --- app/jobs/deploy_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index adba1815..3044b59f 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -57,7 +57,7 @@ class DeployJob < ApplicationJob nil end.compact - unless d == @site.deployment_list.last && !status + if d == @site.deployment_list.last && !status raise DeployException, 'Falló la compilación' end rescue StandardError => e From aaf2ca8757e32cc4d0cd567105ecfd46abf18388 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 23 Nov 2023 18:41:49 -0300 Subject: [PATCH 14/14] fix: deshabilitar njalla #14696 --- Gemfile | 1 - Gemfile.lock | 4 -- app/models/deploy_distributed_press.rb | 60 +------------------------- 3 files changed, 2 insertions(+), 63 deletions(-) diff --git a/Gemfile b/Gemfile index 972560b4..bf9e875c 100644 --- a/Gemfile +++ b/Gemfile @@ -40,7 +40,6 @@ gem 'devise' gem 'devise-i18n' gem 'devise_invitable' gem 'distributed-press-api-client', '~> 0.3.0rc0' -gem 'njalla-api-client', '~> 0.2.0' gem 'email_address', git: 'https://github.com/fauno/email_address', branch: 'i18n' gem 'exception_notification' gem 'fast_blank' diff --git a/Gemfile.lock b/Gemfile.lock index 39394a8c..3faad5e8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -366,9 +366,6 @@ GEM net-ssh (7.1.0) netaddr (2.0.6) nio4r (2.5.9-x86_64-linux-musl) - njalla-api-client (0.2.0) - dry-schema - httparty (~> 0.18) nokogiri (1.15.4-x86_64-linux-musl) mini_portile2 (~> 2.8.2) racc (~> 1.4) @@ -636,7 +633,6 @@ DEPENDENCIES mini_magick mobility net-ssh - njalla-api-client (~> 0.2.0) nokogiri pg pg_search diff --git a/app/models/deploy_distributed_press.rb b/app/models/deploy_distributed_press.rb index 2c892b55..da8fe209 100644 --- a/app/models/deploy_distributed_press.rb +++ b/app/models/deploy_distributed_press.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'distributed_press/v1/client/site' -require 'njalla/v1' # Soportar Distributed Press APIv1 # @@ -15,8 +14,8 @@ require 'njalla/v1' class DeployDistributedPress < Deploy store :values, accessors: %i[hostname remote_site_id remote_info], coder: JSON - before_create :create_remote_site!, :create_njalla_records! - before_destroy :delete_remote_site!, :delete_njalla_records! + before_create :create_remote_site! + before_destroy :delete_remote_site! DEPENDENCIES = %i[deploy_local] @@ -31,17 +30,12 @@ class DeployDistributedPress < Deploy time_start create_remote_site! if remote_site_id.blank? - create_njalla_records! save if remote_site_id.blank? raise DeployJob::DeployException, 'El sitio no se creó en Distributed Press' end - if create_njalla_records? && remote_info[:njalla].blank? - raise DeployJob::DeployException, 'No se pudieron crear los registros necesarios en Njalla' - end - site_client.tap do |c| stdout = Thread.new(publisher.logger_out) do |io| until io.eof? @@ -145,29 +139,6 @@ class DeployDistributedPress < Deploy nil end - # Crea los registros en Njalla - # - # XXX: Esto depende de nuestro DNS actual, cuando lo migremos hay - # que eliminarlo. - # - # @return [nil] - def create_njalla_records! - return unless create_njalla_records? - - self.remote_info ||= {} - self.remote_info[:njalla] ||= {} - self.remote_info[:njalla][:a] ||= njalla.add_record(name: site.name, type: 'CNAME', content: "#{Site.domain}.").to_h - self.remote_info[:njalla][:cname] ||= njalla.add_record(name: "www.#{site.name}", type: 'CNAME', content: "#{Site.domain}.").to_h - self.remote_info[:njalla][:ns] ||= njalla.add_record(name: "_dnslink.#{site.name}", type: 'NS', content: "#{publisher.hostname}.").to_h - - nil - rescue HTTParty::Error => e - ExceptionNotifier.notify_exception(e, data: { site: site.name }) - self.remote_info.delete :njalla - ensure - nil - end - # Registra lo que sucedió # # @param status [Bool] @@ -185,31 +156,4 @@ class DeployDistributedPress < Deploy ExceptionNotifier.notify_exception(e, data: { site: site.name }) nil end - - def delete_njalla_records! - return unless create_njalla_records? - - %w[a ns cname].each do |type| - next if (id = remote_info.dig('njalla', type, 'id')).blank? - - njalla.remove_record(id: id.to_i) - end - end - - # Actualizar registros en Njalla - # - # @return [Njalla::V1::Domain] - def njalla - @njalla ||= - begin - client = Njalla::V1::Client.new(token: Rails.application.credentials.njalla) - - Njalla::V1::Domain.new(domain: Site.domain, client: client) - end - end - - # Detecta si tenemos que crear registros en Njalla - def create_njalla_records? - !site.name.end_with?('.') - end end