From 8376e663ced7c232d18e3575ce4b6d68abc4af89 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 15:39:12 -0300 Subject: [PATCH 1/6] =?UTF-8?q?feat:=20almacenar=20el=20=C3=BAltimo=20comm?= =?UTF-8?q?it=20indexado=20#13780?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../20230927153926_add_last_indexed_commit_to_sites.rb | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb diff --git a/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb b/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb new file mode 100644 index 00000000..2d22cbd7 --- /dev/null +++ b/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# Almacenar el último commit indexado +class AddLastIndexedCommitToSites < ActiveRecord::Migration[6.1] + def change + add_column :sites, :last_indexed_commit, :string, null: true + end +end From f18d0213dfc4d769ce58fc8ad3d898fd8a65f508 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 15:40:14 -0300 Subject: [PATCH 2/6] =?UTF-8?q?feat:=20guardar=20el=20=C3=BAltimo=20commit?= =?UTF-8?q?=20indexado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/site/index.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/models/site/index.rb b/app/models/site/index.rb index e11095e3..f728b48b 100644 --- a/app/models/site/index.rb +++ b/app/models/site/index.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -# Indexa todos los artículos de un sitio -# -# TODO: Hacer opcional class Site + # Indexa todos los artículos de un sitio + # + # TODO: Hacer opcional module Index extend ActiveSupport::Concern @@ -15,6 +15,8 @@ class Site def index_posts! Site.transaction do docs.each(&:index!) + + update(last_indexed_commit: repository.head_commit.oid) end end end From 074cb49752fcbbacd8a3ab9424a4210b062ffbc4 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 16:20:57 -0300 Subject: [PATCH 3/6] feat: reindexar cambios #13780 --- app/models/site/index.rb | 100 +++++++++++++++++++++++++++++++++++++++ config/locales/en.yml | 3 ++ config/locales/es.yml | 3 ++ 3 files changed, 106 insertions(+) diff --git a/app/models/site/index.rb b/app/models/site/index.rb index f728b48b..ed0932bb 100644 --- a/app/models/site/index.rb +++ b/app/models/site/index.rb @@ -12,6 +12,10 @@ class Site after_create :index_posts! has_many :indexed_posts, dependent: :destroy + MODIFIED_STATUSES = %i[added modified].freeze + DELETED_STATUSES = %i[deleted].freeze + LOCALE_FROM_PATH = /\A_/.freeze + def index_posts! Site.transaction do docs.each(&:index!) @@ -19,6 +23,102 @@ class Site update(last_indexed_commit: repository.head_commit.oid) end end + + # Encuentra los artículos modificados entre dos commits y los + # reindexa. + def reindex_changes! + return unless reindexable? + + Site.transaction do + remove_deleted_posts! + reindex_modified_posts! + + update(last_indexed_commit: repository.head_commit.oid) + end + end + + # No hacer nada si el repositorio no cambió o no hubo cambios + # necesarios + def reindexable? + return false if last_indexed_commit.blank? + return false if last_indexed_commit == repository.head_commit.oid + + !indexable_posts.empty? + end + + private + + # Trae el último commit indexado desde el repositorio + # + # @return [Rugged::Commit] + def indexed_commit + @indexed_commit ||= repository.rugged.lookup(last_indexed_commit) + end + + # Calcula la diferencia entre el último commit indexado y el + # actual + # + # XXX: Esto no tiene en cuenta modificaciones en la historia como + # cambio de ramas, reverts y etc, solo asume que se mueve hacia + # adelante en la misma rama o las dos ramas están relacionadas. + # + # @return [Rugged::Diff] + def diff_with_head + @diff_with_head ||= indexed_commit.diff(repository.head_commit) + end + + # Obtiene todos los archivos a reindexar + # + # @return [Array] + def indexable_posts + @indexable_posts ||= + diff_with_head.each_delta.select do |delta| + locales.any? do |locale| + delta.old_file[:path].start_with? "_#{locale}/" + end + end + end + + # Elimina los artículos eliminados o que cambiaron de ubicación + # del índice + def remove_deleted_posts! + indexable_posts.select do |delta| + DELETED_STATUSES.include? delta.status + end.each do |delta| + locale, path = locale_and_path_from(delta.old_file[:path]) + + indexed_posts.destroy_by(locale: locale, path: path).tap do |destroyed_posts| + next unless destroyed_posts.empty? + + Rails.logger.info I18n.t('indexed_posts.deleted', site: name, path: path, records: destroyed_posts.count) + end + end + end + + # Reindexa artículos que cambiaron de ubicación, se agregaron + # o fueron modificados + def reindex_modified_posts! + indexable_posts.select do |delta| + MODIFIED_STATUSES.include? delta.status + end.each do |delta| + locale, path = locale_and_path_from(delta.new_file[:path]) + + site.posts(lang: locale).find(path).index! + end + end + + # Obtiene el idioma y la ruta del post a partir de la ubicación en + # el disco + # + # @return [Array] + def locale_and_path_from(path) + locale, path = path.split(File::SEPARATOR, 2) + + [ + locale.sub(LOCALE_FROM_PATH, ''), + File.basename(path, '.*') + ] + end end end end diff --git a/config/locales/en.yml b/config/locales/en.yml index 5f97a8b9..f2c0d94c 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -167,6 +167,7 @@ en: usuarie: User licencia: License design: Design + indexed_post: Indexed post attributes: usuarie: email: 'E-mail address' @@ -709,3 +710,5 @@ en: build_stats: index: title: "Publications" + indexed_posts: + deleted: "Deleted indexed post %{path} from %{site} (records: %{records})" diff --git a/config/locales/es.yml b/config/locales/es.yml index 9e0b8945..73de7b18 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -167,6 +167,7 @@ es: usuarie: Usuarie licencia: Licencia design: Diseño + indexed_post: Artículo indexado attributes: usuarie: email: 'Correo electrónico' @@ -717,3 +718,5 @@ es: build_stats: index: title: "Publicaciones" + indexed_posts: + deleted: "Eliminado artículo %{path} de %{site} (filas: %{records})" From 1e13985ef82c0dde7068db2bd0e027e8f8419434 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 16:31:23 -0300 Subject: [PATCH 4/6] =?UTF-8?q?fix:=20asumir=20que=20todos=20los=20sitios?= =?UTF-8?q?=20ya=20est=C3=A1n=20indexados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...0230927153926_add_last_indexed_commit_to_sites.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb b/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb index 2d22cbd7..71e08f37 100644 --- a/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb +++ b/db/migrate/20230927153926_add_last_indexed_commit_to_sites.rb @@ -2,7 +2,17 @@ # Almacenar el último commit indexado class AddLastIndexedCommitToSites < ActiveRecord::Migration[6.1] - def change + def up add_column :sites, :last_indexed_commit, :string, null: true + + Site.find_each do |site| + site.update_columns(last_indexed_commit: site.repository.head_commit.oid) + rescue Rugged::Error, Rugged::OSError => e + puts "Falló #{site.name}, ignorando: #{e.message}" + end + end + + def down + remove_column :sites, :last_indexed_commit end end From 997114a89688bb735347f2c12966c24097662a5c Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Sep 2023 16:38:19 -0300 Subject: [PATCH 5/6] fix: ya estamos en el contexto del sitio --- app/models/site/index.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/site/index.rb b/app/models/site/index.rb index ed0932bb..06c8821b 100644 --- a/app/models/site/index.rb +++ b/app/models/site/index.rb @@ -103,7 +103,7 @@ class Site end.each do |delta| locale, path = locale_and_path_from(delta.new_file[:path]) - site.posts(lang: locale).find(path).index! + posts(lang: locale).find(path).index! end end From cb2c1b0e7daeab32fdc3f01fca9f7158eaf6918e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Sep 2023 10:10:18 -0300 Subject: [PATCH 6/6] =?UTF-8?q?fix:=20usar=20la=20codificaci=C3=B3n=20corr?= =?UTF-8?q?ecta?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit los archivos con tildes fallaban --- app/models/site/index.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/site/index.rb b/app/models/site/index.rb index 06c8821b..6f4714c6 100644 --- a/app/models/site/index.rb +++ b/app/models/site/index.rb @@ -108,11 +108,14 @@ class Site end # Obtiene el idioma y la ruta del post a partir de la ubicación en - # el disco + # el disco. + # + # Las rutas vienen en ASCII-9BIT desde Rugged, pero en realidad + # son UTF-8 # # @return [Array] def locale_and_path_from(path) - locale, path = path.split(File::SEPARATOR, 2) + locale, path = path.force_encoding('utf-8').split(File::SEPARATOR, 2) [ locale.sub(LOCALE_FROM_PATH, ''),