From cc2b4a29719c249110908d953d4c330d279c2f06 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 12:26:36 -0300 Subject: [PATCH 01/33] dependencias del buscador MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pg_search: habilita búsquedas en postgresql hairtrigger: permite crear acciones automáticas en postgresql --- Gemfile | 6 +++++- Gemfile.lock | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 63b921c..93a8a22 100644 --- a/Gemfile +++ b/Gemfile @@ -51,7 +51,6 @@ gem 'sutty-liquid' gem 'lockbox' gem 'mini_magick' gem 'mobility' -gem 'pg' gem 'pundit' gem 'rails-i18n' gem 'rails_warden' @@ -67,6 +66,11 @@ gem 'validates_hostname' gem 'webpacker' gem 'yaml_db', git: 'https://0xacab.org/sutty/yaml_db.git' +# database +gem 'hairtrigger' +gem 'pg' +gem 'pg_search' + # performance gem 'flamegraph' gem 'memory_profiler' diff --git a/Gemfile.lock b/Gemfile.lock index 5c1d062..5ddaeaa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -217,6 +217,10 @@ GEM ffi (~> 1.0) globalid (0.4.2) activesupport (>= 4.2.0) + hairtrigger (0.2.24) + activerecord (>= 5.0, < 7) + ruby2ruby (~> 2.4) + ruby_parser (~> 3.10) haml (5.2.1) temple (>= 0.8.0) tilt @@ -387,6 +391,9 @@ GEM forwardable-extended (~> 2.6) pg (1.2.3) pg (1.2.3-x86_64-linux-musl) + pg_search (2.3.5) + activerecord (>= 5.2) + activesupport (>= 5.2) popper_js (1.16.0) prometheus_exporter (0.7.0) webrick @@ -523,7 +530,12 @@ GEM ruby-statistics (2.1.3) ruby-vips (2.1.0) ffi (~> 1.12) + ruby2ruby (2.4.4) + ruby_parser (~> 3.1) + sexp_processor (~> 4.6) ruby_dep (1.5.0) + ruby_parser (3.15.1) + sexp_processor (~> 4.9) rubyzip (2.3.0) rugged (1.1.0) rugged (1.1.0-x86_64-linux-musl) @@ -544,6 +556,7 @@ GEM childprocess (>= 0.5, < 4.0) rubyzip (>= 1.2.2) semantic_range (3.0.0) + sexp_processor (4.15.2) share-to-fediverse-jekyll-theme (0.1.4) jekyll (~> 4.0) jekyll-data (~> 1.1) @@ -677,6 +690,7 @@ DEPENDENCIES fast_jsonparser flamegraph friendly_id + hairtrigger haml-lint hamlit-rails hiredis @@ -699,6 +713,7 @@ DEPENDENCIES mobility net-ssh pg + pg_search prometheus_exporter pry puma From 655e07cd4091a61a54d170af9e0d27b0ecd29acb Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 12:29:48 -0300 Subject: [PATCH 02/33] =?UTF-8?q?configuraci=C3=B3n=20de=20las=20bases=20d?= =?UTF-8?q?e=20datos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ahora el testeo y desarrollo también se hace en postgresql. nos queda pendiente que cada quien pueda usar la base de datos que quiera y que si nos posible indexar documentos, que sea opcional. --- config/database.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/config/database.yml b/config/database.yml index 28e195b..9e72da0 100644 --- a/config/database.yml +++ b/config/database.yml @@ -5,25 +5,24 @@ # gem 'sqlite3' # default: &default - adapter: sqlite3 + adapter: postgresql pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> timeout: 5000 + database: sutty + user: postgres + host: postgresql + encoding: unicode development: <<: *default - database: db/development.sqlite3 # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". # Do not set this db to the same as development or production. test: <<: *default - database: db/test.sqlite3 + database: sutty_test production: - adapter: postgresql - pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> - database: sutty + <<: *default user: sutty - host: postgresql - encoding: unicode From 26d186721d6df1f76a602caf176fd444e54f0a91 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 12:33:28 -0300 Subject: [PATCH 03/33] algunos metadatos son indexables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit solo los de texto que no se refieran a otros artículos por ahora --- app/models/metadata_array.rb | 8 ++++++++ app/models/metadata_content.rb | 8 ++++++++ app/models/metadata_document_date.rb | 4 ++++ app/models/metadata_related_posts.rb | 4 ++++ app/models/metadata_string.rb | 4 ++++ app/models/metadata_template.rb | 6 ++++++ 6 files changed, 34 insertions(+) diff --git a/app/models/metadata_array.rb b/app/models/metadata_array.rb index 5c0b16f..5f43b79 100644 --- a/app/models/metadata_array.rb +++ b/app/models/metadata_array.rb @@ -13,6 +13,14 @@ class MetadataArray < MetadataTemplate false end + def indexable? + true + end + + def to_s + value.join(', ') + end + private # TODO: Sanitizar otros valores diff --git a/app/models/metadata_content.rb b/app/models/metadata_content.rb index 525f38a..546e08c 100644 --- a/app/models/metadata_content.rb +++ b/app/models/metadata_content.rb @@ -19,6 +19,14 @@ class MetadataContent < MetadataTemplate document.content end + def indexable? + true + end + + def to_s + sanitizer.sanitize value, tags: [], attributes: [] + end + private # Detectar si el contenido estaba en Markdown y pasarlo a HTML diff --git a/app/models/metadata_document_date.rb b/app/models/metadata_document_date.rb index 39e6873..a52cd05 100644 --- a/app/models/metadata_document_date.rb +++ b/app/models/metadata_document_date.rb @@ -11,6 +11,10 @@ class MetadataDocumentDate < MetadataTemplate document.date end + def indexable? + true + end + # El valor puede ser un Date, Time o una String en el formato # "yyyy-mm-dd" def value diff --git a/app/models/metadata_related_posts.rb b/app/models/metadata_related_posts.rb index 9a73dea..bcc18d8 100644 --- a/app/models/metadata_related_posts.rb +++ b/app/models/metadata_related_posts.rb @@ -18,6 +18,10 @@ class MetadataRelatedPosts < MetadataArray false end + def indexable? + false + end + private # Obtiene todos los posts y opcionalmente los filtra diff --git a/app/models/metadata_string.rb b/app/models/metadata_string.rb index ed50bc8..724c2ef 100644 --- a/app/models/metadata_string.rb +++ b/app/models/metadata_string.rb @@ -7,6 +7,10 @@ class MetadataString < MetadataTemplate super || '' end + def indexable? + true + end + private # No se permite HTML en las strings diff --git a/app/models/metadata_template.rb b/app/models/metadata_template.rb index 5800059..2fa8a61 100644 --- a/app/models/metadata_template.rb +++ b/app/models/metadata_template.rb @@ -7,6 +7,12 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, :value, :help, :required, :errors, :post, :layout, keyword_init: true) do + + # Determina si el campo es indexable + def indexable? + false + end + def inspect "#<#{self.class} site=#{site.name.inspect} post=#{post.id.inspect} value=#{value.inspect}>" end From ceaadeb7bf00e1907a2ff993253b18a127381bc6 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 12:45:33 -0300 Subject: [PATCH 04/33] =?UTF-8?q?crear=20la=20tabla=20de=20indexaci=C3=B3n?= =?UTF-8?q?=20de=20posts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit los posts se siguen guardando en el sitio jekyll, lo que guardamos en la base de datos es una representación indexable que tiene los datos mínimos de los posts para buscarlos por distintos parámetros. esto nos permite cargar la lista de artículos y filtrarla de distintas formas sin cargar todo jekyll en memoria, lo que reduciría el consumo de recursos y aceleraría el panel. ya tenemos caché así que el problema estaba mitigado, pero igual es un avance. ya que migramos la base de datos a postgresql, aparecieron todas las tablas y campos en el schema.rb, que es lo que usa rails para configurar una base de datos desde cero. --- ...10504224144_create_pg_search_extensions.rb | 9 + .../20210504224343_create_indexed_posts.rb | 44 ++++ db/schema.rb | 203 ++++++++++++++++-- 3 files changed, 235 insertions(+), 21 deletions(-) create mode 100644 db/migrate/20210504224144_create_pg_search_extensions.rb create mode 100644 db/migrate/20210504224343_create_indexed_posts.rb diff --git a/db/migrate/20210504224144_create_pg_search_extensions.rb b/db/migrate/20210504224144_create_pg_search_extensions.rb new file mode 100644 index 0000000..18eebe9 --- /dev/null +++ b/db/migrate/20210504224144_create_pg_search_extensions.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Habilitar las extensiones de búsqueda de texto libre +class CreatePgSearchExtensions < ActiveRecord::Migration[6.1] + def change + enable_extension 'plpgsql' + enable_extension 'pg_trgm' + end +end diff --git a/db/migrate/20210504224343_create_indexed_posts.rb b/db/migrate/20210504224343_create_indexed_posts.rb new file mode 100644 index 0000000..9cf2153 --- /dev/null +++ b/db/migrate/20210504224343_create_indexed_posts.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# Crea la tabla donde se indexa el contenido de los artículos, los +# IndexedPosts van a estar relacionados con un Post del mismo UUID. +# +# Solo contienen la información mínima necesaria para mostrar los +# resultados de búsqueda. +class CreateIndexedPosts < ActiveRecord::Migration[6.1] + def change + # Necesario para gen_random_uuid() + # + # XXX: En realidad no lo necesitamos porque cada IndexedPost va a + # tener el UUID del Post correspondiente. + enable_extension 'pgcrypto' + + create_table :indexed_posts, id: false do |t| + t.primary_key :id, :uuid, default: 'public.gen_random_uuid()' + t.belongs_to :site, index: true + t.timestamps + + # Filtramos por idioma + t.string :locale, default: 'simple', index: true + # Vamos a querer filtrar por layout + t.string :layout, null: false, index: true + # Esta es la ruta al artículo + t.string :path, null: false + # Queremos mostrar el título por separado + t.string :title, default: '' + # También vamos a mostrar las categorías + t.jsonb :front_matter, default: '{}' + t.string :content, default: '' + t.tsvector :indexed_content + + t.index :indexed_content, using: 'gin' + t.index :front_matter, using: 'gin' + end + + # Crea un trigger que actualiza el índice tsvector con el título y + # contenido del artículo y su idioma. + create_trigger(compatibility: 1).on(:indexed_posts).before(:insert, :update) do + "new.indexed_content := to_tsvector(('pg_catalog.' || new.locale)::regconfig, coalesce(new.title, '') || '\n' || coalesce(new.content,''));" + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 2a93c5f..fed6934 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,16 +10,74 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_04_14_152728) do +ActiveRecord::Schema.define(version: 2021_05_04_224343) do -# Could not dump table "access_logs" because of following StandardError -# Unknown type '' for column 'id' + # These are extensions that must be enabled in order to support this database + enable_extension "pg_trgm" + enable_extension "pgcrypto" + enable_extension "plpgsql" + + create_table "access_logs", id: :uuid, default: nil, force: :cascade do |t| + t.string "host" + t.float "msec" + t.string "server_protocol" + t.string "request_method" + t.string "request_completion" + t.string "uri" + t.string "query_string" + t.integer "status" + t.string "sent_http_content_type" + t.string "sent_http_content_encoding" + t.string "sent_http_etag" + t.string "sent_http_last_modified" + t.string "http_accept" + t.string "http_accept_encoding" + t.string "http_accept_language" + t.string "http_pragma" + t.string "http_cache_control" + t.string "http_if_none_match" + t.string "http_dnt" + t.string "http_user_agent" + t.string "http_origin" + t.float "request_time" + t.integer "bytes_sent" + t.integer "body_bytes_sent" + t.integer "request_length" + t.string "http_connection" + t.string "pipe" + t.integer "connection_requests" + t.string "geoip2_data_country_name" + t.string "geoip2_data_city_name" + t.string "ssl_server_name" + t.string "ssl_protocol" + t.string "ssl_early_data" + t.string "ssl_session_reused" + t.string "ssl_curves" + t.string "ssl_ciphers" + t.string "ssl_cipher" + t.string "sent_http_x_xss_protection" + t.string "sent_http_x_frame_options" + t.string "sent_http_x_content_type_options" + t.string "sent_http_strict_transport_security" + t.string "nginx_version" + t.integer "pid" + t.string "remote_user" + t.boolean "crawler", default: false + t.string "http_referer" + t.index ["geoip2_data_city_name"], name: "index_access_logs_on_geoip2_data_city_name" + t.index ["geoip2_data_country_name"], name: "index_access_logs_on_geoip2_data_country_name" + t.index ["host"], name: "index_access_logs_on_host" + t.index ["http_origin"], name: "index_access_logs_on_http_origin" + t.index ["http_user_agent"], name: "index_access_logs_on_http_user_agent" + t.index ["status"], name: "index_access_logs_on_status" + t.index ["uri"], name: "index_access_logs_on_uri" + end create_table "action_text_rich_texts", force: :cascade do |t| t.string "name", null: false t.text "body" t.string "record_type", null: false - t.integer "record_id", null: false + t.bigint "record_id", null: false t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true @@ -28,8 +86,8 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do create_table "active_storage_attachments", force: :cascade do |t| t.string "name", null: false t.string "record_type", null: false - t.integer "record_id", null: false - t.integer "blob_id", null: false + t.bigint "record_id", null: false + t.bigint "blob_id", null: false t.datetime "created_at", null: false t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true @@ -40,7 +98,7 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.string "filename", null: false t.string "content_type" t.text "metadata" - t.integer "byte_size", null: false + t.bigint "byte_size", null: false t.string "checksum", null: false t.datetime "created_at", null: false t.string "service_name", null: false @@ -53,10 +111,65 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true end + create_table "blazer_audits", force: :cascade do |t| + t.bigint "user_id" + t.bigint "query_id" + t.text "statement" + t.string "data_source" + t.datetime "created_at" + t.index ["query_id"], name: "index_blazer_audits_on_query_id" + t.index ["user_id"], name: "index_blazer_audits_on_user_id" + end + + create_table "blazer_checks", force: :cascade do |t| + t.bigint "creator_id" + t.bigint "query_id" + t.string "state" + t.string "schedule" + t.text "emails" + t.text "slack_channels" + t.string "check_type" + t.text "message" + t.datetime "last_run_at" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["creator_id"], name: "index_blazer_checks_on_creator_id" + t.index ["query_id"], name: "index_blazer_checks_on_query_id" + end + + create_table "blazer_dashboard_queries", force: :cascade do |t| + t.bigint "dashboard_id" + t.bigint "query_id" + t.integer "position" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["dashboard_id"], name: "index_blazer_dashboard_queries_on_dashboard_id" + t.index ["query_id"], name: "index_blazer_dashboard_queries_on_query_id" + end + + create_table "blazer_dashboards", force: :cascade do |t| + t.bigint "creator_id" + t.text "name" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["creator_id"], name: "index_blazer_dashboards_on_creator_id" + end + + create_table "blazer_queries", force: :cascade do |t| + t.bigint "creator_id" + t.string "name" + t.text "description" + t.text "statement" + t.string "data_source" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["creator_id"], name: "index_blazer_queries_on_creator_id" + end + create_table "build_stats", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "deploy_id" + t.bigint "deploy_id" t.integer "bytes" t.float "seconds" t.string "action", null: false @@ -65,13 +178,27 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.index ["deploy_id"], name: "index_build_stats_on_deploy_id" end -# Could not dump table "csp_reports" because of following StandardError -# Unknown type 'uuid' for column 'id' + create_table "csp_reports", id: :uuid, default: nil, force: :cascade do |t| + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.string "disposition" + t.string "referrer" + t.string "blocked_uri" + t.string "document_uri" + t.string "effective_directive" + t.string "original_policy" + t.string "script_sample" + t.string "status_code" + t.string "violated_directive" + t.integer "column_number" + t.integer "line_number" + t.string "source_file" + end create_table "deploys", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "site_id" + t.bigint "site_id" t.string "type" t.text "values" t.index ["site_id"], name: "index_deploys_on_site_id" @@ -91,6 +218,24 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.string "designer_url" end + create_table "indexed_posts", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.bigint "site_id" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.string "locale", default: "simple" + t.string "layout", null: false + t.string "path", null: false + t.string "title", default: "" + t.jsonb "front_matter", default: "{}" + t.string "content", default: "" + t.tsvector "indexed_content" + t.index ["front_matter"], name: "index_indexed_posts_on_front_matter", using: :gin + t.index ["indexed_content"], name: "index_indexed_posts_on_indexed_content", using: :gin + t.index ["layout"], name: "index_indexed_posts_on_layout" + t.index ["locale"], name: "index_indexed_posts_on_locale" + t.index ["site_id"], name: "index_indexed_posts_on_site_id" + end + create_table "licencias", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -104,7 +249,7 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do create_table "log_entries", force: :cascade do |t| t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false - t.integer "site_id" + t.bigint "site_id" t.text "text" t.boolean "sent", default: false t.index ["site_id"], name: "index_log_entries_on_site_id" @@ -124,7 +269,7 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.string "key", null: false t.string "value" t.string "translatable_type" - t.integer "translatable_id" + t.bigint "translatable_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["translatable_id", "translatable_type", "key"], name: "index_mobility_string_translations_on_translatable_attribute" @@ -137,7 +282,7 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.string "key", null: false t.text "value" t.string "translatable_type" - t.integer "translatable_id" + t.bigint "translatable_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["translatable_id", "translatable_type", "key"], name: "index_mobility_text_translations_on_translatable_attribute" @@ -147,8 +292,8 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do create_table "roles", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "site_id" - t.integer "usuarie_id" + t.bigint "site_id" + t.bigint "usuarie_id" t.string "rol" t.boolean "temporal" t.index ["site_id", "usuarie_id"], name: "index_roles_on_site_id_and_usuarie_id", unique: true @@ -160,15 +305,14 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.string "name" - t.integer "design_id" - t.integer "licencia_id" + t.bigint "design_id" + t.bigint "licencia_id" t.string "status", default: "waiting" t.text "description" t.string "title" t.boolean "colaboracion_anonima", default: false t.boolean "contact", default: false t.string "private_key_ciphertext" - t.boolean "invitades", default: false t.boolean "acepta_invitades", default: false t.string "tienda_api_key_ciphertext", default: "" t.string "tienda_url", default: "" @@ -200,7 +344,7 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.datetime "invitation_accepted_at" t.integer "invitation_limit" t.string "invited_by_type" - t.integer "invited_by_id" + t.bigint "invited_by_id" t.integer "invitations_count", default: 0 t.string "lang", default: "es" t.index ["confirmation_token"], name: "index_usuaries_on_confirmation_token", unique: true @@ -208,11 +352,28 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.index ["invitation_token"], name: "index_usuaries_on_invitation_token", unique: true t.index ["invitations_count"], name: "index_usuaries_on_invitations_count" t.index ["invited_by_id"], name: "index_usuaries_on_invited_by_id" - t.index ["invited_by_type", "invited_by_id"], name: "index_usuaries_on_invited_by_type_and_invited_by_id" + t.index ["invited_by_type", "invited_by_id"], name: "index_usuaries_on_invited_by" t.index ["reset_password_token"], name: "index_usuaries_on_reset_password_token", unique: true t.index ["unlock_token"], name: "index_usuaries_on_unlock_token", unique: true end add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + # no candidate create_trigger statement could be found, creating an adapter-specific one + execute(<<-SQL) +CREATE OR REPLACE FUNCTION public.indexed_posts_before_insert_update_row_tr() + RETURNS trigger + LANGUAGE plpgsql +AS $function$ +BEGIN + new.indexed_content := to_tsvector(('pg_catalog.' || new.locale)::regconfig, coalesce(new.title, '') || ' + ' || coalesce(new.content,'')); + RETURN NEW; +END; +$function$ + SQL + + # no candidate create_trigger statement could be found, creating an adapter-specific one + execute("CREATE TRIGGER indexed_posts_before_insert_update_row_tr BEFORE INSERT OR UPDATE ON \"indexed_posts\" FOR EACH ROW EXECUTE PROCEDURE indexed_posts_before_insert_update_row_tr()") + end From 86ae7fb8f81bb9cb74c7a76215cbef49624e0d09 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 12:52:30 -0300 Subject: [PATCH 05/33] los posts pueden ser indexados MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `IndexedPost` es una representación indexada por PG de `Post`. ambos están relacionados por el UUID de `Post`, de forma que se puede traer el artículo completo (por ejemplo al previsualizar o editar). cada artículo está indexado según su idioma. para eso convertimos el locale en el equivalente en el diccionario de PG. `Site#index_posts` es un método para indexar todos los artículos en masa. `Post#to_index` genera el `IndexedPost` correspondiente `IndexedPost.search(:es, 'hola')` busca "hola" en todos los artículos utilizando el diccionario de castellano. esto no quiere decir que busque en todos los artículos en castellano. por ahora para eso hay que hacer algo como: ```ruby site = Site.find 1 site.indexed_posts.where(locale: :english).search(:en, 'hello') ``` para encontrar todos los artículos en inglés del sitio con id 1 --- app/models/indexed_post.rb | 38 ++++++++++++++++++++++++ app/models/post.rb | 4 +++ app/models/post/indexable.rb | 56 ++++++++++++++++++++++++++++++++++++ app/models/site.rb | 2 ++ 4 files changed, 100 insertions(+) create mode 100644 app/models/indexed_post.rb create mode 100644 app/models/post/indexable.rb diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb new file mode 100644 index 0000000..6456a82 --- /dev/null +++ b/app/models/indexed_post.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +# La representación indexable de un artículo +class IndexedPost < ApplicationRecord + include PgSearch::Model + + # La traducción del locale según Sutty al locale según PostgreSQL + DICTIONARIES = { + es: 'spanish', + en: 'english' + }.freeze + + # TODO: Los indexed posts tienen que estar scopeados al idioma actual, + # no buscar sobre todos + pg_search_scope :search, + lambda { |locale, query| + { + against: :content, + query: query, + using: { + tsearch: { + dictionary: IndexedPost.to_dictionary(locale: locale), + tsvector_column: 'indexed_content' + } + } + } + } + + belongs_to :site + + # Convertir locale a direccionario de PG + # + # @param [String,Symbol] + # @return [String] + def self.to_dictionary(locale:) + DICTIONARIES[locale.to_sym] || 'simple' + end +end diff --git a/app/models/post.rb b/app/models/post.rb index a0e1670..6ef7a2e 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -17,6 +17,8 @@ class Post attr_reader :attributes, :errors, :layout, :site, :document + include Post::Indexable + class << self # Obtiene el layout sin leer el Document # @@ -191,6 +193,8 @@ class Post post: self, required: true) end + alias locale lang + # TODO: Mover a method_missing def uuid @metadata[:uuid] ||= MetadataUuid.new(document: document, site: site, layout: layout, name: :uuid, type: :uuid, diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb new file mode 100644 index 0000000..f25f28d --- /dev/null +++ b/app/models/post/indexable.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +class Post + # Vuelve indexables a los Posts + module Indexable + extend ActiveSupport::Concern + + included do + # Devuelve una versión indexable del Post + # + # @return [IndexedPosts] + def to_index + @to_index ||= IndexedPost.find_or_create_by(id: uuid.value).tap do |indexed_post| + indexed_post.layout = layout.name + indexed_post.site_id = site.id + indexed_post.path = path.basename + indexed_post.locale = IndexedPost.to_dictionary(locale: locale.value) + indexed_post.title = title.value + indexed_post.front_matter = indexable_front_matter + indexed_post.content = indexable_content + end + end + + private + + # Los metadatos que se almacenan como objetos JSON. Empezamos con + # las categorías porque se usan para filtrar en el listado de + # artículos. + # + # @return [Hash] + def indexable_front_matter + return {} unless attribute? :categories + + { categories: categories.indexable_values } + end + + # Devuelve un documento indexable en texto plano + # + # XXX: No memoizamos para permitir actualizaciones, aunque + # probablemente se indexe una sola vez. + # + # @return [String] + def indexable_content + indexable_attributes.map do |attr| + self[attr].to_s + end.join("\n") + end + + def indexable_attributes + @indexable_attributes ||= attributes.select do |attr| + self[attr].indexable? + end + end + end + end +end diff --git a/app/models/site.rb b/app/models/site.rb index 1c62e9b..14238e1 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -7,6 +7,7 @@ class Site < ApplicationRecord include Site::Forms include Site::FindAndReplace include Site::Api + include Site::Index include Tienda # Cifrar la llave privada que cifra y decifra campos ocultos. Sutty @@ -37,6 +38,7 @@ class Site < ApplicationRecord belongs_to :design belongs_to :licencia + has_many :indexed_posts, dependent: :destroy has_many :log_entries, dependent: :destroy has_many :deploys, dependent: :destroy has_many :build_stats, through: :deploys From a149c870e2023f0b0f41957efebc114b0e50925c Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 13:00:05 -0300 Subject: [PATCH 06/33] =?UTF-8?q?las=20categor=C3=ADas=20se=20guardan=20en?= =?UTF-8?q?=20indexed=20posts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit algunas categorías son otros artículos relacionados, con este método garantizamos que sólo se guardan los títulos. --- app/models/metadata_array.rb | 2 ++ app/models/metadata_related_posts.rb | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/app/models/metadata_array.rb b/app/models/metadata_array.rb index 5f43b79..8d951a1 100644 --- a/app/models/metadata_array.rb +++ b/app/models/metadata_array.rb @@ -21,6 +21,8 @@ class MetadataArray < MetadataTemplate value.join(', ') end + alias indexable_values values + private # TODO: Sanitizar otros valores diff --git a/app/models/metadata_related_posts.rb b/app/models/metadata_related_posts.rb index bcc18d8..4c022ff 100644 --- a/app/models/metadata_related_posts.rb +++ b/app/models/metadata_related_posts.rb @@ -22,6 +22,10 @@ class MetadataRelatedPosts < MetadataArray false end + def indexable_values + values.keys + end + private # Obtiene todos los posts y opcionalmente los filtra From d61d1cad565636c800f4f7642c065ad62d99de9d Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 19:07:11 -0300 Subject: [PATCH 07/33] =?UTF-8?q?respetar=20el=20orden=20de=20los=20art?= =?UTF-8?q?=C3=ADculos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit siempre ordenamos primero por número de orden y fecha de creación, siempre decrecientes. esto permite que les usuaries prioricen contenido usando las herramientas de reordenamiento. pg_search no soporta esto, siempre ordena por cuánto corresponde el resultado con la búsqueda, así que lo emparchamos para que respete el orden que necesitamos. el reporte de error relacionado es este: https://github.com/Casecommons/pg_search/issues/467 --- app/models/indexed_post.rb | 3 +++ app/models/post/indexable.rb | 2 ++ config/initializers/core_extensions.rb | 13 +++++++++++++ .../20210506212356_add_order_to_indexed_posts.rb | 9 +++++++++ db/schema.rb | 3 ++- 5 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20210506212356_add_order_to_indexed_posts.rb diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb index 6456a82..c76a5c8 100644 --- a/app/models/indexed_post.rb +++ b/app/models/indexed_post.rb @@ -26,6 +26,9 @@ class IndexedPost < ApplicationRecord } } + # Trae los IndexedPost en el orden en que van a terminar en el sitio. + default_scope lambda { order(order: :desc, created_at: :desc) } + belongs_to :site # Convertir locale a direccionario de PG diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index f25f28d..9b92111 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -18,6 +18,8 @@ class Post indexed_post.title = title.value indexed_post.front_matter = indexable_front_matter indexed_post.content = indexable_content + indexed_post.created_at = date.value + indexed_post.order = attribute?(:order) ? order.value : 0 end end diff --git a/config/initializers/core_extensions.rb b/config/initializers/core_extensions.rb index e37b2be..84dea42 100644 --- a/config/initializers/core_extensions.rb +++ b/config/initializers/core_extensions.rb @@ -66,3 +66,16 @@ module Jekyll end end end + +# No aplicar el orden por ranking para poder obtener los artículos en el +# orden que tendrían en el sitio final. +module PgSearch + ScopeOptions.class_eval do + def apply(scope) + scope = include_table_aliasing_for_rank(scope) + rank_table_alias = scope.pg_search_rank_table_alias(include_counter: true) + + scope.joins(rank_join(rank_table_alias)) + end + end +end diff --git a/db/migrate/20210506212356_add_order_to_indexed_posts.rb b/db/migrate/20210506212356_add_order_to_indexed_posts.rb new file mode 100644 index 0000000..4b1a9fc --- /dev/null +++ b/db/migrate/20210506212356_add_order_to_indexed_posts.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Agrega el orden y fecha del post en el post indexado, de forma que los +# resultados se puedan obtener en el mismo orden. +class AddOrderToIndexedPosts < ActiveRecord::Migration[6.1] + def change + add_column :indexed_posts, :order, :integer, default: 0 + end +end diff --git a/db/schema.rb b/db/schema.rb index fed6934..ed62940 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_04_224343) do +ActiveRecord::Schema.define(version: 2021_05_06_212356) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -229,6 +229,7 @@ ActiveRecord::Schema.define(version: 2021_05_04_224343) do t.jsonb "front_matter", default: "{}" t.string "content", default: "" t.tsvector "indexed_content" + t.integer "order", default: 0 t.index ["front_matter"], name: "index_indexed_posts_on_front_matter", using: :gin t.index ["indexed_content"], name: "index_indexed_posts_on_indexed_content", using: :gin t.index ["layout"], name: "index_indexed_posts_on_layout" From fd7ab8d7ef1190ce460eb967f922182992bdacdf Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 19:46:36 -0300 Subject: [PATCH 08/33] =?UTF-8?q?actualizar=20el=20=C3=ADndice=20cuando=20?= =?UTF-8?q?se=20agrega=20o=20modifica=20un=20post?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit además, activa callbacks de activerecord de forma que Post se va pareciendo cada vez más a un modelo de rails :D --- app/models/post.rb | 7 +++++-- app/models/post/indexable.rb | 12 +++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/app/models/post.rb b/app/models/post.rb index 790ee1a..7964d4e 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -17,6 +17,7 @@ class Post attr_reader :attributes, :errors, :layout, :site, :document + include ActiveRecord::Callbacks include Post::Indexable class << self @@ -285,8 +286,10 @@ class Post end end - return false unless save_attributes! - return false unless write + run_callbacks :save do + return false unless save_attributes! + return false unless write + end # Vuelve a leer el post para tomar los cambios read diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index 9b92111..b7b5a7f 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -6,11 +6,14 @@ class Post extend ActiveSupport::Concern included do + # Indexa o reindexa el Post + after_save :index! + # Devuelve una versión indexable del Post # # @return [IndexedPosts] def to_index - @to_index ||= IndexedPost.find_or_create_by(id: uuid.value).tap do |indexed_post| + IndexedPost.find_or_create_by(id: uuid.value).tap do |indexed_post| indexed_post.layout = layout.name indexed_post.site_id = site.id indexed_post.path = path.basename @@ -25,6 +28,13 @@ class Post private + # Indexa o reindexa el Post + # + # @return [Boolean] + def index! + to_index.save + end + # Los metadatos que se almacenan como objetos JSON. Empezamos con # las categorías porque se usan para filtrar en el listado de # artículos. From 34a05e860d7248174ad197436cab8adf121107e7 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 19:47:43 -0300 Subject: [PATCH 09/33] =?UTF-8?q?elimina=20el=20espacio=20vac=C3=ADo=20del?= =?UTF-8?q?=20contenido?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/post/indexable.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index b7b5a7f..b887dea 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -54,8 +54,8 @@ class Post # @return [String] def indexable_content indexable_attributes.map do |attr| - self[attr].to_s - end.join("\n") + self[attr].to_s.remove("\n") + end.join("\n").squeeze("\n") end def indexable_attributes From ad871baca63bf0b1e5454575473fcdfa7d04e09d Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 16:17:25 -0300 Subject: [PATCH 10/33] implementar el buscador en el panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ahora el índice de artículos incorporar buscador de texto libre. además todos los filtros de búsqueda se mantienen entre búsquedas, entonces al filtrar por tipo de artículo y término, se aplican ambos y al cambiar el tipo se mantiene la búsqueda de texto. --- app/assets/stylesheets/application.scss | 4 ++ app/controllers/posts_controller.rb | 38 ++++++----- app/models/indexed_post.rb | 2 + app/models/post/indexable.rb | 13 +++- app/policies/post_policy.rb | 4 +- app/views/posts/index.haml | 87 ++++++++++++------------- config/locales/en.yml | 1 + config/locales/es.yml | 1 + 8 files changed, 82 insertions(+), 68 deletions(-) diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index 0a215c4..1f9b634 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -21,6 +21,10 @@ $form-feedback-invalid-color: $magenta; $form-feedback-icon-valid-color: $black; $component-active-bg: $magenta; +$spacers: ( + 2-plus: 0.75rem +); + @import "bootstrap"; @import "editor"; diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index a4b47a1..524335a 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -16,29 +16,25 @@ class PostsController < ApplicationController authorize Post @site = find_site - @category = params.dig(:category) - @layout = params.dig(:layout) - @locale = locale + @q = params[:q] + dictionary = IndexedPost.to_dictionary(locale: locale) # XXX: Cada vez que cambiamos un Post tocamos el sitio con lo que es # más simple saber si hubo cambios. - if @category || @layout || stale?(@site) - @posts = @site.posts(lang: locale) - @posts = @posts.where(categories: @category) if @category - @posts = @posts.where(layout: @layout) if @layout + if filter_params.present? || stale?(@site) + # Todos los artículos de este sitio para el idioma actual + @posts = @site.indexed_posts.where(locale: dictionary) + # De este tipo + @posts = @posts.where(layout: filter_params[:layout]) if filter_params[:layout] + # Que estén dentro de la categoría + @posts = @posts.in_category(filter_params[:category]) if filter_params[:category] + # Aplicar los parámetros de búsqueda + @posts = @posts.search(locale, filter_params[:q]) if filter_params[:q].present? + # A los que este usuarie tiene acceso @posts = PostPolicy::Scope.new(current_usuarie, @posts).resolve - @category_name = if uuid?(@category) - @site.posts(lang: locale).find(@category, uuid: true)&.title&.value - else - @category - end - # Filtrar los posts que les invitades no pueden ver @usuarie = @site.usuarie? current_usuarie - - # Orden descendiente por número y luego por fecha - @posts.sort_by!(:order, :date).reverse! end end @@ -169,4 +165,14 @@ class PostsController < ApplicationController def forget_content flash[:js] = { target: 'editor', action: 'forget-content', keys: (params[:storage_keys] || []).to_json } end + + private + + # Los parámetros de filtros que vamos a mantener en todas las URLs, + # solo los que no estén vacíos. + # + # @return [Hash] + def filter_params + @filter_params ||= params.permit(:q, :category, :layout).to_h.select { |_,v| v.present? } + end end diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb index c76a5c8..7bf1ec7 100644 --- a/app/models/indexed_post.rb +++ b/app/models/indexed_post.rb @@ -28,6 +28,8 @@ class IndexedPost < ApplicationRecord # Trae los IndexedPost en el orden en que van a terminar en el sitio. default_scope lambda { order(order: :desc, created_at: :desc) } + scope :in_category, lambda { |category| where("front_matter->'categories' ? :category", category: category.to_s) } + scope :by_usuarie, lambda { |usuarie| where("front_matter->'usuaries' @> :usuarie::jsonb", usuarie: usuarie.to_s) } belongs_to :site diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index b887dea..bee02e5 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -11,7 +11,7 @@ class Post # Devuelve una versión indexable del Post # - # @return [IndexedPosts] + # @return [IndexedPost] def to_index IndexedPost.find_or_create_by(id: uuid.value).tap do |indexed_post| indexed_post.layout = layout.name @@ -41,9 +41,16 @@ class Post # # @return [Hash] def indexable_front_matter - return {} unless attribute? :categories + {}.tap do |indexable_front_matter| + indexable_front_matter = { + usuaries: usuaries.map(&:id), + draft: attribute?(:draft) ? draft.value : false + } - { categories: categories.indexable_values } + if attribute? :categories + indexable_front_matter[:categories] = categories.indexable_values + end + end end # Devuelve un documento indexable en texto plano diff --git a/app/policies/post_policy.rb b/app/policies/post_policy.rb index c22202a..69ecb18 100644 --- a/app/policies/post_policy.rb +++ b/app/policies/post_policy.rb @@ -59,9 +59,7 @@ class PostPolicy def resolve return scope if scope&.first&.site&.usuarie? usuarie - scope.select do |post| - post.usuaries.include? usuarie - end + scope.by_usuarie(usuarie.id) end end end diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index b2f3f66..9c43e43 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -3,7 +3,7 @@ @site.name, link_to(t('posts.index'), site_posts_path(@site)), - @category_name] + @category] %main.row %aside.menu.col-md-3 @@ -14,15 +14,13 @@ %table.mb-3 - @site.layouts.each do |layout| - next if layout.hidden? - - filter = params[:layout] == layout.value %tr %th= layout.humanized_name - %td.pl-3= link_to t('posts.add'), - new_site_post_path(@site, layout: layout.name), - class: 'badge badge-secondary' - %td= link_to t(filter ? 'posts.remove_filter' : 'posts.filter'), - site_posts_path(@site, layout: (filter ? nil : layout.value)), - class: 'badge badge-' + (filter ? 'primary' : 'secondary') + %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'badge badge-secondary' + - if @filter_params[:layout] == layout.value + %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'badge badge-primary' + - else + %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'badge badge-secondary' - if policy(@site).edit? = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn' @@ -48,19 +46,24 @@ %section.col = render 'layouts/flash' + .d-flex.justify-content-between.align-items-center.pl-2-plus.pr-2-plus.mb-2 + %form + - @filter_params.each do |param, value| + - next if param == 'q' + %input{ type: 'input', name: param, value: value } + .form-group.flex-grow-0.m-0 + %input.form-control.border.border-magenta{ type: 'search', placeholder: 'Buscar', name: 'q', value: @q } + %input.sr-only{ type: 'submit' } + - if @site.locales.size > 1 + + %nav#locales + - @site.locales.each do |locale| + = link_to t("locales.#{locale}.name"), site_posts_path(@site, **@filter_params.merge(locale: locale)), + class: "mr-2 mt-2 mb-2 #{locale == @locale ? 'active font-weight-bold' : ''}" - if @posts.empty? - %h2= t('posts.none') + %h2= t('posts.empty') - else = form_tag site_posts_reorder_path, method: :post do - .d-flex.justify-content-between.align-items-center - -# - TODO: Pensar una interfaz mejor para cuando haya más de tres - idiomas - - unless @site.locales.length == 1 - .locales - - @site.locales.each do |locale| - = link_to t("locales.#{locale}.name"), site_posts_path(@site, locale: locale), - class: "mr-2 mt-2 mb-2#{locale == @locale ? 'active font-weight-bold' : ''}" %table.table{ data: { controller: 'reorder' } } %caption.sr-only= t('posts.caption') %thead @@ -76,6 +79,7 @@ %button.btn{ data: { action: 'reorder#bottom' } }= t('posts.reorder.bottom') %tbody - dir = t("locales.#{@locale}.dir") + - size = @posts.size - @posts.each_with_index do |post, i| -# TODO: Solo les usuaries cachean porque tenemos que separar @@ -84,45 +88,36 @@ TODO: Verificar qué pasa cuando se gestiona el sitio en distintos idiomas a la vez - cache_if @usuarie, post do - - checkbox_id = "checkbox-#{post.uuid.value}" - %tr{ id: post.uuid.value, data: { target: 'reorder.row' } } + - checkbox_id = "checkbox-#{post.id}" + %tr{ id: post.id, data: { target: 'reorder.row' } } %td .custom-control.custom-checkbox %input.custom-control-input{ id: checkbox_id, type: 'checkbox', autocomplete: 'off', data: { action: 'reorder#select' } } %label.custom-control-label{ for: checkbox_id } %span.sr-only= t('posts.reorder.select') -# Orden más alto es mayor prioridad - = hidden_field 'post[reorder]', post.uuid.value, - value: @posts.length - i, + = hidden_field 'post[reorder]', post.id, + value: size - i, data: { reorder: true } %td.w-100{ class: dir } - = link_to site_post_path(@site, post.id) do - %span{ lang: post.lang.value, dir: dir }= post.title.value - - if post.attributes.include? :draft - - if post.draft.value - %span.badge.badge-primary - = post_label_t(:draft, post: post) - - if post.attributes.include? :categories - - unless post.categories.value.empty? - %br - %small - - (post.categories.respond_to?(:belongs_to) ? post.categories.belongs_to : post.categories.value).each do |c| - = link_to site_posts_path(@site, category: (c.respond_to?(:uuid) ? c.uuid.value : c)) do - %span{ lang: post.lang.value, dir: dir }= (c.respond_to?(:title) ? c.title.value : c) + = link_to site_post_path(@site, post.path) do + %span{ lang: post.locale, dir: dir }= post.title + - if post.front_matter['draft'].present? + %span.badge.badge-primary + = post_label_t(:draft, post: post) + - if post.front_matter['categories'].present? + %br + %small + - post.front_matter['categories'].each do |category| + = link_to site_posts_path(@site, **@filter_params.merge(category: category)) do + %span{ lang: post.locale, dir: dir }= category %td - = post.date.value.strftime('%F') + = post.created_at.strftime('%F') %br/ - - if post.attribute? :order - = post.order.value + = post.order %td - if @usuarie || policy(post).edit? - = link_to t('posts.edit'), - edit_site_post_path(@site, post.id), - class: 'btn btn-block' + = link_to t('posts.edit'), edit_site_post_path(@site, post.path), class: 'btn btn-block' - if @usuarie || policy(post).destroy? - = link_to t('posts.destroy'), - site_post_path(@site, post.id), - class: 'btn btn-block', - method: :delete, - data: { confirm: t('posts.confirm_destroy') } + = link_to t('posts.destroy'), site_post_path(@site, post.path), class: 'btn btn-block', method: :delete, data: { confirm: t('posts.confirm_destroy') } diff --git a/config/locales/en.yml b/config/locales/en.yml index dd9f830..54b6115 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -397,6 +397,7 @@ en: en: 'English' ar: 'Arabic' posts: + empty: "There are no results for those search parameters." attribute_ro: file: download: Download file diff --git a/config/locales/es.yml b/config/locales/es.yml index 403389a..b98ae14 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -459,6 +459,7 @@ es: en: 'inglés' ar: 'árabe' posts: + empty: No hay artículos con estos parámetros de búsqueda. caption: Lista de artículos attribute_ro: file: From 4f2e602822bce8db394265c75b5e51abb195db11 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 16:21:44 -0300 Subject: [PATCH 11/33] =?UTF-8?q?tambi=C3=A9n=20buscar=20por=20palabras=20?= =?UTF-8?q?similares?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/indexed_post.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb index 7bf1ec7..ede5b53 100644 --- a/app/models/indexed_post.rb +++ b/app/models/indexed_post.rb @@ -21,6 +21,9 @@ class IndexedPost < ApplicationRecord tsearch: { dictionary: IndexedPost.to_dictionary(locale: locale), tsvector_column: 'indexed_content' + }, + trigram: { + word_similarity: true } } } From 2d3f5b21aef17ddef38c8fa53b8c850936a93a24 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 16:21:59 -0300 Subject: [PATCH 12/33] =?UTF-8?q?al=20eliminar=20los=20saltos=20de=20l?= =?UTF-8?q?=C3=ADnea=20se=20quedaban=20pegadas=20algunas=20palabras?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/post/indexable.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index bee02e5..8a12e40 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -61,7 +61,7 @@ class Post # @return [String] def indexable_content indexable_attributes.map do |attr| - self[attr].to_s.remove("\n") + self[attr].to_s.tr("\n", ' ') end.join("\n").squeeze("\n") end From 6df7f09c26789ea98aa2bb83bd38b69c34348321 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:11:08 -0300 Subject: [PATCH 13/33] =?UTF-8?q?darle=20estilo=20de=20bot=C3=B3n=20a=20lo?= =?UTF-8?q?s=20filtros?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit como eran badges con links no tenían sombra en el foco ni color al pasarles por encima. --- app/views/posts/index.haml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 9c43e43..a9f868e 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -16,11 +16,11 @@ - next if layout.hidden? %tr %th= layout.humanized_name - %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'badge badge-secondary' + %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'btn btn-secondary badge' - if @filter_params[:layout] == layout.value - %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'badge badge-primary' + %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'btn btn-primary badge' - else - %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'badge badge-secondary' + %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary badge' - if policy(@site).edit? = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn' From 418ffb846376852f11bf8d21de67c21a271c3086 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:31:13 -0300 Subject: [PATCH 14/33] =?UTF-8?q?cambiar=20btn-sm=20para=20que=20tenga=20e?= =?UTF-8?q?l=20tama=C3=B1o=20de=20badge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/assets/stylesheets/application.scss | 4 ++++ app/views/posts/index.haml | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index 1f9b634..e059ecd 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -214,6 +214,10 @@ svg { } } +.btn-sm { + @extend .badge +} + .black-bg { color: $white; background-color: $black; diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index a9f868e..11e6efc 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -16,11 +16,11 @@ - next if layout.hidden? %tr %th= layout.humanized_name - %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'btn btn-secondary badge' + %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'btn btn-secondary btn-sm' - if @filter_params[:layout] == layout.value - %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'btn btn-primary badge' + %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'btn btn-primary btn-sm' - else - %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary badge' + %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary btn-sm' - if policy(@site).edit? = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn' From 051e40f64bc876a9bef335743451b3d489da5c3b Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:31:50 -0300 Subject: [PATCH 15/33] el formulario de busqueda tiene que incorporar los filtros MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit para poder volverlos a enviar. me había olvidado de convertirlos en campos ocultos. --- app/views/posts/index.haml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 11e6efc..8fcb86b 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -50,12 +50,12 @@ %form - @filter_params.each do |param, value| - next if param == 'q' - %input{ type: 'input', name: param, value: value } + %input{ type: 'hidden', name: param, value: value } .form-group.flex-grow-0.m-0 %input.form-control.border.border-magenta{ type: 'search', placeholder: 'Buscar', name: 'q', value: @q } %input.sr-only{ type: 'submit' } - - if @site.locales.size > 1 + - if @site.locales.size > 1 %nav#locales - @site.locales.each do |locale| = link_to t("locales.#{locale}.name"), site_posts_path(@site, **@filter_params.merge(locale: locale)), From 754e8fbbcc19b9df0793694f1415f137a8a74c34 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:32:54 -0300 Subject: [PATCH 16/33] =?UTF-8?q?mostrar=20cu=C3=A1les=20son=20los=20filtr?= =?UTF-8?q?os=20actuales=20y=20poder=20removerlos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit además, crear una sección de notas al pie con definiciones disponibles para lectores de pantalla. (no quiere decir que sean completas, `title` solo funciona con usuaries de mouse) --- app/views/posts/index.haml | 16 ++++++++++++++++ config/locales/en.yml | 1 + config/locales/es.yml | 1 + 3 files changed, 18 insertions(+) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 8fcb86b..717460c 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -60,6 +60,16 @@ - @site.locales.each do |locale| = link_to t("locales.#{locale}.name"), site_posts_path(@site, **@filter_params.merge(locale: locale)), class: "mr-2 mt-2 mb-2 #{locale == @locale ? 'active font-weight-bold' : ''}" + .pl-2-plus + - @filter_params.each do |param, value| + - if param == 'layout' + - value = @site.layouts[value.to_sym].humanized_name + = link_to site_posts_path(@site, **@filter_params.reject { |k, _| k == param }), + class: 'btn btn-secondary btn-sm', + title: t('posts.remove_filter_help', filter: value), + aria: { labelledby: "help-filter-#{param}" } do + = value + × - if @posts.empty? %h2= t('posts.empty') - else @@ -121,3 +131,9 @@ = link_to t('posts.edit'), edit_site_post_path(@site, post.path), class: 'btn btn-block' - if @usuarie || policy(post).destroy? = link_to t('posts.destroy'), site_post_path(@site, post.path), class: 'btn btn-block', method: :delete, data: { confirm: t('posts.confirm_destroy') } + +#footnotes{ hidden: true } + - @filter_params.each do |param, value| + - if param == 'layout' + - value = @site.layouts[value.to_sym].humanized_name + %label{ id: "help-filter-#{param}" }= t('posts.remove_filter_help', filter: value) diff --git a/config/locales/en.yml b/config/locales/en.yml index 54b6115..da1028f 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -451,6 +451,7 @@ en: add: 'Add' filter: 'Filter' remove_filter: 'Back' + remove_filter_help: 'Remove the filter: %{filter}' categories: 'Everything' index: 'Posts' edit: 'Edit' diff --git a/config/locales/es.yml b/config/locales/es.yml index b98ae14..a9e8cac 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -515,6 +515,7 @@ es: add: 'Agregar' filter: 'Filtrar' remove_filter: 'Volver' + remove_filter_help: 'Quitar este filtro: %{filter}' index: 'Artículos' edit: 'Editar' preview: From e1055cc91240653ba37cb11dfaa4c767bab43510 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:45:07 -0300 Subject: [PATCH 17/33] necesitamos el locale para poder marcar el idioma actual --- app/controllers/posts_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index 524335a..4acf765 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -16,7 +16,7 @@ class PostsController < ApplicationController authorize Post @site = find_site - @q = params[:q] + @locale = locale dictionary = IndexedPost.to_dictionary(locale: locale) # XXX: Cada vez que cambiamos un Post tocamos el sitio con lo que es From f9a2d12803a132fc0536df2f4726803dfd425544 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:46:49 -0300 Subject: [PATCH 18/33] mostrar el texto buscado en el buscador para poder editarlo si hace falta --- app/views/posts/index.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 717460c..2377223 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -52,7 +52,7 @@ - next if param == 'q' %input{ type: 'hidden', name: param, value: value } .form-group.flex-grow-0.m-0 - %input.form-control.border.border-magenta{ type: 'search', placeholder: 'Buscar', name: 'q', value: @q } + %input.form-control.border.border-magenta{ type: 'search', placeholder: 'Buscar', name: 'q', value: @filter_params[:q] } %input.sr-only{ type: 'submit' } - if @site.locales.size > 1 From 13b6b7a452a3f6ffd2bda523c7b1e905b180f2d7 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:54:45 -0300 Subject: [PATCH 19/33] =?UTF-8?q?cambiar=20la=20leyenda=20del=20bot=C3=B3n?= =?UTF-8?q?=20en=20el=20filtro?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit no cambia el resaltado porque la definición de la clase .btn hace conflicto, para arreglar eso hay que verificar todo el panel. --- app/views/posts/index.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 2377223..26a0f3e 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -17,7 +17,7 @@ %tr %th= layout.humanized_name %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'btn btn-secondary btn-sm' - - if @filter_params[:layout] == layout.value + - if @filter_params[:layout] == layout.name.to_s %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'btn btn-primary btn-sm' - else %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary btn-sm' From d2ae406023af3d54c0201ff73db31af79fd363ab Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 18:06:49 -0300 Subject: [PATCH 20/33] =?UTF-8?q?eliminar=20del=20=C3=ADndice=20al=20elimi?= =?UTF-8?q?nar=20el=20art=C3=ADculo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/post.rb | 15 ++++++++++++--- app/models/post/indexable.rb | 5 +++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/app/models/post.rb b/app/models/post.rb index 7964d4e..a64bd55 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -17,6 +17,9 @@ class Post attr_reader :attributes, :errors, :layout, :site, :document + # TODO: Modificar el historial de Git con callbacks en lugar de + # services. De esta forma podríamos agregar soporte para distintos + # backends. include ActiveRecord::Callbacks include Post::Indexable @@ -258,11 +261,17 @@ class Post end # Eliminar el artículo del repositorio y de la lista de artículos del - # sitio + # sitio. + # + # TODO: Si el callback falla deberíamos recuperar el archivo. + # + # @return [Post] def destroy - FileUtils.rm_f path.absolute + run_callbacks :destroy do + FileUtils.rm_f path.absolute - site.delete_post self + site.delete_post self + end end alias destroy! destroy diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index 8a12e40..0baa801 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -8,6 +8,7 @@ class Post included do # Indexa o reindexa el Post after_save :index! + after_destroy :remove_from_index! # Devuelve una versión indexable del Post # @@ -35,6 +36,10 @@ class Post to_index.save end + def remove_from_index! + to_index.destroy.destroyed? + end + # Los metadatos que se almacenan como objetos JSON. Empezamos con # las categorías porque se usan para filtrar en el listado de # artículos. From 6396841b2c4401d37a345dc351a73c2c3abe4582 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 19:25:09 -0300 Subject: [PATCH 21/33] evitar confusiones entre diccionario y locale el diccionario es lo que usa internamente pg para indexar, el locale es el idioma que asignamos en sutty. --- app/controllers/posts_controller.rb | 3 +- app/models/indexed_post.rb | 2 +- app/models/post/indexable.rb | 3 +- ...7221120_add_dictionary_to_indexed_posts.rb | 29 +++++++++++++++++++ db/schema.rb | 27 +++++++---------- 5 files changed, 43 insertions(+), 21 deletions(-) create mode 100644 db/migrate/20210507221120_add_dictionary_to_indexed_posts.rb diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index 4acf765..f865bd5 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -17,13 +17,12 @@ class PostsController < ApplicationController @site = find_site @locale = locale - dictionary = IndexedPost.to_dictionary(locale: locale) # XXX: Cada vez que cambiamos un Post tocamos el sitio con lo que es # más simple saber si hubo cambios. if filter_params.present? || stale?(@site) # Todos los artículos de este sitio para el idioma actual - @posts = @site.indexed_posts.where(locale: dictionary) + @posts = @site.indexed_posts.where(locale: locale) # De este tipo @posts = @posts.where(layout: filter_params[:layout]) if filter_params[:layout] # Que estén dentro de la categoría diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb index ede5b53..16858a6 100644 --- a/app/models/indexed_post.rb +++ b/app/models/indexed_post.rb @@ -19,7 +19,7 @@ class IndexedPost < ApplicationRecord query: query, using: { tsearch: { - dictionary: IndexedPost.to_dictionary(locale: locale), + dictionary: dictionary, tsvector_column: 'indexed_content' }, trigram: { diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index 0baa801..b1d6250 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -18,7 +18,8 @@ class Post indexed_post.layout = layout.name indexed_post.site_id = site.id indexed_post.path = path.basename - indexed_post.locale = IndexedPost.to_dictionary(locale: locale.value) + indexed_post.locale = locale.value + indexed_post.dictionary = IndexedPost.to_dictionary(locale: locale.value) indexed_post.title = title.value indexed_post.front_matter = indexable_front_matter indexed_post.content = indexable_content diff --git a/db/migrate/20210507221120_add_dictionary_to_indexed_posts.rb b/db/migrate/20210507221120_add_dictionary_to_indexed_posts.rb new file mode 100644 index 0000000..f79309f --- /dev/null +++ b/db/migrate/20210507221120_add_dictionary_to_indexed_posts.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# Para no estar calculando todo el tiempo el diccionario del idioma, +# agregamos una columna más. +class AddDictionaryToIndexedPosts < ActiveRecord::Migration[6.1] + LOCALES = { + 'english' => 'en', + 'spanish' => 'es' + } + + def up + add_column :indexed_posts, :dictionary, :string + + create_trigger(compatibility: 1).on(:indexed_posts).before(:insert, :update) do + "new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig, coalesce(new.title, '') || '\n' || coalesce(new.content,''));" + end + + IndexedPost.find_each do |post| + locale = post.locale + + post.update dictionary: locale, locale: LOCALES[locale] + end + end + + def down + remove_column :indexed_posts, :locale + rename_column :indexed_posts, :dictionary, :locale + end +end diff --git a/db/schema.rb b/db/schema.rb index ed62940..0a8dd08 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_06_212356) do +ActiveRecord::Schema.define(version: 2021_05_07_221120) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -230,6 +230,7 @@ ActiveRecord::Schema.define(version: 2021_05_06_212356) do t.string "content", default: "" t.tsvector "indexed_content" t.integer "order", default: 0 + t.string "dictionary" t.index ["front_matter"], name: "index_indexed_posts_on_front_matter", using: :gin t.index ["indexed_content"], name: "index_indexed_posts_on_indexed_content", using: :gin t.index ["layout"], name: "index_indexed_posts_on_layout" @@ -360,21 +361,13 @@ ActiveRecord::Schema.define(version: 2021_05_06_212356) do add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" - # no candidate create_trigger statement could be found, creating an adapter-specific one - execute(<<-SQL) -CREATE OR REPLACE FUNCTION public.indexed_posts_before_insert_update_row_tr() - RETURNS trigger - LANGUAGE plpgsql -AS $function$ -BEGIN - new.indexed_content := to_tsvector(('pg_catalog.' || new.locale)::regconfig, coalesce(new.title, '') || ' - ' || coalesce(new.content,'')); - RETURN NEW; -END; -$function$ - SQL - - # no candidate create_trigger statement could be found, creating an adapter-specific one - execute("CREATE TRIGGER indexed_posts_before_insert_update_row_tr BEFORE INSERT OR UPDATE ON \"indexed_posts\" FOR EACH ROW EXECUTE PROCEDURE indexed_posts_before_insert_update_row_tr()") + create_trigger("indexed_posts_before_insert_update_row_tr", :compatibility => 1). + on("indexed_posts"). + before(:insert, :update) do + <<-SQL_ACTIONS +new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig, coalesce(new.title, '') || ' +' || coalesce(new.content,'')); + SQL_ACTIONS + end end From 98df0ceb3a1ef23a5f7d444f23176da5de86c446 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 9 May 2021 12:25:16 -0300 Subject: [PATCH 22/33] los datos privados no se indexan --- app/models/metadata_array.rb | 4 +++- app/models/metadata_content.rb | 2 +- app/models/metadata_document_date.rb | 2 +- app/models/metadata_string.rb | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/models/metadata_array.rb b/app/models/metadata_array.rb index 8d951a1..9f5a84b 100644 --- a/app/models/metadata_array.rb +++ b/app/models/metadata_array.rb @@ -13,8 +13,10 @@ class MetadataArray < MetadataTemplate false end + # Solo los datos públicos se indexan, aunque MetadataArray no se cifra + # aun, dejamos esto preparado para la posteridad. def indexable? - true + true && !private? end def to_s diff --git a/app/models/metadata_content.rb b/app/models/metadata_content.rb index 546e08c..437a0dd 100644 --- a/app/models/metadata_content.rb +++ b/app/models/metadata_content.rb @@ -20,7 +20,7 @@ class MetadataContent < MetadataTemplate end def indexable? - true + true && !private? end def to_s diff --git a/app/models/metadata_document_date.rb b/app/models/metadata_document_date.rb index a52cd05..c741e3b 100644 --- a/app/models/metadata_document_date.rb +++ b/app/models/metadata_document_date.rb @@ -12,7 +12,7 @@ class MetadataDocumentDate < MetadataTemplate end def indexable? - true + true && !private? end # El valor puede ser un Date, Time o una String en el formato diff --git a/app/models/metadata_string.rb b/app/models/metadata_string.rb index 724c2ef..95aac4d 100644 --- a/app/models/metadata_string.rb +++ b/app/models/metadata_string.rb @@ -8,7 +8,7 @@ class MetadataString < MetadataTemplate end def indexable? - true + true && !private? end private From 35518ba48abc2833c31640d8f753dfb2b12a6085 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 9 May 2021 12:52:26 -0300 Subject: [PATCH 23/33] =?UTF-8?q?no=20hacen=20falta=20los=20parametros=20a?= =?UTF-8?q?l=20crear=20un=20art=C3=ADculo=20nuevo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/posts/index.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 26a0f3e..814ae04 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -16,7 +16,7 @@ - next if layout.hidden? %tr %th= layout.humanized_name - %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'btn btn-secondary btn-sm' + %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, layout: layout.value), class: 'btn btn-secondary btn-sm' - if @filter_params[:layout] == layout.name.to_s %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'btn btn-primary btn-sm' - else From 219a9985f555ac34f84b61daf31e417270853686 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 14 May 2021 16:59:47 -0300 Subject: [PATCH 24/33] testear el buscador --- app/models/indexed_post.rb | 36 ++++++++++++------------ app/models/post/indexable.rb | 17 ++++------- app/models/site.rb | 5 ++-- test/models/indexed_post_test.rb | 35 +++++++++++++++++++++++ test/models/post/indexable_test.rb | 45 ++++++++++++++++++++++++++++++ 5 files changed, 107 insertions(+), 31 deletions(-) create mode 100644 test/models/indexed_post_test.rb create mode 100644 test/models/post/indexable_test.rb diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb index 16858a6..7f6865f 100644 --- a/app/models/indexed_post.rb +++ b/app/models/indexed_post.rb @@ -13,26 +13,26 @@ class IndexedPost < ApplicationRecord # TODO: Los indexed posts tienen que estar scopeados al idioma actual, # no buscar sobre todos pg_search_scope :search, - lambda { |locale, query| - { - against: :content, - query: query, - using: { - tsearch: { - dictionary: dictionary, - tsvector_column: 'indexed_content' - }, - trigram: { - word_similarity: true - } - } - } - } + lambda { |locale, query| + { + against: :content, + query: query, + using: { + tsearch: { + dictionary: IndexedPost.to_dictionary(locale: locale), + tsvector_column: 'indexed_content' + }, + trigram: { + word_similarity: true + } + } + } + } # Trae los IndexedPost en el orden en que van a terminar en el sitio. - default_scope lambda { order(order: :desc, created_at: :desc) } - scope :in_category, lambda { |category| where("front_matter->'categories' ? :category", category: category.to_s) } - scope :by_usuarie, lambda { |usuarie| where("front_matter->'usuaries' @> :usuarie::jsonb", usuarie: usuarie.to_s) } + default_scope -> { order(order: :desc, created_at: :desc) } + scope :in_category, ->(category) { where("front_matter->'categories' ? :category", category: category.to_s) } + scope :by_usuarie, ->(usuarie) { where("front_matter->'usuaries' @> :usuarie::jsonb", usuarie: usuarie.to_s) } belongs_to :site diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index b1d6250..7757e7f 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -47,15 +47,10 @@ class Post # # @return [Hash] def indexable_front_matter - {}.tap do |indexable_front_matter| - indexable_front_matter = { - usuaries: usuaries.map(&:id), - draft: attribute?(:draft) ? draft.value : false - } - - if attribute? :categories - indexable_front_matter[:categories] = categories.indexable_values - end + {}.tap do |ifm| + ifm[:usuaries] = usuaries.map(&:id) + ifm[:draft] = attribute?(:draft) ? draft.value : false + ifm[:categories] = categories.indexable_values if attribute? :categories end end @@ -73,8 +68,8 @@ class Post def indexable_attributes @indexable_attributes ||= attributes.select do |attr| - self[attr].indexable? - end + self[attr].indexable? + end end end end diff --git a/app/models/site.rb b/app/models/site.rb index 4bfbed4..b3cae93 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -7,7 +7,6 @@ class Site < ApplicationRecord include Site::Forms include Site::FindAndReplace include Site::Api - include Site::Index include Tienda # Cifrar la llave privada que cifra y decifra campos ocultos. Sutty @@ -38,7 +37,6 @@ class Site < ApplicationRecord belongs_to :design belongs_to :licencia - has_many :indexed_posts, dependent: :destroy has_many :log_entries, dependent: :destroy has_many :deploys, dependent: :destroy has_many :build_stats, through: :deploys @@ -70,6 +68,9 @@ class Site < ApplicationRecord # El sitio en Jekyll attr_reader :jekyll + # XXX: Es importante incluir luego de los callbacks de :load_jekyll + include Site::Index + # No permitir HTML en estos atributos def title=(title) super(title.strip_tags) diff --git a/test/models/indexed_post_test.rb b/test/models/indexed_post_test.rb new file mode 100644 index 0000000..27d4e29 --- /dev/null +++ b/test/models/indexed_post_test.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'test_helper' + +class IndexedPostTest < ActiveSupport::TestCase + def site + @site ||= create :site + end + + teardown do + @site&.destroy + end + + test 'se pueden convertir los diccionarios' do + IndexedPost::DICTIONARIES.each do |locale, dict| + assert_equal dict, IndexedPost.to_dictionary(locale: locale) + end + end + + test 'se pueden buscar por categoría' do + assert(post = site.posts.create(title: SecureRandom.hex, description: SecureRandom.hex, + categories: [SecureRandom.hex, SecureRandom.hex])) + assert_not_empty site.indexed_posts.in_category(post.categories.value.sample) + end + + test 'se pueden encontrar por usuarie' do + usuarie = create :usuarie + assert(post = site.posts.create(title: SecureRandom.hex, description: SecureRandom.hex)) + + post.usuaries << usuarie + post.save + + assert_not_empty site.indexed_posts.by_usuarie(usuarie.id) + end +end diff --git a/test/models/post/indexable_test.rb b/test/models/post/indexable_test.rb new file mode 100644 index 0000000..6110bcf --- /dev/null +++ b/test/models/post/indexable_test.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'test_helper' + +class Post::IndexableTest < ActiveSupport::TestCase + setup do + @site = create :site + end + + teardown do + @site&.destroy + end + + test 'los posts se indexan apenas se crean' do + post = @site.posts.create(title: SecureRandom.hex, description: SecureRandom.hex) + indexed_post = @site.indexed_posts.find_by_title post.title.value + + assert indexed_post + assert_equal post.locale.value.to_s, indexed_post.locale + assert_equal post.order.value, indexed_post.order + assert_equal post.path.basename, indexed_post.path + assert_equal post.layout.name.to_s, indexed_post.layout + end + + test 'se pueden encontrar posts' do + post = @site.posts.sample + + assert @site.indexed_posts.where(locale: post.lang.value).search(post.lang.value, post.title.value) + assert @site.indexed_posts.where(locale: post.lang.value).search(post.lang.value, post.description.value) + end + + test 'se pueden actualizar posts' do + post = @site.posts.sample + post.description.value = SecureRandom.hex + + assert post.save + assert @site.indexed_posts.where(locale: post.lang.value).search(post.lang.value, post.description.value) + end + + test 'al borrar el post se borra el indice' do + post = @site.posts.sample + assert post.destroy + assert_not @site.indexed_posts.find_by_id(post.uuid.value) + end +end From 189b94e074cfb63bc14408584e7098eb48149b79 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 14 May 2021 17:19:03 -0300 Subject: [PATCH 25/33] =?UTF-8?q?aplicar=20cach=C3=A9=20a=20los=20par?= =?UTF-8?q?=C3=A1metros=20de=20b=C3=BAsqueda=20tambi=C3=A9n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/posts_controller.rb | 9 +++------ db/schema.rb | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index e197140..3ef2672 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -20,14 +20,11 @@ class PostsController < ApplicationController def index authorize Post - @site = find_site - @locale = locale - # XXX: Cada vez que cambiamos un Post tocamos el sitio con lo que es # más simple saber si hubo cambios. - if filter_params.present? || stale?([current_usuarie, @site]) + if stale?([current_usuarie, site, filter_params]) # Todos los artículos de este sitio para el idioma actual - @posts = @site.indexed_posts.where(locale: locale) + @posts = site.indexed_posts.where(locale: locale) # De este tipo @posts = @posts.where(layout: filter_params[:layout]) if filter_params[:layout] # Que estén dentro de la categoría @@ -38,7 +35,7 @@ class PostsController < ApplicationController @posts = PostPolicy::Scope.new(current_usuarie, @posts).resolve # Filtrar los posts que les invitades no pueden ver - @usuarie = @site.usuarie? current_usuarie + @usuarie = site.usuarie? current_usuarie end end diff --git a/db/schema.rb b/db/schema.rb index 8635424..107e7be 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_11_211357) do +ActiveRecord::Schema.define(version: 2021_05_14_165639) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" From df425bab5aa2c04ce9a79d951e066e58d8fb9ab1 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 17 May 2021 15:33:46 -0300 Subject: [PATCH 26/33] poder indexar los sitios! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit me había olvidado de este archivo --- app/models/site/index.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 app/models/site/index.rb diff --git a/app/models/site/index.rb b/app/models/site/index.rb new file mode 100644 index 0000000..e10fa52 --- /dev/null +++ b/app/models/site/index.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Indexa todos los artículos de un sitio +# +# TODO: Hacer opcional +class Site + module Index + extend ActiveSupport::Concern + + included do + # TODO: Debería ser un Job? + after_create :index_posts! + has_many :indexed_posts, dependent: :destroy + + def index_posts! + Site.transaction do + docs.each do |post| + post.to_index.save + end + end + end + end + end +end From 553b4f15f7719e8c663b927862689ce32f741bbe Mon Sep 17 00:00:00 2001 From: f Date: Mon, 17 May 2021 15:58:37 -0300 Subject: [PATCH 27/33] mostrar si es borrador sin acudir al sitio --- app/views/posts/index.haml | 3 +-- config/locales/en.yml | 2 ++ config/locales/es.yml | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 55ebcce..06287ba 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -103,8 +103,7 @@ = link_to site_post_path(@site, post.path) do %span{ lang: post.locale, dir: dir }= post.title - if post.front_matter['draft'].present? - %span.badge.badge-primary - = post_label_t(:draft, post: post) + %span.badge.badge-primary= I18n.t('posts.attributes.draft.label') - if post.front_matter['categories'].present? %br %small diff --git a/config/locales/en.yml b/config/locales/en.yml index 2705ad9..fc194ea 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -413,6 +413,8 @@ en: destroy: Remove image belongs_to: empty: "(Empty)" + draft: + label: Draft reorder: submit: 'Save order' select: 'Select this post' diff --git a/config/locales/es.yml b/config/locales/es.yml index fe1e318..eca7cee 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -420,6 +420,8 @@ es: destroy: 'Eliminar imagen' belongs_to: empty: "(Vacío)" + draft: + label: Borrador reorder: submit: 'Guardar orden' select: 'Seleccionar este artículo' From 0a23fe1edd475f11121d05e265e87be9d2472494 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 17 May 2021 17:27:01 -0300 Subject: [PATCH 28/33] lo que se indexa son los valores actuales, no todos los valores posibles --- app/models/metadata_belongs_to.rb | 4 ++++ app/models/metadata_related_posts.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/models/metadata_belongs_to.rb b/app/models/metadata_belongs_to.rb index ee182a5..1438c8d 100644 --- a/app/models/metadata_belongs_to.rb +++ b/app/models/metadata_belongs_to.rb @@ -77,6 +77,10 @@ class MetadataBelongsTo < MetadataRelatedPosts @related_methods ||= %i[belongs_to belonged_to].freeze end + def indexable_values + belongs_to&.title&.value + end + private def post_exists? diff --git a/app/models/metadata_related_posts.rb b/app/models/metadata_related_posts.rb index af91c28..092f219 100644 --- a/app/models/metadata_related_posts.rb +++ b/app/models/metadata_related_posts.rb @@ -23,7 +23,7 @@ class MetadataRelatedPosts < MetadataArray end def indexable_values - values.keys + posts.where(uuid: value).map(&:title).map(&:value) end private From 4609ab21b263c1f626fef071463f58bccf22c8bd Mon Sep 17 00:00:00 2001 From: f Date: Tue, 18 May 2021 11:22:09 -0300 Subject: [PATCH 29/33] =?UTF-8?q?separar=20las=20categor=C3=ADas=20con=20b?= =?UTF-8?q?arras?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/posts/index.haml | 1 + 1 file changed, 1 insertion(+) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 06287ba..8b77659 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -110,6 +110,7 @@ - post.front_matter['categories'].each do |category| = link_to site_posts_path(@site, **@filter_params.merge(category: category)) do %span{ lang: post.locale, dir: dir }= category + = '/' unless post.front_matter['categories'].last == category %td = post.created_at.strftime('%F') From 7191baff4abd7d6358769a5aebdd032e84c80aba Mon Sep 17 00:00:00 2001 From: f Date: Mon, 31 May 2021 13:23:40 -0300 Subject: [PATCH 30/33] =?UTF-8?q?reindexar=20despu=C3=A9s=20de=20mergear?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/sites_controller.rb | 2 +- app/services/site_service.rb | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/app/controllers/sites_controller.rb b/app/controllers/sites_controller.rb index 20ce5ba..bdaa901 100644 --- a/app/controllers/sites_controller.rb +++ b/app/controllers/sites_controller.rb @@ -107,7 +107,7 @@ class SitesController < ApplicationController def merge authorize site - if site.repository.merge(current_usuarie) + if SiteService.new(site: site, usuarie: current_usuarie).merge flash[:success] = I18n.t('sites.fetch.merge.success') else flash[:error] = I18n.t('sites.fetch.merge.error') diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 4f3905a..389549c 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -61,6 +61,18 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do commit_config(action: :tor) end + # Trae cambios desde la rama remota y reindexa los artículos. + # + # @return [Boolean] + def merge + result = site.repository.merge(usuarie) + + # TODO: Implementar callbacks + site.try(:index_posts!) if result + + result.present? + end + private # Guarda los cambios de la configuración en el repositorio git From 810c5e32da4157813b905c2320ee4c25ebe1f848 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 16 Jun 2021 09:11:13 -0300 Subject: [PATCH 31/33] convertir valores en arrays por retrocompatibilidad MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit esto permite indexar sitios antiguos y cargar artículos sin generar errores. --- app/models/metadata_array.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models/metadata_array.rb b/app/models/metadata_array.rb index 9f5a84b..96d3be3 100644 --- a/app/models/metadata_array.rb +++ b/app/models/metadata_array.rb @@ -23,6 +23,14 @@ class MetadataArray < MetadataTemplate value.join(', ') end + # Obtiene el valor desde el documento, convirtiéndolo a Array si no lo + # era ya, por retrocompabilidad. + # + # @return [Array] + def document_value + [super].flatten(1) + end + alias indexable_values values private From 78a3cae077fae9bf5415e24db782db34fc5c7477 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 16 Jun 2021 09:13:40 -0300 Subject: [PATCH 32/33] solo indexar los valores actuales MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sin este cambio los artículos se indexaban por todas las categorías posibles! --- app/models/metadata_array.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/metadata_array.rb b/app/models/metadata_array.rb index 96d3be3..0527ccb 100644 --- a/app/models/metadata_array.rb +++ b/app/models/metadata_array.rb @@ -31,7 +31,7 @@ class MetadataArray < MetadataTemplate [super].flatten(1) end - alias indexable_values values + alias indexable_values value private From c3a8b2401ceb249599e7dc39b16a6d0ec9d42b6f Mon Sep 17 00:00:00 2001 From: f Date: Wed, 16 Jun 2021 11:35:37 -0300 Subject: [PATCH 33/33] estandarizar la forma de obtener el valor de los documentos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit teníamos dos métodos que hacían lo mismo y generaban conflictos al obtener el valor por defecto de los arrays cuando no eran arrays. --- app/models/metadata_document_date.rb | 7 ++++--- app/models/metadata_lang.rb | 5 +++-- app/models/metadata_markdown_content.rb | 5 +++-- app/models/metadata_path.rb | 8 ++++++-- app/models/metadata_template.rb | 8 ++------ 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/app/models/metadata_document_date.rb b/app/models/metadata_document_date.rb index 34cbe1c..324e4be 100644 --- a/app/models/metadata_document_date.rb +++ b/app/models/metadata_document_date.rb @@ -7,7 +7,8 @@ class MetadataDocumentDate < MetadataTemplate Date.today.to_time end - def value_from_document + # @return [Time] + def document_value return nil if post.new? document.date @@ -45,10 +46,10 @@ class MetadataDocumentDate < MetadataTemplate begin Date.iso8601(self[:value]).to_time rescue Date::Error - value_from_document || default_value + document_value || default_value end else - self[:value] || value_from_document || default_value + self[:value] || document_value || default_value end end diff --git a/app/models/metadata_lang.rb b/app/models/metadata_lang.rb index 5f31ee9..ff6c08e 100644 --- a/app/models/metadata_lang.rb +++ b/app/models/metadata_lang.rb @@ -6,12 +6,13 @@ class MetadataLang < MetadataTemplate super || I18n.locale end - def value_from_document + # @return [Symbol] + def document_value document.collection.label.to_sym end def value - self[:value] ||= value_from_document || default_value + self[:value] ||= document_value || default_value end def values diff --git a/app/models/metadata_markdown_content.rb b/app/models/metadata_markdown_content.rb index d3cc6de..92a1ab2 100644 --- a/app/models/metadata_markdown_content.rb +++ b/app/models/metadata_markdown_content.rb @@ -9,14 +9,15 @@ class MetadataMarkdownContent < MetadataText end def value - self[:value] || value_from_document || default_value + self[:value] || document_value || default_value end def front_matter? false end - def value_from_document + # @return [String] + def document_value document.content end diff --git a/app/models/metadata_path.rb b/app/models/metadata_path.rb index 3c93cca..95fc7db 100644 --- a/app/models/metadata_path.rb +++ b/app/models/metadata_path.rb @@ -3,12 +3,16 @@ # Este campo representa el archivo donde se almacenan los datos class MetadataPath < MetadataTemplate # :label en este caso es el idioma/colección + # + # @return [String] def default_value File.join(site.path, "_#{lang}", "#{date}-#{slug}#{ext}") end - # El valor no vuelve desde el documento - def value_from_document + # La ruta del archivo según Jekyll + # + # @return [String] + def document_value document.path end diff --git a/app/models/metadata_template.rb b/app/models/metadata_template.rb index 638bdb8..76c3e7b 100644 --- a/app/models/metadata_template.rb +++ b/app/models/metadata_template.rb @@ -49,11 +49,7 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, def value_was return @value_was if instance_variable_defined? '@value_was' - @value_was = value_from_document - end - - def value_from_document - @value_from_document ||= document.data[name.to_s] + @value_was = document_value end def changed? @@ -85,7 +81,7 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, # Valor actual o por defecto. Al memoizarlo podemos modificarlo # usando otros métodos que el de asignación. def value - self[:value] ||= if (data = value_from_document).present? + self[:value] ||= if (data = document_value).present? private? ? decrypt(data) : data else default_value