From cc2b4a29719c249110908d953d4c330d279c2f06 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 12:26:36 -0300 Subject: [PATCH 01/49] dependencias del buscador MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pg_search: habilita búsquedas en postgresql hairtrigger: permite crear acciones automáticas en postgresql --- Gemfile | 6 +++++- Gemfile.lock | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 63b921c2..93a8a22e 100644 --- a/Gemfile +++ b/Gemfile @@ -51,7 +51,6 @@ gem 'sutty-liquid' gem 'lockbox' gem 'mini_magick' gem 'mobility' -gem 'pg' gem 'pundit' gem 'rails-i18n' gem 'rails_warden' @@ -67,6 +66,11 @@ gem 'validates_hostname' gem 'webpacker' gem 'yaml_db', git: 'https://0xacab.org/sutty/yaml_db.git' +# database +gem 'hairtrigger' +gem 'pg' +gem 'pg_search' + # performance gem 'flamegraph' gem 'memory_profiler' diff --git a/Gemfile.lock b/Gemfile.lock index 5c1d0627..5ddaeaaa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -217,6 +217,10 @@ GEM ffi (~> 1.0) globalid (0.4.2) activesupport (>= 4.2.0) + hairtrigger (0.2.24) + activerecord (>= 5.0, < 7) + ruby2ruby (~> 2.4) + ruby_parser (~> 3.10) haml (5.2.1) temple (>= 0.8.0) tilt @@ -387,6 +391,9 @@ GEM forwardable-extended (~> 2.6) pg (1.2.3) pg (1.2.3-x86_64-linux-musl) + pg_search (2.3.5) + activerecord (>= 5.2) + activesupport (>= 5.2) popper_js (1.16.0) prometheus_exporter (0.7.0) webrick @@ -523,7 +530,12 @@ GEM ruby-statistics (2.1.3) ruby-vips (2.1.0) ffi (~> 1.12) + ruby2ruby (2.4.4) + ruby_parser (~> 3.1) + sexp_processor (~> 4.6) ruby_dep (1.5.0) + ruby_parser (3.15.1) + sexp_processor (~> 4.9) rubyzip (2.3.0) rugged (1.1.0) rugged (1.1.0-x86_64-linux-musl) @@ -544,6 +556,7 @@ GEM childprocess (>= 0.5, < 4.0) rubyzip (>= 1.2.2) semantic_range (3.0.0) + sexp_processor (4.15.2) share-to-fediverse-jekyll-theme (0.1.4) jekyll (~> 4.0) jekyll-data (~> 1.1) @@ -677,6 +690,7 @@ DEPENDENCIES fast_jsonparser flamegraph friendly_id + hairtrigger haml-lint hamlit-rails hiredis @@ -699,6 +713,7 @@ DEPENDENCIES mobility net-ssh pg + pg_search prometheus_exporter pry puma From 655e07cd4091a61a54d170af9e0d27b0ecd29acb Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 12:29:48 -0300 Subject: [PATCH 02/49] =?UTF-8?q?configuraci=C3=B3n=20de=20las=20bases=20d?= =?UTF-8?q?e=20datos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ahora el testeo y desarrollo también se hace en postgresql. nos queda pendiente que cada quien pueda usar la base de datos que quiera y que si nos posible indexar documentos, que sea opcional. --- config/database.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/config/database.yml b/config/database.yml index 28e195b8..9e72da07 100644 --- a/config/database.yml +++ b/config/database.yml @@ -5,25 +5,24 @@ # gem 'sqlite3' # default: &default - adapter: sqlite3 + adapter: postgresql pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> timeout: 5000 + database: sutty + user: postgres + host: postgresql + encoding: unicode development: <<: *default - database: db/development.sqlite3 # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". # Do not set this db to the same as development or production. test: <<: *default - database: db/test.sqlite3 + database: sutty_test production: - adapter: postgresql - pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> - database: sutty + <<: *default user: sutty - host: postgresql - encoding: unicode From 26d186721d6df1f76a602caf176fd444e54f0a91 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 12:33:28 -0300 Subject: [PATCH 03/49] algunos metadatos son indexables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit solo los de texto que no se refieran a otros artículos por ahora --- app/models/metadata_array.rb | 8 ++++++++ app/models/metadata_content.rb | 8 ++++++++ app/models/metadata_document_date.rb | 4 ++++ app/models/metadata_related_posts.rb | 4 ++++ app/models/metadata_string.rb | 4 ++++ app/models/metadata_template.rb | 6 ++++++ 6 files changed, 34 insertions(+) diff --git a/app/models/metadata_array.rb b/app/models/metadata_array.rb index 5c0b16f7..5f43b790 100644 --- a/app/models/metadata_array.rb +++ b/app/models/metadata_array.rb @@ -13,6 +13,14 @@ class MetadataArray < MetadataTemplate false end + def indexable? + true + end + + def to_s + value.join(', ') + end + private # TODO: Sanitizar otros valores diff --git a/app/models/metadata_content.rb b/app/models/metadata_content.rb index 525f38ab..546e08c8 100644 --- a/app/models/metadata_content.rb +++ b/app/models/metadata_content.rb @@ -19,6 +19,14 @@ class MetadataContent < MetadataTemplate document.content end + def indexable? + true + end + + def to_s + sanitizer.sanitize value, tags: [], attributes: [] + end + private # Detectar si el contenido estaba en Markdown y pasarlo a HTML diff --git a/app/models/metadata_document_date.rb b/app/models/metadata_document_date.rb index 39e68735..a52cd051 100644 --- a/app/models/metadata_document_date.rb +++ b/app/models/metadata_document_date.rb @@ -11,6 +11,10 @@ class MetadataDocumentDate < MetadataTemplate document.date end + def indexable? + true + end + # El valor puede ser un Date, Time o una String en el formato # "yyyy-mm-dd" def value diff --git a/app/models/metadata_related_posts.rb b/app/models/metadata_related_posts.rb index 9a73dea9..bcc18d86 100644 --- a/app/models/metadata_related_posts.rb +++ b/app/models/metadata_related_posts.rb @@ -18,6 +18,10 @@ class MetadataRelatedPosts < MetadataArray false end + def indexable? + false + end + private # Obtiene todos los posts y opcionalmente los filtra diff --git a/app/models/metadata_string.rb b/app/models/metadata_string.rb index ed50bc88..724c2ef3 100644 --- a/app/models/metadata_string.rb +++ b/app/models/metadata_string.rb @@ -7,6 +7,10 @@ class MetadataString < MetadataTemplate super || '' end + def indexable? + true + end + private # No se permite HTML en las strings diff --git a/app/models/metadata_template.rb b/app/models/metadata_template.rb index 58000596..2fa8a61e 100644 --- a/app/models/metadata_template.rb +++ b/app/models/metadata_template.rb @@ -7,6 +7,12 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, :value, :help, :required, :errors, :post, :layout, keyword_init: true) do + + # Determina si el campo es indexable + def indexable? + false + end + def inspect "#<#{self.class} site=#{site.name.inspect} post=#{post.id.inspect} value=#{value.inspect}>" end From ceaadeb7bf00e1907a2ff993253b18a127381bc6 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 12:45:33 -0300 Subject: [PATCH 04/49] =?UTF-8?q?crear=20la=20tabla=20de=20indexaci=C3=B3n?= =?UTF-8?q?=20de=20posts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit los posts se siguen guardando en el sitio jekyll, lo que guardamos en la base de datos es una representación indexable que tiene los datos mínimos de los posts para buscarlos por distintos parámetros. esto nos permite cargar la lista de artículos y filtrarla de distintas formas sin cargar todo jekyll en memoria, lo que reduciría el consumo de recursos y aceleraría el panel. ya tenemos caché así que el problema estaba mitigado, pero igual es un avance. ya que migramos la base de datos a postgresql, aparecieron todas las tablas y campos en el schema.rb, que es lo que usa rails para configurar una base de datos desde cero. --- ...10504224144_create_pg_search_extensions.rb | 9 + .../20210504224343_create_indexed_posts.rb | 44 ++++ db/schema.rb | 203 ++++++++++++++++-- 3 files changed, 235 insertions(+), 21 deletions(-) create mode 100644 db/migrate/20210504224144_create_pg_search_extensions.rb create mode 100644 db/migrate/20210504224343_create_indexed_posts.rb diff --git a/db/migrate/20210504224144_create_pg_search_extensions.rb b/db/migrate/20210504224144_create_pg_search_extensions.rb new file mode 100644 index 00000000..18eebe95 --- /dev/null +++ b/db/migrate/20210504224144_create_pg_search_extensions.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Habilitar las extensiones de búsqueda de texto libre +class CreatePgSearchExtensions < ActiveRecord::Migration[6.1] + def change + enable_extension 'plpgsql' + enable_extension 'pg_trgm' + end +end diff --git a/db/migrate/20210504224343_create_indexed_posts.rb b/db/migrate/20210504224343_create_indexed_posts.rb new file mode 100644 index 00000000..9cf21538 --- /dev/null +++ b/db/migrate/20210504224343_create_indexed_posts.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# Crea la tabla donde se indexa el contenido de los artículos, los +# IndexedPosts van a estar relacionados con un Post del mismo UUID. +# +# Solo contienen la información mínima necesaria para mostrar los +# resultados de búsqueda. +class CreateIndexedPosts < ActiveRecord::Migration[6.1] + def change + # Necesario para gen_random_uuid() + # + # XXX: En realidad no lo necesitamos porque cada IndexedPost va a + # tener el UUID del Post correspondiente. + enable_extension 'pgcrypto' + + create_table :indexed_posts, id: false do |t| + t.primary_key :id, :uuid, default: 'public.gen_random_uuid()' + t.belongs_to :site, index: true + t.timestamps + + # Filtramos por idioma + t.string :locale, default: 'simple', index: true + # Vamos a querer filtrar por layout + t.string :layout, null: false, index: true + # Esta es la ruta al artículo + t.string :path, null: false + # Queremos mostrar el título por separado + t.string :title, default: '' + # También vamos a mostrar las categorías + t.jsonb :front_matter, default: '{}' + t.string :content, default: '' + t.tsvector :indexed_content + + t.index :indexed_content, using: 'gin' + t.index :front_matter, using: 'gin' + end + + # Crea un trigger que actualiza el índice tsvector con el título y + # contenido del artículo y su idioma. + create_trigger(compatibility: 1).on(:indexed_posts).before(:insert, :update) do + "new.indexed_content := to_tsvector(('pg_catalog.' || new.locale)::regconfig, coalesce(new.title, '') || '\n' || coalesce(new.content,''));" + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 2a93c5f1..fed6934b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,16 +10,74 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_04_14_152728) do +ActiveRecord::Schema.define(version: 2021_05_04_224343) do -# Could not dump table "access_logs" because of following StandardError -# Unknown type '' for column 'id' + # These are extensions that must be enabled in order to support this database + enable_extension "pg_trgm" + enable_extension "pgcrypto" + enable_extension "plpgsql" + + create_table "access_logs", id: :uuid, default: nil, force: :cascade do |t| + t.string "host" + t.float "msec" + t.string "server_protocol" + t.string "request_method" + t.string "request_completion" + t.string "uri" + t.string "query_string" + t.integer "status" + t.string "sent_http_content_type" + t.string "sent_http_content_encoding" + t.string "sent_http_etag" + t.string "sent_http_last_modified" + t.string "http_accept" + t.string "http_accept_encoding" + t.string "http_accept_language" + t.string "http_pragma" + t.string "http_cache_control" + t.string "http_if_none_match" + t.string "http_dnt" + t.string "http_user_agent" + t.string "http_origin" + t.float "request_time" + t.integer "bytes_sent" + t.integer "body_bytes_sent" + t.integer "request_length" + t.string "http_connection" + t.string "pipe" + t.integer "connection_requests" + t.string "geoip2_data_country_name" + t.string "geoip2_data_city_name" + t.string "ssl_server_name" + t.string "ssl_protocol" + t.string "ssl_early_data" + t.string "ssl_session_reused" + t.string "ssl_curves" + t.string "ssl_ciphers" + t.string "ssl_cipher" + t.string "sent_http_x_xss_protection" + t.string "sent_http_x_frame_options" + t.string "sent_http_x_content_type_options" + t.string "sent_http_strict_transport_security" + t.string "nginx_version" + t.integer "pid" + t.string "remote_user" + t.boolean "crawler", default: false + t.string "http_referer" + t.index ["geoip2_data_city_name"], name: "index_access_logs_on_geoip2_data_city_name" + t.index ["geoip2_data_country_name"], name: "index_access_logs_on_geoip2_data_country_name" + t.index ["host"], name: "index_access_logs_on_host" + t.index ["http_origin"], name: "index_access_logs_on_http_origin" + t.index ["http_user_agent"], name: "index_access_logs_on_http_user_agent" + t.index ["status"], name: "index_access_logs_on_status" + t.index ["uri"], name: "index_access_logs_on_uri" + end create_table "action_text_rich_texts", force: :cascade do |t| t.string "name", null: false t.text "body" t.string "record_type", null: false - t.integer "record_id", null: false + t.bigint "record_id", null: false t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true @@ -28,8 +86,8 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do create_table "active_storage_attachments", force: :cascade do |t| t.string "name", null: false t.string "record_type", null: false - t.integer "record_id", null: false - t.integer "blob_id", null: false + t.bigint "record_id", null: false + t.bigint "blob_id", null: false t.datetime "created_at", null: false t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true @@ -40,7 +98,7 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.string "filename", null: false t.string "content_type" t.text "metadata" - t.integer "byte_size", null: false + t.bigint "byte_size", null: false t.string "checksum", null: false t.datetime "created_at", null: false t.string "service_name", null: false @@ -53,10 +111,65 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true end + create_table "blazer_audits", force: :cascade do |t| + t.bigint "user_id" + t.bigint "query_id" + t.text "statement" + t.string "data_source" + t.datetime "created_at" + t.index ["query_id"], name: "index_blazer_audits_on_query_id" + t.index ["user_id"], name: "index_blazer_audits_on_user_id" + end + + create_table "blazer_checks", force: :cascade do |t| + t.bigint "creator_id" + t.bigint "query_id" + t.string "state" + t.string "schedule" + t.text "emails" + t.text "slack_channels" + t.string "check_type" + t.text "message" + t.datetime "last_run_at" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["creator_id"], name: "index_blazer_checks_on_creator_id" + t.index ["query_id"], name: "index_blazer_checks_on_query_id" + end + + create_table "blazer_dashboard_queries", force: :cascade do |t| + t.bigint "dashboard_id" + t.bigint "query_id" + t.integer "position" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["dashboard_id"], name: "index_blazer_dashboard_queries_on_dashboard_id" + t.index ["query_id"], name: "index_blazer_dashboard_queries_on_query_id" + end + + create_table "blazer_dashboards", force: :cascade do |t| + t.bigint "creator_id" + t.text "name" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["creator_id"], name: "index_blazer_dashboards_on_creator_id" + end + + create_table "blazer_queries", force: :cascade do |t| + t.bigint "creator_id" + t.string "name" + t.text "description" + t.text "statement" + t.string "data_source" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["creator_id"], name: "index_blazer_queries_on_creator_id" + end + create_table "build_stats", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "deploy_id" + t.bigint "deploy_id" t.integer "bytes" t.float "seconds" t.string "action", null: false @@ -65,13 +178,27 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.index ["deploy_id"], name: "index_build_stats_on_deploy_id" end -# Could not dump table "csp_reports" because of following StandardError -# Unknown type 'uuid' for column 'id' + create_table "csp_reports", id: :uuid, default: nil, force: :cascade do |t| + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.string "disposition" + t.string "referrer" + t.string "blocked_uri" + t.string "document_uri" + t.string "effective_directive" + t.string "original_policy" + t.string "script_sample" + t.string "status_code" + t.string "violated_directive" + t.integer "column_number" + t.integer "line_number" + t.string "source_file" + end create_table "deploys", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "site_id" + t.bigint "site_id" t.string "type" t.text "values" t.index ["site_id"], name: "index_deploys_on_site_id" @@ -91,6 +218,24 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.string "designer_url" end + create_table "indexed_posts", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.bigint "site_id" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.string "locale", default: "simple" + t.string "layout", null: false + t.string "path", null: false + t.string "title", default: "" + t.jsonb "front_matter", default: "{}" + t.string "content", default: "" + t.tsvector "indexed_content" + t.index ["front_matter"], name: "index_indexed_posts_on_front_matter", using: :gin + t.index ["indexed_content"], name: "index_indexed_posts_on_indexed_content", using: :gin + t.index ["layout"], name: "index_indexed_posts_on_layout" + t.index ["locale"], name: "index_indexed_posts_on_locale" + t.index ["site_id"], name: "index_indexed_posts_on_site_id" + end + create_table "licencias", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -104,7 +249,7 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do create_table "log_entries", force: :cascade do |t| t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false - t.integer "site_id" + t.bigint "site_id" t.text "text" t.boolean "sent", default: false t.index ["site_id"], name: "index_log_entries_on_site_id" @@ -124,7 +269,7 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.string "key", null: false t.string "value" t.string "translatable_type" - t.integer "translatable_id" + t.bigint "translatable_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["translatable_id", "translatable_type", "key"], name: "index_mobility_string_translations_on_translatable_attribute" @@ -137,7 +282,7 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.string "key", null: false t.text "value" t.string "translatable_type" - t.integer "translatable_id" + t.bigint "translatable_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["translatable_id", "translatable_type", "key"], name: "index_mobility_text_translations_on_translatable_attribute" @@ -147,8 +292,8 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do create_table "roles", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "site_id" - t.integer "usuarie_id" + t.bigint "site_id" + t.bigint "usuarie_id" t.string "rol" t.boolean "temporal" t.index ["site_id", "usuarie_id"], name: "index_roles_on_site_id_and_usuarie_id", unique: true @@ -160,15 +305,14 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.string "name" - t.integer "design_id" - t.integer "licencia_id" + t.bigint "design_id" + t.bigint "licencia_id" t.string "status", default: "waiting" t.text "description" t.string "title" t.boolean "colaboracion_anonima", default: false t.boolean "contact", default: false t.string "private_key_ciphertext" - t.boolean "invitades", default: false t.boolean "acepta_invitades", default: false t.string "tienda_api_key_ciphertext", default: "" t.string "tienda_url", default: "" @@ -200,7 +344,7 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.datetime "invitation_accepted_at" t.integer "invitation_limit" t.string "invited_by_type" - t.integer "invited_by_id" + t.bigint "invited_by_id" t.integer "invitations_count", default: 0 t.string "lang", default: "es" t.index ["confirmation_token"], name: "index_usuaries_on_confirmation_token", unique: true @@ -208,11 +352,28 @@ ActiveRecord::Schema.define(version: 2021_04_14_152728) do t.index ["invitation_token"], name: "index_usuaries_on_invitation_token", unique: true t.index ["invitations_count"], name: "index_usuaries_on_invitations_count" t.index ["invited_by_id"], name: "index_usuaries_on_invited_by_id" - t.index ["invited_by_type", "invited_by_id"], name: "index_usuaries_on_invited_by_type_and_invited_by_id" + t.index ["invited_by_type", "invited_by_id"], name: "index_usuaries_on_invited_by" t.index ["reset_password_token"], name: "index_usuaries_on_reset_password_token", unique: true t.index ["unlock_token"], name: "index_usuaries_on_unlock_token", unique: true end add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + # no candidate create_trigger statement could be found, creating an adapter-specific one + execute(<<-SQL) +CREATE OR REPLACE FUNCTION public.indexed_posts_before_insert_update_row_tr() + RETURNS trigger + LANGUAGE plpgsql +AS $function$ +BEGIN + new.indexed_content := to_tsvector(('pg_catalog.' || new.locale)::regconfig, coalesce(new.title, '') || ' + ' || coalesce(new.content,'')); + RETURN NEW; +END; +$function$ + SQL + + # no candidate create_trigger statement could be found, creating an adapter-specific one + execute("CREATE TRIGGER indexed_posts_before_insert_update_row_tr BEFORE INSERT OR UPDATE ON \"indexed_posts\" FOR EACH ROW EXECUTE PROCEDURE indexed_posts_before_insert_update_row_tr()") + end From 86ae7fb8f81bb9cb74c7a76215cbef49624e0d09 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 12:52:30 -0300 Subject: [PATCH 05/49] los posts pueden ser indexados MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `IndexedPost` es una representación indexada por PG de `Post`. ambos están relacionados por el UUID de `Post`, de forma que se puede traer el artículo completo (por ejemplo al previsualizar o editar). cada artículo está indexado según su idioma. para eso convertimos el locale en el equivalente en el diccionario de PG. `Site#index_posts` es un método para indexar todos los artículos en masa. `Post#to_index` genera el `IndexedPost` correspondiente `IndexedPost.search(:es, 'hola')` busca "hola" en todos los artículos utilizando el diccionario de castellano. esto no quiere decir que busque en todos los artículos en castellano. por ahora para eso hay que hacer algo como: ```ruby site = Site.find 1 site.indexed_posts.where(locale: :english).search(:en, 'hello') ``` para encontrar todos los artículos en inglés del sitio con id 1 --- app/models/indexed_post.rb | 38 ++++++++++++++++++++++++ app/models/post.rb | 4 +++ app/models/post/indexable.rb | 56 ++++++++++++++++++++++++++++++++++++ app/models/site.rb | 2 ++ 4 files changed, 100 insertions(+) create mode 100644 app/models/indexed_post.rb create mode 100644 app/models/post/indexable.rb diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb new file mode 100644 index 00000000..6456a824 --- /dev/null +++ b/app/models/indexed_post.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +# La representación indexable de un artículo +class IndexedPost < ApplicationRecord + include PgSearch::Model + + # La traducción del locale según Sutty al locale según PostgreSQL + DICTIONARIES = { + es: 'spanish', + en: 'english' + }.freeze + + # TODO: Los indexed posts tienen que estar scopeados al idioma actual, + # no buscar sobre todos + pg_search_scope :search, + lambda { |locale, query| + { + against: :content, + query: query, + using: { + tsearch: { + dictionary: IndexedPost.to_dictionary(locale: locale), + tsvector_column: 'indexed_content' + } + } + } + } + + belongs_to :site + + # Convertir locale a direccionario de PG + # + # @param [String,Symbol] + # @return [String] + def self.to_dictionary(locale:) + DICTIONARIES[locale.to_sym] || 'simple' + end +end diff --git a/app/models/post.rb b/app/models/post.rb index a0e16706..6ef7a2ec 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -17,6 +17,8 @@ class Post attr_reader :attributes, :errors, :layout, :site, :document + include Post::Indexable + class << self # Obtiene el layout sin leer el Document # @@ -191,6 +193,8 @@ class Post post: self, required: true) end + alias locale lang + # TODO: Mover a method_missing def uuid @metadata[:uuid] ||= MetadataUuid.new(document: document, site: site, layout: layout, name: :uuid, type: :uuid, diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb new file mode 100644 index 00000000..f25f28d2 --- /dev/null +++ b/app/models/post/indexable.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +class Post + # Vuelve indexables a los Posts + module Indexable + extend ActiveSupport::Concern + + included do + # Devuelve una versión indexable del Post + # + # @return [IndexedPosts] + def to_index + @to_index ||= IndexedPost.find_or_create_by(id: uuid.value).tap do |indexed_post| + indexed_post.layout = layout.name + indexed_post.site_id = site.id + indexed_post.path = path.basename + indexed_post.locale = IndexedPost.to_dictionary(locale: locale.value) + indexed_post.title = title.value + indexed_post.front_matter = indexable_front_matter + indexed_post.content = indexable_content + end + end + + private + + # Los metadatos que se almacenan como objetos JSON. Empezamos con + # las categorías porque se usan para filtrar en el listado de + # artículos. + # + # @return [Hash] + def indexable_front_matter + return {} unless attribute? :categories + + { categories: categories.indexable_values } + end + + # Devuelve un documento indexable en texto plano + # + # XXX: No memoizamos para permitir actualizaciones, aunque + # probablemente se indexe una sola vez. + # + # @return [String] + def indexable_content + indexable_attributes.map do |attr| + self[attr].to_s + end.join("\n") + end + + def indexable_attributes + @indexable_attributes ||= attributes.select do |attr| + self[attr].indexable? + end + end + end + end +end diff --git a/app/models/site.rb b/app/models/site.rb index 1c62e9bc..14238e1a 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -7,6 +7,7 @@ class Site < ApplicationRecord include Site::Forms include Site::FindAndReplace include Site::Api + include Site::Index include Tienda # Cifrar la llave privada que cifra y decifra campos ocultos. Sutty @@ -37,6 +38,7 @@ class Site < ApplicationRecord belongs_to :design belongs_to :licencia + has_many :indexed_posts, dependent: :destroy has_many :log_entries, dependent: :destroy has_many :deploys, dependent: :destroy has_many :build_stats, through: :deploys From a149c870e2023f0b0f41957efebc114b0e50925c Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 13:00:05 -0300 Subject: [PATCH 06/49] =?UTF-8?q?las=20categor=C3=ADas=20se=20guardan=20en?= =?UTF-8?q?=20indexed=20posts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit algunas categorías son otros artículos relacionados, con este método garantizamos que sólo se guardan los títulos. --- app/models/metadata_array.rb | 2 ++ app/models/metadata_related_posts.rb | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/app/models/metadata_array.rb b/app/models/metadata_array.rb index 5f43b790..8d951a14 100644 --- a/app/models/metadata_array.rb +++ b/app/models/metadata_array.rb @@ -21,6 +21,8 @@ class MetadataArray < MetadataTemplate value.join(', ') end + alias indexable_values values + private # TODO: Sanitizar otros valores diff --git a/app/models/metadata_related_posts.rb b/app/models/metadata_related_posts.rb index bcc18d86..4c022fff 100644 --- a/app/models/metadata_related_posts.rb +++ b/app/models/metadata_related_posts.rb @@ -22,6 +22,10 @@ class MetadataRelatedPosts < MetadataArray false end + def indexable_values + values.keys + end + private # Obtiene todos los posts y opcionalmente los filtra From d61d1cad565636c800f4f7642c065ad62d99de9d Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 19:07:11 -0300 Subject: [PATCH 07/49] =?UTF-8?q?respetar=20el=20orden=20de=20los=20art?= =?UTF-8?q?=C3=ADculos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit siempre ordenamos primero por número de orden y fecha de creación, siempre decrecientes. esto permite que les usuaries prioricen contenido usando las herramientas de reordenamiento. pg_search no soporta esto, siempre ordena por cuánto corresponde el resultado con la búsqueda, así que lo emparchamos para que respete el orden que necesitamos. el reporte de error relacionado es este: https://github.com/Casecommons/pg_search/issues/467 --- app/models/indexed_post.rb | 3 +++ app/models/post/indexable.rb | 2 ++ config/initializers/core_extensions.rb | 13 +++++++++++++ .../20210506212356_add_order_to_indexed_posts.rb | 9 +++++++++ db/schema.rb | 3 ++- 5 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20210506212356_add_order_to_indexed_posts.rb diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb index 6456a824..c76a5c85 100644 --- a/app/models/indexed_post.rb +++ b/app/models/indexed_post.rb @@ -26,6 +26,9 @@ class IndexedPost < ApplicationRecord } } + # Trae los IndexedPost en el orden en que van a terminar en el sitio. + default_scope lambda { order(order: :desc, created_at: :desc) } + belongs_to :site # Convertir locale a direccionario de PG diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index f25f28d2..9b92111f 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -18,6 +18,8 @@ class Post indexed_post.title = title.value indexed_post.front_matter = indexable_front_matter indexed_post.content = indexable_content + indexed_post.created_at = date.value + indexed_post.order = attribute?(:order) ? order.value : 0 end end diff --git a/config/initializers/core_extensions.rb b/config/initializers/core_extensions.rb index e37b2be4..84dea42e 100644 --- a/config/initializers/core_extensions.rb +++ b/config/initializers/core_extensions.rb @@ -66,3 +66,16 @@ module Jekyll end end end + +# No aplicar el orden por ranking para poder obtener los artículos en el +# orden que tendrían en el sitio final. +module PgSearch + ScopeOptions.class_eval do + def apply(scope) + scope = include_table_aliasing_for_rank(scope) + rank_table_alias = scope.pg_search_rank_table_alias(include_counter: true) + + scope.joins(rank_join(rank_table_alias)) + end + end +end diff --git a/db/migrate/20210506212356_add_order_to_indexed_posts.rb b/db/migrate/20210506212356_add_order_to_indexed_posts.rb new file mode 100644 index 00000000..4b1a9fcf --- /dev/null +++ b/db/migrate/20210506212356_add_order_to_indexed_posts.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Agrega el orden y fecha del post en el post indexado, de forma que los +# resultados se puedan obtener en el mismo orden. +class AddOrderToIndexedPosts < ActiveRecord::Migration[6.1] + def change + add_column :indexed_posts, :order, :integer, default: 0 + end +end diff --git a/db/schema.rb b/db/schema.rb index fed6934b..ed629401 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_04_224343) do +ActiveRecord::Schema.define(version: 2021_05_06_212356) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -229,6 +229,7 @@ ActiveRecord::Schema.define(version: 2021_05_04_224343) do t.jsonb "front_matter", default: "{}" t.string "content", default: "" t.tsvector "indexed_content" + t.integer "order", default: 0 t.index ["front_matter"], name: "index_indexed_posts_on_front_matter", using: :gin t.index ["indexed_content"], name: "index_indexed_posts_on_indexed_content", using: :gin t.index ["layout"], name: "index_indexed_posts_on_layout" From fd7ab8d7ef1190ce460eb967f922182992bdacdf Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 19:46:36 -0300 Subject: [PATCH 08/49] =?UTF-8?q?actualizar=20el=20=C3=ADndice=20cuando=20?= =?UTF-8?q?se=20agrega=20o=20modifica=20un=20post?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit además, activa callbacks de activerecord de forma que Post se va pareciendo cada vez más a un modelo de rails :D --- app/models/post.rb | 7 +++++-- app/models/post/indexable.rb | 12 +++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/app/models/post.rb b/app/models/post.rb index 790ee1a2..7964d4ee 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -17,6 +17,7 @@ class Post attr_reader :attributes, :errors, :layout, :site, :document + include ActiveRecord::Callbacks include Post::Indexable class << self @@ -285,8 +286,10 @@ class Post end end - return false unless save_attributes! - return false unless write + run_callbacks :save do + return false unless save_attributes! + return false unless write + end # Vuelve a leer el post para tomar los cambios read diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index 9b92111f..b7b5a7fa 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -6,11 +6,14 @@ class Post extend ActiveSupport::Concern included do + # Indexa o reindexa el Post + after_save :index! + # Devuelve una versión indexable del Post # # @return [IndexedPosts] def to_index - @to_index ||= IndexedPost.find_or_create_by(id: uuid.value).tap do |indexed_post| + IndexedPost.find_or_create_by(id: uuid.value).tap do |indexed_post| indexed_post.layout = layout.name indexed_post.site_id = site.id indexed_post.path = path.basename @@ -25,6 +28,13 @@ class Post private + # Indexa o reindexa el Post + # + # @return [Boolean] + def index! + to_index.save + end + # Los metadatos que se almacenan como objetos JSON. Empezamos con # las categorías porque se usan para filtrar en el listado de # artículos. From 34a05e860d7248174ad197436cab8adf121107e7 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 6 May 2021 19:47:43 -0300 Subject: [PATCH 09/49] =?UTF-8?q?elimina=20el=20espacio=20vac=C3=ADo=20del?= =?UTF-8?q?=20contenido?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/post/indexable.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index b7b5a7fa..b887deac 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -54,8 +54,8 @@ class Post # @return [String] def indexable_content indexable_attributes.map do |attr| - self[attr].to_s - end.join("\n") + self[attr].to_s.remove("\n") + end.join("\n").squeeze("\n") end def indexable_attributes From ad871baca63bf0b1e5454575473fcdfa7d04e09d Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 16:17:25 -0300 Subject: [PATCH 10/49] implementar el buscador en el panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ahora el índice de artículos incorporar buscador de texto libre. además todos los filtros de búsqueda se mantienen entre búsquedas, entonces al filtrar por tipo de artículo y término, se aplican ambos y al cambiar el tipo se mantiene la búsqueda de texto. --- app/assets/stylesheets/application.scss | 4 ++ app/controllers/posts_controller.rb | 38 ++++++----- app/models/indexed_post.rb | 2 + app/models/post/indexable.rb | 13 +++- app/policies/post_policy.rb | 4 +- app/views/posts/index.haml | 87 ++++++++++++------------- config/locales/en.yml | 1 + config/locales/es.yml | 1 + 8 files changed, 82 insertions(+), 68 deletions(-) diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index 0a215c48..1f9b634e 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -21,6 +21,10 @@ $form-feedback-invalid-color: $magenta; $form-feedback-icon-valid-color: $black; $component-active-bg: $magenta; +$spacers: ( + 2-plus: 0.75rem +); + @import "bootstrap"; @import "editor"; diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index a4b47a16..524335a5 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -16,29 +16,25 @@ class PostsController < ApplicationController authorize Post @site = find_site - @category = params.dig(:category) - @layout = params.dig(:layout) - @locale = locale + @q = params[:q] + dictionary = IndexedPost.to_dictionary(locale: locale) # XXX: Cada vez que cambiamos un Post tocamos el sitio con lo que es # más simple saber si hubo cambios. - if @category || @layout || stale?(@site) - @posts = @site.posts(lang: locale) - @posts = @posts.where(categories: @category) if @category - @posts = @posts.where(layout: @layout) if @layout + if filter_params.present? || stale?(@site) + # Todos los artículos de este sitio para el idioma actual + @posts = @site.indexed_posts.where(locale: dictionary) + # De este tipo + @posts = @posts.where(layout: filter_params[:layout]) if filter_params[:layout] + # Que estén dentro de la categoría + @posts = @posts.in_category(filter_params[:category]) if filter_params[:category] + # Aplicar los parámetros de búsqueda + @posts = @posts.search(locale, filter_params[:q]) if filter_params[:q].present? + # A los que este usuarie tiene acceso @posts = PostPolicy::Scope.new(current_usuarie, @posts).resolve - @category_name = if uuid?(@category) - @site.posts(lang: locale).find(@category, uuid: true)&.title&.value - else - @category - end - # Filtrar los posts que les invitades no pueden ver @usuarie = @site.usuarie? current_usuarie - - # Orden descendiente por número y luego por fecha - @posts.sort_by!(:order, :date).reverse! end end @@ -169,4 +165,14 @@ class PostsController < ApplicationController def forget_content flash[:js] = { target: 'editor', action: 'forget-content', keys: (params[:storage_keys] || []).to_json } end + + private + + # Los parámetros de filtros que vamos a mantener en todas las URLs, + # solo los que no estén vacíos. + # + # @return [Hash] + def filter_params + @filter_params ||= params.permit(:q, :category, :layout).to_h.select { |_,v| v.present? } + end end diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb index c76a5c85..7bf1ec7a 100644 --- a/app/models/indexed_post.rb +++ b/app/models/indexed_post.rb @@ -28,6 +28,8 @@ class IndexedPost < ApplicationRecord # Trae los IndexedPost en el orden en que van a terminar en el sitio. default_scope lambda { order(order: :desc, created_at: :desc) } + scope :in_category, lambda { |category| where("front_matter->'categories' ? :category", category: category.to_s) } + scope :by_usuarie, lambda { |usuarie| where("front_matter->'usuaries' @> :usuarie::jsonb", usuarie: usuarie.to_s) } belongs_to :site diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index b887deac..bee02e54 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -11,7 +11,7 @@ class Post # Devuelve una versión indexable del Post # - # @return [IndexedPosts] + # @return [IndexedPost] def to_index IndexedPost.find_or_create_by(id: uuid.value).tap do |indexed_post| indexed_post.layout = layout.name @@ -41,9 +41,16 @@ class Post # # @return [Hash] def indexable_front_matter - return {} unless attribute? :categories + {}.tap do |indexable_front_matter| + indexable_front_matter = { + usuaries: usuaries.map(&:id), + draft: attribute?(:draft) ? draft.value : false + } - { categories: categories.indexable_values } + if attribute? :categories + indexable_front_matter[:categories] = categories.indexable_values + end + end end # Devuelve un documento indexable en texto plano diff --git a/app/policies/post_policy.rb b/app/policies/post_policy.rb index c22202af..69ecb188 100644 --- a/app/policies/post_policy.rb +++ b/app/policies/post_policy.rb @@ -59,9 +59,7 @@ class PostPolicy def resolve return scope if scope&.first&.site&.usuarie? usuarie - scope.select do |post| - post.usuaries.include? usuarie - end + scope.by_usuarie(usuarie.id) end end end diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index b2f3f661..9c43e431 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -3,7 +3,7 @@ @site.name, link_to(t('posts.index'), site_posts_path(@site)), - @category_name] + @category] %main.row %aside.menu.col-md-3 @@ -14,15 +14,13 @@ %table.mb-3 - @site.layouts.each do |layout| - next if layout.hidden? - - filter = params[:layout] == layout.value %tr %th= layout.humanized_name - %td.pl-3= link_to t('posts.add'), - new_site_post_path(@site, layout: layout.name), - class: 'badge badge-secondary' - %td= link_to t(filter ? 'posts.remove_filter' : 'posts.filter'), - site_posts_path(@site, layout: (filter ? nil : layout.value)), - class: 'badge badge-' + (filter ? 'primary' : 'secondary') + %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'badge badge-secondary' + - if @filter_params[:layout] == layout.value + %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'badge badge-primary' + - else + %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'badge badge-secondary' - if policy(@site).edit? = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn' @@ -48,19 +46,24 @@ %section.col = render 'layouts/flash' + .d-flex.justify-content-between.align-items-center.pl-2-plus.pr-2-plus.mb-2 + %form + - @filter_params.each do |param, value| + - next if param == 'q' + %input{ type: 'input', name: param, value: value } + .form-group.flex-grow-0.m-0 + %input.form-control.border.border-magenta{ type: 'search', placeholder: 'Buscar', name: 'q', value: @q } + %input.sr-only{ type: 'submit' } + - if @site.locales.size > 1 + + %nav#locales + - @site.locales.each do |locale| + = link_to t("locales.#{locale}.name"), site_posts_path(@site, **@filter_params.merge(locale: locale)), + class: "mr-2 mt-2 mb-2 #{locale == @locale ? 'active font-weight-bold' : ''}" - if @posts.empty? - %h2= t('posts.none') + %h2= t('posts.empty') - else = form_tag site_posts_reorder_path, method: :post do - .d-flex.justify-content-between.align-items-center - -# - TODO: Pensar una interfaz mejor para cuando haya más de tres - idiomas - - unless @site.locales.length == 1 - .locales - - @site.locales.each do |locale| - = link_to t("locales.#{locale}.name"), site_posts_path(@site, locale: locale), - class: "mr-2 mt-2 mb-2#{locale == @locale ? 'active font-weight-bold' : ''}" %table.table{ data: { controller: 'reorder' } } %caption.sr-only= t('posts.caption') %thead @@ -76,6 +79,7 @@ %button.btn{ data: { action: 'reorder#bottom' } }= t('posts.reorder.bottom') %tbody - dir = t("locales.#{@locale}.dir") + - size = @posts.size - @posts.each_with_index do |post, i| -# TODO: Solo les usuaries cachean porque tenemos que separar @@ -84,45 +88,36 @@ TODO: Verificar qué pasa cuando se gestiona el sitio en distintos idiomas a la vez - cache_if @usuarie, post do - - checkbox_id = "checkbox-#{post.uuid.value}" - %tr{ id: post.uuid.value, data: { target: 'reorder.row' } } + - checkbox_id = "checkbox-#{post.id}" + %tr{ id: post.id, data: { target: 'reorder.row' } } %td .custom-control.custom-checkbox %input.custom-control-input{ id: checkbox_id, type: 'checkbox', autocomplete: 'off', data: { action: 'reorder#select' } } %label.custom-control-label{ for: checkbox_id } %span.sr-only= t('posts.reorder.select') -# Orden más alto es mayor prioridad - = hidden_field 'post[reorder]', post.uuid.value, - value: @posts.length - i, + = hidden_field 'post[reorder]', post.id, + value: size - i, data: { reorder: true } %td.w-100{ class: dir } - = link_to site_post_path(@site, post.id) do - %span{ lang: post.lang.value, dir: dir }= post.title.value - - if post.attributes.include? :draft - - if post.draft.value - %span.badge.badge-primary - = post_label_t(:draft, post: post) - - if post.attributes.include? :categories - - unless post.categories.value.empty? - %br - %small - - (post.categories.respond_to?(:belongs_to) ? post.categories.belongs_to : post.categories.value).each do |c| - = link_to site_posts_path(@site, category: (c.respond_to?(:uuid) ? c.uuid.value : c)) do - %span{ lang: post.lang.value, dir: dir }= (c.respond_to?(:title) ? c.title.value : c) + = link_to site_post_path(@site, post.path) do + %span{ lang: post.locale, dir: dir }= post.title + - if post.front_matter['draft'].present? + %span.badge.badge-primary + = post_label_t(:draft, post: post) + - if post.front_matter['categories'].present? + %br + %small + - post.front_matter['categories'].each do |category| + = link_to site_posts_path(@site, **@filter_params.merge(category: category)) do + %span{ lang: post.locale, dir: dir }= category %td - = post.date.value.strftime('%F') + = post.created_at.strftime('%F') %br/ - - if post.attribute? :order - = post.order.value + = post.order %td - if @usuarie || policy(post).edit? - = link_to t('posts.edit'), - edit_site_post_path(@site, post.id), - class: 'btn btn-block' + = link_to t('posts.edit'), edit_site_post_path(@site, post.path), class: 'btn btn-block' - if @usuarie || policy(post).destroy? - = link_to t('posts.destroy'), - site_post_path(@site, post.id), - class: 'btn btn-block', - method: :delete, - data: { confirm: t('posts.confirm_destroy') } + = link_to t('posts.destroy'), site_post_path(@site, post.path), class: 'btn btn-block', method: :delete, data: { confirm: t('posts.confirm_destroy') } diff --git a/config/locales/en.yml b/config/locales/en.yml index dd9f8309..54b6115c 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -397,6 +397,7 @@ en: en: 'English' ar: 'Arabic' posts: + empty: "There are no results for those search parameters." attribute_ro: file: download: Download file diff --git a/config/locales/es.yml b/config/locales/es.yml index 403389ab..b98ae149 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -459,6 +459,7 @@ es: en: 'inglés' ar: 'árabe' posts: + empty: No hay artículos con estos parámetros de búsqueda. caption: Lista de artículos attribute_ro: file: From 4f2e602822bce8db394265c75b5e51abb195db11 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 16:21:44 -0300 Subject: [PATCH 11/49] =?UTF-8?q?tambi=C3=A9n=20buscar=20por=20palabras=20?= =?UTF-8?q?similares?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/indexed_post.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb index 7bf1ec7a..ede5b539 100644 --- a/app/models/indexed_post.rb +++ b/app/models/indexed_post.rb @@ -21,6 +21,9 @@ class IndexedPost < ApplicationRecord tsearch: { dictionary: IndexedPost.to_dictionary(locale: locale), tsvector_column: 'indexed_content' + }, + trigram: { + word_similarity: true } } } From 2d3f5b21aef17ddef38c8fa53b8c850936a93a24 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 16:21:59 -0300 Subject: [PATCH 12/49] =?UTF-8?q?al=20eliminar=20los=20saltos=20de=20l?= =?UTF-8?q?=C3=ADnea=20se=20quedaban=20pegadas=20algunas=20palabras?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/post/indexable.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index bee02e54..8a12e40c 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -61,7 +61,7 @@ class Post # @return [String] def indexable_content indexable_attributes.map do |attr| - self[attr].to_s.remove("\n") + self[attr].to_s.tr("\n", ' ') end.join("\n").squeeze("\n") end From 6df7f09c26789ea98aa2bb83bd38b69c34348321 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:11:08 -0300 Subject: [PATCH 13/49] =?UTF-8?q?darle=20estilo=20de=20bot=C3=B3n=20a=20lo?= =?UTF-8?q?s=20filtros?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit como eran badges con links no tenían sombra en el foco ni color al pasarles por encima. --- app/views/posts/index.haml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 9c43e431..a9f868e9 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -16,11 +16,11 @@ - next if layout.hidden? %tr %th= layout.humanized_name - %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'badge badge-secondary' + %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'btn btn-secondary badge' - if @filter_params[:layout] == layout.value - %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'badge badge-primary' + %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'btn btn-primary badge' - else - %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'badge badge-secondary' + %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary badge' - if policy(@site).edit? = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn' From 418ffb846376852f11bf8d21de67c21a271c3086 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:31:13 -0300 Subject: [PATCH 14/49] =?UTF-8?q?cambiar=20btn-sm=20para=20que=20tenga=20e?= =?UTF-8?q?l=20tama=C3=B1o=20de=20badge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/assets/stylesheets/application.scss | 4 ++++ app/views/posts/index.haml | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index 1f9b634e..e059ecd1 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -214,6 +214,10 @@ svg { } } +.btn-sm { + @extend .badge +} + .black-bg { color: $white; background-color: $black; diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index a9f868e9..11e6efcb 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -16,11 +16,11 @@ - next if layout.hidden? %tr %th= layout.humanized_name - %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'btn btn-secondary badge' + %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'btn btn-secondary btn-sm' - if @filter_params[:layout] == layout.value - %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'btn btn-primary badge' + %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'btn btn-primary btn-sm' - else - %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary badge' + %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary btn-sm' - if policy(@site).edit? = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn' From 051e40f64bc876a9bef335743451b3d489da5c3b Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:31:50 -0300 Subject: [PATCH 15/49] el formulario de busqueda tiene que incorporar los filtros MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit para poder volverlos a enviar. me había olvidado de convertirlos en campos ocultos. --- app/views/posts/index.haml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 11e6efcb..8fcb86b5 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -50,12 +50,12 @@ %form - @filter_params.each do |param, value| - next if param == 'q' - %input{ type: 'input', name: param, value: value } + %input{ type: 'hidden', name: param, value: value } .form-group.flex-grow-0.m-0 %input.form-control.border.border-magenta{ type: 'search', placeholder: 'Buscar', name: 'q', value: @q } %input.sr-only{ type: 'submit' } - - if @site.locales.size > 1 + - if @site.locales.size > 1 %nav#locales - @site.locales.each do |locale| = link_to t("locales.#{locale}.name"), site_posts_path(@site, **@filter_params.merge(locale: locale)), From 754e8fbbcc19b9df0793694f1415f137a8a74c34 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:32:54 -0300 Subject: [PATCH 16/49] =?UTF-8?q?mostrar=20cu=C3=A1les=20son=20los=20filtr?= =?UTF-8?q?os=20actuales=20y=20poder=20removerlos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit además, crear una sección de notas al pie con definiciones disponibles para lectores de pantalla. (no quiere decir que sean completas, `title` solo funciona con usuaries de mouse) --- app/views/posts/index.haml | 16 ++++++++++++++++ config/locales/en.yml | 1 + config/locales/es.yml | 1 + 3 files changed, 18 insertions(+) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 8fcb86b5..717460c3 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -60,6 +60,16 @@ - @site.locales.each do |locale| = link_to t("locales.#{locale}.name"), site_posts_path(@site, **@filter_params.merge(locale: locale)), class: "mr-2 mt-2 mb-2 #{locale == @locale ? 'active font-weight-bold' : ''}" + .pl-2-plus + - @filter_params.each do |param, value| + - if param == 'layout' + - value = @site.layouts[value.to_sym].humanized_name + = link_to site_posts_path(@site, **@filter_params.reject { |k, _| k == param }), + class: 'btn btn-secondary btn-sm', + title: t('posts.remove_filter_help', filter: value), + aria: { labelledby: "help-filter-#{param}" } do + = value + × - if @posts.empty? %h2= t('posts.empty') - else @@ -121,3 +131,9 @@ = link_to t('posts.edit'), edit_site_post_path(@site, post.path), class: 'btn btn-block' - if @usuarie || policy(post).destroy? = link_to t('posts.destroy'), site_post_path(@site, post.path), class: 'btn btn-block', method: :delete, data: { confirm: t('posts.confirm_destroy') } + +#footnotes{ hidden: true } + - @filter_params.each do |param, value| + - if param == 'layout' + - value = @site.layouts[value.to_sym].humanized_name + %label{ id: "help-filter-#{param}" }= t('posts.remove_filter_help', filter: value) diff --git a/config/locales/en.yml b/config/locales/en.yml index 54b6115c..da1028f5 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -451,6 +451,7 @@ en: add: 'Add' filter: 'Filter' remove_filter: 'Back' + remove_filter_help: 'Remove the filter: %{filter}' categories: 'Everything' index: 'Posts' edit: 'Edit' diff --git a/config/locales/es.yml b/config/locales/es.yml index b98ae149..a9e8cacb 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -515,6 +515,7 @@ es: add: 'Agregar' filter: 'Filtrar' remove_filter: 'Volver' + remove_filter_help: 'Quitar este filtro: %{filter}' index: 'Artículos' edit: 'Editar' preview: From e1055cc91240653ba37cb11dfaa4c767bab43510 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:45:07 -0300 Subject: [PATCH 17/49] necesitamos el locale para poder marcar el idioma actual --- app/controllers/posts_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index 524335a5..4acf7656 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -16,7 +16,7 @@ class PostsController < ApplicationController authorize Post @site = find_site - @q = params[:q] + @locale = locale dictionary = IndexedPost.to_dictionary(locale: locale) # XXX: Cada vez que cambiamos un Post tocamos el sitio con lo que es From f9a2d12803a132fc0536df2f4726803dfd425544 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:46:49 -0300 Subject: [PATCH 18/49] mostrar el texto buscado en el buscador para poder editarlo si hace falta --- app/views/posts/index.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 717460c3..23772237 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -52,7 +52,7 @@ - next if param == 'q' %input{ type: 'hidden', name: param, value: value } .form-group.flex-grow-0.m-0 - %input.form-control.border.border-magenta{ type: 'search', placeholder: 'Buscar', name: 'q', value: @q } + %input.form-control.border.border-magenta{ type: 'search', placeholder: 'Buscar', name: 'q', value: @filter_params[:q] } %input.sr-only{ type: 'submit' } - if @site.locales.size > 1 From 13b6b7a452a3f6ffd2bda523c7b1e905b180f2d7 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 17:54:45 -0300 Subject: [PATCH 19/49] =?UTF-8?q?cambiar=20la=20leyenda=20del=20bot=C3=B3n?= =?UTF-8?q?=20en=20el=20filtro?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit no cambia el resaltado porque la definición de la clase .btn hace conflicto, para arreglar eso hay que verificar todo el panel. --- app/views/posts/index.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 23772237..26a0f3ee 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -17,7 +17,7 @@ %tr %th= layout.humanized_name %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'btn btn-secondary btn-sm' - - if @filter_params[:layout] == layout.value + - if @filter_params[:layout] == layout.name.to_s %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'btn btn-primary btn-sm' - else %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary btn-sm' From d2ae406023af3d54c0201ff73db31af79fd363ab Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 18:06:49 -0300 Subject: [PATCH 20/49] =?UTF-8?q?eliminar=20del=20=C3=ADndice=20al=20elimi?= =?UTF-8?q?nar=20el=20art=C3=ADculo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/post.rb | 15 ++++++++++++--- app/models/post/indexable.rb | 5 +++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/app/models/post.rb b/app/models/post.rb index 7964d4ee..a64bd551 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -17,6 +17,9 @@ class Post attr_reader :attributes, :errors, :layout, :site, :document + # TODO: Modificar el historial de Git con callbacks en lugar de + # services. De esta forma podríamos agregar soporte para distintos + # backends. include ActiveRecord::Callbacks include Post::Indexable @@ -258,11 +261,17 @@ class Post end # Eliminar el artículo del repositorio y de la lista de artículos del - # sitio + # sitio. + # + # TODO: Si el callback falla deberíamos recuperar el archivo. + # + # @return [Post] def destroy - FileUtils.rm_f path.absolute + run_callbacks :destroy do + FileUtils.rm_f path.absolute - site.delete_post self + site.delete_post self + end end alias destroy! destroy diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index 8a12e40c..0baa8012 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -8,6 +8,7 @@ class Post included do # Indexa o reindexa el Post after_save :index! + after_destroy :remove_from_index! # Devuelve una versión indexable del Post # @@ -35,6 +36,10 @@ class Post to_index.save end + def remove_from_index! + to_index.destroy.destroyed? + end + # Los metadatos que se almacenan como objetos JSON. Empezamos con # las categorías porque se usan para filtrar en el listado de # artículos. From 6396841b2c4401d37a345dc351a73c2c3abe4582 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 7 May 2021 19:25:09 -0300 Subject: [PATCH 21/49] evitar confusiones entre diccionario y locale el diccionario es lo que usa internamente pg para indexar, el locale es el idioma que asignamos en sutty. --- app/controllers/posts_controller.rb | 3 +- app/models/indexed_post.rb | 2 +- app/models/post/indexable.rb | 3 +- ...7221120_add_dictionary_to_indexed_posts.rb | 29 +++++++++++++++++++ db/schema.rb | 27 +++++++---------- 5 files changed, 43 insertions(+), 21 deletions(-) create mode 100644 db/migrate/20210507221120_add_dictionary_to_indexed_posts.rb diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index 4acf7656..f865bd50 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -17,13 +17,12 @@ class PostsController < ApplicationController @site = find_site @locale = locale - dictionary = IndexedPost.to_dictionary(locale: locale) # XXX: Cada vez que cambiamos un Post tocamos el sitio con lo que es # más simple saber si hubo cambios. if filter_params.present? || stale?(@site) # Todos los artículos de este sitio para el idioma actual - @posts = @site.indexed_posts.where(locale: dictionary) + @posts = @site.indexed_posts.where(locale: locale) # De este tipo @posts = @posts.where(layout: filter_params[:layout]) if filter_params[:layout] # Que estén dentro de la categoría diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb index ede5b539..16858a66 100644 --- a/app/models/indexed_post.rb +++ b/app/models/indexed_post.rb @@ -19,7 +19,7 @@ class IndexedPost < ApplicationRecord query: query, using: { tsearch: { - dictionary: IndexedPost.to_dictionary(locale: locale), + dictionary: dictionary, tsvector_column: 'indexed_content' }, trigram: { diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index 0baa8012..b1d62504 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -18,7 +18,8 @@ class Post indexed_post.layout = layout.name indexed_post.site_id = site.id indexed_post.path = path.basename - indexed_post.locale = IndexedPost.to_dictionary(locale: locale.value) + indexed_post.locale = locale.value + indexed_post.dictionary = IndexedPost.to_dictionary(locale: locale.value) indexed_post.title = title.value indexed_post.front_matter = indexable_front_matter indexed_post.content = indexable_content diff --git a/db/migrate/20210507221120_add_dictionary_to_indexed_posts.rb b/db/migrate/20210507221120_add_dictionary_to_indexed_posts.rb new file mode 100644 index 00000000..f79309fd --- /dev/null +++ b/db/migrate/20210507221120_add_dictionary_to_indexed_posts.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# Para no estar calculando todo el tiempo el diccionario del idioma, +# agregamos una columna más. +class AddDictionaryToIndexedPosts < ActiveRecord::Migration[6.1] + LOCALES = { + 'english' => 'en', + 'spanish' => 'es' + } + + def up + add_column :indexed_posts, :dictionary, :string + + create_trigger(compatibility: 1).on(:indexed_posts).before(:insert, :update) do + "new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig, coalesce(new.title, '') || '\n' || coalesce(new.content,''));" + end + + IndexedPost.find_each do |post| + locale = post.locale + + post.update dictionary: locale, locale: LOCALES[locale] + end + end + + def down + remove_column :indexed_posts, :locale + rename_column :indexed_posts, :dictionary, :locale + end +end diff --git a/db/schema.rb b/db/schema.rb index ed629401..0a8dd080 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_06_212356) do +ActiveRecord::Schema.define(version: 2021_05_07_221120) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -230,6 +230,7 @@ ActiveRecord::Schema.define(version: 2021_05_06_212356) do t.string "content", default: "" t.tsvector "indexed_content" t.integer "order", default: 0 + t.string "dictionary" t.index ["front_matter"], name: "index_indexed_posts_on_front_matter", using: :gin t.index ["indexed_content"], name: "index_indexed_posts_on_indexed_content", using: :gin t.index ["layout"], name: "index_indexed_posts_on_layout" @@ -360,21 +361,13 @@ ActiveRecord::Schema.define(version: 2021_05_06_212356) do add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" - # no candidate create_trigger statement could be found, creating an adapter-specific one - execute(<<-SQL) -CREATE OR REPLACE FUNCTION public.indexed_posts_before_insert_update_row_tr() - RETURNS trigger - LANGUAGE plpgsql -AS $function$ -BEGIN - new.indexed_content := to_tsvector(('pg_catalog.' || new.locale)::regconfig, coalesce(new.title, '') || ' - ' || coalesce(new.content,'')); - RETURN NEW; -END; -$function$ - SQL - - # no candidate create_trigger statement could be found, creating an adapter-specific one - execute("CREATE TRIGGER indexed_posts_before_insert_update_row_tr BEFORE INSERT OR UPDATE ON \"indexed_posts\" FOR EACH ROW EXECUTE PROCEDURE indexed_posts_before_insert_update_row_tr()") + create_trigger("indexed_posts_before_insert_update_row_tr", :compatibility => 1). + on("indexed_posts"). + before(:insert, :update) do + <<-SQL_ACTIONS +new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig, coalesce(new.title, '') || ' +' || coalesce(new.content,'')); + SQL_ACTIONS + end end From 98df0ceb3a1ef23a5f7d444f23176da5de86c446 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 9 May 2021 12:25:16 -0300 Subject: [PATCH 22/49] los datos privados no se indexan --- app/models/metadata_array.rb | 4 +++- app/models/metadata_content.rb | 2 +- app/models/metadata_document_date.rb | 2 +- app/models/metadata_string.rb | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/models/metadata_array.rb b/app/models/metadata_array.rb index 8d951a14..9f5a84b6 100644 --- a/app/models/metadata_array.rb +++ b/app/models/metadata_array.rb @@ -13,8 +13,10 @@ class MetadataArray < MetadataTemplate false end + # Solo los datos públicos se indexan, aunque MetadataArray no se cifra + # aun, dejamos esto preparado para la posteridad. def indexable? - true + true && !private? end def to_s diff --git a/app/models/metadata_content.rb b/app/models/metadata_content.rb index 546e08c8..437a0dd9 100644 --- a/app/models/metadata_content.rb +++ b/app/models/metadata_content.rb @@ -20,7 +20,7 @@ class MetadataContent < MetadataTemplate end def indexable? - true + true && !private? end def to_s diff --git a/app/models/metadata_document_date.rb b/app/models/metadata_document_date.rb index a52cd051..c741e3be 100644 --- a/app/models/metadata_document_date.rb +++ b/app/models/metadata_document_date.rb @@ -12,7 +12,7 @@ class MetadataDocumentDate < MetadataTemplate end def indexable? - true + true && !private? end # El valor puede ser un Date, Time o una String en el formato diff --git a/app/models/metadata_string.rb b/app/models/metadata_string.rb index 724c2ef3..95aac4d4 100644 --- a/app/models/metadata_string.rb +++ b/app/models/metadata_string.rb @@ -8,7 +8,7 @@ class MetadataString < MetadataTemplate end def indexable? - true + true && !private? end private From 35518ba48abc2833c31640d8f753dfb2b12a6085 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 9 May 2021 12:52:26 -0300 Subject: [PATCH 23/49] =?UTF-8?q?no=20hacen=20falta=20los=20parametros=20a?= =?UTF-8?q?l=20crear=20un=20art=C3=ADculo=20nuevo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/posts/index.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 26a0f3ee..814ae042 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -16,7 +16,7 @@ - next if layout.hidden? %tr %th= layout.humanized_name - %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, **@filter_params), class: 'btn btn-secondary btn-sm' + %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, layout: layout.value), class: 'btn btn-secondary btn-sm' - if @filter_params[:layout] == layout.name.to_s %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'btn btn-primary btn-sm' - else From 219a9985f555ac34f84b61daf31e417270853686 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 14 May 2021 16:59:47 -0300 Subject: [PATCH 24/49] testear el buscador --- app/models/indexed_post.rb | 36 ++++++++++++------------ app/models/post/indexable.rb | 17 ++++------- app/models/site.rb | 5 ++-- test/models/indexed_post_test.rb | 35 +++++++++++++++++++++++ test/models/post/indexable_test.rb | 45 ++++++++++++++++++++++++++++++ 5 files changed, 107 insertions(+), 31 deletions(-) create mode 100644 test/models/indexed_post_test.rb create mode 100644 test/models/post/indexable_test.rb diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb index 16858a66..7f6865f6 100644 --- a/app/models/indexed_post.rb +++ b/app/models/indexed_post.rb @@ -13,26 +13,26 @@ class IndexedPost < ApplicationRecord # TODO: Los indexed posts tienen que estar scopeados al idioma actual, # no buscar sobre todos pg_search_scope :search, - lambda { |locale, query| - { - against: :content, - query: query, - using: { - tsearch: { - dictionary: dictionary, - tsvector_column: 'indexed_content' - }, - trigram: { - word_similarity: true - } - } - } - } + lambda { |locale, query| + { + against: :content, + query: query, + using: { + tsearch: { + dictionary: IndexedPost.to_dictionary(locale: locale), + tsvector_column: 'indexed_content' + }, + trigram: { + word_similarity: true + } + } + } + } # Trae los IndexedPost en el orden en que van a terminar en el sitio. - default_scope lambda { order(order: :desc, created_at: :desc) } - scope :in_category, lambda { |category| where("front_matter->'categories' ? :category", category: category.to_s) } - scope :by_usuarie, lambda { |usuarie| where("front_matter->'usuaries' @> :usuarie::jsonb", usuarie: usuarie.to_s) } + default_scope -> { order(order: :desc, created_at: :desc) } + scope :in_category, ->(category) { where("front_matter->'categories' ? :category", category: category.to_s) } + scope :by_usuarie, ->(usuarie) { where("front_matter->'usuaries' @> :usuarie::jsonb", usuarie: usuarie.to_s) } belongs_to :site diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index b1d62504..7757e7f7 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -47,15 +47,10 @@ class Post # # @return [Hash] def indexable_front_matter - {}.tap do |indexable_front_matter| - indexable_front_matter = { - usuaries: usuaries.map(&:id), - draft: attribute?(:draft) ? draft.value : false - } - - if attribute? :categories - indexable_front_matter[:categories] = categories.indexable_values - end + {}.tap do |ifm| + ifm[:usuaries] = usuaries.map(&:id) + ifm[:draft] = attribute?(:draft) ? draft.value : false + ifm[:categories] = categories.indexable_values if attribute? :categories end end @@ -73,8 +68,8 @@ class Post def indexable_attributes @indexable_attributes ||= attributes.select do |attr| - self[attr].indexable? - end + self[attr].indexable? + end end end end diff --git a/app/models/site.rb b/app/models/site.rb index 4bfbed4d..b3cae93e 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -7,7 +7,6 @@ class Site < ApplicationRecord include Site::Forms include Site::FindAndReplace include Site::Api - include Site::Index include Tienda # Cifrar la llave privada que cifra y decifra campos ocultos. Sutty @@ -38,7 +37,6 @@ class Site < ApplicationRecord belongs_to :design belongs_to :licencia - has_many :indexed_posts, dependent: :destroy has_many :log_entries, dependent: :destroy has_many :deploys, dependent: :destroy has_many :build_stats, through: :deploys @@ -70,6 +68,9 @@ class Site < ApplicationRecord # El sitio en Jekyll attr_reader :jekyll + # XXX: Es importante incluir luego de los callbacks de :load_jekyll + include Site::Index + # No permitir HTML en estos atributos def title=(title) super(title.strip_tags) diff --git a/test/models/indexed_post_test.rb b/test/models/indexed_post_test.rb new file mode 100644 index 00000000..27d4e29e --- /dev/null +++ b/test/models/indexed_post_test.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'test_helper' + +class IndexedPostTest < ActiveSupport::TestCase + def site + @site ||= create :site + end + + teardown do + @site&.destroy + end + + test 'se pueden convertir los diccionarios' do + IndexedPost::DICTIONARIES.each do |locale, dict| + assert_equal dict, IndexedPost.to_dictionary(locale: locale) + end + end + + test 'se pueden buscar por categoría' do + assert(post = site.posts.create(title: SecureRandom.hex, description: SecureRandom.hex, + categories: [SecureRandom.hex, SecureRandom.hex])) + assert_not_empty site.indexed_posts.in_category(post.categories.value.sample) + end + + test 'se pueden encontrar por usuarie' do + usuarie = create :usuarie + assert(post = site.posts.create(title: SecureRandom.hex, description: SecureRandom.hex)) + + post.usuaries << usuarie + post.save + + assert_not_empty site.indexed_posts.by_usuarie(usuarie.id) + end +end diff --git a/test/models/post/indexable_test.rb b/test/models/post/indexable_test.rb new file mode 100644 index 00000000..6110bcf0 --- /dev/null +++ b/test/models/post/indexable_test.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'test_helper' + +class Post::IndexableTest < ActiveSupport::TestCase + setup do + @site = create :site + end + + teardown do + @site&.destroy + end + + test 'los posts se indexan apenas se crean' do + post = @site.posts.create(title: SecureRandom.hex, description: SecureRandom.hex) + indexed_post = @site.indexed_posts.find_by_title post.title.value + + assert indexed_post + assert_equal post.locale.value.to_s, indexed_post.locale + assert_equal post.order.value, indexed_post.order + assert_equal post.path.basename, indexed_post.path + assert_equal post.layout.name.to_s, indexed_post.layout + end + + test 'se pueden encontrar posts' do + post = @site.posts.sample + + assert @site.indexed_posts.where(locale: post.lang.value).search(post.lang.value, post.title.value) + assert @site.indexed_posts.where(locale: post.lang.value).search(post.lang.value, post.description.value) + end + + test 'se pueden actualizar posts' do + post = @site.posts.sample + post.description.value = SecureRandom.hex + + assert post.save + assert @site.indexed_posts.where(locale: post.lang.value).search(post.lang.value, post.description.value) + end + + test 'al borrar el post se borra el indice' do + post = @site.posts.sample + assert post.destroy + assert_not @site.indexed_posts.find_by_id(post.uuid.value) + end +end From 189b94e074cfb63bc14408584e7098eb48149b79 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 14 May 2021 17:19:03 -0300 Subject: [PATCH 25/49] =?UTF-8?q?aplicar=20cach=C3=A9=20a=20los=20par?= =?UTF-8?q?=C3=A1metros=20de=20b=C3=BAsqueda=20tambi=C3=A9n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/posts_controller.rb | 9 +++------ db/schema.rb | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index e1971400..3ef26720 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -20,14 +20,11 @@ class PostsController < ApplicationController def index authorize Post - @site = find_site - @locale = locale - # XXX: Cada vez que cambiamos un Post tocamos el sitio con lo que es # más simple saber si hubo cambios. - if filter_params.present? || stale?([current_usuarie, @site]) + if stale?([current_usuarie, site, filter_params]) # Todos los artículos de este sitio para el idioma actual - @posts = @site.indexed_posts.where(locale: locale) + @posts = site.indexed_posts.where(locale: locale) # De este tipo @posts = @posts.where(layout: filter_params[:layout]) if filter_params[:layout] # Que estén dentro de la categoría @@ -38,7 +35,7 @@ class PostsController < ApplicationController @posts = PostPolicy::Scope.new(current_usuarie, @posts).resolve # Filtrar los posts que les invitades no pueden ver - @usuarie = @site.usuarie? current_usuarie + @usuarie = site.usuarie? current_usuarie end end diff --git a/db/schema.rb b/db/schema.rb index 86354240..107e7be7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_11_211357) do +ActiveRecord::Schema.define(version: 2021_05_14_165639) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" From df425bab5aa2c04ce9a79d951e066e58d8fb9ab1 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 17 May 2021 15:33:46 -0300 Subject: [PATCH 26/49] poder indexar los sitios! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit me había olvidado de este archivo --- app/models/site/index.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 app/models/site/index.rb diff --git a/app/models/site/index.rb b/app/models/site/index.rb new file mode 100644 index 00000000..e10fa523 --- /dev/null +++ b/app/models/site/index.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Indexa todos los artículos de un sitio +# +# TODO: Hacer opcional +class Site + module Index + extend ActiveSupport::Concern + + included do + # TODO: Debería ser un Job? + after_create :index_posts! + has_many :indexed_posts, dependent: :destroy + + def index_posts! + Site.transaction do + docs.each do |post| + post.to_index.save + end + end + end + end + end +end From 553b4f15f7719e8c663b927862689ce32f741bbe Mon Sep 17 00:00:00 2001 From: f Date: Mon, 17 May 2021 15:58:37 -0300 Subject: [PATCH 27/49] mostrar si es borrador sin acudir al sitio --- app/views/posts/index.haml | 3 +-- config/locales/en.yml | 2 ++ config/locales/es.yml | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 55ebcce1..06287ba5 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -103,8 +103,7 @@ = link_to site_post_path(@site, post.path) do %span{ lang: post.locale, dir: dir }= post.title - if post.front_matter['draft'].present? - %span.badge.badge-primary - = post_label_t(:draft, post: post) + %span.badge.badge-primary= I18n.t('posts.attributes.draft.label') - if post.front_matter['categories'].present? %br %small diff --git a/config/locales/en.yml b/config/locales/en.yml index 2705ad94..fc194eab 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -413,6 +413,8 @@ en: destroy: Remove image belongs_to: empty: "(Empty)" + draft: + label: Draft reorder: submit: 'Save order' select: 'Select this post' diff --git a/config/locales/es.yml b/config/locales/es.yml index fe1e3180..eca7cee8 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -420,6 +420,8 @@ es: destroy: 'Eliminar imagen' belongs_to: empty: "(Vacío)" + draft: + label: Borrador reorder: submit: 'Guardar orden' select: 'Seleccionar este artículo' From 0a23fe1edd475f11121d05e265e87be9d2472494 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 17 May 2021 17:27:01 -0300 Subject: [PATCH 28/49] lo que se indexa son los valores actuales, no todos los valores posibles --- app/models/metadata_belongs_to.rb | 4 ++++ app/models/metadata_related_posts.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/models/metadata_belongs_to.rb b/app/models/metadata_belongs_to.rb index ee182a50..1438c8db 100644 --- a/app/models/metadata_belongs_to.rb +++ b/app/models/metadata_belongs_to.rb @@ -77,6 +77,10 @@ class MetadataBelongsTo < MetadataRelatedPosts @related_methods ||= %i[belongs_to belonged_to].freeze end + def indexable_values + belongs_to&.title&.value + end + private def post_exists? diff --git a/app/models/metadata_related_posts.rb b/app/models/metadata_related_posts.rb index af91c28b..092f219a 100644 --- a/app/models/metadata_related_posts.rb +++ b/app/models/metadata_related_posts.rb @@ -23,7 +23,7 @@ class MetadataRelatedPosts < MetadataArray end def indexable_values - values.keys + posts.where(uuid: value).map(&:title).map(&:value) end private From 4609ab21b263c1f626fef071463f58bccf22c8bd Mon Sep 17 00:00:00 2001 From: f Date: Tue, 18 May 2021 11:22:09 -0300 Subject: [PATCH 29/49] =?UTF-8?q?separar=20las=20categor=C3=ADas=20con=20b?= =?UTF-8?q?arras?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/posts/index.haml | 1 + 1 file changed, 1 insertion(+) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 06287ba5..8b776590 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -110,6 +110,7 @@ - post.front_matter['categories'].each do |category| = link_to site_posts_path(@site, **@filter_params.merge(category: category)) do %span{ lang: post.locale, dir: dir }= category + = '/' unless post.front_matter['categories'].last == category %td = post.created_at.strftime('%F') From fc7c2f31ddcc368d29ba3a23107c1a758a915d68 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 29 May 2021 17:42:45 -0300 Subject: [PATCH 30/49] enviar reportes a gitlab usando la api en lugar del correo --- Gemfile | 1 + Gemfile.lock | 1 + app/jobs/backtrace_job.rb | 2 +- app/jobs/gitlab_notifier_job.rb | 142 ++++++++++++++++++ app/lib/exception_notifier/gitlab_notifier.rb | 17 +++ app/lib/gitlab_api_client.rb | 61 ++++++++ .../exception_notifier/_backtrace.text.erb | 2 +- app/views/exception_notifier/_data.text.erb | 2 +- config/environments/production.rb | 9 +- 9 files changed, 226 insertions(+), 11 deletions(-) create mode 100644 app/jobs/gitlab_notifier_job.rb create mode 100644 app/lib/exception_notifier/gitlab_notifier.rb create mode 100644 app/lib/gitlab_api_client.rb diff --git a/Gemfile b/Gemfile index 3f100e6b..5f1cdc4f 100644 --- a/Gemfile +++ b/Gemfile @@ -41,6 +41,7 @@ gem 'hiredis' gem 'image_processing' gem 'icalendar' gem 'inline_svg' +gem 'httparty' gem 'safe_yaml', source: 'https://gems.sutty.nl' gem 'jekyll', '~> 4.2' gem 'jekyll-data', source: 'https://gems.sutty.nl' diff --git a/Gemfile.lock b/Gemfile.lock index 942d7d6f..5484ad88 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -646,6 +646,7 @@ DEPENDENCIES haml-lint hamlit-rails hiredis + httparty icalendar image_processing inline_svg diff --git a/app/jobs/backtrace_job.rb b/app/jobs/backtrace_job.rb index eab9f226..86a9b2a6 100644 --- a/app/jobs/backtrace_job.rb +++ b/app/jobs/backtrace_job.rb @@ -40,7 +40,7 @@ class BacktraceJob < ApplicationJob begin raise BacktraceException, "#{origin}: #{message}" rescue BacktraceException => e - ExceptionNotifier.notify_exception(e, data: { site: site.name, params: params, _backtrace: true }) + ExceptionNotifier.notify_exception(e, data: { site: site.name, params: params, javascript_backtrace: true }) end end diff --git a/app/jobs/gitlab_notifier_job.rb b/app/jobs/gitlab_notifier_job.rb new file mode 100644 index 00000000..4d676b8d --- /dev/null +++ b/app/jobs/gitlab_notifier_job.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +class GitlabNotifierJob < ApplicationJob + include ExceptionNotifier::BacktraceCleaner + + attr_reader :exception, :options + + queue_as :low_priority + + def perform(exception, **options) + @exception = exception + @options = options + + Rails.logger.info 'Enviando reporte a Gitlab' + + i = client.new_issue confidential: true, title: title, description: description + + Rails.logger.info "Enviado reporte a Gitlab: #{i['iid']}" + rescue Exception => e + Rails.logger.info 'No entrar en loop' + end + + private + + # Define si es una excepción de javascript o local + # + # @see BacktraceJob + def javascript? + @javascript ||= options.dig(:data, :javascript_backtrace).present? + end + + # @return [String] + def title + @title ||= ''.dup.tap do |t| + t << "[#{exception.class}] " unless javascript? + t << exception.message + end + end + + # @return [String] + def description + @description ||= ''.dup.tap do |d| + d << request_section + d << javascript_section + d << javascript_footer + d << backtrace_section + d << data_section + end + end + + # @return [String,Nil] + def backtrace + @backtrace ||= exception.backtrace ? clean_backtrace(exception) : nil + end + + def env + options[:env] + end + + def request + @request ||= ActionDispatch::Request.new(env) if env.present? + end + + # @return [GitlabApiClient] + def client + @client ||= GitlabApiClient.new + end + + def request_section + return '' unless request + + <<~REQUEST + + # Request + + ``` + #{request.request_method} #{request.url}#{' '} + + #{pp request.filtered_parameters} + ``` + + REQUEST + end + + def javascript_section + return '' unless javascript? + + options.dig(:data, :params, 'errors')&.map do |error| + <<~JAVASCRIPT + + ## #{error['type'] || 'NoError'}: #{error['message']} + + + ``` + #{Terminal::Table.new headings: error['backtrace'].first.keys, rows: error['backtrace'].map(&:values)} + ``` + + JAVASCRIPT + end&.join + end + + def javascript_footer + return '' unless javascript? + + <<~JAVASCRIPT + + #{options.dig(:data, :params, 'context', 'userAgent')} + + <#{options.dig(:data, :params, 'context', 'url')}> + + JAVASCRIPT + end + + def backtrace_section + return '' if javascript? + return '' unless backtrace + + <<~BACKTRACE + + ## Backtrace + + ``` + #{backtrace.join("\n")} + ``` + + BACKTRACE + end + + def data_section + return '' unless options[:data] + + <<~DATA + + ## Data + + ``` + #{pp options[:data]} + ``` + + DATA + end +end diff --git a/app/lib/exception_notifier/gitlab_notifier.rb b/app/lib/exception_notifier/gitlab_notifier.rb new file mode 100644 index 00000000..18bfc6d4 --- /dev/null +++ b/app/lib/exception_notifier/gitlab_notifier.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module ExceptionNotifier + # Notifica las excepciones como incidencias en Gitlab + class GitlabNotifier + def initialize(_); end + + # Recibe la excepción y empieza la tarea de notificación en segundo + # plano. + # + # @param [Exception] + # @param [Hash] + def call(exception, **options) + GitlabNotifierJob.perform_async(exception, **options) + end + end +end diff --git a/app/lib/gitlab_api_client.rb b/app/lib/gitlab_api_client.rb new file mode 100644 index 00000000..c8157b47 --- /dev/null +++ b/app/lib/gitlab_api_client.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'httparty' + +class GitlabApiClient + include HTTParty + + # TODO: Hacer configurable por sitio + base_uri ENV.fetch('GITLAB_URI', 'https://0xacab.org') + no_follow true + + # Trae todos los proyectos. Como estamos usando un Project Token, + # siempre va a traer uno solo. + # + # @return [HTTParty::Response] + def projects + self.class.get('/api/v4/projects', { query: params(membership: true) }) + end + + # Obtiene el identificador del proyecto + # + # @return [Integer] + def project_id + @project_id ||= ENV['GITLAB_PROJECT'] || projects&.first&.dig('id') + end + + # Crea un issue + # + # @see https://docs.gitlab.com/ee/api/issues.html#new-issue + # @return [HTTParty::Response] + def new_issue(**args) + self.class.post("/api/v4/projects/#{project_id}/issues", { query: params(**args) }) + end + + # Modifica un issue + # + # @see https://docs.gitlab.com/ee/api/issues.html#edit-issue + # @return [HTTParty::Response] + def edit_issue(**args) + self.class.put("/api/v4/projects/#{project_id}/issues", { query: params(**args) }) + end + + # Crea un comentario + # + # @see https://docs.gitlab.com/ee/api/notes.html#create-new-issue-note + # @return [HTTParty::Response] + def new_note(iid:, **args) + self.class.post("/api/v4/projects/#{project_id}/issues/#{iid}/notes", { query: params(**args) }) + end + + private + + def params(**args) + default_params.merge(args) + end + + # TODO: Que cada sitio tenga su propio token y uri + def default_params + { private_token: ENV['GITLAB_TOKEN'] } + end +end diff --git a/app/views/exception_notifier/_backtrace.text.erb b/app/views/exception_notifier/_backtrace.text.erb index d62b5719..aed7adbe 100644 --- a/app/views/exception_notifier/_backtrace.text.erb +++ b/app/views/exception_notifier/_backtrace.text.erb @@ -1,4 +1,4 @@ -<% unless @data[:_backtrace] %> +<% unless @data[:javascript_backtrace] %> ``` <%= raw @backtrace.join("\n") %> ``` diff --git a/app/views/exception_notifier/_data.text.erb b/app/views/exception_notifier/_data.text.erb index 09313f4c..acb94b89 100644 --- a/app/views/exception_notifier/_data.text.erb +++ b/app/views/exception_notifier/_data.text.erb @@ -1,4 +1,4 @@ -<% if @data[:_backtrace] %> +<% if @data[:javascript_backtrace] %> <% @data.dig(:params, 'errors')&.each do |error| %> # <%= error['type'] %>: <%= error['message'] %> diff --git a/config/environments/production.rb b/config/environments/production.rb index c1269fb2..d121bdbd 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -147,14 +147,7 @@ Rails.application.configure do } config.action_mailer.default_options = { from: ENV.fetch('DEFAULT_FROM', "noreply@sutty.nl") } - config.middleware.use ExceptionNotification::Rack, - error_grouping: true, - email: { - email_prefix: '', - sender_address: ENV.fetch('DEFAULT_FROM', "noreply@sutty.nl"), - exception_recipients: ENV.fetch('EXCEPTION_TO', "errors@sutty.nl"), - normalize_subject: true - } + config.middleware.use ExceptionNotification::Rack, gitlab: {} Rails.application.routes.default_url_options[:host] = "panel.#{ENV.fetch('SUTTY', 'sutty.nl')}" Rails.application.routes.default_url_options[:protocol] = 'https' From 5468a118851b92271e1d081ca386438970101d31 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 31 May 2021 12:08:19 -0300 Subject: [PATCH 31/49] reportar el issue una sola vez y luego actualizarlo --- app/jobs/gitlab_notifier_job.rb | 130 +++++++++++++++++++++++++++++--- app/lib/gitlab_api_client.rb | 22 +++--- 2 files changed, 130 insertions(+), 22 deletions(-) diff --git a/app/jobs/gitlab_notifier_job.rb b/app/jobs/gitlab_notifier_job.rb index 4d676b8d..ac712ab2 100644 --- a/app/jobs/gitlab_notifier_job.rb +++ b/app/jobs/gitlab_notifier_job.rb @@ -1,42 +1,98 @@ # frozen_string_literal: true +# Notifica excepciones a una instancia de Gitlab, como incidencias +# nuevas o como comentarios a las incidencias pre-existentes. class GitlabNotifierJob < ApplicationJob include ExceptionNotifier::BacktraceCleaner - attr_reader :exception, :options + # Variables que vamos a acceder luego + attr_reader :exception, :options, :issue_data, :cached queue_as :low_priority + # @param [Exception] la excepción lanzada + # @param [Hash] opciones de ExceptionNotifier def perform(exception, **options) @exception = exception @options = options + @issue_data = { count: 1 } + # Necesitamos saber si el issue ya existía + @cached = false - Rails.logger.info 'Enviando reporte a Gitlab' + # Traemos los datos desde la caché si existen, sino generamos un + # issue nuevo e inicializamos la caché + @issue_data = Rails.cache.fetch(cache_key) do + issue = client.new_issue confidential: true, title: title, description: description, issue_type: 'incident' + @cached = true - i = client.new_issue confidential: true, title: title, description: description + { + count: 1, + issue: issue['iid'], + user_agents: [user_agent].compact, + params: [request&.filtered_parameters].compact, + urls: [url].compact + } + end - Rails.logger.info "Enviado reporte a Gitlab: #{i['iid']}" + # No seguimos actualizando si acabamos de generar el issue + return if cached + + # Incrementar la cuenta de veces que ocurrió + issue_data[:count] += 1 + # Guardar información útil + issue_data[:urls] << url unless issue_data[:urls].include? url + issue_data[:user_agents] << user_agent unless issue_data[:user_agents].include? user_agent + + # Editar el título para que incluya la cuenta de eventos + client.edit_issue(iid: issue_data[:issue], title: title, state_event: 'reopen') + + # Agregar un comentario con la información posiblemente nueva + client.new_note(iid: issue_data[:issue], body: body) + + # Guardar para después + Rails.cache.write(cache_key, issue_data) + # Si este trabajo genera una excepción va a entrar en un loop + # TODO: Notificarnos por otros medios (mail) rescue Exception => e Rails.logger.info 'No entrar en loop' end private + # La llave en la cache tiene en cuenta la excepción, el mensaje, la + # ruta del backtrace y los errores de JS + # + # @return [String] + def cache_key + @cache_key ||= [ + exception.class.name, + Digest::SHA1.hexdigest(exception.message), + Digest::SHA1.hexdigest(backtrace&.first.to_s), + Digest::SHA1.hexdigest(options.dig(:data, :params, 'errors').to_s) + ].join('/') + end + # Define si es una excepción de javascript o local # # @see BacktraceJob + # @return [Boolean] def javascript? @javascript ||= options.dig(:data, :javascript_backtrace).present? end + # Título + # # @return [String] def title @title ||= ''.dup.tap do |t| t << "[#{exception.class}] " unless javascript? t << exception.message + t << " [#{issue_data[:count]}]" end end + # Descripción + # # @return [String] def description @description ||= ''.dup.tap do |d| @@ -48,24 +104,48 @@ class GitlabNotifierJob < ApplicationJob end end - # @return [String,Nil] + # Comentario + # + # @return [String] + def body + @body ||= ''.dup.tap do |b| + b << request_section + b << javascript_footer + b << data_section + end + end + + # Cadena de archivos donde se produjo el error + # + # @return [Array,Nil] def backtrace @backtrace ||= exception.backtrace ? clean_backtrace(exception) : nil end + # Entorno del error + # + # @return [Hash] def env options[:env] end + # Genera una petición a partir del entorno + # + # @return [ActionDispatch::Request] def request @request ||= ActionDispatch::Request.new(env) if env.present? end + # Cliente de la API de Gitlab + # # @return [GitlabApiClient] def client @client ||= GitlabApiClient.new end + # Muestra información de la petición + # + # @return [String] def request_section return '' unless request @@ -74,7 +154,7 @@ class GitlabNotifierJob < ApplicationJob # Request ``` - #{request.request_method} #{request.url}#{' '} + #{request.request_method} #{url} #{pp request.filtered_parameters} ``` @@ -82,14 +162,19 @@ class GitlabNotifierJob < ApplicationJob REQUEST end + # Muestra información de JavaScript + # + # @return [String] def javascript_section return '' unless javascript? options.dig(:data, :params, 'errors')&.map do |error| + # Algunos errores no son excepciones (?) + error['type'] = 'undefined' if error['type'].blank? + <<~JAVASCRIPT - ## #{error['type'] || 'NoError'}: #{error['message']} - + ## #{error['type']}: #{error['message']} ``` #{Terminal::Table.new headings: error['backtrace'].first.keys, rows: error['backtrace'].map(&:values)} @@ -99,18 +184,24 @@ class GitlabNotifierJob < ApplicationJob end&.join end + # Muestra información de la visita que generó el error en JS + # + # @return [String] def javascript_footer return '' unless javascript? <<~JAVASCRIPT - #{options.dig(:data, :params, 'context', 'userAgent')} + #{user_agent} - <#{options.dig(:data, :params, 'context', 'url')}> + <#{url}> JAVASCRIPT end + # Muestra el historial del error en Ruby + # + # @return [String] def backtrace_section return '' if javascript? return '' unless backtrace @@ -126,6 +217,9 @@ class GitlabNotifierJob < ApplicationJob BACKTRACE end + # Muestra datos extra de la visita + # + # @return [String] def data_section return '' unless options[:data] @@ -139,4 +233,20 @@ class GitlabNotifierJob < ApplicationJob DATA end + + # Obtiene el UA de este error + # + # @return [String] + def user_agent + @user_agent ||= options.dig(:data, :params, 'context', 'userAgent') if javascript? + @user_agent ||= request.headers['user-agent'] if request + @user_agent + end + + # Obtiene la URL actual + # + # @return [String] + def url + @url ||= request&.url || options.dig(:data, :params, 'context', 'url') + end end diff --git a/app/lib/gitlab_api_client.rb b/app/lib/gitlab_api_client.rb index c8157b47..5b1287d6 100644 --- a/app/lib/gitlab_api_client.rb +++ b/app/lib/gitlab_api_client.rb @@ -7,6 +7,9 @@ class GitlabApiClient # TODO: Hacer configurable por sitio base_uri ENV.fetch('GITLAB_URI', 'https://0xacab.org') + # No seguir redirecciones. Si nos olvidamos https:// en la dirección, + # las redirecciones nos pueden llevar a cualquier lado y obtener + # resultados diferentes. no_follow true # Trae todos los proyectos. Como estamos usando un Project Token, @@ -14,7 +17,7 @@ class GitlabApiClient # # @return [HTTParty::Response] def projects - self.class.get('/api/v4/projects', { query: params(membership: true) }) + self.class.get('/api/v4/projects', { query: { membership: true }, headers: headers }) end # Obtiene el identificador del proyecto @@ -29,15 +32,15 @@ class GitlabApiClient # @see https://docs.gitlab.com/ee/api/issues.html#new-issue # @return [HTTParty::Response] def new_issue(**args) - self.class.post("/api/v4/projects/#{project_id}/issues", { query: params(**args) }) + self.class.post("/api/v4/projects/#{project_id}/issues", { body: args, headers: headers }) end # Modifica un issue # # @see https://docs.gitlab.com/ee/api/issues.html#edit-issue # @return [HTTParty::Response] - def edit_issue(**args) - self.class.put("/api/v4/projects/#{project_id}/issues", { query: params(**args) }) + def edit_issue(iid:, **args) + self.class.put("/api/v4/projects/#{project_id}/issues/#{iid}", { body: args, headers: headers }) end # Crea un comentario @@ -45,17 +48,12 @@ class GitlabApiClient # @see https://docs.gitlab.com/ee/api/notes.html#create-new-issue-note # @return [HTTParty::Response] def new_note(iid:, **args) - self.class.post("/api/v4/projects/#{project_id}/issues/#{iid}/notes", { query: params(**args) }) + self.class.post("/api/v4/projects/#{project_id}/issues/#{iid}/notes", { body: args, headers: headers }) end private - def params(**args) - default_params.merge(args) - end - - # TODO: Que cada sitio tenga su propio token y uri - def default_params - { private_token: ENV['GITLAB_TOKEN'] } + def headers(extra = {}) + { 'Authorization' => "Bearer #{ENV['GITLAB_TOKEN']}" }.merge(extra) end end From 4540ea73ad6f3353f63deae5506e1e47991bb8e3 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 31 May 2021 12:18:06 -0300 Subject: [PATCH 32/49] variables de entorno nuevas --- .env.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.env.example b/.env.example index 2ba46219..a1348593 100644 --- a/.env.example +++ b/.env.example @@ -29,3 +29,6 @@ TIENDA=tienda.sutty.local PANEL_URL=https://panel.sutty.nl AIRBRAKE_SITE_ID=1 AIRBRAKE_API_KEY= +GITLAB_URI=https://0xacab.org +GITLAB_PROJECT= +GITLAB_TOKEN= From c6bcf95f345fccc3d7519b8ec98699b33fb9cc42 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 31 May 2021 12:18:23 -0300 Subject: [PATCH 33/49] =?UTF-8?q?si=20la=20notificaci=C3=B3n=20produce=20u?= =?UTF-8?q?na=20excepci=C3=B3n,=20capturarla=20por=20correo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/gitlab_notifier_job.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/app/jobs/gitlab_notifier_job.rb b/app/jobs/gitlab_notifier_job.rb index ac712ab2..7218f68a 100644 --- a/app/jobs/gitlab_notifier_job.rb +++ b/app/jobs/gitlab_notifier_job.rb @@ -51,14 +51,22 @@ class GitlabNotifierJob < ApplicationJob # Guardar para después Rails.cache.write(cache_key, issue_data) - # Si este trabajo genera una excepción va a entrar en un loop - # TODO: Notificarnos por otros medios (mail) + # Si este trabajo genera una excepción va a entrar en un loop, así que + # la notificamos por correo rescue Exception => e - Rails.logger.info 'No entrar en loop' + email_notification.call(e) + email_notification.call(exception, options) end private + # Notificar por correo + # + # @return [ExceptionNotifier::EmailNotifier] + def email_notification + @email_notification ||= ExceptionNotifier::EmailNotifier.new(email_prefix: '[ERROR] ', sender_address: ENV['DEFAULT_FROM'], exception_recipients: ENV['EXCEPTION_TO']) + end + # La llave en la cache tiene en cuenta la excepción, el mensaje, la # ruta del backtrace y los errores de JS # From 7191baff4abd7d6358769a5aebdd032e84c80aba Mon Sep 17 00:00:00 2001 From: f Date: Mon, 31 May 2021 13:23:40 -0300 Subject: [PATCH 34/49] =?UTF-8?q?reindexar=20despu=C3=A9s=20de=20mergear?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/sites_controller.rb | 2 +- app/services/site_service.rb | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/app/controllers/sites_controller.rb b/app/controllers/sites_controller.rb index 20ce5bad..bdaa9011 100644 --- a/app/controllers/sites_controller.rb +++ b/app/controllers/sites_controller.rb @@ -107,7 +107,7 @@ class SitesController < ApplicationController def merge authorize site - if site.repository.merge(current_usuarie) + if SiteService.new(site: site, usuarie: current_usuarie).merge flash[:success] = I18n.t('sites.fetch.merge.success') else flash[:error] = I18n.t('sites.fetch.merge.error') diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 4f3905a5..389549c3 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -61,6 +61,18 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do commit_config(action: :tor) end + # Trae cambios desde la rama remota y reindexa los artículos. + # + # @return [Boolean] + def merge + result = site.repository.merge(usuarie) + + # TODO: Implementar callbacks + site.try(:index_posts!) if result + + result.present? + end + private # Guarda los cambios de la configuración en el repositorio git From 39ffe9e926b2f2722d1e5bb83661ba65d3b77b66 Mon Sep 17 00:00:00 2001 From: Maki Date: Thu, 3 Jun 2021 16:09:26 -0300 Subject: [PATCH 35/49] edit_posts recuperado #1823 --- config/locales/es.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/locales/es.yml b/config/locales/es.yml index 1ce50b09..4361f790 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -180,6 +180,8 @@ es: category: 'Categoría' sites: index: 'Este es el listado de sitios que puedes editar.' + edit_posts: 'Aquí verás el listado de todos los artículos y podrás +- editarlos o crear nuevos' enqueued: 'El sitio está en la cola de espera para ser generado. Una vez que este proceso termine, recibirás un correo indicando el estado y si todo fue bien, se publicarán los cambios en tu sitio From 92a292c42b856935fef87f51048f1716e111500e Mon Sep 17 00:00:00 2001 From: f Date: Tue, 15 Jun 2021 19:04:00 -0300 Subject: [PATCH 36/49] no fallar el container si vendor no existe --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3781772b..0b3253b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,7 @@ RUN cd ../checkout && git checkout $BRANCH WORKDIR /home/app/checkout # Traer las gemas: -RUN rm -r ./vendor +RUN rm -rf ./vendor RUN mv ../sutty/vendor ./vendor RUN mv ../sutty/.bundle ./.bundle From 7b5b24ab2ba8e4ad66b18672cda4a25b2fe82a50 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 15 Jun 2021 20:40:18 -0300 Subject: [PATCH 37/49] actualizar sutty-liquid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit las versiones anteriores no verifican correctamente la fecha en el filtro `date_local` y fallan en fechas incorrectas, especialmente en la vista previa de los artículos. fixes #2100 fixes #2092 --- Gemfile | 2 +- Gemfile.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 5f1cdc4f..aa2d841e 100644 --- a/Gemfile +++ b/Gemfile @@ -48,7 +48,7 @@ gem 'jekyll-data', source: 'https://gems.sutty.nl' gem 'jekyll-commonmark' gem 'jekyll-images' gem 'jekyll-include-cache' -gem 'sutty-liquid' +gem 'sutty-liquid', '>= 0.7.3' gem 'loaf' gem 'lockbox' gem 'mini_magick' diff --git a/Gemfile.lock b/Gemfile.lock index 5484ad88..d96f1fe9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -564,7 +564,7 @@ GEM jekyll-include-cache (~> 0) jekyll-relative-urls (~> 0.0) jekyll-seo-tag (~> 2.1) - sutty-liquid (0.7.2) + sutty-liquid (0.7.3) fast_blank (~> 1.0) jekyll (~> 4) sutty-minima (2.5.0) From 810c5e32da4157813b905c2320ee4c25ebe1f848 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 16 Jun 2021 09:11:13 -0300 Subject: [PATCH 38/49] convertir valores en arrays por retrocompatibilidad MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit esto permite indexar sitios antiguos y cargar artículos sin generar errores. --- app/models/metadata_array.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models/metadata_array.rb b/app/models/metadata_array.rb index 9f5a84b6..96d3be3c 100644 --- a/app/models/metadata_array.rb +++ b/app/models/metadata_array.rb @@ -23,6 +23,14 @@ class MetadataArray < MetadataTemplate value.join(', ') end + # Obtiene el valor desde el documento, convirtiéndolo a Array si no lo + # era ya, por retrocompabilidad. + # + # @return [Array] + def document_value + [super].flatten(1) + end + alias indexable_values values private From 78a3cae077fae9bf5415e24db782db34fc5c7477 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 16 Jun 2021 09:13:40 -0300 Subject: [PATCH 39/49] solo indexar los valores actuales MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sin este cambio los artículos se indexaban por todas las categorías posibles! --- app/models/metadata_array.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/metadata_array.rb b/app/models/metadata_array.rb index 96d3be3c..0527ccb8 100644 --- a/app/models/metadata_array.rb +++ b/app/models/metadata_array.rb @@ -31,7 +31,7 @@ class MetadataArray < MetadataTemplate [super].flatten(1) end - alias indexable_values values + alias indexable_values value private From c3a8b2401ceb249599e7dc39b16a6d0ec9d42b6f Mon Sep 17 00:00:00 2001 From: f Date: Wed, 16 Jun 2021 11:35:37 -0300 Subject: [PATCH 40/49] estandarizar la forma de obtener el valor de los documentos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit teníamos dos métodos que hacían lo mismo y generaban conflictos al obtener el valor por defecto de los arrays cuando no eran arrays. --- app/models/metadata_document_date.rb | 7 ++++--- app/models/metadata_lang.rb | 5 +++-- app/models/metadata_markdown_content.rb | 5 +++-- app/models/metadata_path.rb | 8 ++++++-- app/models/metadata_template.rb | 8 ++------ 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/app/models/metadata_document_date.rb b/app/models/metadata_document_date.rb index 34cbe1c5..324e4be8 100644 --- a/app/models/metadata_document_date.rb +++ b/app/models/metadata_document_date.rb @@ -7,7 +7,8 @@ class MetadataDocumentDate < MetadataTemplate Date.today.to_time end - def value_from_document + # @return [Time] + def document_value return nil if post.new? document.date @@ -45,10 +46,10 @@ class MetadataDocumentDate < MetadataTemplate begin Date.iso8601(self[:value]).to_time rescue Date::Error - value_from_document || default_value + document_value || default_value end else - self[:value] || value_from_document || default_value + self[:value] || document_value || default_value end end diff --git a/app/models/metadata_lang.rb b/app/models/metadata_lang.rb index 5f31ee9d..ff6c08e6 100644 --- a/app/models/metadata_lang.rb +++ b/app/models/metadata_lang.rb @@ -6,12 +6,13 @@ class MetadataLang < MetadataTemplate super || I18n.locale end - def value_from_document + # @return [Symbol] + def document_value document.collection.label.to_sym end def value - self[:value] ||= value_from_document || default_value + self[:value] ||= document_value || default_value end def values diff --git a/app/models/metadata_markdown_content.rb b/app/models/metadata_markdown_content.rb index d3cc6dec..92a1ab21 100644 --- a/app/models/metadata_markdown_content.rb +++ b/app/models/metadata_markdown_content.rb @@ -9,14 +9,15 @@ class MetadataMarkdownContent < MetadataText end def value - self[:value] || value_from_document || default_value + self[:value] || document_value || default_value end def front_matter? false end - def value_from_document + # @return [String] + def document_value document.content end diff --git a/app/models/metadata_path.rb b/app/models/metadata_path.rb index 3c93cca6..95fc7dbb 100644 --- a/app/models/metadata_path.rb +++ b/app/models/metadata_path.rb @@ -3,12 +3,16 @@ # Este campo representa el archivo donde se almacenan los datos class MetadataPath < MetadataTemplate # :label en este caso es el idioma/colección + # + # @return [String] def default_value File.join(site.path, "_#{lang}", "#{date}-#{slug}#{ext}") end - # El valor no vuelve desde el documento - def value_from_document + # La ruta del archivo según Jekyll + # + # @return [String] + def document_value document.path end diff --git a/app/models/metadata_template.rb b/app/models/metadata_template.rb index 638bdb8e..76c3e7b5 100644 --- a/app/models/metadata_template.rb +++ b/app/models/metadata_template.rb @@ -49,11 +49,7 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, def value_was return @value_was if instance_variable_defined? '@value_was' - @value_was = value_from_document - end - - def value_from_document - @value_from_document ||= document.data[name.to_s] + @value_was = document_value end def changed? @@ -85,7 +81,7 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, # Valor actual o por defecto. Al memoizarlo podemos modificarlo # usando otros métodos que el de asignación. def value - self[:value] ||= if (data = value_from_document).present? + self[:value] ||= if (data = document_value).present? private? ? decrypt(data) : data else default_value From 96a16379bdbd2f3afccc03e75da9219cf354e567 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 16 Jun 2021 12:08:05 -0300 Subject: [PATCH 41/49] correr yarn en hain --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index e0da9ddc..ab630b9b 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,9 @@ rake: bundle: $(hain) 'cd /Sutty/sutty; bundle $(args)' +yarn: + $(hain) 'yarn $(args)' + # Servir JS con el dev server. # Esto acelera la compilación del javascript, tiene que correrse por separado # de serve. From bd3b117282bd52cc5d3fc1e663dea34bcbf83fd3 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 16 Jun 2021 12:08:21 -0300 Subject: [PATCH 42/49] aplicar parches en el servidor --- Makefile | 9 +++++++++ ota.sh | 10 ++++++++++ 2 files changed, 19 insertions(+) create mode 100755 ota.sh diff --git a/Makefile b/Makefile index ab630b9b..f3a74f57 100644 --- a/Makefile +++ b/Makefile @@ -124,10 +124,19 @@ ota: assets ssh $(delegate) docker exec $(container) sh -c "cat /srv/http/tmp/puma.pid | xargs -r kill -USR2" # Hotfixes +# +# TODO: Reemplazar esto por git pull en el contenedor commit ?= origin/rails ota-rb: umask 022; git format-patch $(commit) scp ./0*.patch $(delegate):/tmp/ + ssh $(delegate) mkdir -p /tmp/patches-$(commit)/ + scp ./0*.patch $(delegate):/tmp/patches-$(commit)/ + scp ./ota.sh $(delegate):/tmp/ + ssh $(delegate) docker cp /tmp/patches-$(commit) $(container):/tmp/ + ssh $(delegate) docker cp /tmp/ota.sh $(container):/usr/local/bin/ota + ssh $(delegate) docker exec $(container) apk add --no-cache patch + ssh $(delegate) docker exec $(container) ota $(commit) rm ./0*.patch /etc/hosts: always diff --git a/ota.sh b/ota.sh new file mode 100755 index 00000000..68a0642f --- /dev/null +++ b/ota.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -e + +cd /srv/http + +for patch in /tmp/patches-${1}/*.patch; do + su -c "patch -Np 1 -i ${patch}" app && rm $patch +done + +cat tmp/puma.pid | xargs -r kill -USR2 From a4716a0ad1af8f4bd49e263ba5a152ccac4f16ef Mon Sep 17 00:00:00 2001 From: void Date: Fri, 18 Jun 2021 22:47:42 +0000 Subject: [PATCH 43/49] usar certificados de haini.sh para dev en vez de sutty.local, ver https://0xacab.org/sutty/haini.sh/-/merge_requests/18 --- config/puma.rb | 2 +- config/webpacker.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/puma.rb b/config/puma.rb index 60ee5ecc..414507ed 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -26,7 +26,7 @@ if ENV['RAILS_ENV'] == 'production' bind 'tcp://[::]:3100' else sutty = ENV.fetch('SUTTY', 'sutty.local') - bind "ssl://[::]:3000?key=../sutty.local/domain/#{sutty}.key&cert=../sutty.local/domain/#{sutty}.crt" + bind "ssl://[::]:3000?key=/etc/ssl/private/#{sutty}.key&cert=/etc/ssl/certs/#{sutty}.crt" end # Specifies the `environment` that Puma will run in. diff --git a/config/webpacker.yml b/config/webpacker.yml index 25519907..d555770d 100644 --- a/config/webpacker.yml +++ b/config/webpacker.yml @@ -56,8 +56,8 @@ development: # Reference: https://webpack.js.org/configuration/dev-server/ dev_server: https: - key: '../sutty.local/domain/sutty.local.key' - cert: '../sutty.local/domain/sutty.local.crt' + key: '/etc/ssl/private/sutty.local.key' + cert: '/etc/ssl/certs/sutty.local.crt' # XXX: esto está hardcodeado, debería conseguirlo del ENV host: sutty.local port: 3035 From c7b7d660e1d3eaf5a2eb1e41e99a4ef215c30d59 Mon Sep 17 00:00:00 2001 From: void Date: Sat, 19 Jun 2021 14:59:37 +0000 Subject: [PATCH 44/49] arreglar el atributo `markdown` es igual a markdown_content, pero como estaba antes causaba #2102 por que le faltaba .markdown-editor. fixes #2102 --- app/views/posts/attributes/_markdown.haml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/posts/attributes/_markdown.haml b/app/views/posts/attributes/_markdown.haml index 325beb5c..8042009f 100644 --- a/app/views/posts/attributes/_markdown.haml +++ b/app/views/posts/attributes/_markdown.haml @@ -1,8 +1,8 @@ .form-group.markdown-content = label_tag "#{base}_#{attribute}", post_label_t(attribute, post: post) + = render 'posts/attribute_feedback', + post: post, attribute: attribute, metadata: metadata = text_area_tag "#{base}[#{attribute}]", metadata.value, dir: dir, lang: locale, **field_options(attribute, metadata, class: 'content') - .editor.mt-1 - = render 'posts/attribute_feedback', - post: post, attribute: attribute, metadata: metadata + .markdown-editor.mt-1 From 06e958710faf9a03d90ee585b1c1e76fbca15cab Mon Sep 17 00:00:00 2001 From: f Date: Sat, 26 Jun 2021 20:34:13 -0300 Subject: [PATCH 45/49] no guardar valores nulos --- app/models/metadata_array.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/metadata_array.rb b/app/models/metadata_array.rb index 0527ccb8..368aa546 100644 --- a/app/models/metadata_array.rb +++ b/app/models/metadata_array.rb @@ -28,7 +28,7 @@ class MetadataArray < MetadataTemplate # # @return [Array] def document_value - [super].flatten(1) + [super].flatten(1).compact end alias indexable_values value From 66d1846cea03be25fbb3337c711b97edd57fdb8d Mon Sep 17 00:00:00 2001 From: f Date: Sat, 26 Jun 2021 20:34:33 -0300 Subject: [PATCH 46/49] =?UTF-8?q?la=20fecha=20cambi=C3=B3=20si=20se=20acab?= =?UTF-8?q?a=20de=20inicializar=20el=20post?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/models/post_test.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/models/post_test.rb b/test/models/post_test.rb index f98d7af3..52e59a8e 100644 --- a/test/models/post_test.rb +++ b/test/models/post_test.rb @@ -110,7 +110,6 @@ class PostTest < ActiveSupport::TestCase end test 'se puede cambiar la fecha' do - assert_not @post.date.changed? assert @post.date.valid? ex_date = @post.date.value From fd9350cd10af43659befb4a26b81635927f0033f Mon Sep 17 00:00:00 2001 From: f Date: Sat, 26 Jun 2021 20:34:53 -0300 Subject: [PATCH 47/49] permitir autocompletar tests en make --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f3a74f57..c083ddf8 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,8 @@ public/packs/manifest.json.br: $(assets) assets: public/packs/manifest.json.br -test/%_test.rb: always +tests := $(shell find test/ -name "*_test.rb") +$(tests): always $(hain) 'cd /Sutty/sutty; bundle exec rake test TEST="$@" RAILS_ENV=test' test: always From 69577e6495dca362e379f16ace1320dddf739579 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 26 Jun 2021 20:35:19 -0300 Subject: [PATCH 48/49] sutty-liquid 0.7.3 no falla con algunas fechas --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d99ed497..8e7669f7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -710,7 +710,7 @@ DEPENDENCIES sucker_punch sutty-donaciones-jekyll-theme sutty-jekyll-theme - sutty-liquid + sutty-liquid (>= 0.7.3) sutty-minima symbol-fstring terminal-table From f19bfe153931d6100b7da2dfe70438e809f33bad Mon Sep 17 00:00:00 2001 From: Maki Date: Mon, 28 Jun 2021 10:18:20 -0300 Subject: [PATCH 49/49] una linea --- config/locales/es.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/locales/es.yml b/config/locales/es.yml index 4361f790..0d16f3f9 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -180,8 +180,7 @@ es: category: 'Categoría' sites: index: 'Este es el listado de sitios que puedes editar.' - edit_posts: 'Aquí verás el listado de todos los artículos y podrás -- editarlos o crear nuevos' + edit_posts: 'Aquí verás el listado de todos los artículos y podrás editarlos o crear nuevos' enqueued: 'El sitio está en la cola de espera para ser generado. Una vez que este proceso termine, recibirás un correo indicando el estado y si todo fue bien, se publicarán los cambios en tu sitio