diff --git a/.env.example b/.env.example index 2ba46219..a1348593 100644 --- a/.env.example +++ b/.env.example @@ -29,3 +29,6 @@ TIENDA=tienda.sutty.local PANEL_URL=https://panel.sutty.nl AIRBRAKE_SITE_ID=1 AIRBRAKE_API_KEY= +GITLAB_URI=https://0xacab.org +GITLAB_PROJECT= +GITLAB_TOKEN= diff --git a/Dockerfile b/Dockerfile index 3781772b..0b3253b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,7 @@ RUN cd ../checkout && git checkout $BRANCH WORKDIR /home/app/checkout # Traer las gemas: -RUN rm -r ./vendor +RUN rm -rf ./vendor RUN mv ../sutty/vendor ./vendor RUN mv ../sutty/.bundle ./.bundle diff --git a/Gemfile b/Gemfile index 3f100e6b..4256e307 100644 --- a/Gemfile +++ b/Gemfile @@ -41,18 +41,18 @@ gem 'hiredis' gem 'image_processing' gem 'icalendar' gem 'inline_svg' +gem 'httparty' gem 'safe_yaml', source: 'https://gems.sutty.nl' gem 'jekyll', '~> 4.2' gem 'jekyll-data', source: 'https://gems.sutty.nl' gem 'jekyll-commonmark' gem 'jekyll-images' gem 'jekyll-include-cache' -gem 'sutty-liquid' +gem 'sutty-liquid', '>= 0.7.3' gem 'loaf' gem 'lockbox' gem 'mini_magick' gem 'mobility' -gem 'pg' gem 'pundit' gem 'rails-i18n' gem 'rails_warden' @@ -68,6 +68,11 @@ gem 'validates_hostname' gem 'webpacker' gem 'yaml_db', git: 'https://0xacab.org/sutty/yaml_db.git' +# database +gem 'hairtrigger' +gem 'pg' +gem 'pg_search' + # performance gem 'flamegraph' gem 'memory_profiler' diff --git a/Gemfile.lock b/Gemfile.lock index e0c19860..aed60b18 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -205,6 +205,10 @@ GEM ffi (~> 1.0) globalid (0.4.2) activesupport (>= 4.2.0) + hairtrigger (0.2.24) + activerecord (>= 5.0, < 7) + ruby2ruby (~> 2.4) + ruby_parser (~> 3.10) haml (5.2.1) temple (>= 0.8.0) tilt @@ -362,6 +366,9 @@ GEM pathutil (0.16.2) forwardable-extended (~> 2.6) pg (1.2.3-x86_64-linux-musl) + pg_search (2.3.5) + activerecord (>= 5.2) + activesupport (>= 5.2) popper_js (1.16.0) prometheus_exporter (0.7.0) webrick @@ -495,7 +502,12 @@ GEM ruby-statistics (2.1.3) ruby-vips (2.1.2) ffi (~> 1.12) + ruby2ruby (2.4.4) + ruby_parser (~> 3.1) + sexp_processor (~> 4.6) ruby_dep (1.5.0) + ruby_parser (3.15.1) + sexp_processor (~> 4.9) rubyzip (2.3.0) rugged (1.1.0-x86_64-linux-musl) safe_yaml (1.0.6) @@ -513,6 +525,7 @@ GEM childprocess (>= 0.5, < 4.0) rubyzip (>= 1.2.2) semantic_range (3.0.0) + sexp_processor (4.15.2) share-to-fediverse-jekyll-theme (0.1.4) jekyll (~> 4.0) jekyll-data (~> 1.1) @@ -561,7 +574,7 @@ GEM jekyll-include-cache (~> 0) jekyll-relative-urls (~> 0.0) jekyll-seo-tag (~> 2.1) - sutty-liquid (0.7.2) + sutty-liquid (0.7.3) fast_blank (~> 1.0) jekyll (~> 4) sutty-minima (2.5.0) @@ -640,9 +653,11 @@ DEPENDENCIES fast_jsonparser flamegraph friendly_id + hairtrigger haml-lint hamlit-rails hiredis + httparty icalendar image_processing inline_svg @@ -663,6 +678,7 @@ DEPENDENCIES mobility net-ssh pg + pg_search prometheus_exporter pry puma @@ -691,7 +707,7 @@ DEPENDENCIES sucker_punch sutty-donaciones-jekyll-theme sutty-jekyll-theme - sutty-liquid + sutty-liquid (>= 0.7.3) sutty-minima symbol-fstring terminal-table diff --git a/Makefile b/Makefile index e0da9ddc..c083ddf8 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,8 @@ public/packs/manifest.json.br: $(assets) assets: public/packs/manifest.json.br -test/%_test.rb: always +tests := $(shell find test/ -name "*_test.rb") +$(tests): always $(hain) 'cd /Sutty/sutty; bundle exec rake test TEST="$@" RAILS_ENV=test' test: always @@ -54,6 +55,9 @@ rake: bundle: $(hain) 'cd /Sutty/sutty; bundle $(args)' +yarn: + $(hain) 'yarn $(args)' + # Servir JS con el dev server. # Esto acelera la compilación del javascript, tiene que correrse por separado # de serve. @@ -121,10 +125,19 @@ ota: assets ssh $(delegate) docker exec $(container) sh -c "cat /srv/http/tmp/puma.pid | xargs -r kill -USR2" # Hotfixes +# +# TODO: Reemplazar esto por git pull en el contenedor commit ?= origin/rails ota-rb: umask 022; git format-patch $(commit) scp ./0*.patch $(delegate):/tmp/ + ssh $(delegate) mkdir -p /tmp/patches-$(commit)/ + scp ./0*.patch $(delegate):/tmp/patches-$(commit)/ + scp ./ota.sh $(delegate):/tmp/ + ssh $(delegate) docker cp /tmp/patches-$(commit) $(container):/tmp/ + ssh $(delegate) docker cp /tmp/ota.sh $(container):/usr/local/bin/ota + ssh $(delegate) docker exec $(container) apk add --no-cache patch + ssh $(delegate) docker exec $(container) ota $(commit) rm ./0*.patch /etc/hosts: always diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index 5886a2a7..e806a032 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -21,6 +21,10 @@ $form-feedback-invalid-color: $magenta; $form-feedback-icon-valid-color: $black; $component-active-bg: $magenta; +$spacers: ( + 2-plus: 0.75rem +); + @import "bootstrap"; @import "editor"; @@ -210,6 +214,10 @@ svg { } } +.btn-sm { + @extend .badge +} + .black-bg { color: $white; background-color: $black; diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index da0b28aa..3ef26720 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -20,30 +20,22 @@ class PostsController < ApplicationController def index authorize Post - @site = find_site - @category = params.dig(:category) - @layout = params.dig(:layout) - @locale = locale - # XXX: Cada vez que cambiamos un Post tocamos el sitio con lo que es # más simple saber si hubo cambios. - if @category || @layout || stale?([current_usuarie, @site]) - @posts = @site.posts(lang: locale) - @posts = @posts.where(categories: @category) if @category - @posts = @posts.where(layout: @layout) if @layout + if stale?([current_usuarie, site, filter_params]) + # Todos los artículos de este sitio para el idioma actual + @posts = site.indexed_posts.where(locale: locale) + # De este tipo + @posts = @posts.where(layout: filter_params[:layout]) if filter_params[:layout] + # Que estén dentro de la categoría + @posts = @posts.in_category(filter_params[:category]) if filter_params[:category] + # Aplicar los parámetros de búsqueda + @posts = @posts.search(locale, filter_params[:q]) if filter_params[:q].present? + # A los que este usuarie tiene acceso @posts = PostPolicy::Scope.new(current_usuarie, @posts).resolve - @category_name = if uuid?(@category) - @site.posts(lang: locale).find(@category, uuid: true)&.title&.value - else - @category - end - # Filtrar los posts que les invitades no pueden ver - @usuarie = @site.usuarie? current_usuarie - - # Orden descendiente por número y luego por fecha - @posts.sort_by!(:order, :date).reverse! + @usuarie = site.usuarie? current_usuarie end end @@ -157,6 +149,14 @@ class PostsController < ApplicationController private + # Los parámetros de filtros que vamos a mantener en todas las URLs, + # solo los que no estén vacíos. + # + # @return [Hash] + def filter_params + @filter_params ||= params.permit(:q, :category, :layout).to_h.select { |_, v| v.present? } + end + def site @site ||= find_site end diff --git a/app/controllers/sites_controller.rb b/app/controllers/sites_controller.rb index 20ce5bad..bdaa9011 100644 --- a/app/controllers/sites_controller.rb +++ b/app/controllers/sites_controller.rb @@ -107,7 +107,7 @@ class SitesController < ApplicationController def merge authorize site - if site.repository.merge(current_usuarie) + if SiteService.new(site: site, usuarie: current_usuarie).merge flash[:success] = I18n.t('sites.fetch.merge.success') else flash[:error] = I18n.t('sites.fetch.merge.error') diff --git a/app/jobs/backtrace_job.rb b/app/jobs/backtrace_job.rb index eab9f226..86a9b2a6 100644 --- a/app/jobs/backtrace_job.rb +++ b/app/jobs/backtrace_job.rb @@ -40,7 +40,7 @@ class BacktraceJob < ApplicationJob begin raise BacktraceException, "#{origin}: #{message}" rescue BacktraceException => e - ExceptionNotifier.notify_exception(e, data: { site: site.name, params: params, _backtrace: true }) + ExceptionNotifier.notify_exception(e, data: { site: site.name, params: params, javascript_backtrace: true }) end end diff --git a/app/jobs/gitlab_notifier_job.rb b/app/jobs/gitlab_notifier_job.rb new file mode 100644 index 00000000..7218f68a --- /dev/null +++ b/app/jobs/gitlab_notifier_job.rb @@ -0,0 +1,260 @@ +# frozen_string_literal: true + +# Notifica excepciones a una instancia de Gitlab, como incidencias +# nuevas o como comentarios a las incidencias pre-existentes. +class GitlabNotifierJob < ApplicationJob + include ExceptionNotifier::BacktraceCleaner + + # Variables que vamos a acceder luego + attr_reader :exception, :options, :issue_data, :cached + + queue_as :low_priority + + # @param [Exception] la excepción lanzada + # @param [Hash] opciones de ExceptionNotifier + def perform(exception, **options) + @exception = exception + @options = options + @issue_data = { count: 1 } + # Necesitamos saber si el issue ya existía + @cached = false + + # Traemos los datos desde la caché si existen, sino generamos un + # issue nuevo e inicializamos la caché + @issue_data = Rails.cache.fetch(cache_key) do + issue = client.new_issue confidential: true, title: title, description: description, issue_type: 'incident' + @cached = true + + { + count: 1, + issue: issue['iid'], + user_agents: [user_agent].compact, + params: [request&.filtered_parameters].compact, + urls: [url].compact + } + end + + # No seguimos actualizando si acabamos de generar el issue + return if cached + + # Incrementar la cuenta de veces que ocurrió + issue_data[:count] += 1 + # Guardar información útil + issue_data[:urls] << url unless issue_data[:urls].include? url + issue_data[:user_agents] << user_agent unless issue_data[:user_agents].include? user_agent + + # Editar el título para que incluya la cuenta de eventos + client.edit_issue(iid: issue_data[:issue], title: title, state_event: 'reopen') + + # Agregar un comentario con la información posiblemente nueva + client.new_note(iid: issue_data[:issue], body: body) + + # Guardar para después + Rails.cache.write(cache_key, issue_data) + # Si este trabajo genera una excepción va a entrar en un loop, así que + # la notificamos por correo + rescue Exception => e + email_notification.call(e) + email_notification.call(exception, options) + end + + private + + # Notificar por correo + # + # @return [ExceptionNotifier::EmailNotifier] + def email_notification + @email_notification ||= ExceptionNotifier::EmailNotifier.new(email_prefix: '[ERROR] ', sender_address: ENV['DEFAULT_FROM'], exception_recipients: ENV['EXCEPTION_TO']) + end + + # La llave en la cache tiene en cuenta la excepción, el mensaje, la + # ruta del backtrace y los errores de JS + # + # @return [String] + def cache_key + @cache_key ||= [ + exception.class.name, + Digest::SHA1.hexdigest(exception.message), + Digest::SHA1.hexdigest(backtrace&.first.to_s), + Digest::SHA1.hexdigest(options.dig(:data, :params, 'errors').to_s) + ].join('/') + end + + # Define si es una excepción de javascript o local + # + # @see BacktraceJob + # @return [Boolean] + def javascript? + @javascript ||= options.dig(:data, :javascript_backtrace).present? + end + + # Título + # + # @return [String] + def title + @title ||= ''.dup.tap do |t| + t << "[#{exception.class}] " unless javascript? + t << exception.message + t << " [#{issue_data[:count]}]" + end + end + + # Descripción + # + # @return [String] + def description + @description ||= ''.dup.tap do |d| + d << request_section + d << javascript_section + d << javascript_footer + d << backtrace_section + d << data_section + end + end + + # Comentario + # + # @return [String] + def body + @body ||= ''.dup.tap do |b| + b << request_section + b << javascript_footer + b << data_section + end + end + + # Cadena de archivos donde se produjo el error + # + # @return [Array,Nil] + def backtrace + @backtrace ||= exception.backtrace ? clean_backtrace(exception) : nil + end + + # Entorno del error + # + # @return [Hash] + def env + options[:env] + end + + # Genera una petición a partir del entorno + # + # @return [ActionDispatch::Request] + def request + @request ||= ActionDispatch::Request.new(env) if env.present? + end + + # Cliente de la API de Gitlab + # + # @return [GitlabApiClient] + def client + @client ||= GitlabApiClient.new + end + + # Muestra información de la petición + # + # @return [String] + def request_section + return '' unless request + + <<~REQUEST + + # Request + + ``` + #{request.request_method} #{url} + + #{pp request.filtered_parameters} + ``` + + REQUEST + end + + # Muestra información de JavaScript + # + # @return [String] + def javascript_section + return '' unless javascript? + + options.dig(:data, :params, 'errors')&.map do |error| + # Algunos errores no son excepciones (?) + error['type'] = 'undefined' if error['type'].blank? + + <<~JAVASCRIPT + + ## #{error['type']}: #{error['message']} + + ``` + #{Terminal::Table.new headings: error['backtrace'].first.keys, rows: error['backtrace'].map(&:values)} + ``` + + JAVASCRIPT + end&.join + end + + # Muestra información de la visita que generó el error en JS + # + # @return [String] + def javascript_footer + return '' unless javascript? + + <<~JAVASCRIPT + + #{user_agent} + + <#{url}> + + JAVASCRIPT + end + + # Muestra el historial del error en Ruby + # + # @return [String] + def backtrace_section + return '' if javascript? + return '' unless backtrace + + <<~BACKTRACE + + ## Backtrace + + ``` + #{backtrace.join("\n")} + ``` + + BACKTRACE + end + + # Muestra datos extra de la visita + # + # @return [String] + def data_section + return '' unless options[:data] + + <<~DATA + + ## Data + + ``` + #{pp options[:data]} + ``` + + DATA + end + + # Obtiene el UA de este error + # + # @return [String] + def user_agent + @user_agent ||= options.dig(:data, :params, 'context', 'userAgent') if javascript? + @user_agent ||= request.headers['user-agent'] if request + @user_agent + end + + # Obtiene la URL actual + # + # @return [String] + def url + @url ||= request&.url || options.dig(:data, :params, 'context', 'url') + end +end diff --git a/app/lib/exception_notifier/gitlab_notifier.rb b/app/lib/exception_notifier/gitlab_notifier.rb new file mode 100644 index 00000000..18bfc6d4 --- /dev/null +++ b/app/lib/exception_notifier/gitlab_notifier.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module ExceptionNotifier + # Notifica las excepciones como incidencias en Gitlab + class GitlabNotifier + def initialize(_); end + + # Recibe la excepción y empieza la tarea de notificación en segundo + # plano. + # + # @param [Exception] + # @param [Hash] + def call(exception, **options) + GitlabNotifierJob.perform_async(exception, **options) + end + end +end diff --git a/app/lib/gitlab_api_client.rb b/app/lib/gitlab_api_client.rb new file mode 100644 index 00000000..5b1287d6 --- /dev/null +++ b/app/lib/gitlab_api_client.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require 'httparty' + +class GitlabApiClient + include HTTParty + + # TODO: Hacer configurable por sitio + base_uri ENV.fetch('GITLAB_URI', 'https://0xacab.org') + # No seguir redirecciones. Si nos olvidamos https:// en la dirección, + # las redirecciones nos pueden llevar a cualquier lado y obtener + # resultados diferentes. + no_follow true + + # Trae todos los proyectos. Como estamos usando un Project Token, + # siempre va a traer uno solo. + # + # @return [HTTParty::Response] + def projects + self.class.get('/api/v4/projects', { query: { membership: true }, headers: headers }) + end + + # Obtiene el identificador del proyecto + # + # @return [Integer] + def project_id + @project_id ||= ENV['GITLAB_PROJECT'] || projects&.first&.dig('id') + end + + # Crea un issue + # + # @see https://docs.gitlab.com/ee/api/issues.html#new-issue + # @return [HTTParty::Response] + def new_issue(**args) + self.class.post("/api/v4/projects/#{project_id}/issues", { body: args, headers: headers }) + end + + # Modifica un issue + # + # @see https://docs.gitlab.com/ee/api/issues.html#edit-issue + # @return [HTTParty::Response] + def edit_issue(iid:, **args) + self.class.put("/api/v4/projects/#{project_id}/issues/#{iid}", { body: args, headers: headers }) + end + + # Crea un comentario + # + # @see https://docs.gitlab.com/ee/api/notes.html#create-new-issue-note + # @return [HTTParty::Response] + def new_note(iid:, **args) + self.class.post("/api/v4/projects/#{project_id}/issues/#{iid}/notes", { body: args, headers: headers }) + end + + private + + def headers(extra = {}) + { 'Authorization' => "Bearer #{ENV['GITLAB_TOKEN']}" }.merge(extra) + end +end diff --git a/app/models/indexed_post.rb b/app/models/indexed_post.rb new file mode 100644 index 00000000..7f6865f6 --- /dev/null +++ b/app/models/indexed_post.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +# La representación indexable de un artículo +class IndexedPost < ApplicationRecord + include PgSearch::Model + + # La traducción del locale según Sutty al locale según PostgreSQL + DICTIONARIES = { + es: 'spanish', + en: 'english' + }.freeze + + # TODO: Los indexed posts tienen que estar scopeados al idioma actual, + # no buscar sobre todos + pg_search_scope :search, + lambda { |locale, query| + { + against: :content, + query: query, + using: { + tsearch: { + dictionary: IndexedPost.to_dictionary(locale: locale), + tsvector_column: 'indexed_content' + }, + trigram: { + word_similarity: true + } + } + } + } + + # Trae los IndexedPost en el orden en que van a terminar en el sitio. + default_scope -> { order(order: :desc, created_at: :desc) } + scope :in_category, ->(category) { where("front_matter->'categories' ? :category", category: category.to_s) } + scope :by_usuarie, ->(usuarie) { where("front_matter->'usuaries' @> :usuarie::jsonb", usuarie: usuarie.to_s) } + + belongs_to :site + + # Convertir locale a direccionario de PG + # + # @param [String,Symbol] + # @return [String] + def self.to_dictionary(locale:) + DICTIONARIES[locale.to_sym] || 'simple' + end +end diff --git a/app/models/metadata_array.rb b/app/models/metadata_array.rb index 5c0b16f7..368aa546 100644 --- a/app/models/metadata_array.rb +++ b/app/models/metadata_array.rb @@ -13,6 +13,26 @@ class MetadataArray < MetadataTemplate false end + # Solo los datos públicos se indexan, aunque MetadataArray no se cifra + # aun, dejamos esto preparado para la posteridad. + def indexable? + true && !private? + end + + def to_s + value.join(', ') + end + + # Obtiene el valor desde el documento, convirtiéndolo a Array si no lo + # era ya, por retrocompabilidad. + # + # @return [Array] + def document_value + [super].flatten(1).compact + end + + alias indexable_values value + private # TODO: Sanitizar otros valores diff --git a/app/models/metadata_belongs_to.rb b/app/models/metadata_belongs_to.rb index ee182a50..1438c8db 100644 --- a/app/models/metadata_belongs_to.rb +++ b/app/models/metadata_belongs_to.rb @@ -77,6 +77,10 @@ class MetadataBelongsTo < MetadataRelatedPosts @related_methods ||= %i[belongs_to belonged_to].freeze end + def indexable_values + belongs_to&.title&.value + end + private def post_exists? diff --git a/app/models/metadata_content.rb b/app/models/metadata_content.rb index 525f38ab..437a0dd9 100644 --- a/app/models/metadata_content.rb +++ b/app/models/metadata_content.rb @@ -19,6 +19,14 @@ class MetadataContent < MetadataTemplate document.content end + def indexable? + true && !private? + end + + def to_s + sanitizer.sanitize value, tags: [], attributes: [] + end + private # Detectar si el contenido estaba en Markdown y pasarlo a HTML diff --git a/app/models/metadata_document_date.rb b/app/models/metadata_document_date.rb index 7512cbfb..324e4be8 100644 --- a/app/models/metadata_document_date.rb +++ b/app/models/metadata_document_date.rb @@ -7,12 +7,17 @@ class MetadataDocumentDate < MetadataTemplate Date.today.to_time end - def value_from_document + # @return [Time] + def document_value return nil if post.new? document.date end + def indexable? + true && !private? + end + # Siempre es obligatorio def required true @@ -41,10 +46,10 @@ class MetadataDocumentDate < MetadataTemplate begin Date.iso8601(self[:value]).to_time rescue Date::Error - value_from_document || default_value + document_value || default_value end else - self[:value] || value_from_document || default_value + self[:value] || document_value || default_value end end diff --git a/app/models/metadata_lang.rb b/app/models/metadata_lang.rb index 5f31ee9d..ff6c08e6 100644 --- a/app/models/metadata_lang.rb +++ b/app/models/metadata_lang.rb @@ -6,12 +6,13 @@ class MetadataLang < MetadataTemplate super || I18n.locale end - def value_from_document + # @return [Symbol] + def document_value document.collection.label.to_sym end def value - self[:value] ||= value_from_document || default_value + self[:value] ||= document_value || default_value end def values diff --git a/app/models/metadata_markdown_content.rb b/app/models/metadata_markdown_content.rb index d3cc6dec..92a1ab21 100644 --- a/app/models/metadata_markdown_content.rb +++ b/app/models/metadata_markdown_content.rb @@ -9,14 +9,15 @@ class MetadataMarkdownContent < MetadataText end def value - self[:value] || value_from_document || default_value + self[:value] || document_value || default_value end def front_matter? false end - def value_from_document + # @return [String] + def document_value document.content end diff --git a/app/models/metadata_path.rb b/app/models/metadata_path.rb index 3c93cca6..95fc7dbb 100644 --- a/app/models/metadata_path.rb +++ b/app/models/metadata_path.rb @@ -3,12 +3,16 @@ # Este campo representa el archivo donde se almacenan los datos class MetadataPath < MetadataTemplate # :label en este caso es el idioma/colección + # + # @return [String] def default_value File.join(site.path, "_#{lang}", "#{date}-#{slug}#{ext}") end - # El valor no vuelve desde el documento - def value_from_document + # La ruta del archivo según Jekyll + # + # @return [String] + def document_value document.path end diff --git a/app/models/metadata_related_posts.rb b/app/models/metadata_related_posts.rb index 855ea73d..092f219a 100644 --- a/app/models/metadata_related_posts.rb +++ b/app/models/metadata_related_posts.rb @@ -18,6 +18,14 @@ class MetadataRelatedPosts < MetadataArray false end + def indexable? + false + end + + def indexable_values + posts.where(uuid: value).map(&:title).map(&:value) + end + private # Obtiene todos los posts y opcionalmente los filtra diff --git a/app/models/metadata_string.rb b/app/models/metadata_string.rb index ed50bc88..95aac4d4 100644 --- a/app/models/metadata_string.rb +++ b/app/models/metadata_string.rb @@ -7,6 +7,10 @@ class MetadataString < MetadataTemplate super || '' end + def indexable? + true && !private? + end + private # No se permite HTML en las strings diff --git a/app/models/metadata_template.rb b/app/models/metadata_template.rb index cea4d009..5baa7a4a 100644 --- a/app/models/metadata_template.rb +++ b/app/models/metadata_template.rb @@ -7,6 +7,11 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, :value, :help, :required, :errors, :post, :layout, keyword_init: true) do + # Determina si el campo es indexable + def indexable? + false + end + def inspect "#<#{self.class} site=#{site.name.inspect} post=#{post.id.inspect} value=#{value.inspect}>" end @@ -44,11 +49,7 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, def value_was return @value_was if instance_variable_defined? '@value_was' - @value_was = value_from_document - end - - def value_from_document - @value_from_document ||= document.data[name.to_s] + @value_was = document_value end def changed? @@ -80,7 +81,7 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, # Valor actual o por defecto. Al memoizarlo podemos modificarlo # usando otros métodos que el de asignación. def value - self[:value] ||= if (data = value_from_document).present? + self[:value] ||= if (data = document_value).present? private? ? decrypt(data) : data else default_value diff --git a/app/models/post.rb b/app/models/post.rb index d6baee84..cab7665f 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -17,6 +17,12 @@ class Post attr_reader :attributes, :errors, :layout, :site, :document + # TODO: Modificar el historial de Git con callbacks en lugar de + # services. De esta forma podríamos agregar soporte para distintos + # backends. + include ActiveRecord::Callbacks + include Post::Indexable + class << self # Obtiene el layout sin leer el Document # @@ -194,6 +200,8 @@ class Post post: self, required: true) end + alias locale lang + # TODO: Mover a method_missing def uuid @metadata[:uuid] ||= MetadataUuid.new(document: document, site: site, layout: layout, name: :uuid, type: :uuid, @@ -256,11 +264,17 @@ class Post end # Eliminar el artículo del repositorio y de la lista de artículos del - # sitio + # sitio. + # + # TODO: Si el callback falla deberíamos recuperar el archivo. + # + # @return [Post] def destroy - FileUtils.rm_f path.absolute + run_callbacks :destroy do + FileUtils.rm_f path.absolute - site.delete_post self + site.delete_post self + end end alias destroy! destroy @@ -284,8 +298,10 @@ class Post end end - return false unless save_attributes! - return false unless write + run_callbacks :save do + return false unless save_attributes! + return false unless write + end # Vuelve a leer el post para tomar los cambios document.reset diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb new file mode 100644 index 00000000..7757e7f7 --- /dev/null +++ b/app/models/post/indexable.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +class Post + # Vuelve indexables a los Posts + module Indexable + extend ActiveSupport::Concern + + included do + # Indexa o reindexa el Post + after_save :index! + after_destroy :remove_from_index! + + # Devuelve una versión indexable del Post + # + # @return [IndexedPost] + def to_index + IndexedPost.find_or_create_by(id: uuid.value).tap do |indexed_post| + indexed_post.layout = layout.name + indexed_post.site_id = site.id + indexed_post.path = path.basename + indexed_post.locale = locale.value + indexed_post.dictionary = IndexedPost.to_dictionary(locale: locale.value) + indexed_post.title = title.value + indexed_post.front_matter = indexable_front_matter + indexed_post.content = indexable_content + indexed_post.created_at = date.value + indexed_post.order = attribute?(:order) ? order.value : 0 + end + end + + private + + # Indexa o reindexa el Post + # + # @return [Boolean] + def index! + to_index.save + end + + def remove_from_index! + to_index.destroy.destroyed? + end + + # Los metadatos que se almacenan como objetos JSON. Empezamos con + # las categorías porque se usan para filtrar en el listado de + # artículos. + # + # @return [Hash] + def indexable_front_matter + {}.tap do |ifm| + ifm[:usuaries] = usuaries.map(&:id) + ifm[:draft] = attribute?(:draft) ? draft.value : false + ifm[:categories] = categories.indexable_values if attribute? :categories + end + end + + # Devuelve un documento indexable en texto plano + # + # XXX: No memoizamos para permitir actualizaciones, aunque + # probablemente se indexe una sola vez. + # + # @return [String] + def indexable_content + indexable_attributes.map do |attr| + self[attr].to_s.tr("\n", ' ') + end.join("\n").squeeze("\n") + end + + def indexable_attributes + @indexable_attributes ||= attributes.select do |attr| + self[attr].indexable? + end + end + end + end +end diff --git a/app/models/site.rb b/app/models/site.rb index 6466384a..58f20745 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -68,6 +68,9 @@ class Site < ApplicationRecord # El sitio en Jekyll attr_reader :jekyll + # XXX: Es importante incluir luego de los callbacks de :load_jekyll + include Site::Index + # No permitir HTML en estos atributos def title=(title) super(title.strip_tags) diff --git a/app/models/site/index.rb b/app/models/site/index.rb new file mode 100644 index 00000000..e10fa523 --- /dev/null +++ b/app/models/site/index.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Indexa todos los artículos de un sitio +# +# TODO: Hacer opcional +class Site + module Index + extend ActiveSupport::Concern + + included do + # TODO: Debería ser un Job? + after_create :index_posts! + has_many :indexed_posts, dependent: :destroy + + def index_posts! + Site.transaction do + docs.each do |post| + post.to_index.save + end + end + end + end + end +end diff --git a/app/policies/post_policy.rb b/app/policies/post_policy.rb index c22202af..69ecb188 100644 --- a/app/policies/post_policy.rb +++ b/app/policies/post_policy.rb @@ -59,9 +59,7 @@ class PostPolicy def resolve return scope if scope&.first&.site&.usuarie? usuarie - scope.select do |post| - post.usuaries.include? usuarie - end + scope.by_usuarie(usuarie.id) end end end diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 4f3905a5..389549c3 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -61,6 +61,18 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do commit_config(action: :tor) end + # Trae cambios desde la rama remota y reindexa los artículos. + # + # @return [Boolean] + def merge + result = site.repository.merge(usuarie) + + # TODO: Implementar callbacks + site.try(:index_posts!) if result + + result.present? + end + private # Guarda los cambios de la configuración en el repositorio git diff --git a/app/views/exception_notifier/_backtrace.text.erb b/app/views/exception_notifier/_backtrace.text.erb index d62b5719..aed7adbe 100644 --- a/app/views/exception_notifier/_backtrace.text.erb +++ b/app/views/exception_notifier/_backtrace.text.erb @@ -1,4 +1,4 @@ -<% unless @data[:_backtrace] %> +<% unless @data[:javascript_backtrace] %> ``` <%= raw @backtrace.join("\n") %> ``` diff --git a/app/views/exception_notifier/_data.text.erb b/app/views/exception_notifier/_data.text.erb index 09313f4c..acb94b89 100644 --- a/app/views/exception_notifier/_data.text.erb +++ b/app/views/exception_notifier/_data.text.erb @@ -1,4 +1,4 @@ -<% if @data[:_backtrace] %> +<% if @data[:javascript_backtrace] %> <% @data.dig(:params, 'errors')&.each do |error| %> # <%= error['type'] %>: <%= error['message'] %> diff --git a/app/views/posts/attributes/_markdown.haml b/app/views/posts/attributes/_markdown.haml index 325beb5c..8042009f 100644 --- a/app/views/posts/attributes/_markdown.haml +++ b/app/views/posts/attributes/_markdown.haml @@ -1,8 +1,8 @@ .form-group.markdown-content = label_tag "#{base}_#{attribute}", post_label_t(attribute, post: post) + = render 'posts/attribute_feedback', + post: post, attribute: attribute, metadata: metadata = text_area_tag "#{base}[#{attribute}]", metadata.value, dir: dir, lang: locale, **field_options(attribute, metadata, class: 'content') - .editor.mt-1 - = render 'posts/attribute_feedback', - post: post, attribute: attribute, metadata: metadata + .markdown-editor.mt-1 diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 33bb5a7c..8b776590 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -7,15 +7,13 @@ %table.mb-3 - @site.layouts.each do |layout| - next if layout.hidden? - - filter = params[:layout] == layout.value %tr %th= layout.humanized_name - %td.pl-3= link_to t('posts.add'), - new_site_post_path(@site, layout: layout.name), - class: 'badge badge-secondary' - %td= link_to t(filter ? 'posts.remove_filter' : 'posts.filter'), - site_posts_path(@site, layout: (filter ? nil : layout.value)), - class: 'badge badge-' + (filter ? 'primary' : 'secondary') + %td.pl-3= link_to t('posts.add'), new_site_post_path(@site, layout: layout.value), class: 'btn btn-secondary btn-sm' + - if @filter_params[:layout] == layout.name.to_s + %td= link_to t('posts.remove_filter'), site_posts_path(@site, **@filter_params.merge(layout: nil)), class: 'btn btn-primary btn-sm' + - else + %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary btn-sm' - if policy(@site).edit? = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn' @@ -41,19 +39,34 @@ %section.col = render 'layouts/flash' + .d-flex.justify-content-between.align-items-center.pl-2-plus.pr-2-plus.mb-2 + %form + - @filter_params.each do |param, value| + - next if param == 'q' + %input{ type: 'hidden', name: param, value: value } + .form-group.flex-grow-0.m-0 + %input.form-control.border.border-magenta{ type: 'search', placeholder: 'Buscar', name: 'q', value: @filter_params[:q] } + %input.sr-only{ type: 'submit' } + + - if @site.locales.size > 1 + %nav#locales + - @site.locales.each do |locale| + = link_to t("locales.#{locale}.name"), site_posts_path(@site, **@filter_params.merge(locale: locale)), + class: "mr-2 mt-2 mb-2 #{locale == @locale ? 'active font-weight-bold' : ''}" + .pl-2-plus + - @filter_params.each do |param, value| + - if param == 'layout' + - value = @site.layouts[value.to_sym].humanized_name + = link_to site_posts_path(@site, **@filter_params.reject { |k, _| k == param }), + class: 'btn btn-secondary btn-sm', + title: t('posts.remove_filter_help', filter: value), + aria: { labelledby: "help-filter-#{param}" } do + = value + × - if @posts.empty? - %h2= t('posts.none') + %h2= t('posts.empty') - else = form_tag site_posts_reorder_path, method: :post do - .d-flex.justify-content-between.align-items-center - -# - TODO: Pensar una interfaz mejor para cuando haya más de tres - idiomas - - unless @site.locales.length == 1 - .locales - - @site.locales.each do |locale| - = link_to t("locales.#{locale}.name"), site_posts_path(@site, locale: locale), - class: "mr-2 mt-2 mb-2#{locale == @locale ? 'active font-weight-bold' : ''}" %table.table{ data: { controller: 'reorder' } } %caption.sr-only= t('posts.caption') %thead @@ -69,54 +82,48 @@ %button.btn{ data: { action: 'reorder#bottom' } }= t('posts.reorder.bottom') %tbody - dir = t("locales.#{@locale}.dir") + - size = @posts.size - @posts.each_with_index do |post, i| -# TODO: Solo les usuaries cachean porque tenemos que separar les botones por permisos. - cache_if @usuarie, [post, I18n.locale] do - - checkbox_id = "checkbox-#{post.uuid.value}" - %tr{ id: post.uuid.value, data: { target: 'reorder.row' } } + - checkbox_id = "checkbox-#{post.id}" + %tr{ id: post.id, data: { target: 'reorder.row' } } %td .custom-control.custom-checkbox %input.custom-control-input{ id: checkbox_id, type: 'checkbox', autocomplete: 'off', data: { action: 'reorder#select' } } %label.custom-control-label{ for: checkbox_id } %span.sr-only= t('posts.reorder.select') -# Orden más alto es mayor prioridad - = hidden_field 'post[reorder]', post.uuid.value, - value: @posts.length - i, + = hidden_field 'post[reorder]', post.id, + value: size - i, data: { reorder: true } %td.w-100{ class: dir } - = link_to site_post_path(@site, post.id) do - %span{ lang: post.lang.value, dir: dir }= post.title.value - - if post.attributes.include? :draft - - if post.draft.value - %span.badge.badge-primary - = post_label_t(:draft, post: post) - - if post.attributes.include? :categories - - unless post.categories.value.empty? - %br - %small - - categories = post.categories.respond_to?(:has_many) ? post.categories.has_many : post.categories.value - - categories.each do |c| - - c.read if c.respond_to? :read - = link_to site_posts_path(@site, category: (c.respond_to?(:uuid) ? c.uuid.value : c)) do - %span{ lang: post.lang.value, dir: dir }= (c.respond_to?(:title) ? c.title.value : c) - - unless categories.last == c - = '/' + = link_to site_post_path(@site, post.path) do + %span{ lang: post.locale, dir: dir }= post.title + - if post.front_matter['draft'].present? + %span.badge.badge-primary= I18n.t('posts.attributes.draft.label') + - if post.front_matter['categories'].present? + %br + %small + - post.front_matter['categories'].each do |category| + = link_to site_posts_path(@site, **@filter_params.merge(category: category)) do + %span{ lang: post.locale, dir: dir }= category + = '/' unless post.front_matter['categories'].last == category %td - = post.date.value.strftime('%F') + = post.created_at.strftime('%F') %br/ - - if post.attribute? :order - = post.order.value + = post.order %td - if @usuarie || policy(post).edit? - = link_to t('posts.edit'), - edit_site_post_path(@site, post.id), - class: 'btn btn-block' + = link_to t('posts.edit'), edit_site_post_path(@site, post.path), class: 'btn btn-block' - if @usuarie || policy(post).destroy? - = link_to t('posts.destroy'), - site_post_path(@site, post.id), - class: 'btn btn-block', - method: :delete, - data: { confirm: t('posts.confirm_destroy') } + = link_to t('posts.destroy'), site_post_path(@site, post.path), class: 'btn btn-block', method: :delete, data: { confirm: t('posts.confirm_destroy') } + +#footnotes{ hidden: true } + - @filter_params.each do |param, value| + - if param == 'layout' + - value = @site.layouts[value.to_sym].humanized_name + %label{ id: "help-filter-#{param}" }= t('posts.remove_filter_help', filter: value) diff --git a/config/environments/production.rb b/config/environments/production.rb index c1269fb2..d121bdbd 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -147,14 +147,7 @@ Rails.application.configure do } config.action_mailer.default_options = { from: ENV.fetch('DEFAULT_FROM', "noreply@sutty.nl") } - config.middleware.use ExceptionNotification::Rack, - error_grouping: true, - email: { - email_prefix: '', - sender_address: ENV.fetch('DEFAULT_FROM', "noreply@sutty.nl"), - exception_recipients: ENV.fetch('EXCEPTION_TO', "errors@sutty.nl"), - normalize_subject: true - } + config.middleware.use ExceptionNotification::Rack, gitlab: {} Rails.application.routes.default_url_options[:host] = "panel.#{ENV.fetch('SUTTY', 'sutty.nl')}" Rails.application.routes.default_url_options[:protocol] = 'https' diff --git a/config/environments/test.rb b/config/environments/test.rb index c58a65ca..05506587 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -69,8 +69,8 @@ Rails.application.configure do error_grouping: true, email: { email_prefix: '', - sender_address: ENV.fetch('DEFAULT_FROM', "noreply@sutty.nl"), - exception_recipients: ENV.fetch('EXCEPTION_TO', "errors@sutty.nl"), + sender_address: ENV.fetch('DEFAULT_FROM', 'noreply@sutty.nl'), + exception_recipients: ENV.fetch('EXCEPTION_TO', 'errors@sutty.nl'), normalize_subject: true } end diff --git a/config/initializers/core_extensions.rb b/config/initializers/core_extensions.rb index c1e8d652..66d2c92b 100644 --- a/config/initializers/core_extensions.rb +++ b/config/initializers/core_extensions.rb @@ -102,6 +102,19 @@ module Jekyll end end +# No aplicar el orden por ranking para poder obtener los artículos en el +# orden que tendrían en el sitio final. +module PgSearch + ScopeOptions.class_eval do + def apply(scope) + scope = include_table_aliasing_for_rank(scope) + rank_table_alias = scope.pg_search_rank_table_alias(include_counter: true) + + scope.joins(rank_join(rank_table_alias)) + end + end +end + # JekyllData::Reader del plugin jekyll-data modifica Jekyll::Site#reader # para también leer los datos que vienen en el theme. module JekyllData diff --git a/config/locales/en.yml b/config/locales/en.yml index f1f9b7cb..fc194eab 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -376,6 +376,7 @@ en: en: 'English' ar: 'Arabic' posts: + empty: "There are no results for those search parameters." caption: Post list attribute_ro: file: @@ -412,6 +413,8 @@ en: destroy: Remove image belongs_to: empty: "(Empty)" + draft: + label: Draft reorder: submit: 'Save order' select: 'Select this post' @@ -430,6 +433,7 @@ en: add: 'Add' filter: 'Filter' remove_filter: 'Back' + remove_filter_help: 'Remove the filter: %{filter}' categories: 'Everything' index: 'Posts' edit: 'Edit' diff --git a/config/locales/es.yml b/config/locales/es.yml index 1ce50b09..e8185391 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -180,6 +180,7 @@ es: category: 'Categoría' sites: index: 'Este es el listado de sitios que puedes editar.' + edit_posts: 'Aquí verás el listado de todos los artículos y podrás editarlos o crear nuevos' enqueued: 'El sitio está en la cola de espera para ser generado. Una vez que este proceso termine, recibirás un correo indicando el estado y si todo fue bien, se publicarán los cambios en tu sitio @@ -383,6 +384,7 @@ es: en: 'inglés' ar: 'árabe' posts: + empty: No hay artículos con estos parámetros de búsqueda. caption: Lista de artículos attribute_ro: file: @@ -419,6 +421,8 @@ es: destroy: 'Eliminar imagen' belongs_to: empty: "(Vacío)" + draft: + label: Borrador reorder: submit: 'Guardar orden' select: 'Seleccionar este artículo' @@ -438,6 +442,7 @@ es: add: 'Agregar' filter: 'Filtrar' remove_filter: 'Volver' + remove_filter_help: 'Quitar este filtro: %{filter}' index: 'Artículos' edit: 'Editar' preview: diff --git a/config/puma.rb b/config/puma.rb index 60ee5ecc..414507ed 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -26,7 +26,7 @@ if ENV['RAILS_ENV'] == 'production' bind 'tcp://[::]:3100' else sutty = ENV.fetch('SUTTY', 'sutty.local') - bind "ssl://[::]:3000?key=../sutty.local/domain/#{sutty}.key&cert=../sutty.local/domain/#{sutty}.crt" + bind "ssl://[::]:3000?key=/etc/ssl/private/#{sutty}.key&cert=/etc/ssl/certs/#{sutty}.crt" end # Specifies the `environment` that Puma will run in. diff --git a/config/routes.rb b/config/routes.rb index 766df50a..2c5f1c60 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -31,7 +31,8 @@ Rails.application.routes.draw do # detectar el nombre del sitio. get '/sites/private/:site_id(*file)', to: 'private#show', constraints: { site_id: %r{[^/]+} } # Obtener archivos estáticos desde el directorio público - get '/sites/:site_id/static_file/(*file)', to: 'sites#static_file', as: 'site_static_file', constraints: { site_id: %r{[^/]+} } + get '/sites/:site_id/static_file/(*file)', to: 'sites#static_file', as: 'site_static_file', + constraints: { site_id: %r{[^/]+} } get '/env.js', to: 'env#index' match '/api/v3/projects/:site_id/notices' => 'api/v1/notices#create', via: %i[post] diff --git a/config/webpacker.yml b/config/webpacker.yml index 25519907..d555770d 100644 --- a/config/webpacker.yml +++ b/config/webpacker.yml @@ -56,8 +56,8 @@ development: # Reference: https://webpack.js.org/configuration/dev-server/ dev_server: https: - key: '../sutty.local/domain/sutty.local.key' - cert: '../sutty.local/domain/sutty.local.crt' + key: '/etc/ssl/private/sutty.local.key' + cert: '/etc/ssl/certs/sutty.local.crt' # XXX: esto está hardcodeado, debería conseguirlo del ENV host: sutty.local port: 3035 diff --git a/db/migrate/20210504224144_create_pg_search_extensions.rb b/db/migrate/20210504224144_create_pg_search_extensions.rb new file mode 100644 index 00000000..18eebe95 --- /dev/null +++ b/db/migrate/20210504224144_create_pg_search_extensions.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Habilitar las extensiones de búsqueda de texto libre +class CreatePgSearchExtensions < ActiveRecord::Migration[6.1] + def change + enable_extension 'plpgsql' + enable_extension 'pg_trgm' + end +end diff --git a/db/migrate/20210504224343_create_indexed_posts.rb b/db/migrate/20210504224343_create_indexed_posts.rb new file mode 100644 index 00000000..9cf21538 --- /dev/null +++ b/db/migrate/20210504224343_create_indexed_posts.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# Crea la tabla donde se indexa el contenido de los artículos, los +# IndexedPosts van a estar relacionados con un Post del mismo UUID. +# +# Solo contienen la información mínima necesaria para mostrar los +# resultados de búsqueda. +class CreateIndexedPosts < ActiveRecord::Migration[6.1] + def change + # Necesario para gen_random_uuid() + # + # XXX: En realidad no lo necesitamos porque cada IndexedPost va a + # tener el UUID del Post correspondiente. + enable_extension 'pgcrypto' + + create_table :indexed_posts, id: false do |t| + t.primary_key :id, :uuid, default: 'public.gen_random_uuid()' + t.belongs_to :site, index: true + t.timestamps + + # Filtramos por idioma + t.string :locale, default: 'simple', index: true + # Vamos a querer filtrar por layout + t.string :layout, null: false, index: true + # Esta es la ruta al artículo + t.string :path, null: false + # Queremos mostrar el título por separado + t.string :title, default: '' + # También vamos a mostrar las categorías + t.jsonb :front_matter, default: '{}' + t.string :content, default: '' + t.tsvector :indexed_content + + t.index :indexed_content, using: 'gin' + t.index :front_matter, using: 'gin' + end + + # Crea un trigger que actualiza el índice tsvector con el título y + # contenido del artículo y su idioma. + create_trigger(compatibility: 1).on(:indexed_posts).before(:insert, :update) do + "new.indexed_content := to_tsvector(('pg_catalog.' || new.locale)::regconfig, coalesce(new.title, '') || '\n' || coalesce(new.content,''));" + end + end +end diff --git a/db/migrate/20210506212356_add_order_to_indexed_posts.rb b/db/migrate/20210506212356_add_order_to_indexed_posts.rb new file mode 100644 index 00000000..4b1a9fcf --- /dev/null +++ b/db/migrate/20210506212356_add_order_to_indexed_posts.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Agrega el orden y fecha del post en el post indexado, de forma que los +# resultados se puedan obtener en el mismo orden. +class AddOrderToIndexedPosts < ActiveRecord::Migration[6.1] + def change + add_column :indexed_posts, :order, :integer, default: 0 + end +end diff --git a/db/migrate/20210507221120_add_dictionary_to_indexed_posts.rb b/db/migrate/20210507221120_add_dictionary_to_indexed_posts.rb new file mode 100644 index 00000000..f79309fd --- /dev/null +++ b/db/migrate/20210507221120_add_dictionary_to_indexed_posts.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# Para no estar calculando todo el tiempo el diccionario del idioma, +# agregamos una columna más. +class AddDictionaryToIndexedPosts < ActiveRecord::Migration[6.1] + LOCALES = { + 'english' => 'en', + 'spanish' => 'es' + } + + def up + add_column :indexed_posts, :dictionary, :string + + create_trigger(compatibility: 1).on(:indexed_posts).before(:insert, :update) do + "new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig, coalesce(new.title, '') || '\n' || coalesce(new.content,''));" + end + + IndexedPost.find_each do |post| + locale = post.locale + + post.update dictionary: locale, locale: LOCALES[locale] + end + end + + def down + remove_column :indexed_posts, :locale + rename_column :indexed_posts, :dictionary, :locale + end +end diff --git a/db/schema.rb b/db/schema.rb index eeb90ac6..107e7be7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,16 +10,74 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_11_211357) do +ActiveRecord::Schema.define(version: 2021_05_14_165639) do -# Could not dump table "access_logs" because of following StandardError -# Unknown type '' for column 'id' + # These are extensions that must be enabled in order to support this database + enable_extension "pg_trgm" + enable_extension "pgcrypto" + enable_extension "plpgsql" + + create_table "access_logs", id: :uuid, default: nil, force: :cascade do |t| + t.string "host" + t.float "msec" + t.string "server_protocol" + t.string "request_method" + t.string "request_completion" + t.string "uri" + t.string "query_string" + t.integer "status" + t.string "sent_http_content_type" + t.string "sent_http_content_encoding" + t.string "sent_http_etag" + t.string "sent_http_last_modified" + t.string "http_accept" + t.string "http_accept_encoding" + t.string "http_accept_language" + t.string "http_pragma" + t.string "http_cache_control" + t.string "http_if_none_match" + t.string "http_dnt" + t.string "http_user_agent" + t.string "http_origin" + t.float "request_time" + t.integer "bytes_sent" + t.integer "body_bytes_sent" + t.integer "request_length" + t.string "http_connection" + t.string "pipe" + t.integer "connection_requests" + t.string "geoip2_data_country_name" + t.string "geoip2_data_city_name" + t.string "ssl_server_name" + t.string "ssl_protocol" + t.string "ssl_early_data" + t.string "ssl_session_reused" + t.string "ssl_curves" + t.string "ssl_ciphers" + t.string "ssl_cipher" + t.string "sent_http_x_xss_protection" + t.string "sent_http_x_frame_options" + t.string "sent_http_x_content_type_options" + t.string "sent_http_strict_transport_security" + t.string "nginx_version" + t.integer "pid" + t.string "remote_user" + t.boolean "crawler", default: false + t.string "http_referer" + t.index ["geoip2_data_city_name"], name: "index_access_logs_on_geoip2_data_city_name" + t.index ["geoip2_data_country_name"], name: "index_access_logs_on_geoip2_data_country_name" + t.index ["host"], name: "index_access_logs_on_host" + t.index ["http_origin"], name: "index_access_logs_on_http_origin" + t.index ["http_user_agent"], name: "index_access_logs_on_http_user_agent" + t.index ["status"], name: "index_access_logs_on_status" + t.index ["uri"], name: "index_access_logs_on_uri" + end create_table "action_text_rich_texts", force: :cascade do |t| t.string "name", null: false t.text "body" t.string "record_type", null: false - t.integer "record_id", null: false + t.bigint "record_id", null: false t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true @@ -28,8 +86,8 @@ ActiveRecord::Schema.define(version: 2021_05_11_211357) do create_table "active_storage_attachments", force: :cascade do |t| t.string "name", null: false t.string "record_type", null: false - t.integer "record_id", null: false - t.integer "blob_id", null: false + t.bigint "record_id", null: false + t.bigint "blob_id", null: false t.datetime "created_at", null: false t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true @@ -40,7 +98,7 @@ ActiveRecord::Schema.define(version: 2021_05_11_211357) do t.string "filename", null: false t.string "content_type" t.text "metadata" - t.integer "byte_size", null: false + t.bigint "byte_size", null: false t.string "checksum", null: false t.datetime "created_at", null: false t.string "service_name", null: false @@ -53,10 +111,65 @@ ActiveRecord::Schema.define(version: 2021_05_11_211357) do t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true end + create_table "blazer_audits", force: :cascade do |t| + t.bigint "user_id" + t.bigint "query_id" + t.text "statement" + t.string "data_source" + t.datetime "created_at" + t.index ["query_id"], name: "index_blazer_audits_on_query_id" + t.index ["user_id"], name: "index_blazer_audits_on_user_id" + end + + create_table "blazer_checks", force: :cascade do |t| + t.bigint "creator_id" + t.bigint "query_id" + t.string "state" + t.string "schedule" + t.text "emails" + t.text "slack_channels" + t.string "check_type" + t.text "message" + t.datetime "last_run_at" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["creator_id"], name: "index_blazer_checks_on_creator_id" + t.index ["query_id"], name: "index_blazer_checks_on_query_id" + end + + create_table "blazer_dashboard_queries", force: :cascade do |t| + t.bigint "dashboard_id" + t.bigint "query_id" + t.integer "position" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["dashboard_id"], name: "index_blazer_dashboard_queries_on_dashboard_id" + t.index ["query_id"], name: "index_blazer_dashboard_queries_on_query_id" + end + + create_table "blazer_dashboards", force: :cascade do |t| + t.bigint "creator_id" + t.text "name" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["creator_id"], name: "index_blazer_dashboards_on_creator_id" + end + + create_table "blazer_queries", force: :cascade do |t| + t.bigint "creator_id" + t.string "name" + t.text "description" + t.text "statement" + t.string "data_source" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["creator_id"], name: "index_blazer_queries_on_creator_id" + end + create_table "build_stats", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "deploy_id" + t.bigint "deploy_id" t.bigint "bytes" t.float "seconds" t.string "action", null: false @@ -65,13 +178,27 @@ ActiveRecord::Schema.define(version: 2021_05_11_211357) do t.index ["deploy_id"], name: "index_build_stats_on_deploy_id" end -# Could not dump table "csp_reports" because of following StandardError -# Unknown type 'uuid' for column 'id' + create_table "csp_reports", id: :uuid, default: nil, force: :cascade do |t| + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.string "disposition" + t.string "referrer" + t.string "blocked_uri" + t.string "document_uri" + t.string "effective_directive" + t.string "original_policy" + t.string "script_sample" + t.string "status_code" + t.string "violated_directive" + t.integer "column_number" + t.integer "line_number" + t.string "source_file" + end create_table "deploys", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "site_id" + t.bigint "site_id" t.string "type" t.text "values" t.index ["site_id"], name: "index_deploys_on_site_id" @@ -91,6 +218,26 @@ ActiveRecord::Schema.define(version: 2021_05_11_211357) do t.string "designer_url" end + create_table "indexed_posts", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.bigint "site_id" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.string "locale", default: "simple" + t.string "layout", null: false + t.string "path", null: false + t.string "title", default: "" + t.jsonb "front_matter", default: "{}" + t.string "content", default: "" + t.tsvector "indexed_content" + t.integer "order", default: 0 + t.string "dictionary" + t.index ["front_matter"], name: "index_indexed_posts_on_front_matter", using: :gin + t.index ["indexed_content"], name: "index_indexed_posts_on_indexed_content", using: :gin + t.index ["layout"], name: "index_indexed_posts_on_layout" + t.index ["locale"], name: "index_indexed_posts_on_locale" + t.index ["site_id"], name: "index_indexed_posts_on_site_id" + end + create_table "licencias", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -104,7 +251,7 @@ ActiveRecord::Schema.define(version: 2021_05_11_211357) do create_table "log_entries", force: :cascade do |t| t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false - t.integer "site_id" + t.bigint "site_id" t.text "text" t.boolean "sent", default: false t.index ["site_id"], name: "index_log_entries_on_site_id" @@ -124,7 +271,7 @@ ActiveRecord::Schema.define(version: 2021_05_11_211357) do t.string "key", null: false t.string "value" t.string "translatable_type" - t.integer "translatable_id" + t.bigint "translatable_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["translatable_id", "translatable_type", "key"], name: "index_mobility_string_translations_on_translatable_attribute" @@ -137,7 +284,7 @@ ActiveRecord::Schema.define(version: 2021_05_11_211357) do t.string "key", null: false t.text "value" t.string "translatable_type" - t.integer "translatable_id" + t.bigint "translatable_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["translatable_id", "translatable_type", "key"], name: "index_mobility_text_translations_on_translatable_attribute" @@ -147,8 +294,8 @@ ActiveRecord::Schema.define(version: 2021_05_11_211357) do create_table "roles", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "site_id" - t.integer "usuarie_id" + t.bigint "site_id" + t.bigint "usuarie_id" t.string "rol" t.boolean "temporal" t.index ["site_id", "usuarie_id"], name: "index_roles_on_site_id_and_usuarie_id", unique: true @@ -160,15 +307,14 @@ ActiveRecord::Schema.define(version: 2021_05_11_211357) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.string "name" - t.integer "design_id" - t.integer "licencia_id" + t.bigint "design_id" + t.bigint "licencia_id" t.string "status", default: "waiting" t.text "description" t.string "title" t.boolean "colaboracion_anonima", default: false t.boolean "contact", default: false t.string "private_key_ciphertext" - t.boolean "invitades", default: false t.boolean "acepta_invitades", default: false t.string "tienda_api_key_ciphertext", default: "" t.string "tienda_url", default: "" @@ -200,7 +346,7 @@ ActiveRecord::Schema.define(version: 2021_05_11_211357) do t.datetime "invitation_accepted_at" t.integer "invitation_limit" t.string "invited_by_type" - t.integer "invited_by_id" + t.bigint "invited_by_id" t.integer "invitations_count", default: 0 t.string "lang", default: "es" t.index ["confirmation_token"], name: "index_usuaries_on_confirmation_token", unique: true @@ -208,11 +354,20 @@ ActiveRecord::Schema.define(version: 2021_05_11_211357) do t.index ["invitation_token"], name: "index_usuaries_on_invitation_token", unique: true t.index ["invitations_count"], name: "index_usuaries_on_invitations_count" t.index ["invited_by_id"], name: "index_usuaries_on_invited_by_id" - t.index ["invited_by_type", "invited_by_id"], name: "index_usuaries_on_invited_by_type_and_invited_by_id" + t.index ["invited_by_type", "invited_by_id"], name: "index_usuaries_on_invited_by" t.index ["reset_password_token"], name: "index_usuaries_on_reset_password_token", unique: true t.index ["unlock_token"], name: "index_usuaries_on_unlock_token", unique: true end add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + create_trigger("indexed_posts_before_insert_update_row_tr", :compatibility => 1). + on("indexed_posts"). + before(:insert, :update) do + <<-SQL_ACTIONS +new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig, coalesce(new.title, '') || ' +' || coalesce(new.content,'')); + SQL_ACTIONS + end + end diff --git a/ota.sh b/ota.sh new file mode 100755 index 00000000..68a0642f --- /dev/null +++ b/ota.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -e + +cd /srv/http + +for patch in /tmp/patches-${1}/*.patch; do + su -c "patch -Np 1 -i ${patch}" app && rm $patch +done + +cat tmp/puma.pid | xargs -r kill -USR2 diff --git a/test/jobs/backtrace_job_test.rb b/test/jobs/backtrace_job_test.rb index 89c68609..e0061800 100644 --- a/test/jobs/backtrace_job_test.rb +++ b/test/jobs/backtrace_job_test.rb @@ -19,87 +19,87 @@ class DeployJobTest < ActiveSupport::TestCase # setTimeout(() => { throw('Prueba') }, 1000) def notice @notice ||= { - "errors" => [ + 'errors' => [ { - "type" => "", - "message" => "Prueba", - "backtrace" => [ + 'type' => '', + 'message' => 'Prueba', + 'backtrace' => [ { - "function" => "pt "https://tintalimon.com.ar/assets/js/pack.js", - "line" => 89, - "column" => 74094 + 'function' => 'pt 'https://tintalimon.com.ar/assets/js/pack.js', + 'line' => 89, + 'column' => 74_094 }, { - "function" => "pt "https://tintalimon.com.ar/assets/js/pack.js", - "line" => 89, - "column" => 74731 + 'function' => 'pt 'https://tintalimon.com.ar/assets/js/pack.js', + 'line' => 89, + 'column' => 74_731 }, { - "function" => "pt "https://tintalimon.com.ar/assets/js/pack.js", - "line" => 89, - "column" => 71925 + 'function' => 'pt 'https://tintalimon.com.ar/assets/js/pack.js', + 'line' => 89, + 'column' => 71_925 }, { - "function" => "setTimeout handler*", - "file" => "debugger eval code", - "line" => 1, - "column" => 11 + 'function' => 'setTimeout handler*', + 'file' => 'debugger eval code', + 'line' => 1, + 'column' => 11 } ] } ], - "context" => { - "severity" => "error", - "history" => [ + 'context' => { + 'severity' => 'error', + 'history' => [ { - "type" => "error", - "target" => "html. > head. > script.[type=\"text/javascript\"][src=\"//stats.habitapp.org/piwik.js\"]", - "date" => "2021-04-26T22:06:58.390Z" + 'type' => 'error', + 'target' => 'html. > head. > script.[type="text/javascript"][src="//stats.habitapp.org/piwik.js"]', + 'date' => '2021-04-26T22:06:58.390Z' }, { - "type" => "DOMContentLoaded", - "target" => "[object HTMLDocument]", - "date" => "2021-04-26T22:06:58.510Z" + 'type' => 'DOMContentLoaded', + 'target' => '[object HTMLDocument]', + 'date' => '2021-04-26T22:06:58.510Z' }, { - "type" => "load", - "target" => "[object HTMLDocument]", - "date" => "2021-04-26T22:06:58.845Z" + 'type' => 'load', + 'target' => '[object HTMLDocument]', + 'date' => '2021-04-26T22:06:58.845Z' }, { - "type" => "xhr", - "date" => "2021-04-26T22:06:58.343Z", - "method" => "GET", - "url" => "assets/data/site.json", - "statusCode" => 200, - "duration" => 506 + 'type' => 'xhr', + 'date' => '2021-04-26T22:06:58.343Z', + 'method' => 'GET', + 'url' => 'assets/data/site.json', + 'statusCode' => 200, + 'duration' => 506 }, { - "type" => "xhr", - "date" => "2021-04-26T22:06:58.886Z", - "method" => "GET", - "url" => "assets/templates/cart.html", - "statusCode" => 200, - "duration" => 591 + 'type' => 'xhr', + 'date' => '2021-04-26T22:06:58.886Z', + 'method' => 'GET', + 'url' => 'assets/templates/cart.html', + 'statusCode' => 200, + 'duration' => 591 } ], - "windowError" => true, - "notifier" => { - "name" => "airbrake-js/browser", - "version" => "1.4.2", - "url" => "https://github.com/airbrake/airbrake-js/tree/master/packages/browser" + 'windowError' => true, + 'notifier' => { + 'name' => 'airbrake-js/browser', + 'version' => '1.4.2', + 'url' => 'https://github.com/airbrake/airbrake-js/tree/master/packages/browser' }, - "userAgent" => "Mozilla/5.0 (Windows NT 6.1; rv:85.0) Gecko/20100101 Firefox/85.0", - "url" => "https://tintalimon.com.ar/carrito/", - "rootDirectory" => "https://tintalimon.com.ar", - "language" => "JavaScript" + 'userAgent' => 'Mozilla/5.0 (Windows NT 6.1; rv:85.0) Gecko/20100101 Firefox/85.0', + 'url' => 'https://tintalimon.com.ar/carrito/', + 'rootDirectory' => 'https://tintalimon.com.ar', + 'language' => 'JavaScript' }, - "params" => {}, - "environment" => {}, - "session" => {} + 'params' => {}, + 'environment' => {}, + 'session' => {} } # XXX: Siempre devolvemos un duplicado porque BacktraceJob lo @@ -120,8 +120,8 @@ class DeployJobTest < ActiveSupport::TestCase email = ActionMailer::Base.deliveries.first assert email - assert_equal " (BacktraceJob::BacktraceException) \"tintalimon.com.ar: Prueba\"", email.subject - assert (%r{webpack://} =~ email.body.to_s) + assert_equal ' (BacktraceJob::BacktraceException) "tintalimon.com.ar: Prueba"', email.subject + assert(%r{webpack://} =~ email.body.to_s) end test 'los errores se basan en un sitio' do diff --git a/test/models/indexed_post_test.rb b/test/models/indexed_post_test.rb new file mode 100644 index 00000000..27d4e29e --- /dev/null +++ b/test/models/indexed_post_test.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'test_helper' + +class IndexedPostTest < ActiveSupport::TestCase + def site + @site ||= create :site + end + + teardown do + @site&.destroy + end + + test 'se pueden convertir los diccionarios' do + IndexedPost::DICTIONARIES.each do |locale, dict| + assert_equal dict, IndexedPost.to_dictionary(locale: locale) + end + end + + test 'se pueden buscar por categoría' do + assert(post = site.posts.create(title: SecureRandom.hex, description: SecureRandom.hex, + categories: [SecureRandom.hex, SecureRandom.hex])) + assert_not_empty site.indexed_posts.in_category(post.categories.value.sample) + end + + test 'se pueden encontrar por usuarie' do + usuarie = create :usuarie + assert(post = site.posts.create(title: SecureRandom.hex, description: SecureRandom.hex)) + + post.usuaries << usuarie + post.save + + assert_not_empty site.indexed_posts.by_usuarie(usuarie.id) + end +end diff --git a/test/models/post/indexable_test.rb b/test/models/post/indexable_test.rb new file mode 100644 index 00000000..6110bcf0 --- /dev/null +++ b/test/models/post/indexable_test.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'test_helper' + +class Post::IndexableTest < ActiveSupport::TestCase + setup do + @site = create :site + end + + teardown do + @site&.destroy + end + + test 'los posts se indexan apenas se crean' do + post = @site.posts.create(title: SecureRandom.hex, description: SecureRandom.hex) + indexed_post = @site.indexed_posts.find_by_title post.title.value + + assert indexed_post + assert_equal post.locale.value.to_s, indexed_post.locale + assert_equal post.order.value, indexed_post.order + assert_equal post.path.basename, indexed_post.path + assert_equal post.layout.name.to_s, indexed_post.layout + end + + test 'se pueden encontrar posts' do + post = @site.posts.sample + + assert @site.indexed_posts.where(locale: post.lang.value).search(post.lang.value, post.title.value) + assert @site.indexed_posts.where(locale: post.lang.value).search(post.lang.value, post.description.value) + end + + test 'se pueden actualizar posts' do + post = @site.posts.sample + post.description.value = SecureRandom.hex + + assert post.save + assert @site.indexed_posts.where(locale: post.lang.value).search(post.lang.value, post.description.value) + end + + test 'al borrar el post se borra el indice' do + post = @site.posts.sample + assert post.destroy + assert_not @site.indexed_posts.find_by_id(post.uuid.value) + end +end diff --git a/test/models/post_test.rb b/test/models/post_test.rb index f98d7af3..52e59a8e 100644 --- a/test/models/post_test.rb +++ b/test/models/post_test.rb @@ -110,7 +110,6 @@ class PostTest < ActiveSupport::TestCase end test 'se puede cambiar la fecha' do - assert_not @post.date.changed? assert @post.date.valid? ex_date = @post.date.value