From 3a2ce1d47d1d111c5afe9f7d1bfdd482ff256207 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 17:50:32 -0300 Subject: [PATCH 001/251] =?UTF-8?q?Configurar=20Blazer=20para=20mostrar=20?= =?UTF-8?q?estad=C3=ADsticas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/application_controller.rb | 9 +++++++++ config/blazer.yml | 2 +- config/routes.rb | 7 +++---- db/migrate/20200206163257_install_blazer.rb | 2 -- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index acd0134d..f95159f4 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -62,6 +62,15 @@ class ApplicationController < ActionController::Base render 'application/page_not_found', status: :not_found end + # Necesario para poder acceder a Blazer. Solo les usuaries de este + # sitio pueden acceder al panel. + def require_usuarie + unless find_site.usuarie? current_usuarie + redirect_to root_path + return + end + end + protected def configure_permitted_parameters diff --git a/config/blazer.yml b/config/blazer.yml index 2ba0965f..11792ff1 100644 --- a/config/blazer.yml +++ b/config/blazer.yml @@ -50,7 +50,7 @@ user_method: current_usuarie user_name: email # custom before_action to use for auth -# before_action_method: require_admin +before_action_method: require_usuarie # email to send checks from from_email: blazer@<%= ENV.fetch('SUTTY', 'sutty.nl') %> diff --git a/config/routes.rb b/config/routes.rb index 2c5f1c60..186dd66f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -4,8 +4,6 @@ Rails.application.routes.draw do devise_for :usuaries get '/.well-known/change-password', to: redirect('/usuaries/edit') - mount Blazer::Engine, at: 'blazer' - root 'application#index' constraints(Constraints::ApiSubdomain.new) do @@ -38,6 +36,9 @@ Rails.application.routes.draw do match '/api/v3/projects/:site_id/notices' => 'api/v1/notices#create', via: %i[post] resources :sites, constraints: { site_id: %r{[^/]+}, id: %r{[^/]+} } do + # Usar Blazer para mostrar estadísticas + mount Blazer::Engine, at: 'stats', as: 'stats' + # Gestionar actualizaciones del sitio get 'pull', to: 'sites#fetch' post 'pull', to: 'sites#merge' @@ -73,7 +74,5 @@ Rails.application.routes.draw do # Compilar el sitio post 'enqueue', to: 'sites#enqueue' post 'reorder_posts', to: 'sites#reorder_posts' - - resources :stats, only: [:index] end end diff --git a/db/migrate/20200206163257_install_blazer.rb b/db/migrate/20200206163257_install_blazer.rb index 9b084169..32799053 100644 --- a/db/migrate/20200206163257_install_blazer.rb +++ b/db/migrate/20200206163257_install_blazer.rb @@ -3,8 +3,6 @@ # Blazer class InstallBlazer < ActiveRecord::Migration[6.0] def change - return unless Rails.env.production? - create_table :blazer_queries do |t| t.references :creator t.string :name From 7511afbf88575fdc62265d0adfe02b56ac764eb6 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 17:51:35 -0300 Subject: [PATCH 002/251] Les usuaries tienen consultas --- app/models/usuarie.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/usuarie.rb b/app/models/usuarie.rb index 6de7ba4b..c88dcc68 100644 --- a/app/models/usuarie.rb +++ b/app/models/usuarie.rb @@ -11,6 +11,8 @@ class Usuarie < ApplicationRecord has_many :roles has_many :sites, through: :roles + has_many :blazer_audits, foreign_key: 'user_id', class_name: 'Blazer::Audit' + has_many :blazer_queries, foreign_key: 'creator_id', class_name: 'Blazer::Query' def name email.split('@', 2).first From ddc459130a45b9aa15a590bfa276218facb99030 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 17:54:27 -0300 Subject: [PATCH 003/251] Acceder a la lista de consultas disponibles con la identidad de Sutty --- app/controllers/application_controller.rb | 7 +++++++ app/views/blazer/queries/home.haml | 9 +++++++++ app/views/layouts/_breadcrumb.haml | 2 +- app/views/layouts/blazer/application.haml | 14 ++++++++++++++ config/locales/en.yml | 6 ++++++ config/locales/es.yml | 6 ++++++ 6 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 app/views/blazer/queries/home.haml create mode 100644 app/views/layouts/blazer/application.haml diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index f95159f4..2b5088f7 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -69,6 +69,13 @@ class ApplicationController < ActionController::Base redirect_to root_path return end + + # Necesario para los breadcrumbs. + ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions + + breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path + breadcrumb 'sites.index', main_app.sites_path, match: :exact + breadcrumb 'stats.index', root_path, match: :exact end protected diff --git a/app/views/blazer/queries/home.haml b/app/views/blazer/queries/home.haml new file mode 100644 index 00000000..325ba121 --- /dev/null +++ b/app/views/blazer/queries/home.haml @@ -0,0 +1,9 @@ +#queries + %table.table + %tbody.list + - @queries.each do |query| + %tr + -# + Por alguna razón no tenemos acceso a query_path para poder + generar la URL según Rails + %td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query[:to_param]}" diff --git a/app/views/layouts/_breadcrumb.haml b/app/views/layouts/_breadcrumb.haml index c4920bc7..f209787c 100644 --- a/app/views/layouts/_breadcrumb.haml +++ b/app/views/layouts/_breadcrumb.haml @@ -20,5 +20,5 @@ role: 'button', class: 'btn' %li.nav-item - = link_to t('.logout'), destroy_usuarie_session_path, + = link_to t('.logout'), main_app.destroy_usuarie_session_path, method: :delete, role: 'button', class: 'btn' diff --git a/app/views/layouts/blazer/application.haml b/app/views/layouts/blazer/application.haml new file mode 100644 index 00000000..add94190 --- /dev/null +++ b/app/views/layouts/blazer/application.haml @@ -0,0 +1,14 @@ +!!! +%html + %head + %meta{content: 'text/html; charset=UTF-8', 'http-equiv': 'Content-Type'}/ + %title= blazer_title ? blazer_title : 'Sutty' + %meta{charset: 'utf-8'}/ + = favicon_link_tag 'blazer/favicon.png' + = stylesheet_link_tag 'application' + = javascript_pack_tag 'blazer', 'data-turbolinks-track': 'reload' + = csrf_meta_tags + %body{ class: yield(:body) } + .container-fluid#sutty + = render 'layouts/breadcrumb' + = yield diff --git a/config/locales/en.yml b/config/locales/en.yml index fc194eab..7fd83f63 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -576,3 +576,9 @@ en: edit: 'Editing' usuaries: index: 'Users' + stats: + index: 'Statistics' + blazer: + queries: + show: + empty: '(empty)' diff --git a/config/locales/es.yml b/config/locales/es.yml index e8185391..0a48742a 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -584,3 +584,9 @@ es: edit: 'Editando' usuaries: index: 'Usuaries' + stats: + index: 'Estadísticas' + blazer: + queries: + show: + empty: '(vacío)' From 49e5603687af77ac18d20a1f2975baed18e98925 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 17:57:40 -0300 Subject: [PATCH 004/251] Por cuidados, modificar Blazer pero no poder crear consultas --- app/controllers/concerns/blazer_decorator.rb | 44 ++++++++++++++++++++ config/application.rb | 7 ++++ 2 files changed, 51 insertions(+) create mode 100644 app/controllers/concerns/blazer_decorator.rb diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb new file mode 100644 index 00000000..99b162e6 --- /dev/null +++ b/app/controllers/concerns/blazer_decorator.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# Modificaciones para Blazer +module BlazerDecorator + # No poder obtener información de la base de datos. + module DisableDatabaseInfo + extend ActiveSupport::Concern + + included do + def docs; end + + def tables; end + + def schema; end + end + end + + # Deshabilitar edición de consultas y chequeos. + module DisableEdits + extend ActiveSupport::Concern + + included do + def create; end + + def update; end + + def destroy; end + + def run; end + + def refresh; end + + def cancel; end + end + end +end + +classes = [Blazer::QueriesController, Blazer::ChecksController, Blazer::DashboardsController] +modules = [BlazerDecorator::DisableDatabaseInfo, BlazerDecorator::DisableEdits] +classes.each do |klass| + modules.each do |modul| + klass.include modul unless klass.included_modules.include? modul + end +end diff --git a/config/application.rb b/config/application.rb index 7326ae0f..5b6e373c 100644 --- a/config/application.rb +++ b/config/application.rb @@ -38,6 +38,13 @@ module Sutty config.active_storage.variant_processor = :vips + config.to_prepare do + # Load application's model / class decorators + Dir.glob(File.join(File.dirname(__FILE__), '../app/**/*_decorator.rb')) do |c| + Rails.configuration.cache_classes ? require(c) : load(c) + end + end + config.after_initialize do ActiveStorage::DirectUploadsController.include ActiveStorage::AuthenticatedDirectUploadsController From 85ad518d8d4e92fe0254b43e8775b7c0f2b76830 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:01:34 -0300 Subject: [PATCH 005/251] =?UTF-8?q?Correr=20Blazer=20sincr=C3=B3nicamente?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deshabilitamos la funcionalidad asincrónica de Blazer porque ejecuta unos malabares extraños con JS, que filtran la consulta hacia el código. La idea es ejecutar consultas más livianas, con lo que por ahora no lo necesitamos. Podríamos recuperar esto usando ActionCable luego. Además, reimplementa la lógica de generación de gráficos en el controlador, para simplificar la vista. --- app/controllers/concerns/blazer_decorator.rb | 145 +++++++++++++++++++ app/views/blazer/queries/show.haml | 51 +++++++ package.json | 2 + yarn.lock | 35 ++++- 4 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 app/views/blazer/queries/show.haml diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb index 99b162e6..f2598573 100644 --- a/app/controllers/concerns/blazer_decorator.rb +++ b/app/controllers/concerns/blazer_decorator.rb @@ -33,6 +33,149 @@ module BlazerDecorator def cancel; end end end + + # Blazer hace un gran esfuerzo para ejecutar consultas de forma + # asincrónica pero termina enviándolas por JS. + module RunSync + extend ActiveSupport::Concern + + included do + alias_method :original_show, :show + + include Blazer::BaseHelper + + def show + original_show + + options = { user: blazer_user, query: @query, run_id: SecureRandom.uuid, async: false } + @data_source = Blazer.data_sources[@query.data_source] + @result = Blazer::RunStatement.new.perform(@data_source, @statement, options) + chart_data + end + + private + + # blazer-2.4.2/app/views/blazer/queries/run.html.erb + def chart_type + case @result.chart_type + when /\Aline(2)?\z/ + chart_options.merge! min: nil + when /\Abar(2)?\z/ + chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } } + when 'pie' + chart_options + when 'scatter' + chart_options.merge! library: { tooltips: { intersect: false } }, xtitle: @result.columns[0], + ytitle: @result.columns[1] + when nil + else + if @result.column_types.size == 2 + chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } } + else + chart_options.merge! library: { tooltips: { intersect: false } } + end + end + + @result.chart_type + end + + def chart_data + @chart_data ||= + case chart_type + when 'line' + @result.columns[1..-1].each_with_index.map do |k, i| + { + name: blazer_series_name(k), + data: @result.rows.map do |r| + [r[0], r[i + 1]] + end, + library: series_library[i] + } + end + when 'line2' + @result.rows.group_by do |r| + v = r[1] + (@result.boom[@result.columns[1]] || {})[v.to_s] || v + end.each_with_index.map do |(name, v), i| + { + name: blazer_series_name(name), + data: v.map do |v2| + [v2[0], v2[2]] + end, + library: series_library[i] + } + end + when 'pie' + @result.rows.map do |r| + [(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[1]] + end + when 'bar' + (@result.rows.first.size - 1).times.map do |i| + name = @result.columns[i + 1] + + { + name: blazer_series_name(name), + data: @result.rows.first(20).map do |r| + [(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] + end + } + end + when 'bar2' + first_20 = @result.rows.group_by { |r| r[0] }.values.first(20).flatten(1) + labels = first_20.map { |r| r[0] }.uniq + series = first_20.map { |r| r[1] }.uniq + labels.each do |l| + series.each do |s| + first_20 << [l, s, 0] unless first_20.find { |r| r[0] == l && r[1] == s } + end + end + + first_20.group_by do |r| + v = r[1] + (@result.boom[@result.columns[1]] || {})[v.to_s] || v + end.each_with_index.map do |(name, v), _i| + { + name: blazer_series_name(name), + data: v.sort_by do |r2| + labels.index(r2[0]) + end.map do |v2| + v3 = v2[0] + [(@result.boom[@result.columns[0]] || {})[v3.to_s] || v3, v2[2]] + end + } + end + when 'scatter' + @result.rows + end + end + + def target_index + @target_index ||= @result.columns.index do |k| + k.downcase == 'target' + end + end + + def series_library + @series_library ||= {}.tap do |sl| + if target_index + color = '#109618' + sl[target_index - 1] = { + pointStyle: 'line', + hitRadius: 5, + borderColor: color, + pointBackgroundColor: color, + backgroundColor: color, + pointHoverBackgroundColor: color + } + end + end + end + + def chart_options + @chart_options ||= { id: SecureRandom.hex } + end + end + end end classes = [Blazer::QueriesController, Blazer::ChecksController, Blazer::DashboardsController] @@ -42,3 +185,5 @@ classes.each do |klass| klass.include modul unless klass.included_modules.include? modul end end + +Blazer::QueriesController.include BlazerDecorator::RunSync diff --git a/app/views/blazer/queries/show.haml b/app/views/blazer/queries/show.haml new file mode 100644 index 00000000..5ec3da98 --- /dev/null +++ b/app/views/blazer/queries/show.haml @@ -0,0 +1,51 @@ +- blazer_title @query.name +.container + .row + .col-12 + %h1= @query.name + - if @query.description.present? + %p.lead= @query.description + - unless @result.chart_type.blank? + .col-12 + - case @result.chart_type + - when 'line' + = line_chart @chart_data, **@chart_options + - when 'line2' + = line_chart @chart_data, **@chart_options + - when 'pie' + = pie_chart @chart_data, **@chart_options + - when 'bar' + = column_chart @chart_data, **@chart_options + - when 'bar2' + = column_chart @chart_data, **@chart_options + - when 'scatter' + = scatter_chart @chart_data, **@chart_options + .col-12 + %table.table + %thead + %tr + - @result.columns.each do |key| + - next if key.include? 'ciphertext' + - next if key.include? 'encrypted' + %th.position-sticky.background-white= key + %tbody + - @result.rows.each do |row| + %tr + - row.each_with_index do |v, i| + - k = @result.columns[i] + - next if k.include? 'ciphertext' + - next if k.include? 'encrypted' + %td + - if v.is_a?(Time) + - v = blazer_time_value(@data_source, k, v) + + - unless v.nil? + - if v.is_a?(String) && v.empty? + %span.text-muted= t('.empty') + - elsif @data_source.linked_columns[k] + = link_to blazer_format_value(k, v), @data_source.linked_columns[k].gsub('{value}', u(v.to_s)), target: '_blank' + - else + = blazer_format_value(k, v) + + - if (v2 = (@result.boom[k] || {})[v.nil? ? v : v.to_s]) + %span.text-muted= v2 diff --git a/package.json b/package.json index 0a2458a6..6340651f 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "@rails/ujs": "^6.1.3-1", "@rails/webpacker": "5.2.1", "babel-loader": "^8.2.2", + "chart.js": "2.9.3", + "chartkick": "3.2.1", "circular-dependency-plugin": "^5.2.2", "commonmark": "^0.29.0", "fork-awesome": "^1.1.7", diff --git a/yarn.lock b/yarn.lock index 11ff78cb..b478ff67 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2119,6 +2119,34 @@ chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" +chart.js@2.9.3: + version "2.9.3" + resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-2.9.3.tgz#ae3884114dafd381bc600f5b35a189138aac1ef7" + integrity sha512-+2jlOobSk52c1VU6fzkh3UwqHMdSlgH1xFv9FKMqHiNCpXsGPQa/+81AFa+i3jZ253Mq9aAycPwDjnn1XbRNNw== + dependencies: + chartjs-color "^2.1.0" + moment "^2.10.2" + +chartjs-color-string@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz#1df096621c0e70720a64f4135ea171d051402f71" + integrity sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A== + dependencies: + color-name "^1.0.0" + +chartjs-color@^2.1.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/chartjs-color/-/chartjs-color-2.4.1.tgz#6118bba202fe1ea79dd7f7c0f9da93467296c3b0" + integrity sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w== + dependencies: + chartjs-color-string "^0.6.0" + color-convert "^1.9.3" + +chartkick@3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/chartkick/-/chartkick-3.2.1.tgz#a80c2005ae353c5ae011d0a756b6f592fc8fc7a9" + integrity sha512-zV0kUeZNqrX28AmPt10QEDXHKadbVFOTAFkCMyJifHzGFkKzGCDXxVR8orZ0fC1HbePzRn5w6kLCOVxDQbMUCg== + chokidar@^2.1.8: version "2.1.8" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917" @@ -2238,7 +2266,7 @@ collection-visit@^1.0.0: map-visit "^1.0.0" object-visit "^1.0.0" -color-convert@^1.9.0, color-convert@^1.9.1: +color-convert@^1.9.0, color-convert@^1.9.1, color-convert@^1.9.3: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== @@ -5005,6 +5033,11 @@ mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +moment@^2.10.2: + version "2.29.1" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" + integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== + move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92" From 75e6b7a8014ea83362e126dc85dcd681c053514d Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:25:42 -0300 Subject: [PATCH 006/251] =?UTF-8?q?Agregar=20los=20helpers=20de=20Devise?= =?UTF-8?q?=20tambi=C3=A9n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Para que tengamos `current_usuarie` dentro de las vistas de Blazer. --- app/controllers/application_controller.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 2b5088f7..efe1a4a3 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -71,7 +71,9 @@ class ApplicationController < ActionController::Base end # Necesario para los breadcrumbs. - ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions + [Loaf::ViewExtensions, Devise::Controllers::UrlHelpers].each do |helper_module| + ActionView::Base.include helper_module unless ActionView::Base.included_modules.include? helper_module + end breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path breadcrumb 'sites.index', main_app.sites_path, match: :exact From 841279f6cf81d545f064e94dc3aab39e7d2cc454 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:31:17 -0300 Subject: [PATCH 007/251] =?UTF-8?q?fixup!=20Agregar=20los=20helpers=20de?= =?UTF-8?q?=20Devise=20tambi=C3=A9n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/application_controller.rb | 4 +--- app/views/layouts/_breadcrumb.haml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index efe1a4a3..2b5088f7 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -71,9 +71,7 @@ class ApplicationController < ActionController::Base end # Necesario para los breadcrumbs. - [Loaf::ViewExtensions, Devise::Controllers::UrlHelpers].each do |helper_module| - ActionView::Base.include helper_module unless ActionView::Base.included_modules.include? helper_module - end + ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path breadcrumb 'sites.index', main_app.sites_path, match: :exact diff --git a/app/views/layouts/_breadcrumb.haml b/app/views/layouts/_breadcrumb.haml index f209787c..dc0e3158 100644 --- a/app/views/layouts/_breadcrumb.haml +++ b/app/views/layouts/_breadcrumb.haml @@ -12,7 +12,7 @@ - else %span.line-clamp-1= link_to crumb.name, crumb.url - - if current_usuarie + - if @current_usuarie || current_usuarie %ul.navbar-nav - if @site&.tienda? %li.nav-item From af67c39dc477581fb6a0acd2adce5aacb683d0aa Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:37:15 -0300 Subject: [PATCH 008/251] Agregar el nombre del sitio en la miga de pan --- app/controllers/application_controller.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 2b5088f7..c9e5a999 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -65,7 +65,8 @@ class ApplicationController < ActionController::Base # Necesario para poder acceder a Blazer. Solo les usuaries de este # sitio pueden acceder al panel. def require_usuarie - unless find_site.usuarie? current_usuarie + site = find_site + unless site.usuarie? current_usuarie redirect_to root_path return end @@ -75,6 +76,7 @@ class ApplicationController < ActionController::Base breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path breadcrumb 'sites.index', main_app.sites_path, match: :exact + breadcrumb site.title, main_app.site_path(site), match: :exact breadcrumb 'stats.index', root_path, match: :exact end From 44450da5208b55582c2ff287c538d71a452da274 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:41:09 -0300 Subject: [PATCH 009/251] Les usuaries solo pueden ver sus propias consultas. --- app/controllers/concerns/blazer_decorator.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb index f2598573..eee0db64 100644 --- a/app/controllers/concerns/blazer_decorator.rb +++ b/app/controllers/concerns/blazer_decorator.rb @@ -55,6 +55,11 @@ module BlazerDecorator private + # Solo mostrar las consultas de le usuarie + def set_queries(_) + @queries = (@current_usuarie || current_usuarie).blazer_queries + end + # blazer-2.4.2/app/views/blazer/queries/run.html.erb def chart_type case @result.chart_type From 6f08ca6c36b1297d93bd7ba435e5e7d70b930a60 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 3 Aug 2021 10:10:30 -0300 Subject: [PATCH 010/251] =?UTF-8?q?A=20veces=20se=20pasa=20el=20par=C3=A1m?= =?UTF-8?q?etro=20en=20set=5Fqueries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/concerns/blazer_decorator.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb index eee0db64..876f423d 100644 --- a/app/controllers/concerns/blazer_decorator.rb +++ b/app/controllers/concerns/blazer_decorator.rb @@ -56,7 +56,7 @@ module BlazerDecorator private # Solo mostrar las consultas de le usuarie - def set_queries(_) + def set_queries(_ = nil) @queries = (@current_usuarie || current_usuarie).blazer_queries end From 489cbb414ccd4a7fbd5c81f7e6051838a4512ada Mon Sep 17 00:00:00 2001 From: f Date: Tue, 3 Aug 2021 10:15:48 -0300 Subject: [PATCH 011/251] Ya no usamos un hash de queries --- app/views/blazer/queries/home.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/blazer/queries/home.haml b/app/views/blazer/queries/home.haml index 325ba121..977b6bda 100644 --- a/app/views/blazer/queries/home.haml +++ b/app/views/blazer/queries/home.haml @@ -6,4 +6,4 @@ -# Por alguna razón no tenemos acceso a query_path para poder generar la URL según Rails - %td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query[:to_param]}" + %td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query.to_param}" From e87fad33eae8b1225d090a9e6f5464933c69dea6 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 3 Aug 2021 10:16:26 -0300 Subject: [PATCH 012/251] =?UTF-8?q?La=20tabla=20se=20puede=20scrollear=20y?= =?UTF-8?q?=20mantiene=20los=20t=C3=ADtulos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/blazer/queries/show.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/blazer/queries/show.haml b/app/views/blazer/queries/show.haml index 5ec3da98..3b2beed6 100644 --- a/app/views/blazer/queries/show.haml +++ b/app/views/blazer/queries/show.haml @@ -27,7 +27,7 @@ - @result.columns.each do |key| - next if key.include? 'ciphertext' - next if key.include? 'encrypted' - %th.position-sticky.background-white= key + %th.position-sticky.background-white{ style: 'top: 0' }= key %tbody - @result.rows.each do |row| %tr From 71ff9e5e7b22013b5c4f033518871ea9670d52c5 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 3 Aug 2021 10:24:18 -0300 Subject: [PATCH 013/251] =?UTF-8?q?Traducir=20columnas=20o=20mostrarlas=20?= =?UTF-8?q?como=20t=C3=ADtulos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Los nombres más comunas de columnas se pueden agregar al archivo de traducción, si la traducción no existe se convierte a un título. ```ruby "total_horas".titleize => "Total Horas" ``` --- app/views/blazer/queries/show.haml | 2 +- config/locales/en.yml | 5 +++++ config/locales/es.yml | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/app/views/blazer/queries/show.haml b/app/views/blazer/queries/show.haml index 3b2beed6..3b5cb152 100644 --- a/app/views/blazer/queries/show.haml +++ b/app/views/blazer/queries/show.haml @@ -27,7 +27,7 @@ - @result.columns.each do |key| - next if key.include? 'ciphertext' - next if key.include? 'encrypted' - %th.position-sticky.background-white{ style: 'top: 0' }= key + %th.position-sticky.background-white{ style: 'top: 0' }= t("blazer.columns.#{key}", default: key.titleize) %tbody - @result.rows.each do |row| %tr diff --git a/config/locales/en.yml b/config/locales/en.yml index 7fd83f63..18faa8bb 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -579,6 +579,11 @@ en: stats: index: 'Statistics' blazer: + columns: + total: 'Total' + dia: 'Date' + date: 'Date' + visitas: 'Visits' queries: show: empty: '(empty)' diff --git a/config/locales/es.yml b/config/locales/es.yml index 0a48742a..5229a591 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -587,6 +587,11 @@ es: stats: index: 'Estadísticas' blazer: + columns: + total: 'Total' + dia: 'Fecha' + date: 'Fecha' + visitas: 'Visitas' queries: show: empty: '(vacío)' From 71436d3be49cb3d0cdde8e37338369f10fdaf81f Mon Sep 17 00:00:00 2001 From: f Date: Wed, 4 Aug 2021 12:17:49 -0300 Subject: [PATCH 014/251] =?UTF-8?q?Usar=20el=20sistema=20de=20autorizaci?= =?UTF-8?q?=C3=B3n=20de=20Sutty?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Y eliminar código sin utilizar. --- app/controllers/application_controller.rb | 9 ++++----- app/controllers/posts_controller.rb | 3 --- app/controllers/private_controller.rb | 2 -- app/controllers/sites_controller.rb | 3 --- app/controllers/stats_controller.rb | 18 ------------------ app/models/{site_stat.rb => site_blazer.rb} | 2 +- app/policies/site_blazer_policy.rb | 10 ++++++++++ app/policies/site_stat_policy.rb | 15 --------------- 8 files changed, 15 insertions(+), 47 deletions(-) delete mode 100644 app/controllers/stats_controller.rb rename app/models/{site_stat.rb => site_blazer.rb} (50%) create mode 100644 app/policies/site_blazer_policy.rb delete mode 100644 app/policies/site_stat_policy.rb diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index c9e5a999..d8498218 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -3,6 +3,7 @@ # Forma de ingreso a Sutty class ApplicationController < ActionController::Base include ExceptionHandler + include Pundit protect_from_forgery with: :null_session, prepend: true @@ -10,6 +11,7 @@ class ApplicationController < ActionController::Base before_action :configure_permitted_parameters, if: :devise_controller? around_action :set_locale + rescue_from Pundit::NilPolicyError, with: :page_not_found rescue_from ActionController::RoutingError, with: :page_not_found rescue_from ActionController::ParameterMissing, with: :page_not_found @@ -33,7 +35,7 @@ class ApplicationController < ActionController::Base def find_site id = params[:site_id] || params[:id] - unless (site = current_usuarie.sites.find_by_name(id)) + unless (site = current_usuarie&.sites&.find_by_name(id)) raise SiteNotFound end @@ -66,10 +68,7 @@ class ApplicationController < ActionController::Base # sitio pueden acceder al panel. def require_usuarie site = find_site - unless site.usuarie? current_usuarie - redirect_to root_path - return - end + authorize SiteBlazer.new(site) # Necesario para los breadcrumbs. ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index 3ef26720..448592de 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -2,9 +2,6 @@ # Controlador para artículos class PostsController < ApplicationController - include Pundit - rescue_from Pundit::NilPolicyError, with: :page_not_found - before_action :authenticate_usuarie! # TODO: Traer los comunes desde ApplicationController diff --git a/app/controllers/private_controller.rb b/app/controllers/private_controller.rb index bb4d782d..01b6888c 100644 --- a/app/controllers/private_controller.rb +++ b/app/controllers/private_controller.rb @@ -6,8 +6,6 @@ class PrivateController < ApplicationController # XXX: Permite ejecutar JS skip_forgery_protection - include Pundit - # Enviar el archivo si existe, agregar una / al final siempre para no # romper las direcciones relativas. def show diff --git a/app/controllers/sites_controller.rb b/app/controllers/sites_controller.rb index bdaa9011..b4826226 100644 --- a/app/controllers/sites_controller.rb +++ b/app/controllers/sites_controller.rb @@ -2,9 +2,6 @@ # Controlador de sitios class SitesController < ApplicationController - include Pundit - rescue_from Pundit::NilPolicyError, with: :page_not_found - before_action :authenticate_usuarie! breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb deleted file mode 100644 index 07baaf1a..00000000 --- a/app/controllers/stats_controller.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -# Estadísticas del sitio -class StatsController < ApplicationController - include Pundit - before_action :authenticate_usuarie! - - def index - @site = find_site - authorize SiteStat.new(@site) - - # Solo queremos el promedio de tiempo de compilación, no de - # instalación de dependencias. - stats = @site.build_stats.jekyll - @build_avg = stats.average(:seconds).to_f.round(2) - @build_max = stats.maximum(:seconds).to_f.round(2) - end -end diff --git a/app/models/site_stat.rb b/app/models/site_blazer.rb similarity index 50% rename from app/models/site_stat.rb rename to app/models/site_blazer.rb index 73503aca..76dee12a 100644 --- a/app/models/site_stat.rb +++ b/app/models/site_blazer.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -SiteStat = Struct.new(:site) +SiteBlazer = Struct.new(:site) diff --git a/app/policies/site_blazer_policy.rb b/app/policies/site_blazer_policy.rb new file mode 100644 index 00000000..a6ea01b7 --- /dev/null +++ b/app/policies/site_blazer_policy.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# Les invitades no pueden ver las estadísticas (aun) +SiteBlazerPolicy = Struct.new(:usuarie, :site_blazer) do + def home? + site_blazer&.site&.usuarie? usuarie + end + + alias_method :show?, :home? +end diff --git a/app/policies/site_stat_policy.rb b/app/policies/site_stat_policy.rb deleted file mode 100644 index a797034c..00000000 --- a/app/policies/site_stat_policy.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -# Política de acceso a las estadísticas -class SiteStatPolicy - attr_reader :site_stat, :usuarie - - def initialize(usuarie, site_stat) - @usuarie = usuarie - @site_stat = site_stat - end - - def index? - site_stat.site.usuarie? usuarie - end -end From 249b115af848f02fb11dd1bbebd55c3a31f5fa28 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 4 Aug 2021 20:16:56 -0300 Subject: [PATCH 015/251] No enviar URLs en los chequeos. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Necesita más trabajo hacer esto y no tenemos un uso inmediato para chequeos específicos de sitios. fixes #2331 --- .../blazer/check_mailer/failing_checks.haml | 5 ++++ .../blazer/check_mailer/state_change.haml | 30 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 app/views/blazer/check_mailer/failing_checks.haml create mode 100644 app/views/blazer/check_mailer/state_change.haml diff --git a/app/views/blazer/check_mailer/failing_checks.haml b/app/views/blazer/check_mailer/failing_checks.haml new file mode 100644 index 00000000..8624ae9b --- /dev/null +++ b/app/views/blazer/check_mailer/failing_checks.haml @@ -0,0 +1,5 @@ +%ul + - @checks.each do |check| + %li + = link_to check.query.name + = check.state diff --git a/app/views/blazer/check_mailer/state_change.haml b/app/views/blazer/check_mailer/state_change.haml new file mode 100644 index 00000000..48418a58 --- /dev/null +++ b/app/views/blazer/check_mailer/state_change.haml @@ -0,0 +1,30 @@ +!!! +%html + %head + %meta{:content => "text/html; charset=UTF-8", "http-equiv" => "Content-Type"}/ + %body{:style => "font-family: 'Helvetica Neue', Arial, Helvetica; font-size: 14px; color: #333;"} + - if @error + %p= @error + - elsif @rows_count > 0 && @check_type == "bad_data" + %p + - if @rows_count <= 10 + = pluralize(@rows_count, "row") + - else + Showing 10 of #{@rows_count} rows + %table{:style => "width: 100%; border-spacing: 0; border-collapse: collapse;"} + %thead + %tr + - @columns.first(5).each do |column| + %th{:style => "padding: 8px; line-height: 1.4; text-align: left; vertical-align: bottom; border-bottom: 2px solid #ddd; width: #{(100 / @columns.size).round(2)}%;"} + = column + %tbody + - @rows.first(10).each do |row| + %tr + - @columns.first(5).each_with_index do |column, i| + %td{:style => "padding: 8px; line-height: 1.4; vertical-align: top; border-top: 1px solid #ddd;"} + - value = row[i] + - if @column_types[i] == "time" && value.to_s.length > 10 + - value = Time.parse(value).in_time_zone(Blazer.time_zone) rescue value + = value + - if @columns.size > 5 + %p{:style => "color: #999;"} Only first 5 columns shown From 5885dd7e9615288bbe36cf0b413c3aeda042853e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 6 Aug 2021 17:47:27 -0300 Subject: [PATCH 016/251] fixup! No enviar URLs en los chequeos. --- app/views/blazer/check_mailer/failing_checks.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/blazer/check_mailer/failing_checks.haml b/app/views/blazer/check_mailer/failing_checks.haml index 8624ae9b..c28c3936 100644 --- a/app/views/blazer/check_mailer/failing_checks.haml +++ b/app/views/blazer/check_mailer/failing_checks.haml @@ -1,5 +1,5 @@ %ul - @checks.each do |check| %li - = link_to check.query.name + = check.query.name = check.state From aa3cbc209ee30ad3efc3db9e477d57c19d60e2a0 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 10 Feb 2022 15:18:49 -0300 Subject: [PATCH 017/251] =?UTF-8?q?vincular=20a=20un=20idioma=20seg=C3=BAn?= =?UTF-8?q?=20el=20dominio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #4545 --- app/models/deploy_localized_domain.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 app/models/deploy_localized_domain.rb diff --git a/app/models/deploy_localized_domain.rb b/app/models/deploy_localized_domain.rb new file mode 100644 index 00000000..6e89c794 --- /dev/null +++ b/app/models/deploy_localized_domain.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Soportar dominios localizados +class DeployLocalizedDomain < DeployAlternativeDomain + store :values, accessors: %i[hostname locale], coder: JSON + + # Generar un link simbólico del sitio principal al alternativo + def deploy + File.symlink?(destination) || + File.symlink(File.join(site.hostname, locale), destination).zero? + end +end From 958306ce06213b852e7b6bf2e917eeb7b63d588c Mon Sep 17 00:00:00 2001 From: f Date: Thu, 10 Feb 2022 15:26:20 -0300 Subject: [PATCH 018/251] expedir certificados para los dominios localizados --- app/controllers/api/v1/sites_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/v1/sites_controller.rb b/app/controllers/api/v1/sites_controller.rb index 6abff704..0de21669 100644 --- a/app/controllers/api/v1/sites_controller.rb +++ b/app/controllers/api/v1/sites_controller.rb @@ -46,7 +46,7 @@ module Api # Dominios alternativos def alternative_names - DeployAlternativeDomain.all.map(&:hostname) + DeployAlternativeDomain.all.map(&:hostname) + DeployLocalizedDomain.all.map(&:hostname) end # Obtener todos los sitios con API habilitada, es decir formulario From 460635e940fca637bf41499cb8d4070beb9a3db7 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:03:02 -0300 Subject: [PATCH 019/251] ya no hace falta analizar archivos en primer plano --- config/initializers/analyze_job.rb | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 config/initializers/analyze_job.rb diff --git a/config/initializers/analyze_job.rb b/config/initializers/analyze_job.rb deleted file mode 100644 index f268e0dd..00000000 --- a/config/initializers/analyze_job.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -# TODO: Estamos procesando el análisis de los archivos en el momento -# porque queremos obtener la ruta del archivo en el momento y no -# después. Necesitaríamos poder generar el vínculo en el -# repositorio a destiempo, modificando el Job de ActiveStorage -ActiveStorage::AnalyzeJob.queue_adapter = :inline From 29950e4380b4c6d5122a0bfd3f8024178b44a40d Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:05:11 -0300 Subject: [PATCH 020/251] cargar decorators --- config/application.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/application.rb b/config/application.rb index 7326ae0f..0e6e7f44 100644 --- a/config/application.rb +++ b/config/application.rb @@ -38,6 +38,12 @@ module Sutty config.active_storage.variant_processor = :vips + config.to_prepare do + Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')) do |c| + Rails.configuration.cache_classes ? require(c) : load(c) + end + end + config.after_initialize do ActiveStorage::DirectUploadsController.include ActiveStorage::AuthenticatedDirectUploadsController From 3de1228c26541185a62fa7a528dfc170308018bf Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:08:09 -0300 Subject: [PATCH 021/251] implementa un servicio de carga de archivos en jekyll --- .../active_storage/service/jekyll_service.rb | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 app/lib/active_storage/service/jekyll_service.rb diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb new file mode 100644 index 00000000..686c3979 --- /dev/null +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module ActiveStorage + module Service + # Sube los archivos a cada repositorio y los agrega al LFS de su + # repositorio git. + # + # @todo: Implementar LFS. No nos gusta mucho la idea porque duplica + # el espacio en disco, pero es la única forma que tenemos (hasta que + # implementemos IPFS) para poder transferir los archivos junto con el + # sitio. + class JekyllService < Service::DiskService + BLOB_NAME = 'blob' + + # Para poder guardar el archivo con el nombre original pero poder + # recuperarlo durante el download, luego de subirlo le cambiamos el + # nombre y creamos un link simbólico a un nombre conocido. + def upload(key, io, checksum: nil, **options) + super.tap do + path = path_for(key) + filename = options[:filename].to_s + + FileUtils.mv path, path.sub(/#{BLOB_NAME}\z/, filename) + FileUtils.ln_s filename, path + end + end + + # Mantener retrocompatibilidad con cómo gestionamos los archivos + # subidos hasta ahora. + # + # @param :key [String] + # @return [String] + def folder_for(key) + key + end + + # Crea una ruta para la llave con un nombre conocido. + def path_for(key) + File.join root, folder_for(key), BLOB_NAME + end + end + end +end From 873f2c7bcb8db83a1bdc46580158ff28ce3e67e9 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:09:06 -0300 Subject: [PATCH 022/251] informar cuando no se pudo cargar el archivo --- app/models/metadata_file.rb | 1 + config/locales/en.yml | 2 ++ config/locales/es.yml | 2 ++ 3 files changed, 5 insertions(+) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 80cefa27..b020d078 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -19,6 +19,7 @@ class MetadataFile < MetadataTemplate errors << I18n.t("metadata.#{type}.site_invalid") if site.invalid? errors << I18n.t("metadata.#{type}.path_required") if path_missing? errors << I18n.t("metadata.#{type}.no_file_for_description") if no_file_for_description? + errors << I18n.t("metadata.#{type}.attachment_missing") unless static_file errors.compact! errors.empty? diff --git a/config/locales/en.yml b/config/locales/en.yml index b814796d..8bbe3621 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -41,10 +41,12 @@ en: not_an_image: 'Not an image' path_required: 'Missing image for upload' no_file_for_description: "Description with no associated image" + attachment_missing: "I couldn't save the image :(" file: site_invalid: 'The file cannot be stored if the site configuration is not valid' path_required: "Missing file for upload" no_file_for_description: "Description with no associated file" + attachment_missing: "I couldn't save the file :(" event: zone_missing: 'Inexistent timezone' date_missing: 'Event date is required' diff --git a/config/locales/es.yml b/config/locales/es.yml index a6fbd407..93dd162e 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -41,10 +41,12 @@ es: not_an_image: 'No es una imagen' path_required: 'Se necesita una imagen' no_file_for_description: 'Se envió una descripción sin imagen asociada' + attachment_missing: 'no pude guardar el archivo :(' file: site_invalid: 'El archivo no se puede almacenar si la configuración del sitio no es válida' path_required: 'Se necesita un archivo' no_file_for_description: 'se envió una descripción sin archivo asociado' + attachment_missing: 'no pude guardar el archivo :(' event: zone_missing: 'El huso horario no es correcto' date_missing: 'La fecha es obligatoria' From c4139c4b92746c84cdbdaaffe1fea31a69afc9a8 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:12:06 -0300 Subject: [PATCH 023/251] =?UTF-8?q?eliminar=20c=C3=B3digo=20que=20no=20se?= =?UTF-8?q?=20usa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/metadata_file.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index b020d078..58adc857 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -35,12 +35,6 @@ class MetadataFile < MetadataTemplate value['path'].is_a?(String) end - # Determina si la ruta es opcional pero deja pasar si la ruta se - # especifica - def path_optional? - !required && !path? - end - # Asociar la imagen subida al sitio y obtener la ruta # # XXX: Si evitamos guardar cambios con changed? no tenemos forma de From 3d5267451eec0e879fad28ef4086cd240884d683 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:15:38 -0300 Subject: [PATCH 024/251] =?UTF-8?q?eliminar=20c=C3=B3digo=20que=20ya=20no?= =?UTF-8?q?=20se=20usa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/metadata_file.rb | 65 ------------------------------------- 1 file changed, 65 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 58adc857..86644fcc 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -93,71 +93,6 @@ class MetadataFile < MetadataTemplate private - def filemagic - @filemagic ||= FileMagic.new(FileMagic::MAGIC_MIME) - end - - # @return [Pathname] - def path - @path ||= Pathname.new(File.join(site.path, value['path'])) - end - - def file - return unless path? - - @file ||= - case value['path'] - when ActionDispatch::Http::UploadedFile then value['path'].tempfile.path - when String then File.join(site.path, value['path']) - end - end - - # Hacemos un link duro para colocar el archivo dentro del repositorio - # y no duplicar el espacio que ocupan. Esto requiere que ambos - # directorios estén dentro del mismo punto de montaje. - # - # XXX: Asumimos que el archivo destino no existe porque siempre - # contiene una key única. - # - # @return [Boolean] - def hardlink - return if hardlink? - return if File.exist? destination_path - - FileUtils.mkdir_p(File.dirname(destination_path)) - FileUtils.ln(uploaded_path, destination_path).zero? - end - - def hardlink? - File.stat(uploaded_path).ino == File.stat(destination_path).ino - rescue Errno::ENOENT - false - end - - # Obtener la ruta al archivo - # https://stackoverflow.com/a/53908358 - def uploaded_relative_path - ActiveStorage::Blob.service.path_for(static_file.key) - end - - # @return [String] - def uploaded_path - Rails.root.join uploaded_relative_path - end - - # La ruta del archivo mantiene el nombre original pero contiene el - # nombre interno y único del archivo para poder relacionarlo con el - # archivo subido en Sutty. - # - # @return [String] - def relative_destination_path - @relative_destination_path ||= File.join('public', static_file.key, static_file.filename.to_s) - end - - # @return [String] - def destination_path - @destination_path ||= File.join(site.path, relative_destination_path) - end # No hay archivo pero se lo describió def no_file_for_description? From d8fd25a7cb32ab6cdec028f4d054cf398df4b97c Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:16:36 -0300 Subject: [PATCH 025/251] =?UTF-8?q?chequear=20la=20descripci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/metadata_file.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 86644fcc..20d093b2 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -91,11 +91,14 @@ class MetadataFile < MetadataTemplate value['path'].present? end - private + def description? + value['description'].present? + end + private # No hay archivo pero se lo describió def no_file_for_description? - value['description'].present? && value['path'].blank? + !path? && description? end end From 63f8b869eb185dd9c607ed4b684baf9ab478a08d Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:17:03 -0300 Subject: [PATCH 026/251] fixup! cargar decorators --- config/application.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/application.rb b/config/application.rb index 0e6e7f44..031c1909 100644 --- a/config/application.rb +++ b/config/application.rb @@ -39,7 +39,7 @@ module Sutty config.active_storage.variant_processor = :vips config.to_prepare do - Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')) do |c| + Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')).sort.each do |c| Rails.configuration.cache_classes ? require(c) : load(c) end end From 37deb361785d9f79c2d721898c3545a5c7323985 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:19:40 -0300 Subject: [PATCH 027/251] conseguir la ruta al archivo --- app/models/metadata_file.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 20d093b2..89e6d461 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -83,6 +83,15 @@ class MetadataFile < MetadataTemplate end end + # Obtiene la ruta absoluta al archivo + # + # @return [Pathname] + def pathname + raise NoMethodError unless uploaded? + + @pathname ||= Pathname.new(File.join(site.path, value['path'])) + end + def key_from_path path.dirname.basename.to_s end From 03d9913f58f4be0bd0e2c3417577bab6f8a67c4a Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:21:03 -0300 Subject: [PATCH 028/251] obtener el archivo subido --- app/models/metadata_file.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 89e6d461..e4c0d037 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -66,18 +66,19 @@ class MetadataFile < MetadataTemplate # XXX: La última opción provoca archivos duplicados, pero es lo mejor # que tenemos hasta que resolvamos https://0xacab.org/sutty/sutty/-/issues/213 # - # @return [ActiveStorage::Attachment] + # @todo encontrar una forma de obtener el attachment sin tener que + # recurrir al último subido. + # + # @return [ActiveStorage::Attachment,nil] def static_file - return unless path? - @static_file ||= case value['path'] when ActionDispatch::Http::UploadedFile site.static_files.last if site.static_files.attach(value['path']) when String - if (blob = ActiveStorage::Blob.where(key: key_from_path).pluck(:id).first) - site.static_files.find_by(blob_id: blob) - elsif site.static_files.attach(io: path.open, filename: path.basename) + if (blob_id = ActiveStorage::Blob.where(key: key_from_path).pluck(:id).first) + site.static_files.find_by(blob_id: blob_id) + elsif path? && pathname.exist? && site.static_files.attach(io: pathname.open, filename: pathname.basename) site.static_files.last end end @@ -92,8 +93,11 @@ class MetadataFile < MetadataTemplate @pathname ||= Pathname.new(File.join(site.path, value['path'])) end + # Obtiene la key del attachment a partir de la ruta + # + # @return [String] def key_from_path - path.dirname.basename.to_s + pathname.dirname.basename.to_s end def path? From fdbe724f7c92953e9ba2cfa74f8470a6fa617c2e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:21:26 -0300 Subject: [PATCH 029/251] asociar el archivo subido al post --- app/models/metadata_file.rb | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index e4c0d037..1c859481 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -43,13 +43,7 @@ class MetadataFile < MetadataTemplate # repetida. def save value['description'] = sanitize value['description'] - - if path? - hardlink - value['path'] = relative_destination_path - else - value['path'] = nil - end + value['path'] = static_file ? relative_destination_path_with_filename.to_s : nil true end @@ -110,6 +104,28 @@ class MetadataFile < MetadataTemplate private + # Obtener la ruta al archivo relativa al sitio + # + # @return [Pathname] + def destination_path + Pathname.new(static_file_path) + end + + # Agrega el nombre de archivo a la ruta para tener retrocompatibilidad + # + # @return [Pathname] + def destination_path_with_filename + destination_path.realpath + end + + def relative_destination_path_with_filename + destination_path_with_filename.relative_path_from(site.path) + end + + def static_file_path + static_file.blob.service.path_for(static_file.key) + end + # No hay archivo pero se lo describió def no_file_for_description? !path? && description? From f3df5504944cbdbf2f07aa0cd33f8fd8cd7f55a0 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:21:48 -0300 Subject: [PATCH 030/251] =?UTF-8?q?no=20permitir=20subir=20im=C3=A1genes?= =?UTF-8?q?=20que=20no=20son=20para=20web?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/metadata_image.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/models/metadata_image.rb b/app/models/metadata_image.rb index f91a6273..f86c5c26 100644 --- a/app/models/metadata_image.rb +++ b/app/models/metadata_image.rb @@ -13,8 +13,6 @@ class MetadataImage < MetadataFile # Determina si es una imagen def image? - return true unless file - - filemagic.file(file).starts_with? 'image/' + static_file&.blob&.send(:web_image?) end end From 02b52b23b91b0c88a9c48b01604eb9ee22692d9f Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:24:04 -0300 Subject: [PATCH 031/251] compartir el nombre de archivo con JekyllService --- app/models/active_storage/blob_decorator.rb | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 app/models/active_storage/blob_decorator.rb diff --git a/app/models/active_storage/blob_decorator.rb b/app/models/active_storage/blob_decorator.rb new file mode 100644 index 00000000..9c01251a --- /dev/null +++ b/app/models/active_storage/blob_decorator.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module ActiveStorage + # Modificaciones a ActiveStorage::Blob + module BlobDecorator + extend ActiveSupport::Concern + + included do + # Permitir que llegue el nombre de archivo al servicio de subida de + # archivos. + # + # @return [Hash] + def service_metadata + if forcibly_serve_as_binary? + { content_type: ActiveStorage.binary_content_type, disposition: :attachment, filename: filename } + elsif !allowed_inline? + { content_type: content_type, disposition: :attachment, filename: filename } + else + { content_type: content_type, filename: filename } + end + end + end + end +end + +ActiveStorage::Blob.include ActiveStorage::BlobDecorator From 69e7df4c31b8f3b72d9e2cde2b7430bb3cef4f66 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:26:56 -0300 Subject: [PATCH 032/251] instanciar JekyllService para cada sitio --- .../service/registry_decorator.rb | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 app/lib/active_storage/service/registry_decorator.rb diff --git a/app/lib/active_storage/service/registry_decorator.rb b/app/lib/active_storage/service/registry_decorator.rb new file mode 100644 index 00000000..f7f20784 --- /dev/null +++ b/app/lib/active_storage/service/registry_decorator.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module ActiveStorage + module Service + # Modificaciones a ActiveStorage::Service::Registry + module RegistryDecorator + extend ActiveSupport::Concern + + included do + # El mismo comportamiento que #fetch con el agregado de generar + # un {JekyllService} para cada sitio. + def fetch(name) + services.fetch(name.to_sym) do |key| + if configurations.include?(key) + services[key] = configurator.build(key) + elsif (site = Site.find_by_name(key)) + root = File.join(site.path, 'public') + services[key] = ActiveStorage::Service::JekyllService.new(root: root, public: true).tap do |s| + s.name = key.to_sym + end + elsif block_given? + yield key + else + raise KeyError, "Missing configuration for the #{key} Active Storage service. " \ + "Configurations available for the #{configurations.keys.to_sentence} services." + end + end + end + end + end + end +end + +ActiveStorage::Service::Registry.include ActiveStorage::Service::RegistryDecorator From 15e978c877cacc8b5eb3a6c79c5e0e9cc3349eda Mon Sep 17 00:00:00 2001 From: f Date: Sat, 5 Mar 2022 20:08:55 -0300 Subject: [PATCH 033/251] eran clases --- app/lib/active_storage/service/jekyll_service.rb | 2 +- app/lib/active_storage/service/registry_decorator.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 686c3979..173f6898 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module ActiveStorage - module Service + class Service # Sube los archivos a cada repositorio y los agrega al LFS de su # repositorio git. # diff --git a/app/lib/active_storage/service/registry_decorator.rb b/app/lib/active_storage/service/registry_decorator.rb index f7f20784..f6794607 100644 --- a/app/lib/active_storage/service/registry_decorator.rb +++ b/app/lib/active_storage/service/registry_decorator.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module ActiveStorage - module Service + class Service # Modificaciones a ActiveStorage::Service::Registry module RegistryDecorator extend ActiveSupport::Concern From 7f6063475bb63eed2cab8523d5f15d1ef71b15ea Mon Sep 17 00:00:00 2001 From: f Date: Sat, 5 Mar 2022 20:09:09 -0300 Subject: [PATCH 034/251] =?UTF-8?q?m=C3=A9todo=20gen=C3=A9rico=20para=20in?= =?UTF-8?q?stanciar=20el=20servicio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/lib/active_storage/service/jekyll_service.rb | 7 +++++++ app/lib/active_storage/service/registry_decorator.rb | 5 +---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 173f6898..20dda523 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -12,6 +12,13 @@ module ActiveStorage class JekyllService < Service::DiskService BLOB_NAME = 'blob' + # Genera un servicio para un sitio determinado + def self.build_for_site(site:) + new(root: File.join(site.path, 'public'), public: true).tap do |js| + js.name = site.name.to_sym + end + end + # Para poder guardar el archivo con el nombre original pero poder # recuperarlo durante el download, luego de subirlo le cambiamos el # nombre y creamos un link simbólico a un nombre conocido. diff --git a/app/lib/active_storage/service/registry_decorator.rb b/app/lib/active_storage/service/registry_decorator.rb index f6794607..c7096356 100644 --- a/app/lib/active_storage/service/registry_decorator.rb +++ b/app/lib/active_storage/service/registry_decorator.rb @@ -14,10 +14,7 @@ module ActiveStorage if configurations.include?(key) services[key] = configurator.build(key) elsif (site = Site.find_by_name(key)) - root = File.join(site.path, 'public') - services[key] = ActiveStorage::Service::JekyllService.new(root: root, public: true).tap do |s| - s.name = key.to_sym - end + services[key] = ActiveStorage::Service::JekyllService.build_for_site(site: site) elsif block_given? yield key else From 10eef47ce8bfb0f233083e070207f7d1c8bbf578 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 5 Mar 2022 20:09:37 -0300 Subject: [PATCH 035/251] todos los archivos subidos se asocian al sitio --- .../attached/changes/create_one_decorator.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 app/lib/active_storage/attached/changes/create_one_decorator.rb diff --git a/app/lib/active_storage/attached/changes/create_one_decorator.rb b/app/lib/active_storage/attached/changes/create_one_decorator.rb new file mode 100644 index 00000000..bfb92478 --- /dev/null +++ b/app/lib/active_storage/attached/changes/create_one_decorator.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module ActiveStorage + module Attached::Changes::CreateOneDecorator + extend ActiveSupport::Concern + + included do + private + + # A partir de ahora todos los archivos se suben al servicio de + # cada sitio. + def attachment_service_name + record.name.to_sym + end + end + end +end + +ActiveStorage::Attached::Changes::CreateOne.include ActiveStorage::Attached::Changes::CreateOneDecorator From bfb8e95599762f73cc5c8b093932d1750b55c5ea Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 12:51:06 -0300 Subject: [PATCH 036/251] obtener el nombre de archivo desde la key esto agrega una query pero permite no tener que agregar una ruta falsa --- .../active_storage/service/jekyll_service.rb | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 20dda523..02316e65 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -10,8 +10,6 @@ module ActiveStorage # implementemos IPFS) para poder transferir los archivos junto con el # sitio. class JekyllService < Service::DiskService - BLOB_NAME = 'blob' - # Genera un servicio para un sitio determinado def self.build_for_site(site:) new(root: File.join(site.path, 'public'), public: true).tap do |js| @@ -19,16 +17,7 @@ module ActiveStorage end end - # Para poder guardar el archivo con el nombre original pero poder - # recuperarlo durante el download, luego de subirlo le cambiamos el - # nombre y creamos un link simbólico a un nombre conocido. - def upload(key, io, checksum: nil, **options) - super.tap do - path = path_for(key) - filename = options[:filename].to_s - FileUtils.mv path, path.sub(/#{BLOB_NAME}\z/, filename) - FileUtils.ln_s filename, path end end @@ -41,9 +30,17 @@ module ActiveStorage key end + # Obtiene el nombre de archivo para esta key + # + # @param :key [String] + # @return [String] + def filename_for(key) + @filename_for ||= ActiveStorage::Blob.where(key: key).limit(1).pluck(:filename).first + end + # Crea una ruta para la llave con un nombre conocido. def path_for(key) - File.join root, folder_for(key), BLOB_NAME + File.join root, folder_for(key), filename_for(key) end end end From 85cab49208cbfae13b1876ab5a57495544b4220c Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 15:25:15 -0300 Subject: [PATCH 037/251] subir los archivos desde el editor al sitio --- .../direct_uploads_controller_decorator.rb | 18 ++++++++++++++++++ app/controllers/posts_controller.rb | 6 ++++++ .../active_storage/service/jekyll_service.rb | 4 ---- 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 app/controllers/active_storage/direct_uploads_controller_decorator.rb diff --git a/app/controllers/active_storage/direct_uploads_controller_decorator.rb b/app/controllers/active_storage/direct_uploads_controller_decorator.rb new file mode 100644 index 00000000..f27c4cfb --- /dev/null +++ b/app/controllers/active_storage/direct_uploads_controller_decorator.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module ActiveStorage + # Modifica la creación de un blob antes de subir el archivo para que + # incluya el JekyllService adecuado. + module DirectUploadsControllerDecorator + extend ActiveSupport::Concern + + included do + def create + blob = ActiveStorage::Blob.create_before_direct_upload!(service_name: session[:service_name], **blob_args) + render json: direct_upload_json(blob) + end + end + end +end + +ActiveStorage::DirectUploadsController.include ActiveStorage::DirectUploadsControllerDecorator diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index dbdd4d0a..2aff9ac9 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -6,6 +6,7 @@ class PostsController < ApplicationController rescue_from Pundit::NilPolicyError, with: :page_not_found before_action :authenticate_usuarie! + before_action :service_for_direct_upload, only: %i[new edit] # TODO: Traer los comunes desde ApplicationController breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path @@ -166,4 +167,9 @@ class PostsController < ApplicationController def post @post ||= site.posts(lang: locale).find(params[:post_id] || params[:id]) end + + # Recuerda el nombre del servicio de subida de archivos + def service_for_direct_upload + session[:service_name] = site.name.to_sym + end end diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 02316e65..601b6f2f 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -17,10 +17,6 @@ module ActiveStorage end end - - end - end - # Mantener retrocompatibilidad con cómo gestionamos los archivos # subidos hasta ahora. # From c13c021fe92d749cf4a98673bc3cc504c6e90d78 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 15:28:25 -0300 Subject: [PATCH 038/251] asignar el nombre de archivo en la subida directa --- .../active_storage/service/jekyll_service.rb | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 601b6f2f..a2e04336 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -17,6 +17,32 @@ module ActiveStorage end end + # Lo mismo que en DiskService agregando el nombre de archivo en la + # firma. Esto permite que luego podamos guardar el archivo donde + # corresponde. + def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) + instrument :url, key: key do |payload| + verified_token_with_expiration = ActiveStorage.verifier.generate( + { + key: key, + content_type: content_type, + content_length: content_length, + checksum: checksum, + service_name: name, + filename: filename_for(key) + }, + expires_in: expires_in, + purpose: :blob_token + ) + + generated_url = url_helpers.update_rails_disk_service_url(verified_token_with_expiration, host: current_host) + + payload[:url] = generated_url + + generated_url + end + end + # Mantener retrocompatibilidad con cómo gestionamos los archivos # subidos hasta ahora. # From 211fb308e39ddd1a380b711e98d240c59e9576d6 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 15:59:57 -0300 Subject: [PATCH 039/251] asignar archivos subidos desde el editor al sitio --- .../disk_controller_decorator.rb | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 app/controllers/active_storage/disk_controller_decorator.rb diff --git a/app/controllers/active_storage/disk_controller_decorator.rb b/app/controllers/active_storage/disk_controller_decorator.rb new file mode 100644 index 00000000..14366a15 --- /dev/null +++ b/app/controllers/active_storage/disk_controller_decorator.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module ActiveStorage + # Modificar {DiskController} para poder asociar el blob a un sitio + module DiskControllerDecorator + extend ActiveSupport::Concern + + included do + # Asociar el archivo subido al sitio correspondiente. Cada sitio + # tiene su propio servicio de subida de archivos. + def update + if (token = decode_verified_token) + if acceptable_content?(token) + named_disk_service(token[:service_name]).upload token[:key], request.body, checksum: token[:checksum] + + blob = ActiveStorage::Blob.find_by_key token[:key] + site = Site.find_by_name token[:service_name] + + site.static_files.attach(blob) + else + head :unprocessable_entity + end + else + head :not_found + end + rescue ActiveStorage::IntegrityError + head :unprocessable_entity + end + end + end +end + +ActiveStorage::DiskController.include ActiveStorage::DiskControllerDecorator From 61622a4c416b73288cc897b89b8721fe3bb700c9 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 16:00:21 -0300 Subject: [PATCH 040/251] =?UTF-8?q?documentaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/lib/active_storage/service/jekyll_service.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index a2e04336..3b1db7ec 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -11,6 +11,9 @@ module ActiveStorage # sitio. class JekyllService < Service::DiskService # Genera un servicio para un sitio determinado + # + # @param :site [Site] + # @return [ActiveStorage::Service::JekyllService] def self.build_for_site(site:) new(root: File.join(site.path, 'public'), public: true).tap do |js| js.name = site.name.to_sym @@ -20,6 +23,13 @@ module ActiveStorage # Lo mismo que en DiskService agregando el nombre de archivo en la # firma. Esto permite que luego podamos guardar el archivo donde # corresponde. + # + # @param :key [String] + # @param :expires_in [Integer] + # @param :content_type [String] + # @param :content_length [Integer] + # @param :checksum [String] + # @return [String] def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) instrument :url, key: key do |payload| verified_token_with_expiration = ActiveStorage.verifier.generate( @@ -61,6 +71,9 @@ module ActiveStorage end # Crea una ruta para la llave con un nombre conocido. + # + # @param :key [String] + # @return [String] def path_for(key) File.join root, folder_for(key), filename_for(key) end From 6c9288b03bd64f907760d63a5c0ddd638b9ea74f Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 16:01:16 -0300 Subject: [PATCH 041/251] =?UTF-8?q?deprecar=20la=20migraci=C3=B3n=20de=20a?= =?UTF-8?q?rchivos=20est=C3=A1ticos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit siempre fue super lenta y falible --- app/models/site.rb | 7 +--- app/models/site/static_file_migration.rb | 52 ------------------------ 2 files changed, 1 insertion(+), 58 deletions(-) delete mode 100644 app/models/site/static_file_migration.rb diff --git a/app/models/site.rb b/app/models/site.rb index 5b78d625..7d4875e5 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -57,7 +57,7 @@ class Site < ApplicationRecord # Carga el sitio Jekyll una vez que se inicializa el modelo o después # de crearlo after_initialize :load_jekyll - after_create :load_jekyll, :static_file_migration! + after_create :load_jekyll # Cambiar el nombre del directorio before_update :update_name! before_save :add_private_key_if_missing! @@ -474,11 +474,6 @@ class Site < ApplicationRecord config.hostname = hostname end - # Migra los archivos a Sutty - def static_file_migration! - Site::StaticFileMigration.new(site: self).migrate! - end - # Valida si el sitio tiene al menos una forma de alojamiento asociada # y es la local # diff --git a/app/models/site/static_file_migration.rb b/app/models/site/static_file_migration.rb deleted file mode 100644 index 36a882bf..00000000 --- a/app/models/site/static_file_migration.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -class Site - # Obtiene todos los archivos relacionados en artículos del sitio y los - # sube a Sutty. - class StaticFileMigration - # Tipos de metadatos que contienen archivos - STATIC_TYPES = %i[file image].freeze - - attr_reader :site - - def initialize(site:) - @site = site - end - - def migrate! - modified = site.docs.map do |doc| - next unless STATIC_TYPES.map do |field| - next unless doc.attribute? field - next unless doc[field].path? - next unless doc[field].static_file - - true - end.any? - - log.write "#{doc.path.relative};no se pudo guardar\n" unless doc.save(validate: false) - - doc.path.absolute - end.compact - - log.close - - return if modified.empty? - - # TODO: Hacer la migración desde el servicio de creación de sitios? - site.repository.commit(file: modified, - message: I18n.t('sites.static_file_migration'), - usuarie: author) - end - - private - - def author - @author ||= GitAuthor.new email: "sutty@#{Site.domain}", - name: 'Sutty' - end - - def log - @log ||= File.open(File.join(site.path, 'migration.csv'), 'w') - end - end -end From f7d8a3ecf9e0142292ea819e7ab94c6b6b096a56 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 17:07:37 -0300 Subject: [PATCH 042/251] fixup! obtener el nombre de archivo desde la key --- app/lib/active_storage/service/jekyll_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 3b1db7ec..92b26e0e 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -67,7 +67,7 @@ module ActiveStorage # @param :key [String] # @return [String] def filename_for(key) - @filename_for ||= ActiveStorage::Blob.where(key: key).limit(1).pluck(:filename).first + ActiveStorage::Blob.where(key: key).limit(1).pluck(:filename).first end # Crea una ruta para la llave con un nombre conocido. From 81cbafef66e31ba60ac6bf396adc88696d85f142 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 7 Mar 2022 20:52:33 -0300 Subject: [PATCH 043/251] validaciones --- app/models/metadata_file.rb | 2 +- app/models/metadata_image.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 1c859481..5be7f84c 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -19,7 +19,7 @@ class MetadataFile < MetadataTemplate errors << I18n.t("metadata.#{type}.site_invalid") if site.invalid? errors << I18n.t("metadata.#{type}.path_required") if path_missing? errors << I18n.t("metadata.#{type}.no_file_for_description") if no_file_for_description? - errors << I18n.t("metadata.#{type}.attachment_missing") unless static_file + errors << I18n.t("metadata.#{type}.attachment_missing") if path? && !static_file errors.compact! errors.empty? diff --git a/app/models/metadata_image.rb b/app/models/metadata_image.rb index f86c5c26..85ee062a 100644 --- a/app/models/metadata_image.rb +++ b/app/models/metadata_image.rb @@ -5,7 +5,7 @@ class MetadataImage < MetadataFile def validate super - errors << I18n.t('metadata.image.not_an_image') unless image? + errors << I18n.t('metadata.image.not_an_image') unless path? && image? errors.compact! errors.empty? From 7de96a4581a1d1483628c3da11f99e1a4e454ec0 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 8 Mar 2022 13:10:34 -0300 Subject: [PATCH 044/251] =?UTF-8?q?usar=20la=20ruta=20absoluta=20tambi?= =?UTF-8?q?=C3=A9n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit en producción usamos links simbólicos entre directorios y sin resolver la ubicación real de los sitios estábamos generando rutas erróneas. --- app/models/metadata_file.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 5be7f84c..31becc64 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -119,7 +119,7 @@ class MetadataFile < MetadataTemplate end def relative_destination_path_with_filename - destination_path_with_filename.relative_path_from(site.path) + destination_path_with_filename.relative_path_from(Pathname.new(site.path).realpath) end def static_file_path From 79974b710497c2db0498de7f9f4375f2802711c9 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 8 Mar 2022 13:21:13 -0300 Subject: [PATCH 045/251] =?UTF-8?q?validar=20im=C3=A1genes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/metadata_image.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/metadata_image.rb b/app/models/metadata_image.rb index 85ee062a..fc61a4d7 100644 --- a/app/models/metadata_image.rb +++ b/app/models/metadata_image.rb @@ -5,7 +5,7 @@ class MetadataImage < MetadataFile def validate super - errors << I18n.t('metadata.image.not_an_image') unless path? && image? + errors << I18n.t('metadata.image.not_an_image') if path? && !image? errors.compact! errors.empty? From d7f9f0c9b1f1c3d43632cc92872805d5608687cc Mon Sep 17 00:00:00 2001 From: f Date: Tue, 8 Mar 2022 15:09:31 -0300 Subject: [PATCH 046/251] devolver todos los dominios --- app/controllers/api/v1/sites_controller.rb | 28 ++++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/v1/sites_controller.rb b/app/controllers/api/v1/sites_controller.rb index 6abff704..32fdae1d 100644 --- a/app/controllers/api/v1/sites_controller.rb +++ b/app/controllers/api/v1/sites_controller.rb @@ -9,7 +9,7 @@ module Api # Lista de nombres de dominios a emitir certificados def index - render json: sites_names + alternative_names + api_names + render json: sites_names + alternative_names + api_names + www_names end # Sitios con hidden service de Tor @@ -28,7 +28,7 @@ module Api site = Site.find_by(name: params[:name]) if site - usuarie = GitAuthor.new email: 'tor@' + Site.domain, name: 'Tor' + usuarie = GitAuthor.new email: "tor@#{Site.domain}", name: 'Tor' service = SiteService.new site: site, usuarie: usuarie, params: params service.add_onion @@ -39,14 +39,22 @@ module Api private + def canonicalize(name) + name.end_with?('.') ? name[0..-2] : "#{name}.#{Site.domain}" + end + # Nombres de los sitios def sites_names - Site.all.order(:name).pluck(:name) + Site.all.order(:name).pluck(:name).map do |name| + canonicalize name + end end # Dominios alternativos def alternative_names - DeployAlternativeDomain.all.map(&:hostname) + (DeployAlternativeDomain.all.map(&:hostname) + DeployLocalizedDomain.all.map(&:hostname)).map do |name| + canonicalize name + end end # Obtener todos los sitios con API habilitada, es decir formulario @@ -56,7 +64,17 @@ module Api def api_names Site.where(contact: true) .or(Site.where(colaboracion_anonima: true)) - .select("'api.' || name as name").map(&:name) + .select("'api.' || name as name").map(&:name).map do |name| + canonicalize name + end + end + + def www_names + Site.where(contact: true) + .or(Site.where(colaboracion_anonima: true)) + .select("'www.' || name as name").map(&:name).map do |name| + canonicalize name + end end end end From d4d1acc8d91e5cf23db4fa056a5c0520d7179c43 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 9 Mar 2022 19:32:53 -0300 Subject: [PATCH 047/251] retrocompatibilidad los archivos subidos como archivos locales ya estaban copiados al sitio --- app/models/metadata_file.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 31becc64..0f50d643 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -123,7 +123,12 @@ class MetadataFile < MetadataTemplate end def static_file_path - static_file.blob.service.path_for(static_file.key) + case static_file.blob.service.name + when :local + File.join(site.path, 'public', static_file.key, static_file.filename.to_s) + else + static_file.blob.service.path_for(static_file.key) + end end # No hay archivo pero se lo describió From 263394baef931995b78e4a962a6bebfbc5f76b27 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 18 Mar 2022 15:42:39 -0300 Subject: [PATCH 048/251] no fallar al serializar Site#everything_of el problema era que los archivos subidos no se pueden marshalear y estamos usando MetadataTemplate#values para poder cachearlos. closes #1315 closes #1316 closes #1842 closes #2368 closes #2369 closes #2370 closes #2371 closes #2372 closes #2373 closes #2393 closes #2396 closes #2922 closes #3347 closes #3350 closes #3353 closes #3356 closes #3437 closes #3486 closes #3487 closes #3488 closes #3551 closes #3554 closes #4229 closes #4256 closes #4259 closes #4262 closes #4265 closes #4553 closes #4556 closes #4603 closes #4606 closes #4647 closes #4759 closes #4767 closes #4787 closes #4794 closes #4833 closes #4835 closes #4836 closes #4875 closes #4876 closes #4877 closes #4879 closes #4881 closes #4882 closes #4883 closes #4884 closes #4885 closes #4886 closes #4887 closes #4888 closes #4889 closes #4890 closes #4891 closes #4892 closes #4906 closes #4907 closes #4908 closes #4976 closes #4977 closes #4978 closes #4979 closes #4980 closes #4981 closes #4982 closes #4983 closes #4984 closes #4985 closes #4986 closes #4987 --- app/models/metadata_file.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 0f50d643..2c1a6d59 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -13,6 +13,11 @@ class MetadataFile < MetadataTemplate value == default_value end + # No hay valores sugeridos para archivos subidos. + # + # XXX: Esto ayuda a deserializar en {Site#everything_of} + def values; end + def validate super From c5896a63cfdd48128592087aac35aaf1a6523df1 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 22 Mar 2022 20:15:59 -0300 Subject: [PATCH 049/251] resiliencia closes #5061 closes #5060 closes #5058 closes #5057 closes #5056 closes #5054 closes #5053 closes #5051 closes #5050 closes #5048 closes #5047 closes #5045 closes #5042 closes #5040 closes #5038 closes #5036 closes #5034 closes #5033 closes #5032 closes #5030 closes #4946 closes #3856 closes #3562 closes #3558 closes #3557 closes #2707 closes #2706 closes #2705 closes #2703 closes #2702 closes #2297 closes #2296 closes #2295 closes #1970 closes #1969 closes #1768 --- app/models/metadata_file.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 2c1a6d59..eca87478 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -121,6 +121,13 @@ class MetadataFile < MetadataTemplate # @return [Pathname] def destination_path_with_filename destination_path.realpath + # Si el archivo no llegara a existir, en lugar de hacer fallar todo, + # devolvemos la ruta original, que puede ser el archivo que no existe + # o vacía si se está subiendo uno. + rescue Errno::ENOENT => e + ExceptionNotifier.notify_exception(e) + + value['path'] end def relative_destination_path_with_filename From ecb823f40746b4eae0d61cc3acca63ac620881f0 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 23 Mar 2022 11:37:00 -0300 Subject: [PATCH 050/251] al migrar un archivo reutilizar la key --- app/models/metadata_file.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index eca87478..71d3f049 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -78,7 +78,9 @@ class MetadataFile < MetadataTemplate if (blob_id = ActiveStorage::Blob.where(key: key_from_path).pluck(:id).first) site.static_files.find_by(blob_id: blob_id) elsif path? && pathname.exist? && site.static_files.attach(io: pathname.open, filename: pathname.basename) - site.static_files.last + site.static_files.last.tap do |s| + s.blob.update(key: key_from_path) + end end end end From b4532e94ca3ed5c33f50b236c31df8d38fdfb200 Mon Sep 17 00:00:00 2001 From: Nulo Date: Sat, 2 Apr 2022 14:34:12 +0000 Subject: [PATCH 051/251] Agregar citas al editor --- app/assets/stylesheets/editor.scss | 6 +++++- app/javascript/editor/types/blocks.ts | 8 +++++++- app/javascript/editor/utils.ts | 1 + app/models/metadata_template.rb | 2 +- app/views/posts/attributes/_content.haml | 3 +++ config/locales/en.yml | 1 + config/locales/es.yml | 1 + 7 files changed, 19 insertions(+), 3 deletions(-) diff --git a/app/assets/stylesheets/editor.scss b/app/assets/stylesheets/editor.scss index 30fab60a..85bf4471 100644 --- a/app/assets/stylesheets/editor.scss +++ b/app/assets/stylesheets/editor.scss @@ -67,7 +67,11 @@ .editor-content { min-height: 480px; - p, h1, h2, h3, h4, h5, h6, ul, li, figcaption { outline: #ccc solid thin; } + p, h1, h2, h3, h4, h5, h6, ul, li, blockquote, figcaption { outline: #ccc solid thin; } + blockquote { + border-left: #555 solid .25em; + padding: .75em; + } strong, em, del, u, sub, sup, small { background: #0002; } a { background: #13fefe50; } [data-editor-selected] { outline: #f206f9 solid thick; } diff --git a/app/javascript/editor/types/blocks.ts b/app/javascript/editor/types/blocks.ts index 2e2dea7e..956b79d9 100644 --- a/app/javascript/editor/types/blocks.ts +++ b/app/javascript/editor/types/blocks.ts @@ -21,11 +21,12 @@ function makeBlock(tag: string): EditorBlock { } export const li: EditorBlock = makeBlock("li"); +const paragraph: EditorBlock = makeBlock("p"); // XXX: si agregás algo acá, agregalo a blockNames // (y probablemente le quieras hacer un botón en app/views/posts/attributes/_content.haml) export const blocks: { [propName: string]: EditorBlock } = { - paragraph: makeBlock("p"), + paragraph, h1: makeBlock("h1"), h2: makeBlock("h2"), h3: makeBlock("h3"), @@ -42,6 +43,11 @@ export const blocks: { [propName: string]: EditorBlock } = { allowedChildren: ["li"], handleEmpty: li, }, + blockquote: { + ...makeBlock("blockquote"), + allowedChildren: blockNames, + handleEmpty: paragraph, + }, }; export function setupButtons(editor: Editor): void { diff --git a/app/javascript/editor/utils.ts b/app/javascript/editor/utils.ts index 167c0a6d..b0bed66e 100644 --- a/app/javascript/editor/utils.ts +++ b/app/javascript/editor/utils.ts @@ -10,6 +10,7 @@ export const blockNames = [ "h6", "unordered_list", "ordered_list", + "blockquote", ]; export const markNames = [ "bold", diff --git a/app/models/metadata_template.rb b/app/models/metadata_template.rb index 5baa7a4a..26351249 100644 --- a/app/models/metadata_template.rb +++ b/app/models/metadata_template.rb @@ -199,7 +199,7 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, end def allowed_tags - @allowed_tags ||= %w[strong em del u mark p h1 h2 h3 h4 h5 h6 ul ol li img iframe audio video div figure + @allowed_tags ||= %w[strong em del u mark p h1 h2 h3 h4 h5 h6 ul ol li img iframe audio video div figure blockquote figcaption a sub sup small].freeze end diff --git a/app/views/posts/attributes/_content.haml b/app/views/posts/attributes/_content.haml index 4ae70ba0..36b88872 100644 --- a/app/views/posts/attributes/_content.haml +++ b/app/views/posts/attributes/_content.haml @@ -95,6 +95,9 @@ %button.btn{ type: 'button', title: t('editor.right'), data: { editor_button: 'parentBlock-right' } }> %i.fa.fa-fw.fa-align-right> %span.sr-only>= t('editor.right') + %button.btn{ type: 'button', title: t('editor.blockquote'), data: { editor_button: 'block-blockquote' } }> + %i.fa.fa-fw.fa-quote-left> + %span.sr-only>= t('editor.blockquote') -# HAML cringe .editor-auxiliary-toolbar.mt-1.scrollbar-black{ data: { editor_auxiliary_toolbar: '' } } diff --git a/config/locales/en.yml b/config/locales/en.yml index b814796d..99fbc8b1 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -568,6 +568,7 @@ en: left: Left right: Right center: Center + blockquote: Quote color: Color text-color: Text color multimedia: Media diff --git a/config/locales/es.yml b/config/locales/es.yml index a6fbd407..3ed72d63 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -576,6 +576,7 @@ es: left: Izquierda right: Derecha center: Centro + blockquote: Cita color: Color text-color: Color del texto multimedia: Multimedia From 455070f2ea576775fe28f0bb2d01a6b62d2dc7d0 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 4 Apr 2022 13:56:40 -0300 Subject: [PATCH 052/251] normalizar todas las strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit convierte la codificación de windows/osx en unicode normalizado. --- app/models/metadata_markdown.rb | 2 +- app/models/metadata_markdown_content.rb | 2 +- app/models/metadata_permalink.rb | 2 +- app/models/metadata_string.rb | 2 +- app/models/metadata_template.rb | 9 ++++++--- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/app/models/metadata_markdown.rb b/app/models/metadata_markdown.rb index 1e8b4fc8..7816ec33 100644 --- a/app/models/metadata_markdown.rb +++ b/app/models/metadata_markdown.rb @@ -12,6 +12,6 @@ class MetadataMarkdown < MetadataText # markdown y se eliminan autolinks. Mejor es habilitar la generación # SAFE de CommonMark en la configuración del sitio. def sanitize(string) - string + string.unicode_normalize(:nfkc) end end diff --git a/app/models/metadata_markdown_content.rb b/app/models/metadata_markdown_content.rb index 92a1ab21..cb4124db 100644 --- a/app/models/metadata_markdown_content.rb +++ b/app/models/metadata_markdown_content.rb @@ -25,6 +25,6 @@ class MetadataMarkdownContent < MetadataText # markdown y se eliminan autolinks. Mejor es deshabilitar la # generación SAFE de CommonMark en la configuración del sitio. def sanitize(string) - string.tr("\r", '') + string.tr("\r", '').unicode_normalize(:nfkc) end end diff --git a/app/models/metadata_permalink.rb b/app/models/metadata_permalink.rb index 59b68461..9b0c063c 100644 --- a/app/models/metadata_permalink.rb +++ b/app/models/metadata_permalink.rb @@ -19,7 +19,7 @@ class MetadataPermalink < MetadataString # puntos suspensivos, la primera / para que siempre sea relativa y # agregamos una / al final si la ruta no tiene extensión. def sanitize(value) - value = value.strip.gsub('..', '/').gsub('./', '').squeeze('/') + value = value.strip.unicode_normalize(:nfkc).gsub('..', '/').gsub('./', '').squeeze('/') value = value[1..-1] if value.start_with? '/' value += '/' if File.extname(value).blank? diff --git a/app/models/metadata_string.rb b/app/models/metadata_string.rb index 95aac4d4..28bfe82a 100644 --- a/app/models/metadata_string.rb +++ b/app/models/metadata_string.rb @@ -17,7 +17,7 @@ class MetadataString < MetadataTemplate def sanitize(string) return '' if string.blank? - sanitizer.sanitize(string.strip, + sanitizer.sanitize(string.strip.unicode_normalize(:nfkc), tags: [], attributes: []).strip.html_safe end diff --git a/app/models/metadata_template.rb b/app/models/metadata_template.rb index 5baa7a4a..a72f8e83 100644 --- a/app/models/metadata_template.rb +++ b/app/models/metadata_template.rb @@ -184,9 +184,12 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, return if string.nil? return string unless string.is_a? String - sanitizer.sanitize(string.tr("\r", ''), - tags: allowed_tags, - attributes: allowed_attributes).strip.html_safe + sanitizer + .sanitize(string.tr("\r", '').unicode_normalize(:nfkc), + tags: allowed_tags, + attributes: allowed_attributes) + .strip + .html_safe end def sanitizer From 30bc14d83e790c6540a6fb19e8afa8184eef6c37 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 4 Apr 2022 14:27:00 -0300 Subject: [PATCH 053/251] normalizar los nombres de archivo enviados por subida directa --- .../direct_uploads_controller_decorator.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/controllers/active_storage/direct_uploads_controller_decorator.rb b/app/controllers/active_storage/direct_uploads_controller_decorator.rb index f27c4cfb..3052f974 100644 --- a/app/controllers/active_storage/direct_uploads_controller_decorator.rb +++ b/app/controllers/active_storage/direct_uploads_controller_decorator.rb @@ -11,6 +11,17 @@ module ActiveStorage blob = ActiveStorage::Blob.create_before_direct_upload!(service_name: session[:service_name], **blob_args) render json: direct_upload_json(blob) end + + private + + # Normalizar los caracteres unicode en los nombres de archivos + # para que puedan propagarse correctamente a través de todo el + # stack. + def blob_args + params.require(:blob).permit(:filename, :byte_size, :checksum, :content_type, metadata: {}).to_h.symbolize_keys.tap do |ba| + ba[:filename] = ba[:filename].unicode_normalize(:nfkc) + end + end end end end From 4d6a26d67178ec90e4b8f580889cee528f74d195 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 4 Apr 2022 14:32:58 -0300 Subject: [PATCH 054/251] normalizar todos los nombres de archivos subidos --- .../http/uploaded_file_decorator.rb | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 app/lib/action_dispatch/http/uploaded_file_decorator.rb diff --git a/app/lib/action_dispatch/http/uploaded_file_decorator.rb b/app/lib/action_dispatch/http/uploaded_file_decorator.rb new file mode 100644 index 00000000..c171c81c --- /dev/null +++ b/app/lib/action_dispatch/http/uploaded_file_decorator.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module ActionDispatch + module Http + # Normaliza los nombres de archivo para que se propaguen + # correctamente a través de todo el stack. + module UploadedFileDecorator + extend ActiveSupport::Concern + + included do + # Devolver el nombre de archivo con caracteres unicode + # normalizados + def original_filename + @original_filename.unicode_normalize(:nfkc) + end + end + end + end +end + +ActionDispatch::Http::UploadedFile.include ActionDispatch::Http::UploadedFileDecorator From accb559f014ccc287ffcc5e6f15ce5ddae918a53 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 16:39:56 -0300 Subject: [PATCH 055/251] hacer deploys remotos --- app/models/deploy_rsync.rb | 88 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 app/models/deploy_rsync.rb diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb new file mode 100644 index 00000000..40018dbd --- /dev/null +++ b/app/models/deploy_rsync.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +# Sincroniza sitios a servidores remotos usando Rsync. El servidor +# remoto tiene que tener rsync instalado. +class DeployRsync < Deploy + store :values, accessors: %i[flags destination host_keys], coder: JSON + + def deploy + ssh? && rsync + end + + # No se ocupa espacio local + # + # @return [Integer] + def size + 0 + end + + private + + # Verificar la conexión SSH implementando Trust On First Use + # + # TODO: Medir el tiempo que tarda en iniciarse la conexión + # + # @return [Boolean] + def ssh? + user, host = user_host + ssh_available = false + + Net::SSH.start(host, user, verify_host_key: tofu) do |ssh| + if values[:host_keys].blank? + # Guardar las llaves que se encontraron en la primera conexión + values[:host_keys] = ssh.transport.host_keys.map do |host_key| + "#{host_key.ssh_type} #{host_key.fingerprint}" + end + + ssh_available = save + else + ssh_available = true + end + end + + ssh_available + rescue Exception => e + ExceptionNotifier.notify_exception(e, data: { site: site.id, hostname: host, user: user }) + + false + end + + # Confiar en la primera llave que encontremos, fallar si cambian + # + # @return [Symbol] + def tofu + values[:host_keys].present? ? :always : :accept_new + end + + # Devuelve el par user host + # + # @return [Array] + def user_host + destination.split(':', 2).first.split('@', 2).tap do |d| + next unless d.size == 1 + + d.insert(0, nil) + end + end + + # Sincroniza hacia el directorio remoto, usando las flags opcionales. + # + # @return [Boolean] + def rsync + run %(rsync -av #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + end + + # El origen es el destino de la compilación + # + # @return [String] + def source + site.deploys.find_by(type: 'DeployLocal').destination + end + + # Devolver el destino o lanzar un error si no está configurado + def destination + values[:destination].tap do |_d| + raise ArgumentError, 'destination no está configurado' + end + end +end From f10b65173a9542500e3d206a2167fb3524b59ed0 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 17:05:51 -0300 Subject: [PATCH 056/251] fixup! hacer deploys remotos --- app/models/deploy_rsync.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 40018dbd..9e855f57 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -81,8 +81,8 @@ class DeployRsync < Deploy # Devolver el destino o lanzar un error si no está configurado def destination - values[:destination].tap do |_d| - raise ArgumentError, 'destination no está configurado' + values[:destination].tap do |d| + raise(ArgumentError, 'destination no está configurado') if d.blank? end end end From 6fcdeb52f381bbde75b5156671f228016209f6ea Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 17:23:42 -0300 Subject: [PATCH 057/251] entorno --- app/models/deploy_rsync.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 9e855f57..4191b20d 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -47,6 +47,14 @@ class DeployRsync < Deploy false end + def env + { + 'HOME' => home_dir, + 'PATH' => '/usr/bin', + 'LANG' => ENV['LANG'] + } + end + # Confiar en la primera llave que encontremos, fallar si cambian # # @return [Symbol] From ea198f185b655a353755b097cdbc6c264378db34 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:07:45 -0300 Subject: [PATCH 058/251] bajar el timeout para no bloquear el deploy --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 4191b20d..2387366a 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -27,7 +27,7 @@ class DeployRsync < Deploy user, host = user_host ssh_available = false - Net::SSH.start(host, user, verify_host_key: tofu) do |ssh| + Net::SSH.start(host, user, verify_host_key: tofu, timeout: 5) do |ssh| if values[:host_keys].blank? # Guardar las llaves que se encontraron en la primera conexión values[:host_keys] = ssh.transport.host_keys.map do |host_key| From c49c63f77661413a208a47bd7a80f7b4b8d49535 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:11:21 -0300 Subject: [PATCH 059/251] timeout de rsync --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 2387366a..5d360c4c 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -77,7 +77,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -av #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -av --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From 8edc9b460af993ad4c52b1192962cb548c39a82a Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:17:55 -0300 Subject: [PATCH 060/251] obtener listado de nodos de un lugar central --- app/models/sutty.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 app/models/sutty.rb diff --git a/app/models/sutty.rb b/app/models/sutty.rb new file mode 100644 index 00000000..7ec8432c --- /dev/null +++ b/app/models/sutty.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Configuración general de Sutty +class Sutty + extend self + + # Los nodos son otros servidores de Sutty hacia los que se sincronizan + # sitios. + # + # @return [Array] + def nodes + @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') + end +end From dcbd1c02ac32c1b183d6c90255da693ad67bbc9e Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:26:41 -0300 Subject: [PATCH 061/251] =?UTF-8?q?el=20m=C3=A9todo=20es=20p=C3=BAblico?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_rsync.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 5d360c4c..50719cd1 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -16,6 +16,13 @@ class DeployRsync < Deploy 0 end + # Devolver el destino o lanzar un error si no está configurado + def destination + values[:destination].tap do |d| + raise(ArgumentError, 'destination no está configurado') if d.blank? + end + end + private # Verificar la conexión SSH implementando Trust On First Use @@ -86,11 +93,4 @@ class DeployRsync < Deploy def source site.deploys.find_by(type: 'DeployLocal').destination end - - # Devolver el destino o lanzar un error si no está configurado - def destination - values[:destination].tap do |d| - raise(ArgumentError, 'destination no está configurado') if d.blank? - end - end end From 5fac827ca09fd46231490c59f845b6ec6a2e6121 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:27:15 -0300 Subject: [PATCH 062/251] registrar los cambios --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 50719cd1..7733406e 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -84,7 +84,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -av --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -avi --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From 20915443d84d7ddb50ef8417d957d2a2dbb2dbcd Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:27:52 -0300 Subject: [PATCH 063/251] deprecar las flags por ahora --- app/models/deploy_rsync.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 7733406e..12f4ce0e 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -3,7 +3,7 @@ # Sincroniza sitios a servidores remotos usando Rsync. El servidor # remoto tiene que tener rsync instalado. class DeployRsync < Deploy - store :values, accessors: %i[flags destination host_keys], coder: JSON + store :values, accessors: %i[destination host_keys], coder: JSON def deploy ssh? && rsync @@ -80,11 +80,11 @@ class DeployRsync < Deploy end end - # Sincroniza hacia el directorio remoto, usando las flags opcionales. + # Sincroniza hacia el directorio remoto # # @return [Boolean] def rsync - run %(rsync -avi --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -avi --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From a5e90257a30566ecd00fc36a63bf8fcb5e789528 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:38:03 -0300 Subject: [PATCH 064/251] generar un deploy rsync para el sitio con todos los nodos --- app/services/site_service.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 5e2fc706..2ffb8cdd 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -144,4 +144,11 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do PostService.new(site: site, usuarie: usuarie, post: post, params: params).update end + + # Crea los deploys necesarios para sincronizar a otros nodos de Sutty + def sync_nodes + Sutty.nodes.each do |node| + site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") + end + end end From 10d950689b614f6ca2a99ace3cf14dce44b3a037 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:39:52 -0300 Subject: [PATCH 065/251] crear rsyncs al crear el sitio --- app/services/site_service.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 2ffb8cdd..89c5796d 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -9,6 +9,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do self.site = Site.new params add_role temporal: false, rol: 'usuarie' + sync_nodes I18n.with_locale(usuarie&.lang&.to_sym || I18n.default_locale) do site.save && From a0134cc052578e2ec66025b3d51b2879fc144217 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 19:05:50 -0300 Subject: [PATCH 066/251] =?UTF-8?q?ya=20existe=20un=20lugar=20donde=20guar?= =?UTF-8?q?dar=20la=20configuraci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/sutty.rb | 14 -------------- app/services/site_service.rb | 2 +- config/application.rb | 4 ++++ 3 files changed, 5 insertions(+), 15 deletions(-) delete mode 100644 app/models/sutty.rb diff --git a/app/models/sutty.rb b/app/models/sutty.rb deleted file mode 100644 index 7ec8432c..00000000 --- a/app/models/sutty.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -# Configuración general de Sutty -class Sutty - extend self - - # Los nodos son otros servidores de Sutty hacia los que se sincronizan - # sitios. - # - # @return [Array] - def nodes - @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') - end -end diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 89c5796d..22423bb8 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -148,7 +148,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do # Crea los deploys necesarios para sincronizar a otros nodos de Sutty def sync_nodes - Sutty.nodes.each do |node| + Rails.application.nodes.each do |node| site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") end end diff --git a/config/application.rb b/config/application.rb index 7326ae0f..bc948936 100644 --- a/config/application.rb +++ b/config/application.rb @@ -49,5 +49,9 @@ module Sutty EmailAddress::Config.error_messages translations.transform_keys(&:to_s), locale.to_s end end + + def nodes + @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') + end end end From aa0c359554917ef93547d3446e55ac73394587a8 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 19:06:11 -0300 Subject: [PATCH 067/251] crear un rsync por cada sitio existente --- .../20220406211042_add_deploy_rsync_to_sites.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 db/migrate/20220406211042_add_deploy_rsync_to_sites.rb diff --git a/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb b/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb new file mode 100644 index 00000000..92b6f17b --- /dev/null +++ b/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# Agrega un DeployRsync hacia los servidores alternativos para cada +# sitio +class AddDeployRsyncToSites < ActiveRecord::Migration[6.1] + def up + Site.find_each do |site| + SiteService.new(site: site).send :sync_nodes + site.save + end + end + + def down + DeployRsync.destroy_all + end +end From f51c051af81da89345e66f137be419b00800ebe5 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 19:35:35 -0300 Subject: [PATCH 068/251] mensajes --- config/locales/en.yml | 4 ++++ config/locales/es.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/config/locales/en.yml b/config/locales/en.yml index b814796d..647f3ee8 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -100,6 +100,10 @@ en: title: Alternative domain name success: Success! error: Error + deploy_rsync: + title: Synchronize to backup server + success: Success! + error: Error help: You can contact us by replying to this e-mail maintenance_mailer: notice: diff --git a/config/locales/es.yml b/config/locales/es.yml index a6fbd407..1ccf5047 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -100,6 +100,10 @@ es: title: Dominio alternativo success: ¡Éxito! error: Hubo un error + deploy_rsync: + title: Sincronizar al servidor alternativo + success: ¡Éxito! + error: Hubo un error help: Por cualquier duda, responde este correo para contactarte con nosotres. maintenance_mailer: notice: From 87cdba9bec0a45ab48802c409830478ca045ff6e Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:07:14 -0300 Subject: [PATCH 069/251] =?UTF-8?q?enviar=20m=C3=A1s=20informaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/deploy_job.rb | 14 ++++++++++++-- app/models/deploy.rb | 4 ++++ app/models/deploy_alternative_domain.rb | 6 +++++- app/models/deploy_hidden_service.rb | 2 +- app/models/deploy_local.rb | 4 ++++ app/models/deploy_private.rb | 4 ++++ app/models/deploy_www.rb | 4 ++++ app/models/deploy_zip.rb | 4 ++++ app/views/deploy_mailer/deployed.html.haml | 9 +++++++-- app/views/deploy_mailer/deployed.text.haml | 7 +++---- config/locales/en.yml | 2 ++ config/locales/es.yml | 2 ++ 12 files changed, 52 insertions(+), 10 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 70997ce1..db91672b 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -27,7 +27,13 @@ class DeployJob < ApplicationJob @site.update status: 'building' # Asegurarse que DeployLocal sea el primero! - @deployed = { deploy_local: deploy_locally } + @deployed = { + deploy_local: { + status: deploy_locally, + seconds: deploy_local.build_stats.last.seconds, + url: site.url + } + } # No es opcional unless @deployed[:deploy_local] @@ -60,7 +66,11 @@ class DeployJob < ApplicationJob def deploy_others @site.deploys.where.not(type: 'DeployLocal').find_each do |d| - @deployed[d.type.underscore.to_sym] = d.deploy + @deployed[d.type.underscore.to_sym] = { + status: d.deploy, + seconds: d.build_stats.last.seconds, + url: d.url + } end end diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 3f034ad5..1aefe126 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -15,6 +15,10 @@ class Deploy < ApplicationRecord raise NotImplementedError end + def url + raise NotImplementedError + end + def limit raise NotImplementedError end diff --git a/app/models/deploy_alternative_domain.rb b/app/models/deploy_alternative_domain.rb index e4960e65..d245ac21 100644 --- a/app/models/deploy_alternative_domain.rb +++ b/app/models/deploy_alternative_domain.rb @@ -18,6 +18,10 @@ class DeployAlternativeDomain < Deploy end def destination - File.join(Rails.root, '_deploy', hostname.gsub(/\.\z/, '')) + @destination ||= File.join(Rails.root, '_deploy', hostname.gsub(/\.\z/, '')) + end + + def url + "https://#{File.basename destination}" end end diff --git a/app/models/deploy_hidden_service.rb b/app/models/deploy_hidden_service.rb index d4d2b822..8df46c2e 100644 --- a/app/models/deploy_hidden_service.rb +++ b/app/models/deploy_hidden_service.rb @@ -13,6 +13,6 @@ class DeployHiddenService < DeployWww end def url - 'http://' + fqdn + "http://#{fqdn}" end end diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 4fa588f5..b6fa407f 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -25,6 +25,10 @@ class DeployLocal < Deploy 1 end + def url + site.url + end + # Obtener el tamaño de todos los archivos y directorios (los # directorios son archivos :) def size diff --git a/app/models/deploy_private.rb b/app/models/deploy_private.rb index 3a6595f9..0d79811d 100644 --- a/app/models/deploy_private.rb +++ b/app/models/deploy_private.rb @@ -16,6 +16,10 @@ class DeployPrivate < DeployLocal File.join(Rails.root, '_private', site.name) end + def url + "#{ENV['PANEL_URL']}/sites/private/#{site.name}" + end + # No usar recursos en compresión y habilitar los datos privados def env @env ||= super.merge({ diff --git a/app/models/deploy_www.rb b/app/models/deploy_www.rb index 5602b0fc..3c9feb3b 100644 --- a/app/models/deploy_www.rb +++ b/app/models/deploy_www.rb @@ -27,6 +27,10 @@ class DeployWww < Deploy "www.#{site.hostname}" end + def url + "https://www.#{site.hostname}/" + end + private def remove_destination! diff --git a/app/models/deploy_zip.rb b/app/models/deploy_zip.rb index ec8973d1..0a66b602 100644 --- a/app/models/deploy_zip.rb +++ b/app/models/deploy_zip.rb @@ -49,6 +49,10 @@ class DeployZip < Deploy "#{site.hostname}.zip" end + def url + "#{site.url}#{file}" + end + def path File.join(destination, file) end diff --git a/app/views/deploy_mailer/deployed.html.haml b/app/views/deploy_mailer/deployed.html.haml index e8b2e7af..23e99469 100644 --- a/app/views/deploy_mailer/deployed.html.haml +++ b/app/views/deploy_mailer/deployed.html.haml @@ -8,10 +8,15 @@ %tr %th= t('.th.type') %th= t('.th.status') + %th= t('.th.url') + %th= t('.th.seconds') %tbody - - @deploys.each do |deploy, value| + - @deploys.each_pair do |deploy, value| %tr %td= t(".#{deploy}.title") - %td= value ? t(".#{deploy}.success") : t(".#{deploy}.error") + %td= value[:status] ? t(".#{deploy}.success") : t(".#{deploy}.error") + %td= link_to value[:url], value[:url] + %td + %time{ datetime: "PT#{value[:seconds]}S" }= distance_of_time_in_words value[:seconds].seconds = sanitize_markdown t('.help'), tags: %w[p a strong em] diff --git a/app/views/deploy_mailer/deployed.text.haml b/app/views/deploy_mailer/deployed.text.haml index 53a9b008..a3073a61 100644 --- a/app/views/deploy_mailer/deployed.text.haml +++ b/app/views/deploy_mailer/deployed.text.haml @@ -3,10 +3,9 @@ = t('.explanation', fqdn: @deploy_local.site.hostname) \ = Terminal::Table.new do |table| - - table << [t('.th.type'), t('.th.status')] + - table << [t('.th.type'), t('.th.status'), t('.th.url'), t('.th.seconds')] - table.add_separator - - @deploys.each do |deploy, value| - - table << [t(".#{deploy}.title"), - value ? t(".#{deploy}.success") : t(".#{deploy}.error")] + - @deploys.each_pair do |deploy, value| + - table << [ t(".#{deploy}.title"), value[:status] ? t(".#{deploy}.success") : t(".#{deploy}.error"), value[:url], distance_of_time_in_words(value[:seconds].seconds) ] \ = t('.help') diff --git a/config/locales/en.yml b/config/locales/en.yml index b814796d..c8df9150 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -76,6 +76,8 @@ en: th: type: Type status: Status + seconds: Duration + url: Address deploy_local: title: Build the site success: Success! diff --git a/config/locales/es.yml b/config/locales/es.yml index a6fbd407..e94f623a 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -76,6 +76,8 @@ es: th: type: Tipo status: Estado + seconds: Duración + url: Dirección deploy_local: title: Generar el sitio success: ¡Éxito! From c47b4f179cfa07b492edd3834c162508e2c198dc Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:16:31 -0300 Subject: [PATCH 070/251] =?UTF-8?q?fixup!=20enviar=20m=C3=A1s=20informaci?= =?UTF-8?q?=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/deploy_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index db91672b..79cfa134 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -31,7 +31,7 @@ class DeployJob < ApplicationJob deploy_local: { status: deploy_locally, seconds: deploy_local.build_stats.last.seconds, - url: site.url + url: deploy_local.url } } From 8c9bd6aa88c9c584896c9109d4c04aad5d9b1cfd Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:31:49 -0300 Subject: [PATCH 071/251] por ahora no es configurable --- app/views/deploys/_deploy_rsync.haml | 1 + 1 file changed, 1 insertion(+) create mode 100644 app/views/deploys/_deploy_rsync.haml diff --git a/app/views/deploys/_deploy_rsync.haml b/app/views/deploys/_deploy_rsync.haml new file mode 100644 index 00000000..0aab9802 --- /dev/null +++ b/app/views/deploys/_deploy_rsync.haml @@ -0,0 +1 @@ +-# nada From 92866704249f0fa8d8156557e5e8e8365bab5b79 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:42:40 -0300 Subject: [PATCH 072/251] mostrar espacio utilizado --- app/jobs/deploy_job.rb | 2 ++ app/views/deploy_mailer/deployed.html.haml | 2 ++ app/views/deploy_mailer/deployed.text.haml | 2 +- config/locales/en.yml | 1 + config/locales/es.yml | 1 + 5 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 79cfa134..38bb0429 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -31,6 +31,7 @@ class DeployJob < ApplicationJob deploy_local: { status: deploy_locally, seconds: deploy_local.build_stats.last.seconds, + size: deploy_local.build_stats.last.size, url: deploy_local.url } } @@ -69,6 +70,7 @@ class DeployJob < ApplicationJob @deployed[d.type.underscore.to_sym] = { status: d.deploy, seconds: d.build_stats.last.seconds, + size: d.build_stats.last.size, url: d.url } end diff --git a/app/views/deploy_mailer/deployed.html.haml b/app/views/deploy_mailer/deployed.html.haml index 23e99469..0053accf 100644 --- a/app/views/deploy_mailer/deployed.html.haml +++ b/app/views/deploy_mailer/deployed.html.haml @@ -10,6 +10,7 @@ %th= t('.th.status') %th= t('.th.url') %th= t('.th.seconds') + %th= t('.th.size') %tbody - @deploys.each_pair do |deploy, value| %tr @@ -18,5 +19,6 @@ %td= link_to value[:url], value[:url] %td %time{ datetime: "PT#{value[:seconds]}S" }= distance_of_time_in_words value[:seconds].seconds + %td= number_to_human_size value[:size], precision: 2 = sanitize_markdown t('.help'), tags: %w[p a strong em] diff --git a/app/views/deploy_mailer/deployed.text.haml b/app/views/deploy_mailer/deployed.text.haml index a3073a61..0188c303 100644 --- a/app/views/deploy_mailer/deployed.text.haml +++ b/app/views/deploy_mailer/deployed.text.haml @@ -6,6 +6,6 @@ - table << [t('.th.type'), t('.th.status'), t('.th.url'), t('.th.seconds')] - table.add_separator - @deploys.each_pair do |deploy, value| - - table << [ t(".#{deploy}.title"), value[:status] ? t(".#{deploy}.success") : t(".#{deploy}.error"), value[:url], distance_of_time_in_words(value[:seconds].seconds) ] + - table << [t(".#{deploy}.title"), value[:status] ? t(".#{deploy}.success") : t(".#{deploy}.error"), value[:url], distance_of_time_in_words(value[:seconds].seconds), number_to_human_size(value[:size], precision: 2)] \ = t('.help') diff --git a/config/locales/en.yml b/config/locales/en.yml index c8df9150..75fbcc94 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -77,6 +77,7 @@ en: type: Type status: Status seconds: Duration + size: Space used url: Address deploy_local: title: Build the site diff --git a/config/locales/es.yml b/config/locales/es.yml index e94f623a..73467149 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -77,6 +77,7 @@ es: type: Tipo status: Estado seconds: Duración + size: Espacio ocupado url: Dirección deploy_local: title: Generar el sitio From 35e41b729ad35e86b1b4d4ce63ba2d4dc0ad97d6 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:44:46 -0300 Subject: [PATCH 073/251] el espacio ocupado es el mismo que el local --- app/models/deploy_rsync.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 12f4ce0e..c19d279b 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -9,11 +9,11 @@ class DeployRsync < Deploy ssh? && rsync end - # No se ocupa espacio local + # El espacio remoto es el mismo que el local # # @return [Integer] def size - 0 + deploy_local.build_stats.last.size end # Devolver el destino o lanzar un error si no está configurado @@ -91,6 +91,10 @@ class DeployRsync < Deploy # # @return [String] def source - site.deploys.find_by(type: 'DeployLocal').destination + deploy_local.destination + end + + def deploy_local + @deploy_local ||= site.deploys.find_by(type: 'DeployLocal') end end From 9024a13490288013f852a57e2a546161d9e47420 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:48:14 -0300 Subject: [PATCH 074/251] algunos deploys no generan build_stats --- app/jobs/deploy_job.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 38bb0429..97438650 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -67,10 +67,13 @@ class DeployJob < ApplicationJob def deploy_others @site.deploys.where.not(type: 'DeployLocal').find_each do |d| + status = d.deploy + build_stat = d.build_stats.last + @deployed[d.type.underscore.to_sym] = { - status: d.deploy, - seconds: d.build_stats.last.seconds, - size: d.build_stats.last.size, + status: status, + seconds: build_stat.try(:seconds) || 0, + size: build_stat.try(:size) || 0, url: d.url } end From ef2589a4f162552d7903f98005cb43303fbdcf65 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:51:45 -0300 Subject: [PATCH 075/251] =?UTF-8?q?no=20es=20necesario=20traer=20el=20tama?= =?UTF-8?q?=C3=B1o=20de=20los=20build=5Fstats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/deploy_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 97438650..ae6cb279 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -31,7 +31,7 @@ class DeployJob < ApplicationJob deploy_local: { status: deploy_locally, seconds: deploy_local.build_stats.last.seconds, - size: deploy_local.build_stats.last.size, + size: deploy_local.size, url: deploy_local.url } } @@ -73,7 +73,7 @@ class DeployJob < ApplicationJob @deployed[d.type.underscore.to_sym] = { status: status, seconds: build_stat.try(:seconds) || 0, - size: build_stat.try(:size) || 0, + size: d.size, url: d.url } end From ea25d27f240cacad4fac4a1bf97455aa03c4c8f5 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:53:45 -0300 Subject: [PATCH 076/251] =?UTF-8?q?calcular=20el=20tama=C3=B1o=20a=20parti?= =?UTF-8?q?r=20de=20deploy=20local?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index c19d279b..b2ffba7c 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -13,7 +13,7 @@ class DeployRsync < Deploy # # @return [Integer] def size - deploy_local.build_stats.last.size + deploy_local.size end # Devolver el destino o lanzar un error si no está configurado From bb7c50e24b4fb9a9ce16666d53a77fc666a22506 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:54:35 -0300 Subject: [PATCH 077/251] calcular el espacio una sola vez --- app/models/deploy_local.rb | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 4fa588f5..1b661059 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -28,15 +28,17 @@ class DeployLocal < Deploy # Obtener el tamaño de todos los archivos y directorios (los # directorios son archivos :) def size - paths = [destination, File.join(destination, '**', '**')] + @size ||= begin + paths = [destination, File.join(destination, '**', '**')] - Dir.glob(paths).map do |file| - if File.symlink? file - 0 - else - File.size(file) - end - end.inject(:+) + Dir.glob(paths).map do |file| + if File.symlink? file + 0 + else + File.size(file) + end + end.inject(:+) + end end def destination From 106029b48f4bba2576e92928dfb9cba857d99de3 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 7 Apr 2022 13:34:12 -0300 Subject: [PATCH 078/251] header --- app/views/deploy_mailer/deployed.text.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/deploy_mailer/deployed.text.haml b/app/views/deploy_mailer/deployed.text.haml index 0188c303..48e8000c 100644 --- a/app/views/deploy_mailer/deployed.text.haml +++ b/app/views/deploy_mailer/deployed.text.haml @@ -3,7 +3,7 @@ = t('.explanation', fqdn: @deploy_local.site.hostname) \ = Terminal::Table.new do |table| - - table << [t('.th.type'), t('.th.status'), t('.th.url'), t('.th.seconds')] + - table << [t('.th.type'), t('.th.status'), t('.th.url'), t('.th.seconds'), t('.th.size')] - table.add_separator - @deploys.each_pair do |deploy, value| - table << [t(".#{deploy}.title"), value[:status] ? t(".#{deploy}.success") : t(".#{deploy}.error"), value[:url], distance_of_time_in_words(value[:seconds].seconds), number_to_human_size(value[:size], precision: 2)] From 78c0bdcc3930c5d6035682ecc2982498f88cd0d6 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Apr 2022 10:39:44 -0300 Subject: [PATCH 079/251] cada deploy puede ejecutar tareas de limpieza MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pero la mayoría no lo va a necesitar --- app/models/deploy.rb | 3 +++ app/models/deploy_local.rb | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/app/models/deploy.rb b/app/models/deploy.rb index 3f034ad5..42bfd345 100644 --- a/app/models/deploy.rb +++ b/app/models/deploy.rb @@ -23,6 +23,9 @@ class Deploy < ApplicationRecord raise NotImplementedError end + # Realizar tareas de limpieza. + def cleanup!; end + def time_start @start = Time.now end diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 4fa588f5..66412df2 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -43,6 +43,17 @@ class DeployLocal < Deploy File.join(Rails.root, '_deploy', site.hostname) end + # Libera espacio eliminando archivos temporales + # + # @return [nil] + def cleanup! + FileUtils.rm_rf(gems_dir) + FileUtils.rm_rf(yarn_cache_dir) + FileUtils.rm_rf(File.join(site.path, 'node_modules')) + FileUtils.rm_rf(File.join(site.path, '.sass-cache')) + FileUtils.rm_rf(File.join(site.path, '.jekyll-cache')) + end + private def mkdir From 1e0fb95825b7aed00d7a5c77512e7cb6c1efa770 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Apr 2022 10:40:18 -0300 Subject: [PATCH 080/251] =?UTF-8?q?correr=20la=20recolecci=C3=B3n=20de=20b?= =?UTF-8?q?asura=20de=20git?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/site/repository.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/app/models/site/repository.rb b/app/models/site/repository.rb index 74db2549..f63288d4 100644 --- a/app/models/site/repository.rb +++ b/app/models/site/repository.rb @@ -147,6 +147,23 @@ class Site rugged.index.remove(relativize(file)) end + # Garbage collection + # + # @return [Boolean] + def gc + env = { 'PATH' => '/usr/bin', 'LANG' => ENV['LANG'], 'HOME' => path } + cmd = 'git gc' + + r = nil + Dir.chdir(path) do + Open3.popen2e(env, cmd, unsetenv_others: true) do |_, _, t| + r = t.value + end + end + + r&.success? + end + private # Si Sutty tiene una llave privada de tipo ED25519, devuelve las From 9fefc0f554e7142de3c70efac2a3e66d53b61782 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Apr 2022 10:41:22 -0300 Subject: [PATCH 081/251] abl --- app/services/cleanup_service.rb | 47 +++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 app/services/cleanup_service.rb diff --git a/app/services/cleanup_service.rb b/app/services/cleanup_service.rb new file mode 100644 index 00000000..ad87cf9a --- /dev/null +++ b/app/services/cleanup_service.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +# Realiza tareas de limpieza en todos los sitios, para optimizar y +# liberar espacio. +class CleanupService + # Días de antigüedad de los sitios + attr_reader :before + + # @param :before [ActiveSupport::TimeWithZone] Cuánto tiempo lleva sin usarse un sitio. + def initialize(before: 30.days.ago) + @before = before + end + + # Limpieza general + # + # @return [nil] + def cleanup_everything! + cleanup_older_sites! + cleanup_newer_sites! + end + + # Encuentra todos los sitios sin actualizar y realiza limpieza. + # + # @return [nil] + def cleanup_older_sites! + Site.where('updated_at < ?', before).find_each do |site| + next unless File.directory? site.path + + site.deploys.find_each(&:cleanup!) + + site.repository.gc + site.touch + end + end + + # Tareas para los sitios en uso + # + # @return [nil] + def cleanup_newer_sites! + Site.where('updated_at >= ?', before).find_each do |site| + next unless File.directory? site.path + + site.repository.gc + site.touch + end + end +end From 170cbeb69eeddde6e6176a7dcd8f5dbdb13fe7d2 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Apr 2022 10:41:53 -0300 Subject: [PATCH 082/251] poder hacer limpieza desde la terminal --- lib/tasks/cleanup.rake | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 lib/tasks/cleanup.rake diff --git a/lib/tasks/cleanup.rake b/lib/tasks/cleanup.rake new file mode 100644 index 00000000..e14693bc --- /dev/null +++ b/lib/tasks/cleanup.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +namespace :cleanup do + desc 'Cleanup sites' + task everything: :environment do + before = ENV.fetch('BEFORE', '30').to_i.days.ago + service = CleanupService.new(before: before) + + service.cleanup_everything! + end +end From 3b616198eaf64f122e1a2684ed0e2d345ed38d1e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Apr 2022 10:57:51 -0300 Subject: [PATCH 083/251] realizar la limpieza mensualmente --- .dockerignore | 1 + Dockerfile | 2 ++ Procfile | 8 +------- monit.conf | 30 ++++-------------------------- 4 files changed, 8 insertions(+), 33 deletions(-) diff --git a/.dockerignore b/.dockerignore index afe4e8d7..7b84d429 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,4 @@ * # Solo agregar lo que usamos en COPY # !./archivo +!./monit.conf diff --git a/Dockerfile b/Dockerfile index ecf43cbc..342c2750 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,8 @@ RUN apk add --no-cache libxslt libxml2 postgresql-libs libssh2 \ RUN gem install --no-document --no-user-install foreman RUN wget https://github.com/jgm/pandoc/releases/download/${PANDOC_VERSION}/pandoc-${PANDOC_VERSION}-linux-amd64.tar.gz -O - | tar --strip-components 1 -xvzf - pandoc-${PANDOC_VERSION}/bin/pandoc && mv /bin/pandoc /usr/bin/pandoc +COPY ./monit.conf /etc/monit.d/sutty.conf + VOLUME "/srv" EXPOSE 3000 diff --git a/Procfile b/Procfile index b308ffd5..48931e8d 100644 --- a/Procfile +++ b/Procfile @@ -1,7 +1 @@ -migrate: bundle exec rake db:prepare db:seed -sutty: bundle exec puma config.ru -blazer_5m: bundle exec rake blazer:run_checks SCHEDULE="5 minutes" -blazer_1h: bundle exec rake blazer:run_checks SCHEDULE="1 hour" -blazer_1d: bundle exec rake blazer:run_checks SCHEDULE="1 day" -blazer: bundle exec rake blazer:send_failing_checks -prometheus: bundle exec prometheus_exporter -b 0.0.0.0 --prefix "sutty_" +cleanup: rake cleanup:everything diff --git a/monit.conf b/monit.conf index 96c08d8a..07cd1c1e 100644 --- a/monit.conf +++ b/monit.conf @@ -1,27 +1,5 @@ -check process sutty with pidfile /srv/tmp/puma.pid - start program = "/usr/local/bin/sutty start" - stop program = "/usr/local/bin/sutty stop" - -check process prometheus with pidfile /tmp/prometheus.pid - start program = "/usr/local/bin/sutty prometheus start" - stop program = "/usr/local/bin/sutty prometheus start" - -check program blazer_5m - with path "/usr/local/bin/sutty blazer 5m" - every 5 cycles - if status != 0 then alert - -check program blazer_1h - with path "/usr/local/bin/sutty blazer 1h" - every 60 cycles - if status != 0 then alert - -check program blazer_1d - with path "/usr/local/bin/sutty blazer 1d" - every 1440 cycles - if status != 0 then alert - -check program blazer - with path "/usr/local/bin/sutty blazer" - every 61 cycles +# Limpiar mensualmente +check program cleanup + with path "/usr/bin/foreman run -f /srv/Procfile -d /srv " as uid "rails" gid "http-data" + every "0 3 1 * *" if status != 0 then alert From 13561a5f717af796371a244daf11b2e7d3d029d7 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 19 Apr 2022 10:07:10 -0300 Subject: [PATCH 084/251] =?UTF-8?q?no=20generar=20errores=20si=20se=20env?= =?UTF-8?q?=C3=ADa=20el=20reporte=20incompleto?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #5136 --- app/controllers/api/v1/csp_reports_controller.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/v1/csp_reports_controller.rb b/app/controllers/api/v1/csp_reports_controller.rb index bc6cfae0..f1d7a376 100644 --- a/app/controllers/api/v1/csp_reports_controller.rb +++ b/app/controllers/api/v1/csp_reports_controller.rb @@ -6,6 +6,9 @@ module Api class CspReportsController < BaseController skip_forgery_protection + # No queremos indicar que algo salió mal + rescue_from ActionController::ParameterMissing, with: :csp_report_created + # Crea un reporte de CSP intercambiando los guiones medios por # bajos # @@ -18,7 +21,7 @@ module Api csp.id = SecureRandom.uuid csp.save - render json: {}, status: :created + csp_report_created end private @@ -39,6 +42,10 @@ module Api :'column-number', :'source-file') end + + def csp_report_created + render json: {}, status: :created + end end end end From 52f446fcf1e8fc9c31a1ee9ef1028f320bb24304 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 19 Apr 2022 10:57:01 -0300 Subject: [PATCH 085/251] refactorizacion --- app/controllers/api/v1/csp_reports_controller.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/v1/csp_reports_controller.rb b/app/controllers/api/v1/csp_reports_controller.rb index f1d7a376..ea186729 100644 --- a/app/controllers/api/v1/csp_reports_controller.rb +++ b/app/controllers/api/v1/csp_reports_controller.rb @@ -14,9 +14,9 @@ module Api # # TODO: Aplicar rate_limit def create - csp = CspReport.new(csp_report_params.to_h.map do |k, v| - [k.tr('-', '_'), v] - end.to_h) + csp = CspReport.new(csp_report_params.to_h.transform_keys do |k| + k.tr('-', '_') + end) csp.id = SecureRandom.uuid csp.save From ed3f15391982e93c2902c2a33582c0ce063c5c35 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:06:00 -0300 Subject: [PATCH 086/251] =?UTF-8?q?es=20una=20tarea=20peri=C3=B3dica?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 2aa8d702..16f04738 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -2,7 +2,7 @@ # Genera resúmenes de información para poder mostrar estadísticas y se # corre regularmente a sí misma. -class StatCollectionJob < ApplicationJob +class StatCollectionJob < PeriodicJob STAT_NAME = 'stat_collection_job' def perform(site_id:, once: true) From 30861d240ea08e415c709c639d9b8cc87495b239 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:06:17 -0300 Subject: [PATCH 087/251] =?UTF-8?q?aplicar=20la=20operaci=C3=B3n!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 16f04738..e70544c3 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -44,7 +44,7 @@ class StatCollectionJob < PeriodicJob .where_dimensions(site_id: site.id) .group("dimensions->'site_id'") .rollup(name, interval: interval, update: true) do |rollup| - rollup.try(:operation, :value) + rollup.try(operation, :value) end end From 22742cf058cef5ebf2d562b4e91ee6da2abe833f Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:06:55 -0300 Subject: [PATCH 088/251] migraciones faltantes --- db/migrate/20211022224008_add_site_to_stats.rb | 8 ++++++++ db/migrate/20211022225449_add_name_to_stats.rb | 9 +++++++++ 2 files changed, 17 insertions(+) create mode 100644 db/migrate/20211022224008_add_site_to_stats.rb create mode 100644 db/migrate/20211022225449_add_name_to_stats.rb diff --git a/db/migrate/20211022224008_add_site_to_stats.rb b/db/migrate/20211022224008_add_site_to_stats.rb new file mode 100644 index 00000000..db2b43ab --- /dev/null +++ b/db/migrate/20211022224008_add_site_to_stats.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# La recolección de estadísticas podría pertenecer a un sitio +class AddSiteToStats < ActiveRecord::Migration[6.1] + def change + add_belongs_to :stats, :site, index: true, null: true + end +end diff --git a/db/migrate/20211022225449_add_name_to_stats.rb b/db/migrate/20211022225449_add_name_to_stats.rb new file mode 100644 index 00000000..89a17ee0 --- /dev/null +++ b/db/migrate/20211022225449_add_name_to_stats.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Agregarle un nombre a la estadística +class AddNameToStats < ActiveRecord::Migration[6.1] + def change + add_column :stats, :name, :string, null: false + add_index :stats, :name, using: 'hash' + end +end From 9b2194ee950354573fdef604a49cdafb8141d0e7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:24:06 -0300 Subject: [PATCH 089/251] detener la tarea --- app/jobs/periodic_job.rb | 8 ++++++++ app/jobs/uri_collection_job.rb | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/jobs/periodic_job.rb b/app/jobs/periodic_job.rb index 8d9453a3..2f60a2b3 100644 --- a/app/jobs/periodic_job.rb +++ b/app/jobs/periodic_job.rb @@ -52,4 +52,12 @@ class PeriodicJob < ApplicationJob def beginning_of_interval @beginning_of_interval ||= last_stat.created_at.try(:"beginning_of_#{starting_interval}") end + + def stop_file + @stop_file ||= Rails.root.join('tmp', self.class.to_s.tableize) + end + + def stop? + File.exist? stop_file + end end diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 9ec333cd..b66a804d 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -21,7 +21,7 @@ class UriCollectionJob < PeriodicJob hostnames.each do |hostname| uris.each do |uri| - return if File.exist? Rails.root.join('tmp', 'uri_collection_job_stop') + return if stop? AccessLog.where(host: hostname, uri: uri) .where('created_at >= ?', beginning_of_interval) From ef32a6d7f6e94b266f19c35ea565db063eb3ced7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:30:03 -0300 Subject: [PATCH 090/251] schema --- db/schema.rb | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index 107e7be7..a395329d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_14_165639) do +ActiveRecord::Schema.define(version: 2021_10_22_225449) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -64,6 +64,7 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.string "remote_user" t.boolean "crawler", default: false t.string "http_referer" + t.datetime "created_at", precision: 6 t.index ["geoip2_data_city_name"], name: "index_access_logs_on_geoip2_data_city_name" t.index ["geoip2_data_country_name"], name: "index_access_logs_on_geoip2_data_country_name" t.index ["host"], name: "index_access_logs_on_host" @@ -303,6 +304,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.index ["usuarie_id"], name: "index_roles_on_usuarie_id" end + create_table "rollups", force: :cascade do |t| + t.string "name", null: false + t.string "interval", null: false + t.datetime "time", null: false + t.jsonb "dimensions", default: {}, null: false + t.float "value" + t.index ["name", "interval", "time", "dimensions"], name: "index_rollups_on_name_and_interval_and_time_and_dimensions", unique: true + end + create_table "sites", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -324,6 +334,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.index ["name"], name: "index_sites_on_name", unique: true end + create_table "stats", force: :cascade do |t| + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.bigint "site_id" + t.string "name", null: false + t.index ["name"], name: "index_stats_on_name", using: :hash + t.index ["site_id"], name: "index_stats_on_site_id" + end + create_table "usuaries", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -370,4 +389,10 @@ new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig, SQL_ACTIONS end + create_trigger("access_logs_before_insert_row_tr", :compatibility => 1). + on("access_logs"). + before(:insert) do + "new.created_at := to_timestamp(new.msec);" + end + end From cd1263a67211d103ccaab46b1bfe89cdc97ed536 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:46:24 -0300 Subject: [PATCH 091/251] usar fork por ahora --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 2b304ee0..1e476dde 100644 --- a/Gemfile +++ b/Gemfile @@ -64,7 +64,7 @@ gem 'rails-i18n' gem 'rails_warden' gem 'redis', require: %w[redis redis/connection/hiredis] gem 'redis-rails' -gem 'rollups', git: 'https://github.com/ankane/rollup.git', branch: 'master' +gem 'rollups', git: 'https://github.com/fauno/rollup.git', branch: 'update' gem 'rubyzip' gem 'rugged' gem 'concurrent-ruby-ext' From f96c7aff7d7dde7e3c874fb7f7b78e16a11c198f Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 16:28:32 -0300 Subject: [PATCH 092/251] =?UTF-8?q?volcar=20y=20eliminar=20los=20logs=20de?= =?UTF-8?q?l=20d=C3=ADa=20anterior?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/access_logs | 12 ++++++++++++ monit.conf | 5 +++++ 2 files changed, 17 insertions(+) create mode 100755 bin/access_logs diff --git a/bin/access_logs b/bin/access_logs new file mode 100755 index 00000000..2b4fd99f --- /dev/null +++ b/bin/access_logs @@ -0,0 +1,12 @@ +#!/bin/sh +set -e + +# Volcar y eliminar todos los access logs del día anterior +date=`date +%F` + +psql -h postgresql "${DATABASE:-sutty}" sutty < "/srv/http/_storage/${date}.psql.gz" +begin; +copy (select * from access_logs where created_at < '${date}') to stdout; +delete from access_logs where created_at < '${date}'; +commit; +SQL diff --git a/monit.conf b/monit.conf index f574c56d..27605d7c 100644 --- a/monit.conf +++ b/monit.conf @@ -29,3 +29,8 @@ check program blazer as uid "app" and gid "www-data" every 61 cycles if status != 0 then alert + +check program access_logs + with path "/srv/http/bin/access_logs" as uid "app" and gid "www-data" + every "0 0 * * *" + if status != 0 then alert From 83b9325d0c76e74a5cf525df45eb67b660f0246e Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 17:50:36 -0300 Subject: [PATCH 093/251] =?UTF-8?q?grabar=20la=20duraci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index b66a804d..7ca9a4d9 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -19,9 +19,12 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + # Recordar la última vez que se corrió la tarea + stat = site.stats.create! name: STAT_NAME + hostnames.each do |hostname| uris.each do |uri| - return if stop? + next if stop? AccessLog.where(host: hostname, uri: uri) .where('created_at >= ?', beginning_of_interval) @@ -46,8 +49,7 @@ class UriCollectionJob < PeriodicJob end end - # Recordar la última vez que se corrió la tarea - site.stats.create! name: STAT_NAME + stat.touch run_again! unless once end From 0cd71ad779f05923de7f1c94cc3a89e880c9aa35 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 17:54:36 -0300 Subject: [PATCH 094/251] eliminar blazer de las rutas por sitio --- config/routes.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/routes.rb b/config/routes.rb index c323f556..8bab18af 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -36,9 +36,6 @@ Rails.application.routes.draw do match '/api/v3/projects/:site_id/notices' => 'api/v1/notices#create', via: %i[post] resources :sites, constraints: { site_id: %r{[^/]+}, id: %r{[^/]+} } do - # Usar Blazer para mostrar estadísticas - mount Blazer::Engine, at: 'stats', as: 'stats' - # Gestionar actualizaciones del sitio get 'pull', to: 'sites#fetch' post 'pull', to: 'sites#merge' From 3edf45e27fcdd51fdf64730bc856fcf021224dc2 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 18:04:51 -0300 Subject: [PATCH 095/251] fixup! usar fork por ahora --- Gemfile.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8df2d77e..87812726 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,15 +6,6 @@ GIT rails (>= 3.0) rake (>= 0.8.7) -GIT - remote: https://github.com/ankane/rollup.git - revision: 0ab6c603450175eb1004f7793e86486943cb9f72 - branch: master - specs: - rollups (0.1.3) - activesupport (>= 5.1) - groupdate (>= 5.2) - GIT remote: https://github.com/fauno/email_address revision: 536b51f7071b68a55140c0c1726b4cd401d1c04d @@ -24,6 +15,15 @@ GIT netaddr (>= 2.0.4, < 3) simpleidn +GIT + remote: https://github.com/fauno/rollup.git + revision: ddbb345aa57e63b4cfdf7557267efa89ba60caac + branch: update + specs: + rollups (0.1.3) + activesupport (>= 5.1) + groupdate (>= 5.2) + GEM remote: https://gems.sutty.nl/ specs: @@ -214,8 +214,8 @@ GEM ffi (~> 1.0) globalid (0.6.0) activesupport (>= 5.0) - groupdate (5.2.2) - activesupport (>= 5) + groupdate (6.1.0) + activesupport (>= 5.2) hairtrigger (0.2.24) activerecord (>= 5.0, < 7) ruby2ruby (~> 2.4) From bfdd90088f76fb88560f9745218eb60e1c43ceed Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 18:29:18 -0300 Subject: [PATCH 096/251] permisos --- app/models/site_stat.rb | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 app/models/site_stat.rb diff --git a/app/models/site_stat.rb b/app/models/site_stat.rb new file mode 100644 index 00000000..73503aca --- /dev/null +++ b/app/models/site_stat.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +SiteStat = Struct.new(:site) From 0c34613cb579712045696c5d6a213c02491ddf5a Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 18:56:34 -0300 Subject: [PATCH 097/251] encontrar todos los idiomas --- app/jobs/uri_collection_job.rb | 50 ++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 7ca9a4d9..ada61bab 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -13,7 +13,7 @@ class UriCollectionJob < PeriodicJob # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las # estadísticas. - IMAGES = %w[.png .jpg .jpeg .gif .webp].freeze + IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze STAT_NAME = 'uri_collection_job' def perform(site_id:, once: true) @@ -91,18 +91,46 @@ class UriCollectionJob < PeriodicJob # # @return [Array] def uris - @uris ||= Dir.chdir destination do - (Dir.glob('**/*.html') + Dir.glob('public/**/*').reject do |p| - File.directory? p - end.reject do |p| - p = p.downcase + @uris ||= + locales.map do |locale| + uri = "/#{locale}/".squeeze('/') + dir = File.join(destination, locale) - IMAGES.any? do |i| - p.end_with? i + files(dir).map do |f| + uri + f end - end).map do |uri| - "/#{uri}" - end + end.flatten(2) + end + + # @return [Array] + def locales + @locales ||= ['', site.locales.map(&:to_s)].flatten(1) + end + + # @param :dir [String] + # @return [Array] + def files(dir) + Dir.chdir(dir) do + pages = Dir.glob('**/*.html') + files = Dir.glob('public/**/*') + files = remove_directories files + files = remove_images files + + [pages, files].flatten(1) + end + end + + # @param :files [Array] + # @return [Array] + def remove_directories(files) + files.reject do |f| + File.directory? f + end + end + + def remove_images(files) + files.reject do |f| + IMAGES.include? File.extname(f).downcase end end end From f94ece78cfe9c9cbf1a2ae65effed69b9000acff Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 19:06:17 -0300 Subject: [PATCH 098/251] optimizar sql usando indices --- app/jobs/uri_collection_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ada61bab..1bc6f768 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -37,6 +37,7 @@ class UriCollectionJob < PeriodicJob # intervalo más amplio. Stat::INTERVALS.reduce do |previous, current| Rollup.where(name: 'host|uri', interval: previous) + .where('time >= ?', beginning_of_interval) .where_dimensions(host: hostname, uri: uri) .group("dimensions->'host'", "dimensions->'uri'") .rollup('host|uri', interval: current, update: true) do |rollup| From 7cbc48d45b139c25656d8c0d8315d513d133181c Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 19:24:06 -0300 Subject: [PATCH 099/251] actualizar todo el intervalo --- app/jobs/uri_collection_job.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 1bc6f768..4d369c2d 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -36,8 +36,10 @@ class UriCollectionJob < PeriodicJob # Reducir las estadísticas calculadas aplicando un rollup sobre el # intervalo más amplio. Stat::INTERVALS.reduce do |previous, current| + beginning_of_this_interval = beginning_of_interval.try(:"beginning_of_#{current}") + Rollup.where(name: 'host|uri', interval: previous) - .where('time >= ?', beginning_of_interval) + .where('time >= ?', beginning_of_this_interval) .where_dimensions(host: hostname, uri: uri) .group("dimensions->'host'", "dimensions->'uri'") .rollup('host|uri', interval: current, update: true) do |rollup| From d8060728be781f074d9a8442408d651ef44d570c Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 19:32:55 -0300 Subject: [PATCH 100/251] =?UTF-8?q?mantener=20el=20d=C3=ADa=20anterior=20c?= =?UTF-8?q?ompleto=20en=20la=20base=20de=20datos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/access_logs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/access_logs b/bin/access_logs index 2b4fd99f..a7c151cc 100755 --- a/bin/access_logs +++ b/bin/access_logs @@ -1,8 +1,8 @@ #!/bin/sh set -e -# Volcar y eliminar todos los access logs del día anterior -date=`date +%F` +# Volcar y eliminar todos los access logs de dos días atrás +date=`bundle exec rails runner "puts (Date.today - 2.days)"` psql -h postgresql "${DATABASE:-sutty}" sutty < "/srv/http/_storage/${date}.psql.gz" begin; From 6b191d2d91541df6ef40e36a354e3a3a071b3ef5 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 19:37:08 -0300 Subject: [PATCH 101/251] =?UTF-8?q?usar=20la=20misma=20optimizaci=C3=B3n?= =?UTF-8?q?=20para=20stats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index e70544c3..3b68b61e 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -41,6 +41,7 @@ class StatCollectionJob < PeriodicJob # @return [NilClass] def rollup(name:, interval_previous:, interval:, operation: :sum) Rollup.where(name: name, interval: interval_previous) + .where('time >= ?', beginning_of_interval.try(:"beginning_of_#{interval}")) .where_dimensions(site_id: site.id) .group("dimensions->'site_id'") .rollup(name, interval: interval, update: true) do |rollup| From aad5c82cb7b7de9b673d2f4d57b41c3d39bef961 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 20:32:20 -0300 Subject: [PATCH 102/251] refactorizar rollups recursivos --- app/jobs/concerns/recursive_rollup.rb | 38 +++++++++++++ app/jobs/stat_collection_job.rb | 24 +++----- app/jobs/uri_collection_job.rb | 82 +++++++++++++++++++-------- 3 files changed, 102 insertions(+), 42 deletions(-) create mode 100644 app/jobs/concerns/recursive_rollup.rb diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb new file mode 100644 index 00000000..3163558e --- /dev/null +++ b/app/jobs/concerns/recursive_rollup.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module RecursiveRollup + extend ActiveSupport::Concern + + included do + private + + # Genera un rollup recursivo en base al período anterior y aplica una + # operación. + # + # @param :name [String] + # @param :interval_previous [String] + # @param :interval [String] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @param :new_name [String] + # @param :beginning [Time] + # @return [Rollup] + def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum, new_name: nil) + Rollup.where(name: name, interval: interval_previous) + .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) + .where_dimensions(**dimensions) + .group(*dimensions_to_jsonb_query(dimensions)) + .rollup(new_name || name, interval: interval, update: true) do |rollup| + rollup.try(operation, :value) + end + end + + # @param :dimensions [Hash] + # @return [Array] + def dimensions_to_jsonb_query(dimensions) + dimensions.keys.map do |key| + "dimensions->'#{key}'" + end + end + end +end diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 3b68b61e..c94320d5 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -3,6 +3,8 @@ # Genera resúmenes de información para poder mostrar estadísticas y se # corre regularmente a sí misma. class StatCollectionJob < PeriodicJob + include RecursiveRollup + STAT_NAME = 'stat_collection_job' def perform(site_id:, once: true) @@ -20,9 +22,11 @@ class StatCollectionJob < PeriodicJob # XXX: Es correcto promediar promedios? Stat::INTERVALS.reduce do |previous, current| - rollup(name: 'builds', interval_previous: previous, interval: current) - rollup(name: 'space_used', interval_previous: previous, interval: current, operation: :average) - rollup(name: 'build_time', interval_previous: previous, interval: current, operation: :average) + opts = { interval_previous: previous, interval: current, dimensions: { site_id: site.id } } + + recursive_rollup(name: 'builds', **opts) + recursive_rollup(name: 'space_used', operation: :average, **opts) + recursive_rollup(name: 'build_time', operation: :average, **opts) current end @@ -35,20 +39,6 @@ class StatCollectionJob < PeriodicJob private - # Genera un rollup recursivo en base al período anterior y aplica una - # operación. - # - # @return [NilClass] - def rollup(name:, interval_previous:, interval:, operation: :sum) - Rollup.where(name: name, interval: interval_previous) - .where('time >= ?', beginning_of_interval.try(:"beginning_of_#{interval}")) - .where_dimensions(site_id: site.id) - .group("dimensions->'site_id'") - .rollup(name, interval: interval, update: true) do |rollup| - rollup.try(operation, :value) - end - end - # Los registros a procesar # # @return [ActiveRecord::Relation] diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 4d369c2d..cad05cbb 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -11,6 +11,8 @@ # Los hostnames de un sitio van a poder obtenerse a partir de # Site#hostnames con la garantía de que son únicos. class UriCollectionJob < PeriodicJob + include RecursiveRollup + # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las # estadísticas. IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze @@ -21,35 +23,19 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME + name = 'host|uri' + beginning = beginning_of_interval + + hostnames.each do |host| + break if stop? - hostnames.each do |hostname| uris.each do |uri| - next if stop? + break if stop? - AccessLog.where(host: hostname, uri: uri) - .where('created_at >= ?', beginning_of_interval) - .completed_requests - .non_robots - .group(:host, :uri) - .rollup('host|uri', interval: starting_interval, update: true) - - # Reducir las estadísticas calculadas aplicando un rollup sobre el - # intervalo más amplio. - Stat::INTERVALS.reduce do |previous, current| - beginning_of_this_interval = beginning_of_interval.try(:"beginning_of_#{current}") - - Rollup.where(name: 'host|uri', interval: previous) - .where('time >= ?', beginning_of_this_interval) - .where_dimensions(host: hostname, uri: uri) - .group("dimensions->'host'", "dimensions->'uri'") - .rollup('host|uri', interval: current, update: true) do |rollup| - rollup.sum(:value) - end - - # Devolver el intervalo actual - current - end + rollup_uri(uri, host, name, beginning) end + + rollup_host(host, name, beginning) end stat.touch @@ -59,6 +45,52 @@ class UriCollectionJob < PeriodicJob private + def rollup_uri(uri, host, name, beginning) + dimensions = { host: host, uri: uri } + + AccessLog.where(**dimensions) + .where('created_at >= ?', beginning) + .completed_requests + .non_robots + .group(*dimensions.keys) + .rollup(name, interval: starting_interval, update: true) + + # Reducir las estadísticas calculadas aplicando un rollup sobre el + # intervalo más amplio. + Stat::INTERVALS.reduce do |previous, current| + recursive_rollup(name: name, + interval_previous: previous, + interval: current, + dimensions: dimensions, + beginning: beginning) + + # Devolver el intervalo actual + current + end + end + + def rollup_host(host, name, beginning) + dimensions = { host: host } + new_name = 'host' + + recursive_rollup(name: name, + new_name: new_name, + interval_previous: starting_interval, + interval: starting_interval, + dimensions: dimensions, + beginning: beginning) + + Stat::INTERVALS.reduce do |previous, current| + recursive_rollup(name: new_name, + interval_previous: previous, + interval: current, + dimensions: dimensions, + beginning: beginning) + + current + end + end + def stat_name STAT_NAME end From c09df3b87b8fa563e9225de4bed244ed4e280937 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 20:33:02 -0300 Subject: [PATCH 103/251] =?UTF-8?q?registrar=20el=20tiempo=20que=20pas?= =?UTF-8?q?=C3=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index c94320d5..03dcd292 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -10,6 +10,9 @@ class StatCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + # Registrar que se hicieron todas las recolecciones + stat = site.stats.create! name: STAT_NAME + scope.rollup('builds', **options) scope.rollup('space_used', **options) do |rollup| @@ -31,9 +34,7 @@ class StatCollectionJob < PeriodicJob current end - # Registrar que se hicieron todas las recolecciones - site.stats.create! name: STAT_NAME - + stat.touch run_again! unless once end From 47a74b584824001e81d1bc1be86c6333ba2099b0 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 20:33:58 -0300 Subject: [PATCH 104/251] fixup! refactorizar rollups recursivos --- app/jobs/stat_collection_job.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 03dcd292..fcb4d6e1 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -12,6 +12,7 @@ class StatCollectionJob < PeriodicJob # Registrar que se hicieron todas las recolecciones stat = site.stats.create! name: STAT_NAME + beginning = beginning_of_interval scope.rollup('builds', **options) @@ -25,7 +26,7 @@ class StatCollectionJob < PeriodicJob # XXX: Es correcto promediar promedios? Stat::INTERVALS.reduce do |previous, current| - opts = { interval_previous: previous, interval: current, dimensions: { site_id: site.id } } + opts = { interval_previous: previous, interval: current, beginning: beginning, dimensions: { site_id: site.id } } recursive_rollup(name: 'builds', **opts) recursive_rollup(name: 'space_used', operation: :average, **opts) From 298fd5bed14fa31c04dcf8d9cb5785c8c86c3c36 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:46:48 -0300 Subject: [PATCH 105/251] centralizar la forma en que se obtienen los hostnames relacionado con !50 --- app/jobs/uri_collection_job.rb | 22 +--------------------- app/models/site.rb | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index cad05cbb..ea968fa0 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -26,7 +26,7 @@ class UriCollectionJob < PeriodicJob name = 'host|uri' beginning = beginning_of_interval - hostnames.each do |host| + site.hostnames.each do |host| break if stop? uris.each do |uri| @@ -102,26 +102,6 @@ class UriCollectionJob < PeriodicJob @destination ||= site.deploys.find_by(type: 'DeployLocal').destination end - # TODO: Cambiar al mergear origin-referer - # - # @return [Array] - def hostnames - @hostnames ||= site.deploys.map do |deploy| - case deploy - when DeployLocal - site.hostname - when DeployWww - deploy.fqdn - when DeployAlternativeDomain - deploy.hostname.dup.tap do |h| - h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") - end - when DeployHiddenService - deploy.onion - end - end.compact - end - # Recolecta todas las URIs menos imágenes # # @return [Array] diff --git a/app/models/site.rb b/app/models/site.rb index 5b78d625..bcc9e60b 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -101,6 +101,26 @@ class Site < ApplicationRecord "https://#{hostname}#{slash ? '/' : ''}" end + # TODO: Cambiar al mergear origin-referer + # + # @return [Array] + def hostnames + @hostnames ||= deploys.map do |deploy| + case deploy + when DeployLocal + hostname + when DeployWww + deploy.fqdn + when DeployAlternativeDomain + deploy.hostname.dup.tap do |h| + h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") + end + when DeployHiddenService + deploy.onion + end + end.compact + end + # Obtiene los dominios alternativos # # @return Array @@ -123,7 +143,9 @@ class Site < ApplicationRecord # # @return Array def urls(slash: true) - alternative_urls(slash: slash) << url(slash: slash) + @urls ||= hostnames.map do |h| + "https://#{h}#{slash ? '/' : ''}" + end end def invitade?(usuarie) From fb5cc0aaa7c10749d78ce18b2246c7b20945d273 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:48:25 -0300 Subject: [PATCH 106/251] breadcrumbs --- app/controllers/stats_controller.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 44073c1f..ba9113ff 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -8,6 +8,10 @@ class StatsController < ApplicationController before_action :authenticate_usuarie! before_action :authorize_stats + breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path + breadcrumb 'sites.index', :sites_path, match: :exact + breadcrumb -> { site.title }, -> { site_posts_path(site, locale: locale) }, match: :exact + EXTRA_OPTIONS = { builds: {}, space_used: { bytes: true }, @@ -21,6 +25,8 @@ class StatsController < ApplicationController end def index + breadcrumb I18n.t('stats.index.title'), '' + @chart_params = { interval: interval } hostnames last_stat From f72d27c3083ee32b50da0baa8e948566de65e79f Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:49:26 -0300 Subject: [PATCH 107/251] =?UTF-8?q?optimizaci=C3=B3n:=20usar=20las=20dimen?= =?UTF-8?q?siones=20como=20=C3=ADndices=20y=20no=20como=20texto?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Rollup#where_dimensions` no aprovecha los índices. --- app/jobs/concerns/recursive_rollup.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 3163558e..b3327a73 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Implementa rollups recursivos module RecursiveRollup extend ActiveSupport::Concern @@ -18,9 +19,8 @@ module RecursiveRollup # @param :beginning [Time] # @return [Rollup] def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum, new_name: nil) - Rollup.where(name: name, interval: interval_previous) + Rollup.where(name: name, interval: interval_previous, dimensions: dimensions.to_json) .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) - .where_dimensions(**dimensions) .group(*dimensions_to_jsonb_query(dimensions)) .rollup(new_name || name, interval: interval, update: true) do |rollup| rollup.try(operation, :value) From 7c7ae2f75041b667a8f77ecbbf43fe2c569809d8 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:52:27 -0300 Subject: [PATCH 108/251] =?UTF-8?q?mostrar=20un=20per=C3=ADodo=20por=20def?= =?UTF-8?q?ecto?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit todavía no hay interfaz para cambiarlos --- app/controllers/stats_controller.rb | 30 +++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index ba9113ff..ded3a05c 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -24,6 +24,13 @@ class StatsController < ApplicationController policy.script_src :self, :unsafe_inline end + # Parámetros por defecto + # + # @return [Hash] + def default_url_options + { interval: 'day', period_start: Date.today.beginning_of_year, period_end: Date.today } + end + def index breadcrumb I18n.t('stats.index.title'), '' @@ -36,9 +43,9 @@ class StatsController < ApplicationController # Genera un gráfico de visitas por dominio asociado a este sitio def host - return unless stale? [last_stat, hostnames, interval] + return unless stale? [last_stat, hostnames, interval, period] - stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| + stats = rollup_scope.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie.dig(:dimensions, 'host') serie[:data].transform_values! do |value| @@ -51,7 +58,7 @@ class StatsController < ApplicationController end def resources - return unless stale? [last_stat, interval, resource] + return unless stale? [last_stat, interval, resource, period] options = { interval: interval, @@ -60,14 +67,14 @@ class StatsController < ApplicationController } } - render json: Rollup.series(resource, **options) + render json: rollup_scope.series(resource, **options) end def uris - return unless stale? [last_stat, hostnames, interval, normalized_urls] + return unless stale? [last_stat, hostnames, interval, normalized_urls, period] options = { host: hostnames, uri: normalized_paths } - stats = Rollup.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| + stats = rollup_scope.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/') serie[:data].transform_values! do |value| @@ -81,6 +88,10 @@ class StatsController < ApplicationController private + def rollup_scope + Rollup.where(time: period) + end + def last_stat @last_stat ||= Stat.last end @@ -171,4 +182,11 @@ class StatsController < ApplicationController def nodes @nodes ||= ENV.fetch('NODES', 1).to_i end + + def period + @period ||= begin + p = params.permit(:period_start, :period_end) + p[:period_start]..p[:period_end] + end + end end From 3a32422e26695feb58d4c45b5526d1bca227dfb8 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:54:04 -0300 Subject: [PATCH 109/251] =?UTF-8?q?usar=20m=C3=A9todos=20del=20sitio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index ded3a05c..116bca7c 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -103,23 +103,29 @@ class StatsController < ApplicationController # TODO: Eliminar cuando mergeemos referer-origin def hostnames - @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten + @hostnames ||= site.hostnames end # Normalizar las URLs # # @return [Array] def normalized_urls - @normalized_urls ||= params.permit(:urls).try(:[], - :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| - uri.start_with? 'https://' - end&.map do |u| - # XXX: Eliminar - # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} - next u unless u.end_with? '/' + @normalized_urls ||= + begin + urls = params.permit(:urls).try(:[], :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| + uri.start_with? 'https://' + end - "#{u}index.html" - end&.uniq || [@site.url, @site.urls].flatten.uniq + urls ||= [site.url] + + urls.map do |u| + # XXX: Eliminar al deployear + # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} + next u unless u.end_with? '/' + + "#{u}index.html" + end.uniq + end end def normalized_paths From 027d24d66a83dbd00273c3310e121090ade72097 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:55:54 -0300 Subject: [PATCH 110/251] memoizar el sitio --- app/controllers/stats_controller.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 116bca7c..10b75819 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -97,8 +97,7 @@ class StatsController < ApplicationController end def authorize_stats - @site = find_site - authorize SiteStat.new(@site) + authorize SiteStat.new(site) end # TODO: Eliminar cuando mergeemos referer-origin @@ -195,4 +194,8 @@ class StatsController < ApplicationController p[:period_start]..p[:period_end] end end + + def site + @site ||= find_site + end end From 0754e29a5a272a276ebefb93ab620838e9cc2fca Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:56:25 -0300 Subject: [PATCH 111/251] =?UTF-8?q?cada=20sitio=20tiene=20su=20propio=20co?= =?UTF-8?q?ntrol=20de=20estad=C3=ADsticas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 10b75819..4933c821 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -93,7 +93,7 @@ class StatsController < ApplicationController end def last_stat - @last_stat ||= Stat.last + @last_stat ||= site.stats.last end def authorize_stats From 7e575372cf1a7a42e4817f2ebe858941163daf6a Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:56:45 -0300 Subject: [PATCH 112/251] parametros por defecto --- app/controllers/stats_controller.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 4933c821..9a8a345d 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -162,14 +162,15 @@ class StatsController < ApplicationController def interval @interval ||= begin i = params[:interval]&.to_sym - Stat::INTERVALS.include?(i) ? i : :day + Stat::INTERVALS.include?(i) ? i : Stat::INTERVALS.first end end + # @return [Symbol] def resource @resource ||= begin r = params[:resource].to_sym - Stat::RESOURCES.include?(r) ? r : :builds + Stat::RESOURCES.include?(r) ? r : Stat::RESOURCES.first end end From a703eb409643fa6fa0d068d596472e7cfff5dac3 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:57:16 -0300 Subject: [PATCH 113/251] no fallar si el sitio no existe --- app/jobs/uri_collection_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ea968fa0..796a289c 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -20,6 +20,7 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + return unless File.directory? destination # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME From 5186892a87c0ec15aa1a4f9cb1dafa1b5173444c Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:57:57 -0300 Subject: [PATCH 114/251] =?UTF-8?q?fixup!=20usar=20m=C3=A9todos=20del=20si?= =?UTF-8?q?tio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index bfcf33ef..2b5d643e 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -25,12 +25,12 @@ %input{ type: 'hidden', name: 'interval', value: @interval } .form-group %label{ for: 'urls' }= t('.urls.label') - %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") + %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size + 1, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") %small#help-urls.feedback.form-text.text-muted= t('.urls.help') .form-group %button.btn{ type: 'submit' }= t('.urls.submit') - if @normalized_urls.present? - = line_chart site_stats_uris_path(urls: params[:urls], **@chart_params), **@chart_options + = line_chart site_stats_uris_path(urls: @normalized_urls, **@chart_params), **@chart_options .mb-5 %h2= t('.resources.title') From be6224ddf173f35d45a26df68abe71aea44282f2 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:59:02 -0300 Subject: [PATCH 115/251] =?UTF-8?q?recolectar=20estad=C3=ADsticas=20de=20r?= =?UTF-8?q?eferers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 46 +++++++++++++--------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 796a289c..12f25eb6 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -24,7 +24,6 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME - name = 'host|uri' beginning = beginning_of_interval site.hostnames.each do |host| @@ -33,10 +32,18 @@ class UriCollectionJob < PeriodicJob uris.each do |uri| break if stop? - rollup_uri(uri, host, name, beginning) + rollup('host|uri', beginning, host: host, uri: uri) + + AccessLog.where(host: host, uri: uri).distinct(:http_referer).pluck(:http_referer).each do |http_referer| + rollup('host|uri|referer', beginning, host: host, uri: uri, http_referer: http_referer) + end end - rollup_host(host, name, beginning) + rollup('host', beginning, host: host) + + AccessLog.where(host: host).distinct(:http_referer).pluck(:http_referer).each do |http_referer| + rollup('host|referer', beginning, host: host, http_referer: http_referer) + end end stat.touch @@ -46,9 +53,14 @@ class UriCollectionJob < PeriodicJob private - def rollup_uri(uri, host, name, beginning) - dimensions = { host: host, uri: uri } - + # Generar un rollup a partir de unas dimensiones que también sirven de + # filtro. + # + # @param :name [String] + # @param :beginning [Time] + # @param :dimensions [Hash] + # @return [nil] + def rollup(name, beginning, **dimensions) AccessLog.where(**dimensions) .where('created_at >= ?', beginning) .completed_requests @@ -70,28 +82,6 @@ class UriCollectionJob < PeriodicJob end end - def rollup_host(host, name, beginning) - dimensions = { host: host } - new_name = 'host' - - recursive_rollup(name: name, - new_name: new_name, - interval_previous: starting_interval, - interval: starting_interval, - dimensions: dimensions, - beginning: beginning) - - Stat::INTERVALS.reduce do |previous, current| - recursive_rollup(name: new_name, - interval_previous: previous, - interval: current, - dimensions: dimensions, - beginning: beginning) - - current - end - end - def stat_name STAT_NAME end From 3b19d8f792f743964e7b4e2a2119fcb770bb8a97 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 18:50:00 -0300 Subject: [PATCH 116/251] =?UTF-8?q?actualizaci=C3=B3n=20diaria?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/locales/en.yml | 2 +- config/locales/es.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 639d9184..732eb4bd 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -252,7 +252,7 @@ en: help: | These statistics show information about how your site is generated and how many resources it uses. - last_update: 'Updated every hour. Last update on ' + last_update: 'Updated daily. Last update on ' empty: 'There is no enough information yet. We invite you to come back in %{please_return_at}!' loading: 'Loading...' hour: 'Hourly' diff --git a/config/locales/es.yml b/config/locales/es.yml index 86e156df..e47c1fb5 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -257,7 +257,7 @@ es: help: | Las estadísticas visibilizan información sobre cómo se genera y cuántos recursos utiliza tu sitio. - last_update: 'Actualizadas cada hora. Última actualización hace ' + last_update: 'Actualizadas diariamente. Última actualización hace ' empty: 'Todavía no hay información suficiente. Te invitamos a volver en %{please_return_at} :)' loading: 'Cargando...' hour: 'Por hora' From 479ca7430f44269b643cd26ee6684f35544a9ceb Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 18:50:11 -0300 Subject: [PATCH 117/251] nombre --- app/jobs/stat_collection_job.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index fcb4d6e1..1ee2f2b3 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -57,4 +57,8 @@ class StatCollectionJob < PeriodicJob def options @options ||= { interval: starting_interval, update: true } end + + def stat_name + STAT_NAME + end end From 752935afb414d864c8acfe28184b0984c021b38f Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Apr 2022 13:57:33 -0300 Subject: [PATCH 118/251] calcular bien la fecha --- Dockerfile | 2 +- app/jobs/stat_collection_job.rb | 2 +- bin/access_logs | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ee6ba871..b3fd8e78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -114,7 +114,7 @@ RUN ln -s data/_private /srv/http/_private USER root # Instalar la configuración de monit RUN install -m 640 -o root -g root /srv/http/monit.conf /etc/monit.d/sutty.conf -RUN apk add --no-cache daemonize ruby-webrick +RUN apk add --no-cache daemonize ruby-webrick dateutils RUN install -m 755 /srv/http/entrypoint.sh /usr/local/bin/sutty # Mantener estos directorios! diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 1ee2f2b3..a67a3c50 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -47,7 +47,7 @@ class StatCollectionJob < PeriodicJob def scope @scope ||= site.build_stats .jekyll - .where('created_at => ?', beginning_of_interval) + .where('created_at >= ?', beginning_of_interval) .group(:site_id) end diff --git a/bin/access_logs b/bin/access_logs index a7c151cc..cfeeb57a 100755 --- a/bin/access_logs +++ b/bin/access_logs @@ -2,9 +2,12 @@ set -e # Volcar y eliminar todos los access logs de dos días atrás -date=`bundle exec rails runner "puts (Date.today - 2.days)"` +date="`dateadd today -1d`" +file="/srv/http/_storage/${date}.psql.gz" +test -n "${date}" +test ! -f "${file}" -psql -h postgresql "${DATABASE:-sutty}" sutty < "/srv/http/_storage/${date}.psql.gz" +psql -h postgresql "${DATABASE:-sutty}" sutty < "${file}" begin; copy (select * from access_logs where created_at < '${date}') to stdout; delete from access_logs where created_at < '${date}'; From eb7d0679bbc411d8f2afe831a23d958bc0e91abd Mon Sep 17 00:00:00 2001 From: f Date: Thu, 28 Apr 2022 10:34:57 -0300 Subject: [PATCH 119/251] normalizar strings sin romperlas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit en castellano no nos afectó pero en árabe rompe el significado --- .../active_storage/direct_uploads_controller_decorator.rb | 2 +- app/lib/action_dispatch/http/uploaded_file_decorator.rb | 2 +- app/models/metadata_markdown.rb | 2 +- app/models/metadata_markdown_content.rb | 2 +- app/models/metadata_permalink.rb | 2 +- app/models/metadata_string.rb | 2 +- app/models/metadata_template.rb | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/controllers/active_storage/direct_uploads_controller_decorator.rb b/app/controllers/active_storage/direct_uploads_controller_decorator.rb index 3052f974..c62dae2a 100644 --- a/app/controllers/active_storage/direct_uploads_controller_decorator.rb +++ b/app/controllers/active_storage/direct_uploads_controller_decorator.rb @@ -19,7 +19,7 @@ module ActiveStorage # stack. def blob_args params.require(:blob).permit(:filename, :byte_size, :checksum, :content_type, metadata: {}).to_h.symbolize_keys.tap do |ba| - ba[:filename] = ba[:filename].unicode_normalize(:nfkc) + ba[:filename] = ba[:filename].unicode_normalize end end end diff --git a/app/lib/action_dispatch/http/uploaded_file_decorator.rb b/app/lib/action_dispatch/http/uploaded_file_decorator.rb index c171c81c..0bdebdc0 100644 --- a/app/lib/action_dispatch/http/uploaded_file_decorator.rb +++ b/app/lib/action_dispatch/http/uploaded_file_decorator.rb @@ -11,7 +11,7 @@ module ActionDispatch # Devolver el nombre de archivo con caracteres unicode # normalizados def original_filename - @original_filename.unicode_normalize(:nfkc) + @original_filename.unicode_normalize end end end diff --git a/app/models/metadata_markdown.rb b/app/models/metadata_markdown.rb index 7816ec33..a09e351c 100644 --- a/app/models/metadata_markdown.rb +++ b/app/models/metadata_markdown.rb @@ -12,6 +12,6 @@ class MetadataMarkdown < MetadataText # markdown y se eliminan autolinks. Mejor es habilitar la generación # SAFE de CommonMark en la configuración del sitio. def sanitize(string) - string.unicode_normalize(:nfkc) + string.unicode_normalize end end diff --git a/app/models/metadata_markdown_content.rb b/app/models/metadata_markdown_content.rb index cb4124db..75088e30 100644 --- a/app/models/metadata_markdown_content.rb +++ b/app/models/metadata_markdown_content.rb @@ -25,6 +25,6 @@ class MetadataMarkdownContent < MetadataText # markdown y se eliminan autolinks. Mejor es deshabilitar la # generación SAFE de CommonMark en la configuración del sitio. def sanitize(string) - string.tr("\r", '').unicode_normalize(:nfkc) + string.tr("\r", '').unicode_normalize end end diff --git a/app/models/metadata_permalink.rb b/app/models/metadata_permalink.rb index 9b0c063c..30ad32cc 100644 --- a/app/models/metadata_permalink.rb +++ b/app/models/metadata_permalink.rb @@ -19,7 +19,7 @@ class MetadataPermalink < MetadataString # puntos suspensivos, la primera / para que siempre sea relativa y # agregamos una / al final si la ruta no tiene extensión. def sanitize(value) - value = value.strip.unicode_normalize(:nfkc).gsub('..', '/').gsub('./', '').squeeze('/') + value = value.strip.unicode_normalize.gsub('..', '/').gsub('./', '').squeeze('/') value = value[1..-1] if value.start_with? '/' value += '/' if File.extname(value).blank? diff --git a/app/models/metadata_string.rb b/app/models/metadata_string.rb index 28bfe82a..c1d888b1 100644 --- a/app/models/metadata_string.rb +++ b/app/models/metadata_string.rb @@ -17,7 +17,7 @@ class MetadataString < MetadataTemplate def sanitize(string) return '' if string.blank? - sanitizer.sanitize(string.strip.unicode_normalize(:nfkc), + sanitizer.sanitize(string.strip.unicode_normalize, tags: [], attributes: []).strip.html_safe end diff --git a/app/models/metadata_template.rb b/app/models/metadata_template.rb index a72f8e83..ddcd100e 100644 --- a/app/models/metadata_template.rb +++ b/app/models/metadata_template.rb @@ -185,7 +185,7 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, return string unless string.is_a? String sanitizer - .sanitize(string.tr("\r", '').unicode_normalize(:nfkc), + .sanitize(string.tr("\r", '').unicode_normalize, tags: allowed_tags, attributes: allowed_attributes) .strip From f4ec30eac7c41d7ef189768cd929ac3928a7d184 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 28 Apr 2022 12:06:27 -0300 Subject: [PATCH 120/251] jekyll no soporta urls con barra al final MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit y genera un montón de problemas! --- app/models/site.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/site.rb b/app/models/site.rb index 5b78d625..af10b004 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -470,7 +470,7 @@ class Site < ApplicationRecord config.theme = design.gem unless design.no_theme? config.description = description config.title = title - config.url = url + config.url = url(slash: false) config.hostname = hostname end From 957c810aea703d7c877ca5521fca8a9c8f9f306f Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 15:04:42 -0300 Subject: [PATCH 121/251] =?UTF-8?q?recolectar=20informaci=C3=B3n=20acumul?= =?UTF-8?q?=C3=A1ndola?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 48 +++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 12f25eb6..2939b9f9 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -26,23 +26,43 @@ class UriCollectionJob < PeriodicJob stat = site.stats.create! name: STAT_NAME beginning = beginning_of_interval + # Recorremos todos los hostnames y uris posibles y luego agrupamos + # recursivamente para no tener que recalcular, asumiendo que es más + # rápido buscar en los rollups indexados que en la tabla en bruto. + # + # Los referers solo se agrupan por host. site.hostnames.each do |host| break if stop? + dimensions = { host: host } + uris.each do |uri| break if stop? - rollup('host|uri', beginning, host: host, uri: uri) + dimensions[:uri] = uri - AccessLog.where(host: host, uri: uri).distinct(:http_referer).pluck(:http_referer).each do |http_referer| - rollup('host|uri|referer', beginning, host: host, uri: uri, http_referer: http_referer) - end + rollup('host|uri', beginning, **dimensions) + reduce_rollup('host|uri', beginning, **dimensions) end - rollup('host', beginning, host: host) + dimensions.delete(:uri) + # Reducir todas las visitas a cantidad de visitas por host + recursive_rollup(name: 'host|uri', + new_name: 'host', + interval_previous: starting_interval, + interval: starting_interval, + dimensions: dimensions) + + # Acumular por mes y año + reduce_rollup('host', beginning, **dimensions) + + # Obtener orígenes de visitas por host AccessLog.where(host: host).distinct(:http_referer).pluck(:http_referer).each do |http_referer| - rollup('host|referer', beginning, host: host, http_referer: http_referer) + dimensions[:http_referer] = http_referer + + rollup('host|referer', beginning, **dimensions) + reduce_rollup('host|referer', beginning, **dimensions) end end @@ -53,8 +73,7 @@ class UriCollectionJob < PeriodicJob private - # Generar un rollup a partir de unas dimensiones que también sirven de - # filtro. + # Generar un rollup de access logs # # @param :name [String] # @param :beginning [Time] @@ -67,9 +86,16 @@ class UriCollectionJob < PeriodicJob .non_robots .group(*dimensions.keys) .rollup(name, interval: starting_interval, update: true) + end - # Reducir las estadísticas calculadas aplicando un rollup sobre el - # intervalo más amplio. + # Reducir las estadísticas calculadas aplicando un rollup sobre el + # intervalo más amplio. + # + # @param :name [String] + # @param :beginning [Time] + # @param :dimensions [Hash] + # @return [nil] + def reduce_rollup(name, beginning, **dimensions) Stat::INTERVALS.reduce do |previous, current| recursive_rollup(name: name, interval_previous: previous, @@ -80,6 +106,8 @@ class UriCollectionJob < PeriodicJob # Devolver el intervalo actual current end + + nil end def stat_name From 23260791e44cf747aa388d7cd1a496d9005639d7 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 15:29:27 -0300 Subject: [PATCH 122/251] =?UTF-8?q?fixup!=20recolectar=20informaci=C3=B3n?= =?UTF-8?q?=20acumul=C3=A1ndola?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 2939b9f9..bea388a7 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -52,7 +52,8 @@ class UriCollectionJob < PeriodicJob new_name: 'host', interval_previous: starting_interval, interval: starting_interval, - dimensions: dimensions) + dimensions: dimensions, + beginning: beginning) # Acumular por mes y año reduce_rollup('host', beginning, **dimensions) From cc839affb866b040e84e836a95be2294ab872934 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 17:43:06 -0300 Subject: [PATCH 123/251] tener en cuenta todos los directorios posibles por ejemplo DeployLocalizedDomain en la otra rama tiene otras rutas --- app/jobs/uri_collection_job.rb | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index bea388a7..8504f006 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -115,11 +115,17 @@ class UriCollectionJob < PeriodicJob STAT_NAME end + # Obtiene todas las ubicaciones de archivos + # # @return [String] # # TODO: Cambiar al mergear origin-referer - def destination - @destination ||= site.deploys.find_by(type: 'DeployLocal').destination + def destinations + @destinations ||= site.deploys.map(&:destination).select do |d| + File.directory?(d) + end.map do |d| + File.realpath(d) + end.uniq end # Recolecta todas las URIs menos imágenes @@ -127,14 +133,16 @@ class UriCollectionJob < PeriodicJob # @return [Array] def uris @uris ||= - locales.map do |locale| - uri = "/#{locale}/".squeeze('/') - dir = File.join(destination, locale) + destinations.map do |destination| + locales.map do |locale| + uri = "/#{locale}/".squeeze('/') + dir = File.join(destination, locale) - files(dir).map do |f| - uri + f + files(dir).map do |f| + uri + f + end end - end.flatten(2) + end.flatten(3) end # @return [Array] From 9998f19087318910b70ad69e998bc1c0aaf809a2 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 17:57:29 -0300 Subject: [PATCH 124/251] =?UTF-8?q?acumular=20visitas=20por=20pa=C3=ADs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 8504f006..eaa3b153 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -58,12 +58,20 @@ class UriCollectionJob < PeriodicJob # Acumular por mes y año reduce_rollup('host', beginning, **dimensions) - # Obtener orígenes de visitas por host - AccessLog.where(host: host).distinct(:http_referer).pluck(:http_referer).each do |http_referer| - dimensions[:http_referer] = http_referer + columns = %i[http_referer geoip2_data_country_name] + columns.each do |column| + # Obtener orígenes de visitas por host + AccessLog.where(host: host).distinct(column).pluck(column).each do |value| + dimensions.delete_if do |k, _| + columns.include? k + end - rollup('host|referer', beginning, **dimensions) - reduce_rollup('host|referer', beginning, **dimensions) + name = "host|#{column}" + dimensions[column] = value + + rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, **dimensions) + end end end From 23e2edf259759ab0b85bddbb57bfa81bc2c0b677 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 18:01:07 -0300 Subject: [PATCH 125/251] =?UTF-8?q?la=20validaci=C3=B3n=20de=20directorio?= =?UTF-8?q?=20pasa=20por=20otro=20lado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index eaa3b153..0c5b6b1f 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -20,7 +20,6 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id - return unless File.directory? destination # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME From d56eeb465a8c91f8874a60d5babfe7baad8071ef Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 18:04:43 -0300 Subject: [PATCH 126/251] =?UTF-8?q?comprobar=20que=20los=20idiomas=20tambi?= =?UTF-8?q?=C3=A9n=20existan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 0c5b6b1f..b6d4a21b 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -145,11 +145,13 @@ class UriCollectionJob < PeriodicJob uri = "/#{locale}/".squeeze('/') dir = File.join(destination, locale) + next unless File.directory? dir + files(dir).map do |f| uri + f end end - end.flatten(3) + end.flatten(3).compact end # @return [Array] From 7dad3e9e54f523f7d4b1e801aede6dd4bfb40e5e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 20:06:07 -0300 Subject: [PATCH 127/251] =?UTF-8?q?ser=20m=C3=A1s=20explicitxs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index b6d4a21b..d8b24e3a 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -33,39 +33,30 @@ class UriCollectionJob < PeriodicJob site.hostnames.each do |host| break if stop? - dimensions = { host: host } - uris.each do |uri| break if stop? - dimensions[:uri] = uri - - rollup('host|uri', beginning, **dimensions) - reduce_rollup('host|uri', beginning, **dimensions) + rollup('host|uri', beginning, host: host, uri: uri) + reduce_rollup('host|uri', beginning, host: host, uri: uri) end - dimensions.delete(:uri) - # Reducir todas las visitas a cantidad de visitas por host recursive_rollup(name: 'host|uri', new_name: 'host', interval_previous: starting_interval, interval: starting_interval, - dimensions: dimensions, + dimensions: { host: host }, beginning: beginning) # Acumular por mes y año - reduce_rollup('host', beginning, **dimensions) + reduce_rollup('host', beginning, host: host) columns = %i[http_referer geoip2_data_country_name] columns.each do |column| # Obtener orígenes de visitas por host AccessLog.where(host: host).distinct(column).pluck(column).each do |value| - dimensions.delete_if do |k, _| - columns.include? k - end - name = "host|#{column}" + dimensions = { host: host } dimensions[column] = value rollup(name, beginning, **dimensions) From 595a8759a4379ea2e7bad904afe3bb46142207a7 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 20:32:21 -0300 Subject: [PATCH 128/251] usar las uris como fuente de verdad --- app/jobs/uri_collection_job.rb | 45 ++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index d8b24e3a..310984a0 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -24,6 +24,7 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME beginning = beginning_of_interval + columns = %i[http_referer geoip2_data_country_name] # Recorremos todos los hostnames y uris posibles y luego agrupamos # recursivamente para no tener que recalcular, asumiendo que es más @@ -33,11 +34,31 @@ class UriCollectionJob < PeriodicJob site.hostnames.each do |host| break if stop? + host_dimensions = { host: host } + + # Las URIs son la fuente de verdad de las visitas, porque son las + # que indican las páginas y recursos descargables, el resto son + # imágenes, CSS, JS y tipografías que no nos aportan números + # significativos. uris.each do |uri| break if stop? - rollup('host|uri', beginning, host: host, uri: uri) - reduce_rollup('host|uri', beginning, host: host, uri: uri) + name = 'host|uri' + dimensions = { host: host, uri: uri } + + rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, **dimensions) + + columns.each do |column| + # Obtener orígenes de visitas por host + AccessLog.where(**host_dimensions).distinct(column).pluck(column).each do |value| + name = "#{name}|#{column}" + dimensions[column] = value + + rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, **dimensions) + end + end end # Reducir todas las visitas a cantidad de visitas por host @@ -45,23 +66,21 @@ class UriCollectionJob < PeriodicJob new_name: 'host', interval_previous: starting_interval, interval: starting_interval, - dimensions: { host: host }, + dimensions: host_dimensions, beginning: beginning) # Acumular por mes y año - reduce_rollup('host', beginning, host: host) + reduce_rollup('host', beginning, **host_dimensions) - columns = %i[http_referer geoip2_data_country_name] columns.each do |column| - # Obtener orígenes de visitas por host - AccessLog.where(host: host).distinct(column).pluck(column).each do |value| - name = "host|#{column}" - dimensions = { host: host } - dimensions[column] = value + recursive_rollup(name: "host|uri|#{column}", + new_name: "host|#{column}", + interval_previous: starting_interval, + interval: starting_interval, + dimensions: host_dimensions, + beginning: beginning) - rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, **dimensions) - end + reduce_rollup("host|#{column}", beginning, **host_dimensions) end end From 121d4f49e26505f2f4edb2ef095a6ead487d83ab Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 09:56:19 -0300 Subject: [PATCH 129/251] no se puede renombrar un rollup usando dimensiones exactas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit además era un método que hacía dos cosas distintas según un parámetro --- app/jobs/concerns/recursive_rollup.rb | 27 +++++++++++++++++++++++---- app/jobs/uri_collection_job.rb | 22 ++++++++++------------ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index b3327a73..5ba9dd8b 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -15,14 +15,33 @@ module RecursiveRollup # @param :interval [String] # @param :operation [Symbol] # @param :dimensions [Hash] - # @param :new_name [String] # @param :beginning [Time] # @return [Rollup] - def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum, new_name: nil) - Rollup.where(name: name, interval: interval_previous, dimensions: dimensions.to_json) + def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum) + Rollup.where(name: name, interval: interval_previous, dimensions: dimensions) .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) .group(*dimensions_to_jsonb_query(dimensions)) - .rollup(new_name || name, interval: interval, update: true) do |rollup| + .rollup(name, interval: interval, update: true) do |rollup| + rollup.try(operation, :value) + end + end + + # Genera un nuevo rollup a partir de uno anterior. + # + # @param :name [String] + # @param :new_name [String] + # @param :interval_previous [String] + # @param :interval [String] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @param :beginning [Time] + # @return [Rollup] + def square_rollup(name:, new_name:, interval:, dimensions:, beginning:, operation: :sum) + Rollup.where(name: name, interval: interval_previous) + .where_dimensions(**dimensions) + .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) + .group(*dimensions_to_jsonb_query(dimensions)) + .rollup(new_name, interval: interval, update: true) do |rollup| rollup.try(operation, :value) end end diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 310984a0..5de95d43 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -62,23 +62,21 @@ class UriCollectionJob < PeriodicJob end # Reducir todas las visitas a cantidad de visitas por host - recursive_rollup(name: 'host|uri', - new_name: 'host', - interval_previous: starting_interval, - interval: starting_interval, - dimensions: host_dimensions, - beginning: beginning) + square_rollup(name: 'host|uri', + new_name: 'host', + interval: starting_interval, + dimensions: host_dimensions, + beginning: beginning) # Acumular por mes y año reduce_rollup('host', beginning, **host_dimensions) columns.each do |column| - recursive_rollup(name: "host|uri|#{column}", - new_name: "host|#{column}", - interval_previous: starting_interval, - interval: starting_interval, - dimensions: host_dimensions, - beginning: beginning) + square_rollup(name: "host|uri|#{column}", + new_name: "host|#{column}", + interval: starting_interval, + dimensions: host_dimensions, + beginning: beginning) reduce_rollup("host|#{column}", beginning, **host_dimensions) end From b36bdc976324d7359c8ac2a0574ff6e949fdb559 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:03:14 -0300 Subject: [PATCH 130/251] fixup! no se puede renombrar un rollup usando dimensiones exactas --- app/jobs/concerns/recursive_rollup.rb | 6 +++--- app/jobs/uri_collection_job.rb | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 5ba9dd8b..3539d7d2 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -30,14 +30,14 @@ module RecursiveRollup # # @param :name [String] # @param :new_name [String] - # @param :interval_previous [String] - # @param :interval [String] + # @param :interval_previous [String, Symbol] + # @param :interval [String, Symbol] # @param :operation [Symbol] # @param :dimensions [Hash] # @param :beginning [Time] # @return [Rollup] def square_rollup(name:, new_name:, interval:, dimensions:, beginning:, operation: :sum) - Rollup.where(name: name, interval: interval_previous) + Rollup.where(name: name, interval: interval) .where_dimensions(**dimensions) .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) .group(*dimensions_to_jsonb_query(dimensions)) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 5de95d43..a9549efe 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -52,11 +52,11 @@ class UriCollectionJob < PeriodicJob columns.each do |column| # Obtener orígenes de visitas por host AccessLog.where(**host_dimensions).distinct(column).pluck(column).each do |value| - name = "#{name}|#{column}" + column_name = "host|#{column}" dimensions[column] = value - rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, **dimensions) + rollup(column_name, beginning, **dimensions) + reduce_rollup(column_name, beginning, **dimensions) end end end @@ -145,6 +145,9 @@ class UriCollectionJob < PeriodicJob # Recolecta todas las URIs menos imágenes # + # TODO: Para los sitios con DeployLocalizedDomain estamos buscando + # URIs de más. + # # @return [Array] def uris @uris ||= From ce0d2306f6f8b14ea046c77d41ff9a106559bae4 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:13:49 -0300 Subject: [PATCH 131/251] fixup! fixup! no se puede renombrar un rollup usando dimensiones exactas --- app/jobs/uri_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index a9549efe..1f635997 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -52,7 +52,7 @@ class UriCollectionJob < PeriodicJob columns.each do |column| # Obtener orígenes de visitas por host AccessLog.where(**host_dimensions).distinct(column).pluck(column).each do |value| - column_name = "host|#{column}" + column_name = "host|uri|#{column}" dimensions[column] = value rollup(column_name, beginning, **dimensions) From deb3f0f0709bed4d586e8580cb1987722b310094 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:23:58 -0300 Subject: [PATCH 132/251] =?UTF-8?q?usar=20el=20=C3=BAltimo=20intervalo,=20?= =?UTF-8?q?no=20el=20que=20acabamos=20de=20crear?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 1f635997..e5105e5a 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -21,9 +21,10 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + # Obtener el principio del intervalo anterior + beginning = beginning_of_interval # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME - beginning = beginning_of_interval columns = %i[http_referer geoip2_data_country_name] # Recorremos todos los hostnames y uris posibles y luego agrupamos From 80f2df362b3091b8c9856766ceb1bf4bd5dc4f64 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:24:21 -0300 Subject: [PATCH 133/251] precalcular las columnas --- app/jobs/uri_collection_job.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index e5105e5a..196d2e50 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -25,7 +25,8 @@ class UriCollectionJob < PeriodicJob beginning = beginning_of_interval # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME - columns = %i[http_referer geoip2_data_country_name] + # Columnas a agrupar + columns = %i[http_referer geoip2_data_country_name].zip([nil]).to_h # Recorremos todos los hostnames y uris posibles y luego agrupamos # recursivamente para no tener que recalcular, asumiendo que es más @@ -36,6 +37,9 @@ class UriCollectionJob < PeriodicJob break if stop? host_dimensions = { host: host } + columns.each_key do |column| + columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) + end # Las URIs son la fuente de verdad de las visitas, porque son las # que indican las páginas y recursos descargables, el resto son @@ -50,9 +54,9 @@ class UriCollectionJob < PeriodicJob rollup(name, beginning, **dimensions) reduce_rollup(name, beginning, **dimensions) - columns.each do |column| + columns.each_pair do |column, values| # Obtener orígenes de visitas por host - AccessLog.where(**host_dimensions).distinct(column).pluck(column).each do |value| + values.each do |value| column_name = "host|uri|#{column}" dimensions[column] = value @@ -72,7 +76,7 @@ class UriCollectionJob < PeriodicJob # Acumular por mes y año reduce_rollup('host', beginning, **host_dimensions) - columns.each do |column| + columns.each_key do |column| square_rollup(name: "host|uri|#{column}", new_name: "host|#{column}", interval: starting_interval, From e9d13a7c5931b9ad4e86bf218ff6aff79ac132c0 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:49:32 -0300 Subject: [PATCH 134/251] mostrar demora --- app/jobs/uri_collection_job.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 196d2e50..ae4b098e 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -38,7 +38,10 @@ class UriCollectionJob < PeriodicJob host_dimensions = { host: host } columns.each_key do |column| + time_start = Time.now columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) + + puts "#{column}: #{Time.now - time_start}" end # Las URIs son la fuente de verdad de las visitas, porque son las @@ -51,39 +54,58 @@ class UriCollectionJob < PeriodicJob name = 'host|uri' dimensions = { host: host, uri: uri } + time_start = Time.now rollup(name, beginning, **dimensions) + puts "#{dimensions.values.join}: #{Time.now - time_start}" + + time_start = Time.now reduce_rollup(name, beginning, **dimensions) + puts "reduced: #{Time.now - time_start}" columns.each_pair do |column, values| # Obtener orígenes de visitas por host values.each do |value| + time_start = Time.now + column_name = "host|uri|#{column}" dimensions[column] = value rollup(column_name, beginning, **dimensions) + puts "#{dimensions.values.join}: #{Time.now - time_start}" + + time_start = Time.now reduce_rollup(column_name, beginning, **dimensions) + puts "reduced: #{Time.now - time_start}" end end end # Reducir todas las visitas a cantidad de visitas por host + time_start = Time.now square_rollup(name: 'host|uri', new_name: 'host', interval: starting_interval, dimensions: host_dimensions, beginning: beginning) + puts "#{host}: #{Time.now - time_start}" # Acumular por mes y año + time_start = Time.now reduce_rollup('host', beginning, **host_dimensions) + puts "reduced: #{Time.now - time_start}" columns.each_key do |column| + time_start = Time.now square_rollup(name: "host|uri|#{column}", new_name: "host|#{column}", interval: starting_interval, dimensions: host_dimensions, beginning: beginning) + puts "#{host} #{column}: #{Time.now - time_start}" + time_start = Time.now reduce_rollup("host|#{column}", beginning, **host_dimensions) + puts "reduced: #{Time.now - time_start}" end end From 1962175cc66502848aa0e17116959c3e0986f140 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:58:41 -0300 Subject: [PATCH 135/251] Revert "mostrar demora" This reverts commit e9d13a7c5931b9ad4e86bf218ff6aff79ac132c0. --- app/jobs/uri_collection_job.rb | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ae4b098e..196d2e50 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -38,10 +38,7 @@ class UriCollectionJob < PeriodicJob host_dimensions = { host: host } columns.each_key do |column| - time_start = Time.now columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) - - puts "#{column}: #{Time.now - time_start}" end # Las URIs son la fuente de verdad de las visitas, porque son las @@ -54,58 +51,39 @@ class UriCollectionJob < PeriodicJob name = 'host|uri' dimensions = { host: host, uri: uri } - time_start = Time.now rollup(name, beginning, **dimensions) - puts "#{dimensions.values.join}: #{Time.now - time_start}" - - time_start = Time.now reduce_rollup(name, beginning, **dimensions) - puts "reduced: #{Time.now - time_start}" columns.each_pair do |column, values| # Obtener orígenes de visitas por host values.each do |value| - time_start = Time.now - column_name = "host|uri|#{column}" dimensions[column] = value rollup(column_name, beginning, **dimensions) - puts "#{dimensions.values.join}: #{Time.now - time_start}" - - time_start = Time.now reduce_rollup(column_name, beginning, **dimensions) - puts "reduced: #{Time.now - time_start}" end end end # Reducir todas las visitas a cantidad de visitas por host - time_start = Time.now square_rollup(name: 'host|uri', new_name: 'host', interval: starting_interval, dimensions: host_dimensions, beginning: beginning) - puts "#{host}: #{Time.now - time_start}" # Acumular por mes y año - time_start = Time.now reduce_rollup('host', beginning, **host_dimensions) - puts "reduced: #{Time.now - time_start}" columns.each_key do |column| - time_start = Time.now square_rollup(name: "host|uri|#{column}", new_name: "host|#{column}", interval: starting_interval, dimensions: host_dimensions, beginning: beginning) - puts "#{host} #{column}: #{Time.now - time_start}" - time_start = Time.now reduce_rollup("host|#{column}", beginning, **host_dimensions) - puts "reduced: #{Time.now - time_start}" end end From 6699db838078d141cc3eaab7815b09d22d1862ec Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 11:34:21 -0300 Subject: [PATCH 136/251] otra vez el mismo error de hashes acumulados hice benchmarks, generar hashes nuevos es mas rapido que duplicarlos, y eliminar las keys que no corresponden es apenas mas rapido pero con mas posibilidad de bugs --- app/jobs/uri_collection_job.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 196d2e50..620123db 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -58,10 +58,11 @@ class UriCollectionJob < PeriodicJob # Obtener orígenes de visitas por host values.each do |value| column_name = "host|uri|#{column}" - dimensions[column] = value + column_dimensions = { host: host, uri: uri } + column_dimensions[column] = value - rollup(column_name, beginning, **dimensions) - reduce_rollup(column_name, beginning, **dimensions) + rollup(column_name, beginning, **column_dimensions) + reduce_rollup(column_name, beginning, **column_dimensions) end end end From 6bbb5b48c0c6f63fa3ee2601affcdd51d94cb79e Mon Sep 17 00:00:00 2001 From: Nulo Date: Sat, 30 Apr 2022 14:56:04 +0000 Subject: [PATCH 137/251] editor: Borrar comentario viejo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ahora la blockquote está implementada por https://0xacab.org/sutty/sutty/-/merge_requests/83, está implementada como bloque normal ya que una blockquote puede tener varios blockquotes adentro --- app/javascript/editor/types/parentBlocks.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/javascript/editor/types/parentBlocks.ts b/app/javascript/editor/types/parentBlocks.ts index ffe40bdf..5ba8f366 100644 --- a/app/javascript/editor/types/parentBlocks.ts +++ b/app/javascript/editor/types/parentBlocks.ts @@ -20,7 +20,6 @@ function makeParentBlock( }; } -// TODO: añadir blockquote // XXX: si agregás algo acá, probablemente le quieras hacer un botón // en app/views/posts/attributes/_content.haml export const parentBlocks: { [propName: string]: EditorNode } = { From 001443c1be55fc0f7f893392fd9e119a1c07c125 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 14:07:34 -0300 Subject: [PATCH 138/251] =?UTF-8?q?la=20=C3=BAltima=20actualizaci=C3=B3n?= =?UTF-8?q?=20es=20cuando=20se=20cierra=20el=20stat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 2b5d643e..02683781 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -6,8 +6,8 @@ %p %small = t('.last_update') - %time{ datetime: @last_stat.created_at } - #{time_ago_in_words @last_stat.created_at}. + %time{ datetime: @last_stat.updated_at } + #{time_ago_in_words @last_stat.updated_at}. .mb-5 - Stat::INTERVALS.each do |interval| From 6bf286077bba19858665314f9fd850133229c7eb Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 14:39:31 -0300 Subject: [PATCH 139/251] =?UTF-8?q?generar=20estad=C3=ADsticas=20de=20uso?= =?UTF-8?q?=20de=20infra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/concerns/recursive_rollup.rb | 24 +++++++++++++++++++++ app/jobs/stat_collection_job.rb | 22 +++++--------------- app/jobs/uri_collection_job.rb | 30 ++++----------------------- 3 files changed, 33 insertions(+), 43 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 3539d7d2..28c2e66a 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -46,6 +46,30 @@ module RecursiveRollup end end + # Reducir las estadísticas calculadas aplicando un rollup sobre el + # intervalo más amplio. + # + # @param :name [String] + # @param :beginning [Time] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @return [nil] + def reduce_rollup(name, beginning, operation, **dimensions) + Stat::INTERVALS.reduce do |previous, current| + recursive_rollup(name: name, + interval_previous: previous, + interval: current, + dimensions: dimensions, + beginning: beginning, + operation: operation) + + # Devolver el intervalo actual + current + end + + nil + end + # @param :dimensions [Hash] # @return [Array] def dimensions_to_jsonb_query(dimensions) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index a67a3c50..65226b5c 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -9,10 +9,8 @@ class StatCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id - - # Registrar que se hicieron todas las recolecciones - stat = site.stats.create! name: STAT_NAME beginning = beginning_of_interval + stat = site.stats.create! name: STAT_NAME scope.rollup('builds', **options) @@ -24,16 +22,9 @@ class StatCollectionJob < PeriodicJob rollup.average(:seconds) end - # XXX: Es correcto promediar promedios? - Stat::INTERVALS.reduce do |previous, current| - opts = { interval_previous: previous, interval: current, beginning: beginning, dimensions: { site_id: site.id } } - - recursive_rollup(name: 'builds', **opts) - recursive_rollup(name: 'space_used', operation: :average, **opts) - recursive_rollup(name: 'build_time', operation: :average, **opts) - - current - end + reduce_rollup('builds', beginning, :sum, site_id: site_id) + reduce_rollup('space_used', beginning, :average, site_id: site_id) + reduce_rollup('build_time', beginning, :average, site_id: site_id) stat.touch run_again! unless once @@ -45,10 +36,7 @@ class StatCollectionJob < PeriodicJob # # @return [ActiveRecord::Relation] def scope - @scope ||= site.build_stats - .jekyll - .where('created_at >= ?', beginning_of_interval) - .group(:site_id) + @scope ||= site.build_stats.jekyll.where('created_at >= ?', beginning_of_interval) end # Las opciones por defecto diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 620123db..1553fb22 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -52,7 +52,7 @@ class UriCollectionJob < PeriodicJob dimensions = { host: host, uri: uri } rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, :sum, **dimensions) columns.each_pair do |column, values| # Obtener orígenes de visitas por host @@ -62,7 +62,7 @@ class UriCollectionJob < PeriodicJob column_dimensions[column] = value rollup(column_name, beginning, **column_dimensions) - reduce_rollup(column_name, beginning, **column_dimensions) + reduce_rollup(column_name, beginning, :sum, **column_dimensions) end end end @@ -75,7 +75,7 @@ class UriCollectionJob < PeriodicJob beginning: beginning) # Acumular por mes y año - reduce_rollup('host', beginning, **host_dimensions) + reduce_rollup('host', beginning, :sum, **host_dimensions) columns.each_key do |column| square_rollup(name: "host|uri|#{column}", @@ -84,7 +84,7 @@ class UriCollectionJob < PeriodicJob dimensions: host_dimensions, beginning: beginning) - reduce_rollup("host|#{column}", beginning, **host_dimensions) + reduce_rollup("host|#{column}", beginning, :sum, **host_dimensions) end end @@ -110,28 +110,6 @@ class UriCollectionJob < PeriodicJob .rollup(name, interval: starting_interval, update: true) end - # Reducir las estadísticas calculadas aplicando un rollup sobre el - # intervalo más amplio. - # - # @param :name [String] - # @param :beginning [Time] - # @param :dimensions [Hash] - # @return [nil] - def reduce_rollup(name, beginning, **dimensions) - Stat::INTERVALS.reduce do |previous, current| - recursive_rollup(name: name, - interval_previous: previous, - interval: current, - dimensions: dimensions, - beginning: beginning) - - # Devolver el intervalo actual - current - end - - nil - end - def stat_name STAT_NAME end From 8d4958af536a6e9a0dedcd9381811e5898fee91f Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 14:44:27 -0300 Subject: [PATCH 140/251] =?UTF-8?q?fixup!=20generar=20estad=C3=ADsticas=20?= =?UTF-8?q?de=20uso=20de=20infra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 65226b5c..3f7cdd48 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -36,7 +36,7 @@ class StatCollectionJob < PeriodicJob # # @return [ActiveRecord::Relation] def scope - @scope ||= site.build_stats.jekyll.where('created_at >= ?', beginning_of_interval) + @scope ||= site.build_stats.jekyll.where('created_at >= ?', beginning_of_interval).group(:site_id) end # Las opciones por defecto From 92cd2ac05236c206d69cf081bec8d9e3fd658dc2 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 14:48:42 -0300 Subject: [PATCH 141/251] =?UTF-8?q?fixup!=20fixup!=20generar=20estad=C3=AD?= =?UTF-8?q?sticas=20de=20uso=20de=20infra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 3f7cdd48..63d44999 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -36,7 +36,7 @@ class StatCollectionJob < PeriodicJob # # @return [ActiveRecord::Relation] def scope - @scope ||= site.build_stats.jekyll.where('created_at >= ?', beginning_of_interval).group(:site_id) + @scope ||= site.build_stats.jekyll.where('build_stats.created_at >= ?', beginning_of_interval).group(:site_id) end # Las opciones por defecto From 756fb8f1cbc32561bfd1433b0698267024d870ff Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 16:38:16 -0300 Subject: [PATCH 142/251] =?UTF-8?q?usar=20el=20id=20del=20sitio=20como=20d?= =?UTF-8?q?imensi=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 9a8a345d..3e1ec76b 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -60,12 +60,7 @@ class StatsController < ApplicationController def resources return unless stale? [last_stat, interval, resource, period] - options = { - interval: interval, - dimensions: { - deploy_id: @site.deploys.where(type: 'DeployLocal').pluck(:id).first - } - } + options = { interval: interval, dimensions: { site_id: site.id } } render json: rollup_scope.series(resource, **options) end From d181c179ff48948143526f4e63c6ac8b9bc40d75 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 16:38:32 -0300 Subject: [PATCH 143/251] nota --- app/controllers/stats_controller.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 3e1ec76b..35f7f108 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -69,6 +69,8 @@ class StatsController < ApplicationController return unless stale? [last_stat, hostnames, interval, normalized_urls, period] options = { host: hostnames, uri: normalized_paths } + # XXX: where_dimensions es más corto pero no aprovecha los índices + # de Rollup stats = rollup_scope.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/') From 604ee233bd1ef7bbbfa5ccd293a7d2edd6ca248e Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 16:38:50 -0300 Subject: [PATCH 144/251] =?UTF-8?q?agregar=20la=20dimensi=C3=B3n=20al=20ag?= =?UTF-8?q?rupar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 1553fb22..44642c1f 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -78,13 +78,18 @@ class UriCollectionJob < PeriodicJob reduce_rollup('host', beginning, :sum, **host_dimensions) columns.each_key do |column| - square_rollup(name: "host|uri|#{column}", - new_name: "host|#{column}", - interval: starting_interval, - dimensions: host_dimensions, - beginning: beginning) + values.each do |value| + column_dimensions = { host: host } + column_dimensions[column] = value - reduce_rollup("host|#{column}", beginning, :sum, **host_dimensions) + square_rollup(name: "host|uri|#{column}", + new_name: "host|#{column}", + interval: starting_interval, + dimensions: column_dimensions, + beginning: beginning) + + reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions) + end end end From d766ee4eff68284e28462f1afe3e9c6c6d061698 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 16:44:23 -0300 Subject: [PATCH 145/251] =?UTF-8?q?fixup!=20agregar=20la=20dimensi=C3=B3n?= =?UTF-8?q?=20al=20agrupar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 44642c1f..a0b1181e 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -77,7 +77,7 @@ class UriCollectionJob < PeriodicJob # Acumular por mes y año reduce_rollup('host', beginning, :sum, **host_dimensions) - columns.each_key do |column| + columns.each_pair do |column, values| values.each do |value| column_dimensions = { host: host } column_dimensions[column] = value From fb34823ce31606c23823d66365ef85438e0e12fa Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 17:51:05 -0300 Subject: [PATCH 146/251] deprecar square_rollup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit es muy lento buscar dentro de cada dimensión --- app/jobs/concerns/recursive_rollup.rb | 20 -------------------- app/jobs/uri_collection_job.rb | 13 ++----------- 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 28c2e66a..4d9e11b0 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -26,26 +26,6 @@ module RecursiveRollup end end - # Genera un nuevo rollup a partir de uno anterior. - # - # @param :name [String] - # @param :new_name [String] - # @param :interval_previous [String, Symbol] - # @param :interval [String, Symbol] - # @param :operation [Symbol] - # @param :dimensions [Hash] - # @param :beginning [Time] - # @return [Rollup] - def square_rollup(name:, new_name:, interval:, dimensions:, beginning:, operation: :sum) - Rollup.where(name: name, interval: interval) - .where_dimensions(**dimensions) - .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) - .group(*dimensions_to_jsonb_query(dimensions)) - .rollup(new_name, interval: interval, update: true) do |rollup| - rollup.try(operation, :value) - end - end - # Reducir las estadísticas calculadas aplicando un rollup sobre el # intervalo más amplio. # diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index a0b1181e..fe50929d 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -68,11 +68,7 @@ class UriCollectionJob < PeriodicJob end # Reducir todas las visitas a cantidad de visitas por host - square_rollup(name: 'host|uri', - new_name: 'host', - interval: starting_interval, - dimensions: host_dimensions, - beginning: beginning) + rollup('host', beginning, :sum, **host_dimensions) # Acumular por mes y año reduce_rollup('host', beginning, :sum, **host_dimensions) @@ -82,12 +78,7 @@ class UriCollectionJob < PeriodicJob column_dimensions = { host: host } column_dimensions[column] = value - square_rollup(name: "host|uri|#{column}", - new_name: "host|#{column}", - interval: starting_interval, - dimensions: column_dimensions, - beginning: beginning) - + rollup("host|#{column}", beginning, :sum, **column_dimensions) reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions) end end From e00780c7f0d7fee3e0364ffb1fc7cfc7ac5b237c Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 17:55:53 -0300 Subject: [PATCH 147/251] mostrar la cantidad de visitas por origen y pais --- app/controllers/stats_controller.rb | 14 ++++++++++++++ app/jobs/uri_collection_job.rb | 2 +- app/models/stat.rb | 8 ++++++++ app/views/stats/index.haml | 21 +++++++++++++++++++++ config/locales/en.yml | 11 +++++++++++ config/locales/es.yml | 11 +++++++++++ 6 files changed, 66 insertions(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 35f7f108..84ca3028 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -39,6 +39,20 @@ class StatsController < ApplicationController last_stat chart_options normalized_urls + + expires_in = Time.now.try(:"end_of_#{Stat.default_interval}") - Time.now + @columns = {} + + Stat::COLUMNS.each do |column| + @columns[column] = + Rails.cache.fetch("stats/#{column}/#{site.id}", expires_in: expires_in) do + Rollup.where(interval: interval, name: "host|#{column}") + .where_dimensions(host: hostnames) + .group("dimensions->>'#{column}'") + .sum(:value) + .transform_values(&:to_i) + end + end end # Genera un gráfico de visitas por dominio asociado a este sitio diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index fe50929d..a7dd2a00 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -26,7 +26,7 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME # Columnas a agrupar - columns = %i[http_referer geoip2_data_country_name].zip([nil]).to_h + columns = Stat::COLUMNS.zip([nil]).to_h # Recorremos todos los hostnames y uris posibles y luego agrupamos # recursivamente para no tener que recalcular, asumiendo que es más diff --git a/app/models/stat.rb b/app/models/stat.rb index 5f72ccd0..f1016026 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -5,6 +5,14 @@ class Stat < ApplicationRecord # XXX: Los intervalos van en orden de mayor especificidad a menor INTERVALS = %i[day month year].freeze RESOURCES = %i[builds space_used build_time].freeze + COLUMNS = %i[http_referer geoip2_data_country_name].freeze belongs_to :site + + # El intervalo por defecto + # + # @return [Symbol] + def self.default_interval + INTERVALS.first + end end diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 02683781..be0c1426 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -32,6 +32,27 @@ - if @normalized_urls.present? = line_chart site_stats_uris_path(urls: @normalized_urls, **@chart_params), **@chart_options + .row.mb-5.row-cols-1.row-cols-md-2 + - @columns.each_pair do |column, values| + - next if values.blank? + .col.mb-5 + %h2= t(".columns.#{column}.title") + %p.lead= t(".columns.#{column}.description") + + %table.table + %colgroup + %col + %col + %thead + %tr.sticky-top.background-white + %th{ scope: 'col' }= t(".columns.#{column}.column") + %th{ scope: 'col' }= t('.columns.visits') + %tfoot + %tbody + - values.each_pair do |col, val| + %tr + %th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t('.columns.empty') : col + %td= val .mb-5 %h2= t('.resources.title') %p.lead= t('.resources.description') diff --git a/config/locales/en.yml b/config/locales/en.yml index 732eb4bd..76760958 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -284,6 +284,17 @@ en: build_time: title: 'Publication time' description: 'Average time your site takes to build.' + columns: + visits: "Visits" + empty: "(nothing)" + http_referer: + title: "Referers" + description: "Visits by origin" + column: "Referer" + geoip2_data_country_name: + title: "Countries" + description: "Visits by country" + column: "Country" sites: donations: url: 'https://donaciones.sutty.nl/en/' diff --git a/config/locales/es.yml b/config/locales/es.yml index e47c1fb5..82bfd253 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -289,6 +289,17 @@ es: build_time: title: 'Tiempo de publicación' description: 'Tiempo promedio que toma en publicarse tu sitio.' + columns: + visits: "Visitas" + empty: "(nada)" + http_referer: + title: "Referencias" + description: "Orígenes de las visitas" + column: "Referencia" + geoip2_data_country_name: + title: "Países" + description: "Cantidad de visitas por país" + column: "País" sites: donations: url: 'https://donaciones.sutty.nl/' From 224e599de2f9b5acf1ae75ebfc5463b37364d179 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:05:08 -0300 Subject: [PATCH 148/251] fixup! mostrar la cantidad de visitas por origen y pais --- app/jobs/uri_collection_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index a7dd2a00..c3e48ecd 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -68,7 +68,7 @@ class UriCollectionJob < PeriodicJob end # Reducir todas las visitas a cantidad de visitas por host - rollup('host', beginning, :sum, **host_dimensions) + rollup('host', beginning, **host_dimensions) # Acumular por mes y año reduce_rollup('host', beginning, :sum, **host_dimensions) @@ -78,7 +78,7 @@ class UriCollectionJob < PeriodicJob column_dimensions = { host: host } column_dimensions[column] = value - rollup("host|#{column}", beginning, :sum, **column_dimensions) + rollup("host|#{column}", beginning, **column_dimensions) reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions) end end From 99e3f093fcd1fda9447ebcfefd6661e6053f76a4 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:20:45 -0300 Subject: [PATCH 149/251] ordenar de menor a mayor --- app/controllers/stats_controller.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 84ca3028..e00b9822 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -51,6 +51,9 @@ class StatsController < ApplicationController .group("dimensions->>'#{column}'") .sum(:value) .transform_values(&:to_i) + .sort_by do |_, v| + v.last + end.reverse.to_h end end end From 09696529411acdbf190ad45ce7fd3fbe11f7cd31 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:22:35 -0300 Subject: [PATCH 150/251] acomodar el valor a la cantidad de nodos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit en realidad para estas visitas no tiene mucho sentido y de hecho es en las que estaríamos perdiendo información, porque nada indica que no hubo otros países y referencias que quedaron en otros nodos... --- app/controllers/stats_controller.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index e00b9822..07755c73 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -51,6 +51,9 @@ class StatsController < ApplicationController .group("dimensions->>'#{column}'") .sum(:value) .transform_values(&:to_i) + .transform_values do |v| + v * nodes + end .sort_by do |_, v| v.last end.reverse.to_h From 8244532c5a0fa5e00a3501b376c5d0d4fe329af7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:30:24 -0300 Subject: [PATCH 151/251] fixup! ordenar de menor a mayor --- app/controllers/stats_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 07755c73..36b72721 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -55,7 +55,7 @@ class StatsController < ApplicationController v * nodes end .sort_by do |_, v| - v.last + v end.reverse.to_h end end From e4009e59dcf73bf84dd9e6f340618e49799ee60d Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:42:50 -0300 Subject: [PATCH 152/251] las tablas con cabeceras sticky usan el color de fondo del theme --- app/assets/stylesheets/application.scss | 1 + app/views/posts/index.haml | 4 ++-- app/views/stats/index.haml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index e806a032..b756759a 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -126,6 +126,7 @@ ol.breadcrumb { color: var(--foreground); } +.table tr.sticky-top, .form-control, .custom-file-label { background-color: var(--background); diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 90d65670..accfdb34 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -71,8 +71,8 @@ %table.table{ data: { controller: 'reorder' } } %caption.sr-only= t('posts.caption') %thead - %tr - %th.border-0.background-white.position-sticky{ style: 'top: 0; z-index: 2', colspan: '4' } + %tr.sticky-top + %th.border-0{ colspan: '4' } .d-flex.flex-row.justify-content-between %div = submit_tag t('posts.reorder.submit'), class: 'btn' diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index be0c1426..6bbf596e 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -44,7 +44,7 @@ %col %col %thead - %tr.sticky-top.background-white + %tr.sticky-top %th{ scope: 'col' }= t(".columns.#{column}.column") %th{ scope: 'col' }= t('.columns.visits') %tfoot From e8300364d1e49e1b5877512f8695fda955e44da3 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 19:11:57 -0300 Subject: [PATCH 153/251] pasar todas las uris a la base de datos --- app/jobs/uri_collection_job.rb | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index c3e48ecd..90789aba 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -45,25 +45,21 @@ class UriCollectionJob < PeriodicJob # que indican las páginas y recursos descargables, el resto son # imágenes, CSS, JS y tipografías que no nos aportan números # significativos. - uris.each do |uri| - break if stop? + name = 'host|uri' + dimensions = { host: host, uri: uris } - name = 'host|uri' - dimensions = { host: host, uri: uri } + rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, :sum, **dimensions) - rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, :sum, **dimensions) + columns.each_pair do |column, values| + # Obtener orígenes de visitas por host + values.each do |value| + column_name = "host|uri|#{column}" + column_dimensions = { host: host, uri: uris } + column_dimensions[column] = value - columns.each_pair do |column, values| - # Obtener orígenes de visitas por host - values.each do |value| - column_name = "host|uri|#{column}" - column_dimensions = { host: host, uri: uri } - column_dimensions[column] = value - - rollup(column_name, beginning, **column_dimensions) - reduce_rollup(column_name, beginning, :sum, **column_dimensions) - end + rollup(column_name, beginning, **column_dimensions) + reduce_rollup(column_name, beginning, :sum, **column_dimensions) end end From 26b2d34b4d031dd21a31c490ecd11bb5b3cb8064 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 19:35:14 -0300 Subject: [PATCH 154/251] =?UTF-8?q?simplificar=20el=20c=C3=B3digo!!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit envía todos los parámetros a la base de datos para acelerar por un montón la generación de datos --- app/jobs/concerns/recursive_rollup.rb | 5 +- app/jobs/stat_collection_job.rb | 8 ++-- app/jobs/uri_collection_job.rb | 69 ++++++++++----------------- 3 files changed, 33 insertions(+), 49 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 4d9e11b0..1da14c01 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -30,17 +30,16 @@ module RecursiveRollup # intervalo más amplio. # # @param :name [String] - # @param :beginning [Time] # @param :operation [Symbol] # @param :dimensions [Hash] # @return [nil] - def reduce_rollup(name, beginning, operation, **dimensions) + def reduce_rollup(name:, dimensions:, operation: :sum) Stat::INTERVALS.reduce do |previous, current| recursive_rollup(name: name, interval_previous: previous, interval: current, dimensions: dimensions, - beginning: beginning, + beginning: beginning_of_interval, operation: operation) # Devolver el intervalo actual diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 63d44999..e402e3b5 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -22,9 +22,11 @@ class StatCollectionJob < PeriodicJob rollup.average(:seconds) end - reduce_rollup('builds', beginning, :sum, site_id: site_id) - reduce_rollup('space_used', beginning, :average, site_id: site_id) - reduce_rollup('build_time', beginning, :average, site_id: site_id) + dimensions = { site_id: site_id } + + reduce_rollup(name: 'builds', operation: :sum, dimensions: dimensions) + reduce_rollup(name: 'space_used', operation: :average, dimensions: dimensions) + reduce_rollup(name: 'build_time', operation: :average, dimensions: dimensions) stat.touch run_again! unless once diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 90789aba..a8c51d84 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -22,7 +22,7 @@ class UriCollectionJob < PeriodicJob @site = Site.find site_id # Obtener el principio del intervalo anterior - beginning = beginning_of_interval + beginning_of_interval # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME # Columnas a agrupar @@ -33,51 +33,33 @@ class UriCollectionJob < PeriodicJob # rápido buscar en los rollups indexados que en la tabla en bruto. # # Los referers solo se agrupan por host. - site.hostnames.each do |host| - break if stop? + columns.each_key do |column| + columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) + end - host_dimensions = { host: host } - columns.each_key do |column| - columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) - end + # Las URIs son la fuente de verdad de las visitas, porque son las + # que indican las páginas y recursos descargables, el resto son + # imágenes, CSS, JS y tipografías que no nos aportan números + # significativos. + uri_dimensions = { host: site.hostnames, uri: uris } + host_dimensions = { host: site.hostnames } - # Las URIs son la fuente de verdad de las visitas, porque son las - # que indican las páginas y recursos descargables, el resto son - # imágenes, CSS, JS y tipografías que no nos aportan números - # significativos. - name = 'host|uri' - dimensions = { host: host, uri: uris } + # Cantidad de visitas por host + rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) + reduce_rollup(name: 'host', operation: :sum, dimensions: uri_dimensions) - rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, :sum, **dimensions) + # Cantidad de visitas por página/recurso + rollup(name: 'host|uri', dimensions: uri_dimensions) + reduce_rollup(name: 'host|uri', operation: :sum, dimensions: uri_dimensions) - columns.each_pair do |column, values| - # Obtener orígenes de visitas por host - values.each do |value| - column_name = "host|uri|#{column}" - column_dimensions = { host: host, uri: uris } - column_dimensions[column] = value + # Cantidad de visitas host y parámetro + columns.each_pair do |column, values| + column_name = "host|#{column}" + column_dimensions = { host: site.hostnames } + column_dimensions[column] = values - rollup(column_name, beginning, **column_dimensions) - reduce_rollup(column_name, beginning, :sum, **column_dimensions) - end - end - - # Reducir todas las visitas a cantidad de visitas por host - rollup('host', beginning, **host_dimensions) - - # Acumular por mes y año - reduce_rollup('host', beginning, :sum, **host_dimensions) - - columns.each_pair do |column, values| - values.each do |value| - column_dimensions = { host: host } - column_dimensions[column] = value - - rollup("host|#{column}", beginning, **column_dimensions) - reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions) - end - end + rollup(name: column_name, dimensions: column_dimensions, filter: uri_dimensions) + reduce_rollup(name: column_name, dimensions: column_dimensions) end stat.touch @@ -92,9 +74,10 @@ class UriCollectionJob < PeriodicJob # @param :name [String] # @param :beginning [Time] # @param :dimensions [Hash] + # @param :filter [Hash] # @return [nil] - def rollup(name, beginning, **dimensions) - AccessLog.where(**dimensions) + def rollup(name:, beginning:, dimensions:, filter: nil) + AccessLog.where(**(filter || dimensions)) .where('created_at >= ?', beginning) .completed_requests .non_robots From 6464f35625f26ef1627682ad97ca4a143d0d8179 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 19:38:32 -0300 Subject: [PATCH 155/251] =?UTF-8?q?fixup!=20simplificar=20el=20c=C3=B3digo?= =?UTF-8?q?!!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index a8c51d84..bab85d0d 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -28,6 +28,13 @@ class UriCollectionJob < PeriodicJob # Columnas a agrupar columns = Stat::COLUMNS.zip([nil]).to_h + # Las URIs son la fuente de verdad de las visitas, porque son las + # que indican las páginas y recursos descargables, el resto son + # imágenes, CSS, JS y tipografías que no nos aportan números + # significativos. + uri_dimensions = { host: site.hostnames, uri: uris } + host_dimensions = { host: site.hostnames } + # Recorremos todos los hostnames y uris posibles y luego agrupamos # recursivamente para no tener que recalcular, asumiendo que es más # rápido buscar en los rollups indexados que en la tabla en bruto. @@ -37,13 +44,6 @@ class UriCollectionJob < PeriodicJob columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) end - # Las URIs son la fuente de verdad de las visitas, porque son las - # que indican las páginas y recursos descargables, el resto son - # imágenes, CSS, JS y tipografías que no nos aportan números - # significativos. - uri_dimensions = { host: site.hostnames, uri: uris } - host_dimensions = { host: site.hostnames } - # Cantidad de visitas por host rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) reduce_rollup(name: 'host', operation: :sum, dimensions: uri_dimensions) From 8a3c9a76a72d2e384005f81af0013f3dad54e23e Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 19:44:02 -0300 Subject: [PATCH 156/251] =?UTF-8?q?fixup!=20fixup!=20simplificar=20el=20c?= =?UTF-8?q?=C3=B3digo!!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index bab85d0d..ca755ca1 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -76,9 +76,9 @@ class UriCollectionJob < PeriodicJob # @param :dimensions [Hash] # @param :filter [Hash] # @return [nil] - def rollup(name:, beginning:, dimensions:, filter: nil) + def rollup(name:, dimensions:, filter: nil) AccessLog.where(**(filter || dimensions)) - .where('created_at >= ?', beginning) + .where('created_at >= ?', beginning_of_interval) .completed_requests .non_robots .group(*dimensions.keys) From 158c2244490fc79467d915e4841b6d54a34e4a49 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:40:54 -0300 Subject: [PATCH 157/251] generar los rollups para los siguientes intervalos --- app/jobs/uri_collection_job.rb | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ca755ca1..0ea468da 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -11,8 +11,6 @@ # Los hostnames de un sitio van a poder obtenerse a partir de # Site#hostnames con la garantía de que son únicos. class UriCollectionJob < PeriodicJob - include RecursiveRollup - # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las # estadísticas. IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze @@ -76,13 +74,29 @@ class UriCollectionJob < PeriodicJob # @param :dimensions [Hash] # @param :filter [Hash] # @return [nil] - def rollup(name:, dimensions:, filter: nil) + def rollup(name:, dimensions:, interval: starting_interval, filter: nil) AccessLog.where(**(filter || dimensions)) .where('created_at >= ?', beginning_of_interval) .completed_requests .non_robots .group(*dimensions.keys) - .rollup(name, interval: starting_interval, update: true) + .rollup(name, interval: interval, update: true) + end + + # Generar rollups con el resto de la información + # + # @param :name [String] + # @param :dimensions [Hash] + # @param :filter [Hash] + # @return [nil] + def reduce_rollup(name:, dimensions:, filter: nil) + Stat::INTERVALS.reduce do |_previous, current| + rollup(name: name, dimensions: dimensions, filter: filter, interval: current) + + current + end + + nil end def stat_name From 3f29d87acdb91d59f79fd38f105ea038c1909a94 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:41:38 -0300 Subject: [PATCH 158/251] =?UTF-8?q?las=20tablas=20tambi=C3=A9n=20est=C3=A1?= =?UTF-8?q?n=20filtradas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 36b72721..c2eb0425 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -46,17 +46,17 @@ class StatsController < ApplicationController Stat::COLUMNS.each do |column| @columns[column] = Rails.cache.fetch("stats/#{column}/#{site.id}", expires_in: expires_in) do - Rollup.where(interval: interval, name: "host|#{column}") - .where_dimensions(host: hostnames) - .group("dimensions->>'#{column}'") - .sum(:value) - .transform_values(&:to_i) - .transform_values do |v| - v * nodes - end - .sort_by do |_, v| - v - end.reverse.to_h + rollup_scope.where(interval: interval, name: "host|#{column}") + .where_dimensions(host: hostnames) + .group("dimensions->>'#{column}'") + .sum(:value) + .transform_values(&:to_i) + .transform_values do |v| + v * nodes + end + .sort_by do |_, v| + v + end.reverse.to_h end end end From c8ff6de9ae5ab32ce58b754cbd449b9fb5182ce7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:42:10 -0300 Subject: [PATCH 159/251] trasladar los parametros a los graficos --- app/controllers/stats_controller.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index c2eb0425..03d5fc9b 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -34,7 +34,14 @@ class StatsController < ApplicationController def index breadcrumb I18n.t('stats.index.title'), '' - @chart_params = { interval: interval } + params.with_defaults! default_url_options + + @chart_params = { + interval: interval, + period_start: params[:period_start], + period_end: params[:period_end] + } + hostnames last_stat chart_options From 62bdfff98c5ef174e8c7ed9246aee1fbd44f9f4d Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:42:38 -0300 Subject: [PATCH 160/251] poder ver y filtrar por rangos --- app/views/stats/index.haml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 6bbf596e..8987560f 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -9,9 +9,13 @@ %time{ datetime: @last_stat.updated_at } #{time_ago_in_words @last_stat.updated_at}. - .mb-5 + %form.mb-5.form-inline{ method: 'get' } - Stat::INTERVALS.each do |interval| - = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls]), class: "btn #{'btn-primary active' if @interval == interval}" + = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls], period_start: params[:period_start].to_date.try(:"beginning_of_#{interval}").to_date, period_end: params[:period_end]), class: "mb-0 btn #{'btn-primary active' if @interval == interval}" + + %input.form-control{ type: 'date', name: :period_start, value: params[:period_start] } + %input.form-control{ type: 'date', name: :period_end, value: params[:period_end] } + %button.btn.mb-0{ type: 'submit' }= t('.filter') .mb-5 %h2= t('.host.title', count: @hostnames.size) From 8ca5085aae92c72492a6ff6380f130aa0edc0012 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:48:53 -0300 Subject: [PATCH 161/251] fixup! generar los rollups para los siguientes intervalos --- app/jobs/uri_collection_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 0ea468da..42c05963 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -44,11 +44,11 @@ class UriCollectionJob < PeriodicJob # Cantidad de visitas por host rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) - reduce_rollup(name: 'host', operation: :sum, dimensions: uri_dimensions) + reduce_rollup(name: 'host', dimensions: uri_dimensions) # Cantidad de visitas por página/recurso rollup(name: 'host|uri', dimensions: uri_dimensions) - reduce_rollup(name: 'host|uri', operation: :sum, dimensions: uri_dimensions) + reduce_rollup(name: 'host|uri', dimensions: uri_dimensions) # Cantidad de visitas host y parámetro columns.each_pair do |column, values| From 35d16b11397c5ae5ee3a442cd30827080a7c4717 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 21:21:58 -0300 Subject: [PATCH 162/251] fixup! fixup! generar los rollups para los siguientes intervalos --- app/jobs/uri_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 42c05963..59f753ec 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -44,7 +44,7 @@ class UriCollectionJob < PeriodicJob # Cantidad de visitas por host rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) - reduce_rollup(name: 'host', dimensions: uri_dimensions) + reduce_rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) # Cantidad de visitas por página/recurso rollup(name: 'host|uri', dimensions: uri_dimensions) From 755db601a161a5cba148ec6567959fa2c6b1dace Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 May 2022 13:41:37 -0300 Subject: [PATCH 163/251] ordenar en la base de datos --- app/controllers/stats_controller.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 03d5fc9b..cccdb8b6 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -56,14 +56,12 @@ class StatsController < ApplicationController rollup_scope.where(interval: interval, name: "host|#{column}") .where_dimensions(host: hostnames) .group("dimensions->>'#{column}'") + .order('sum(value) desc') .sum(:value) .transform_values(&:to_i) .transform_values do |v| v * nodes end - .sort_by do |_, v| - v - end.reverse.to_h end end end From 6a7511a5e1c1398f770ca67194acc7b00f935c72 Mon Sep 17 00:00:00 2001 From: Nulo Date: Mon, 2 May 2022 18:55:35 +0000 Subject: [PATCH 164/251] Arreglar scroll al reordenar posts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Utilizaba https://developer.mozilla.org/en-US/docs/Web/API/Element/scrollIntoViewIfNeeded que es propietario. Ahora usa scrollIntoView siempre, configurado para que prácticamente siempre se pueda ver el post al reordenar. --- app/javascript/controllers/reorder_controller.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/app/javascript/controllers/reorder_controller.js b/app/javascript/controllers/reorder_controller.js index dca6e166..2cba4163 100644 --- a/app/javascript/controllers/reorder_controller.js +++ b/app/javascript/controllers/reorder_controller.js @@ -103,11 +103,7 @@ export default class extends Controller { this.reorder() // Mantenemos el primero a la vista - if ("scrollIntoViewIfNeeded" in rows[0].row) { - rows[0].row.scrollIntoViewIfNeeded() - } else { - rows[0].row.scrollIntoView() - } + rows[0].row.scrollIntoView({ block: "center" }); } counter () { @@ -146,7 +142,7 @@ export default class extends Controller { this.reorder() // Mantenemos el primero a la vista - rows[0].row.scrollIntoViewIfNeeded() + rows[0].row.scrollIntoView({ block: "center" }); } bottom (event) { @@ -167,7 +163,7 @@ export default class extends Controller { this.reorder() // Mantenemos el primero a la vista - rows[0].row.scrollIntoViewIfNeeded() + rows[0].row.scrollIntoView({ block: "center" }); } /* From 844ba75d08b213c42e128ea6578c91d91b8369e7 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 30 May 2022 15:10:52 -0300 Subject: [PATCH 165/251] =?UTF-8?q?indicar=20qu=C3=A9=20significa=20lo=20v?= =?UTF-8?q?ac=C3=ADo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 2 +- config/locales/en.yml | 3 ++- config/locales/es.yml | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 8987560f..058f17ef 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -55,7 +55,7 @@ %tbody - values.each_pair do |col, val| %tr - %th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t('.columns.empty') : col + %th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t(".columns.#{column}.empty") : col %td= val .mb-5 %h2= t('.resources.title') diff --git a/config/locales/en.yml b/config/locales/en.yml index ec8a5792..87a18ff6 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -288,15 +288,16 @@ en: description: 'Average time your site takes to build.' columns: visits: "Visits" - empty: "(nothing)" http_referer: title: "Referers" description: "Visits by origin" column: "Referer" + empty: "(direct visit)" geoip2_data_country_name: title: "Countries" description: "Visits by country" column: "Country" + empty: "(couldn't detect country)" sites: donations: url: 'https://donaciones.sutty.nl/en/' diff --git a/config/locales/es.yml b/config/locales/es.yml index 30ea15fc..3c9d11b5 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -293,15 +293,16 @@ es: description: 'Tiempo promedio que toma en publicarse tu sitio.' columns: visits: "Visitas" - empty: "(nada)" http_referer: title: "Referencias" description: "Orígenes de las visitas" column: "Referencia" + empty: "(visita directa)" geoip2_data_country_name: title: "Países" description: "Cantidad de visitas por país" column: "País" + empty: "(no se pudo detectar el país)" sites: donations: url: 'https://donaciones.sutty.nl/' From 472085301960d6829aa4c1d195cda4bc96fcac78 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 11 Jun 2022 19:33:02 -0300 Subject: [PATCH 166/251] =?UTF-8?q?al=20recargar=20la=20p=C3=A1gina=20volv?= =?UTF-8?q?er=20al=20gr=C3=A1fico?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 058f17ef..49fbd023 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -22,10 +22,10 @@ %p.lead= t('.host.description') = line_chart site_stats_host_path(@chart_params), **@chart_options - .mb-5 + #custom-urls.mb-5 %h2= t('.urls.title') %p.lead= t('.urls.description') - %form + %form{ method: 'get', action: '#custom-urls' } %input{ type: 'hidden', name: 'interval', value: @interval } .form-group %label{ for: 'urls' }= t('.urls.label') From 6f45756e6d6b5ea12386288821fd45d6b4ebed1b Mon Sep 17 00:00:00 2001 From: f Date: Sat, 11 Jun 2022 19:33:21 -0300 Subject: [PATCH 167/251] =?UTF-8?q?mejorar=20el=20mensaje=20de=20tiempo=20?= =?UTF-8?q?de=20compilaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/locales/en.yml | 2 +- config/locales/es.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 87a18ff6..f89778ed 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -285,7 +285,7 @@ en: description: 'Average storage space used by your site.' build_time: title: 'Publication time' - description: 'Average time your site takes to build.' + description: 'Average time your site takes to build, from pressing "Publish changes" to actually being available on your site.' columns: visits: "Visits" http_referer: diff --git a/config/locales/es.yml b/config/locales/es.yml index 3c9d11b5..bf3e93f0 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -290,7 +290,7 @@ es: description: 'Espacio en disco que ocupa en promedio tu sitio.' build_time: title: 'Tiempo de publicación' - description: 'Tiempo promedio que toma en publicarse tu sitio.' + description: 'Tiempo que tarda el sitio en generarse, desde que usas el botón "Publicar cambios" hasta que los puedes ver en el sitio' columns: visits: "Visitas" http_referer: From 00f303cf20f8b7338e402eac0d9baf8465b822e5 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 11 Jun 2022 19:42:55 -0300 Subject: [PATCH 168/251] =?UTF-8?q?procesar=20las=20estad=C3=ADsticas=20au?= =?UTF-8?q?tom=C3=A1ticamente?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Procfile | 1 + lib/tasks/stats.rake | 11 +++++++++++ monit.conf | 5 +++++ 3 files changed, 17 insertions(+) create mode 100644 lib/tasks/stats.rake diff --git a/Procfile b/Procfile index b308ffd5..8f6c7741 100644 --- a/Procfile +++ b/Procfile @@ -5,3 +5,4 @@ blazer_1h: bundle exec rake blazer:run_checks SCHEDULE="1 hour" blazer_1d: bundle exec rake blazer:run_checks SCHEDULE="1 day" blazer: bundle exec rake blazer:send_failing_checks prometheus: bundle exec prometheus_exporter -b 0.0.0.0 --prefix "sutty_" +stats: bundle exec rake stats:process_all diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake new file mode 100644 index 00000000..9461782a --- /dev/null +++ b/lib/tasks/stats.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +namespace :stats do + desc 'Process stats' + task process_all: :environment do + Site.all.pluck(:id).each do |site_id| + UriCollectionJob.perform_now site_id: site_id, once: true + StatCollectionJob.perform_now site_id: site_id, once: true + end + end +end diff --git a/monit.conf b/monit.conf index 28db0cf0..83d17449 100644 --- a/monit.conf +++ b/monit.conf @@ -30,3 +30,8 @@ check program access_logs with path "/srv/http/bin/access_logs" as uid "app" and gid "www-data" every "0 0 * * *" if status != 0 then alert + +check program stats + with path "/usr/bin/foreman run -f /srv/Procfile -d /srv stats" as uid "rails" gid "www-data" + every "0 1 * * *" + if status != 0 then alert From a0441a858e960360997d412ce44192f4393e49a3 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 11 Jun 2022 19:51:11 -0300 Subject: [PATCH 169/251] comandos correctos --- Procfile | 2 +- monit.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Procfile b/Procfile index 48931e8d..112b6042 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -cleanup: rake cleanup:everything +cleanup: bundle exec rake cleanup:everything diff --git a/monit.conf b/monit.conf index 07cd1c1e..ea6925fd 100644 --- a/monit.conf +++ b/monit.conf @@ -1,5 +1,5 @@ # Limpiar mensualmente check program cleanup - with path "/usr/bin/foreman run -f /srv/Procfile -d /srv " as uid "rails" gid "http-data" + with path "/usr/bin/foreman run -f /srv/Procfile -d /srv cleanup" as uid "rails" gid "http-data" every "0 3 1 * *" if status != 0 then alert From 369b0c9937ea1b7baf8112d36e17b7df2f5dd45c Mon Sep 17 00:00:00 2001 From: f Date: Sat, 11 Jun 2022 19:52:38 -0300 Subject: [PATCH 170/251] fixup! comandos correctos --- monit.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monit.conf b/monit.conf index ea6925fd..20e4c703 100644 --- a/monit.conf +++ b/monit.conf @@ -1,5 +1,5 @@ # Limpiar mensualmente check program cleanup - with path "/usr/bin/foreman run -f /srv/Procfile -d /srv cleanup" as uid "rails" gid "http-data" + with path "/usr/bin/foreman run -f /srv/Procfile -d /srv cleanup" as uid "rails" gid "www-data" every "0 3 1 * *" if status != 0 then alert From dd34d176ea53cd24b157ce3aaf5da238f9c47217 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 15 Jun 2022 13:34:46 -0300 Subject: [PATCH 171/251] sincronizar links duros --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index b2ffba7c..996f8cdd 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -84,7 +84,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -avi --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -aviH --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From 24c0161aac6451afc65f7f222bc4acbe00b11d2e Mon Sep 17 00:00:00 2001 From: f Date: Wed, 29 Jun 2022 18:17:23 -0300 Subject: [PATCH 172/251] =?UTF-8?q?no=20fallar=20si=20los=20layouts=20no?= =?UTF-8?q?=20tienen=20traducci=C3=B3n=20#6841?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/site.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/site.rb b/app/models/site.rb index 7d4875e5..58947263 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -166,8 +166,10 @@ class Site < ApplicationRecord end # Similar a site.i18n en jekyll-locales + # + # @return [Hash] def i18n - data[I18n.locale.to_s] + data[I18n.locale.to_s] || {} end # Devuelve el idioma por defecto del sitio, el primero de la lista. From 655c6b65d59c37be310c4c48f0b4c2cad7939a9c Mon Sep 17 00:00:00 2001 From: f Date: Fri, 1 Jul 2022 14:43:30 -0300 Subject: [PATCH 173/251] procesar todas las urls a veces las urls vienen como arrays, con lo que no hay que procesarlas. --- app/controllers/stats_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index cccdb8b6..c2c7bc58 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -133,7 +133,8 @@ class StatsController < ApplicationController def normalized_urls @normalized_urls ||= begin - urls = params.permit(:urls).try(:[], :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| + urls = params[:urls].is_a?(Array) ? params[:urls] : params[:urls]&.split("\n") + urls = urls&.map(&:strip)&.select(&:present?)&.select do |uri| uri.start_with? 'https://' end From e6c03df13f9bf7d0d6e27e1fbc7cecb54cd280f0 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 1 Jul 2022 15:01:57 -0300 Subject: [PATCH 174/251] =?UTF-8?q?mostrar=20las=20estadisticas=20en=20el?= =?UTF-8?q?=20men=C3=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/posts/index.haml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index accfdb34..c6ffc2f5 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -21,6 +21,9 @@ - if policy(@site).private? = link_to t('sites.private'), '../private/' + @site.name, class: 'btn', target: '_blank', rel: 'noopener' + - if policy(SiteStat.new(@site)).index? + = link_to t('stats.index.title'), site_stats_path(@site), class: 'btn' + - if policy(SiteUsuarie.new(@site, current_usuarie)).index? = render 'layouts/btn_with_tooltip', tooltip: t('usuaries.index.help.self'), From f1137c6ed28c5245f27865c489904a1012f119b3 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 1 Jul 2022 15:02:17 -0300 Subject: [PATCH 175/251] error de sintaxis --- config/application.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/application.rb b/config/application.rb index 7c530e69..f1bf8365 100644 --- a/config/application.rb +++ b/config/application.rb @@ -38,6 +38,7 @@ module Sutty config.active_storage.variant_processor = :vips + config.to_prepare do # Load application's model / class decorators Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')).sort.each do |c| Rails.configuration.cache_classes ? require(c) : load(c) From 9e531e49955801bd41a4a8a48ca8d764ebfe342a Mon Sep 17 00:00:00 2001 From: f Date: Tue, 12 Jul 2022 15:38:16 -0300 Subject: [PATCH 176/251] no cambiar el valor si no se pudo cargar el archivo esto hace que sutilmente desaparezcan archivos --- app/models/metadata_file.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 71d3f049..2b628a57 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -41,14 +41,10 @@ class MetadataFile < MetadataTemplate end # Asociar la imagen subida al sitio y obtener la ruta - # - # XXX: Si evitamos guardar cambios con changed? no tenemos forma de - # saber que un archivo subido manualmente se convirtió en - # un Attachment y cada vez que lo editemos vamos a subir una imagen - # repetida. + # @return [Boolean] def save value['description'] = sanitize value['description'] - value['path'] = static_file ? relative_destination_path_with_filename.to_s : nil + value['path'] = relative_destination_path_with_filename.to_s if static_file true end From 0b06bf35417b1f23176f798611c70ffc49822985 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 12 Jul 2022 15:39:08 -0300 Subject: [PATCH 177/251] notificar cuando no se pudo cargar el archivo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit no perder información silenciosamente --- app/models/metadata_file.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 2b628a57..942b448d 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -58,9 +58,6 @@ class MetadataFile < MetadataTemplate # * El archivo es una ruta que apunta a un archivo asociado al sitio # * El archivo es una ruta a un archivo dentro del repositorio # - # XXX: La última opción provoca archivos duplicados, pero es lo mejor - # que tenemos hasta que resolvamos https://0xacab.org/sutty/sutty/-/issues/213 - # # @todo encontrar una forma de obtener el attachment sin tener que # recurrir al último subido. # @@ -77,8 +74,12 @@ class MetadataFile < MetadataTemplate site.static_files.last.tap do |s| s.blob.update(key: key_from_path) end + else + raise ArgumentError, 'No se pudo subir el archivo' end end + rescue ArgumentError => e + ExceptionNotifier.notify_exception(e, data: { site: site.name, path: value['path'] }) end # Obtiene la ruta absoluta al archivo From def4402313e0d142b72321192c1b09783fdfe520 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 12 Jul 2022 15:39:42 -0300 Subject: [PATCH 178/251] permitir cargar archivos con la misma key podemos tener sitios distintos con la misma llave identificatoria. ahora permitimos solo una llave por cada sitio distinto. si subimos un archivo duplicado en un mismo sitio el error va a seguir siendo el mismo. --- app/models/metadata_file.rb | 2 +- ...nge_blob_key_uniqueness_to_include_service_name.rb | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20220712135053_change_blob_key_uniqueness_to_include_service_name.rb diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 942b448d..fa570ac0 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -68,7 +68,7 @@ class MetadataFile < MetadataTemplate when ActionDispatch::Http::UploadedFile site.static_files.last if site.static_files.attach(value['path']) when String - if (blob_id = ActiveStorage::Blob.where(key: key_from_path).pluck(:id).first) + if (blob_id = ActiveStorage::Blob.where(key: key_from_path, service_name: site.name).pluck(:id).first) site.static_files.find_by(blob_id: blob_id) elsif path? && pathname.exist? && site.static_files.attach(io: pathname.open, filename: pathname.basename) site.static_files.last.tap do |s| diff --git a/db/migrate/20220712135053_change_blob_key_uniqueness_to_include_service_name.rb b/db/migrate/20220712135053_change_blob_key_uniqueness_to_include_service_name.rb new file mode 100644 index 00000000..00bae7ea --- /dev/null +++ b/db/migrate/20220712135053_change_blob_key_uniqueness_to_include_service_name.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +# Cambia el índice único para incluir el nombre del servicio, de forma +# que podamos tener varias copias del mismo sitio (por ejemplo para +# test) sin que falle la creación de archivos. +class ChangeBlobKeyUniquenessToIncludeServiceName < ActiveRecord::Migration[6.1] + def change + remove_index :active_storage_blobs, %i[key], unique: true + add_index :active_storage_blobs, %i[key service_name], unique: true + end +end From cd523babade074fd69471e9c54cf1c391a2730da Mon Sep 17 00:00:00 2001 From: f Date: Tue, 12 Jul 2022 15:58:31 -0300 Subject: [PATCH 179/251] siempre devolver Pathname closes #6982 closes #6979 closes #6974 closes #6973 closes #6972 closes #6970 closes #6968 closes #6967 --- app/models/metadata_file.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index fa570ac0..4ea026b4 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -126,7 +126,7 @@ class MetadataFile < MetadataTemplate rescue Errno::ENOENT => e ExceptionNotifier.notify_exception(e) - value['path'] + Pathname.new(value['path']) end def relative_destination_path_with_filename From aab8ed9fed5c1411da527814a78e5ad7db132bd6 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 13 Jul 2022 13:34:45 -0300 Subject: [PATCH 180/251] =?UTF-8?q?informar=20cuando=20el=20nombre=20de=20?= =?UTF-8?q?archivo=20est=C3=A1=20vac=C3=ADo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/lib/active_storage/service/jekyll_service.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 92b26e0e..3edd2653 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -67,7 +67,9 @@ module ActiveStorage # @param :key [String] # @return [String] def filename_for(key) - ActiveStorage::Blob.where(key: key).limit(1).pluck(:filename).first + ActiveStorage::Blob.where(key: key).limit(1).pluck(:filename).first.tap do |filename| + raise ArgumentError, "Filename for key #{key} is blank" if filename.blank? + end end # Crea una ruta para la llave con un nombre conocido. From 5397ae66a2709aeb30da2dbb2b9064cf3e42e8c7 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 13 Jul 2022 14:18:39 -0300 Subject: [PATCH 181/251] informar con mas detalle cuando el archivo no existe closes #6991 closes #6990 closes #6985 closes #6969 closes #6966 --- app/models/metadata_file.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 4ea026b4..00ac7af0 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -124,13 +124,15 @@ class MetadataFile < MetadataTemplate # devolvemos la ruta original, que puede ser el archivo que no existe # o vacía si se está subiendo uno. rescue Errno::ENOENT => e - ExceptionNotifier.notify_exception(e) + ExceptionNotifier.notify_exception(e, data: { site: site.name, path: value['path'] }) Pathname.new(value['path']) end def relative_destination_path_with_filename destination_path_with_filename.relative_path_from(Pathname.new(site.path).realpath) + rescue ArgumentError => e + ExceptionNotifier.notify_exception(e, data: { site: site.name, path: value['path'] }) end def static_file_path From 2a416b4e670fc8690538a2de70d501b72e0ee436 Mon Sep 17 00:00:00 2001 From: Nulo Date: Wed, 13 Jul 2022 21:43:44 +0000 Subject: [PATCH 182/251] editor: No reportar error cuando no se selecciona un archivo para subir --- app/javascript/editor/types/multimedia.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/javascript/editor/types/multimedia.ts b/app/javascript/editor/types/multimedia.ts index 2af9643a..29d2a0ab 100644 --- a/app/javascript/editor/types/multimedia.ts +++ b/app/javascript/editor/types/multimedia.ts @@ -137,8 +137,10 @@ export function setupAuxiliaryToolbar(editor: Editor): void { "click", (event) => { const files = editor.toolbar.auxiliary.multimedia.fileEl.files; - if (!files || !files.length) - throw new Error("no hay archivos para subir"); + if (!files || !files.length) { + console.info("no hay archivos para subir"); + return; + } const file = files[0]; const selectedEl = editor.contentEl.querySelector( From 332ae66b8a54636b9bafea7f992719deb6af8122 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Jul 2022 13:19:08 -0300 Subject: [PATCH 183/251] =?UTF-8?q?no=20juntar=20estad=C3=ADsticas=20por?= =?UTF-8?q?=20mes=20y=20a=C3=B1o=20todav=C3=ADa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit la forma en que las estamos recopilando es rápida pero no acumulativa, entonces los números siempre dan mal. --- app/models/stat.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/stat.rb b/app/models/stat.rb index f1016026..6c681aa4 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -3,7 +3,7 @@ # Registran cuándo fue la última recolección de datos. class Stat < ApplicationRecord # XXX: Los intervalos van en orden de mayor especificidad a menor - INTERVALS = %i[day month year].freeze + INTERVALS = %i[day].freeze RESOURCES = %i[builds space_used build_time].freeze COLUMNS = %i[http_referer geoip2_data_country_name].freeze From 091ceac788595edf537d81ffd929c0dc5084f230 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Jul 2022 14:17:46 -0300 Subject: [PATCH 184/251] =?UTF-8?q?bot=C3=B3n=20de=20estad=C3=ADsticas=20m?= =?UTF-8?q?=C3=A1s=20arriba?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/posts_controller.rb | 2 ++ app/views/posts/index.haml | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index e6f836bb..c5dc0f54 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -35,6 +35,8 @@ class PostsController < ApplicationController # Filtrar los posts que les invitades no pueden ver @usuarie = site.usuarie? current_usuarie + + @site_stat = SiteStat.new(site) end def show diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index c6ffc2f5..bc5c826c 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -15,15 +15,15 @@ - else %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary btn-sm' + - if policy(@site_stat).index? + = link_to t('stats.index.title'), site_stats_path(@site), class: 'btn' + - if policy(@site).edit? = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn' - if policy(@site).private? = link_to t('sites.private'), '../private/' + @site.name, class: 'btn', target: '_blank', rel: 'noopener' - - if policy(SiteStat.new(@site)).index? - = link_to t('stats.index.title'), site_stats_path(@site), class: 'btn' - - if policy(SiteUsuarie.new(@site, current_usuarie)).index? = render 'layouts/btn_with_tooltip', tooltip: t('usuaries.index.help.self'), From 83384b686af0ddb9be18ee3fbbb10d1a9d3ea899 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Jul 2022 18:22:06 -0300 Subject: [PATCH 185/251] solo copiar el archivo si no existe --- app/lib/active_storage/service/jekyll_service.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 3edd2653..88ffa83c 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -20,6 +20,18 @@ module ActiveStorage end end + # Solo copiamos el archivo si no existe + # + # @param :key [String] + # @param :io [IO] + # @param :checksum [String] + def upload(key, io, checksum: nil, **) + instrument :upload, key: key, checksum: checksum do + IO.copy_stream(io, make_path_for(key)) unless exist?(key) + ensure_integrity_of(key, checksum) if checksum + end + end + # Lo mismo que en DiskService agregando el nombre de archivo en la # firma. Esto permite que luego podamos guardar el archivo donde # corresponde. From 6c62c20572a1a763ceb23b62eea924fed4b86e25 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Jul 2022 19:32:06 -0300 Subject: [PATCH 186/251] subir directamente los archivos a la base de datos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cuando subimos un archivo usando attach en realidad estábamos generando una copia y dejándola huérfana en el repositorio del sitio. usando blob podemos imitar un servicio de subida de archivos sin generar io innecesario. --- app/models/metadata_file.rb | 42 ++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 00ac7af0..c7296de4 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -68,18 +68,8 @@ class MetadataFile < MetadataTemplate when ActionDispatch::Http::UploadedFile site.static_files.last if site.static_files.attach(value['path']) when String - if (blob_id = ActiveStorage::Blob.where(key: key_from_path, service_name: site.name).pluck(:id).first) - site.static_files.find_by(blob_id: blob_id) - elsif path? && pathname.exist? && site.static_files.attach(io: pathname.open, filename: pathname.basename) - site.static_files.last.tap do |s| - s.blob.update(key: key_from_path) - end - else - raise ArgumentError, 'No se pudo subir el archivo' - end + site.static_files.find_by(blob_id: blob_id) || migrate_static_file! end - rescue ArgumentError => e - ExceptionNotifier.notify_exception(e, data: { site: site.name, path: value['path'] }) end # Obtiene la ruta absoluta al archivo @@ -95,7 +85,7 @@ class MetadataFile < MetadataTemplate # # @return [String] def key_from_path - pathname.dirname.basename.to_s + @key_from_path ||= pathname.dirname.basename.to_s end def path? @@ -148,4 +138,32 @@ class MetadataFile < MetadataTemplate def no_file_for_description? !path? && description? end + + # Obtiene el id del blob asociado + # + # @return [Integer,nil] + def blob_id + @blob_id ||= ActiveStorage::Blob.where(key: key_from_path, service_name: site.name).pluck(:id).first + end + + # Genera el blob para un archivo que ya se encuentra en el + # repositorio y lo agrega a la base de datos. + # + # @return [ActiveStorage::Attachment] + def migrate_static_file! + raise ArgumentError, 'El archivo no existe' unless path? && pathname.exist? + + Site.transaction do + blob = + ActiveStorage::Blob.create_after_unfurling!(key: key_from_path, + io: pathname.open, + filename: pathname.basename, + service_name: site.name) + + ActiveStorage::Attachment.create!(name: 'static_files', record: site, blob: blob) + end + rescue ArgumentError => e + ExceptionNotifier.notify_exception(e, data: { site: site.name, path: value['path'] }) + nil + end end From 69712a306e131e48f6c7416002e2e8f1f1cee515 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 25 Jul 2022 16:01:08 -0300 Subject: [PATCH 187/251] equiparar file con image #7076 --- app/views/posts/attributes/_file.haml | 16 +++++++++------- app/views/posts/attributes/_image.haml | 3 +-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/app/views/posts/attributes/_file.haml b/app/views/posts/attributes/_file.haml index 19175b9c..54f9f81a 100644 --- a/app/views/posts/attributes/_file.haml +++ b/app/views/posts/attributes/_file.haml @@ -1,7 +1,5 @@ .form-group - if metadata.static_file - = hidden_field_tag "#{base}[#{attribute}][path]", metadata.value['path'] - - case metadata.static_file.blob.content_type - when %r{\Avideo/} = video_tag url_for(metadata.static_file), @@ -14,13 +12,17 @@ - else = link_to t('posts.attribute_ro.file.download'), url_for(metadata.static_file) - .custom-control.custom-switch - = check_box_tag "#{base}[#{attribute}][path]", '', false, id: "#{base}_#{attribute}_destroy", class: 'custom-control-input' - = label_tag "#{base}_#{attribute}_destroy", t('posts.attributes.file.destroy'), class: 'custom-control-label' + -# Mantener el valor si no enviamos ninguna imagen + = hidden_field_tag "#{base}[#{attribute}][path]", metadata.value['path'] + -# Los archivos requeridos solo se pueden reemplazar + - unless metadata.required + .custom-control.custom-switch + = check_box_tag "#{base}[#{attribute}][path]", '', false, id: "#{base}_#{attribute}_destroy", class: 'custom-control-input' + = label_tag "#{base}_#{attribute}_destroy", t('posts.attributes.file.destroy'), class: 'custom-control-label' .custom-file = file_field(*field_name_for(base, attribute, :path), - **field_options(attribute, metadata), + **field_options(attribute, metadata, required: (metadata.required && !metadata.path?)), class: "custom-file-input #{invalid(post, attribute)}", data: { preview: "#{attribute}-preview" }) = label_tag "#{base}_#{attribute}_path", @@ -30,7 +32,7 @@ .form-group = label_tag "#{base}_#{attribute}_description", - post_label_t(attribute, :description, post: post) + post_label_t(attribute, :description, post: post, required: false) = text_field(*field_name_for(base, attribute, :description), value: metadata.value['description'], dir: dir, lang: locale, diff --git a/app/views/posts/attributes/_image.haml b/app/views/posts/attributes/_image.haml index c11f4196..f4d9bb3d 100644 --- a/app/views/posts/attributes/_image.haml +++ b/app/views/posts/attributes/_image.haml @@ -1,5 +1,5 @@ .form-group - - if metadata.uploaded? + - if metadata.static_file = image_tag url_for(metadata.static_file), alt: metadata.value['description'], class: 'img-fluid', @@ -37,4 +37,3 @@ **field_options(attribute, metadata, required: false)) = render 'posts/attribute_feedback', post: post, attribute: [attribute, :description], metadata: metadata - From 0d6a2c020c0a7efa0f16c3af41d70cafc41f688a Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Jul 2022 13:49:59 -0300 Subject: [PATCH 188/251] reindexar los sitios al terminar de compilarlos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit para poder soportar modificaciones durante la generación del sitio --- app/models/deploy_reindex.rb | 19 +++++++++++++++++++ app/views/deploys/_deploy_reindex.haml | 1 + config/locales/en.yml | 4 ++++ config/locales/es.yml | 4 ++++ 4 files changed, 28 insertions(+) create mode 100644 app/models/deploy_reindex.rb create mode 100644 app/views/deploys/_deploy_reindex.haml diff --git a/app/models/deploy_reindex.rb b/app/models/deploy_reindex.rb new file mode 100644 index 00000000..d2ff0679 --- /dev/null +++ b/app/models/deploy_reindex.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# Reindexa los artículos al terminar la compilación +class DeployReindex < Deploy + def deploy + site.reset + site.indexed_posts.destroy_all + site.index_posts! + site.touch + end + + def limit + 1 + end + + def hostname + nil + end +end diff --git a/app/views/deploys/_deploy_reindex.haml b/app/views/deploys/_deploy_reindex.haml new file mode 100644 index 00000000..af058968 --- /dev/null +++ b/app/views/deploys/_deploy_reindex.haml @@ -0,0 +1 @@ +-# NADA diff --git a/config/locales/en.yml b/config/locales/en.yml index 530a9381..b08ddc95 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -102,6 +102,10 @@ en: title: Alternative domain name success: Success! error: Error + deploy_reindex: + title: Reindex + success: Success! + error: Error help: You can contact us by replying to this e-mail maintenance_mailer: notice: diff --git a/config/locales/es.yml b/config/locales/es.yml index eaa23137..026f2bc5 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -102,6 +102,10 @@ es: title: Dominio alternativo success: ¡Éxito! error: Hubo un error + deploy_reindex: + title: Reindexación + success: ¡Éxito! + error: Hubo un error help: Por cualquier duda, responde este correo para contactarte con nosotres. maintenance_mailer: notice: From b983353223c8f46e8c03a4d9a6336500e011fc57 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Jul 2022 14:13:54 -0300 Subject: [PATCH 189/251] =?UTF-8?q?log=20de=20reindexaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_reindex.rb | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/app/models/deploy_reindex.rb b/app/models/deploy_reindex.rb index d2ff0679..2bc8f642 100644 --- a/app/models/deploy_reindex.rb +++ b/app/models/deploy_reindex.rb @@ -3,12 +3,29 @@ # Reindexa los artículos al terminar la compilación class DeployReindex < Deploy def deploy + time_start + site.reset - site.indexed_posts.destroy_all - site.index_posts! + + Site.transaction do + site.indexed_posts.destroy_all + site.index_posts! + end + + time_stop + + build_stats.create action: 'reindex', + log: 'Reindex', + seconds: time_spent_in_seconds, + bytes: size, + status: true site.touch end + def size + 0 + end + def limit 1 end From 70ee31325467db20b0f33fcf67d999846e838c85 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Jul 2022 14:50:46 -0300 Subject: [PATCH 190/251] tampoco tienen url --- app/models/deploy_reindex.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/models/deploy_reindex.rb b/app/models/deploy_reindex.rb index 2bc8f642..921451d9 100644 --- a/app/models/deploy_reindex.rb +++ b/app/models/deploy_reindex.rb @@ -33,4 +33,8 @@ class DeployReindex < Deploy def hostname nil end + + def url + nil + end end From 64e93506b77de088f3696a1588e5dfad2873bed8 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 3 Aug 2022 12:53:50 -0300 Subject: [PATCH 191/251] =?UTF-8?q?eliminar=20c=C3=B3digo=20que=20no=20se?= =?UTF-8?q?=20usa=20m=C3=A1s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/post/template_field.rb | 360 ------------------------------ 1 file changed, 360 deletions(-) delete mode 100644 app/models/post/template_field.rb diff --git a/app/models/post/template_field.rb b/app/models/post/template_field.rb deleted file mode 100644 index 08da933b..00000000 --- a/app/models/post/template_field.rb +++ /dev/null @@ -1,360 +0,0 @@ -# frozen_string_literal: true - -# Representa los distintos tipos de campos que pueden venir de una -# plantilla compleja -class Post - class TemplateField - attr_reader :post, :contents, :key - - STRING_VALUES = %w[string text url number email password date year - image video audio document].freeze - - # Tipo de valores que son archivos - FILE_TYPES = %w[image video audio document].freeze - - def initialize(post, key, contents) - @post = post - @key = key - @contents = contents - end - - def title - contents.dig('title') if complex? - end - - def subtitle - contents.dig('subtitle') if complex? - end - - # Obtiene el valor - def value - complex? ? contents.dig('value') : contents - end - - def max - return 0 if simple? - - contents.fetch('max', 0) - end - - def min - return 0 if simple? - - contents.fetch('min', 0) - end - - # TODO: volver elegante! - def type - return @type if @type - - if image? - @type = 'image' - elsif email? - @type = 'email' - elsif url? - @type = 'url' - elsif number? - @type = 'number' - elsif password? - @type = 'password' - elsif date? - @type = 'date' - elsif year? - @type = 'year' - elsif text_area? - @type = 'text_area' - elsif check_box_group? - @type = 'check_box_group' - elsif radio_group? - @type = 'radio_group' - elsif string? - @type = 'text' - # TODO: volver a hacer funcionar esto y ahorranos los multiple: - # false - elsif string? && contents.split('/', 2).count == 2 - @type = 'select' - elsif nested? - @type = 'table' - elsif array? - @type = 'select' - elsif boolean? - @type = 'check_box' - end - - @type - end - - # Devuelve los valores vacíos según el tipo - def empty_value - if string? - '' - elsif nested? - # TODO: devolver las keys también - {} - elsif array? - [] - elsif boolean? - false - end - end - - def cols - complex? && contents.dig('cols') - end - - def align - complex? && contents.dig('align') - end - - # El campo es requerido si es complejo y se especifica que lo sea - def required? - complex? && contents.dig('required') - end - - def boolean? - value.is_a?(FalseClass) || value.is_a?(TrueClass) - end - - def string? - value.is_a? String - end - - def text_area? - value == 'text' - end - - def url? - value == 'url' - end - - def email? - value == 'email' || value == 'mail' - end - alias mail? email? - - def date? - value == 'date' - end - - def password? - value == 'password' - end - - def number? - value == 'number' - end - - def year? - value == 'year' - end - - def file? - string? && FILE_TYPES.include?(value) - end - - def image? - array? ? value.first == 'image' : value == 'image' - end - - # Si la plantilla es simple no está admitiendo Hashes como valores - def simple? - !complex? - end - - def complex? - contents.is_a? Hash - end - - # XXX Retrocompatibilidad - def to_s - key - end - - # Convierte el campo en un parámetro - def to_param - if nested? - { key.to_sym => {} } - elsif array? && multiple? - { key.to_sym => [] } - else - key.to_sym - end - end - - # Convierte la plantilla en el formato de front_matter - def to_front_matter - { key => empty_value } - end - - def check_box_group? - array? && (complex? && contents.fetch('checkbox', false)) - end - - def radio_group? - array? && (complex? && contents.fetch('radio', false)) - end - - def array? - value.is_a? Array - end - - # TODO: detectar cuando es complejo y tomar el valor de :multiple - def multiple? - # si la plantilla es simple, es multiple cuando tenemos un array - return array? if simple? - - array? && contents.fetch('multiple', true) - end - - # Detecta si el valor es una tabla de campos - def nested? - value.is_a?(Hash) || (array? && value.first.is_a?(Hash)) - end - - # Un campo acepta valores abiertos si no es un array con múltiples - # elementos - def open? - # Todos los valores simples son abiertos - return true unless complex? - return false unless array? - - # La cosa se complejiza cuando tenemos valores complejos - # - # Si tenemos una lista cerrada de valores, necesitamos saber si el - # campo es abierto o cerrado. Si la lista tiene varios elementos, - # es una lista cerrada, opcionalmente abierta. Si la lista tiene - # un elemento, quiere decir que estamos autocompletando desde otro - # lado. - contents.fetch('open', value.count < 2) - end - - def closed? - !open? - end - - # Determina si los valores del campo serán públicos después - # - # XXX Esto es solo una indicación, el theme Jekyll tiene que - # respetarlos por su lado luego - def public? - # Todos los campos son públicos a menos que se indique lo - # contrario - simple? || contents.fetch('public', true) - end - - def private? - !public? - end - - def human - h = key.humanize - - h - end - - def label - h = (complex? && contents.dig('label')) || human - h += ' *' if required? - - h - end - - def help - complex? && contents.dig('help') - end - - def nested_fields - return unless nested? - - v = value - v = value.first if array? - - @nested_fields ||= v.map do |k, sv| - Post::TemplateField.new post, k, sv - end - end - - # Obtiene los valores posibles para el campo de la plantilla - def values - return 'false' if value == false - return 'true' if value == true - # XXX por alguna razón `value` no refiere a value() :/ - return '' if STRING_VALUES.include? value - # Las listas cerradas no necesitan mayor procesamiento - return value if array? && closed? && value.count > 1 - # Y las vacías tampoco - return value if array? && value.empty? - # Ahorrarnos el trabajo - return @values if @values - - # Duplicar el valor para no tener efectos secundarios luego (?) - value = self.value.dup - - # Para obtener los valores posibles, hay que procesar la string y - # convertirla a parametros - - # Si es una array de un solo elemento, es un indicador de que - # tenemos que rellenarla con los valores que indica. - # - # El primer valor es el que trae la string de autocompletado - values = array? ? value.shift : value - - # Si el valor es un array con más de un elemento, queremos usar - # esas opciones. Pero si además es abierto, queremos traer los - # valores cargados anteriormente. - - # Procesamos el valor, buscando : como separador de campos que - # queremos encontrar y luego los unimos - _value = (values&.split(':', 2) || []).map do |v| - # Tenemos hasta tres niveles de búsqueda - collection, attr, subattr = v.split('/', 3) - - if collection == 'site' - # TODO: puede ser peligroso permitir acceder a cualquier - # atributo de site? No estamos trayendo nada fuera de - # lo normal - post.site.send(attr.to_sym) - # Si hay un subatributo, tenemos que averiguar todos los - # valores dentro de el - # TODO volver elegante! - # TODO volver recursivo! - elsif subattr - post.site.everything_of(attr, lang: collection) - .compact - .map { |sv| sv[subattr] } - .flatten - .compact - .uniq - else - post.site.everything_of(attr, lang: collection).compact - end - end - - # Si el valor es abierto, sumar los valores auto-completados a - # lo pre-cargados. - # - # En este punto _value es un array de 1 o 2 arrays, si es de uno, - # value tambien tiene que serlo. Si es de 2, hay que unir cada - # una - if open? - if _value.count == 1 - _value = [(_value.first + value).uniq] - elsif _value.count == 2 - _value = _value.each_with_index.map do |v, i| - v + value.fetch(i, []) - end - end - end - - # Crea un array de arrays, útil para los select - # [ [ 1, a ], [ 2, b ] ] - # aunque si no hay un : en el autocompletado, el array queda - # [ [ 1, 1 ], [ 2, 2 ] ] - values = _value.empty? ? [] : _value.last.zip(_value.first) - - # En última instancia, traer el valor por defecto y ahorrarnos - # volver a procesar - @values = values - end - end -end From d5f6d3c61b5ccee54f1ef58f7bfdbe1063fd09c8 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 3 Aug 2022 12:56:39 -0300 Subject: [PATCH 192/251] =?UTF-8?q?no=20compartir=20el=20uuid=20del=20post?= =?UTF-8?q?=20con=20su=20indexaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aunque los uuid son únicos, en el contexto de un sitio los posts son únicos por cada sitio. si tenemos sitios duplicados (por ejemplo un sitio de testing), la indexación funciona pero va moviendo posts de un panel a otro. ahora los uuid de cada post se guardan por separado y las indexaciones tienen sus propios uuids únicos. por ahora no los estamos usando para nada, pero cuando tengamos permalinks, los registros van a estar relacionados con sus posts indexados. --- app/models/post/indexable.rb | 3 +-- ...53308_indexed_posts_by_uuid_and_site_id.rb | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20220802153308_indexed_posts_by_uuid_and_site_id.rb diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index 7757e7f7..48dc6b0d 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -14,9 +14,8 @@ class Post # # @return [IndexedPost] def to_index - IndexedPost.find_or_create_by(id: uuid.value).tap do |indexed_post| + IndexedPost.find_or_create_by(post_id: uuid.value, site_id: site.id).tap do |indexed_post| indexed_post.layout = layout.name - indexed_post.site_id = site.id indexed_post.path = path.basename indexed_post.locale = locale.value indexed_post.dictionary = IndexedPost.to_dictionary(locale: locale.value) diff --git a/db/migrate/20220802153308_indexed_posts_by_uuid_and_site_id.rb b/db/migrate/20220802153308_indexed_posts_by_uuid_and_site_id.rb new file mode 100644 index 00000000..e6572ffb --- /dev/null +++ b/db/migrate/20220802153308_indexed_posts_by_uuid_and_site_id.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# No podemos compartir el uuid entre indexed_posts y posts porque +# podemos tener sitios duplicados. Al menos hasta que los sitios de +# testeo estén integrados en el panel vamos a tener que generar otros +# UUID. +class IndexedPostsByUuidAndSiteId < ActiveRecord::Migration[6.1] + def up + add_column :indexed_posts, :post_id, :uuid, index: true + + IndexedPost.transaction do + ActiveRecord::Base.connection.execute('update indexed_posts set post_id = id where post_id is null') + end + end + + def down + remove_column :indexed_posts, :post_id + end +end From 3e442865ab8d5f7559f5fb42e80dc27eed003647 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 3 Aug 2022 13:11:06 -0300 Subject: [PATCH 193/251] permitir indexar --- app/models/post/indexable.rb | 6 +++--- app/models/site/index.rb | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/app/models/post/indexable.rb b/app/models/post/indexable.rb index 48dc6b0d..4e46d7b2 100644 --- a/app/models/post/indexable.rb +++ b/app/models/post/indexable.rb @@ -14,7 +14,7 @@ class Post # # @return [IndexedPost] def to_index - IndexedPost.find_or_create_by(post_id: uuid.value, site_id: site.id).tap do |indexed_post| + IndexedPost.find_or_initialize_by(post_id: uuid.value, site_id: site.id).tap do |indexed_post| indexed_post.layout = layout.name indexed_post.path = path.basename indexed_post.locale = locale.value @@ -27,8 +27,6 @@ class Post end end - private - # Indexa o reindexa el Post # # @return [Boolean] @@ -40,6 +38,8 @@ class Post to_index.destroy.destroyed? end + private + # Los metadatos que se almacenan como objetos JSON. Empezamos con # las categorías porque se usan para filtrar en el listado de # artículos. diff --git a/app/models/site/index.rb b/app/models/site/index.rb index e10fa523..e11095e3 100644 --- a/app/models/site/index.rb +++ b/app/models/site/index.rb @@ -14,9 +14,7 @@ class Site def index_posts! Site.transaction do - docs.each do |post| - post.to_index.save - end + docs.each(&:index!) end end end From fff679b98184f5ab0ee20e30bb40382f3aed483c Mon Sep 17 00:00:00 2001 From: f Date: Wed, 3 Aug 2022 13:23:03 -0300 Subject: [PATCH 194/251] no devolver un destino closes #7151 closes #7152 closes #7153 closes #7154 closes #7167 closes #7168 closes #7169 closes #7170 closes #7380 closes #7381 closes #7382 closes #7383 closes #7394 closes #7395 closes #7396 closes #7397 closes #7405 closes #7406 closes #7407 closes #7408 closes #7491 closes #7492 closes #7493 closes #7494 closes #7509 closes #7510 closes #7511 closes #7512 closes #7527 closes #7528 closes #7529 closes #7530 --- app/models/deploy_reindex.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app/models/deploy_reindex.rb b/app/models/deploy_reindex.rb index 921451d9..d6b2be65 100644 --- a/app/models/deploy_reindex.rb +++ b/app/models/deploy_reindex.rb @@ -30,11 +30,9 @@ class DeployReindex < Deploy 1 end - def hostname - nil - end + def hostname; end - def url - nil - end + def url; end + + def destination; end end From cacb355b620cb5547ca21ec3743aa15c5a6d8834 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 3 Aug 2022 13:24:36 -0300 Subject: [PATCH 195/251] los deploys pueden no tener destino closes #7151 closes #7152 closes #7153 closes #7154 closes #7167 closes #7168 closes #7169 closes #7170 closes #7380 closes #7381 closes #7382 closes #7383 closes #7394 closes #7395 closes #7396 closes #7397 closes #7405 closes #7406 closes #7407 closes #7408 closes #7491 closes #7492 closes #7493 closes #7494 closes #7509 closes #7510 closes #7511 closes #7512 closes #7527 closes #7528 closes #7529 closes #7530 --- app/jobs/uri_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 59f753ec..4cbbf593 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -109,7 +109,7 @@ class UriCollectionJob < PeriodicJob # # TODO: Cambiar al mergear origin-referer def destinations - @destinations ||= site.deploys.map(&:destination).select do |d| + @destinations ||= site.deploys.map(&:destination).compact.select do |d| File.directory?(d) end.map do |d| File.realpath(d) From 2d0090c488b2befff5b7eca8b01a8641b2489e4b Mon Sep 17 00:00:00 2001 From: f Date: Wed, 24 Aug 2022 18:00:18 -0300 Subject: [PATCH 196/251] usar el uuid del post para reordenar --- app/views/posts/index.haml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 90d65670..70748570 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -93,15 +93,15 @@ TODO: Solo les usuaries cachean porque tenemos que separar les botones por permisos. - cache_if @usuarie, [post, I18n.locale] do - - checkbox_id = "checkbox-#{post.id}" - %tr{ id: post.id, data: { target: 'reorder.row' } } + - checkbox_id = "checkbox-#{post.post_id}" + %tr{ id: post.post_id, data: { target: 'reorder.row' } } %td .custom-control.custom-checkbox %input.custom-control-input{ id: checkbox_id, type: 'checkbox', autocomplete: 'off', data: { action: 'reorder#select' } } %label.custom-control-label{ for: checkbox_id } %span.sr-only= t('posts.reorder.select') -# Orden más alto es mayor prioridad - = hidden_field 'post[reorder]', post.id, + = hidden_field 'post[reorder]', post.post_id, value: size - i, data: { reorder: true } %td.w-100{ class: dir } From 0afe4b6aba50264634045b05bee6ecf3ec033425 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 27 Aug 2022 19:30:52 -0300 Subject: [PATCH 197/251] permitir a les usuaries elegir idioma y registrarse con ese --- app/controllers/application_controller.rb | 2 +- app/models/usuarie.rb | 9 +++++++++ app/views/devise/registrations/new.haml | 2 +- app/views/devise/shared/_links.haml | 14 ++++++++------ app/views/layouts/_breadcrumb.haml | 3 +++ 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index acd0134d..823e3264 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -54,7 +54,7 @@ class ApplicationController < ActionController::Base # corresponde con el idioma de los artículos, porque puede querer # traducirlos. def set_locale(&action) - I18n.with_locale(current_locale(include_params: false), &action) + I18n.with_locale(current_locale(include_params: params[:controller].start_with?('devise')), &action) end # Muestra una página 404 diff --git a/app/models/usuarie.rb b/app/models/usuarie.rb index 6de7ba4b..9126885a 100644 --- a/app/models/usuarie.rb +++ b/app/models/usuarie.rb @@ -9,6 +9,8 @@ class Usuarie < ApplicationRecord validates_uniqueness_of :email validates_with EmailAddress::ActiveRecordValidator, field: :email + before_create :lang_from_locale! + has_many :roles has_many :sites, through: :roles @@ -36,4 +38,11 @@ class Usuarie < ApplicationRecord increment_failed_attempts lock_access! if attempts_exceeded? && !access_locked? end + + private + + def lang_from_locale! + binding.pry + self.lang = I18n.locale.to_s + end end diff --git a/app/views/devise/registrations/new.haml b/app/views/devise/registrations/new.haml index cb6ff0d1..21676556 100644 --- a/app/views/devise/registrations/new.haml +++ b/app/views/devise/registrations/new.haml @@ -8,7 +8,7 @@ = form_for(resource, as: resource_name, - url: registration_path(resource_name)) do |f| + url: registration_path(resource_name, params: { locale: params[:locale] })) do |f| = render 'devise/shared/error_messages', resource: resource diff --git a/app/views/devise/shared/_links.haml b/app/views/devise/shared/_links.haml index c182d323..c8a6c041 100644 --- a/app/views/devise/shared/_links.haml +++ b/app/views/devise/shared/_links.haml @@ -1,35 +1,37 @@ %hr/ +- locale = params.permit(:locale) + - if controller_name != 'sessions' - = link_to t('.sign_in'), new_session_path(resource_name) + = link_to t('.sign_in'), new_session_path(resource_name, params: locale) %br/ - if devise_mapping.registerable? && controller_name != 'registrations' - = link_to t('.sign_up'), new_registration_path(resource_name), + = link_to t('.sign_up'), new_registration_path(resource_name, params: locale), class: 'btn btn-lg btn-block btn-success' %br/ - if devise_mapping.recoverable? - unless %w[passwords registrations].include?(controller_name) = link_to t('.forgot_your_password'), - new_password_path(resource_name) + new_password_path(resource_name, params: locale) %br/ - if devise_mapping.confirmable? && controller_name != 'confirmations' = link_to t('.didn_t_receive_confirmation_instructions'), - new_confirmation_path(resource_name) + new_confirmation_path(resource_name, params: locale) %br/ - if devise_mapping.lockable? - if resource_class.unlock_strategy_enabled?(:email) - if controller_name != 'unlocks' = link_to t('.didn_t_receive_unlock_instructions'), - new_unlock_path(resource_name) + new_unlock_path(resource_name, params: locale) %br/ - if devise_mapping.omniauthable? - resource_class.omniauth_providers.each do |provider| = link_to t('.sign_in_with_provider', provider: OmniAuth::Utils.camelize(provider)), - omniauth_authorize_path(resource_name, provider) + omniauth_authorize_path(resource_name, provider, params: locale) %br/ diff --git a/app/views/layouts/_breadcrumb.haml b/app/views/layouts/_breadcrumb.haml index c4920bc7..04375ca4 100644 --- a/app/views/layouts/_breadcrumb.haml +++ b/app/views/layouts/_breadcrumb.haml @@ -22,3 +22,6 @@ %li.nav-item = link_to t('.logout'), destroy_usuarie_session_path, method: :delete, role: 'button', class: 'btn' + - else + - other_locale = I18n.available_locales.find { |locale| locale != I18n.locale } + = link_to t(other_locale), "?locale=#{other_locale}" From 3595342d3a2df840e7b040c50b6151a010fe3f9d Mon Sep 17 00:00:00 2001 From: f Date: Sat, 3 Sep 2022 10:51:58 -0300 Subject: [PATCH 198/251] pry! --- app/models/usuarie.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/models/usuarie.rb b/app/models/usuarie.rb index 9126885a..605eb819 100644 --- a/app/models/usuarie.rb +++ b/app/models/usuarie.rb @@ -42,7 +42,6 @@ class Usuarie < ApplicationRecord private def lang_from_locale! - binding.pry self.lang = I18n.locale.to_s end end From 6c7ddb082df96249cf0c20b7e021b1ba2e8d48a2 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 1 Nov 2022 13:08:06 -0300 Subject: [PATCH 199/251] fix: no guardar el valor en texto plano si estuvo cifrado MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cuando un campo está cifrado y no fue modificado, al guardar se guardaba en texto plano y luego salían errores de decifrado. closes #1515 closes #1938 closes #1939 closes #1940 closes #1942 closes #1943 closes #1944 closes #8204 --- app/models/metadata_template.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/models/metadata_template.rb b/app/models/metadata_template.rb index c778e1b2..5de54be1 100644 --- a/app/models/metadata_template.rb +++ b/app/models/metadata_template.rb @@ -134,7 +134,11 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, # En caso de que algún campo necesite realizar acciones antes de ser # guardado def save - return true unless changed? + if !changed? + self[:value] = document_value if private? + + return true + end self[:value] = sanitize value self[:value] = encrypt(value) if private? From 3d8c9b4031d4290a27c3faaecd02323f6b9c6a77 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 1 Nov 2022 13:17:03 -0300 Subject: [PATCH 200/251] feat: implementar campo de tipo password #8342 --- app/models/metadata_password.rb | 25 +++++++++++++++++++++ app/views/posts/attribute_ro/_password.haml | 6 +++++ app/views/posts/attributes/_password.haml | 7 ++++++ config/locales/en.yml | 2 ++ config/locales/es.yml | 2 ++ 5 files changed, 42 insertions(+) create mode 100644 app/models/metadata_password.rb create mode 100644 app/views/posts/attribute_ro/_password.haml create mode 100644 app/views/posts/attributes/_password.haml diff --git a/app/models/metadata_password.rb b/app/models/metadata_password.rb new file mode 100644 index 00000000..1e0e2698 --- /dev/null +++ b/app/models/metadata_password.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# Almacena una contraseña +class MetadataPassword < MetadataString + # Las contraseñas no son indexables + # + # @return [boolean] + def indexable? + false + end + + private + + alias_method :original_sanitize, :sanitize + + # Sanitizar la string y generar un hash Bcrypt + # + # @param :string [String] + # @return [String] + def sanitize(string) + string = original_sanitize string + + ::BCrypt::Password.create(string).to_s + end +end diff --git a/app/views/posts/attribute_ro/_password.haml b/app/views/posts/attribute_ro/_password.haml new file mode 100644 index 00000000..e55b021f --- /dev/null +++ b/app/views/posts/attribute_ro/_password.haml @@ -0,0 +1,6 @@ +%tr{ id: attribute } + %th= post_label_t(attribute, post: post) + %td{ dir: dir, lang: locale } + = metadata.value + %br/ + %small= t('.safety') diff --git a/app/views/posts/attributes/_password.haml b/app/views/posts/attributes/_password.haml new file mode 100644 index 00000000..0aace30f --- /dev/null +++ b/app/views/posts/attributes/_password.haml @@ -0,0 +1,7 @@ +.form-group + = label_tag "#{base}_#{attribute}", post_label_t(attribute, post: post) + = password_field base, attribute, value: metadata.value, + dir: dir, lang: locale, + **field_options(attribute, metadata) + = render 'posts/attribute_feedback', + post: post, attribute: attribute, metadata: metadata diff --git a/config/locales/en.yml b/config/locales/en.yml index 530a9381..b9d4d8f9 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -414,6 +414,8 @@ en: attribute_ro: file: download: Download file + password: + safety: Passwords are stored safely show: front_matter: Post metadata submit: diff --git a/config/locales/es.yml b/config/locales/es.yml index eaa23137..b737f5c2 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -422,6 +422,8 @@ es: attribute_ro: file: download: Descargar archivo + password: + safety: Las contraseñas se almacenan de forma segura show: front_matter: Metadatos del artículo submit: From f582e692ef784fcea3fd2a264afc324d4c9fb5bb Mon Sep 17 00:00:00 2001 From: f Date: Wed, 9 Nov 2022 18:43:55 -0300 Subject: [PATCH 201/251] =?UTF-8?q?fix:=20explicar=20mejor=20por=20qu?= =?UTF-8?q?=C3=A9=20no=20es=20una=20preview=20completa=20#8400?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/locales/en.yml | 2 +- config/locales/es.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 530a9381..ec5171ec 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -475,7 +475,7 @@ en: preview: btn: 'Preliminary version' alert: 'Not every article type has a preliminary version' - message: 'This is a preliminary version, use the Publish changes button back on the panel to publish the article onto your site.' + message: 'This is a preview of your post with some contextual elements from your site.' open: 'Tip: You can add new options by typing them and pressing Enter' private: '🔒 The values of this field will remain private' select: diff --git a/config/locales/es.yml b/config/locales/es.yml index eaa23137..b9f45032 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -483,7 +483,7 @@ es: preview: btn: 'Versión preliminar' alert: 'No todos los tipos de artículos poseen vista preliminar :)' - message: 'Esta es una versión preliminar, para que el artículo aparezca en tu sitio utiliza el botón Publicar cambios en el panel' + message: 'Esta es la vista previa de tu artículo, con algunos elementos contextuales del sitio' open: 'Nota: Puedes agregar más opciones a medida que las escribes y presionas Entrar' private: '🔒 Los valores de este campo serán privados' select: From b141b09ee452aa75e92b5755ffe24b549333a345 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 9 Nov 2022 18:44:27 -0300 Subject: [PATCH 202/251] fix: poder mostrar la vista previa sin errores de liquid closes #8397 closes #8263 closes #8153 --- app/models/post.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/models/post.rb b/app/models/post.rb index cab7665f..3e824108 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -90,6 +90,10 @@ class Post 'page' => document.to_liquid } + # No tener errores de Liquid + site.jekyll.config['liquid']['strict_filters'] = false + site.jekyll.config['liquid']['strict_variables'] = false + # Renderizar lo estrictamente necesario y convertir a HTML para # poder reemplazar valores. html = Nokogiri::HTML document.renderer.render_document From 5a5d41448237870e5c8c5d1d763b3502d1f56e28 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 11 Nov 2022 19:09:38 -0300 Subject: [PATCH 203/251] =?UTF-8?q?fix:=20traducci=C3=B3n=20de=20algunos?= =?UTF-8?q?=20locales=20en=20uso=20#8427?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/locales/en.yml | 9 +++++++++ config/locales/es.yml | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/config/locales/en.yml b/config/locales/en.yml index 530a9381..a07533a9 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -13,6 +13,15 @@ en: ar: name: Arabic dir: rtl + zh: + name: Chinese + dir: ltr + de: + name: German + dir: ltr + fr: + name: French + dir: ltr login: email: E-mail address password: Password diff --git a/config/locales/es.yml b/config/locales/es.yml index eaa23137..3d6a2e10 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -13,6 +13,15 @@ es: ar: name: Árabe dir: rtl + zh: + name: Chino + dir: ltr + de: + name: Alemán + dir: ltr + fr: + name: Francés + dir: ltr login: email: Correo electrónico password: Contraseña From 4fc2ece2d5bd7f913559a9a501bd6a724ef01fa4 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 28 Nov 2022 19:30:20 -0300 Subject: [PATCH 204/251] =?UTF-8?q?fix:=20permitir=20borrar=20la=20imagen?= =?UTF-8?q?=20sin=20vaciar=20la=20descripci=C3=B3n=20#8347?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sin embargo no estamos pudiendo distinguir entre imagen borrada e imagen sin subir, con lo que perdimos una verificación --- app/models/metadata_file.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index c7296de4..e67761b2 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -23,7 +23,6 @@ class MetadataFile < MetadataTemplate errors << I18n.t("metadata.#{type}.site_invalid") if site.invalid? errors << I18n.t("metadata.#{type}.path_required") if path_missing? - errors << I18n.t("metadata.#{type}.no_file_for_description") if no_file_for_description? errors << I18n.t("metadata.#{type}.attachment_missing") if path? && !static_file errors.compact! @@ -134,11 +133,6 @@ class MetadataFile < MetadataTemplate end end - # No hay archivo pero se lo describió - def no_file_for_description? - !path? && description? - end - # Obtiene el id del blob asociado # # @return [Integer,nil] From 581e8e10c2af693a9d0b43fc38e398bc12c8b000 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 28 Nov 2022 19:31:09 -0300 Subject: [PATCH 205/251] fix: vaciar la imagen al borrarla --- app/models/metadata_file.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index e67761b2..a155a414 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -42,8 +42,12 @@ class MetadataFile < MetadataTemplate # Asociar la imagen subida al sitio y obtener la ruta # @return [Boolean] def save - value['description'] = sanitize value['description'] - value['path'] = relative_destination_path_with_filename.to_s if static_file + if value['path'].blank? + self[:value] = default_value + else + value['description'] = sanitize value['description'] + value['path'] = relative_destination_path_with_filename.to_s if static_file + end true end From 7e8e47387bda0f4d07d07566fcada290b7fc15cc Mon Sep 17 00:00:00 2001 From: f Date: Tue, 27 Dec 2022 17:35:37 -0300 Subject: [PATCH 206/251] feat: eliminar dependencia en plantillas instaladas #1849 --- Gemfile | 12 ------- Gemfile.lock | 99 ---------------------------------------------------- 2 files changed, 111 deletions(-) diff --git a/Gemfile b/Gemfile index 2b304ee0..f7744b30 100644 --- a/Gemfile +++ b/Gemfile @@ -99,18 +99,6 @@ gem 'net-ssh' gem 'ed25519' gem 'bcrypt_pbkdf' -group :themes do - gem 'adhesiones-jekyll-theme', require: false - gem 'editorial-autogestiva-jekyll-theme', require: false - gem 'minima', require: false - gem 'sutty-minima', require: false - gem 'radios-comunitarias-jekyll-theme', require: false - gem 'share-to-fediverse-jekyll-theme', require: false - gem 'sutty-donaciones-jekyll-theme', require: false - gem 'sutty-jekyll-theme', require: false - gem 'recursero-jekyll-theme', require: false -end - group :production do gem 'lograge' end diff --git a/Gemfile.lock b/Gemfile.lock index 8df2d77e..00520efa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -88,15 +88,6 @@ GEM zeitwerk (~> 2.3) addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) - adhesiones-jekyll-theme (0.2.1) - jekyll (~> 4.0) - jekyll-data (~> 1.1) - jekyll-feed (~> 0.9) - jekyll-images (~> 0.2) - jekyll-include-cache (~> 0) - jekyll-locales (~> 0.1) - jekyll-relative-urls (~> 0.0) - jekyll-seo-tag (~> 2.1) ast (2.4.2) autoprefixer-rails (10.3.3.0) execjs (~> 2) @@ -169,25 +160,6 @@ GEM down (5.2.4) addressable (~> 2.8) ed25519 (1.2.4-x86_64-linux-musl) - editorial-autogestiva-jekyll-theme (0.3.4) - jekyll (~> 4) - jekyll-commonmark (~> 1.3) - jekyll-data (~> 1.1) - jekyll-dotenv (>= 0.2) - jekyll-feed (~> 0.15) - jekyll-hardlinks (~> 0) - jekyll-ignore-layouts (~> 0) - jekyll-images (~> 0.2) - jekyll-include-cache (~> 0) - jekyll-linked-posts (~> 0) - jekyll-locales (~> 0.1) - jekyll-order (~> 0) - jekyll-relative-urls (~> 0) - jekyll-seo-tag (~> 2) - jekyll-spree-client (~> 0) - jekyll-unique-urls (~> 0) - jekyll-write-and-commit-changes (~> 0) - sutty-liquid (~> 0) em-websocket (0.5.3) eventmachine (>= 0.12.9) http_parser.rb (~> 0) @@ -369,10 +341,6 @@ GEM mini_magick (4.11.0) mini_mime (1.1.2) mini_portile2 (2.6.1) - minima (2.5.1) - jekyll (>= 3.5, < 5.0) - jekyll-feed (~> 0.9) - jekyll-seo-tag (~> 2.1) minitest (5.14.4) mobility (1.2.4) i18n (>= 0.6.10, < 2) @@ -415,17 +383,6 @@ GEM rack rack-test (1.1.0) rack (>= 1.0, < 3) - radios-comunitarias-jekyll-theme (0.1.5) - jekyll (~> 4.0) - jekyll-data (~> 1.1) - jekyll-feed (~> 0.9) - jekyll-images (~> 0.2) - jekyll-include-cache (~> 0) - jekyll-linked-posts (~> 0) - jekyll-locales (~> 0.1) - jekyll-relative-urls (~> 0.0) - jekyll-seo-tag (~> 2.1) - jekyll-turbolinks (~> 0) rails (6.1.4.1) actioncable (= 6.1.4.1) actionmailbox (= 6.1.4.1) @@ -462,24 +419,6 @@ GEM rb-fsevent (0.11.0) rb-inotify (0.10.1) ffi (~> 1.0) - recursero-jekyll-theme (0.2.0) - jekyll (~> 4) - jekyll-commonmark (~> 1.3) - jekyll-data (~> 1.1) - jekyll-dotenv (>= 0.2) - jekyll-feed (~> 0.15) - jekyll-ignore-layouts (~> 0) - jekyll-images (~> 0.2) - jekyll-include-cache (~> 0) - jekyll-linked-posts (~> 0) - jekyll-locales (~> 0.1) - jekyll-lunr (~> 0.1) - jekyll-order (~> 0) - jekyll-relative-urls (~> 0) - jekyll-seo-tag (~> 2) - jekyll-unique-urls (~> 0.1) - sutty-archives (~> 2.2) - sutty-liquid (~> 0) redis (4.5.1) redis-actionpack (5.2.0) actionpack (>= 5, < 7) @@ -552,14 +491,6 @@ GEM rubyzip (>= 1.2.2) semantic_range (3.0.0) sexp_processor (4.16.0) - share-to-fediverse-jekyll-theme (0.1.4) - jekyll (~> 4.0) - jekyll-data (~> 1.1) - jekyll-feed (~> 0.9) - jekyll-images (~> 0.2) - jekyll-include-cache (~> 0) - jekyll-relative-urls (~> 0.0) - jekyll-seo-tag (~> 2.1) simpleidn (0.2.1) unf (~> 0.1.4) sourcemap (0.1.1) @@ -583,30 +514,9 @@ GEM concurrent-ruby (~> 1.0) sutty-archives (2.5.4) jekyll (>= 3.6, < 5.0) - sutty-donaciones-jekyll-theme (0.1.2) - jekyll (~> 4.0) - jekyll-data (~> 1.1) - jekyll-feed (~> 0.9) - jekyll-images (~> 0.2) - jekyll-include-cache (~> 0) - jekyll-locales (~> 0.1) - jekyll-relative-urls (~> 0.0) - jekyll-seo-tag (~> 2.1) - sutty-archives (~> 2.2) - sutty-jekyll-theme (0.1.2) - jekyll (~> 4.0) - jekyll-feed (~> 0.9) - jekyll-images (~> 0.2) - jekyll-include-cache (~> 0) - jekyll-relative-urls (~> 0.0) - jekyll-seo-tag (~> 2.1) sutty-liquid (0.7.4) fast_blank (~> 1.0) jekyll (~> 4) - sutty-minima (2.5.0) - jekyll (>= 3.5, < 5.0) - jekyll-feed (~> 0.9) - jekyll-seo-tag (~> 2.1) symbol-fstring (1.0.2-x86_64-linux-musl) sysexits (1.2.0) temple (0.8.2) @@ -654,7 +564,6 @@ PLATFORMS x86_64-linux-musl DEPENDENCIES - adhesiones-jekyll-theme bcrypt (~> 3.1.7) bcrypt_pbkdf blazer @@ -672,7 +581,6 @@ DEPENDENCIES dotenv-rails down ed25519 - editorial-autogestiva-jekyll-theme email_address! exception_notification factory_bot_rails @@ -702,7 +610,6 @@ DEPENDENCIES lograge memory_profiler mini_magick - minima mobility net-ssh nokogiri @@ -714,11 +621,9 @@ DEPENDENCIES pundit rack-cors rack-mini-profiler - radios-comunitarias-jekyll-theme rails (~> 6) rails-i18n rails_warden - recursero-jekyll-theme redis redis-rails rollups! @@ -728,17 +633,13 @@ DEPENDENCIES safe_yaml! sassc-rails selenium-webdriver - share-to-fediverse-jekyll-theme sourcemap spring spring-watcher-listen (~> 2.0.0) sqlite3 stackprof sucker_punch - sutty-donaciones-jekyll-theme - sutty-jekyll-theme sutty-liquid (>= 0.7.3) - sutty-minima symbol-fstring terminal-table timecop From e6068fa7ecbde7826193fa80e44b994a5217e560 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 27 Dec 2022 17:36:32 -0300 Subject: [PATCH 207/251] fix: traer gemas del mismo repositorio --- Gemfile | 4 ++-- Gemfile.lock | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index f7744b30..0f26071b 100644 --- a/Gemfile +++ b/Gemfile @@ -48,9 +48,9 @@ gem 'image_processing' gem 'icalendar' gem 'inline_svg' gem 'httparty' -gem 'safe_yaml', source: 'https://gems.sutty.nl' +gem 'safe_yaml' gem 'jekyll', '~> 4.2' -gem 'jekyll-data', source: 'https://gems.sutty.nl' +gem 'jekyll-data' gem 'jekyll-commonmark' gem 'jekyll-images' gem 'jekyll-include-cache' diff --git a/Gemfile.lock b/Gemfile.lock index 00520efa..db7e601a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -599,7 +599,7 @@ DEPENDENCIES jbuilder (~> 2.5) jekyll (~> 4.2) jekyll-commonmark - jekyll-data! + jekyll-data jekyll-images jekyll-include-cache kaminari @@ -630,7 +630,7 @@ DEPENDENCIES rubocop-rails rubyzip rugged - safe_yaml! + safe_yaml sassc-rails selenium-webdriver sourcemap From 072b01e4d94e5a22c1d5646bfc63cd25c1affedf Mon Sep 17 00:00:00 2001 From: f Date: Tue, 27 Dec 2022 17:37:47 -0300 Subject: [PATCH 208/251] fix: siempre instalar las gemas localmente --- app/models/deploy_local.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 4fa588f5..88b84930 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -91,11 +91,7 @@ class DeployLocal < Deploy end def bundle - if Rails.env.production? run %(bundle install --no-cache --path="#{gems_dir}") - else - run %(bundle install) - end end def jekyll_build From 47f1a8e530053b81e5978f36f71763fd4e822228 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 27 Dec 2022 17:42:00 -0300 Subject: [PATCH 209/251] feat: leer datos de las plantillas directamente #1849 usando el lockfile de cada repositorio, acceder directamente al directorio de datos usando jekyll-data. --- app/models/site.rb | 22 +++--------- config/initializers/core_extensions.rb | 47 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/app/models/site.rb b/app/models/site.rb index 638b3f47..5dee0b85 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -335,7 +335,10 @@ class Site < ApplicationRecord def jekyll run_in_path do - @jekyll ||= Jekyll::Site.new(configuration) + @jekyll ||= + Jekyll::Site.new(configuration).tap do |site| + site.reader = JekyllData::Reader.new(site) if site.theme + end end end @@ -382,9 +385,6 @@ class Site < ApplicationRecord @configuration[unneeded] = [] if @configuration.key? unneeded end - # Eliminar el theme si no es una gema válida - @configuration.delete('theme') unless theme_available? - # Si estamos usando nuestro propio plugin de i18n, los posts están # en "colecciones" locales.map(&:to_s).each do |i| @@ -394,20 +394,6 @@ class Site < ApplicationRecord @configuration end - # Lista los nombres de las plantillas disponibles como gemas, - # tomándolas dinámicamente de las que agreguemos en el grupo :themes - # del Gemfile. - def available_themes - @available_themes ||= Bundler.load.current_dependencies.select do |gem| - gem.groups.include? :themes - end.map(&:name) - end - - # Detecta si el tema actual es una gema - def theme_available? - available_themes.include? design&.gem - end - # Devuelve el dominio actual def self.domain ENV.fetch('SUTTY', 'sutty.nl') diff --git a/config/initializers/core_extensions.rb b/config/initializers/core_extensions.rb index 66d2c92b..1516a43a 100644 --- a/config/initializers/core_extensions.rb +++ b/config/initializers/core_extensions.rb @@ -37,6 +37,13 @@ end # # TODO: Aplicar monkey patches en otro lado... module Jekyll + Site.class_eval do + def configure_theme + self.theme = nil + self.theme = Jekyll::Theme.new(config['theme'], self) unless config['theme'].nil? + end + end + Reader.class_eval do # No necesitamos otros posts def retrieve_posts(_); end @@ -69,6 +76,46 @@ module Jekyll end end + Theme.class_eval do + attr_reader :site + + def initialize(name, site) + @name = name.downcase.strip + @site = site + end + + def root + @root ||= begin + lockfile = Bundler::LockfileParser.new(File.read(site.in_source_dir('Gemfile.lock'))) + spec = lockfile.specs.find do |spec| + spec.name == name + end + + ruby_version = Gem::Version.new(RUBY_VERSION) + ruby_version.canonical_segments[2] = 0 + base_path = Rails.root.join('_storage', 'gems', File.basename(site.source), 'ruby', + ruby_version.canonical_segments.join('.')) + + File.realpath( + case spec.source + when Bundler::Source::Git + File.join(base_path, 'bundler', 'gems', spec.source.extension_dir_name) + when Bundler::Source::Rubygems + File.join(base_path, 'gems', spec.full_name) + end + ) + end + end + + def runtime_dependencies + [] + end + + private + + def gemspec; end + end + # No necesitamos los archivos de la plantilla ThemeAssetsReader.class_eval do def read; end From e258e6e2d9928a8d721bdbfc991f5c390bf46327 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 27 Dec 2022 18:41:36 -0300 Subject: [PATCH 210/251] =?UTF-8?q?fix:=20asegurarse=20que=20las=20gemas?= =?UTF-8?q?=20est=C3=A1n=20instaladas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/site.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/models/site.rb b/app/models/site.rb index 5dee0b85..dfc4f034 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -56,8 +56,8 @@ class Site < ApplicationRecord before_destroy :remove_directories! # Carga el sitio Jekyll una vez que se inicializa el modelo o después # de crearlo - after_initialize :load_jekyll - after_create :load_jekyll + after_initialize :install_gems, :load_jekyll + after_create :install_gems, :load_jekyll # Cambiar el nombre del directorio before_update :update_name! before_save :add_private_key_if_missing! @@ -515,4 +515,11 @@ class Site < ApplicationRecord def run_in_path(&block) Dir.chdir path, &block end + + def install_gems + return unless persisted? + return if Rails.root.join('_storage', 'gems', name).directory? + + deploys.find_by_type('DeployLocal').send(:bundle) + end end From a265caad9770a76b5c4c0091e889f0bcb4ea34ef Mon Sep 17 00:00:00 2001 From: f Date: Thu, 29 Dec 2022 16:38:17 -0300 Subject: [PATCH 211/251] feat: eliminar archivos que no se usan de las gemas para liberar espacio --- app/models/deploy_local.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 88b84930..5bee9ff5 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -92,6 +92,11 @@ class DeployLocal < Deploy def bundle run %(bundle install --no-cache --path="#{gems_dir}") + + # Eliminar archivos que no se usan + Dir.glob("#{gems_dir}/ruby/2.7.0/{cache/*,gems/*/{spec,test,ext,vendor}}").each do |dir| + FileUtils.rm_rf(dir) + end end def jekyll_build From d896a111b8128ff24be33b6233f8c85bd5ffdd0a Mon Sep 17 00:00:00 2001 From: f Date: Fri, 30 Dec 2022 10:55:10 -0300 Subject: [PATCH 212/251] fix: no cargar el sitio hasta que no se usen los posts --- app/models/site.rb | 6 +++++- app/views/sites/index.haml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models/site.rb b/app/models/site.rb index dfc4f034..ac290403 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -333,6 +333,10 @@ class Site < ApplicationRecord status == 'building' end + def jekyll? + File.directory? path + end + def jekyll run_in_path do @jekyll ||= @@ -432,7 +436,7 @@ class Site < ApplicationRecord # Clona el esqueleto de Sutty para crear el sitio nuevo, no pasa nada # si el sitio ya existe def clone_skel! - return if File.directory? path + return if jekyll? Rugged::Repository.clone_at ENV['SKEL_SUTTY'], path end diff --git a/app/views/sites/index.haml b/app/views/sites/index.haml index d69dbeac..56178775 100644 --- a/app/views/sites/index.haml +++ b/app/views/sites/index.haml @@ -14,7 +14,7 @@ %table.table.table-condensed %tbody - @sites.each do |site| - - next unless site.jekyll + - next unless site.jekyll? - rol = current_usuarie.rol_for_site(site) -# TODO: Solo les usuaries cachean porque tenemos que separar From f35f4c0f065196efe273ab2be70636545e9d4bea Mon Sep 17 00:00:00 2001 From: f Date: Fri, 30 Dec 2022 10:56:00 -0300 Subject: [PATCH 213/251] fix: cargar jekyll cuando realmente se lo usa --- app/models/site.rb | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/app/models/site.rb b/app/models/site.rb index ac290403..f38500eb 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -54,10 +54,6 @@ class Site < ApplicationRecord before_create :clone_skel! # Elimina el directorio al destruir un sitio before_destroy :remove_directories! - # Carga el sitio Jekyll una vez que se inicializa el modelo o después - # de crearlo - after_initialize :install_gems, :load_jekyll - after_create :install_gems, :load_jekyll # Cambiar el nombre del directorio before_update :update_name! before_save :add_private_key_if_missing! @@ -338,12 +334,14 @@ class Site < ApplicationRecord end def jekyll - run_in_path do - @jekyll ||= + @jekyll ||= + begin + install_gems + Jekyll::Site.new(configuration).tap do |site| site.reader = JekyllData::Reader.new(site) if site.theme end - end + end end # Cargar el sitio Jekyll From 53b11cba155e4cef51dd5e0801220c5112eae318 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 11 Jan 2023 16:54:45 -0300 Subject: [PATCH 214/251] fix: no devolver 'true' si no existe el archivo closes #9380 closes #9379 closes #9375 closes #9374 closes #9366 closes #8888 closes #8765 closes #8764 closes #8674 closes #8099 closes #8098 closes #7845 closes #7844 closes #7612 closes #7611 closes #9378 closes #9377 closes #9364 closes #9362 --- app/models/metadata_file.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index a155a414..a55549cb 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -122,10 +122,17 @@ class MetadataFile < MetadataTemplate Pathname.new(value['path']) end + # Obtener la ruta relativa al sitio. + # + # Si algo falla, devolver la ruta original para no romper el archivo. + # + # @return [String, nil] def relative_destination_path_with_filename destination_path_with_filename.relative_path_from(Pathname.new(site.path).realpath) rescue ArgumentError => e ExceptionNotifier.notify_exception(e, data: { site: site.name, path: value['path'] }) + + value['path'] end def static_file_path From 5117ffbce3e41ee3315803e25cf3d2f05c8259a9 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 11 Jan 2023 17:04:22 -0300 Subject: [PATCH 215/251] fix: siempre devolver la ruta completa closes #6984 closes #7053 closes #7807 closes #7998 closes #8262 closes #8690 closes #8887 closes #9363 closes #9376 --- app/models/metadata_file.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index a55549cb..3ac89c9b 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -119,7 +119,7 @@ class MetadataFile < MetadataTemplate rescue Errno::ENOENT => e ExceptionNotifier.notify_exception(e, data: { site: site.name, path: value['path'] }) - Pathname.new(value['path']) + Pathname.new(File.join(site.path, value['path'])) end # Obtener la ruta relativa al sitio. From 2352ce52ec4ee784ca5e1b82859e7070e0c06871 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 12 Jan 2023 13:40:43 -0300 Subject: [PATCH 216/251] fix: no filtrar la url del panel en sitios proxeados #9386 --- app/controllers/api/v1/notices_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/v1/notices_controller.rb b/app/controllers/api/v1/notices_controller.rb index cd44130c..436c78b5 100644 --- a/app/controllers/api/v1/notices_controller.rb +++ b/app/controllers/api/v1/notices_controller.rb @@ -15,7 +15,7 @@ module Api params: airbrake_params.to_h end - render status: 201, json: { id: 1, url: root_url } + render status: 201, json: { id: 1, url: '' } end private From 247e069f2b3a7bc7907f2c388467f0c8d8d082cc Mon Sep 17 00:00:00 2001 From: f Date: Thu, 12 Jan 2023 14:24:21 -0300 Subject: [PATCH 217/251] fix: fast_jsonparser 0.6.0 da segfaults #9386 --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 1e476dde..6ecb7456 100644 --- a/Gemfile +++ b/Gemfile @@ -89,7 +89,7 @@ gem 'stackprof' gem 'prometheus_exporter' # debug -gem 'fast_jsonparser' +gem 'fast_jsonparser', '~> 0.5.0' gem 'down' gem 'sourcemap' gem 'rack-cors' From 75cbed8f6d9610d1a03ef0e299d2a8083c91925b Mon Sep 17 00:00:00 2001 From: f Date: Mon, 6 Feb 2023 16:23:41 -0300 Subject: [PATCH 218/251] fix: no eliminar archivos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gem-compiler sabe mejor cuales se pueden eliminar y cuáles no sutty/gems#3 sutty/gems#1 --- app/models/deploy_local.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 5bee9ff5..02ce97da 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -91,12 +91,7 @@ class DeployLocal < Deploy end def bundle - run %(bundle install --no-cache --path="#{gems_dir}") - - # Eliminar archivos que no se usan - Dir.glob("#{gems_dir}/ruby/2.7.0/{cache/*,gems/*/{spec,test,ext,vendor}}").each do |dir| - FileUtils.rm_rf(dir) - end + run %(bundle install --no-cache --path="#{gems_dir}") end def jekyll_build From 8dcc8596e2e9dfb2aa1c58a5dc16e8c9e2c69b1a Mon Sep 17 00:00:00 2001 From: f Date: Tue, 7 Feb 2023 09:19:38 -0300 Subject: [PATCH 219/251] feat: informar la rama actual del repositorio #9804 --- .profile | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .profile diff --git a/.profile b/.profile new file mode 100644 index 00000000..3c73ffa9 --- /dev/null +++ b/.profile @@ -0,0 +1,9 @@ +Color_Off='\e[0m' +BPurple='\e[1;35m' +BBlue='\e[1;34m' + +is_git() { + git rev-parse --abbrev-ref HEAD 2>/dev/null +} + +PS1="\[${BPurple}\]\$(is_git) \[${BBlue}\]\W\[${Color_Off}\] >_ " From b091b3212a71fa182be5ddb884f700faec05f3ae Mon Sep 17 00:00:00 2001 From: f Date: Wed, 8 Feb 2023 18:55:05 -0300 Subject: [PATCH 220/251] feat: refactorizar mailer para que use menos recursors y soporte varias urls --- app/jobs/deploy_job.rb | 4 +- app/mailers/deploy_mailer.rb | 63 ++++++++++++++++++---- app/views/deploy_mailer/deployed.html.haml | 31 +++++------ app/views/deploy_mailer/deployed.text.haml | 12 ++--- 4 files changed, 73 insertions(+), 37 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index ae6cb279..6250cf73 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -32,7 +32,7 @@ class DeployJob < ApplicationJob status: deploy_locally, seconds: deploy_local.build_stats.last.seconds, size: deploy_local.size, - url: deploy_local.url + urls: [deploy_local.url] } } @@ -74,7 +74,7 @@ class DeployJob < ApplicationJob status: status, seconds: build_stat.try(:seconds) || 0, size: d.size, - url: d.url + urls: d.respond_to?(:urls) ? d.urls : [d.url] } end end diff --git a/app/mailers/deploy_mailer.rb b/app/mailers/deploy_mailer.rb index 1d0c7308..aeb13676 100644 --- a/app/mailers/deploy_mailer.rb +++ b/app/mailers/deploy_mailer.rb @@ -8,21 +8,64 @@ # TODO: Agregar firma GPG y header Autocrypt # TODO: Cifrar con GPG si le usuarie nos dio su llave class DeployMailer < ApplicationMailer + include ActionView::Helpers::NumberHelper + include ActionView::Helpers::DateHelper + # rubocop:disable Metrics/AbcSize - def deployed(which_ones) - @usuarie = Usuarie.find(params[:usuarie]) - @site = @usuarie.sites.find(params[:site]) - @deploys = which_ones - @deploy_local = @site.deploys.find_by(type: 'DeployLocal') + def deployed(deploys) + usuarie = Usuarie.find(params[:usuarie]) + site = usuarie.sites.find(params[:site]) + subject = t('.subject', site: site.name) + hostname = site.hostname + + @hi = t('.hi') + @explanation = t('.explanation', fqdn: hostname) + @help = t('.help') + + @headers = %w[type status url seconds size].map do |header| + t(".th.#{header}") + end + + @table = deploys.each_pair.map do |deploy, value| + { + title: t(".#{deploy}.title"), + status: t(".#{deploy}.#{value[:status] ? 'success' : 'error'}"), + urls: value[:urls], + seconds: { + human: distance_of_time_in_words(value[:seconds].seconds), + machine: "PT#{value[:seconds]}S" + }, + size: number_to_human_size(value[:size], precision: 2) + } + end + + @terminal_table = Terminal::Table.new do |t| + t << @headers + t.add_separator + @table.each do |row| + row[:urls].each do |url| + t << (row.map do |k, v| + case k + when :seconds then v[:human] + when :urls then url + else v + end + end) + end + end + end # Informamos a cada quien en su idioma y damos una dirección de # respuesta porque a veces les usuaries nos escriben - I18n.with_locale(@usuarie.lang) do - mail(to: @usuarie.email, - reply_to: "sutty@#{Site.domain}", - subject: I18n.t('deploy_mailer.deployed.subject', - site: @site.name)) + I18n.with_locale(usuarie.lang) do + mail(to: usuarie.email, reply_to: "sutty@#{Site.domain}", subject: subject) end end # rubocop:enable Metrics/AbcSize + + private + + def t(key, **args) + I18n.t("deploy_mailer.deployed#{key}", **args) + end end diff --git a/app/views/deploy_mailer/deployed.html.haml b/app/views/deploy_mailer/deployed.html.haml index 0053accf..20548f16 100644 --- a/app/views/deploy_mailer/deployed.html.haml +++ b/app/views/deploy_mailer/deployed.html.haml @@ -1,24 +1,21 @@ -%h1= t('.hi') +%h1= @hi -= sanitize_markdown t('.explanation', fqdn: @deploy_local.site.hostname), - tags: %w[p a strong em] += sanitize_markdown @explanation, tags: %w[p a strong em] %table %thead %tr - %th= t('.th.type') - %th= t('.th.status') - %th= t('.th.url') - %th= t('.th.seconds') - %th= t('.th.size') + - @headers.each do |header| + %th= header %tbody - - @deploys.each_pair do |deploy, value| - %tr - %td= t(".#{deploy}.title") - %td= value[:status] ? t(".#{deploy}.success") : t(".#{deploy}.error") - %td= link_to value[:url], value[:url] - %td - %time{ datetime: "PT#{value[:seconds]}S" }= distance_of_time_in_words value[:seconds].seconds - %td= number_to_human_size value[:size], precision: 2 + - @table.each do |row| + - row[:urls].each do |url| + %tr + %td= row[:title] + %td= row[:status] + %td= link_to url, url + %td + %time{ datetime: row[:seconds][:machine] }= row[:seconds][:human] + %td= row[:size] -= sanitize_markdown t('.help'), tags: %w[p a strong em] += sanitize_markdown @help, tags: %w[p a strong em] diff --git a/app/views/deploy_mailer/deployed.text.haml b/app/views/deploy_mailer/deployed.text.haml index 48e8000c..b2d0416f 100644 --- a/app/views/deploy_mailer/deployed.text.haml +++ b/app/views/deploy_mailer/deployed.text.haml @@ -1,11 +1,7 @@ -= '# ' + t('.hi') += "# #{@hi}" \ -= t('.explanation', fqdn: @deploy_local.site.hostname) += @explanation \ -= Terminal::Table.new do |table| - - table << [t('.th.type'), t('.th.status'), t('.th.url'), t('.th.seconds'), t('.th.size')] - - table.add_separator - - @deploys.each_pair do |deploy, value| - - table << [t(".#{deploy}.title"), value[:status] ? t(".#{deploy}.success") : t(".#{deploy}.error"), value[:url], distance_of_time_in_words(value[:seconds].seconds), number_to_human_size(value[:size], precision: 2)] += @terminal_table \ -= t('.help') += @help From e81f188147e062400721dcd609eef5d62d8f089c Mon Sep 17 00:00:00 2001 From: f Date: Wed, 8 Feb 2023 19:30:05 -0300 Subject: [PATCH 221/251] =?UTF-8?q?fix:=20no=20generar=20links=20si=20la?= =?UTF-8?q?=20url=20viene=20vac=C3=ADa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/deploy_mailer/deployed.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/deploy_mailer/deployed.html.haml b/app/views/deploy_mailer/deployed.html.haml index 20548f16..f5afe5de 100644 --- a/app/views/deploy_mailer/deployed.html.haml +++ b/app/views/deploy_mailer/deployed.html.haml @@ -13,7 +13,7 @@ %tr %td= row[:title] %td= row[:status] - %td= link_to url, url + %td= link_to_if url.present?, url, url %td %time{ datetime: row[:seconds][:machine] }= row[:seconds][:human] %td= row[:size] From d821f143058eb515062a55e397a25ad58cd687fd Mon Sep 17 00:00:00 2001 From: f Date: Thu, 9 Mar 2023 15:08:48 -0300 Subject: [PATCH 222/251] fix: temporalmente deshabilitar el boton #9989 --- app/views/posts/show.haml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/app/views/posts/show.haml b/app/views/posts/show.haml index e46114af..64daee41 100644 --- a/app/views/posts/show.haml +++ b/app/views/posts/show.haml @@ -6,13 +6,6 @@ edit_site_post_path(@site, @post.id), class: 'btn btn-block' - - unless @post.layout.ignored? - = link_to t('posts.preview.btn'), - site_post_preview_path(@site, @post.id), - class: 'btn btn-block', - target: '_blank', - rel: 'noopener' - %table.table.table-condensed %thead %tr From 611a1f0eb043807d8ebc96cca88fb37fa6d5cf83 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 9 Mar 2023 15:08:48 -0300 Subject: [PATCH 223/251] fix: temporalmente deshabilitar el boton #9989 --- app/views/posts/show.haml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/app/views/posts/show.haml b/app/views/posts/show.haml index e46114af..64daee41 100644 --- a/app/views/posts/show.haml +++ b/app/views/posts/show.haml @@ -6,13 +6,6 @@ edit_site_post_path(@site, @post.id), class: 'btn btn-block' - - unless @post.layout.ignored? - = link_to t('posts.preview.btn'), - site_post_preview_path(@site, @post.id), - class: 'btn btn-block', - target: '_blank', - rel: 'noopener' - %table.table.table-condensed %thead %tr From ffce90daf3af38634a5892fb6bf4111f4f29f2d9 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 9 Mar 2023 15:13:43 -0300 Subject: [PATCH 224/251] =?UTF-8?q?wip:=20no=20mostrar=20errores=20al=20te?= =?UTF-8?q?ner=20errores=20de=20renderizaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pero no muestra nada de nada --- app/models/post.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/models/post.rb b/app/models/post.rb index cab7665f..bb2afd3c 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -108,6 +108,10 @@ class Post # Cacofonía html.to_html.html_safe + rescue Liquid::Error => e + ExceptionNotifier.notify(e, data: { site: site.name, post: post.id }) + + '' end end From 866b11ff74ff3eba6095389340f62761029cff2a Mon Sep 17 00:00:00 2001 From: f Date: Mon, 13 Mar 2023 17:29:13 -0300 Subject: [PATCH 225/251] fix: poder distinguir entre errores de deploy --- app/jobs/deploy_job.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 70997ce1..84119416 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -3,6 +3,7 @@ # Realiza el deploy de un sitio class DeployJob < ApplicationJob class DeployException < StandardError; end + class DeployTimedOutException < DeployException; end # rubocop:disable Metrics/MethodLength def perform(site, notify = true, time = Time.now) @@ -17,7 +18,7 @@ class DeployJob < ApplicationJob if @site.building? if 10.minutes.ago >= time @site.update status: 'waiting' - raise DeployException, + raise DeployTimedOutException, "#{@site.name} la tarea estuvo más de 10 minutos esperando, volviendo al estado original" end From e62172e37a188effecdf1319f9e76991dbaa3192 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 13 Mar 2023 18:46:22 -0300 Subject: [PATCH 226/251] =?UTF-8?q?chore:=20secar=20el=20c=C3=B3digo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit la primera hipótesis sobre #10031 es que las excepciones cancelan la actualización de la información, como si fueran una transacción, pero haciendo pruebas manuales no pasa. con este cambio al menos el código queda más limpio. --- app/jobs/deploy_job.rb | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 84119416..9c0ed1dd 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -17,7 +17,6 @@ class DeployJob < ApplicationJob # hora original para poder ir haciendo timeouts. if @site.building? if 10.minutes.ago >= time - @site.update status: 'waiting' raise DeployTimedOutException, "#{@site.name} la tarea estuvo más de 10 minutos esperando, volviendo al estado original" end @@ -32,17 +31,13 @@ class DeployJob < ApplicationJob # No es opcional unless @deployed[:deploy_local] - @site.update status: 'waiting' - notify_usuaries if notify - # Hacer fallar la tarea raise DeployException, deploy_local.build_stats.last.log end deploy_others - - # Volver a la espera - @site.update status: 'waiting' + ensure + @site&.update status: 'waiting' notify_usuaries if notify end From 6a7a0dddda284fde2e2eb9f6c8b493e45b407d50 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 13 Mar 2023 18:46:56 -0300 Subject: [PATCH 227/251] fix: no notificar al liberar la tarea --- app/jobs/deploy_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 9c0ed1dd..2165ed11 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -17,6 +17,7 @@ class DeployJob < ApplicationJob # hora original para poder ir haciendo timeouts. if @site.building? if 10.minutes.ago >= time + notify = false raise DeployTimedOutException, "#{@site.name} la tarea estuvo más de 10 minutos esperando, volviendo al estado original" end From 955786be0e6a48a88e55510b0e16de595b7680ea Mon Sep 17 00:00:00 2001 From: f Date: Mon, 13 Mar 2023 18:52:15 -0300 Subject: [PATCH 228/251] =?UTF-8?q?fix:=20cancelar=20la=20compilaci=C3=B3n?= =?UTF-8?q?=20al=20primer=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recupera el comportamiento de #1716 #1730 #1735 #1738 #1739 #1740 #1741 #1743 #1744 #1746 #1832 #1952 #2057 #2058 #2059 #2062 #1104 #1124 #1152 #1153 #1154 #1175 #1191 #1230 #1303 #1461 #1478 #1609 #1610 #1667 #504 --- app/jobs/deploy_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 2165ed11..9f58e3a1 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -31,7 +31,7 @@ class DeployJob < ApplicationJob @deployed = { deploy_local: deploy_locally } # No es opcional - unless @deployed[:deploy_local] + unless @deployed[:deploy_local][:status] # Hacer fallar la tarea raise DeployException, deploy_local.build_stats.last.log end From 1d08b3fdcd080c28702b6a692827399a1c828bd8 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 13 Mar 2023 19:04:55 -0300 Subject: [PATCH 229/251] fix: llegar hasta el final aunque fallen algunos metodos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit al usar ensure en el commit anterior, podemos garantizar que cualquier excepción en otros deploys no deja la compilación en estado de compilación permanente, pero no se notifica a les usuaries. --- app/jobs/deploy_job.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 9f58e3a1..2dacf08e 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -58,6 +58,10 @@ class DeployJob < ApplicationJob def deploy_others @site.deploys.where.not(type: 'DeployLocal').find_each do |d| @deployed[d.type.underscore.to_sym] = d.deploy + rescue StandardError => e + @deployed[d.type.underscore.to_sym] = false + + ExceptionNotifier.notify_exception(e, data: { site: site.id, deploy: d.type }) end end From efe3474f5f4db5bb0b47ab9c4bab6db84691022f Mon Sep 17 00:00:00 2001 From: f Date: Mon, 13 Mar 2023 19:14:36 -0300 Subject: [PATCH 230/251] fix: hay deploys sin url --- app/jobs/deploy_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 48366bb9..6dd93129 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -78,7 +78,7 @@ class DeployJob < ApplicationJob status: status, seconds: seconds || 0, size: d.size, - urls: d.respond_to?(:urls) ? d.urls : [d.url] + urls: d.respond_to?(:urls) ? d.urls : [d.url].compact } end end From e0a7232643bbad491887d121bb7a237ac95b6923 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 13 Mar 2023 19:27:03 -0300 Subject: [PATCH 231/251] fix: notificar las excepciones manualmente MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit por alguna razón sucker punch dejó de enviarlas! --- app/jobs/deploy_job.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index 2dacf08e..e187b6bd 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -37,6 +37,8 @@ class DeployJob < ApplicationJob end deploy_others + rescue DeployException => e + notify_exception e ensure @site&.update status: 'waiting' @@ -47,6 +49,12 @@ class DeployJob < ApplicationJob private + # @param :exception [StandardError] + # @param :deploy [Deploy] + def notify_exception(exception, deploy = nil) + ExceptionNotifier.notify_exception(exception, data: { site: @site.id, deploy: deploy&.type }) + end + def deploy_local @deploy_local ||= @site.deploys.find_by(type: 'DeployLocal') end From 8c5a1096107d7fb904ba327d83e3af12f2f4743d Mon Sep 17 00:00:00 2001 From: f Date: Mon, 13 Mar 2023 20:04:18 -0300 Subject: [PATCH 232/251] fix: informar cuando no se puede procesar un issue --- app/jobs/gitlab_notifier_job.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app/jobs/gitlab_notifier_job.rb b/app/jobs/gitlab_notifier_job.rb index 7218f68a..129697ba 100644 --- a/app/jobs/gitlab_notifier_job.rb +++ b/app/jobs/gitlab_notifier_job.rb @@ -3,6 +3,8 @@ # Notifica excepciones a una instancia de Gitlab, como incidencias # nuevas o como comentarios a las incidencias pre-existentes. class GitlabNotifierJob < ApplicationJob + class GitlabNotifierError < StandardError; end + include ExceptionNotifier::BacktraceCleaner # Variables que vamos a acceder luego @@ -18,22 +20,28 @@ class GitlabNotifierJob < ApplicationJob @issue_data = { count: 1 } # Necesitamos saber si el issue ya existía @cached = false + @issue = {} # Traemos los datos desde la caché si existen, sino generamos un # issue nuevo e inicializamos la caché @issue_data = Rails.cache.fetch(cache_key) do - issue = client.new_issue confidential: true, title: title, description: description, issue_type: 'incident' + @issue = client.new_issue confidential: true, title: title, description: description, issue_type: 'incident' @cached = true { count: 1, - issue: issue['iid'], + issue: @issue['iid'], user_agents: [user_agent].compact, params: [request&.filtered_parameters].compact, urls: [url].compact } end + unless @issue['iid'] + Rails.cache.delete(cache_key) + raise GitlabNotifierError, @issue.dig('message', 'title')&.join(', ') + end + # No seguimos actualizando si acabamos de generar el issue return if cached From a0263ad3fbaed7238a9e017e29a89665e009d6aa Mon Sep 17 00:00:00 2001 From: f Date: Mon, 13 Mar 2023 20:04:53 -0300 Subject: [PATCH 233/251] =?UTF-8?q?fix:=20el=20log=20completo=20hace=20un?= =?UTF-8?q?=20t=C3=ADtulo=20demasiado=20largo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/deploy_job.rb | 10 ++++++++-- app/jobs/gitlab_notifier_job.rb | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index e187b6bd..b411568e 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -33,7 +33,7 @@ class DeployJob < ApplicationJob # No es opcional unless @deployed[:deploy_local][:status] # Hacer fallar la tarea - raise DeployException, deploy_local.build_stats.last.log + raise DeployException, 'Falló la compilación' end deploy_others @@ -52,7 +52,13 @@ class DeployJob < ApplicationJob # @param :exception [StandardError] # @param :deploy [Deploy] def notify_exception(exception, deploy = nil) - ExceptionNotifier.notify_exception(exception, data: { site: @site.id, deploy: deploy&.type }) + data = { + site: @site.id, + deploy: deploy&.type, + log: deploy&.build_stats&.last&.log + } + + ExceptionNotifier.notify_exception(exception, data: data) end def deploy_local diff --git a/app/jobs/gitlab_notifier_job.rb b/app/jobs/gitlab_notifier_job.rb index 129697ba..701c6789 100644 --- a/app/jobs/gitlab_notifier_job.rb +++ b/app/jobs/gitlab_notifier_job.rb @@ -112,6 +112,7 @@ class GitlabNotifierJob < ApplicationJob # @return [String] def description @description ||= ''.dup.tap do |d| + d << log_section d << request_section d << javascript_section d << javascript_footer @@ -159,6 +160,19 @@ class GitlabNotifierJob < ApplicationJob @client ||= GitlabApiClient.new end + # @return [String] + def log_section + return '' unless options[:log] + + <<~LOG + # Log + + ``` + #{options[:log]} + ``` + LOG + end + # Muestra información de la petición # # @return [String] From 6bac8ed65dbb413715631a9477bbd2abc038f8b3 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 13 Mar 2023 20:11:12 -0300 Subject: [PATCH 234/251] fix: agrupar por sitio y enviar el log --- app/jobs/deploy_job.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/jobs/deploy_job.rb b/app/jobs/deploy_job.rb index b411568e..961f8d69 100644 --- a/app/jobs/deploy_job.rb +++ b/app/jobs/deploy_job.rb @@ -33,12 +33,14 @@ class DeployJob < ApplicationJob # No es opcional unless @deployed[:deploy_local][:status] # Hacer fallar la tarea - raise DeployException, 'Falló la compilación' + raise DeployException, "#{@site.name}: Falló la compilación" end deploy_others - rescue DeployException => e + rescue DeployTimedOutException => e notify_exception e + rescue DeployException => e + notify_exception e, deploy_local ensure @site&.update status: 'waiting' From 222cc8f62656486c08d6f48e1e3a95c164053f4d Mon Sep 17 00:00:00 2001 From: f Date: Tue, 14 Mar 2023 16:45:39 -0300 Subject: [PATCH 235/251] fix: volver a traducir los correos #9941 --- app/mailers/deploy_mailer.rb | 74 ++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/app/mailers/deploy_mailer.rb b/app/mailers/deploy_mailer.rb index aeb13676..7d939940 100644 --- a/app/mailers/deploy_mailer.rb +++ b/app/mailers/deploy_mailer.rb @@ -18,46 +18,46 @@ class DeployMailer < ApplicationMailer subject = t('.subject', site: site.name) hostname = site.hostname - @hi = t('.hi') - @explanation = t('.explanation', fqdn: hostname) - @help = t('.help') - - @headers = %w[type status url seconds size].map do |header| - t(".th.#{header}") - end - - @table = deploys.each_pair.map do |deploy, value| - { - title: t(".#{deploy}.title"), - status: t(".#{deploy}.#{value[:status] ? 'success' : 'error'}"), - urls: value[:urls], - seconds: { - human: distance_of_time_in_words(value[:seconds].seconds), - machine: "PT#{value[:seconds]}S" - }, - size: number_to_human_size(value[:size], precision: 2) - } - end - - @terminal_table = Terminal::Table.new do |t| - t << @headers - t.add_separator - @table.each do |row| - row[:urls].each do |url| - t << (row.map do |k, v| - case k - when :seconds then v[:human] - when :urls then url - else v - end - end) - end - end - end - # Informamos a cada quien en su idioma y damos una dirección de # respuesta porque a veces les usuaries nos escriben I18n.with_locale(usuarie.lang) do + @hi = t('.hi') + @explanation = t('.explanation', fqdn: hostname) + @help = t('.help') + + @headers = %w[type status url seconds size].map do |header| + t(".th.#{header}") + end + + @table = deploys.each_pair.map do |deploy, value| + { + title: t(".#{deploy}.title"), + status: t(".#{deploy}.#{value[:status] ? 'success' : 'error'}"), + urls: value[:urls], + seconds: { + human: distance_of_time_in_words(value[:seconds].seconds), + machine: "PT#{value[:seconds]}S" + }, + size: number_to_human_size(value[:size], precision: 2) + } + end + + @terminal_table = Terminal::Table.new do |t| + t << @headers + t.add_separator + @table.each do |row| + row[:urls].each do |url| + t << (row.map do |k, v| + case k + when :seconds then v[:human] + when :urls then url + else v + end + end) + end + end + end + mail(to: usuarie.email, reply_to: "sutty@#{Site.domain}", subject: subject) end end From 51c7c57572177632f1971dd8fba61fe1533e2fdd Mon Sep 17 00:00:00 2001 From: f Date: Tue, 14 Mar 2023 16:48:44 -0300 Subject: [PATCH 236/251] fix: strings #10030 --- config/locales/en.yml | 4 ++++ config/locales/es.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/config/locales/en.yml b/config/locales/en.yml index b814796d..6aa9acd3 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -100,6 +100,10 @@ en: title: Alternative domain name success: Success! error: Error + deploy_localized_domain: + title: Domain name by language + success: Success! + error: Error help: You can contact us by replying to this e-mail maintenance_mailer: notice: diff --git a/config/locales/es.yml b/config/locales/es.yml index a6fbd407..ff916b07 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -100,6 +100,10 @@ es: title: Dominio alternativo success: ¡Éxito! error: Hubo un error + deploy_localized_domain: + title: Dominio según idioma + success: ¡Éxito! + error: Hubo un error help: Por cualquier duda, responde este correo para contactarte con nosotres. maintenance_mailer: notice: From 77e209ac9704185fad57271df9a5f058e0583f4e Mon Sep 17 00:00:00 2001 From: f Date: Tue, 14 Mar 2023 19:23:17 -0300 Subject: [PATCH 237/251] fix: traducir el asunto #9941 --- app/mailers/deploy_mailer.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/mailers/deploy_mailer.rb b/app/mailers/deploy_mailer.rb index 7d939940..25efb18d 100644 --- a/app/mailers/deploy_mailer.rb +++ b/app/mailers/deploy_mailer.rb @@ -15,12 +15,13 @@ class DeployMailer < ApplicationMailer def deployed(deploys) usuarie = Usuarie.find(params[:usuarie]) site = usuarie.sites.find(params[:site]) - subject = t('.subject', site: site.name) hostname = site.hostname # Informamos a cada quien en su idioma y damos una dirección de # respuesta porque a veces les usuaries nos escriben I18n.with_locale(usuarie.lang) do + subject = t('.subject', site: site.name) + @hi = t('.hi') @explanation = t('.explanation', fqdn: hostname) @help = t('.help') From 8b479d1743a2dd94603b550024ccff9639ad5304 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 15 Mar 2023 18:22:20 -0300 Subject: [PATCH 238/251] refactor: convertir deploy en un servicio --- app/controllers/sites_controller.rb | 4 +--- app/services/site_service.rb | 5 +++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/app/controllers/sites_controller.rb b/app/controllers/sites_controller.rb index b4826226..f0eff0dc 100644 --- a/app/controllers/sites_controller.rb +++ b/app/controllers/sites_controller.rb @@ -68,9 +68,7 @@ class SitesController < ApplicationController def enqueue authorize site - # XXX: Convertir en una máquina de estados? - site.enqueue! - DeployJob.perform_async site.id + SiteService.new(site: site).deploy redirect_to site_posts_path(site, locale: site.default_locale) end diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 22423bb8..e9a134b5 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -3,6 +3,11 @@ # Se encargar de guardar cambios en sitios # TODO: Implementar rollback en la configuración SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do + def deploy + site.enqueue! + DeployJob.perform_async site.id + end + # Crea un sitio, agrega un rol nuevo y guarda los cambios a la # configuración en el repositorio git def create From 805eb28273abf02fce6853e8c792fb81532dbd17 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 15 Mar 2023 18:23:06 -0300 Subject: [PATCH 239/251] feat: publicar cambios apenas se crea el sitio #2466 --- app/services/site_service.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/services/site_service.rb b/app/services/site_service.rb index e9a134b5..9e5a4988 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -24,6 +24,8 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do add_licencias + deploy + site end From bf45707be48b9f850975eefabf118df40a759663 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 16 Mar 2023 13:17:39 -0300 Subject: [PATCH 240/251] fix: obtener correctamente los dominios con www #10452 --- app/controllers/api/v1/sites_controller.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/v1/sites_controller.rb b/app/controllers/api/v1/sites_controller.rb index 32fdae1d..10a92907 100644 --- a/app/controllers/api/v1/sites_controller.rb +++ b/app/controllers/api/v1/sites_controller.rb @@ -69,12 +69,11 @@ module Api end end + # Todos los dominios con WWW habilitado def www_names - Site.where(contact: true) - .or(Site.where(colaboracion_anonima: true)) - .select("'www.' || name as name").map(&:name).map do |name| - canonicalize name - end + Site.where(id: DeployWww.all.pluck(:site_id)).select("'www.' || name as name").map(&:name).map do |name| + canonicalize name + end end end end From 74af7512648f997f34a4ad4ced4b0a7c268101d3 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 16 Mar 2023 18:26:17 -0300 Subject: [PATCH 241/251] feat: al crear un sitio configurarlo con el idioma de le usuarie #10022 --- app/models/site.rb | 11 +++++++++++ app/services/site_service.rb | 9 ++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/app/models/site.rb b/app/models/site.rb index 8cab0ae0..b8050591 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -179,10 +179,20 @@ class Site < ApplicationRecord # Siempre tiene que tener algo porque las traducciones están # incorporadas a los sitios de Sutty, aunque les usuaries no traduzcan # sus sitios. + # + # @return [Array] def locales @locales ||= config.fetch('locales', I18n.available_locales).map(&:to_sym) end + # Modificar los locales disponibles + # + # @param :new_locales [Array] + # @return [Array] + def locales=(new_locales) + @locales = new_locales.map(&:to_sym).uniq + end + # Similar a site.i18n en jekyll-locales # # @return [Hash] @@ -484,6 +494,7 @@ class Site < ApplicationRecord config.title = title config.url = url(slash: false) config.hostname = hostname + config.locales = locales.map(&:to_s) end # Valida si el sitio tiene al menos una forma de alojamiento asociada diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 22423bb8..d21bd597 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -11,7 +11,14 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do add_role temporal: false, rol: 'usuarie' sync_nodes - I18n.with_locale(usuarie&.lang&.to_sym || I18n.default_locale) do + I18n.with_locale(usuarie.lang.to_sym || I18n.default_locale) do + # No se puede llamar a site.config antes de save porque el sitio + # todavía no existe. + # + # TODO: hacer que el repositorio se cree cuando es necesario, para + # que no haya estados intermedios. + site.locales = [usuarie.lang] + I18n.available_locales + site.save && site.config.write && commit_config(action: :create) From 59bbc04230baeacee6a4dc0ec9b8a2ce3bf6b1ed Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 14:05:47 -0300 Subject: [PATCH 242/251] fix: actualizar el hash solo cuando se escribir cambios y devolver el resultado de Site::Writer#save --- app/models/site/config.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/models/site/config.rb b/app/models/site/config.rb index 3215277e..d2e78d98 100644 --- a/app/models/site/config.rb +++ b/app/models/site/config.rb @@ -33,10 +33,10 @@ class Site def write return if persisted? - @saved = Site::Writer.new(site: site, file: path, - content: content.to_yaml).save - # Actualizar el hash para no escribir dos veces - @hash = content.hash + @saved = Site::Writer.new(site: site, file: path, content: content.to_yaml).save.tap do |result| + # Actualizar el hash para no escribir dos veces + @hash = content.hash + end end alias save write From 2f3a596fa5033d99c859e2b8cfde0d4175623b8a Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 15:27:06 -0300 Subject: [PATCH 243/251] =?UTF-8?q?feat:=20cambiar=20el=20idioma=20en=20to?= =?UTF-8?q?da=20la=20sesi=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/application_controller.rb | 18 ++++++++++++++---- app/views/layouts/_breadcrumb.haml | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 79fc7d73..c72392c7 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -46,17 +46,19 @@ class ApplicationController < ActionController::Base # defecto. # # Esto se refiere al idioma de la interfaz, no de los artículos. - def current_locale(include_params: true, site: nil) - return params[:locale] if include_params && params[:locale].present? + # + # @return [String,Symbol] + def current_locale + session[:locale] ||= params[:change_locale_to] - current_usuarie&.lang || I18n.locale + session[:locale] || current_usuarie&.lang || I18n.locale end # El idioma es el preferido por le usuarie, pero no necesariamente se # corresponde con el idioma de los artículos, porque puede querer # traducirlos. def set_locale(&action) - I18n.with_locale(current_locale(include_params: params[:controller].start_with?('devise')), &action) + I18n.with_locale(current_locale, &action) end # Muestra una página 404 @@ -88,4 +90,12 @@ class ApplicationController < ActionController::Base def prepare_exception_notifier request.env['exception_notifier.exception_data'] = { usuarie: current_usuarie } end + + # Olvidar el idioma elegido antes de iniciar la sesión y reenviar a + # los sitios en el idioma de le usuarie. + def after_sign_in_path_for(resource) + session[:locale] = nil + + sites_path + end end diff --git a/app/views/layouts/_breadcrumb.haml b/app/views/layouts/_breadcrumb.haml index 131c3281..0823b1be 100644 --- a/app/views/layouts/_breadcrumb.haml +++ b/app/views/layouts/_breadcrumb.haml @@ -24,4 +24,4 @@ method: :delete, role: 'button', class: 'btn' - else - other_locale = I18n.available_locales.find { |locale| locale != I18n.locale } - = link_to t(other_locale), "?locale=#{other_locale}" + = link_to t(other_locale), "?change_locale_to=#{other_locale}" From 5ac72f8b2bec443ef3009164e4dfc4a276df4b44 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 15:32:56 -0300 Subject: [PATCH 244/251] feat: soportar varios idiomas --- app/views/layouts/_breadcrumb.haml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/views/layouts/_breadcrumb.haml b/app/views/layouts/_breadcrumb.haml index 0823b1be..099ddde4 100644 --- a/app/views/layouts/_breadcrumb.haml +++ b/app/views/layouts/_breadcrumb.haml @@ -23,5 +23,6 @@ = link_to t('.logout'), main_app.destroy_usuarie_session_path, method: :delete, role: 'button', class: 'btn' - else - - other_locale = I18n.available_locales.find { |locale| locale != I18n.locale } - = link_to t(other_locale), "?change_locale_to=#{other_locale}" + - I18n.available_locales.each do |locale| + - next if locale == I18n.locale + = link_to t(locale), "?change_locale_to=#{locale}" From da721ddaaf611cffd33f862d22291b0ffacb7ecd Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 15:49:26 -0300 Subject: [PATCH 245/251] =?UTF-8?q?feat:=20enviar=20el=20idioma=20en=20el?= =?UTF-8?q?=20correo=20de=20confirmaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/devise/mailer/confirmation_instructions.html.haml | 2 +- app/views/devise/mailer/confirmation_instructions.text.haml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/devise/mailer/confirmation_instructions.html.haml b/app/views/devise/mailer/confirmation_instructions.html.haml index 46706c40..76b10d7f 100644 --- a/app/views/devise/mailer/confirmation_instructions.html.haml +++ b/app/views/devise/mailer/confirmation_instructions.html.haml @@ -1,3 +1,3 @@ %p= t('.greeting', recipient: @email) %p= t('.instruction') -%p= link_to t('.action'), confirmation_url(@resource, confirmation_token: @token) +%p= link_to t('.action'), confirmation_url(@resource, confirmation_token: @token, change_locale_to: @resource.lang) diff --git a/app/views/devise/mailer/confirmation_instructions.text.haml b/app/views/devise/mailer/confirmation_instructions.text.haml index 38e4c548..7123a738 100644 --- a/app/views/devise/mailer/confirmation_instructions.text.haml +++ b/app/views/devise/mailer/confirmation_instructions.text.haml @@ -2,4 +2,4 @@ \ = t('.instruction') \ -= confirmation_url(@resource, confirmation_token: @token) += confirmation_url(@resource, confirmation_token: @token, change_locale_to: @resource.lang) From 6887e5319a8f0c07f493a5543a5849163f3d0e18 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 15:52:10 -0300 Subject: [PATCH 246/251] =?UTF-8?q?feat:=20mantener=20el=20idioma=20de=20l?= =?UTF-8?q?e=20usuarie=20en=20todos=20los=20links=20que=20env=C3=ADe=20dev?= =?UTF-8?q?ise?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/devise/mailer/invitation_instructions.html.haml | 2 +- app/views/devise/mailer/invitation_instructions.text.haml | 2 +- app/views/devise/mailer/reset_password_instructions.html.haml | 2 +- app/views/devise/mailer/reset_password_instructions.text.haml | 2 +- app/views/devise/mailer/unlock_instructions.html.haml | 2 +- app/views/devise/mailer/unlock_instructions.text.haml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/views/devise/mailer/invitation_instructions.html.haml b/app/views/devise/mailer/invitation_instructions.html.haml index 74193878..a2434abd 100644 --- a/app/views/devise/mailer/invitation_instructions.html.haml +++ b/app/views/devise/mailer/invitation_instructions.html.haml @@ -9,7 +9,7 @@ %p= site.description %p= link_to t('devise.mailer.invitation_instructions.accept'), - accept_invitation_url(@resource, invitation_token: @token) + accept_invitation_url(@resource, invitation_token: @token, change_locale_to: @resource.lang) - if @resource.invitation_due_at %p= t('devise.mailer.invitation_instructions.accept_until', diff --git a/app/views/devise/mailer/invitation_instructions.text.haml b/app/views/devise/mailer/invitation_instructions.text.haml index 16a9f0a8..27b1580c 100644 --- a/app/views/devise/mailer/invitation_instructions.text.haml +++ b/app/views/devise/mailer/invitation_instructions.text.haml @@ -9,7 +9,7 @@ \ = site.description \ -= accept_invitation_url(@resource, invitation_token: @token) += accept_invitation_url(@resource, invitation_token: @token, change_locale_to: @resource.lang) \ - if @resource.invitation_due_at = t('devise.mailer.invitation_instructions.accept_until', diff --git a/app/views/devise/mailer/reset_password_instructions.html.haml b/app/views/devise/mailer/reset_password_instructions.html.haml index ccc4aa55..8d8f5919 100644 --- a/app/views/devise/mailer/reset_password_instructions.html.haml +++ b/app/views/devise/mailer/reset_password_instructions.html.haml @@ -1,5 +1,5 @@ %p= t('.greeting', recipient: @resource.email) %p= t('.instruction') -%p= link_to t('.action'), edit_password_url(@resource, reset_password_token: @token) +%p= link_to t('.action'), edit_password_url(@resource, reset_password_token: @token, change_locale_to: @resource.lang) %p= t('.instruction_2') %p= t('.instruction_3') diff --git a/app/views/devise/mailer/reset_password_instructions.text.haml b/app/views/devise/mailer/reset_password_instructions.text.haml index 3d0fe64d..923c2a0c 100644 --- a/app/views/devise/mailer/reset_password_instructions.text.haml +++ b/app/views/devise/mailer/reset_password_instructions.text.haml @@ -2,7 +2,7 @@ \ = t('.instruction') \ -= edit_password_url(@resource, reset_password_token: @token) += edit_password_url(@resource, reset_password_token: @token, change_locale_to: @resource.lang) \ = t('.instruction_2') \ diff --git a/app/views/devise/mailer/unlock_instructions.html.haml b/app/views/devise/mailer/unlock_instructions.html.haml index d68bf7c7..9f8cd492 100644 --- a/app/views/devise/mailer/unlock_instructions.html.haml +++ b/app/views/devise/mailer/unlock_instructions.html.haml @@ -1,4 +1,4 @@ %p= t('.greeting', recipient: @resource.email) %p= t('.message') %p= t('.instruction') -%p= link_to t('.action'), unlock_url(@resource, unlock_token: @token) +%p= link_to t('.action'), unlock_url(@resource, unlock_token: @token, change_locale_to: @resource.lang) diff --git a/app/views/devise/mailer/unlock_instructions.text.haml b/app/views/devise/mailer/unlock_instructions.text.haml index cf06927b..950e04b7 100644 --- a/app/views/devise/mailer/unlock_instructions.text.haml +++ b/app/views/devise/mailer/unlock_instructions.text.haml @@ -4,4 +4,4 @@ \ = t('.instruction') \ -= unlock_url(@resource, unlock_token: @token) += unlock_url(@resource, unlock_token: @token, change_locale_to: @resource.lang) From 0910753437697a1f2e6f1f158a56574fef97390d Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 15:53:56 -0300 Subject: [PATCH 247/251] =?UTF-8?q?fix:=20login=20como=20bot=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/devise/shared/_links.haml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/views/devise/shared/_links.haml b/app/views/devise/shared/_links.haml index c8a6c041..b4b89175 100644 --- a/app/views/devise/shared/_links.haml +++ b/app/views/devise/shared/_links.haml @@ -3,7 +3,8 @@ - locale = params.permit(:locale) - if controller_name != 'sessions' - = link_to t('.sign_in'), new_session_path(resource_name, params: locale) + = link_to t('.sign_in'), new_session_path(resource_name, params: locale), + class: 'btn btn-lg btn-block btn-success' %br/ - if devise_mapping.registerable? && controller_name != 'registrations' From a2660485b8bc3ae7e59435e1755efadf86580d03 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 17 Mar 2023 16:03:11 -0300 Subject: [PATCH 248/251] fix: poder cambiar el idioma! --- app/controllers/application_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index c72392c7..e80c279d 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -49,7 +49,7 @@ class ApplicationController < ActionController::Base # # @return [String,Symbol] def current_locale - session[:locale] ||= params[:change_locale_to] + session[:locale] = params[:change_locale_to] if params[:change_locale_to].present? session[:locale] || current_usuarie&.lang || I18n.locale end From 50f1ce51d9219149da38aaba5eef0fb37554da17 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 19:14:12 -0300 Subject: [PATCH 249/251] fix: el locale de los posts es distinto al de le usuarie #10454 --- app/controllers/posts_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index c5dc0f54..3c529c24 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -12,7 +12,7 @@ class PostsController < ApplicationController # Las URLs siempre llevan el idioma actual o el de le usuarie def default_url_options - { locale: current_locale } + { locale: locale } end def index From 7160cb01df19c39fe150d3a6479813105acc5ded Mon Sep 17 00:00:00 2001 From: f Date: Sat, 18 Mar 2023 19:22:08 -0300 Subject: [PATCH 250/251] fix: la tienda se activa luego de crear el sitio #10512 --- app/views/sites/_form.haml | 40 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/app/views/sites/_form.haml b/app/views/sites/_form.haml index 6f15d570..9a044c7f 100644 --- a/app/views/sites/_form.haml +++ b/app/views/sites/_form.haml @@ -104,27 +104,27 @@ %hr/ - .form-group#tienda - %h2= t('.tienda.title') - %p.lead - - if site.tienda? - = t('.tienda.help') - - else - = t('.tienda.first_time_html') - - .row - .col - .form-group - = f.label :tienda_url - = f.url_field :tienda_url, class: 'form-control' - .col - .form-group - = f.label :tienda_api_key - = f.text_field :tienda_api_key, class: 'form-control' - - %hr/ - - if site.persisted? + .form-group#tienda + %h2= t('.tienda.title') + %p.lead + - if site.tienda? + = t('.tienda.help') + - else + = t('.tienda.first_time_html') + + .row + .col + .form-group + = f.label :tienda_url + = f.url_field :tienda_url, class: 'form-control' + .col + .form-group + = f.label :tienda_api_key + = f.text_field :tienda_api_key, class: 'form-control' + + %hr/ + .form-group#contact %h2= t('.contact.title') %p.lead= t('.contact.help') From 1496a6f04bb6ce3c0842bb1791057ee91ce8d1f7 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 23 Mar 2023 11:59:27 -0300 Subject: [PATCH 251/251] fix: a veces no se puede deployear nada closes ##12735 closes ##12733 closes ##12722 closes ##12719 closes ##12716 closes ##10417 --- app/mailers/deploy_mailer.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/mailers/deploy_mailer.rb b/app/mailers/deploy_mailer.rb index 25efb18d..b7b464cb 100644 --- a/app/mailers/deploy_mailer.rb +++ b/app/mailers/deploy_mailer.rb @@ -12,10 +12,11 @@ class DeployMailer < ApplicationMailer include ActionView::Helpers::DateHelper # rubocop:disable Metrics/AbcSize - def deployed(deploys) + def deployed(deploys = {}) usuarie = Usuarie.find(params[:usuarie]) site = usuarie.sites.find(params[:site]) hostname = site.hostname + deploys ||= {} # Informamos a cada quien en su idioma y damos una dirección de # respuesta porque a veces les usuaries nos escriben